BUGZ-829 DEV-168 - domain backup/content management improvements

BUGZ-829 - improve error reporting during restore of content

DEV-168 - show last installed archive information

domain-server/resources/describe-settings.json

@@ -1788,6 +1788,29 @@
           "default": false
         }
       ]
+    },
+    {
+      "name": "installed_archive",
+      "label": "Installed Archive",
+      "hidden": true,
+      "settings": [
+        {
+          "name": "filename",
+          "content_setting": true,
+          "default": ""
+        },
+        {
+          "name": "install_time",
+          "type": "int",
+          "content_setting": true,
+          "default": 0
+        },
+        {
+          "name": "installed_by",
+          "content_setting": true,
+          "default": ""
+        }
+      ]
     }
   ]
 }

domain-server/resources/web/content/js/content.js

@@ -4,8 +4,13 @@ $(document).ready(function(){
   var RESTORE_SETTINGS_FILE_ID = 'restore-settings-file';
   var UPLOAD_CONTENT_ALLOWED_DIV_ID = 'upload-content-allowed';
   var UPLOAD_CONTENT_RECOVERING_DIV_ID = 'upload-content-recovering';
+  var INSTALLED_ARCHIVE_NAME_ID = 'installed-archive-name';
+  var INSTALLED_ARCHIVE_CREATED_ID = 'installed-archive-created';
+  var INSTALLED_ARCHIVE_INSTALLED_ID = 'installed-archive-installed';
+  var INSTALLED_ARCHIVE_INSTALLED_BY_ID = 'installed-archive-installed-by';
 
   var isRestoring = false;
+  var restoreErrorShown = false;
 
   function progressBarHTML(extraClass, label) {
     var html = "<div class='progress'>";
@@ -103,10 +108,23 @@ $(document).ready(function(){
     html += "<span class='help-block'>Restore in progress</span>";
     html += progressBarHTML('recovery', 'Restoring');
     html += "</div></div>";
-
     $('#' + Settings.UPLOAD_CONTENT_BACKUP_PANEL_ID + ' .panel-body').html(html);
   }
 
+  function setupInstalledContentInfo() {
+    var html = "<table class='table table-bordered'><tbody>";
+    html += "<tr class='headers'><td class='data'><strong>Name</strong></td>";
+    html += "<td class='data'><strong>Created</strong></td>";
+    html += "<td class='data'><strong>Installed</strong></td>";
+    //html += "<td class='data'><strong>Installed By</strong></td></tr>";
+    html += "<tr><td class='data' id='" + INSTALLED_ARCHIVE_NAME_ID + "'/>";
+    html += "<td class='data' id='" + INSTALLED_ARCHIVE_CREATED_ID + "'/>";
+    html += "<td class='data' id='" + INSTALLED_ARCHIVE_INSTALLED_ID + "'/>";
+    //html += "<td class='data' id='" + INSTALLED_ARCHIVE_INSTALLED_BY_ID + "'/></tr>";
+    html += "</tbody></table>";
+    $('#' + Settings.INSTALLED_ARCHIVE_INFO + ' .panel-body').html(html);
+  }
+
   // handle content archive or entity file upload
 
   // when the selected file is changed, enable the button if there's a selected file
@@ -135,6 +153,7 @@ $(document).ready(function(){
 
   var GENERATE_ARCHIVE_BUTTON_ID = 'generate-archive-button';
   var CONTENT_ARCHIVES_NORMAL_ID = 'content-archives-success';
+  var CONTENT_ARCHIVES_CONTENT_INFO_ID = 'content-archives-content-info';
   var CONTENT_ARCHIVES_ERROR_ID = 'content-archives-error';
   var AUTOMATIC_ARCHIVES_TABLE_ID = 'automatic-archives-table';
   var AUTOMATIC_ARCHIVES_TBODY_ID = 'automatic-archives-tbody';
@@ -230,13 +249,28 @@ $(document).ready(function(){
       url: '/api/backups',
       cache: false
     }).done(function(data) {
-
+console.log(data);
       // split the returned data into manual and automatic manual backups
       var splitBackups = _.partition(data.backups, function(value, index) {
         return value.isManualBackup;
       });
-
+      if (data.status.recoveryError && !restoreErrorShown) {
-      if (isRestoring && !data.status.isRecovering) {
+        restoreErrorShown = true;
+        swal({
+          title: "Error",
+          text: "There was a problem restoring domain content.\n"
+          + data.status.recoveryError,
+          type: "error",
+          showCancelButton: false,
+          confirmButtonText: "Restart",
+          closeOnConfirm: true,
+        },
+        function() {
+          $.get("/restart");
+          showRestartModal();
+        });
+      }
+      if (isRestoring && !data.status.isRecovering && !data.status.recoveryError) {
         // we were recovering and we finished - the DS is going to restart so show the restart modal
         showRestartModal();
         return;
@@ -327,6 +361,12 @@ $(document).ready(function(){
       $('#' + UPLOAD_CONTENT_ALLOWED_DIV_ID).toggle(!data.status.isRecovering);
       $('#' + UPLOAD_CONTENT_RECOVERING_DIV_ID).toggle(data.status.isRecovering);
 
+
+      $('#' + INSTALLED_ARCHIVE_NAME_ID).text(data.installed_archive.name);
+      $('#' + INSTALLED_ARCHIVE_CREATED_ID).text(moment(data.installed_archive.creation_time).format('lll'));
+      $('#' + INSTALLED_ARCHIVE_INSTALLED_ID).text(moment(data.installed_archive.install_time).format('lll'));
+      //$('#' + INSTALLED_ARCHIVE_INSTALLED_BY_ID).text(data.current_archive.installed_by);
+
       // update the progress bars for current restore status
       if (data.status.isRecovering) {
         updateProgressBars($('.recovery.progress-bar'), data.status.recoveryProgress * 100);
@@ -514,6 +554,7 @@ $(document).ready(function(){
   Settings.afterReloadActions = function() {
     setupBackupUpload();
     setupContentArchives();
+    setupInstalledContentInfo();
 
     // load the latest backups immediately
     reloadBackupInformation();

domain-server/resources/web/js/domain-server.js

@@ -57,10 +57,14 @@ $(document).ready(function(){
   // define extra groups to add to setting panels, with their splice index
   Settings.extraContentGroupsAtIndex = {
     0: {
+      html_id: Settings.INSTALLED_ARCHIVE_INFO,
+      label: 'Installed Archive'
+    },
+    1: {
       html_id: Settings.CONTENT_ARCHIVES_PANEL_ID,
       label: 'Content Archives'
     },
-    1: {
+    2: {
       html_id: Settings.UPLOAD_CONTENT_BACKUP_PANEL_ID,
       label: 'Upload Content'
     }

domain-server/resources/web/js/shared.js

@@ -44,7 +44,8 @@ $.extend(Settings, {
   INVALID_ROW_CLASS: 'invalid-input',
   DATA_ROW_INDEX: 'data-row-index',
   CONTENT_ARCHIVES_PANEL_ID: 'content_archives',
-  UPLOAD_CONTENT_BACKUP_PANEL_ID: 'upload_content'
+  UPLOAD_CONTENT_BACKUP_PANEL_ID: 'upload_content',
+  INSTALLED_ARCHIVE_INFO: 'installed_archive_info'
 });
 
 var URLs = {

domain-server/src/AssetsBackupHandler.cpp

@@ -240,12 +240,12 @@ void AssetsBackupHandler::createBackup(const QString& backupName, QuaZip& zip) {
     Q_ASSERT(QThread::currentThread() == thread());
 
     if (operationInProgress()) {
-        qCWarning(asset_backup) << "There is already an operation in progress.";
+        qCWarning(asset_backup) << "There is already an operation in progress.  Please wait.";
         return;
     }
 
     if (_assetServerEnabled && _lastMappingsRefresh.time_since_epoch().count() == 0) {
-        qCWarning(asset_backup) << "Current mappings not yet loaded.";
+        qCWarning(asset_backup) << "Current mappings not yet loaded.  Please wait.";
         _backups.emplace_back(backupName, AssetUtils::Mappings(), true);
         return;
     }
@@ -278,17 +278,19 @@ void AssetsBackupHandler::createBackup(const QString& backupName, QuaZip& zip) {
     _backups.emplace_back(backupName, mappings, false);
 }
 
-void AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
+std::pair<bool, QString> AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
     Q_ASSERT(QThread::currentThread() == thread());
 
     if (operationInProgress()) {
-        qCWarning(asset_backup) << "There is already a backup/restore in progress.";
+        QString errorStr ("There is already a backup/restore in progress.  Please wait.");
-        return;
+        qWarning() << errorStr;
+        return { false, errorStr };
     }
 
     if (_lastMappingsRefresh.time_since_epoch().count() == 0) {
-        qCWarning(asset_backup) << "Current mappings not yet loaded.";
+        QString errorStr ("Current mappings not yet loaded.  Please wait.");
-        return;
+        qWarning() << errorStr;
+        return { false, errorStr };
     }
 
     if ((p_high_resolution_clock::now() - _lastMappingsRefresh) > MAX_REFRESH_TIME) {
@@ -301,6 +303,16 @@ void AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip)
     if (it == end(_backups)) {
         loadBackup(backupName, zip);
 
+        auto emplaced_backup = find_if(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+            return backup.name == backupName;
+        });
+
+        if(emplaced_backup->corruptedBackup) {
+            QString errorStr ("Current mappings file is corrupted.");
+            qWarning() << errorStr;
+            return { false, errorStr };
+        }
+
         QuaZipDir zipDir { &zip, ZIP_ASSETS_FOLDER };
 
         auto assetNames = zipDir.entryList(QDir::Files);
@@ -330,8 +342,9 @@ void AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip)
         });
 
         if (it == end(_backups)) {
-            qCCritical(asset_backup) << "Failed to recover backup:" << backupName;
+            QString errorStr ("Failed to recover backup: " + backupName);
-            return;
+            qWarning() << errorStr;
+            return { false, errorStr };
         }
     }
 
@@ -339,6 +352,7 @@ void AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip)
     computeServerStateDifference(_currentMappings, newMappings);
 
     restoreAllAssets();
+    return { true, QString() };
 }
 
 void AssetsBackupHandler::deleteBackup(const QString& backupName) {

domain-server/src/AssetsBackupHandler.h

@@ -38,7 +38,7 @@ public:
     void loadBackup(const QString& backupName, QuaZip& zip) override;
     void loadingComplete() override;
     void createBackup(const QString& backupName, QuaZip& zip) override;
-    void recoverBackup(const QString& backupName, QuaZip& zip) override;
+    std::pair<bool, QString> recoverBackup(const QString& backupName, QuaZip& zip) override;
     void deleteBackup(const QString& backupName) override;
     void consolidateBackup(const QString& backupName, QuaZip& zip) override;
     bool isCorruptedBackup(const QString& backupName) override;

domain-server/src/BackupHandler.h

@@ -30,7 +30,7 @@ public:
     virtual void loadBackup(const QString& backupName, QuaZip& zip) = 0;
     virtual void loadingComplete() = 0;
     virtual void createBackup(const QString& backupName, QuaZip& zip) = 0;
-    virtual void recoverBackup(const QString& backupName, QuaZip& zip) = 0;
+    virtual std::pair<bool, QString> recoverBackup(const QString& backupName, QuaZip& zip) = 0;
     virtual void deleteBackup(const QString& backupName) = 0;
     virtual void consolidateBackup(const QString& backupName, QuaZip& zip) = 0;
     virtual bool isCorruptedBackup(const QString& backupName) = 0;

domain-server/src/ContentSettingsBackupHandler.cpp

@@ -24,6 +24,7 @@
 #pragma GCC diagnostic pop
 #endif
 
+static const QString DATETIME_FORMAT { "yyyy-MM-dd_HH-mm-ss" };
 
 ContentSettingsBackupHandler::ContentSettingsBackupHandler(DomainServerSettingsManager& domainServerSettingsManager) :
     _settingsManager(domainServerSettingsManager)
@@ -31,6 +32,10 @@ ContentSettingsBackupHandler::ContentSettingsBackupHandler(DomainServerSettingsM
 }
 
 static const QString CONTENT_SETTINGS_BACKUP_FILENAME = "content-settings.json";
+static const QString INSTALLED_ARCHIVE = "installed_archive";
+static const QString INSTALLED_ARCHIVE_FILENAME = "filename";
+static const QString INSTALLED_ARCHIVE_INSTALL_TIME = "install_time";
+static const QString INSTALLED_ARCHIVE_INSTALLED_BY = "installed_by";
 
 void ContentSettingsBackupHandler::createBackup(const QString& backupName, QuaZip& zip) {
 
@@ -42,6 +47,15 @@ void ContentSettingsBackupHandler::createBackup(const QString& backupName, QuaZi
         DomainServerSettingsManager::ForBackup
     );
 
+    // save the filename internally as it's used to determine the name/creation time
+    // of the archive, and we don't want that to change if the actual file is renamed
+    // after download.
+    QJsonObject installed_archive {
+        { INSTALLED_ARCHIVE_FILENAME, backupName },
+    };
+
+    contentSettingsJSON.insert(INSTALLED_ARCHIVE, installed_archive);
+
     // make a QJsonDocument using the object
     QJsonDocument contentSettingsDocument { contentSettingsJSON };
 
@@ -62,24 +76,46 @@ void ContentSettingsBackupHandler::createBackup(const QString& backupName, QuaZi
     }
 }
 
-void ContentSettingsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
+std::pair<bool, QString> ContentSettingsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
     if (!zip.setCurrentFile(CONTENT_SETTINGS_BACKUP_FILENAME)) {
-        qWarning() << "Failed to find" << CONTENT_SETTINGS_BACKUP_FILENAME << "while recovering backup";
+        QString errorStr("Failed to find " + CONTENT_SETTINGS_BACKUP_FILENAME + " while recovering backup");
-        return;
+        qWarning() << errorStr;
+        return { false, errorStr };
     }
 
     QuaZipFile zipFile { &zip };
     if (!zipFile.open(QIODevice::ReadOnly)) {
-        qCritical() << "Failed to open" << CONTENT_SETTINGS_BACKUP_FILENAME << "in backup";
+        QString errorStr("Failed to open " + CONTENT_SETTINGS_BACKUP_FILENAME + " in backup");
-        return;
+        qCritical() << errorStr;
+        return { false, errorStr };
     }
 
     auto rawData = zipFile.readAll();
     zipFile.close();
 
-    QJsonDocument jsonDocument = QJsonDocument::fromJson(rawData);
+    if (zipFile.getZipError() != UNZ_OK) {
-
+        QString errorStr("Failed to unzip " + CONTENT_SETTINGS_BACKUP_FILENAME + ": " + zipFile.getZipError());
-    if (!_settingsManager.restoreSettingsFromObject(jsonDocument.object(), ContentSettings)) {
+        qCritical() << errorStr;
-        qCritical() << "Failed to restore settings from" << CONTENT_SETTINGS_BACKUP_FILENAME << "in content archive";
+        return { false, errorStr };
     }
+
+    QJsonDocument jsonDocument = QJsonDocument::fromJson(rawData);
+    QJsonObject jsonObject = jsonDocument.object();
+
+    auto archiveJson = jsonObject.find(INSTALLED_ARCHIVE)->toObject();
+
+    QJsonObject installed_archive {
+        { INSTALLED_ARCHIVE_FILENAME, archiveJson[INSTALLED_ARCHIVE_FILENAME]},
+        { INSTALLED_ARCHIVE_INSTALL_TIME, QDateTime::currentDateTime().currentMSecsSinceEpoch() },
+        { INSTALLED_ARCHIVE_INSTALLED_BY, ""}
+    };
+
+    jsonObject.insert(INSTALLED_ARCHIVE, installed_archive);
+
+    if (!_settingsManager.restoreSettingsFromObject(jsonObject, ContentSettings)) {
+        QString errorStr("Failed to restore settings from " + CONTENT_SETTINGS_BACKUP_FILENAME + " in content archive");
+        qCritical() << errorStr;
+        return { false, errorStr };
+    }
+    return { true, QString() };
 }

domain-server/src/ContentSettingsBackupHandler.h

@@ -28,7 +28,7 @@ public:
 
     void createBackup(const QString& backupName, QuaZip& zip) override;
 
-    void recoverBackup(const QString& backupName, QuaZip& zip) override;
+    std::pair<bool, QString> recoverBackup(const QString& backupName, QuaZip& zip) override;
 
     void deleteBackup(const QString& backupName) override {}
 

domain-server/src/DomainContentBackupManager.cpp

@@ -45,6 +45,7 @@ static const QString DATETIME_FORMAT { "yyyy-MM-dd_HH-mm-ss" };
 static const QString DATETIME_FORMAT_RE { "\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}" };
 static const QString AUTOMATIC_BACKUP_PREFIX { "autobackup-" };
 static const QString MANUAL_BACKUP_PREFIX { "backup-" };
+static const QString PRE_UPLOAD_SUFFIX{ "pre_upload" };
 static const QString MANUAL_BACKUP_NAME_RE { "[a-zA-Z0-9\\-_ ]+" };
 
 void DomainContentBackupManager::addBackupHandler(BackupHandlerPointer handler) {
@@ -52,9 +53,10 @@ void DomainContentBackupManager::addBackupHandler(BackupHandlerPointer handler)
 }
 
 DomainContentBackupManager::DomainContentBackupManager(const QString& backupDirectory,
-                                                       const QVariantList& backupRules,
+                                                       DomainServerSettingsManager& domainServerSettingsManager,
                                                        std::chrono::milliseconds persistInterval,
                                                        bool debugTimestampNow) :
+    _settingsManager(domainServerSettingsManager),
     _consolidatedBackupDirectory(PathUtils::generateTemporaryDir()),
     _backupDirectory(backupDirectory), _persistInterval(persistInterval), _lastCheck(p_high_resolution_clock::now())
 {
@@ -63,7 +65,8 @@ DomainContentBackupManager::DomainContentBackupManager(const QString& backupDire
     // Make sure the backup directory exists.
     QDir(_backupDirectory).mkpath(".");
 
-    parseBackupRules(backupRules);
+    static const QString BACKUP_RULES_KEYPATH = AUTOMATIC_CONTENT_ARCHIVES_GROUP + ".backup_rules";
+    parseBackupRules(_settingsManager.valueOrDefaultValueForKeyPath(BACKUP_RULES_KEYPATH).toList());
 
     constexpr int CONSOLIDATED_BACKUP_CLEANER_INTERVAL_MSECS = 30 * 1000;
     _consolidatedBackupCleanupTimer.setInterval(CONSOLIDATED_BACKUP_CLEANER_INTERVAL_MSECS);
@@ -170,7 +173,7 @@ bool DomainContentBackupManager::process() {
                 return handler->getRecoveryStatus().first;
             });
 
-            if (!isStillRecovering) {
+            if (!isStillRecovering && _recoveryError.isEmpty()) {
                 _isRecovering = false;
                 _recoveryFilename = "";
                 emit recoveryCompleted();
@@ -277,7 +280,7 @@ void DomainContentBackupManager::deleteBackup(MiniPromise::Promise promise, cons
     });
 }
 
-bool DomainContentBackupManager::recoverFromBackupZip(const QString& backupName, QuaZip& zip) {
+bool DomainContentBackupManager::recoverFromBackupZip(const QString& backupName, QuaZip& zip, bool rollingBack) {
     if (!zip.open(QuaZip::Mode::mdUnzip)) {
         qWarning() << "Failed to unzip file: " << backupName;
         return false;
@@ -286,7 +289,15 @@ bool DomainContentBackupManager::recoverFromBackupZip(const QString& backupName,
         _recoveryFilename = backupName;
 
         for (auto& handler : _backupHandlers) {
-            handler->recoverBackup(backupName, zip);
+            bool success;
+            QString errorStr;
+            std::tie(success, errorStr) = handler->recoverBackup(backupName, zip);
+            if (!success) {
+                if (!rollingBack) {
+                    _recoveryError = errorStr;
+                }
+                return false;
+            }
         }
 
         qDebug() << "Successfully started recovering from " << backupName;
@@ -309,7 +320,7 @@ void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise,
     }
 
     qDebug() << "Recovering from" << backupName;
-
+    _recoveryError = "";
     bool success { false };
     QDir backupDir { _backupDirectory };
     auto backupFilePath { backupDir.filePath(backupName) };
@@ -360,14 +371,36 @@ void DomainContentBackupManager::recoverFromUploadedFile(MiniPromise::Promise pr
     }
 
     qDebug() << "Recovering from uploaded file -" << uploadedFilename;
+    bool success;
+    QString path;
+    std::tie(success, path) = createBackup(AUTOMATIC_BACKUP_PREFIX, PRE_UPLOAD_SUFFIX);
+    if(!success) {
+        _recoveryError = "Failed to create backup for " + PRE_UPLOAD_SUFFIX + " at " + path;
+        qCWarning(domain_server) << _recoveryError;
+    } else {
+        QFile uploadedFile(uploadedFilename);
+        QuaZip uploadedZip { &uploadedFile };
 
-    QFile uploadedFile(uploadedFilename);
+        QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip";
-    QuaZip uploadedZip { &uploadedFile };
 
-    QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip";
+        bool success = recoverFromBackupZip(backupName, uploadedZip);
 
-    bool success = recoverFromBackupZip(backupName, uploadedZip);
+        if (!success) {
 
+            // attempt to rollback to
+            QString filename;
+            QDateTime filetime;
+            if (getMostRecentBackup(PRE_UPLOAD_SUFFIX, filename, filetime)) {
+                QFile uploadedFile(uploadedFilename);
+                QuaZip uploadedZip { &uploadedFile };
+
+                QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip";
+                recoverFromBackupZip(backupName, uploadedZip, true);
+
+            }
+        }
+
+    }
     promise->resolve({
         { "success", success }
         });
@@ -455,9 +488,32 @@ void DomainContentBackupManager::getAllBackupsAndStatus(MiniPromise::Promise pro
         { "recoveryProgress", recoveryProgress }
     };
 
+    if(!_recoveryError.isEmpty()) {
+        status["recoveryError"] = _recoveryError;
+    }
+
+    QVariantMap currentArchive;
+
+    QString fileName = _settingsManager.valueForKeyPath(CONTENT_SETTINGS_ARCHIVE_FILENAME).toString();
+    QString prefixFormat = "(" + QRegExp::escape(AUTOMATIC_BACKUP_PREFIX) + "|" + QRegExp::escape(MANUAL_BACKUP_PREFIX) + ")";
+    QString nameFormat = "(.+)";
+    QString dateTimeFormat = "(" + DATETIME_FORMAT_RE + ")";
+    QRegExp backupNameFormat { prefixFormat + nameFormat + "-" + dateTimeFormat + "\\.zip" };
+    if (backupNameFormat.exactMatch(fileName)) {
+        auto type = backupNameFormat.cap(1);
+        auto name = backupNameFormat.cap(2);
+        auto dateTime = backupNameFormat.cap(3);
+        auto createdAt = QDateTime::fromString(dateTime, DATETIME_FORMAT);
+        currentArchive["name"] = name;
+        currentArchive["creation_time"] = createdAt.toMSecsSinceEpoch();
+        currentArchive["install_time"] = _settingsManager.valueForKeyPath(CONTENT_SETTINGS_ARCHIVE_INSTALL_TIME).toULongLong();
+        currentArchive["installed_by"] = _settingsManager.valueForKeyPath(CONTENT_SETTINGS_ARCHIVE_INSTALLED_BY).toString();
+    }
+
     QVariantMap info {
         { "backups", variantBackups },
-        { "status", status }
+        { "status", status },
+        { "installed_archive", currentArchive }
     };
 
     promise->resolve(info);

domain-server/src/DomainContentBackupManager.h

@@ -28,6 +28,7 @@
 #include <GenericThread.h>
 
 #include "BackupHandler.h"
+#include "DomainServerSettingsManager.h"
 
 #include <shared/MiniPromises.h>
 
@@ -71,7 +72,7 @@ public:
     static const std::chrono::seconds DEFAULT_PERSIST_INTERVAL;
 
     DomainContentBackupManager(const QString& rootBackupDirectory,
-                               const QVariantList& settings,
+                               DomainServerSettingsManager& domainServerSettingsManager,
                                std::chrono::milliseconds persistInterval = DEFAULT_PERSIST_INTERVAL,
                                bool debugTimestampNow = false);
 
@@ -108,13 +109,15 @@ protected:
 
     std::pair<bool, QString> createBackup(const QString& prefix, const QString& name);
 
-    bool recoverFromBackupZip(const QString& backupName, QuaZip& backupZip);
+    bool recoverFromBackupZip(const QString& backupName, QuaZip& backupZip, bool rollingBack = false);
 
 private slots:
     void removeOldConsolidatedBackups();
     void consolidateBackupInternal(QString fileName);
 
 private:
+    DomainServerSettingsManager& _settingsManager;
+
     QTimer _consolidatedBackupCleanupTimer;
 
     const QString _consolidatedBackupDirectory;
@@ -126,6 +129,7 @@ private:
     std::unordered_map<QString, ConsolidatedBackupInfo> _consolidatedBackups;
 
     std::atomic<bool> _isRecovering { false };
+    QString _recoveryError;
     QString _recoveryFilename { };
 
     p_high_resolution_clock::time_point _lastCheck;

domain-server/src/DomainServer.cpp

@@ -307,11 +307,7 @@ DomainServer::DomainServer(int argc, char* argv[]) :
     }
     maybeHandleReplacementEntityFile();
 
-
+    _contentManager.reset(new DomainContentBackupManager(getContentBackupDir(), _settingsManager));
-    static const QString BACKUP_RULES_KEYPATH = AUTOMATIC_CONTENT_ARCHIVES_GROUP + ".backup_rules";
-    auto backupRulesVariant = _settingsManager.valueOrDefaultValueForKeyPath(BACKUP_RULES_KEYPATH);
-
-    _contentManager.reset(new DomainContentBackupManager(getContentBackupDir(), backupRulesVariant.toList()));
 
     connect(_contentManager.get(), &DomainContentBackupManager::started, _contentManager.get(), [this](){
         _contentManager->addBackupHandler(BackupHandlerPointer(new EntitiesBackupHandler(getEntitiesFilePath(), getEntitiesReplacementFilePath())));
@@ -2333,8 +2329,7 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
 
                 QJsonObject rootJSON;
                 auto success = result["success"].toBool();
-                rootJSON["success"] = success;
+                QJsonDocument docJSON(QJsonObject::fromVariantMap(result));
-                QJsonDocument docJSON(rootJSON);
                 connectionPtr->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
                                     JSON_MIME_TYPE.toUtf8());
             });
@@ -2362,8 +2357,7 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
 
                 QJsonObject rootJSON;
                 auto success = result["success"].toBool();
-                rootJSON["success"] = success;
+                QJsonDocument docJSON(QJsonObject::fromVariantMap(result));
-                QJsonDocument docJSON(rootJSON);
                 connectionPtr->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
                                     JSON_MIME_TYPE.toUtf8());
             });
@@ -2467,8 +2461,7 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
 
                 QJsonObject rootJSON;
                 auto success = result["success"].toBool();
-                rootJSON["success"] = success;
+                QJsonDocument docJSON(QJsonObject::fromVariantMap(result));
-                QJsonDocument docJSON(rootJSON);
                 connectionPtr->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
                                     JSON_MIME_TYPE.toUtf8());
             });
@@ -2687,11 +2680,12 @@ void DomainServer::profileRequestFinished() {
 
 bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl& url) {
 
-    const QByteArray HTTP_COOKIE_HEADER_KEY = "Cookie";
+    static const QByteArray HTTP_COOKIE_HEADER_KEY = "Cookie";
-    const QString ADMIN_USERS_CONFIG_KEY = "admin-users";
+    static const QString ADMIN_USERS_CONFIG_KEY = "admin-users";
-    const QString ADMIN_ROLES_CONFIG_KEY = "admin-roles";
+    static const QString ADMIN_ROLES_CONFIG_KEY = "admin-roles";
-    const QString BASIC_AUTH_USERNAME_KEY_PATH = "security.http_username";
+    static const QString BASIC_AUTH_USERNAME_KEY_PATH = "security.http_username";
-    const QString BASIC_AUTH_PASSWORD_KEY_PATH = "security.http_password";
+    static const QString BASIC_AUTH_PASSWORD_KEY_PATH = "security.http_password";
+    const QString COOKIE_UUID_REGEX_STRING = HIFI_SESSION_COOKIE_KEY + "=([\\d\\w-]+)($|;)";
 
     const QByteArray UNAUTHENTICATED_BODY = "You do not have permission to access this domain-server.";
 
@@ -2702,7 +2696,6 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl
         && (adminUsersVariant.isValid() || adminRolesVariant.isValid())) {
         QString cookieString = connection->requestHeader(HTTP_COOKIE_HEADER_KEY);
 
-        const QString COOKIE_UUID_REGEX_STRING = HIFI_SESSION_COOKIE_KEY + "=([\\d\\w-]+)($|;)";
         QRegExp cookieUUIDRegex(COOKIE_UUID_REGEX_STRING);
 
         QUuid cookieUUID;

domain-server/src/DomainServer.h

@@ -194,6 +194,7 @@ private:
     QUrl oauthRedirectURL();
     QUrl oauthAuthorizationURL(const QUuid& stateUUID = QUuid::createUuid());
 
+    QString getWebSessionUsername(HTTPConnection* connection);
     bool isAuthenticatedRequest(HTTPConnection* connection, const QUrl& url);
 
     QNetworkReply* profileRequestGivenTokenReply(QNetworkReply* tokenReply);

domain-server/src/DomainServerSettingsManager.h

@@ -35,6 +35,9 @@ const QString MACHINE_FINGERPRINT_PERMISSIONS_KEYPATH = "security.machine_finger
 const QString GROUP_PERMISSIONS_KEYPATH = "security.group_permissions";
 const QString GROUP_FORBIDDENS_KEYPATH = "security.group_forbiddens";
 const QString AUTOMATIC_CONTENT_ARCHIVES_GROUP = "automatic_content_archives";
+const QString CONTENT_SETTINGS_ARCHIVE_FILENAME = "installed_archive.filename";
+const QString CONTENT_SETTINGS_ARCHIVE_INSTALL_TIME = "installed_archive.install_time";
+const QString CONTENT_SETTINGS_ARCHIVE_INSTALLED_BY = "installed_archive.installed_by";
 
 using GroupByUUIDKey = QPair<QUuid, QUuid>; // groupID, rankID
 

domain-server/src/EntitiesBackupHandler.cpp

@@ -57,36 +57,41 @@ void EntitiesBackupHandler::createBackup(const QString& backupName, QuaZip& zip)
     }
 }
 
-void EntitiesBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
+std::pair<bool, QString> EntitiesBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
     if (!zip.setCurrentFile(ENTITIES_BACKUP_FILENAME)) {
-        qWarning() << "Failed to find" << ENTITIES_BACKUP_FILENAME << "while recovering backup";
+        QString errorStr("Failed to find " + ENTITIES_BACKUP_FILENAME + " while recovering backup");
-        return;
+        qWarning() << errorStr;
+        return { false, errorStr };
     }
     QuaZipFile zipFile { &zip };
     if (!zipFile.open(QIODevice::ReadOnly)) {
-        qCritical() << "Failed to open" << ENTITIES_BACKUP_FILENAME << "in backup";
+        QString errorStr("Failed to open " + ENTITIES_BACKUP_FILENAME + " in backup");
-        return;
+        qCritical() << errorStr;
+        return { false, errorStr };
     }
     auto rawData = zipFile.readAll();
 
     zipFile.close();
 
+    if (zipFile.getZipError() != UNZ_OK) {
+        QString errorStr("Failed to unzip " + ENTITIES_BACKUP_FILENAME + ": " + zipFile.getZipError());
+        qCritical() << errorStr;
+        return { false, errorStr };
+    }
+
     OctreeUtils::RawEntityData data;
     if (!data.readOctreeDataInfoFromData(rawData)) {
-        qCritical() << "Unable to parse octree data during backup recovery";
+        QString errorStr("Unable to parse octree data during backup recovery");
-        return;
+        qCritical() << errorStr;
+        return { false, errorStr };
     }
 
     data.resetIdAndVersion();
 
-    if (zipFile.getZipError() != UNZ_OK) {
-        qCritical().nospace() << "Failed to unzip " << ENTITIES_BACKUP_FILENAME << ": " << zipFile.getZipError();
-        return;
-    }
-
     QFile entitiesFile { _entitiesReplacementFilePath };
 
     if (entitiesFile.open(QIODevice::WriteOnly)) {
         entitiesFile.write(data.toGzippedByteArray());
     }
+    return { true, QString() };
 }

domain-server/src/EntitiesBackupHandler.h

@@ -29,7 +29,7 @@ public:
     void createBackup(const QString& backupName, QuaZip& zip) override;
 
     // Recover from a full backup
-    void recoverBackup(const QString& backupName, QuaZip& zip) override;
+    std::pair<bool, QString> recoverBackup(const QString& backupName, QuaZip& zip) override;
 
     // Delete a skeleton backup
     void deleteBackup(const QString& backupName) override {}