Update content backup downloads to be asynchronous

Previously backups would be consolidated into a full .zip when the
backup was requested. Because of the potential for a large number of
assets, this could take awhile, causing a browser to fail due to timeout
before the backup was available. This change splits the download
endpoint into 2 parts - one to request information about a backup and
possibly kick off a consolidation, and another to request the actual
file once the consolidate backup is available.

domain-server/resources/web/content/js/content.js

@@ -132,6 +132,41 @@ $(document).ready(function(){
   var ACTIVE_BACKUP_ROW_CLASS = 'active-backup';
   var CORRUPTED_ROW_CLASS = 'danger';
 
+  $('body').on('click',  '.' + BACKUP_DOWNLOAD_LINK_CLASS, function(ev) {
+    ev.preventDefault();
+    var backupID = $(this).data('backup-id')
+
+    showSpinnerAlert("Preparing backup...");
+    function checkBackupStatus() {
+      $.ajax({
+        url: "/api/backups/" + backupID,
+        dataType: 'json',
+        success: function(data) {
+          if (data.complete) {
+            if (data.error == '') {
+              location.href = "/api/backups/download/" + backupID;
+              swal.close();
+            } else {
+              showErrorMessage(
+                "Error",
+                "There was an error preparing your backup. Please refresh the page and try again."
+              );
+            }
+          } else {
+            setTimeout(checkBackupStatus, 500);
+          }
+        },
+        error: function() {
+          showErrorMessage(
+            "Error",
+            "There was an error preparing your backup."
+          );
+        },
+      });
+    }
+    checkBackupStatus();
+  });
+
   function reloadBackupInformation() {
     // make a GET request to get backup information to populate the table
     $.ajax({
@@ -164,7 +199,7 @@ $(document).ready(function(){
           + "<div class='dropdown'><div class='dropdown-toggle' data-toggle='dropdown' aria-expanded='false'><span class='glyphicon glyphicon-option-vertical'></span></div>"
           + "<ul class='dropdown-menu dropdown-menu-right'>"
           + "<li><a class='" + BACKUP_RESTORE_LINK_CLASS + "' href='#'>Restore from here</a></li><li class='divider'></li>"
-          + "<li><a class='" + BACKUP_DOWNLOAD_LINK_CLASS + "' href='/api/backups/" + backup.id + "'>Download</a></li><li class='divider'></li>"
+          + "<li><a class='" + BACKUP_DOWNLOAD_LINK_CLASS + "' data-backup-id='" + backup.id + "' href='#'>Download</a></li><li class='divider'></li>"
           + "<li><a class='" + BACKUP_DELETE_LINK_CLASS + "' href='#' target='_blank'>Delete</a></li></ul></div></td>";
       }
 

domain-server/src/DomainContentBackupManager.cpp

@@ -55,9 +55,9 @@ DomainContentBackupManager::DomainContentBackupManager(const QString& backupDire
                                                        const QVariantList& backupRules,
                                                        std::chrono::milliseconds persistInterval,
                                                        bool debugTimestampNow) :
+    _consolidatedBackupDirectory(PathUtils::generateTemporaryDir()),
     _backupDirectory(backupDirectory), _persistInterval(persistInterval), _lastCheck(p_high_resolution_clock::now())
 {
-
     setObjectName("DomainContentBackupManager");
 
     // Make sure the backup directory exists.
@@ -498,23 +498,63 @@ void DomainContentBackupManager::backup() {
     }
 }
 
-void DomainContentBackupManager::consolidateBackup(MiniPromise::Promise promise, QString fileName) {
+ConsolidatedBackupInfo DomainContentBackupManager::consolidateBackup(QString fileName) {
-    if (QThread::currentThread() != thread()) {
+    {
-        QMetaObject::invokeMethod(this, "consolidateBackup", Q_ARG(MiniPromise::Promise, promise),
+        std::lock_guard<std::mutex> lock { _consolidatedBackupsMutex };
-                                  Q_ARG(QString, fileName));
+        auto it = _consolidatedBackups.find(fileName);
-        return;
+
+        if (it != _consolidatedBackups.end()) {
+            return it->second;
+        }
+    }
+    QMetaObject::invokeMethod(this, "consolidateBackupInternal", Q_ARG(QString, fileName));
+    return {
+        ConsolidatedBackupInfo::CONSOLIDATING,
+        "",
+        ""
+    };
+}
+
+void DomainContentBackupManager::consolidateBackupInternal(QString fileName) {
+    auto markFailure = [this, &fileName](QString error) {
+        qWarning() << "Failed to consolidate backup:" << fileName << error;
+        {
+            std::lock_guard<std::mutex> lock { _consolidatedBackupsMutex };
+            auto& consolidatedBackup = _consolidatedBackups[fileName];
+            consolidatedBackup.state = ConsolidatedBackupInfo::COMPLETE_WITH_ERROR;
+            consolidatedBackup.error = error;
+        }
+    };
+
+    {
+        std::lock_guard<std::mutex> lock { _consolidatedBackupsMutex };
+
+        auto it = _consolidatedBackups.find(fileName);
+        if (it != _consolidatedBackups.end()) {
+            return;
+        }
+
+        _consolidatedBackups[fileName] = {
+            ConsolidatedBackupInfo::CONSOLIDATING,
+            "",
+            ""
+        };
     }
 
     QDir backupDir { _backupDirectory };
     if (!backupDir.exists()) {
-        qCritical() << "Backup directory does not exist, bailing consolidation of backup";
+        markFailure("Backup directory does not exist, bailing consolidation of backup");
-        promise->resolve({ { "success", false } });
         return;
     }
 
     auto filePath = backupDir.absoluteFilePath(fileName);
+    
+    if (!QFile::exists(filePath)) {
+        markFailure("Backup does not exist");
+        return;
+    }
 
-    auto copyFilePath = QDir::tempPath() + "/" + fileName;
+    auto copyFilePath = _consolidatedBackupDirectory + "/" + fileName;
 
     {
         QFile copyFile(copyFilePath);
@@ -523,8 +563,7 @@ void DomainContentBackupManager::consolidateBackup(MiniPromise::Promise promise,
     }
     auto copySuccess = QFile::copy(filePath, copyFilePath);
     if (!copySuccess) {
-        qCritical() << "Failed to create copy of backup.";
+        markFailure("Failed to create copy of backup.");
-        promise->resolve({ { "success", false } });
         return;
     }
 
@@ -532,7 +571,7 @@ void DomainContentBackupManager::consolidateBackup(MiniPromise::Promise promise,
     if (!zip.open(QuaZip::mdAdd)) {
         qCritical() << "Could not open backup archive:" << filePath;
         qCritical() << "    ERROR:" << zip.getZipError();
-        promise->resolve({ { "success", false } });
+        markFailure("Could not open backup archive");
         return;
     }
 
@@ -544,14 +583,17 @@ void DomainContentBackupManager::consolidateBackup(MiniPromise::Promise promise,
 
     if (zip.getZipError() != UNZ_OK) {
         qCritical() << "Failed to consolidate backup: " << zip.getZipError();
-        promise->resolve({ { "success", false } });
+        markFailure("Failed to consolidate backup");
         return;
     }
 
-    promise->resolve({
+    {
-        { "success", true },
+        std::lock_guard<std::mutex> lock { _consolidatedBackupsMutex };
-        { "backupFilePath", copyFilePath }
+        auto& consolidatedBackup = _consolidatedBackups[fileName];
-    });
+        consolidatedBackup.state = ConsolidatedBackupInfo::COMPLETE_WITH_SUCCESS;
+        consolidatedBackup.absoluteFilePath = copyFilePath;
+    }
+
 }
 
 void DomainContentBackupManager::createManualBackup(MiniPromise::Promise promise, const QString& name) {

domain-server/src/DomainContentBackupManager.h

@@ -15,10 +15,15 @@
 #ifndef hifi_DomainContentBackupManager_h
 #define hifi_DomainContentBackupManager_h
 
+#include <RegisteredMetaTypes.h>
+
 #include <QString>
 #include <QVector>
 #include <QDateTime>
 
+#include <mutex>
+#include <unordered_map>
+
 #include <GenericThread.h>
 
 #include "BackupHandler.h"
@@ -38,6 +43,17 @@ struct BackupItemInfo {
     bool isManualBackup;
 };
 
+struct ConsolidatedBackupInfo {
+    enum State {
+        CONSOLIDATING,
+        COMPLETE_WITH_ERROR,
+        COMPLETE_WITH_SUCCESS
+    };
+    State state;
+    QString error;
+    QString absoluteFilePath;
+};
+
 class DomainContentBackupManager : public GenericThread {
     Q_OBJECT
 public:
@@ -61,6 +77,7 @@ public:
     void addBackupHandler(BackupHandlerPointer handler);
     void aboutToFinish();  /// call this to inform the persist thread that the owner is about to finish to support final persist
     void replaceData(QByteArray data);
+    ConsolidatedBackupInfo consolidateBackup(QString fileName);
 
 public slots:
     void getAllBackupsAndStatus(MiniPromise::Promise promise);
@@ -68,7 +85,6 @@ public slots:
     void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName);
     void recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup);
     void deleteBackup(MiniPromise::Promise promise, const QString& backupName);
-    void consolidateBackup(MiniPromise::Promise promise, QString fileName);
 
 signals:
     void loadCompleted();
@@ -91,11 +107,18 @@ protected:
 
     bool recoverFromBackupZip(const QString& backupName, QuaZip& backupZip);
 
+private slots:
+    void consolidateBackupInternal(QString fileName);
+
 private:
+    const QString _consolidatedBackupDirectory;
     const QString _backupDirectory;
     std::vector<BackupHandlerPointer> _backupHandlers;
     std::chrono::milliseconds _persistInterval { 0 };
 
+    std::mutex _consolidatedBackupsMutex;
+    std::unordered_map<QString, ConsolidatedBackupInfo> _consolidatedBackups;
+
     std::atomic<bool> _isRecovering { false };
     QString _recoveryFilename { };
 

domain-server/src/DomainServer.cpp

@@ -164,6 +164,8 @@ DomainServer::DomainServer(int argc, char* argv[]) :
     _iceServerAddr(ICE_SERVER_DEFAULT_HOSTNAME),
     _iceServerPort(ICE_SERVER_DEFAULT_PORT)
 {
+    PathUtils::removeTemporaryApplicationDirs();
+
     parseCommandLine();
 
     DependencyManager::set<tracing::Tracer>();
@@ -1933,6 +1935,7 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
     const QString URI_API_DOMAINS_ID = "/api/domains/";
     const QString URI_API_BACKUPS = "/api/backups";
     const QString URI_API_BACKUPS_ID = "/api/backups/";
+    const QString URI_API_BACKUPS_DOWNLOAD_ID = "/api/backups/download/";
     const QString URI_API_BACKUPS_RECOVER = "/api/backups/recover/";
 
     const QString UUID_REGEX_STRING = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
@@ -2133,37 +2136,40 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
             });
             _contentManager->getAllBackupsAndStatus(deferred);
             return true;
+        } else if (url.path().startsWith(URI_API_BACKUPS_DOWNLOAD_ID)) {
+            auto id = url.path().mid(QString(URI_API_BACKUPS_DOWNLOAD_ID).length());
+            auto info = _contentManager->consolidateBackup(id);
+
+            if (info.state == ConsolidatedBackupInfo::COMPLETE_WITH_SUCCESS) {
+                auto file { std::unique_ptr<QFile>(new QFile(info.absoluteFilePath)) };
+                if (file->open(QIODevice::ReadOnly)) {
+                    constexpr const char* CONTENT_TYPE_ZIP = "application/zip";
+                    auto downloadedFilename = id;
+                    downloadedFilename.replace(QRegularExpression(".zip$"), ".content.zip");
+                    auto contentDisposition = "attachment; filename=\"" + downloadedFilename + "\"";
+                    connectionPtr->respond(HTTPConnection::StatusCode200, std::move(file), CONTENT_TYPE_ZIP, {
+                        { "Content-Disposition", contentDisposition.toUtf8() }
+                    });
+                } else {
+                    qCritical(domain_server) << "Unable to load consolidated backup at:" << info.absoluteFilePath;
+                    connectionPtr->respond(HTTPConnection::StatusCode500, "Error opening backup");
+                }
+            } else if (info.state == ConsolidatedBackupInfo::COMPLETE_WITH_ERROR) {
+                connectionPtr->respond(HTTPConnection::StatusCode500, ("Error creating backup: " + info.error).toUtf8());
+            } else {
+                connectionPtr->respond(HTTPConnection::StatusCode400, "Backup unavailable");
+            }
+            return true;
         } else if (url.path().startsWith(URI_API_BACKUPS_ID)) {
             auto id = url.path().mid(QString(URI_API_BACKUPS_ID).length());
-            auto deferred = makePromise("consolidateBackup");
+            auto info = _contentManager->consolidateBackup(id);
-            deferred->then([connectionPtr, JSON_MIME_TYPE, id](QString error, QVariantMap result) {
-                if (!connectionPtr) {
-                    return;
-                }
 
-                QJsonObject rootJSON;
+            QJsonObject rootJSON {
-                auto success = result["success"].toBool();
+                { "complete", info.state == ConsolidatedBackupInfo::COMPLETE_WITH_SUCCESS },
-                if (success) {
+                { "error", info.error }
-                    auto path = result["backupFilePath"].toString();
+            };
-                    auto file { std::unique_ptr<QFile>(new QFile(path)) };
+            QJsonDocument docJSON { rootJSON };
-                    if (file->open(QIODevice::ReadOnly)) {
+            connectionPtr->respond(HTTPConnection::StatusCode200, docJSON.toJson(), JSON_MIME_TYPE.toUtf8());
-                        constexpr const char* CONTENT_TYPE_ZIP = "application/zip";
-
-                        auto downloadedFilename = id;
-                        downloadedFilename.replace(QRegularExpression(".zip$"), ".content.zip");
-                        auto contentDisposition = "attachment; filename=\"" + downloadedFilename + "\"";
-                        connectionPtr->respond(HTTPConnection::StatusCode200, std::move(file), CONTENT_TYPE_ZIP, {
-                            { "Content-Disposition", contentDisposition.toUtf8() }
-                        });
-                    } else {
-                        qCritical(domain_server) << "Unable to load consolidated backup at:" << path << result;
-                        connectionPtr->respond(HTTPConnection::StatusCode500, "Error opening backup");
-                    }
-                } else {
-                    connectionPtr->respond(HTTPConnection::StatusCode400);
-                }
-            });
-            _contentManager->consolidateBackup(deferred, id);
 
             return true;
         } else if (url.path() == URI_RESTART) {