Merge pull request #14391 from SimonWalton-HiFi/upload-to-ds-chunked

Upload content to domain server in multiple chunks

domain-server/resources/web/content/js/content.js

@@ -10,10 +10,85 @@ $(document).ready(function(){
   function progressBarHTML(extraClass, label) {
     var html = "<div class='progress'>";
     html += "<div class='" + extraClass + " progress-bar progress-bar-success progress-bar-striped active' role='progressbar' aria-valuemin='0' aria-valuemax='100'>";
-    html += label + "<span class='sr-only'></span></div></div>";
+    html += "<span class='ongoing-msg'></span></div></div>";
     return html;
   }
 
+  function showUploadProgress(title) {
+    swal({
+      title: title,
+      text: progressBarHTML('upload-content-progress', 'Upload'),
+      html: true,
+      showConfirmButton: false,
+      allowEscapeKey: false
+    });
+  }
+
+  function uploadNextChunk(file, offset, id) {
+      if (offset == undefined) {
+          offset = 0;
+      }
+      if (id == undefined) {
+          // Identify this upload session
+          id = Math.round(Math.random() * 2147483647);
+      }
+
+      var fileSize = file.size;
+      var filename = file.name;
+
+      var CHUNK_SIZE = 1048576; // 1 MiB
+
+      var isFinal = Boolean(fileSize - offset <= CHUNK_SIZE);
+      var nextChunkSize = Math.min(fileSize - offset, CHUNK_SIZE);
+      var chunk = file.slice(offset, offset + nextChunkSize, file.type);
+      var chunkFormData = new FormData();
+
+      var formItemName = 'restore-file-chunk';
+      if (offset == 0) {
+          formItemName = isFinal ? 'restore-file-chunk-only' : 'restore-file-chunk-initial';
+      } else if (isFinal) {
+          formItemName = 'restore-file-chunk-final';
+      }
+
+      chunkFormData.append(formItemName, chunk, filename);
+      var ajaxParams = {
+        url: '/content/upload',
+        type: 'POST',
+        timeout: 30000, // 30 s
+        headers: {"X-Session-Id": id},
+        cache: false,
+        processData: false,
+        contentType: false,
+        data: chunkFormData
+      };
+
+      var ajaxObject = $.ajax(ajaxParams);
+      ajaxObject.fail(function (jqXHR, textStatus, errorThrown) {
+        showErrorMessage(
+          "Error",
+          "There was a problem restoring domain content.\n"
+          + "Please ensure that the content archive or entity file is valid and try again."
+        );
+      });
+
+      updateProgressBars($('.upload-content-progress'), (offset + nextChunkSize) * 100 / fileSize);
+
+      if (!isFinal) {
+        ajaxObject.done(function (data, textStatus, jqXHR)
+          { uploadNextChunk(file, offset + CHUNK_SIZE, id); });
+      } else {
+        ajaxObject.done(function(data, textStatus, jqXHR) {
+          isRestoring = true;
+
+          // immediately reload backup information since one should be restoring now
+          reloadBackupInformation();
+
+          swal.close();
+        });
+      }
+      
+    }
+  
   function setupBackupUpload() {
     // construct the HTML needed for the settings backup panel
     var html = "<div class='form-group'><div id='" + UPLOAD_CONTENT_ALLOWED_DIV_ID + "'>";
@@ -50,34 +125,10 @@ $(document).ready(function(){
       "Restore content",
       function() {
         var files = $('#' + RESTORE_SETTINGS_FILE_ID).prop('files');
+        var file = files[0];
 
-        var fileFormData = new FormData();
+        showUploadProgress("Uploading " + file.name);
-        fileFormData.append('restore-file', files[0]);
+        uploadNextChunk(file);
-
-        showSpinnerAlert("Uploading content to restore");
-
-        $.ajax({
-          url: '/content/upload',
-          type: 'POST',
-          timeout: 3600000, // Set timeout to 1h
-          cache: false,
-          processData: false,
-          contentType: false,
-          data: fileFormData
-        }).done(function(data, textStatus, jqXHR) {
-          isRestoring = true;
-
-          // immediately reload backup information since one should be restoring now
-          reloadBackupInformation();
-
-          swal.close();
-        }).fail(function(jqXHR, textStatus, errorThrown) {
-          showErrorMessage(
-            "Error",
-            "There was a problem restoring domain content.\n"
-            + "Please ensure that the content archive or entity file is valid and try again."
-          );
-        });
       }
     );
   });
@@ -168,6 +219,11 @@ $(document).ready(function(){
     checkBackupStatus();
   });
 
+  function updateProgressBars($progressBar, value) {
+    $progressBar.attr('aria-valuenow', value).attr('style', 'width: ' + value + '%');
+    $progressBar.find('.ongoing-msg').html(" " + Math.round(value) + "%");
+  }
+
   function reloadBackupInformation() {
     // make a GET request to get backup information to populate the table
     $.ajax({
@@ -204,11 +260,6 @@ $(document).ready(function(){
           + "<li><a class='" + BACKUP_DELETE_LINK_CLASS + "' href='#' target='_blank'>Delete</a></li></ul></div></td>";
       }
 
-      function updateProgressBars($progressBar, value) {
-        $progressBar.attr('aria-valuenow', value).attr('style', 'width: ' + value + '%');
-        $progressBar.find('.sr-only').html(value + "% Complete");
-      }
-
       // before we add any new rows and update existing ones
       // remove our flag for active rows
       $('.' + ACTIVE_BACKUP_ROW_CLASS).removeClass(ACTIVE_BACKUP_ROW_CLASS);

domain-server/src/DomainContentBackupManager.cpp

@@ -348,6 +348,27 @@ void DomainContentBackupManager::recoverFromUploadedBackup(MiniPromise::Promise
     });
 }
 
+void DomainContentBackupManager::recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "recoverFromUploadedFile", Q_ARG(MiniPromise::Promise, promise),
+            Q_ARG(QString, uploadedFilename));
+        return;
+    }
+
+    qDebug() << "Recovering from uploaded file -" << uploadedFilename;
+
+    QFile uploadedFile(uploadedFilename);
+    QuaZip uploadedZip { &uploadedFile };
+
+    QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip";
+
+    bool success = recoverFromBackupZip(backupName, uploadedZip);
+
+    promise->resolve({
+        { "success", success }
+        });
+}
+
 std::vector<BackupItemInfo> DomainContentBackupManager::getAllBackups() {
 
     QDir backupDir { _backupDirectory };

domain-server/src/DomainContentBackupManager.h

@@ -86,6 +86,7 @@ public slots:
     void createManualBackup(MiniPromise::Promise promise, const QString& name);
     void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName);
     void recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup);
+    void recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename);
     void deleteBackup(MiniPromise::Promise promise, const QString& backupName);
 
 signals:

domain-server/src/DomainServer.cpp

@@ -2258,46 +2258,18 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
                 // check the file extension to see what kind of file this is
                 // to make sure we handle this filetype for a content restore
                 auto dispositionValue = QString(firstFormData.first.value("Content-Disposition"));
-                auto formDataFilenameRegex = QRegExp("filename=\"(.+)\"");
+                QRegExp formDataFieldsRegex(R":(name="(restore-file.*)".*filename="(.+)"):");
-                auto matchIndex = formDataFilenameRegex.indexIn(dispositionValue);
+                auto matchIndex = formDataFieldsRegex.indexIn(dispositionValue);
 
+                QString formItemName = "";
                 QString uploadedFilename = "";
                 if (matchIndex != -1) {
-                    uploadedFilename = formDataFilenameRegex.cap(1);
+                    formItemName = formDataFieldsRegex.cap(1);
-                }
+                    uploadedFilename = formDataFieldsRegex.cap(2);
-
-                if (uploadedFilename.endsWith(".json", Qt::CaseInsensitive)
-                    || uploadedFilename.endsWith(".json.gz", Qt::CaseInsensitive)) {
-                    // invoke our method to hand the new octree file off to the octree server
-                    QMetaObject::invokeMethod(this, "handleOctreeFileReplacement",
-                                              Qt::QueuedConnection, Q_ARG(QByteArray, firstFormData.second));
-
-                    // respond with a 200 for success
-                    connection->respond(HTTPConnection::StatusCode200);
-                } else if (uploadedFilename.endsWith(".zip", Qt::CaseInsensitive)) {
-                    auto deferred = makePromise("recoverFromUploadedBackup");
-
-                    deferred->then([connectionPtr, JSON_MIME_TYPE](QString error, QVariantMap result) {
-                        if (!connectionPtr) {
-                            return;
-                        }
-
-                        QJsonObject rootJSON;
-                        auto success = result["success"].toBool();
-                        rootJSON["success"] = success;
-                        QJsonDocument docJSON(rootJSON);
-                        connectionPtr->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
-                                            JSON_MIME_TYPE.toUtf8());
-                    });
-
-                    _contentManager->recoverFromUploadedBackup(deferred, firstFormData.second);
-
-                    return true;
-                } else {
-                    // we don't have handling for this filetype, send back a 400 for failure
-                    connection->respond(HTTPConnection::StatusCode400);
                 }
 
+                // Received a chunk
+                processPendingContent(connection, formItemName, uploadedFilename, firstFormData.second);
             } else {
                 // respond with a 400 for failure
                 connection->respond(HTTPConnection::StatusCode400);
@@ -2546,6 +2518,72 @@ bool DomainServer::handleHTTPSRequest(HTTPSConnection* connection, const QUrl &u
     }
 }
 
+bool DomainServer::processPendingContent(HTTPConnection* connection, QString itemName, QString filename, QByteArray dataChunk) {
+    static const QString UPLOAD_SESSION_KEY { "X-Session-Id" };
+    QByteArray sessionIdBytes = connection->requestHeader(UPLOAD_SESSION_KEY);
+    int sessionId = sessionIdBytes.toInt();
+
+    bool newUpload = itemName == "restore-file" || itemName == "restore-file-chunk-initial" || itemName == "restore-file-chunk-only";
+
+    if (filename.endsWith(".zip", Qt::CaseInsensitive)) {
+        static const QString TEMPORARY_CONTENT_FILEPATH { QDir::tempPath() + "/hifiUploadContent_XXXXXX.zip" };
+
+        if (_pendingContentFiles.find(sessionId) == _pendingContentFiles.end()) {
+            if (!newUpload) {
+                return false;
+            }
+            std::unique_ptr<QTemporaryFile> newTemp(new QTemporaryFile(TEMPORARY_CONTENT_FILEPATH));
+            _pendingContentFiles[sessionId] = std::move(newTemp);
+        } else if (newUpload) {
+            qCDebug(domain_server) << "New upload received using existing session ID";
+            _pendingContentFiles[sessionId]->resize(0);
+        }
+
+        QTemporaryFile& _pendingFileContent = *_pendingContentFiles[sessionId];
+        if (!_pendingFileContent.open()) {
+            _pendingContentFiles.erase(sessionId);
+            connection->respond(HTTPConnection::StatusCode400);
+            return false;
+        }
+        _pendingFileContent.seek(_pendingFileContent.size());
+        _pendingFileContent.write(dataChunk);
+        _pendingFileContent.close();
+        
+        // Respond immediately - will timeout if we wait for restore.
+        connection->respond(HTTPConnection::StatusCode200);
+        if (itemName == "restore-file" || itemName == "restore-file-chunk-final" || itemName == "restore-file-chunk-only") {
+            auto deferred = makePromise("recoverFromUploadedBackup");
+
+            deferred->then([this, sessionId](QString error, QVariantMap result) {
+                _pendingContentFiles.erase(sessionId);
+            });
+
+            _contentManager->recoverFromUploadedFile(deferred, _pendingFileContent.fileName());
+        }
+    } else if (filename.endsWith(".json", Qt::CaseInsensitive)
+        || filename.endsWith(".json.gz", Qt::CaseInsensitive)) {
+        if (_pendingUploadedContents.find(sessionId) == _pendingUploadedContents.end() && !newUpload) {
+            qCDebug(domain_server) << "Json upload with invalid session ID received";
+            return false;
+        }
+        QByteArray& _pendingUploadedContent = _pendingUploadedContents[sessionId];
+        _pendingUploadedContent += dataChunk;
+        connection->respond(HTTPConnection::StatusCode200);
+
+        if (itemName == "restore-file" || itemName == "restore-file-chunk-final" || itemName == "restore-file-chunk-only") {
+            // invoke our method to hand the new octree file off to the octree server
+            QMetaObject::invokeMethod(this, "handleOctreeFileReplacement",
+                Qt::QueuedConnection, Q_ARG(QByteArray, _pendingUploadedContent));
+            _pendingUploadedContents.erase(sessionId);
+        }
+    } else {
+        connection->respond(HTTPConnection::StatusCode400);
+        return false;
+    }
+
+    return true;
+}
+
 HTTPSConnection* DomainServer::connectionFromReplyWithState(QNetworkReply* reply) {
     // grab the UUID state property from the reply
     QUuid stateUUID = reply->property(STATE_QUERY_KEY.toLocal8Bit()).toUuid();

domain-server/src/DomainServer.h

@@ -20,6 +20,7 @@
 #include <QtCore/QStringList>
 #include <QtCore/QThread>
 #include <QtCore/QUrl>
+#include <QHostAddress>
 #include <QAbstractNativeEventFilter>
 
 #include <Assignment.h>
@@ -209,6 +210,8 @@ private:
 
     HTTPSConnection* connectionFromReplyWithState(QNetworkReply* reply);
 
+    bool processPendingContent(HTTPConnection* connection, QString itemName, QString filename, QByteArray dataChunk);
+
     bool forwardMetaverseAPIRequest(HTTPConnection* connection,
                                     const QString& metaversePath,
                                     const QString& requestSubobject,
@@ -281,6 +284,9 @@ private:
 
     QHash<QUuid, QPointer<HTTPSConnection>> _pendingOAuthConnections;
 
+    std::unordered_map<int, QByteArray> _pendingUploadedContents;
+    std::unordered_map<int, std::unique_ptr<QTemporaryFile>> _pendingContentFiles;
+
     QThread _assetClientThread;
 };