diff --git a/domain-server/resources/web/content/js/content.js b/domain-server/resources/web/content/js/content.js index 346e846748..85bd9e68b3 100644 --- a/domain-server/resources/web/content/js/content.js +++ b/domain-server/resources/web/content/js/content.js @@ -10,10 +10,85 @@ $(document).ready(function(){ function progressBarHTML(extraClass, label) { var html = "
"; html += "
"; - html += label + "
"; + html += ""; return html; } + function showUploadProgress(title) { + swal({ + title: title, + text: progressBarHTML('upload-content-progress', 'Upload'), + html: true, + showConfirmButton: false, + allowEscapeKey: false + }); + } + + function uploadNextChunk(file, offset, id) { + if (offset == undefined) { + offset = 0; + } + if (id == undefined) { + // Identify this upload session + id = Math.round(Math.random() * 2147483647); + } + + var fileSize = file.size; + var filename = file.name; + + var CHUNK_SIZE = 1048576; // 1 MiB + + var isFinal = Boolean(fileSize - offset <= CHUNK_SIZE); + var nextChunkSize = Math.min(fileSize - offset, CHUNK_SIZE); + var chunk = file.slice(offset, offset + nextChunkSize, file.type); + var chunkFormData = new FormData(); + + var formItemName = 'restore-file-chunk'; + if (offset == 0) { + formItemName = isFinal ? 'restore-file-chunk-only' : 'restore-file-chunk-initial'; + } else if (isFinal) { + formItemName = 'restore-file-chunk-final'; + } + + chunkFormData.append(formItemName, chunk, filename); + var ajaxParams = { + url: '/content/upload', + type: 'POST', + timeout: 30000, // 30 s + headers: {"X-Session-Id": id}, + cache: false, + processData: false, + contentType: false, + data: chunkFormData + }; + + var ajaxObject = $.ajax(ajaxParams); + ajaxObject.fail(function (jqXHR, textStatus, errorThrown) { + showErrorMessage( + "Error", + "There was a problem restoring domain content.\n" + + "Please ensure that the content archive or entity file is valid and try again." + ); + }); + + updateProgressBars($('.upload-content-progress'), (offset + nextChunkSize) * 100 / fileSize); + + if (!isFinal) { + ajaxObject.done(function (data, textStatus, jqXHR) + { uploadNextChunk(file, offset + CHUNK_SIZE, id); }); + } else { + ajaxObject.done(function(data, textStatus, jqXHR) { + isRestoring = true; + + // immediately reload backup information since one should be restoring now + reloadBackupInformation(); + + swal.close(); + }); + } + + } + function setupBackupUpload() { // construct the HTML needed for the settings backup panel var html = "
"; @@ -50,34 +125,10 @@ $(document).ready(function(){ "Restore content", function() { var files = $('#' + RESTORE_SETTINGS_FILE_ID).prop('files'); + var file = files[0]; - var fileFormData = new FormData(); - fileFormData.append('restore-file', files[0]); - - showSpinnerAlert("Uploading content to restore"); - - $.ajax({ - url: '/content/upload', - type: 'POST', - timeout: 3600000, // Set timeout to 1h - cache: false, - processData: false, - contentType: false, - data: fileFormData - }).done(function(data, textStatus, jqXHR) { - isRestoring = true; - - // immediately reload backup information since one should be restoring now - reloadBackupInformation(); - - swal.close(); - }).fail(function(jqXHR, textStatus, errorThrown) { - showErrorMessage( - "Error", - "There was a problem restoring domain content.\n" - + "Please ensure that the content archive or entity file is valid and try again." - ); - }); + showUploadProgress("Uploading " + file.name); + uploadNextChunk(file); } ); }); @@ -168,6 +219,11 @@ $(document).ready(function(){ checkBackupStatus(); }); + function updateProgressBars($progressBar, value) { + $progressBar.attr('aria-valuenow', value).attr('style', 'width: ' + value + '%'); + $progressBar.find('.ongoing-msg').html(" " + Math.round(value) + "%"); + } + function reloadBackupInformation() { // make a GET request to get backup information to populate the table $.ajax({ @@ -204,11 +260,6 @@ $(document).ready(function(){ + "
  • Delete
  • "; } - function updateProgressBars($progressBar, value) { - $progressBar.attr('aria-valuenow', value).attr('style', 'width: ' + value + '%'); - $progressBar.find('.sr-only').html(value + "% Complete"); - } - // before we add any new rows and update existing ones // remove our flag for active rows $('.' + ACTIVE_BACKUP_ROW_CLASS).removeClass(ACTIVE_BACKUP_ROW_CLASS); diff --git a/domain-server/src/DomainContentBackupManager.cpp b/domain-server/src/DomainContentBackupManager.cpp index 518ed73f9e..3b8180e49e 100644 --- a/domain-server/src/DomainContentBackupManager.cpp +++ b/domain-server/src/DomainContentBackupManager.cpp @@ -348,6 +348,27 @@ void DomainContentBackupManager::recoverFromUploadedBackup(MiniPromise::Promise }); } +void DomainContentBackupManager::recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename) { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "recoverFromUploadedFile", Q_ARG(MiniPromise::Promise, promise), + Q_ARG(QString, uploadedFilename)); + return; + } + + qDebug() << "Recovering from uploaded file -" << uploadedFilename; + + QFile uploadedFile(uploadedFilename); + QuaZip uploadedZip { &uploadedFile }; + + QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip"; + + bool success = recoverFromBackupZip(backupName, uploadedZip); + + promise->resolve({ + { "success", success } + }); +} + std::vector DomainContentBackupManager::getAllBackups() { QDir backupDir { _backupDirectory }; diff --git a/domain-server/src/DomainContentBackupManager.h b/domain-server/src/DomainContentBackupManager.h index 2b07afe0b3..4af3ae5bfd 100644 --- a/domain-server/src/DomainContentBackupManager.h +++ b/domain-server/src/DomainContentBackupManager.h @@ -86,6 +86,7 @@ public slots: void createManualBackup(MiniPromise::Promise promise, const QString& name); void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName); void recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup); + void recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename); void deleteBackup(MiniPromise::Promise promise, const QString& backupName); signals: diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 8cf033c130..69f16af8b3 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -2258,46 +2258,18 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url // check the file extension to see what kind of file this is // to make sure we handle this filetype for a content restore auto dispositionValue = QString(firstFormData.first.value("Content-Disposition")); - auto formDataFilenameRegex = QRegExp("filename=\"(.+)\""); - auto matchIndex = formDataFilenameRegex.indexIn(dispositionValue); + QRegExp formDataFieldsRegex(R":(name="(restore-file.*)".*filename="(.+)"):"); + auto matchIndex = formDataFieldsRegex.indexIn(dispositionValue); + QString formItemName = ""; QString uploadedFilename = ""; if (matchIndex != -1) { - uploadedFilename = formDataFilenameRegex.cap(1); - } - - if (uploadedFilename.endsWith(".json", Qt::CaseInsensitive) - || uploadedFilename.endsWith(".json.gz", Qt::CaseInsensitive)) { - // invoke our method to hand the new octree file off to the octree server - QMetaObject::invokeMethod(this, "handleOctreeFileReplacement", - Qt::QueuedConnection, Q_ARG(QByteArray, firstFormData.second)); - - // respond with a 200 for success - connection->respond(HTTPConnection::StatusCode200); - } else if (uploadedFilename.endsWith(".zip", Qt::CaseInsensitive)) { - auto deferred = makePromise("recoverFromUploadedBackup"); - - deferred->then([connectionPtr, JSON_MIME_TYPE](QString error, QVariantMap result) { - if (!connectionPtr) { - return; - } - - QJsonObject rootJSON; - auto success = result["success"].toBool(); - rootJSON["success"] = success; - QJsonDocument docJSON(rootJSON); - connectionPtr->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(), - JSON_MIME_TYPE.toUtf8()); - }); - - _contentManager->recoverFromUploadedBackup(deferred, firstFormData.second); - - return true; - } else { - // we don't have handling for this filetype, send back a 400 for failure - connection->respond(HTTPConnection::StatusCode400); + formItemName = formDataFieldsRegex.cap(1); + uploadedFilename = formDataFieldsRegex.cap(2); } + // Received a chunk + processPendingContent(connection, formItemName, uploadedFilename, firstFormData.second); } else { // respond with a 400 for failure connection->respond(HTTPConnection::StatusCode400); @@ -2546,6 +2518,72 @@ bool DomainServer::handleHTTPSRequest(HTTPSConnection* connection, const QUrl &u } } +bool DomainServer::processPendingContent(HTTPConnection* connection, QString itemName, QString filename, QByteArray dataChunk) { + static const QString UPLOAD_SESSION_KEY { "X-Session-Id" }; + QByteArray sessionIdBytes = connection->requestHeader(UPLOAD_SESSION_KEY); + int sessionId = sessionIdBytes.toInt(); + + bool newUpload = itemName == "restore-file" || itemName == "restore-file-chunk-initial" || itemName == "restore-file-chunk-only"; + + if (filename.endsWith(".zip", Qt::CaseInsensitive)) { + static const QString TEMPORARY_CONTENT_FILEPATH { QDir::tempPath() + "/hifiUploadContent_XXXXXX.zip" }; + + if (_pendingContentFiles.find(sessionId) == _pendingContentFiles.end()) { + if (!newUpload) { + return false; + } + std::unique_ptr newTemp(new QTemporaryFile(TEMPORARY_CONTENT_FILEPATH)); + _pendingContentFiles[sessionId] = std::move(newTemp); + } else if (newUpload) { + qCDebug(domain_server) << "New upload received using existing session ID"; + _pendingContentFiles[sessionId]->resize(0); + } + + QTemporaryFile& _pendingFileContent = *_pendingContentFiles[sessionId]; + if (!_pendingFileContent.open()) { + _pendingContentFiles.erase(sessionId); + connection->respond(HTTPConnection::StatusCode400); + return false; + } + _pendingFileContent.seek(_pendingFileContent.size()); + _pendingFileContent.write(dataChunk); + _pendingFileContent.close(); + + // Respond immediately - will timeout if we wait for restore. + connection->respond(HTTPConnection::StatusCode200); + if (itemName == "restore-file" || itemName == "restore-file-chunk-final" || itemName == "restore-file-chunk-only") { + auto deferred = makePromise("recoverFromUploadedBackup"); + + deferred->then([this, sessionId](QString error, QVariantMap result) { + _pendingContentFiles.erase(sessionId); + }); + + _contentManager->recoverFromUploadedFile(deferred, _pendingFileContent.fileName()); + } + } else if (filename.endsWith(".json", Qt::CaseInsensitive) + || filename.endsWith(".json.gz", Qt::CaseInsensitive)) { + if (_pendingUploadedContents.find(sessionId) == _pendingUploadedContents.end() && !newUpload) { + qCDebug(domain_server) << "Json upload with invalid session ID received"; + return false; + } + QByteArray& _pendingUploadedContent = _pendingUploadedContents[sessionId]; + _pendingUploadedContent += dataChunk; + connection->respond(HTTPConnection::StatusCode200); + + if (itemName == "restore-file" || itemName == "restore-file-chunk-final" || itemName == "restore-file-chunk-only") { + // invoke our method to hand the new octree file off to the octree server + QMetaObject::invokeMethod(this, "handleOctreeFileReplacement", + Qt::QueuedConnection, Q_ARG(QByteArray, _pendingUploadedContent)); + _pendingUploadedContents.erase(sessionId); + } + } else { + connection->respond(HTTPConnection::StatusCode400); + return false; + } + + return true; +} + HTTPSConnection* DomainServer::connectionFromReplyWithState(QNetworkReply* reply) { // grab the UUID state property from the reply QUuid stateUUID = reply->property(STATE_QUERY_KEY.toLocal8Bit()).toUuid(); diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index e2bddc1aa5..f0c20241a2 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -209,6 +210,8 @@ private: HTTPSConnection* connectionFromReplyWithState(QNetworkReply* reply); + bool processPendingContent(HTTPConnection* connection, QString itemName, QString filename, QByteArray dataChunk); + bool forwardMetaverseAPIRequest(HTTPConnection* connection, const QString& metaversePath, const QString& requestSubobject, @@ -281,6 +284,9 @@ private: QHash> _pendingOAuthConnections; + std::unordered_map _pendingUploadedContents; + std::unordered_map> _pendingContentFiles; + QThread _assetClientThread; };