From 22c3f5239adbc48fbd3ed05b4617a8f09cd1a0b1 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Thu, 15 Nov 2018 16:59:52 -0800 Subject: [PATCH] Relay upoaded content.zip chunks to temp file Entities uploads still build in-memory. Move out chunk handling to new routine. --- .../resources/web/content/js/content.js | 4 +- .../src/DomainContentBackupManager.cpp | 21 +++++ .../src/DomainContentBackupManager.h | 1 + domain-server/src/DomainServer.cpp | 77 ++++++++++--------- domain-server/src/DomainServer.h | 3 +- 5 files changed, 67 insertions(+), 39 deletions(-) diff --git a/domain-server/resources/web/content/js/content.js b/domain-server/resources/web/content/js/content.js index 78e78266d3..365c5e8403 100644 --- a/domain-server/resources/web/content/js/content.js +++ b/domain-server/resources/web/content/js/content.js @@ -60,7 +60,7 @@ $(document).ready(function(){ ); }); - updateProgressBars($('.upload-content-progress'), Math.round((offset + nextChunkSize) * 100 / fileSize)); + updateProgressBars($('.upload-content-progress'), (offset + nextChunkSize) * 100 / fileSize); if (!isFinal) { ajaxObject.done(function (data, textStatus, jqXHR) @@ -210,7 +210,7 @@ $(document).ready(function(){ function updateProgressBars($progressBar, value) { $progressBar.attr('aria-valuenow', value).attr('style', 'width: ' + value + '%'); - $progressBar.find('.ongoing-msg').html(" " + value + "% Complete"); + $progressBar.find('.ongoing-msg').html(" " + Math.round(value) + "% Complete"); } function reloadBackupInformation() { diff --git a/domain-server/src/DomainContentBackupManager.cpp b/domain-server/src/DomainContentBackupManager.cpp index 518ed73f9e..3b8180e49e 100644 --- a/domain-server/src/DomainContentBackupManager.cpp +++ b/domain-server/src/DomainContentBackupManager.cpp @@ -348,6 +348,27 @@ void DomainContentBackupManager::recoverFromUploadedBackup(MiniPromise::Promise }); } +void DomainContentBackupManager::recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename) { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "recoverFromUploadedFile", Q_ARG(MiniPromise::Promise, promise), + Q_ARG(QString, uploadedFilename)); + return; + } + + qDebug() << "Recovering from uploaded file -" << uploadedFilename; + + QFile uploadedFile(uploadedFilename); + QuaZip uploadedZip { &uploadedFile }; + + QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip"; + + bool success = recoverFromBackupZip(backupName, uploadedZip); + + promise->resolve({ + { "success", success } + }); +} + std::vector DomainContentBackupManager::getAllBackups() { QDir backupDir { _backupDirectory }; diff --git a/domain-server/src/DomainContentBackupManager.h b/domain-server/src/DomainContentBackupManager.h index 2b07afe0b3..4af3ae5bfd 100644 --- a/domain-server/src/DomainContentBackupManager.h +++ b/domain-server/src/DomainContentBackupManager.h @@ -86,6 +86,7 @@ public slots: void createManualBackup(MiniPromise::Promise promise, const QString& name); void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName); void recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup); + void recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename); void deleteBackup(MiniPromise::Promise promise, const QString& backupName); signals: diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 117313462b..3e68cd0fc4 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -2268,17 +2268,8 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url uploadedFilename = formDataFieldsRegex.cap(2); } - _pendingUploadedContent += firstFormData.second; - if (formItemName == "restore-file-chunk") { - // Received another chunk - connection->respond(HTTPConnection::StatusCode200); - } else if (formItemName == "restore-file-chunk-final") { - readPendingContent(connection, uploadedFilename); - } else if (formItemName == "restore-file") { - readPendingContent(connection, uploadedFilename); - } else { - connection->respond(HTTPConnection::StatusCode400); - } + // Received a chunk + processPendingContent(connection, formItemName, uploadedFilename, firstFormData.second); } else { // respond with a 400 for failure connection->respond(HTTPConnection::StatusCode400); @@ -2527,37 +2518,51 @@ bool DomainServer::handleHTTPSRequest(HTTPSConnection* connection, const QUrl &u } } -void DomainServer::readPendingContent(HTTPConnection* connection, QString filename) { - if (filename.endsWith(".json", Qt::CaseInsensitive) - || filename.endsWith(".json.gz", Qt::CaseInsensitive)) { - // invoke our method to hand the new octree file off to the octree server - QMetaObject::invokeMethod(this, "handleOctreeFileReplacement", - Qt::QueuedConnection, Q_ARG(QByteArray, _pendingUploadedContent)); +bool DomainServer::processPendingContent(HTTPConnection* connection, QString itemName, QString filename, QByteArray dataChunk) { + if (filename.endsWith(".zip", Qt::CaseInsensitive)) { + static const QString TEMPORARY_CONTENT_FILEPATH { QDir::tempPath() + "/hifiUploadContent_XXXXXX.zip" }; - // respond with a 200 for success + if (!_pendingFileContent) { + _pendingFileContent = std::make_unique(TEMPORARY_CONTENT_FILEPATH); + } + if (!_pendingFileContent->open()) { + _pendingFileContent = nullptr; + connection->respond(HTTPConnection::StatusCode400); + return false; + } + _pendingFileContent->seek(_pendingFileContent->size()); + _pendingFileContent->write(dataChunk); + _pendingFileContent->close(); + + // Respond immediately - will timeout if we wait for restore. connection->respond(HTTPConnection::StatusCode200); - _pendingUploadedContent.clear(); - } else if (filename.endsWith(".zip", Qt::CaseInsensitive)) { - auto deferred = makePromise("recoverFromUploadedBackup"); + if (itemName == "restore-file-chunk-final" || itemName == "restore-file") { + auto deferred = makePromise("recoverFromUploadedBackup"); - QPointer connectionPtr(connection); - const QString JSON_MIME_TYPE = "application/json"; - deferred->then([connectionPtr, JSON_MIME_TYPE, this](QString error, QVariantMap result) { - if (!connectionPtr) { - return; - } + deferred->then([this](QString error, QVariantMap result) { + _pendingFileContent = nullptr; + }); - QJsonObject rootJSON; - auto success = result["success"].toBool(); - rootJSON["success"] = success; - QJsonDocument docJSON(rootJSON); - connectionPtr->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(), - JSON_MIME_TYPE.toUtf8()); + _contentManager->recoverFromUploadedFile(deferred, _pendingFileContent->fileName()); + } else { + } + } else if (filename.endsWith(".json", Qt::CaseInsensitive) + || filename.endsWith(".json.gz", Qt::CaseInsensitive)) { + _pendingUploadedContent += dataChunk; + connection->respond(HTTPConnection::StatusCode200); + + if (itemName == "restore-file-chunk-final" || itemName == "restore-file") { + // invoke our method to hand the new octree file off to the octree server + QMetaObject::invokeMethod(this, "handleOctreeFileReplacement", + Qt::QueuedConnection, Q_ARG(QByteArray, _pendingUploadedContent)); _pendingUploadedContent.clear(); - }); - - _contentManager->recoverFromUploadedBackup(deferred, _pendingUploadedContent); + } + } else { + connection->respond(HTTPConnection::StatusCode400); + return false; } + + return true; } HTTPSConnection* DomainServer::connectionFromReplyWithState(QNetworkReply* reply) { diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index b2ef933bc3..73e23bb3fe 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -209,7 +209,7 @@ private: HTTPSConnection* connectionFromReplyWithState(QNetworkReply* reply); - void readPendingContent(HTTPConnection* connection, QString filename); + bool processPendingContent(HTTPConnection* connection, QString itemName, QString filename, QByteArray dataChunk); bool forwardMetaverseAPIRequest(HTTPConnection* connection, const QString& metaversePath, @@ -284,6 +284,7 @@ private: QHash> _pendingOAuthConnections; QByteArray _pendingUploadedContent; + std::unique_ptr _pendingFileContent; QThread _assetClientThread; };