mirror of
https://github.com/overte-org/overte.git
synced 2025-08-09 12:28:02 +02:00
Merge pull request #14391 from SimonWalton-HiFi/upload-to-ds-chunked
Upload content to domain server in multiple chunks
This commit is contained in:
commit
9b886ff725
5 changed files with 185 additions and 68 deletions
|
@ -10,10 +10,85 @@ $(document).ready(function(){
|
||||||
function progressBarHTML(extraClass, label) {
|
function progressBarHTML(extraClass, label) {
|
||||||
var html = "<div class='progress'>";
|
var html = "<div class='progress'>";
|
||||||
html += "<div class='" + extraClass + " progress-bar progress-bar-success progress-bar-striped active' role='progressbar' aria-valuemin='0' aria-valuemax='100'>";
|
html += "<div class='" + extraClass + " progress-bar progress-bar-success progress-bar-striped active' role='progressbar' aria-valuemin='0' aria-valuemax='100'>";
|
||||||
html += label + "<span class='sr-only'></span></div></div>";
|
html += "<span class='ongoing-msg'></span></div></div>";
|
||||||
return html;
|
return html;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function showUploadProgress(title) {
|
||||||
|
swal({
|
||||||
|
title: title,
|
||||||
|
text: progressBarHTML('upload-content-progress', 'Upload'),
|
||||||
|
html: true,
|
||||||
|
showConfirmButton: false,
|
||||||
|
allowEscapeKey: false
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function uploadNextChunk(file, offset, id) {
|
||||||
|
if (offset == undefined) {
|
||||||
|
offset = 0;
|
||||||
|
}
|
||||||
|
if (id == undefined) {
|
||||||
|
// Identify this upload session
|
||||||
|
id = Math.round(Math.random() * 2147483647);
|
||||||
|
}
|
||||||
|
|
||||||
|
var fileSize = file.size;
|
||||||
|
var filename = file.name;
|
||||||
|
|
||||||
|
var CHUNK_SIZE = 1048576; // 1 MiB
|
||||||
|
|
||||||
|
var isFinal = Boolean(fileSize - offset <= CHUNK_SIZE);
|
||||||
|
var nextChunkSize = Math.min(fileSize - offset, CHUNK_SIZE);
|
||||||
|
var chunk = file.slice(offset, offset + nextChunkSize, file.type);
|
||||||
|
var chunkFormData = new FormData();
|
||||||
|
|
||||||
|
var formItemName = 'restore-file-chunk';
|
||||||
|
if (offset == 0) {
|
||||||
|
formItemName = isFinal ? 'restore-file-chunk-only' : 'restore-file-chunk-initial';
|
||||||
|
} else if (isFinal) {
|
||||||
|
formItemName = 'restore-file-chunk-final';
|
||||||
|
}
|
||||||
|
|
||||||
|
chunkFormData.append(formItemName, chunk, filename);
|
||||||
|
var ajaxParams = {
|
||||||
|
url: '/content/upload',
|
||||||
|
type: 'POST',
|
||||||
|
timeout: 30000, // 30 s
|
||||||
|
headers: {"X-Session-Id": id},
|
||||||
|
cache: false,
|
||||||
|
processData: false,
|
||||||
|
contentType: false,
|
||||||
|
data: chunkFormData
|
||||||
|
};
|
||||||
|
|
||||||
|
var ajaxObject = $.ajax(ajaxParams);
|
||||||
|
ajaxObject.fail(function (jqXHR, textStatus, errorThrown) {
|
||||||
|
showErrorMessage(
|
||||||
|
"Error",
|
||||||
|
"There was a problem restoring domain content.\n"
|
||||||
|
+ "Please ensure that the content archive or entity file is valid and try again."
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
updateProgressBars($('.upload-content-progress'), (offset + nextChunkSize) * 100 / fileSize);
|
||||||
|
|
||||||
|
if (!isFinal) {
|
||||||
|
ajaxObject.done(function (data, textStatus, jqXHR)
|
||||||
|
{ uploadNextChunk(file, offset + CHUNK_SIZE, id); });
|
||||||
|
} else {
|
||||||
|
ajaxObject.done(function(data, textStatus, jqXHR) {
|
||||||
|
isRestoring = true;
|
||||||
|
|
||||||
|
// immediately reload backup information since one should be restoring now
|
||||||
|
reloadBackupInformation();
|
||||||
|
|
||||||
|
swal.close();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
function setupBackupUpload() {
|
function setupBackupUpload() {
|
||||||
// construct the HTML needed for the settings backup panel
|
// construct the HTML needed for the settings backup panel
|
||||||
var html = "<div class='form-group'><div id='" + UPLOAD_CONTENT_ALLOWED_DIV_ID + "'>";
|
var html = "<div class='form-group'><div id='" + UPLOAD_CONTENT_ALLOWED_DIV_ID + "'>";
|
||||||
|
@ -50,34 +125,10 @@ $(document).ready(function(){
|
||||||
"Restore content",
|
"Restore content",
|
||||||
function() {
|
function() {
|
||||||
var files = $('#' + RESTORE_SETTINGS_FILE_ID).prop('files');
|
var files = $('#' + RESTORE_SETTINGS_FILE_ID).prop('files');
|
||||||
|
var file = files[0];
|
||||||
|
|
||||||
var fileFormData = new FormData();
|
showUploadProgress("Uploading " + file.name);
|
||||||
fileFormData.append('restore-file', files[0]);
|
uploadNextChunk(file);
|
||||||
|
|
||||||
showSpinnerAlert("Uploading content to restore");
|
|
||||||
|
|
||||||
$.ajax({
|
|
||||||
url: '/content/upload',
|
|
||||||
type: 'POST',
|
|
||||||
timeout: 3600000, // Set timeout to 1h
|
|
||||||
cache: false,
|
|
||||||
processData: false,
|
|
||||||
contentType: false,
|
|
||||||
data: fileFormData
|
|
||||||
}).done(function(data, textStatus, jqXHR) {
|
|
||||||
isRestoring = true;
|
|
||||||
|
|
||||||
// immediately reload backup information since one should be restoring now
|
|
||||||
reloadBackupInformation();
|
|
||||||
|
|
||||||
swal.close();
|
|
||||||
}).fail(function(jqXHR, textStatus, errorThrown) {
|
|
||||||
showErrorMessage(
|
|
||||||
"Error",
|
|
||||||
"There was a problem restoring domain content.\n"
|
|
||||||
+ "Please ensure that the content archive or entity file is valid and try again."
|
|
||||||
);
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
@ -168,6 +219,11 @@ $(document).ready(function(){
|
||||||
checkBackupStatus();
|
checkBackupStatus();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
function updateProgressBars($progressBar, value) {
|
||||||
|
$progressBar.attr('aria-valuenow', value).attr('style', 'width: ' + value + '%');
|
||||||
|
$progressBar.find('.ongoing-msg').html(" " + Math.round(value) + "%");
|
||||||
|
}
|
||||||
|
|
||||||
function reloadBackupInformation() {
|
function reloadBackupInformation() {
|
||||||
// make a GET request to get backup information to populate the table
|
// make a GET request to get backup information to populate the table
|
||||||
$.ajax({
|
$.ajax({
|
||||||
|
@ -204,11 +260,6 @@ $(document).ready(function(){
|
||||||
+ "<li><a class='" + BACKUP_DELETE_LINK_CLASS + "' href='#' target='_blank'>Delete</a></li></ul></div></td>";
|
+ "<li><a class='" + BACKUP_DELETE_LINK_CLASS + "' href='#' target='_blank'>Delete</a></li></ul></div></td>";
|
||||||
}
|
}
|
||||||
|
|
||||||
function updateProgressBars($progressBar, value) {
|
|
||||||
$progressBar.attr('aria-valuenow', value).attr('style', 'width: ' + value + '%');
|
|
||||||
$progressBar.find('.sr-only').html(value + "% Complete");
|
|
||||||
}
|
|
||||||
|
|
||||||
// before we add any new rows and update existing ones
|
// before we add any new rows and update existing ones
|
||||||
// remove our flag for active rows
|
// remove our flag for active rows
|
||||||
$('.' + ACTIVE_BACKUP_ROW_CLASS).removeClass(ACTIVE_BACKUP_ROW_CLASS);
|
$('.' + ACTIVE_BACKUP_ROW_CLASS).removeClass(ACTIVE_BACKUP_ROW_CLASS);
|
||||||
|
|
|
@ -348,6 +348,27 @@ void DomainContentBackupManager::recoverFromUploadedBackup(MiniPromise::Promise
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void DomainContentBackupManager::recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename) {
|
||||||
|
if (QThread::currentThread() != thread()) {
|
||||||
|
QMetaObject::invokeMethod(this, "recoverFromUploadedFile", Q_ARG(MiniPromise::Promise, promise),
|
||||||
|
Q_ARG(QString, uploadedFilename));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
qDebug() << "Recovering from uploaded file -" << uploadedFilename;
|
||||||
|
|
||||||
|
QFile uploadedFile(uploadedFilename);
|
||||||
|
QuaZip uploadedZip { &uploadedFile };
|
||||||
|
|
||||||
|
QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip";
|
||||||
|
|
||||||
|
bool success = recoverFromBackupZip(backupName, uploadedZip);
|
||||||
|
|
||||||
|
promise->resolve({
|
||||||
|
{ "success", success }
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
std::vector<BackupItemInfo> DomainContentBackupManager::getAllBackups() {
|
std::vector<BackupItemInfo> DomainContentBackupManager::getAllBackups() {
|
||||||
|
|
||||||
QDir backupDir { _backupDirectory };
|
QDir backupDir { _backupDirectory };
|
||||||
|
|
|
@ -86,6 +86,7 @@ public slots:
|
||||||
void createManualBackup(MiniPromise::Promise promise, const QString& name);
|
void createManualBackup(MiniPromise::Promise promise, const QString& name);
|
||||||
void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName);
|
void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName);
|
||||||
void recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup);
|
void recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup);
|
||||||
|
void recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename);
|
||||||
void deleteBackup(MiniPromise::Promise promise, const QString& backupName);
|
void deleteBackup(MiniPromise::Promise promise, const QString& backupName);
|
||||||
|
|
||||||
signals:
|
signals:
|
||||||
|
|
|
@ -2258,46 +2258,18 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
|
||||||
// check the file extension to see what kind of file this is
|
// check the file extension to see what kind of file this is
|
||||||
// to make sure we handle this filetype for a content restore
|
// to make sure we handle this filetype for a content restore
|
||||||
auto dispositionValue = QString(firstFormData.first.value("Content-Disposition"));
|
auto dispositionValue = QString(firstFormData.first.value("Content-Disposition"));
|
||||||
auto formDataFilenameRegex = QRegExp("filename=\"(.+)\"");
|
QRegExp formDataFieldsRegex(R":(name="(restore-file.*)".*filename="(.+)"):");
|
||||||
auto matchIndex = formDataFilenameRegex.indexIn(dispositionValue);
|
auto matchIndex = formDataFieldsRegex.indexIn(dispositionValue);
|
||||||
|
|
||||||
|
QString formItemName = "";
|
||||||
QString uploadedFilename = "";
|
QString uploadedFilename = "";
|
||||||
if (matchIndex != -1) {
|
if (matchIndex != -1) {
|
||||||
uploadedFilename = formDataFilenameRegex.cap(1);
|
formItemName = formDataFieldsRegex.cap(1);
|
||||||
}
|
uploadedFilename = formDataFieldsRegex.cap(2);
|
||||||
|
|
||||||
if (uploadedFilename.endsWith(".json", Qt::CaseInsensitive)
|
|
||||||
|| uploadedFilename.endsWith(".json.gz", Qt::CaseInsensitive)) {
|
|
||||||
// invoke our method to hand the new octree file off to the octree server
|
|
||||||
QMetaObject::invokeMethod(this, "handleOctreeFileReplacement",
|
|
||||||
Qt::QueuedConnection, Q_ARG(QByteArray, firstFormData.second));
|
|
||||||
|
|
||||||
// respond with a 200 for success
|
|
||||||
connection->respond(HTTPConnection::StatusCode200);
|
|
||||||
} else if (uploadedFilename.endsWith(".zip", Qt::CaseInsensitive)) {
|
|
||||||
auto deferred = makePromise("recoverFromUploadedBackup");
|
|
||||||
|
|
||||||
deferred->then([connectionPtr, JSON_MIME_TYPE](QString error, QVariantMap result) {
|
|
||||||
if (!connectionPtr) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
QJsonObject rootJSON;
|
|
||||||
auto success = result["success"].toBool();
|
|
||||||
rootJSON["success"] = success;
|
|
||||||
QJsonDocument docJSON(rootJSON);
|
|
||||||
connectionPtr->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
|
|
||||||
JSON_MIME_TYPE.toUtf8());
|
|
||||||
});
|
|
||||||
|
|
||||||
_contentManager->recoverFromUploadedBackup(deferred, firstFormData.second);
|
|
||||||
|
|
||||||
return true;
|
|
||||||
} else {
|
|
||||||
// we don't have handling for this filetype, send back a 400 for failure
|
|
||||||
connection->respond(HTTPConnection::StatusCode400);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Received a chunk
|
||||||
|
processPendingContent(connection, formItemName, uploadedFilename, firstFormData.second);
|
||||||
} else {
|
} else {
|
||||||
// respond with a 400 for failure
|
// respond with a 400 for failure
|
||||||
connection->respond(HTTPConnection::StatusCode400);
|
connection->respond(HTTPConnection::StatusCode400);
|
||||||
|
@ -2546,6 +2518,72 @@ bool DomainServer::handleHTTPSRequest(HTTPSConnection* connection, const QUrl &u
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool DomainServer::processPendingContent(HTTPConnection* connection, QString itemName, QString filename, QByteArray dataChunk) {
|
||||||
|
static const QString UPLOAD_SESSION_KEY { "X-Session-Id" };
|
||||||
|
QByteArray sessionIdBytes = connection->requestHeader(UPLOAD_SESSION_KEY);
|
||||||
|
int sessionId = sessionIdBytes.toInt();
|
||||||
|
|
||||||
|
bool newUpload = itemName == "restore-file" || itemName == "restore-file-chunk-initial" || itemName == "restore-file-chunk-only";
|
||||||
|
|
||||||
|
if (filename.endsWith(".zip", Qt::CaseInsensitive)) {
|
||||||
|
static const QString TEMPORARY_CONTENT_FILEPATH { QDir::tempPath() + "/hifiUploadContent_XXXXXX.zip" };
|
||||||
|
|
||||||
|
if (_pendingContentFiles.find(sessionId) == _pendingContentFiles.end()) {
|
||||||
|
if (!newUpload) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
std::unique_ptr<QTemporaryFile> newTemp(new QTemporaryFile(TEMPORARY_CONTENT_FILEPATH));
|
||||||
|
_pendingContentFiles[sessionId] = std::move(newTemp);
|
||||||
|
} else if (newUpload) {
|
||||||
|
qCDebug(domain_server) << "New upload received using existing session ID";
|
||||||
|
_pendingContentFiles[sessionId]->resize(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
QTemporaryFile& _pendingFileContent = *_pendingContentFiles[sessionId];
|
||||||
|
if (!_pendingFileContent.open()) {
|
||||||
|
_pendingContentFiles.erase(sessionId);
|
||||||
|
connection->respond(HTTPConnection::StatusCode400);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
_pendingFileContent.seek(_pendingFileContent.size());
|
||||||
|
_pendingFileContent.write(dataChunk);
|
||||||
|
_pendingFileContent.close();
|
||||||
|
|
||||||
|
// Respond immediately - will timeout if we wait for restore.
|
||||||
|
connection->respond(HTTPConnection::StatusCode200);
|
||||||
|
if (itemName == "restore-file" || itemName == "restore-file-chunk-final" || itemName == "restore-file-chunk-only") {
|
||||||
|
auto deferred = makePromise("recoverFromUploadedBackup");
|
||||||
|
|
||||||
|
deferred->then([this, sessionId](QString error, QVariantMap result) {
|
||||||
|
_pendingContentFiles.erase(sessionId);
|
||||||
|
});
|
||||||
|
|
||||||
|
_contentManager->recoverFromUploadedFile(deferred, _pendingFileContent.fileName());
|
||||||
|
}
|
||||||
|
} else if (filename.endsWith(".json", Qt::CaseInsensitive)
|
||||||
|
|| filename.endsWith(".json.gz", Qt::CaseInsensitive)) {
|
||||||
|
if (_pendingUploadedContents.find(sessionId) == _pendingUploadedContents.end() && !newUpload) {
|
||||||
|
qCDebug(domain_server) << "Json upload with invalid session ID received";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
QByteArray& _pendingUploadedContent = _pendingUploadedContents[sessionId];
|
||||||
|
_pendingUploadedContent += dataChunk;
|
||||||
|
connection->respond(HTTPConnection::StatusCode200);
|
||||||
|
|
||||||
|
if (itemName == "restore-file" || itemName == "restore-file-chunk-final" || itemName == "restore-file-chunk-only") {
|
||||||
|
// invoke our method to hand the new octree file off to the octree server
|
||||||
|
QMetaObject::invokeMethod(this, "handleOctreeFileReplacement",
|
||||||
|
Qt::QueuedConnection, Q_ARG(QByteArray, _pendingUploadedContent));
|
||||||
|
_pendingUploadedContents.erase(sessionId);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
connection->respond(HTTPConnection::StatusCode400);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
HTTPSConnection* DomainServer::connectionFromReplyWithState(QNetworkReply* reply) {
|
HTTPSConnection* DomainServer::connectionFromReplyWithState(QNetworkReply* reply) {
|
||||||
// grab the UUID state property from the reply
|
// grab the UUID state property from the reply
|
||||||
QUuid stateUUID = reply->property(STATE_QUERY_KEY.toLocal8Bit()).toUuid();
|
QUuid stateUUID = reply->property(STATE_QUERY_KEY.toLocal8Bit()).toUuid();
|
||||||
|
|
|
@ -20,6 +20,7 @@
|
||||||
#include <QtCore/QStringList>
|
#include <QtCore/QStringList>
|
||||||
#include <QtCore/QThread>
|
#include <QtCore/QThread>
|
||||||
#include <QtCore/QUrl>
|
#include <QtCore/QUrl>
|
||||||
|
#include <QHostAddress>
|
||||||
#include <QAbstractNativeEventFilter>
|
#include <QAbstractNativeEventFilter>
|
||||||
|
|
||||||
#include <Assignment.h>
|
#include <Assignment.h>
|
||||||
|
@ -209,6 +210,8 @@ private:
|
||||||
|
|
||||||
HTTPSConnection* connectionFromReplyWithState(QNetworkReply* reply);
|
HTTPSConnection* connectionFromReplyWithState(QNetworkReply* reply);
|
||||||
|
|
||||||
|
bool processPendingContent(HTTPConnection* connection, QString itemName, QString filename, QByteArray dataChunk);
|
||||||
|
|
||||||
bool forwardMetaverseAPIRequest(HTTPConnection* connection,
|
bool forwardMetaverseAPIRequest(HTTPConnection* connection,
|
||||||
const QString& metaversePath,
|
const QString& metaversePath,
|
||||||
const QString& requestSubobject,
|
const QString& requestSubobject,
|
||||||
|
@ -281,6 +284,9 @@ private:
|
||||||
|
|
||||||
QHash<QUuid, QPointer<HTTPSConnection>> _pendingOAuthConnections;
|
QHash<QUuid, QPointer<HTTPSConnection>> _pendingOAuthConnections;
|
||||||
|
|
||||||
|
std::unordered_map<int, QByteArray> _pendingUploadedContents;
|
||||||
|
std::unordered_map<int, std::unique_ptr<QTemporaryFile>> _pendingContentFiles;
|
||||||
|
|
||||||
QThread _assetClientThread;
|
QThread _assetClientThread;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue