Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-08 07:12:40 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Relay upoaded content.zip chunks to temp file

Browse source 
Entities uploads still build in-memory. Move out chunk
handling to new routine.
This commit is contained in:
Simon Walton 2018-11-15 16:59:52 -08:00
parent cd00abd216
commit 22c3f5239a
5 changed files with 67 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
domain-server/resources/web/content/js/content.js
View file

@ -60,7 +60,7 @@ $(document).ready(function(){
);
});
updateProgressBars($('.upload-content-progress'), Math.round((offset + nextChunkSize) * 100 / fileSize));
updateProgressBars($('.upload-content-progress'), (offset + nextChunkSize) * 100 / fileSize);
if (!isFinal) {
ajaxObject.done(function (data, textStatus, jqXHR)
@ -210,7 +210,7 @@ $(document).ready(function(){
function updateProgressBars($progressBar, value) {
$progressBar.attr('aria-valuenow', value).attr('style', 'width: ' + value + '%');
$progressBar.find('.ongoing-msg').html(" " + value + "% Complete");
$progressBar.find('.ongoing-msg').html(" " + Math.round(value) + "% Complete");
}
function reloadBackupInformation() {

21
domain-server/src/DomainContentBackupManager.cpp
View file

@ -348,6 +348,27 @@ void DomainContentBackupManager::recoverFromUploadedBackup(MiniPromise::Promise
});
}
void DomainContentBackupManager::recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename) {
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(this, "recoverFromUploadedFile", Q_ARG(MiniPromise::Promise, promise),
Q_ARG(QString, uploadedFilename));
return;
}
qDebug() << "Recovering from uploaded file -" << uploadedFilename;
QFile uploadedFile(uploadedFilename);
QuaZip uploadedZip { &uploadedFile };
QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip";
bool success = recoverFromBackupZip(backupName, uploadedZip);
promise->resolve({
{ "success", success }
});
}
std::vector<BackupItemInfo> DomainContentBackupManager::getAllBackups() {
QDir backupDir { _backupDirectory };

1
domain-server/src/DomainContentBackupManager.h
View file

@ -86,6 +86,7 @@ public slots:
void createManualBackup(MiniPromise::Promise promise, const QString& name);
void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName);
void recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup);
void recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename);
void deleteBackup(MiniPromise::Promise promise, const QString& backupName);
signals:

77
domain-server/src/DomainServer.cpp
View file

@ -2268,17 +2268,8 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
uploadedFilename = formDataFieldsRegex.cap(2);
}
_pendingUploadedContent += firstFormData.second;
if (formItemName == "restore-file-chunk") {
// Received another chunk
connection->respond(HTTPConnection::StatusCode200);
} else if (formItemName == "restore-file-chunk-final") {
readPendingContent(connection, uploadedFilename);
} else if (formItemName == "restore-file") {
readPendingContent(connection, uploadedFilename);
} else {
connection->respond(HTTPConnection::StatusCode400);
}
// Received a chunk
processPendingContent(connection, formItemName, uploadedFilename, firstFormData.second);
} else {
// respond with a 400 for failure
connection->respond(HTTPConnection::StatusCode400);
@ -2527,37 +2518,51 @@ bool DomainServer::handleHTTPSRequest(HTTPSConnection* connection, const QUrl &u
}
}
void DomainServer::readPendingContent(HTTPConnection* connection, QString filename) {
if (filename.endsWith(".json", Qt::CaseInsensitive)
|| filename.endsWith(".json.gz", Qt::CaseInsensitive)) {
// invoke our method to hand the new octree file off to the octree server
QMetaObject::invokeMethod(this, "handleOctreeFileReplacement",
Qt::QueuedConnection, Q_ARG(QByteArray, _pendingUploadedContent));
bool DomainServer::processPendingContent(HTTPConnection* connection, QString itemName, QString filename, QByteArray dataChunk) {
if (filename.endsWith(".zip", Qt::CaseInsensitive)) {
static const QString TEMPORARY_CONTENT_FILEPATH { QDir::tempPath() + "/hifiUploadContent_XXXXXX.zip" };
// respond with a 200 for success
if (!_pendingFileContent) {
_pendingFileContent = std::make_unique<QTemporaryFile>(TEMPORARY_CONTENT_FILEPATH);
}
if (!_pendingFileContent->open()) {
_pendingFileContent = nullptr;
connection->respond(HTTPConnection::StatusCode400);
return false;
}
_pendingFileContent->seek(_pendingFileContent->size());
_pendingFileContent->write(dataChunk);
_pendingFileContent->close();
// Respond immediately - will timeout if we wait for restore.
connection->respond(HTTPConnection::StatusCode200);
_pendingUploadedContent.clear();
} else if (filename.endsWith(".zip", Qt::CaseInsensitive)) {
auto deferred = makePromise("recoverFromUploadedBackup");
if (itemName == "restore-file-chunk-final" || itemName == "restore-file") {
auto deferred = makePromise("recoverFromUploadedBackup");
QPointer<HTTPConnection> connectionPtr(connection);
const QString JSON_MIME_TYPE = "application/json";
deferred->then([connectionPtr, JSON_MIME_TYPE, this](QString error, QVariantMap result) {
if (!connectionPtr) {
return;
}
deferred->then([this](QString error, QVariantMap result) {
_pendingFileContent = nullptr;
});
QJsonObject rootJSON;
auto success = result["success"].toBool();
rootJSON["success"] = success;
QJsonDocument docJSON(rootJSON);
connectionPtr->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
JSON_MIME_TYPE.toUtf8());
_contentManager->recoverFromUploadedFile(deferred, _pendingFileContent->fileName());
} else {
}
} else if (filename.endsWith(".json", Qt::CaseInsensitive)
|| filename.endsWith(".json.gz", Qt::CaseInsensitive)) {
_pendingUploadedContent += dataChunk;
connection->respond(HTTPConnection::StatusCode200);
if (itemName == "restore-file-chunk-final" || itemName == "restore-file") {
// invoke our method to hand the new octree file off to the octree server
QMetaObject::invokeMethod(this, "handleOctreeFileReplacement",
Qt::QueuedConnection, Q_ARG(QByteArray, _pendingUploadedContent));
_pendingUploadedContent.clear();
});
_contentManager->recoverFromUploadedBackup(deferred, _pendingUploadedContent);
}
} else {
connection->respond(HTTPConnection::StatusCode400);
return false;
}
return true;
}
HTTPSConnection* DomainServer::connectionFromReplyWithState(QNetworkReply* reply) {

3
domain-server/src/DomainServer.h
View file

@ -209,7 +209,7 @@ private:
HTTPSConnection* connectionFromReplyWithState(QNetworkReply* reply);
void readPendingContent(HTTPConnection* connection, QString filename);
bool processPendingContent(HTTPConnection* connection, QString itemName, QString filename, QByteArray dataChunk);
bool forwardMetaverseAPIRequest(HTTPConnection* connection,
const QString& metaversePath,
@ -284,6 +284,7 @@ private:
QHash<QUuid, QPointer<HTTPSConnection>> _pendingOAuthConnections;
QByteArray _pendingUploadedContent;
std::unique_ptr<QTemporaryFile> _pendingFileContent;
QThread _assetClientThread;
};