Mirrors/overte-JulianGro
3
0
Fork 0
mirror of https://github.com/JulianGro/overte.git synced 2025-04-13 22:27:13 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update content backup downloads to be asynchronous

Browse source 
Previously backups would be consolidated into a full .zip when the
backup was requested. Because of the potential for a large number of
assets, this could take awhile, causing a browser to fail due to timeout
before the backup was available. This change splits the download
endpoint into 2 parts - one to request information about a backup and
possibly kick off a consolidation, and another to request the actual
file once the consolidate backup is available.
This commit is contained in:
Ryan Huffman 2018-03-20 10:19:07 -07:00
parent 2d63afbe28
commit dc694fb0b7
4 changed files with 153 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
domain-server/resources/web/content/js/content.js
View file

@ -132,6 +132,41 @@ $(document).ready(function(){
var ACTIVE_BACKUP_ROW_CLASS = 'active-backup';
var CORRUPTED_ROW_CLASS = 'danger';
$('body').on('click', '.' + BACKUP_DOWNLOAD_LINK_CLASS, function(ev) {
ev.preventDefault();
var backupID = $(this).data('backup-id')
showSpinnerAlert("Preparing backup...");
function checkBackupStatus() {
$.ajax({
url: "/api/backups/" + backupID,
dataType: 'json',
success: function(data) {
if (data.complete) {
if (data.error == '') {
location.href = "/api/backups/download/" + backupID;
swal.close();
} else {
showErrorMessage(
"Error",
"There was an error preparing your backup. Please refresh the page and try again."
);
}
} else {
setTimeout(checkBackupStatus, 500);
}
},
error: function() {
showErrorMessage(
"Error",
"There was an error preparing your backup."
);
},
});
}
checkBackupStatus();
});
function reloadBackupInformation() {
// make a GET request to get backup information to populate the table
$.ajax({
@ -164,7 +199,7 @@ $(document).ready(function(){
+ "<div class='dropdown'><div class='dropdown-toggle' data-toggle='dropdown' aria-expanded='false'><span class='glyphicon glyphicon-option-vertical'></span></div>"
+ "<ul class='dropdown-menu dropdown-menu-right'>"
+ "<li><a class='" + BACKUP_RESTORE_LINK_CLASS + "' href='#'>Restore from here</a></li><li class='divider'></li>"
+ "<li><a class='" + BACKUP_DOWNLOAD_LINK_CLASS + "' href='/api/backups/" + backup.id + "'>Download</a></li><li class='divider'></li>"
+ "<li><a class='" + BACKUP_DOWNLOAD_LINK_CLASS + "' data-backup-id='" + backup.id + "' href='#'>Download</a></li><li class='divider'></li>"
+ "<li><a class='" + BACKUP_DELETE_LINK_CLASS + "' href='#' target='_blank'>Delete</a></li></ul></div></td>";
}

76
domain-server/src/DomainContentBackupManager.cpp
View file

@ -55,9 +55,9 @@ DomainContentBackupManager::DomainContentBackupManager(const QString& backupDire
const QVariantList& backupRules,
std::chrono::milliseconds persistInterval,
bool debugTimestampNow) :
_consolidatedBackupDirectory(PathUtils::generateTemporaryDir()),
_backupDirectory(backupDirectory), _persistInterval(persistInterval), _lastCheck(p_high_resolution_clock::now())
{
setObjectName("DomainContentBackupManager");
// Make sure the backup directory exists.
@ -498,23 +498,63 @@ void DomainContentBackupManager::backup() {
}
}
void DomainContentBackupManager::consolidateBackup(MiniPromise::Promise promise, QString fileName) {
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(this, "consolidateBackup", Q_ARG(MiniPromise::Promise, promise),
Q_ARG(QString, fileName));
return;
ConsolidatedBackupInfo DomainContentBackupManager::consolidateBackup(QString fileName) {
{
std::lock_guard<std::mutex> lock { _consolidatedBackupsMutex };
auto it = _consolidatedBackups.find(fileName);
if (it != _consolidatedBackups.end()) {
return it->second;
}
}
QMetaObject::invokeMethod(this, "consolidateBackupInternal", Q_ARG(QString, fileName));
return {
ConsolidatedBackupInfo::CONSOLIDATING,
"",
""
};
}
void DomainContentBackupManager::consolidateBackupInternal(QString fileName) {
auto markFailure = [this, &fileName](QString error) {
qWarning() << "Failed to consolidate backup:" << fileName << error;
{
std::lock_guard<std::mutex> lock { _consolidatedBackupsMutex };
auto& consolidatedBackup = _consolidatedBackups[fileName];
consolidatedBackup.state = ConsolidatedBackupInfo::COMPLETE_WITH_ERROR;
consolidatedBackup.error = error;
}
};
{
std::lock_guard<std::mutex> lock { _consolidatedBackupsMutex };
auto it = _consolidatedBackups.find(fileName);
if (it != _consolidatedBackups.end()) {
return;
}
_consolidatedBackups[fileName] = {
ConsolidatedBackupInfo::CONSOLIDATING,
"",
""
};
}
QDir backupDir { _backupDirectory };
if (!backupDir.exists()) {
qCritical() << "Backup directory does not exist, bailing consolidation of backup";
promise->resolve({ { "success", false } });
markFailure("Backup directory does not exist, bailing consolidation of backup");
return;
}
auto filePath = backupDir.absoluteFilePath(fileName);
if (!QFile::exists(filePath)) {
markFailure("Backup does not exist");
return;
}
auto copyFilePath = QDir::tempPath() + "/" + fileName;
auto copyFilePath = _consolidatedBackupDirectory + "/" + fileName;
{
QFile copyFile(copyFilePath);
@ -523,8 +563,7 @@ void DomainContentBackupManager::consolidateBackup(MiniPromise::Promise promise,
}
auto copySuccess = QFile::copy(filePath, copyFilePath);
if (!copySuccess) {
qCritical() << "Failed to create copy of backup.";
promise->resolve({ { "success", false } });
markFailure("Failed to create copy of backup.");
return;
}
@ -532,7 +571,7 @@ void DomainContentBackupManager::consolidateBackup(MiniPromise::Promise promise,
if (!zip.open(QuaZip::mdAdd)) {
qCritical() << "Could not open backup archive:" << filePath;
qCritical() << " ERROR:" << zip.getZipError();
promise->resolve({ { "success", false } });
markFailure("Could not open backup archive");
return;
}
@ -544,14 +583,17 @@ void DomainContentBackupManager::consolidateBackup(MiniPromise::Promise promise,
if (zip.getZipError() != UNZ_OK) {
qCritical() << "Failed to consolidate backup: " << zip.getZipError();
promise->resolve({ { "success", false } });
markFailure("Failed to consolidate backup");
return;
}
promise->resolve({
{ "success", true },
{ "backupFilePath", copyFilePath }
});
{
std::lock_guard<std::mutex> lock { _consolidatedBackupsMutex };
auto& consolidatedBackup = _consolidatedBackups[fileName];
consolidatedBackup.state = ConsolidatedBackupInfo::COMPLETE_WITH_SUCCESS;
consolidatedBackup.absoluteFilePath = copyFilePath;
}
}
void DomainContentBackupManager::createManualBackup(MiniPromise::Promise promise, const QString& name) {

25
domain-server/src/DomainContentBackupManager.h
View file

@ -15,10 +15,15 @@
#ifndef hifi_DomainContentBackupManager_h
#define hifi_DomainContentBackupManager_h
#include <RegisteredMetaTypes.h>
#include <QString>
#include <QVector>
#include <QDateTime>
#include <mutex>
#include <unordered_map>
#include <GenericThread.h>
#include "BackupHandler.h"
@ -38,6 +43,17 @@ struct BackupItemInfo {
bool isManualBackup;
};
struct ConsolidatedBackupInfo {
enum State {
CONSOLIDATING,
COMPLETE_WITH_ERROR,
COMPLETE_WITH_SUCCESS
};
State state;
QString error;
QString absoluteFilePath;
};
class DomainContentBackupManager : public GenericThread {
Q_OBJECT
public:
@ -61,6 +77,7 @@ public:
void addBackupHandler(BackupHandlerPointer handler);
void aboutToFinish(); /// call this to inform the persist thread that the owner is about to finish to support final persist
void replaceData(QByteArray data);
ConsolidatedBackupInfo consolidateBackup(QString fileName);
public slots:
void getAllBackupsAndStatus(MiniPromise::Promise promise);
@ -68,7 +85,6 @@ public slots:
void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName);
void recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup);
void deleteBackup(MiniPromise::Promise promise, const QString& backupName);
void consolidateBackup(MiniPromise::Promise promise, QString fileName);
signals:
void loadCompleted();
@ -91,11 +107,18 @@ protected:
bool recoverFromBackupZip(const QString& backupName, QuaZip& backupZip);
private slots:
void consolidateBackupInternal(QString fileName);
private:
const QString _consolidatedBackupDirectory;
const QString _backupDirectory;
std::vector<BackupHandlerPointer> _backupHandlers;
std::chrono::milliseconds _persistInterval { 0 };
std::mutex _consolidatedBackupsMutex;
std::unordered_map<QString, ConsolidatedBackupInfo> _consolidatedBackups;
std::atomic<bool> _isRecovering { false };
QString _recoveryFilename { };

62
domain-server/src/DomainServer.cpp
View file

@ -164,6 +164,8 @@ DomainServer::DomainServer(int argc, char* argv[]) :
_iceServerAddr(ICE_SERVER_DEFAULT_HOSTNAME),
_iceServerPort(ICE_SERVER_DEFAULT_PORT)
{
PathUtils::removeTemporaryApplicationDirs();
parseCommandLine();
DependencyManager::set<tracing::Tracer>();
@ -1933,6 +1935,7 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
const QString URI_API_DOMAINS_ID = "/api/domains/";
const QString URI_API_BACKUPS = "/api/backups";
const QString URI_API_BACKUPS_ID = "/api/backups/";
const QString URI_API_BACKUPS_DOWNLOAD_ID = "/api/backups/download/";
const QString URI_API_BACKUPS_RECOVER = "/api/backups/recover/";
const QString UUID_REGEX_STRING = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
@ -2133,37 +2136,40 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
});
_contentManager->getAllBackupsAndStatus(deferred);
return true;
} else if (url.path().startsWith(URI_API_BACKUPS_DOWNLOAD_ID)) {
auto id = url.path().mid(QString(URI_API_BACKUPS_DOWNLOAD_ID).length());
auto info = _contentManager->consolidateBackup(id);
if (info.state == ConsolidatedBackupInfo::COMPLETE_WITH_SUCCESS) {
auto file { std::unique_ptr<QFile>(new QFile(info.absoluteFilePath)) };
if (file->open(QIODevice::ReadOnly)) {
constexpr const char* CONTENT_TYPE_ZIP = "application/zip";
auto downloadedFilename = id;
downloadedFilename.replace(QRegularExpression(".zip$"), ".content.zip");
auto contentDisposition = "attachment; filename=\"" + downloadedFilename + "\"";
connectionPtr->respond(HTTPConnection::StatusCode200, std::move(file), CONTENT_TYPE_ZIP, {
{ "Content-Disposition", contentDisposition.toUtf8() }
});
} else {
qCritical(domain_server) << "Unable to load consolidated backup at:" << info.absoluteFilePath;
connectionPtr->respond(HTTPConnection::StatusCode500, "Error opening backup");
}
} else if (info.state == ConsolidatedBackupInfo::COMPLETE_WITH_ERROR) {
connectionPtr->respond(HTTPConnection::StatusCode500, ("Error creating backup: " + info.error).toUtf8());
} else {
connectionPtr->respond(HTTPConnection::StatusCode400, "Backup unavailable");
}
return true;
} else if (url.path().startsWith(URI_API_BACKUPS_ID)) {
auto id = url.path().mid(QString(URI_API_BACKUPS_ID).length());
auto deferred = makePromise("consolidateBackup");
deferred->then([connectionPtr, JSON_MIME_TYPE, id](QString error, QVariantMap result) {
if (!connectionPtr) {
return;
}
auto info = _contentManager->consolidateBackup(id);
QJsonObject rootJSON;
auto success = result["success"].toBool();
if (success) {
auto path = result["backupFilePath"].toString();
auto file { std::unique_ptr<QFile>(new QFile(path)) };
if (file->open(QIODevice::ReadOnly)) {
constexpr const char* CONTENT_TYPE_ZIP = "application/zip";
auto downloadedFilename = id;
downloadedFilename.replace(QRegularExpression(".zip$"), ".content.zip");
auto contentDisposition = "attachment; filename=\"" + downloadedFilename + "\"";
connectionPtr->respond(HTTPConnection::StatusCode200, std::move(file), CONTENT_TYPE_ZIP, {
{ "Content-Disposition", contentDisposition.toUtf8() }
});
} else {
qCritical(domain_server) << "Unable to load consolidated backup at:" << path << result;
connectionPtr->respond(HTTPConnection::StatusCode500, "Error opening backup");
}
} else {
connectionPtr->respond(HTTPConnection::StatusCode400);
}
});
_contentManager->consolidateBackup(deferred, id);
QJsonObject rootJSON {
{ "complete", info.state == ConsolidatedBackupInfo::COMPLETE_WITH_SUCCESS },
{ "error", info.error }
};
QJsonDocument docJSON { rootJSON };
connectionPtr->respond(HTTPConnection::StatusCode200, docJSON.toJson(), JSON_MIME_TYPE.toUtf8());
return true;
} else if (url.path() == URI_RESTART) {