Merge pull request #15978 from roxanneskelly/bugz829

BUGZ-829 DEV-168 - domain backup/content management improvements
This commit is contained in:
Shannon Romano 2019-07-29 15:28:49 -07:00 committed by GitHub
commit f29284198c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 371 additions and 104 deletions

View file

@ -1788,6 +1788,39 @@
"default": false
}
]
},
{
"name": "installed_content",
"label": "Installed Content",
"hidden": true,
"settings": [
{
"name": "filename",
"content_setting": true,
"default": ""
},
{
"name": "name",
"content_setting": true,
"default": ""
},
{
"name": "creation_time",
"content_setting": true,
"default": 0
},
{
"name": "install_time",
"type": "int",
"content_setting": true,
"default": 0
},
{
"name": "installed_by",
"content_setting": true,
"default": ""
}
]
}
]
}

View file

@ -4,8 +4,14 @@ $(document).ready(function(){
var RESTORE_SETTINGS_FILE_ID = 'restore-settings-file';
var UPLOAD_CONTENT_ALLOWED_DIV_ID = 'upload-content-allowed';
var UPLOAD_CONTENT_RECOVERING_DIV_ID = 'upload-content-recovering';
var INSTALLED_CONTENT_FILENAME_ID = 'installed-content-filename';
var INSTALLED_CONTENT_NAME_ID = 'installed-content-name';
var INSTALLED_CONTENT_CREATED_ID = 'installed-content-created';
var INSTALLED_CONTENT_INSTALLED_ID = 'installed-content-installed';
var INSTALLED_CONTENT_INSTALLED_BY_ID = 'installed-content-installed-by';
var isRestoring = false;
var restoreErrorShown = false;
function progressBarHTML(extraClass, label) {
var html = "<div class='progress'>";
@ -64,11 +70,23 @@ $(document).ready(function(){
var ajaxObject = $.ajax(ajaxParams);
ajaxObject.fail(function (jqXHR, textStatus, errorThrown) {
showErrorMessage(
"Error",
"There was a problem restoring domain content.\n"
+ "Please ensure that the content archive or entity file is valid and try again."
);
// status of 0 means the connection was reset, which
// happens after the content is parsed and the server restarts
// in the case of json and json.gz files
if (jqXHR.status != 0) {
showErrorMessage(
"Error",
"There was a problem restoring domain content.\n"
+ "Please ensure that the content archive or entity file is valid and try again."
);
} else {
isRestoring = true;
// immediately reload backup information since one should be restoring now
reloadBackupInformation();
swal.close();
}
});
updateProgressBars($('.upload-content-progress'), (offset + nextChunkSize) * 100 / fileSize);
@ -103,10 +121,25 @@ $(document).ready(function(){
html += "<span class='help-block'>Restore in progress</span>";
html += progressBarHTML('recovery', 'Restoring');
html += "</div></div>";
$('#' + Settings.UPLOAD_CONTENT_BACKUP_PANEL_ID + ' .panel-body').html(html);
}
function setupInstalledContentInfo() {
var html = "<table class='table table-bordered'><tbody>";
html += "<tr class='headers'><td class='data'><strong>Name</strong></td>";
html += "<td class='data'><strong>File Name</strong></td>";
html += "<td class='data'><strong>Created</strong></td>";
html += "<td class='data'><strong>Installed</strong></td>";
//html += "<td class='data'><strong>Installed By</strong></td></tr>";
html += "<tr><td class='data' id='" + INSTALLED_CONTENT_NAME_ID + "'/>";
html += "<td class='data' id='" + INSTALLED_CONTENT_FILENAME_ID + "'/>";
html += "<td class='data' id='" + INSTALLED_CONTENT_CREATED_ID + "'/>";
html += "<td class='data' id='" + INSTALLED_CONTENT_INSTALLED_ID + "'/>";
//html += "<td class='data' id='" + INSTALLED_CONTENT_INSTALLED_BY_ID + "'/></tr>";
html += "</tbody></table>";
$('#' + Settings.INSTALLED_CONTENT + ' .panel-body').html(html);
}
// handle content archive or entity file upload
// when the selected file is changed, enable the button if there's a selected file
@ -135,6 +168,7 @@ $(document).ready(function(){
var GENERATE_ARCHIVE_BUTTON_ID = 'generate-archive-button';
var CONTENT_ARCHIVES_NORMAL_ID = 'content-archives-success';
var CONTENT_ARCHIVES_CONTENT_INFO_ID = 'content-archives-content-info';
var CONTENT_ARCHIVES_ERROR_ID = 'content-archives-error';
var AUTOMATIC_ARCHIVES_TABLE_ID = 'automatic-archives-table';
var AUTOMATIC_ARCHIVES_TBODY_ID = 'automatic-archives-tbody';
@ -230,13 +264,27 @@ $(document).ready(function(){
url: '/api/backups',
cache: false
}).done(function(data) {
// split the returned data into manual and automatic manual backups
var splitBackups = _.partition(data.backups, function(value, index) {
return value.isManualBackup;
});
if (isRestoring && !data.status.isRecovering) {
if (data.status.recoveryError && !restoreErrorShown) {
restoreErrorShown = true;
swal({
title: "Error",
text: "There was a problem restoring domain content.\n"
+ data.status.recoveryError,
type: "error",
showCancelButton: false,
confirmButtonText: "Restart",
closeOnConfirm: true,
},
function () {
$.get("/restart");
showRestartModal();
});
}
if (isRestoring && !data.status.isRecovering && !data.status.recoveryError) {
// we were recovering and we finished - the DS is going to restart so show the restart modal
showRestartModal();
return;
@ -327,6 +375,12 @@ $(document).ready(function(){
$('#' + UPLOAD_CONTENT_ALLOWED_DIV_ID).toggle(!data.status.isRecovering);
$('#' + UPLOAD_CONTENT_RECOVERING_DIV_ID).toggle(data.status.isRecovering);
$('#' + INSTALLED_CONTENT_NAME_ID).text(data.installed_content.name);
$('#' + INSTALLED_CONTENT_FILENAME_ID).text(data.installed_content.filename);
$('#' + INSTALLED_CONTENT_CREATED_ID).text(data.installed_content.creation_time ? moment(data.installed_content.creation_time).format('lll') : "");
$('#' + INSTALLED_CONTENT_INSTALLED_ID).text(data.installed_content.install_time ? moment(data.installed_content.install_time).format('lll') : "");
//$('#' + INSTALLED_CONTENT_INSTALLED_BY_ID).text(data.installed_content.installed_by);
// update the progress bars for current restore status
if (data.status.isRecovering) {
updateProgressBars($('.recovery.progress-bar'), data.status.recoveryProgress * 100);
@ -514,6 +568,7 @@ $(document).ready(function(){
Settings.afterReloadActions = function() {
setupBackupUpload();
setupContentArchives();
setupInstalledContentInfo();
// load the latest backups immediately
reloadBackupInformation();

View file

@ -57,10 +57,14 @@ $(document).ready(function(){
// define extra groups to add to setting panels, with their splice index
Settings.extraContentGroupsAtIndex = {
0: {
html_id: Settings.INSTALLED_CONTENT,
label: 'Installed Content'
},
1: {
html_id: Settings.CONTENT_ARCHIVES_PANEL_ID,
label: 'Content Archives'
},
1: {
2: {
html_id: Settings.UPLOAD_CONTENT_BACKUP_PANEL_ID,
label: 'Upload Content'
}

View file

@ -44,7 +44,8 @@ $.extend(Settings, {
INVALID_ROW_CLASS: 'invalid-input',
DATA_ROW_INDEX: 'data-row-index',
CONTENT_ARCHIVES_PANEL_ID: 'content_archives',
UPLOAD_CONTENT_BACKUP_PANEL_ID: 'upload_content'
UPLOAD_CONTENT_BACKUP_PANEL_ID: 'upload_content',
INSTALLED_CONTENT: 'installed_content'
});
var URLs = {

View file

@ -278,17 +278,19 @@ void AssetsBackupHandler::createBackup(const QString& backupName, QuaZip& zip) {
_backups.emplace_back(backupName, mappings, false);
}
void AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
std::pair<bool, QString> AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip, const QString& sourceFilename) {
Q_ASSERT(QThread::currentThread() == thread());
if (operationInProgress()) {
qCWarning(asset_backup) << "There is already a backup/restore in progress.";
return;
QString errorStr ("There is already a backup/restore in progress. Please wait.");
qWarning() << errorStr;
return { false, errorStr };
}
if (_lastMappingsRefresh.time_since_epoch().count() == 0) {
qCWarning(asset_backup) << "Current mappings not yet loaded.";
return;
QString errorStr ("Current mappings not yet loaded. Please wait.");
qWarning() << errorStr;
return { false, errorStr };
}
if ((p_high_resolution_clock::now() - _lastMappingsRefresh) > MAX_REFRESH_TIME) {
@ -301,6 +303,16 @@ void AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip)
if (it == end(_backups)) {
loadBackup(backupName, zip);
auto emplaced_backup = find_if(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
return backup.name == backupName;
});
if(emplaced_backup->corruptedBackup) {
QString errorStr ("Current mappings file is corrupted.");
qWarning() << errorStr;
return { false, errorStr };
}
QuaZipDir zipDir { &zip, ZIP_ASSETS_FOLDER };
auto assetNames = zipDir.entryList(QDir::Files);
@ -330,8 +342,9 @@ void AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip)
});
if (it == end(_backups)) {
qCCritical(asset_backup) << "Failed to recover backup:" << backupName;
return;
QString errorStr ("Failed to recover backup: " + backupName);
qWarning() << errorStr;
return { false, errorStr };
}
}
@ -339,6 +352,7 @@ void AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip)
computeServerStateDifference(_currentMappings, newMappings);
restoreAllAssets();
return { true, QString() };
}
void AssetsBackupHandler::deleteBackup(const QString& backupName) {

View file

@ -38,7 +38,7 @@ public:
void loadBackup(const QString& backupName, QuaZip& zip) override;
void loadingComplete() override;
void createBackup(const QString& backupName, QuaZip& zip) override;
void recoverBackup(const QString& backupName, QuaZip& zip) override;
std::pair<bool, QString> recoverBackup(const QString& backupName, QuaZip& zip, const QString& sourceFilename) override;
void deleteBackup(const QString& backupName) override;
void consolidateBackup(const QString& backupName, QuaZip& zip) override;
bool isCorruptedBackup(const QString& backupName) override;

View file

@ -30,7 +30,7 @@ public:
virtual void loadBackup(const QString& backupName, QuaZip& zip) = 0;
virtual void loadingComplete() = 0;
virtual void createBackup(const QString& backupName, QuaZip& zip) = 0;
virtual void recoverBackup(const QString& backupName, QuaZip& zip) = 0;
virtual std::pair<bool, QString> recoverBackup(const QString& backupName, QuaZip& zip, const QString& sourceFilename) = 0;
virtual void deleteBackup(const QString& backupName) = 0;
virtual void consolidateBackup(const QString& backupName, QuaZip& zip) = 0;
virtual bool isCorruptedBackup(const QString& backupName) = 0;

View file

@ -10,6 +10,7 @@
//
#include "ContentSettingsBackupHandler.h"
#include "DomainContentBackupManager.h"
#if !defined(__clang__) && defined(__GNUC__)
#pragma GCC diagnostic push
@ -24,6 +25,7 @@
#pragma GCC diagnostic pop
#endif
static const QString DATETIME_FORMAT { "yyyy-MM-dd_HH-mm-ss" };
ContentSettingsBackupHandler::ContentSettingsBackupHandler(DomainServerSettingsManager& domainServerSettingsManager) :
_settingsManager(domainServerSettingsManager)
@ -41,6 +43,26 @@ void ContentSettingsBackupHandler::createBackup(const QString& backupName, QuaZi
DomainServerSettingsManager::IncludeContentSettings, DomainServerSettingsManager::NoDefaultSettings,
DomainServerSettingsManager::ForBackup
);
QString prefixFormat = "(" + QRegExp::escape(AUTOMATIC_BACKUP_PREFIX) + "|" + QRegExp::escape(MANUAL_BACKUP_PREFIX) + ")";
QString nameFormat = "(.+)";
QString dateTimeFormat = "(" + DATETIME_FORMAT_RE + ")";
QRegExp backupNameFormat { prefixFormat + nameFormat + "-" + dateTimeFormat + "\\.zip" };
QString name{ "" };
QDateTime createdAt;
if (backupNameFormat.exactMatch(backupName)) {
name = backupNameFormat.cap(2);
auto dateTime = backupNameFormat.cap(3);
createdAt = QDateTime::fromString(dateTime, DATETIME_FORMAT);
}
QJsonObject installed_content {
{ INSTALLED_CONTENT_NAME, name},
{ INSTALLED_CONTENT_CREATION_TIME, createdAt.currentMSecsSinceEpoch()}
};
contentSettingsJSON.insert(INSTALLED_CONTENT, installed_content);
// make a QJsonDocument using the object
QJsonDocument contentSettingsDocument { contentSettingsJSON };
@ -62,24 +84,48 @@ void ContentSettingsBackupHandler::createBackup(const QString& backupName, QuaZi
}
}
void ContentSettingsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
std::pair<bool, QString> ContentSettingsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip, const QString& sourceFilename) {
if (!zip.setCurrentFile(CONTENT_SETTINGS_BACKUP_FILENAME)) {
qWarning() << "Failed to find" << CONTENT_SETTINGS_BACKUP_FILENAME << "while recovering backup";
return;
QString errorStr("Failed to find " + CONTENT_SETTINGS_BACKUP_FILENAME + " while recovering backup");
qWarning() << errorStr;
return { false, errorStr };
}
QuaZipFile zipFile { &zip };
if (!zipFile.open(QIODevice::ReadOnly)) {
qCritical() << "Failed to open" << CONTENT_SETTINGS_BACKUP_FILENAME << "in backup";
return;
QString errorStr("Failed to open " + CONTENT_SETTINGS_BACKUP_FILENAME + " in backup");
qCritical() << errorStr;
return { false, errorStr };
}
auto rawData = zipFile.readAll();
zipFile.close();
QJsonDocument jsonDocument = QJsonDocument::fromJson(rawData);
if (!_settingsManager.restoreSettingsFromObject(jsonDocument.object(), ContentSettings)) {
qCritical() << "Failed to restore settings from" << CONTENT_SETTINGS_BACKUP_FILENAME << "in content archive";
if (zipFile.getZipError() != UNZ_OK) {
QString errorStr("Failed to unzip " + CONTENT_SETTINGS_BACKUP_FILENAME + ": " + zipFile.getZipError());
qCritical() << errorStr;
return { false, errorStr };
}
QJsonDocument jsonDocument = QJsonDocument::fromJson(rawData);
QJsonObject jsonObject = jsonDocument.object();
auto archiveJson = jsonObject.find(INSTALLED_CONTENT)->toObject();
QJsonObject installed_content {
{ INSTALLED_CONTENT_FILENAME, sourceFilename },
{ INSTALLED_CONTENT_NAME, archiveJson[INSTALLED_CONTENT_NAME].toString()},
{ INSTALLED_CONTENT_CREATION_TIME, archiveJson[INSTALLED_CONTENT_CREATION_TIME].toVariant().toLongLong() },
{ INSTALLED_CONTENT_INSTALL_TIME, QDateTime::currentDateTime().currentMSecsSinceEpoch() },
{ INSTALLED_CONTENT_INSTALLED_BY, "" }
};
jsonObject.insert(INSTALLED_CONTENT, installed_content);
if (!_settingsManager.restoreSettingsFromObject(jsonObject, ContentSettings)) {
QString errorStr("Failed to restore settings from " + CONTENT_SETTINGS_BACKUP_FILENAME + " in content archive");
qCritical() << errorStr;
return { false, errorStr };
}
return { true, QString() };
}

View file

@ -28,7 +28,7 @@ public:
void createBackup(const QString& backupName, QuaZip& zip) override;
void recoverBackup(const QString& backupName, QuaZip& zip) override;
std::pair<bool, QString> recoverBackup(const QString& backupName, QuaZip& zip, const QString& sourceFilename) override;
void deleteBackup(const QString& backupName) override {}

View file

@ -42,9 +42,7 @@ const std::chrono::seconds DomainContentBackupManager::DEFAULT_PERSIST_INTERVAL
// Backup format looks like: daily_backup-TIMESTAMP.zip
static const QString DATETIME_FORMAT { "yyyy-MM-dd_HH-mm-ss" };
static const QString DATETIME_FORMAT_RE { "\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}" };
static const QString AUTOMATIC_BACKUP_PREFIX { "autobackup-" };
static const QString MANUAL_BACKUP_PREFIX { "backup-" };
static const QString PRE_UPLOAD_SUFFIX{ "pre_upload" };
static const QString MANUAL_BACKUP_NAME_RE { "[a-zA-Z0-9\\-_ ]+" };
void DomainContentBackupManager::addBackupHandler(BackupHandlerPointer handler) {
@ -52,9 +50,10 @@ void DomainContentBackupManager::addBackupHandler(BackupHandlerPointer handler)
}
DomainContentBackupManager::DomainContentBackupManager(const QString& backupDirectory,
const QVariantList& backupRules,
DomainServerSettingsManager& domainServerSettingsManager,
std::chrono::milliseconds persistInterval,
bool debugTimestampNow) :
_settingsManager(domainServerSettingsManager),
_consolidatedBackupDirectory(PathUtils::generateTemporaryDir()),
_backupDirectory(backupDirectory), _persistInterval(persistInterval), _lastCheck(p_high_resolution_clock::now())
{
@ -63,7 +62,8 @@ DomainContentBackupManager::DomainContentBackupManager(const QString& backupDire
// Make sure the backup directory exists.
QDir(_backupDirectory).mkpath(".");
parseBackupRules(backupRules);
static const QString BACKUP_RULES_KEYPATH = AUTOMATIC_CONTENT_ARCHIVES_GROUP + ".backup_rules";
parseBackupRules(_settingsManager.valueOrDefaultValueForKeyPath(BACKUP_RULES_KEYPATH).toList());
constexpr int CONSOLIDATED_BACKUP_CLEANER_INTERVAL_MSECS = 30 * 1000;
_consolidatedBackupCleanupTimer.setInterval(CONSOLIDATED_BACKUP_CLEANER_INTERVAL_MSECS);
@ -170,7 +170,9 @@ bool DomainContentBackupManager::process() {
return handler->getRecoveryStatus().first;
});
if (!isStillRecovering) {
// if an error occurred, don't restart the server so that the user
// can be notified of the error and take action.
if (!isStillRecovering && _recoveryError.isEmpty()) {
_isRecovering = false;
_recoveryFilename = "";
emit recoveryCompleted();
@ -277,7 +279,7 @@ void DomainContentBackupManager::deleteBackup(MiniPromise::Promise promise, cons
});
}
bool DomainContentBackupManager::recoverFromBackupZip(const QString& backupName, QuaZip& zip) {
bool DomainContentBackupManager::recoverFromBackupZip(const QString& backupName, QuaZip& zip, const QString& sourceFilename, bool rollingBack) {
if (!zip.open(QuaZip::Mode::mdUnzip)) {
qWarning() << "Failed to unzip file: " << backupName;
return false;
@ -286,7 +288,15 @@ bool DomainContentBackupManager::recoverFromBackupZip(const QString& backupName,
_recoveryFilename = backupName;
for (auto& handler : _backupHandlers) {
handler->recoverBackup(backupName, zip);
bool success;
QString errorStr;
std::tie(success, errorStr) = handler->recoverBackup(backupName, zip, sourceFilename);
if (!success) {
if (!rollingBack) {
_recoveryError = errorStr;
}
return false;
}
}
qDebug() << "Successfully started recovering from " << backupName;
@ -309,7 +319,7 @@ void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise,
}
qDebug() << "Recovering from" << backupName;
_recoveryError = "";
bool success { false };
QDir backupDir { _backupDirectory };
auto backupFilePath { backupDir.filePath(backupName) };
@ -317,7 +327,7 @@ void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise,
if (backupFile.open(QIODevice::ReadOnly)) {
QuaZip zip { &backupFile };
success = recoverFromBackupZip(backupName, zip);
success = recoverFromBackupZip(backupName, zip, backupName);
backupFile.close();
} else {
@ -345,29 +355,51 @@ void DomainContentBackupManager::recoverFromUploadedBackup(MiniPromise::Promise
QuaZip uploadedZip { &uploadedBackupBuffer };
QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip";
bool success = recoverFromBackupZip(backupName, uploadedZip);
bool success = recoverFromBackupZip(backupName, uploadedZip, QString());
promise->resolve({
{ "success", success }
});
}
void DomainContentBackupManager::recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename) {
void DomainContentBackupManager::recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename, QString sourceFilename) {
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(this, "recoverFromUploadedFile", Q_ARG(MiniPromise::Promise, promise),
Q_ARG(QString, uploadedFilename));
Q_ARG(QString, uploadedFilename), Q_ARG(QString, sourceFilename));
return;
}
qDebug() << "Recovering from uploaded file -" << uploadedFilename;
qDebug() << "Recovering from uploaded file -" << uploadedFilename << "source" << sourceFilename;
bool success;
QString path;
std::tie(success, path) = createBackup(AUTOMATIC_BACKUP_PREFIX, PRE_UPLOAD_SUFFIX);
if(!success) {
_recoveryError = "Failed to create backup for " + PRE_UPLOAD_SUFFIX + " at " + path;
qCWarning(domain_server) << _recoveryError;
} else {
QFile uploadedFile(uploadedFilename);
QuaZip uploadedZip { &uploadedFile };
QFile uploadedFile(uploadedFilename);
QuaZip uploadedZip { &uploadedFile };
QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip";
QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip";
bool success = recoverFromBackupZip(backupName, uploadedZip, sourceFilename);
bool success = recoverFromBackupZip(backupName, uploadedZip);
if (!success) {
// attempt to rollback to
QString filename;
QDateTime filetime;
if (getMostRecentBackup(PRE_UPLOAD_SUFFIX, filename, filetime)) {
QFile uploadedFile(uploadedFilename);
QuaZip uploadedZip { &uploadedFile };
QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip";
recoverFromBackupZip(backupName, uploadedZip, sourceFilename, true);
}
}
}
promise->resolve({
{ "success", success }
});
@ -455,9 +487,44 @@ void DomainContentBackupManager::getAllBackupsAndStatus(MiniPromise::Promise pro
{ "recoveryProgress", recoveryProgress }
};
if(!_recoveryError.isEmpty()) {
status["recoveryError"] = _recoveryError;
}
QString filename = _settingsManager.valueForKeyPath(CONTENT_SETTINGS_INSTALLED_CONTENT_FILENAME).toString();
QString name = _settingsManager.valueForKeyPath(CONTENT_SETTINGS_INSTALLED_CONTENT_NAME).toString();
auto creationTime = _settingsManager.valueForKeyPath(CONTENT_SETTINGS_INSTALLED_CONTENT_CREATION_TIME).toULongLong();
if (name.isEmpty() || creationTime == 0) {
QString prefixFormat = "(" + QRegExp::escape(AUTOMATIC_BACKUP_PREFIX) + "|" + QRegExp::escape(MANUAL_BACKUP_PREFIX) + ")";
QString nameFormat = "(.+)";
QString dateTimeFormat = "(" + DATETIME_FORMAT_RE + ")";
QRegExp backupNameFormat { prefixFormat + nameFormat + "-" + dateTimeFormat + "\\.zip" };
if (backupNameFormat.exactMatch(filename)) {
if (name.isEmpty()) {
name = backupNameFormat.cap(2);
}
if (creationTime == 0) {
auto dateTime = backupNameFormat.cap(3);
creationTime = QDateTime::fromString(dateTime, DATETIME_FORMAT).toMSecsSinceEpoch();
}
}
}
QVariantMap currentArchive;
currentArchive["filename"] = filename;
currentArchive["name"] = name;
currentArchive["creation_time"] = creationTime;
currentArchive["install_time"] = _settingsManager.valueForKeyPath(CONTENT_SETTINGS_INSTALLED_CONTENT_INSTALL_TIME).toULongLong();
currentArchive["installed_by"] = _settingsManager.valueForKeyPath(CONTENT_SETTINGS_INSTALLED_CONTENT_INSTALLED_BY).toString();
QVariantMap info {
{ "backups", variantBackups },
{ "status", status }
{ "status", status },
{ "installed_content", currentArchive }
};
promise->resolve(info);

View file

@ -28,11 +28,22 @@
#include <GenericThread.h>
#include "BackupHandler.h"
#include "DomainServerSettingsManager.h"
#include <shared/MiniPromises.h>
#include <PortableHighResolutionClock.h>
const QString DATETIME_FORMAT_RE { "\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}" };
const QString AUTOMATIC_BACKUP_PREFIX { "autobackup-" };
const QString MANUAL_BACKUP_PREFIX { "backup-" };
const QString INSTALLED_CONTENT = "installed_content";
const QString INSTALLED_CONTENT_FILENAME = "filename";
const QString INSTALLED_CONTENT_NAME = "name";
const QString INSTALLED_CONTENT_CREATION_TIME = "creation_time";
const QString INSTALLED_CONTENT_INSTALL_TIME = "install_time";
const QString INSTALLED_CONTENT_INSTALLED_BY = "installed_by";
struct BackupItemInfo {
BackupItemInfo(QString pId, QString pName, QString pAbsolutePath, QDateTime pCreatedAt, bool pIsManualBackup) :
id(pId), name(pName), absolutePath(pAbsolutePath), createdAt(pCreatedAt), isManualBackup(pIsManualBackup) { };
@ -71,7 +82,7 @@ public:
static const std::chrono::seconds DEFAULT_PERSIST_INTERVAL;
DomainContentBackupManager(const QString& rootBackupDirectory,
const QVariantList& settings,
DomainServerSettingsManager& domainServerSettingsManager,
std::chrono::milliseconds persistInterval = DEFAULT_PERSIST_INTERVAL,
bool debugTimestampNow = false);
@ -86,7 +97,7 @@ public slots:
void createManualBackup(MiniPromise::Promise promise, const QString& name);
void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName);
void recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup);
void recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename);
void recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename, QString sourceFilename);
void deleteBackup(MiniPromise::Promise promise, const QString& backupName);
signals:
@ -108,13 +119,15 @@ protected:
std::pair<bool, QString> createBackup(const QString& prefix, const QString& name);
bool recoverFromBackupZip(const QString& backupName, QuaZip& backupZip);
bool recoverFromBackupZip(const QString& backupName, QuaZip& backupZip, const QString& sourceFilename, bool rollingBack = false);
private slots:
void removeOldConsolidatedBackups();
void consolidateBackupInternal(QString fileName);
private:
DomainServerSettingsManager& _settingsManager;
QTimer _consolidatedBackupCleanupTimer;
const QString _consolidatedBackupDirectory;
@ -126,6 +139,7 @@ private:
std::unordered_map<QString, ConsolidatedBackupInfo> _consolidatedBackups;
std::atomic<bool> _isRecovering { false };
QString _recoveryError;
QString _recoveryFilename { };
p_high_resolution_clock::time_point _lastCheck;

View file

@ -307,11 +307,7 @@ DomainServer::DomainServer(int argc, char* argv[]) :
}
maybeHandleReplacementEntityFile();
static const QString BACKUP_RULES_KEYPATH = AUTOMATIC_CONTENT_ARCHIVES_GROUP + ".backup_rules";
auto backupRulesVariant = _settingsManager.valueOrDefaultValueForKeyPath(BACKUP_RULES_KEYPATH);
_contentManager.reset(new DomainContentBackupManager(getContentBackupDir(), backupRulesVariant.toList()));
_contentManager.reset(new DomainContentBackupManager(getContentBackupDir(), _settingsManager));
connect(_contentManager.get(), &DomainContentBackupManager::started, _contentManager.get(), [this](){
_contentManager->addBackupHandler(BackupHandlerPointer(new EntitiesBackupHandler(getEntitiesFilePath(), getEntitiesReplacementFilePath())));
@ -2194,7 +2190,7 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
return true;
} else if (url.path() == URI_RESTART) {
connection->respond(HTTPConnection::StatusCode200);
connection->respond(HTTPConnection::StatusCode204);
restart();
return true;
} else if (url.path() == URI_API_METAVERSE_INFO) {
@ -2333,8 +2329,7 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
QJsonObject rootJSON;
auto success = result["success"].toBool();
rootJSON["success"] = success;
QJsonDocument docJSON(rootJSON);
QJsonDocument docJSON(QJsonObject::fromVariantMap(result));
connectionPtr->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
JSON_MIME_TYPE.toUtf8());
});
@ -2362,8 +2357,7 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
QJsonObject rootJSON;
auto success = result["success"].toBool();
rootJSON["success"] = success;
QJsonDocument docJSON(rootJSON);
QJsonDocument docJSON(QJsonObject::fromVariantMap(result));
connectionPtr->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
JSON_MIME_TYPE.toUtf8());
});
@ -2467,8 +2461,7 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
QJsonObject rootJSON;
auto success = result["success"].toBool();
rootJSON["success"] = success;
QJsonDocument docJSON(rootJSON);
QJsonDocument docJSON(QJsonObject::fromVariantMap(result));
connectionPtr->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
JSON_MIME_TYPE.toUtf8());
});
@ -2590,7 +2583,7 @@ bool DomainServer::processPendingContent(HTTPConnection* connection, QString ite
_pendingFileContent.close();
// Respond immediately - will timeout if we wait for restore.
connection->respond(HTTPConnection::StatusCode200);
connection->respond(HTTPConnection::StatusCode204);
if (itemName == "restore-file" || itemName == "restore-file-chunk-final" || itemName == "restore-file-chunk-only") {
auto deferred = makePromise("recoverFromUploadedBackup");
@ -2598,7 +2591,7 @@ bool DomainServer::processPendingContent(HTTPConnection* connection, QString ite
_pendingContentFiles.erase(sessionId);
});
_contentManager->recoverFromUploadedFile(deferred, _pendingFileContent.fileName());
_contentManager->recoverFromUploadedFile(deferred, _pendingFileContent.fileName(), filename);
}
} else if (filename.endsWith(".json", Qt::CaseInsensitive)
|| filename.endsWith(".json.gz", Qt::CaseInsensitive)) {
@ -2608,14 +2601,16 @@ bool DomainServer::processPendingContent(HTTPConnection* connection, QString ite
}
QByteArray& _pendingUploadedContent = _pendingUploadedContents[sessionId];
_pendingUploadedContent += dataChunk;
connection->respond(HTTPConnection::StatusCode200);
if (itemName == "restore-file" || itemName == "restore-file-chunk-final" || itemName == "restore-file-chunk-only") {
// invoke our method to hand the new octree file off to the octree server
QMetaObject::invokeMethod(this, "handleOctreeFileReplacement",
Qt::QueuedConnection, Q_ARG(QByteArray, _pendingUploadedContent));
if (!handleOctreeFileReplacement(_pendingUploadedContent, filename, QString())) {
connection->respond(HTTPConnection::StatusCode400);
return false;
}
_pendingUploadedContents.erase(sessionId);
}
connection->respond(HTTPConnection::StatusCode204);
} else {
connection->respond(HTTPConnection::StatusCode400);
return false;
@ -2687,11 +2682,12 @@ void DomainServer::profileRequestFinished() {
bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl& url) {
const QByteArray HTTP_COOKIE_HEADER_KEY = "Cookie";
const QString ADMIN_USERS_CONFIG_KEY = "admin-users";
const QString ADMIN_ROLES_CONFIG_KEY = "admin-roles";
const QString BASIC_AUTH_USERNAME_KEY_PATH = "security.http_username";
const QString BASIC_AUTH_PASSWORD_KEY_PATH = "security.http_password";
static const QByteArray HTTP_COOKIE_HEADER_KEY = "Cookie";
static const QString ADMIN_USERS_CONFIG_KEY = "admin-users";
static const QString ADMIN_ROLES_CONFIG_KEY = "admin-roles";
static const QString BASIC_AUTH_USERNAME_KEY_PATH = "security.http_username";
static const QString BASIC_AUTH_PASSWORD_KEY_PATH = "security.http_password";
const QString COOKIE_UUID_REGEX_STRING = HIFI_SESSION_COOKIE_KEY + "=([\\d\\w-]+)($|;)";
const QByteArray UNAUTHENTICATED_BODY = "You do not have permission to access this domain-server.";
@ -2702,7 +2698,6 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl
&& (adminUsersVariant.isValid() || adminRolesVariant.isValid())) {
QString cookieString = connection->requestHeader(HTTP_COOKIE_HEADER_KEY);
const QString COOKIE_UUID_REGEX_STRING = HIFI_SESSION_COOKIE_KEY + "=([\\d\\w-]+)($|;)";
QRegExp cookieUUIDRegex(COOKIE_UUID_REGEX_STRING);
QUuid cookieUUID;
@ -3498,7 +3493,7 @@ void DomainServer::maybeHandleReplacementEntityFile() {
}
}
void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) {
bool DomainServer::handleOctreeFileReplacement(QByteArray octreeFile, QString sourceFilename, QString name) {
OctreeUtils::RawEntityData data;
if (data.readOctreeDataInfoFromData(octreeFile)) {
data.resetIdAndVersion();
@ -3514,15 +3509,32 @@ void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) {
// process it when it comes back up
qInfo() << "Wrote octree replacement file to" << replacementFilePath << "- stopping server";
QJsonObject installed_content {
{ INSTALLED_CONTENT_FILENAME, sourceFilename },
{ INSTALLED_CONTENT_NAME, name },
{ INSTALLED_CONTENT_CREATION_TIME, 0 },
{ INSTALLED_CONTENT_INSTALL_TIME, QDateTime::currentDateTime().currentMSecsSinceEpoch() },
{ INSTALLED_CONTENT_INSTALLED_BY, "" }
};
QJsonObject jsonObject { { INSTALLED_CONTENT, installed_content } };
_settingsManager.recurseJSONObjectAndOverwriteSettings(jsonObject, ContentSettings);
QMetaObject::invokeMethod(this, "restart", Qt::QueuedConnection);
return true;
} else {
qWarning() << "Could not write replacement octree data to file - refusing to process";
return false;
}
} else {
qDebug() << "Received replacement octree file that is invalid - refusing to process";
return false;
}
}
static const QString CONTENT_SET_NAME_QUERY_PARAM = "name";
void DomainServer::handleDomainContentReplacementFromURLRequest(QSharedPointer<ReceivedMessage> message) {
qInfo() << "Received request to replace content from a url";
auto node = DependencyManager::get<LimitedNodeList>()->findNodeWithAddr(message->getSenderSockAddr());
@ -3534,13 +3546,16 @@ void DomainServer::handleDomainContentReplacementFromURLRequest(QSharedPointer<R
QNetworkRequest request(modelsURL);
QNetworkReply* reply = networkAccessManager.get(request);
qDebug() << "Downloading JSON from: " << modelsURL;
qDebug() << "Downloading JSON from: " << modelsURL.toString(QUrl::FullyEncoded);
connect(reply, &QNetworkReply::finished, [this, reply, modelsURL]() {
QNetworkReply::NetworkError networkError = reply->error();
if (networkError == QNetworkReply::NoError) {
if (modelsURL.fileName().endsWith(".json.gz")) {
handleOctreeFileReplacement(reply->readAll());
QUrlQuery urlQuery(modelsURL.query(QUrl::FullyEncoded));
QString itemName = urlQuery.queryItemValue(CONTENT_SET_NAME_QUERY_PARAM);
handleOctreeFileReplacement(reply->readAll(), modelsURL.fileName(), itemName);
} else if (modelsURL.fileName().endsWith(".zip")) {
auto deferred = makePromise("recoverFromUploadedBackup");
_contentManager->recoverFromUploadedBackup(deferred, reply->readAll());
@ -3555,6 +3570,6 @@ void DomainServer::handleDomainContentReplacementFromURLRequest(QSharedPointer<R
void DomainServer::handleOctreeFileReplacementRequest(QSharedPointer<ReceivedMessage> message) {
auto node = DependencyManager::get<NodeList>()->nodeWithLocalID(message->getSourceID());
if (node->getCanReplaceContent()) {
handleOctreeFileReplacement(message->readAll());
handleOctreeFileReplacement(message->readAll(), QString(), QString());
}
}

View file

@ -99,7 +99,7 @@ private slots:
void handleDomainContentReplacementFromURLRequest(QSharedPointer<ReceivedMessage> message);
void handleOctreeFileReplacementRequest(QSharedPointer<ReceivedMessage> message);
void handleOctreeFileReplacement(QByteArray octreeFile);
bool handleOctreeFileReplacement(QByteArray octreeFile, QString sourceFilename, QString name);
void processOctreeDataRequestMessage(QSharedPointer<ReceivedMessage> message);
void processOctreeDataPersistMessage(QSharedPointer<ReceivedMessage> message);

View file

@ -35,6 +35,11 @@ const QString MACHINE_FINGERPRINT_PERMISSIONS_KEYPATH = "security.machine_finger
const QString GROUP_PERMISSIONS_KEYPATH = "security.group_permissions";
const QString GROUP_FORBIDDENS_KEYPATH = "security.group_forbiddens";
const QString AUTOMATIC_CONTENT_ARCHIVES_GROUP = "automatic_content_archives";
const QString CONTENT_SETTINGS_INSTALLED_CONTENT_FILENAME = "installed_content.filename";
const QString CONTENT_SETTINGS_INSTALLED_CONTENT_NAME = "installed_content.name";
const QString CONTENT_SETTINGS_INSTALLED_CONTENT_CREATION_TIME = "installed_content.creation_time";
const QString CONTENT_SETTINGS_INSTALLED_CONTENT_INSTALL_TIME = "installed_content.install_time";
const QString CONTENT_SETTINGS_INSTALLED_CONTENT_INSTALLED_BY = "installed_content.installed_by";
using GroupByUUIDKey = QPair<QUuid, QUuid>; // groupID, rankID

View file

@ -57,36 +57,41 @@ void EntitiesBackupHandler::createBackup(const QString& backupName, QuaZip& zip)
}
}
void EntitiesBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
std::pair<bool, QString> EntitiesBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip, const QString& sourceFilename) {
if (!zip.setCurrentFile(ENTITIES_BACKUP_FILENAME)) {
qWarning() << "Failed to find" << ENTITIES_BACKUP_FILENAME << "while recovering backup";
return;
QString errorStr("Failed to find " + ENTITIES_BACKUP_FILENAME + " while recovering backup");
qWarning() << errorStr;
return { false, errorStr };
}
QuaZipFile zipFile { &zip };
if (!zipFile.open(QIODevice::ReadOnly)) {
qCritical() << "Failed to open" << ENTITIES_BACKUP_FILENAME << "in backup";
return;
QString errorStr("Failed to open " + ENTITIES_BACKUP_FILENAME + " in backup");
qCritical() << errorStr;
return { false, errorStr };
}
auto rawData = zipFile.readAll();
zipFile.close();
if (zipFile.getZipError() != UNZ_OK) {
QString errorStr("Failed to unzip " + ENTITIES_BACKUP_FILENAME + ": " + zipFile.getZipError());
qCritical() << errorStr;
return { false, errorStr };
}
OctreeUtils::RawEntityData data;
if (!data.readOctreeDataInfoFromData(rawData)) {
qCritical() << "Unable to parse octree data during backup recovery";
return;
QString errorStr("Unable to parse octree data during backup recovery");
qCritical() << errorStr;
return { false, errorStr };
}
data.resetIdAndVersion();
if (zipFile.getZipError() != UNZ_OK) {
qCritical().nospace() << "Failed to unzip " << ENTITIES_BACKUP_FILENAME << ": " << zipFile.getZipError();
return;
}
QFile entitiesFile { _entitiesReplacementFilePath };
if (entitiesFile.open(QIODevice::WriteOnly)) {
entitiesFile.write(data.toGzippedByteArray());
}
return { true, QString() };
}

View file

@ -29,7 +29,7 @@ public:
void createBackup(const QString& backupName, QuaZip& zip) override;
// Recover from a full backup
void recoverBackup(const QString& backupName, QuaZip& zip) override;
std::pair<bool, QString> recoverBackup(const QString& backupName, QuaZip& zip, const QString& sourceFilename) override;
// Delete a skeleton backup
void deleteBackup(const QString& backupName) override {}

View file

@ -786,7 +786,7 @@ Rectangle {
}
lightboxPopup.button2text = "CONFIRM";
lightboxPopup.button2method = function() {
Commerce.replaceContentSet(root.itemHref, root.certificateId);
Commerce.replaceContentSet(root.itemHref, root.certificateId, root.itemName);
lightboxPopup.visible = false;
rezzedNotifContainer.visible = true;
rezzedNotifContainerTimer.start();

View file

@ -36,7 +36,7 @@ Rectangle {
break;
case "content set":
urlHandler.handleUrl("hifi://localhost/0,0,0");
Commerce.replaceContentSet(toUrl(resource), "");
Commerce.replaceContentSet(toUrl(resource), "", "");
break;
case "entity":
case "wearable":

View file

@ -729,7 +729,7 @@ Item {
onClicked: {
Tablet.playSound(TabletEnums.ButtonClick);
if (root.itemType === "contentSet") {
sendToPurchases({method: 'showReplaceContentLightbox', itemHref: root.itemHref, certID: root.certificateId});
sendToPurchases({method: 'showReplaceContentLightbox', itemHref: root.itemHref, certID: root.certificateId, itemName: root.itemName});
} else if (root.itemType === "avatar") {
sendToPurchases({method: 'showChangeAvatarLightbox', itemName: root.itemName, itemHref: root.itemHref});
} else if (root.itemType === "app") {

View file

@ -609,7 +609,7 @@ Rectangle {
}
lightboxPopup.button2text = "CONFIRM";
lightboxPopup.button2method = function() {
Commerce.replaceContentSet(msg.itemHref, msg.certID);
Commerce.replaceContentSet(msg.itemHref, msg.certID, msg.itemName);
lightboxPopup.visible = false;
};
lightboxPopup.visible = true;

View file

@ -7789,9 +7789,15 @@ bool Application::askToWearAvatarAttachmentUrl(const QString& url) {
return true;
}
void Application::replaceDomainContent(const QString& url) {
static const QString CONTENT_SET_NAME_QUERY_PARAM = "name";
void Application::replaceDomainContent(const QString& url, const QString& itemName) {
qCDebug(interfaceapp) << "Attempting to replace domain content";
QByteArray urlData(url.toUtf8());
QUrl msgUrl(url);
QUrlQuery urlQuery(msgUrl.query());
urlQuery.addQueryItem(CONTENT_SET_NAME_QUERY_PARAM, itemName);
msgUrl.setQuery(urlQuery.query(QUrl::QUrl::FullyEncoded));
QByteArray urlData(msgUrl.toString(QUrl::QUrl::FullyEncoded).toUtf8());
auto limitedNodeList = DependencyManager::get<NodeList>();
const auto& domainHandler = limitedNodeList->getDomainHandler();
@ -7825,7 +7831,7 @@ bool Application::askToReplaceDomainContent(const QString& url) {
QString details;
if (static_cast<QMessageBox::StandardButton>(answer.toInt()) == QMessageBox::Yes) {
// Given confirmation, send request to domain server to replace content
replaceDomainContent(url);
replaceDomainContent(url, QString());
details = "SuccessfulRequestToReplaceContent";
} else {
details = "UserDeclinedToReplaceContent";

View file

@ -326,7 +326,7 @@ public:
bool isInterstitialMode() const { return _interstitialMode; }
bool failedToConnectToEntityServer() const { return _failedToConnectToEntityServer; }
void replaceDomainContent(const QString& url);
void replaceDomainContent(const QString& url, const QString& itemName);
void loadAvatarScripts(const QVector<QString>& urls);
void unloadAvatarScripts();

View file

@ -259,7 +259,7 @@ void QmlCommerce::authorizeAssetTransfer(const QString& couponID,
ledger->authorizeAssetTransfer(key, couponID, certificateID, amount, optionalMessage);
}
void QmlCommerce::replaceContentSet(const QString& itemHref, const QString& certificateID) {
void QmlCommerce::replaceContentSet(const QString& itemHref, const QString& certificateID, const QString& itemName) {
if (!certificateID.isEmpty()) {
auto ledger = DependencyManager::get<Ledger>();
ledger->updateLocation(
@ -267,7 +267,7 @@ void QmlCommerce::replaceContentSet(const QString& itemHref, const QString& cert
DependencyManager::get<AddressManager>()->getPlaceName(),
true);
}
qApp->replaceDomainContent(itemHref);
qApp->replaceDomainContent(itemHref, itemName);
QJsonObject messageProperties = {
{ "status", "SuccessfulRequestToReplaceContent" },
{ "content_set_url", itemHref } };

View file

@ -90,7 +90,7 @@ protected:
Q_INVOKABLE void transferAssetToUsername(const QString& username, const QString& certificateID, const int& amount, const QString& optionalMessage);
Q_INVOKABLE void authorizeAssetTransfer(const QString& couponID, const QString& certificateID, const int& amount, const QString& optionalMessage);
Q_INVOKABLE void replaceContentSet(const QString& itemHref, const QString& certificateID);
Q_INVOKABLE void replaceContentSet(const QString& itemHref, const QString& certificateID, const QString& itemName);
Q_INVOKABLE QString getInstalledApps(const QString& justInstalledAppID = "");
Q_INVOKABLE bool installApp(const QString& appHref, const bool& alsoOpenImmediately = false);

View file

@ -22,6 +22,7 @@
#include "HTTPManager.h"
const char* HTTPConnection::StatusCode200 = "200 OK";
const char* HTTPConnection::StatusCode204 = "204 No Content";
const char* HTTPConnection::StatusCode301 = "301 Moved Permanently";
const char* HTTPConnection::StatusCode302 = "302 Found";
const char* HTTPConnection::StatusCode400 = "400 Bad Request";

View file

@ -45,6 +45,7 @@ class HTTPConnection : public QObject {
public:
static const char* StatusCode200;
static const char* StatusCode204;
static const char* StatusCode301;
static const char* StatusCode302;
static const char* StatusCode400;