Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-19 15:43:50 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #12423 from Atlante45/feat/backups-integration

Browse source 
Return Backup and Recovery status and progress
This commit is contained in:
Stephen Birarda 2018-02-15 19:26:56 -07:00 committed by GitHub
commit 61ccd82916
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 269 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
assignment-client/src/assets/AssetServer.cpp
View file

@ -416,9 +416,9 @@ void AssetServer::completeSetup() {
PathUtils::removeTemporaryApplicationDirs();
PathUtils::removeTemporaryApplicationDirs("Oven");
// We're fully setup, remove the request queueing and replay all requests
qCDebug(asset_server) << "Overriding temporary queuing packet handler.";
// We're fully setup, override the request queueing handler and replay all requests
auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
packetReceiver.unregisterListener(this);
packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet");
packetReceiver.registerListener(PacketType::AssetGetInfo, this, "handleAssetGetInfo");
packetReceiver.registerListener(PacketType::AssetUpload, this, "handleAssetUpload");
@ -428,11 +428,30 @@ void AssetServer::completeSetup() {
}
void AssetServer::queueRequests(QSharedPointer<ReceivedMessage> packet, SharedNodePointer senderNode) {
qCDebug(asset_server) << "Queuing requests until fully setup";
QMutexLocker lock { &_queuedRequestsMutex };
_queuedRequests.push_back({ packet, senderNode });
// If we've stopped queueing but the callback was already in flight,
// then replay it immediately.
if (!_isQueueingRequests) {
lock.unlock();
replayRequests();
}
}
void AssetServer::replayRequests() {
for (const auto& request : _queuedRequests) {
RequestQueue queue;
{
QMutexLocker lock { &_queuedRequestsMutex };
qSwap(queue, _queuedRequests);
_isQueueingRequests = false;
}
qCDebug(asset_server) << "Replaying" << queue.size() << "requests.";
for (const auto& request : queue) {
switch (request.first->getType()) {
case PacketType::AssetGet:
handleAssetGet(request.first, request.second);
@ -447,11 +466,10 @@ void AssetServer::replayRequests() {
handleAssetMappingOperation(request.first, request.second);
break;
default:
qWarning() << "Unknown queued request type:" << request.first->getType();
qCWarning(asset_server) << "Unknown queued request type:" << request.first->getType();
break;
}
}
_queuedRequests.clear();
}
void AssetServer::cleanupUnmappedFiles() {

5
assignment-client/src/assets/AssetServer.h
View file

@ -123,7 +123,10 @@ private:
QHash<AssetUtils::AssetHash, std::shared_ptr<BakeAssetTask>> _pendingBakes;
QThreadPool _bakingTaskPool;
QVector<QPair<QSharedPointer<ReceivedMessage>, SharedNodePointer>> _queuedRequests;
QMutex _queuedRequestsMutex;
bool _isQueueingRequests { true };
using RequestQueue = QVector<QPair<QSharedPointer<ReceivedMessage>, SharedNodePointer>>;
RequestQueue _queuedRequests;
bool _wasColorTextureCompressionEnabled { false };
bool _wasGrayscaleTextureCompressionEnabled { false };

51
domain-server/src/AssetsBackupHandler.cpp
View file

@ -111,6 +111,42 @@ void AssetsBackupHandler::checkForAssetsToDelete() {
}
}
std::pair<bool, float> AssetsBackupHandler::isAvailable(QString filePath) {
auto it = find_if(begin(_backups), end(_backups), [&](const std::vector<AssetServerBackup>::value_type& value) {
return value.filePath == filePath;
});
if (it == end(_backups)) {
return { true, 1.0f };
}
float progress = (float)it->mappings.size();
for (const auto& mapping : it->mappings) {
if (_assetsLeftToRequest.find(mapping.second) != end(_assetsLeftToRequest)) {
progress -= 1.0f;
}
}
progress /= (float)it->mappings.size();
return { false, progress };
}
std::pair<bool, float> AssetsBackupHandler::getRecoveryStatus() {
if (_assetsLeftToUpload.empty() &&
_mappingsLeftToSet.empty() &&
_mappingsLeftToDelete.empty() &&
_mappingRequestsInFlight == 0) {
return { false, 1.0f };
}
float progress = (float)_numRestoreOperations;
progress -= (float)_assetsLeftToUpload.size();
progress -= (float)_mappingRequestsInFlight;
progress /= (float)_numRestoreOperations;
return { true, progress };
}
void AssetsBackupHandler::loadBackup(QuaZip& zip) {
Q_ASSERT(QThread::currentThread() == thread());
@ -451,6 +487,11 @@ void AssetsBackupHandler::computeServerStateDifference(const AssetUtils::Mapping
}
}
_numRestoreOperations = (int)_assetsLeftToUpload.size() + (int)_mappingsLeftToSet.size();
if (!_mappingsLeftToDelete.empty()) {
++_numRestoreOperations;
}
qCDebug(asset_backup) << "Mappings to set:" << _mappingsLeftToSet.size();
qCDebug(asset_backup) << "Mappings to del:" << _mappingsLeftToDelete.size();
qCDebug(asset_backup) << "Assets to upload:" << _assetsLeftToUpload.size();
@ -461,8 +502,6 @@ void AssetsBackupHandler::restoreAllAssets() {
}
void AssetsBackupHandler::restoreNextAsset() {
startOperation();
if (_assetsLeftToUpload.empty()) {
updateMappings();
return;
@ -500,9 +539,7 @@ void AssetsBackupHandler::updateMappings() {
qCCritical(asset_backup) << " Error:" << request->getErrorString();
}
if (--_mappingRequestsInFlight == 0) {
stopOperation();
}
--_mappingRequestsInFlight;
request->deleteLater();
});
@ -519,9 +556,7 @@ void AssetsBackupHandler::updateMappings() {
qCCritical(asset_backup) << " Error:" << request->getErrorString();
}
if (--_mappingRequestsInFlight == 0) {
stopOperation();
}
--_mappingRequestsInFlight;
request->deleteLater();
});

21
domain-server/src/AssetsBackupHandler.h
View file

@ -31,13 +31,16 @@ class AssetsBackupHandler : public QObject, public BackupHandlerInterface {
public:
AssetsBackupHandler(const QString& backupDirectory);
void loadBackup(QuaZip& zip);
void createBackup(QuaZip& zip);
void recoverBackup(QuaZip& zip);
void deleteBackup(QuaZip& zip);
void consolidateBackup(QuaZip& zip);
std::pair<bool, float> isAvailable(QString filePath) override;
std::pair<bool, float> getRecoveryStatus() override;
bool operationInProgress() const { return _operationInProgress; }
void loadBackup(QuaZip& zip) override;
void createBackup(QuaZip& zip) override;
void recoverBackup(QuaZip& zip) override;
void deleteBackup(QuaZip& zip) override;
void consolidateBackup(QuaZip& zip) override;
bool operationInProgress() { return getRecoveryStatus().first; }
private:
void setupRefreshTimer();
@ -48,9 +51,6 @@ private:
void checkForMissingAssets();
void checkForAssetsToDelete();
void startOperation() { _operationInProgress = true; }
void stopOperation() { _operationInProgress = false; }
void downloadMissingFiles(const AssetUtils::Mappings& mappings);
void downloadNextMissingFile();
bool writeAssetFile(const AssetUtils::AssetHash& hash, const QByteArray& data);
@ -73,8 +73,6 @@ private:
bool corruptedBackup;
};
bool _operationInProgress { false };
// Internal storage for backups on disk
bool _allBackupsLoadedSuccessfully { false };
std::vector<AssetServerBackup> _backups;
@ -89,6 +87,7 @@ private:
std::vector<std::pair<AssetUtils::AssetPath, AssetUtils::AssetHash>> _mappingsLeftToSet;
AssetUtils::AssetPathList _mappingsLeftToDelete;
int _mappingRequestsInFlight { 0 };
int _numRestoreOperations { 0 }; // Used to compute a restore progress.
};
#endif /* hifi_AssetsBackupHandler_h */

8
domain-server/src/BackupHandler.h
View file

@ -14,19 +14,25 @@
#include <memory>
#include <QString>
class QuaZip;
class BackupHandlerInterface {
public:
virtual ~BackupHandlerInterface() = default;
virtual std::pair<bool, float> isAvailable(QString filePath) = 0;
// Returns whether a recovery is ongoing and a progress between 0 and 1 if one is.
virtual std::pair<bool, float> getRecoveryStatus() = 0;
virtual void loadBackup(QuaZip& zip) = 0;
virtual void createBackup(QuaZip& zip) = 0;
virtual void recoverBackup(QuaZip& zip) = 0;
virtual void deleteBackup(QuaZip& zip) = 0;
virtual void consolidateBackup(QuaZip& zip) = 0;
};
using BackupHandlerPointer = std::unique_ptr<BackupHandlerInterface>;
#endif /* hifi_BackupHandler_h */

13
domain-server/src/ContentSettingsBackupHandler.h
View file

@ -19,15 +19,18 @@ class ContentSettingsBackupHandler : public BackupHandlerInterface {
public:
ContentSettingsBackupHandler(DomainServerSettingsManager& domainServerSettingsManager);
void loadBackup(QuaZip& zip) {};
std::pair<bool, float> isAvailable(QString filePath) override { return { true, 1.0f }; }
std::pair<bool, float> getRecoveryStatus() override { return { false, 1.0f }; }
void createBackup(QuaZip& zip);
void loadBackup(QuaZip& zip) override {}
void recoverBackup(QuaZip& zip);
void createBackup(QuaZip& zip) override;
void deleteBackup(QuaZip& zip) {};
void recoverBackup(QuaZip& zip) override;
void consolidateBackup(QuaZip& zip) {};
void deleteBackup(QuaZip& zip) override {}
void consolidateBackup(QuaZip& zip) override {}
private:
DomainServerSettingsManager& _settingsManager;
};

162
domain-server/src/DomainContentBackupManager.cpp
View file

@ -147,7 +147,21 @@ bool DomainContentBackupManager::process() {
if (sinceLastSave > intervalToCheck) {
_lastCheck = now;
backup();
if (_isRecovering) {
using Value = std::vector<BackupHandlerPointer>::value_type;
bool isStillRecovering = std::any_of(begin(_backupHandlers), end(_backupHandlers), [](const Value& handler) {
return handler->getRecoveryStatus().first;
});
if (!isStillRecovering) {
_isRecovering = false;
emit recoveryCompleted();
}
}
if (!_isRecovering) {
backup();
}
}
}
@ -213,6 +227,13 @@ void DomainContentBackupManager::deleteBackup(MiniPromise::Promise promise, cons
return;
}
if (_isRecovering && backupName == _recoveryFilename) {
promise->resolve({
{ "success", false }
});
return;
}
QDir backupDir { _backupDirectory };
QFile backupFile { backupDir.filePath(backupName) };
auto success = backupFile.remove();
@ -222,6 +243,13 @@ void DomainContentBackupManager::deleteBackup(MiniPromise::Promise promise, cons
}
void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise, const QString& backupName) {
if (_isRecovering) {
promise->resolve({
{ "success", false }
});
return;
};
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(this, "recoverFromBackup", Q_ARG(MiniPromise::Promise, promise),
Q_ARG(const QString&, backupName));
@ -239,11 +267,12 @@ void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise,
qWarning() << "Failed to unzip file: " << backupName;
success = false;
} else {
_isRecovering = true;
for (auto& handler : _backupHandlers) {
handler->recoverBackup(zip);
}
qDebug() << "Successfully recovered from " << backupName;
qDebug() << "Successfully started recovering from " << backupName;
success = true;
}
backupFile.close();
@ -257,8 +286,11 @@ void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise,
});
}
std::vector<BackupItemInfo> DomainContentBackupManager::getAllBackups() {
std::vector<BackupItemInfo> backups;
void DomainContentBackupManager::getAllBackupInformation(MiniPromise::Promise promise) {
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(this, "getAllBackupInformation", Q_ARG(MiniPromise::Promise, promise));
return;
}
QDir backupDir { _backupDirectory };
auto matchingFiles =
@ -269,6 +301,8 @@ std::vector<BackupItemInfo> DomainContentBackupManager::getAllBackups() {
QString dateTimeFormat = "(" + DATETIME_FORMAT_RE + ")";
QRegExp backupNameFormat { prefixFormat + nameFormat + "-" + dateTimeFormat + "\\.zip" };
QVariantList backups;
for (const auto& fileInfo : matchingFiles) {
auto fileName = fileInfo.fileName();
if (backupNameFormat.exactMatch(fileName)) {
@ -280,12 +314,48 @@ std::vector<BackupItemInfo> DomainContentBackupManager::getAllBackups() {
continue;
}
BackupItemInfo backup { fileInfo.fileName(), name, fileInfo.absoluteFilePath(), createdAt, type == MANUAL_BACKUP_PREFIX };
backups.push_back(backup);
bool isAvailable { true };
float availabilityProgress { 0.0f };
for (auto& handler : _backupHandlers) {
bool handlerIsAvailable { true };
float progress { 0.0f };
std::tie(handlerIsAvailable, progress) = handler->isAvailable(fileInfo.absoluteFilePath());
isAvailable &= handlerIsAvailable;
availabilityProgress += progress / _backupHandlers.size();
}
backups.push_back(QVariantMap({
{ "id", fileInfo.fileName() },
{ "name", name },
{ "createdAtMillis", createdAt.toMSecsSinceEpoch() },
{ "isAvailable", isAvailable },
{ "availabilityProgress", availabilityProgress },
{ "isManualBackup", type == MANUAL_BACKUP_PREFIX }
}));
}
}
return backups;
float recoveryProgress = 0.0f;
bool isRecovering = _isRecovering.load();
if (_isRecovering) {
for (auto& handler : _backupHandlers) {
float progress = handler->getRecoveryStatus().second;
recoveryProgress += progress / _backupHandlers.size();
}
}
QVariantMap status {
{ "isRecovering", isRecovering },
{ "recoveringBackupId", _recoveryFilename },
{ "recoveryProgress", recoveryProgress }
};
QVariantMap info {
{ "backups", backups },
{ "status", status }
};
promise->resolve(info);
}
void DomainContentBackupManager::removeOldBackupVersions(const BackupRule& rule) {
@ -383,32 +453,60 @@ void DomainContentBackupManager::backup() {
}
}
void DomainContentBackupManager::consolidate(QString fileName) {
QDir backupDir { _backupDirectory };
if (backupDir.exists()) {
auto filePath = backupDir.absoluteFilePath(fileName);
auto copyFilePath = QDir::tempPath() + "/" + fileName;
auto copySuccess = QFile::copy(filePath, copyFilePath);
if (!copySuccess) {
qCritical() << "Failed to create full backup.";
return;
}
QuaZip zip(copyFilePath);
if (!zip.open(QuaZip::mdAdd)) {
qCritical() << "Could not open backup archive:" << filePath;
qCritical() << " ERROR:" << zip.getZipError();
return;
}
for (auto& handler : _backupHandlers) {
handler->consolidateBackup(zip);
}
zip.close();
void DomainContentBackupManager::consolidateBackup(MiniPromise::Promise promise, QString fileName) {
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(this, "consolidateBackup", Q_ARG(MiniPromise::Promise, promise),
Q_ARG(const QString&, fileName));
return;
}
QDir backupDir { _backupDirectory };
if (!backupDir.exists()) {
qCritical() << "Backup directory does not exist, bailing consolidation of backup";
promise->resolve({ { "success", false } });
return;
}
auto filePath = backupDir.absoluteFilePath(fileName);
auto copyFilePath = QDir::tempPath() + "/" + fileName;
{
QFile copyFile(copyFilePath);
copyFile.remove();
copyFile.close();
}
auto copySuccess = QFile::copy(filePath, copyFilePath);
if (!copySuccess) {
qCritical() << "Failed to create copy of backup.";
promise->resolve({ { "success", false } });
return;
}
QuaZip zip(copyFilePath);
if (!zip.open(QuaZip::mdAdd)) {
qCritical() << "Could not open backup archive:" << filePath;
qCritical() << " ERROR:" << zip.getZipError();
promise->resolve({ { "success", false } });
return;
}
for (auto& handler : _backupHandlers) {
handler->consolidateBackup(zip);
}
zip.close();
if (zip.getZipError() != UNZ_OK) {
qCritical() << "Failed to consolidate backup: " << zip.getZipError();
promise->resolve({ { "success", false } });
return;
}
promise->resolve({
{ "success", true },
{ "backupFilePath", copyFilePath }
});
}
void DomainContentBackupManager::createManualBackup(MiniPromise::Promise promise, const QString& name) {

16
domain-server/src/DomainContentBackupManager.h
View file

@ -24,14 +24,6 @@
#include <shared/MiniPromises.h>
struct BackupItemInfo {
QString id;
QString name;
QString absolutePath;
QDateTime createdAt;
bool isManualBackup;
};
class DomainContentBackupManager : public GenericThread {
Q_OBJECT
public:
@ -52,19 +44,21 @@ public:
bool debugTimestampNow = false);
void addBackupHandler(BackupHandlerPointer handler);
std::vector<BackupItemInfo> getAllBackups();
void aboutToFinish(); /// call this to inform the persist thread that the owner is about to finish to support final persist
void replaceData(QByteArray data);
public slots:
void getAllBackupInformation(MiniPromise::Promise promise);
void createManualBackup(MiniPromise::Promise promise, const QString& name);
void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName);
void deleteBackup(MiniPromise::Promise promise, const QString& backupName);
void consolidateBackup(MiniPromise::Promise promise, QString fileName);
signals:
void loadCompleted();
void recoveryCompleted();
protected:
/// Implements generic processing behavior for this thread.
@ -73,7 +67,6 @@ protected:
void load();
void backup();
void consolidate(QString fileName);
void removeOldBackupVersions(const BackupRule& rule);
bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime);
int64_t getMostRecentBackupTimeInSecs(const QString& format);
@ -86,6 +79,9 @@ private:
std::vector<BackupHandlerPointer> _backupHandlers;
int _persistInterval { 0 };
std::atomic<bool> _isRecovering { false };
QString _recoveryFilename { };
int64_t _lastCheck { 0 };
std::vector<BackupRule> _backupRules;
};

50
domain-server/src/DomainServer.cpp
View file

@ -307,10 +307,7 @@ DomainServer::DomainServer(int argc, char* argv[]) :
_contentManager->initialize(true);
qDebug() << "Existing backups:";
for (auto& backup : _contentManager->getAllBackups()) {
qDebug() << " Backup: " << backup.name << backup.createdAt;
}
connect(_contentManager.get(), &DomainContentBackupManager::recoveryCompleted, this, &DomainServer::restart);
}
void DomainServer::parseCommandLine() {
@ -1785,6 +1782,8 @@ QString DomainServer::getEntitiesReplacementFilePath() {
void DomainServer::processOctreeDataRequestMessage(QSharedPointer<ReceivedMessage> message) {
qDebug() << "Got request for octree data from " << message->getSenderSockAddr();
maybeHandleReplacementEntityFile();
bool remoteHasExistingData { false };
QUuid id;
int version;
@ -2129,24 +2128,35 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
return true;
} else if (url.path() == URI_API_BACKUPS) {
QJsonObject rootJSON;
QJsonArray backupsJSON;
auto deferred = makePromise("getAllBackupInformation");
deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
QJsonDocument docJSON(QJsonObject::fromVariantMap(result));
auto backups = _contentManager->getAllBackups();
connection->respond(HTTPConnection::StatusCode200, docJSON.toJson(), JSON_MIME_TYPE.toUtf8());
});
_contentManager->getAllBackupInformation(deferred);
return true;
} else if (url.path().startsWith(URI_API_BACKUPS_ID)) {
auto id = url.path().mid(QString(URI_API_BACKUPS_ID).length());
auto deferred = makePromise("consolidateBackup");
deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
QJsonObject rootJSON;
auto success = result["success"].toBool();
if (success) {
auto path = result["backupFilePath"].toString();
auto file { std::unique_ptr<QFile>(new QFile(path)) };
if (file->open(QIODevice::ReadOnly)) {
connection->respond(HTTPConnection::StatusCode200, std::move(file));
} else {
qCritical(domain_server) << "Unable to load consolidated backup at:" << path << result;
connection->respond(HTTPConnection::StatusCode500, "Error opening backup");
}
} else {
connection->respond(HTTPConnection::StatusCode400);
}
});
_contentManager->consolidateBackup(deferred, id);
for (const auto& backup : backups) {
QJsonObject obj;
obj["id"] = backup.id;
obj["name"] = backup.name;
obj["createdAtMillis"] = backup.createdAt.toMSecsSinceEpoch();
obj["isManualBackup"] = backup.isManualBackup;
backupsJSON.push_back(obj);
}
rootJSON["backups"] = backupsJSON;
QJsonDocument docJSON(rootJSON);
connection->respond(HTTPConnection::StatusCode200, docJSON.toJson(), JSON_MIME_TYPE.toUtf8());
return true;
} else if (url.path() == URI_RESTART) {
connection->respond(HTTPConnection::StatusCode200);

15
domain-server/src/EntitiesBackupHandler.h
View file

@ -12,27 +12,28 @@
#ifndef hifi_EntitiesBackupHandler_h
#define hifi_EntitiesBackupHandler_h
#include <QString>
#include "BackupHandler.h"
class EntitiesBackupHandler : public BackupHandlerInterface {
public:
EntitiesBackupHandler(QString entitiesFilePath, QString entitiesReplacementFilePath);
void loadBackup(QuaZip& zip) {}
std::pair<bool, float> isAvailable(QString filePath) override { return { true, 1.0f }; }
std::pair<bool, float> getRecoveryStatus() override { return { false, 1.0f }; }
void loadBackup(QuaZip& zip) override {}
// Create a skeleton backup
void createBackup(QuaZip& zip);
void createBackup(QuaZip& zip) override;
// Recover from a full backup
void recoverBackup(QuaZip& zip);
void recoverBackup(QuaZip& zip) override;
// Delete a skeleton backup
void deleteBackup(QuaZip& zip) {}
void deleteBackup(QuaZip& zip) override {}
// Create a full backup
void consolidateBackup(QuaZip& zip) {}
void consolidateBackup(QuaZip& zip) override {}
private:
QString _entitiesFilePath;