Integrate new backup systems

This commit is contained in:
Atlante45 2018-02-06 15:37:48 -08:00
parent 0bbbff95cd
commit 11b7fb89a9
7 changed files with 251 additions and 73 deletions

View file

@ -0,0 +1,110 @@
//
// BackupHandler.h
// assignment-client
//
// Created by Clement Brisset on 2/5/18.
// Copyright 2018 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_BackupHandler_h
#define hifi_BackupHandler_h
#include <memory>
#include <QDebug>
#include <quazip5/quazip.h>
class BackupHandler {
public:
template <typename T>
BackupHandler(T x) : _self(std::make_shared<Model<T>>(std::move(x))) {}
void loadBackup(const QuaZip& zip) {
_self->loadBackup(zip);
}
void createBackup(QuaZip& zip) const {
_self->createBackup(zip);
}
void recoverBackup(const QuaZip& zip) const {
_self->recoverBackup(zip);
}
void deleteBackup(const QuaZip& zip) {
_self->deleteBackup(zip);
}
void consolidateBackup(QuaZip& zip) const {
_self->consolidateBackup(zip);
}
private:
struct Concept {
virtual ~Concept() = default;
virtual void loadBackup(const QuaZip& zip) = 0;
virtual void createBackup(QuaZip& zip) const = 0;
virtual void recoverBackup(const QuaZip& zip) const = 0;
virtual void deleteBackup(const QuaZip& zip) = 0;
virtual void consolidateBackup(QuaZip& zip) const = 0;
};
template <typename T>
struct Model : Concept {
Model(T x) : data(std::move(x)) {}
void loadBackup(const QuaZip& zip) {
data.loadBackup(zip);
}
void createBackup(QuaZip& zip) const {
data.createBackup(zip);
}
void recoverBackup(const QuaZip& zip) const {
data.recoverBackup(zip);
}
void deleteBackup(const QuaZip& zip) {
data.deleteBackup(zip);
}
void consolidateBackup(QuaZip& zip) const {
data.consolidateBackup(zip);
}
T data;
};
std::shared_ptr<Concept> _self;
};
#include <quazip5/quazipfile.h>
class EntitiesBackupHandler {
public:
EntitiesBackupHandler(QString entitiesFilePath) : _entitiesFilePath(entitiesFilePath) {}
void loadBackup(const QuaZip& zip) {}
void createBackup(QuaZip& zip) const {
qDebug() << "Creating a backup from handler";
QFile entitiesFile { _entitiesFilePath };
if (entitiesFile.open(QIODevice::ReadOnly)) {
QuaZipFile zipFile { &zip };
zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo("models.json.gz", _entitiesFilePath));
zipFile.write(entitiesFile.readAll());
zipFile.close();
if (zipFile.getZipError() != UNZ_OK) {
qDebug() << "testCreate(): outFile.close(): " << zipFile.getZipError();
}
}
}
void recoverBackup(const QuaZip& zip) const {}
void deleteBackup(const QuaZip& zip) {}
void consolidateBackup(QuaZip& zip) const {}
private:
QString _entitiesFilePath;
};
#endif /* hifi_BackupHandler_h */

View file

@ -40,6 +40,39 @@ BackupSupervisor::BackupSupervisor() {
}
loadAllBackups();
static constexpr int MAPPINGS_REFRESH_INTERVAL = 30 * 1000;
_mappingsRefreshTimer.setInterval(MAPPINGS_REFRESH_INTERVAL);
_mappingsRefreshTimer.setTimerType(Qt::CoarseTimer);
_mappingsRefreshTimer.setSingleShot(false);
QObject::connect(&_mappingsRefreshTimer, &QTimer::timeout, this, &BackupSupervisor::refreshMappings);
_mappingsRefreshTimer.start();
}
void BackupSupervisor::refreshMappings() {
auto assetClient = DependencyManager::get<AssetClient>();
auto request = assetClient->createGetAllMappingsRequest();
QObject::connect(request, &GetAllMappingsRequest::finished, this, [this](GetAllMappingsRequest* request) {
if (request->getError() == MappingRequest::NoError) {
const auto& mappings = request->getMappings();
qDebug() << "Refreshed" << mappings.size() << "asset mappings!";
_currentMappings.clear();
for (const auto& mapping : mappings) {
_currentMappings.insert({ mapping.first, mapping.second.hash });
}
_lastMappingsRefresh = usecTimestampNow();
} else {
qCritical() << "Could not refresh asset server mappings.";
qCritical() << " Error:" << request->getErrorString();
}
request->deleteLater();
});
request->start();
}
void BackupSupervisor::loadAllBackups() {
@ -138,35 +171,26 @@ void BackupSupervisor::backupAssetServer() {
return;
}
auto assetClient = DependencyManager::get<AssetClient>();
auto request = assetClient->createGetAllMappingsRequest();
if (_lastMappingsRefresh == 0) {
qWarning() << "Current mappings not yet loaded, ";
return;
}
connect(request, &GetAllMappingsRequest::finished, this, [this](GetAllMappingsRequest* request) {
qDebug() << "Got" << request->getMappings().size() << "mappings!";
if (request->getError() != MappingRequest::NoError) {
qCritical() << "Could not complete backup.";
qCritical() << " Error:" << request->getErrorString();
finishBackup();
request->deleteLater();
return;
}
if (!writeBackupFile(request->getMappings())) {
finishBackup();
request->deleteLater();
return;
}
assert(!_backups.empty());
const auto& mappings = _backups.back().mappings;
backupMissingFiles(mappings);
request->deleteLater();
});
static constexpr quint64 MAX_REFRESH_TIME = 15 * 60 * 1000 * 1000;
if (usecTimestampNow() - _lastMappingsRefresh > MAX_REFRESH_TIME) {
qWarning() << "Backing up asset mappings that appear old.";
}
startBackup();
request->start();
if (!writeBackupFile(_currentMappings)) {
finishBackup();
return;
}
assert(!_backups.empty());
const auto& mappings = _backups.back().mappings;
backupMissingFiles(mappings);
}
void BackupSupervisor::backupMissingFiles(const AssetUtils::Mappings& mappings) {
@ -193,7 +217,7 @@ void BackupSupervisor::backupNextMissingFile() {
auto assetClient = DependencyManager::get<AssetClient>();
auto assetRequest = assetClient->createRequest(hash);
connect(assetRequest, &AssetRequest::finished, this, [this](AssetRequest* request) {
QObject::connect(assetRequest, &AssetRequest::finished, this, [this](AssetRequest* request) {
if (request->getError() == AssetRequest::NoError) {
qDebug() << "Got" << request->getHash();
@ -213,7 +237,7 @@ void BackupSupervisor::backupNextMissingFile() {
assetRequest->start();
}
bool BackupSupervisor::writeBackupFile(const AssetUtils::AssetMappings& mappings) {
bool BackupSupervisor::writeBackupFile(const AssetUtils::Mappings& mappings) {
auto filename = MAPPINGS_PREFIX + QDateTime::currentDateTimeUtc().toString(Qt::ISODate) + ".json";
QFile file { PathUtils::getAppDataPath() + BACKUPS_DIR + filename };
if (!file.open(QFile::WriteOnly)) {
@ -224,9 +248,9 @@ bool BackupSupervisor::writeBackupFile(const AssetUtils::AssetMappings& mappings
AssetServerBackup backup;
QJsonObject jsonObject;
for (auto& mapping : mappings) {
backup.mappings[mapping.first] = mapping.second.hash;
_assetsInBackups.insert(mapping.second.hash);
jsonObject.insert(mapping.first, mapping.second.hash);
backup.mappings[mapping.first] = mapping.second;
_assetsInBackups.insert(mapping.second);
jsonObject.insert(mapping.first, mapping.second);
}
QJsonDocument document(jsonObject);
@ -262,7 +286,7 @@ void BackupSupervisor::restoreAssetServer(int backupIndex) {
auto assetClient = DependencyManager::get<AssetClient>();
auto request = assetClient->createGetAllMappingsRequest();
connect(request, &GetAllMappingsRequest::finished, this, [this, backupIndex](GetAllMappingsRequest* request) {
QObject::connect(request, &GetAllMappingsRequest::finished, this, [this, backupIndex](GetAllMappingsRequest* request) {
if (request->getError() == MappingRequest::NoError) {
const auto& newMappings = _backups.at(backupIndex).mappings;
computeServerStateDifference(request->getMappings(), newMappings);
@ -332,7 +356,7 @@ void BackupSupervisor::restoreNextAsset() {
auto assetClient = DependencyManager::get<AssetClient>();
auto request = assetClient->createUpload(assetFilename);
connect(request, &AssetUpload::finished, this, [this](AssetUpload* request) {
QObject::connect(request, &AssetUpload::finished, this, [this](AssetUpload* request) {
if (request->getError() != AssetUpload::NoError) {
qCritical() << "Failed to restore asset:" << request->getFilename();
qCritical() << " Error:" << request->getErrorString();
@ -350,7 +374,7 @@ void BackupSupervisor::updateMappings() {
auto assetClient = DependencyManager::get<AssetClient>();
for (const auto& mapping : _mappingsLeftToSet) {
auto request = assetClient->createSetMappingRequest(mapping.first, mapping.second);
connect(request, &SetMappingRequest::finished, this, [this](SetMappingRequest* request) {
QObject::connect(request, &SetMappingRequest::finished, this, [this](SetMappingRequest* request) {
if (request->getError() != MappingRequest::NoError) {
qCritical() << "Failed to set mapping:" << request->getPath();
qCritical() << " Error:" << request->getErrorString();
@ -369,7 +393,7 @@ void BackupSupervisor::updateMappings() {
_mappingsLeftToSet.clear();
auto request = assetClient->createDeleteMappingsRequest(_mappingsLeftToDelete);
connect(request, &DeleteMappingsRequest::finished, this, [this](DeleteMappingsRequest* request) {
QObject::connect(request, &DeleteMappingsRequest::finished, this, [this](DeleteMappingsRequest* request) {
if (request->getError() != MappingRequest::NoError) {
qCritical() << "Failed to delete mappings";
qCritical() << " Error:" << request->getErrorString();

View file

@ -16,11 +16,17 @@
#include <map>
#include <QObject>
#include <QTimer>
#include <QJsonDocument>
#include <QJsonObject>
#include <AssetUtils.h>
#include <SharedUtil.h>
#include <ReceivedMessage.h>
class QuaZip;
struct AssetServerBackup {
std::string filePath;
AssetUtils::Mappings mappings;
@ -42,7 +48,12 @@ public:
bool backupInProgress() const { return _backupInProgress; }
bool restoreInProgress() const { return _restoreInProgress; }
AssetUtils::Mappings getCurrentMappings() const { return _currentMappings; }
quint64 getLastRefreshTimestamp() const { return _lastMappingsRefresh; }
private:
void refreshMappings();
void loadAllBackups();
bool loadBackup(const QString& backupFile);
@ -50,7 +61,7 @@ private:
void finishBackup() { _backupInProgress = false; }
void backupMissingFiles(const AssetUtils::Mappings& mappings);
void backupNextMissingFile();
bool writeBackupFile(const AssetUtils::AssetMappings& mappings);
bool writeBackupFile(const AssetUtils::Mappings& mappings);
bool writeAssetFile(const AssetUtils::AssetHash& hash, const QByteArray& data);
void startRestore() { _restoreInProgress = true; }
@ -64,6 +75,10 @@ private:
QString _backupsDirectory;
QString _assetsDirectory;
quint64 _lastMappingsRefresh { 0 };
AssetUtils::Mappings _currentMappings;
// Internal storage for backups on disk
bool _allBackupsLoadedSuccessfully { false };
std::vector<AssetServerBackup> _backups;
@ -80,6 +95,55 @@ private:
std::vector<std::pair<AssetUtils::AssetPath, AssetUtils::AssetHash>> _mappingsLeftToSet;
AssetUtils::AssetPathList _mappingsLeftToDelete;
int _mappingRequestsInFlight { 0 };
QTimer _mappingsRefreshTimer;
};
#include <quazip5/quazipfile.h>
class AssetsBackupHandler {
public:
AssetsBackupHandler(BackupSupervisor* backupSupervisor) : _backupSupervisor(backupSupervisor) {}
void loadBackup(const QuaZip& zip) {}
void createBackup(QuaZip& zip) const {
quint64 lastRefreshTimestamp = _backupSupervisor->getLastRefreshTimestamp();
AssetUtils::Mappings mappings = _backupSupervisor->getCurrentMappings();
if (lastRefreshTimestamp == 0) {
qWarning() << "Current mappings not yet loaded, ";
return;
}
static constexpr quint64 MAX_REFRESH_TIME = 15 * 60 * 1000 * 1000;
if (usecTimestampNow() - lastRefreshTimestamp > MAX_REFRESH_TIME) {
qWarning() << "Backing up asset mappings that appear old.";
}
QJsonObject jsonObject;
for (const auto& mapping : mappings) {
jsonObject.insert(mapping.first, mapping.second);
}
QJsonDocument document(jsonObject);
QuaZipFile zipFile { &zip };
if (!zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo("mappings.json"))) {
qDebug() << "testCreate(): outFile.open()";
}
zipFile.write(document.toJson());
zipFile.close();
if (zipFile.getZipError() != UNZ_OK) {
qDebug() << "testCreate(): outFile.close(): " << zipFile.getZipError();
}
}
void recoverBackup(const QuaZip& zip) const {}
void deleteBackup(const QuaZip& zip) {}
void consolidateBackup(QuaZip& zip) const {}
private:
BackupSupervisor* _backupSupervisor;
};
#endif /* hifi_BackupSupervisor_h */

View file

@ -37,8 +37,8 @@ const int DomainContentBackupManager::DEFAULT_PERSIST_INTERVAL = 1000 * 30; //
const static QString DATETIME_FORMAT { "yyyy-MM-dd_HH-mm-ss" };
const static QString DATETIME_FORMAT_RE("\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}");
void DomainContentBackupManager::addCreateBackupHandler(CreateBackupHandler handler) {
_backupHandlers.push_back(handler);
void DomainContentBackupManager::addBackupHandler(BackupHandler handler) {
_backupHandlers.push_back(std::move(handler));
}
DomainContentBackupManager::DomainContentBackupManager(const QString& backupDirectory,
@ -48,7 +48,6 @@ DomainContentBackupManager::DomainContentBackupManager(const QString& backupDire
: _backupDirectory(backupDirectory),
_persistInterval(persistInterval),
_initialLoadComplete(false),
_loadTimeUSecs(0),
_lastCheck(0),
_debugTimestampNow(debugTimestampNow),
_lastTimeDebug(0) {
@ -268,14 +267,14 @@ void DomainContentBackupManager::backup() {
auto timestamp = QDateTime::currentDateTime().toString(DATETIME_FORMAT);
auto fileName = "backup-" + rule.extensionFormat + timestamp + ".zip";
auto zip = new QuaZip(_backupDirectory + "/" + fileName);
zip->open(QuaZip::mdAdd);
QuaZip zip(_backupDirectory + "/" + fileName);
zip.open(QuaZip::mdAdd);
for (auto& handler : _backupHandlers) {
handler(zip);
for (const auto& handler : _backupHandlers) {
handler.createBackup(zip);
}
zip->close();
zip.close();
qDebug() << "Created backup: " << fileName;

View file

@ -15,17 +15,11 @@
#define hifi_DomainContentBackupManager_h
#include <QString>
#include <GenericThread.h>
#include <QVector>
#include <quazip5/quazip.h>
#include <quazip5/quazipfile.h>
#include <GenericThread.h>
#include <functional>
using BackupResult = std::vector<QString>;
using CreateBackupHandler = std::function<void(QuaZip* quazip)>;
using RecoverBackupHandler = std::function<void()>;
#include "BackupHandler.h"
class DomainContentBackupManager : public GenericThread {
Q_OBJECT
@ -46,9 +40,8 @@ public:
int persistInterval = DEFAULT_PERSIST_INTERVAL,
bool debugTimestampNow = false);
void addCreateBackupHandler(CreateBackupHandler handler);
void addBackupHandler(BackupHandler handler);
bool isInitialLoadComplete() const { return _initialLoadComplete; }
int64_t getLoadElapsedTime() const { return _loadTimeUSecs; }
void aboutToFinish(); /// call this to inform the persist thread that the owner is about to finish to support final persist
@ -70,12 +63,10 @@ protected:
private:
QString _backupDirectory;
std::vector<CreateBackupHandler> _backupHandlers;
std::vector<BackupHandler> _backupHandlers;
int _persistInterval;
bool _initialLoadComplete;
int64_t _loadTimeUSecs;
time_t _lastPersistTime;
int64_t _lastCheck;
bool _wantBackup{ true };

View file

@ -45,6 +45,7 @@
#include <Trace.h>
#include <StatTracker.h>
#include "BackupSupervisor.h"
#include "DomainServerNodeData.h"
#include "NodeConnectionData.h"
@ -293,23 +294,10 @@ DomainServer::DomainServer(int argc, char* argv[]) :
qCDebug(domain_server) << "Created entities data directory";
}
maybeHandleReplacementEntityFile();
auto entitiesFilePath = getEntitiesFilePath();
_contentManager.reset(new DomainContentBackupManager(getContentBackupDir(), _settingsManager.responseObjectForType("6")["entity_server_settings"].toObject()));
_contentManager->addCreateBackupHandler([entitiesFilePath](QuaZip* zip) {
qDebug() << "Creating a backup from handler";
QFile entitiesFile { entitiesFilePath };
if (entitiesFile.open(QIODevice::ReadOnly)) {
QuaZipFile zipFile { zip };
zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo("models.json.gz", entitiesFilePath));
zipFile.write(entitiesFile.readAll());
zipFile.close();
if (zipFile.getZipError() != UNZ_OK) {
qDebug() << "Failed to write entities file to backup:" << zipFile.getZipError();
}
}
});
_contentManager->addBackupHandler(EntitiesBackupHandler(getEntitiesFilePath()));
_contentManager->addBackupHandler(AssetsBackupHandler(&_backupSupervisor));
_contentManager->initialize(true);
}

View file

@ -275,6 +275,8 @@ private:
QHash<QUuid, QPointer<HTTPSConnection>> _pendingOAuthConnections;
QThread _assetClientThread;
BackupSupervisor _backupSupervisor;
};