From e63b692d80f584eac5d70751b9e01d2a1a8b8cf9 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 7 Feb 2018 16:53:29 -0800 Subject: [PATCH] Add BackupHandler for entity file backups --- domain-server/src/BackupHandler.h | 34 ++++++++++++-- domain-server/src/BackupSupervisor.h | 47 +++++++++++++++++++ .../src/DomainContentBackupManager.cpp | 31 ++++++++++-- .../src/DomainContentBackupManager.h | 5 ++ domain-server/src/DomainServer.cpp | 2 + libraries/entities/src/EntityTree.cpp | 1 + 6 files changed, 113 insertions(+), 7 deletions(-) diff --git a/domain-server/src/BackupHandler.h b/domain-server/src/BackupHandler.h index 4643d183b2..b790591bea 100644 --- a/domain-server/src/BackupHandler.h +++ b/domain-server/src/BackupHandler.h @@ -83,7 +83,10 @@ public: void loadBackup(QuaZip& zip) {} - void createBackup(QuaZip& zip) const { + // Create a skeleton backup + void createBackup(QuaZip& zip) { + qDebug() << "Creating a backup from handler"; + QFile entitiesFile { _entitiesFilePath }; if (entitiesFile.open(QIODevice::ReadOnly)) { @@ -97,9 +100,32 @@ public: } } - void recoverBackup(QuaZip& zip) const {} - void deleteBackup(QuaZip& zip) {} - void consolidateBackup(QuaZip& zip) const {} + // Recover from a full backup + void recoverBackup(QuaZip& zip) { + if (!zip.setCurrentFile("models.json.gz")) { + qWarning() << "Failed to find models.json.gz while recovering backup"; + return; + } + QuaZipFile zipFile { &zip }; + zipFile.open(QIODevice::ReadOnly); + auto data = zipFile.readAll(); + + QFile entitiesFile { _entitiesFilePath }; + + if (entitiesFile.open(QIODevice::WriteOnly)) { + entitiesFile.write(data); + } + + zipFile.close(); + } + + // Delete a skeleton backup + void deleteBackup(QuaZip& zip) { + } + + // Create a full backup + void consolidateBackup(QuaZip& zip) { + } private: QString _entitiesFilePath; diff --git a/domain-server/src/BackupSupervisor.h b/domain-server/src/BackupSupervisor.h index 9fedcca19b..1023622971 100644 --- a/domain-server/src/BackupSupervisor.h +++ b/domain-server/src/BackupSupervisor.h @@ -91,4 +91,51 @@ private: int _mappingRequestsInFlight { 0 }; }; + +#include +class AssetsBackupHandler { +public: + AssetsBackupHandler(BackupSupervisor* backupSupervisor) : _backupSupervisor(backupSupervisor) {} + + void loadBackup(QuaZip& zip) {} + + void createBackup(QuaZip& zip) { + quint64 lastRefreshTimestamp = _backupSupervisor->getLastRefreshTimestamp(); + AssetUtils::Mappings mappings = _backupSupervisor->getCurrentMappings(); + + if (lastRefreshTimestamp == 0) { + qWarning() << "Current mappings not yet loaded, "; + return; + } + + static constexpr quint64 MAX_REFRESH_TIME = 15 * 60 * 1000 * 1000; + if (usecTimestampNow() - lastRefreshTimestamp > MAX_REFRESH_TIME) { + qWarning() << "Backing up asset mappings that appear old."; + } + + QJsonObject jsonObject; + for (const auto& mapping : mappings) { + jsonObject.insert(mapping.first, mapping.second); + } + QJsonDocument document(jsonObject); + + QuaZipFile zipFile { &zip }; + if (!zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo("mappings.json"))) { + qDebug() << "testCreate(): outFile.open()"; + } + zipFile.write(document.toJson()); + zipFile.close(); + if (zipFile.getZipError() != UNZ_OK) { + qDebug() << "testCreate(): outFile.close(): " << zipFile.getZipError(); + } + } + + void recoverBackup(QuaZip& zip) {} + void deleteBackup(QuaZip& zip) {} + void consolidateBackup(QuaZip& zip) {} + +private: + BackupSupervisor* _backupSupervisor; +}; + #endif /* hifi_BackupSupervisor_h */ diff --git a/domain-server/src/DomainContentBackupManager.cpp b/domain-server/src/DomainContentBackupManager.cpp index ed5d99f927..b0a80531f8 100644 --- a/domain-server/src/DomainContentBackupManager.cpp +++ b/domain-server/src/DomainContentBackupManager.cpp @@ -216,10 +216,35 @@ bool DomainContentBackupManager::getMostRecentBackup(const QString& format, return bestBackupFound; } +bool DomainContentBackupManager::recoverFromBackup(const QString& backupName) { + qDebug() << "Recoving from" << backupName; + + QDir backupDir { _backupDirectory }; + QFile backupFile { backupDir.filePath(backupName) }; + if (backupFile.open(QIODevice::ReadOnly)) { + QuaZip zip { &backupFile }; + if (!zip.open(QuaZip::Mode::mdUnzip)) { + qWarning() << "Failed to unzip file: " << backupName; + backupFile.close(); + return false; + } + + for (auto& handler : _backupHandlers) { + handler.recoverBackup(zip); + } + + backupFile.close(); + } + + qDebug() << "Successfully recovered from " << backupName; + + return true; +} + void DomainContentBackupManager::removeOldBackupVersions(const BackupRule& rule) { QDir backupDir { _backupDirectory }; if (backupDir.exists() && rule.maxBackupVersions > 0) { - qCDebug(domain_server) << "Rolling old backup versions for rule" << rule.name << "..."; + qCDebug(domain_server) << "Rolling old backup versions for rule" << rule.name; auto matchingFiles = backupDir.entryInfoList({ rule.extensionFormat + "*.zip" }, QDir::Files | QDir::NoSymLinks, QDir::Name); @@ -235,11 +260,11 @@ void DomainContentBackupManager::removeOldBackupVersions(const BackupRule& rule) } } - qCDebug(domain_server) << "Done rolling old backup versions..."; + qCDebug(domain_server) << "Done removing old backup versions"; } else { qCDebug(domain_server) << "Rolling backups for rule" << rule.name << "." << " Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]." - << " No need to roll backups..."; + << " No need to roll backups"; } } diff --git a/domain-server/src/DomainContentBackupManager.h b/domain-server/src/DomainContentBackupManager.h index 69163b4ead..d0dd9cf2c6 100644 --- a/domain-server/src/DomainContentBackupManager.h +++ b/domain-server/src/DomainContentBackupManager.h @@ -46,6 +46,11 @@ public: void replaceData(QByteArray data); + bool recoverFromBackup(const QString& backupName); + +signals: + void loadCompleted(); + protected: /// Implements generic processing behavior for this thread. virtual void setup() override; diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 8ccba3d942..fe6a303e08 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -299,6 +299,8 @@ DomainServer::DomainServer(int argc, char* argv[]) : _contentManager->addBackupHandler(new EntitiesBackupHandler(getEntitiesFilePath())); _contentManager->addBackupHandler(new BackupSupervisor(getContentBackupDir())); _contentManager->initialize(true); + + _contentManager->recoverFromBackup("backup-daily_rolling-2018-02-06_15-13-50.zip"); } void DomainServer::parseCommandLine() { diff --git a/libraries/entities/src/EntityTree.cpp b/libraries/entities/src/EntityTree.cpp index f632bcf140..fc3e793b45 100644 --- a/libraries/entities/src/EntityTree.cpp +++ b/libraries/entities/src/EntityTree.cpp @@ -2246,6 +2246,7 @@ bool EntityTree::writeToMap(QVariantMap& entityDescription, OctreeElementPointer } entityDescription["DataVersion"] = _persistDataVersion; entityDescription["Id"] = _persistID; + qDebug() << "Writing to map: " << _persistDataVersion << _persistID; QScriptEngine scriptEngine; RecurseOctreeToMapOperator theOperator(entityDescription, element, &scriptEngine, skipDefaultValues, skipThoseWithBadParents, _myAvatar);