From d6e281408144f0db5bb102ca4a0b99dd460cd63f Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 15 Feb 2018 11:25:07 -0800 Subject: [PATCH] Write assets to disk when recovering full backup --- domain-server/src/AssetsBackupHandler.cpp | 152 ++++++++++++++-------- domain-server/src/AssetsBackupHandler.h | 1 + 2 files changed, 99 insertions(+), 54 deletions(-) diff --git a/domain-server/src/AssetsBackupHandler.cpp b/domain-server/src/AssetsBackupHandler.cpp index 3dc4851762..a9f56a0c5b 100644 --- a/domain-server/src/AssetsBackupHandler.cpp +++ b/domain-server/src/AssetsBackupHandler.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -23,13 +24,14 @@ #include #include -const QString ASSETS_DIR = "/assets/"; -const QString MAPPINGS_FILE = "mappings.json"; +static const QString ASSETS_DIR = "/assets/"; +static const QString MAPPINGS_FILE = "mappings.json"; +static const QString ZIP_ASSETS_FOLDER = "files"; using namespace std; -Q_DECLARE_LOGGING_CATEGORY(backup_supervisor) -Q_LOGGING_CATEGORY(backup_supervisor, "hifi.backup-supervisor"); +Q_DECLARE_LOGGING_CATEGORY(asset_backup) +Q_LOGGING_CATEGORY(asset_backup, "hifi.asset-backup"); AssetsBackupHandler::AssetsBackupHandler(const QString& backupDirectory) : _assetsDirectory(backupDirectory + ASSETS_DIR) @@ -39,6 +41,10 @@ AssetsBackupHandler::AssetsBackupHandler(const QString& backupDirectory) : refreshAssetsOnDisk(); + setupRefreshTimer(); +} + +void AssetsBackupHandler::setupRefreshTimer() { _mappingsRefreshTimer.setTimerType(Qt::CoarseTimer); _mappingsRefreshTimer.setSingleShot(true); QObject::connect(&_mappingsRefreshTimer, &QTimer::timeout, this, &AssetsBackupHandler::refreshMappings); @@ -50,9 +56,14 @@ AssetsBackupHandler::AssetsBackupHandler(const QString& backupDirectory) : _mappingsRefreshTimer.start(0); } }); + QObject::connect(nodeList.data(), &LimitedNodeList::nodeKilled, this, [this](SharedNodePointer node) { + if (node->getType() == NodeType::AssetServer) { + // run immediately for the first time. + _mappingsRefreshTimer.stop(); + } + }); } - void AssetsBackupHandler::refreshAssetsOnDisk() { QDir assetsDir { _assetsDirectory }; auto assetNames = assetsDir.entryList(QDir::Files); @@ -79,7 +90,7 @@ void AssetsBackupHandler::checkForMissingAssets() { begin(_assetsOnDisk), end(_assetsOnDisk), back_inserter(missingAssets)); if (missingAssets.size() > 0) { - qCWarning(backup_supervisor) << "Found" << missingAssets.size() << "assets missing."; + qCWarning(asset_backup) << "Found" << missingAssets.size() << "backup assets missing from disk."; } } @@ -90,24 +101,26 @@ void AssetsBackupHandler::checkForAssetsToDelete() { back_inserter(deprecatedAssets)); if (deprecatedAssets.size() > 0) { - qCDebug(backup_supervisor) << "Found" << deprecatedAssets.size() << "assets to delete."; + qCDebug(asset_backup) << "Found" << deprecatedAssets.size() << "backup assets to delete from disk."; if (_allBackupsLoadedSuccessfully) { for (const auto& hash : deprecatedAssets) { QFile::remove(_assetsDirectory + hash); } } else { - qCWarning(backup_supervisor) << "Some backups did not load properly, aborting deleting for safety."; + qCWarning(asset_backup) << "Some backups did not load properly, aborting delete operation for safety."; } } } void AssetsBackupHandler::loadBackup(QuaZip& zip) { + Q_ASSERT(QThread::currentThread() == thread()); + _backups.push_back({ zip.getZipName(), {}, false }); auto& backup = _backups.back(); if (!zip.setCurrentFile(MAPPINGS_FILE)) { - qCCritical(backup_supervisor) << "Failed to find" << MAPPINGS_FILE << "while recovering backup"; - qCCritical(backup_supervisor) << " Error:" << zip.getZipError(); + qCCritical(asset_backup) << "Failed to find" << MAPPINGS_FILE << "while loading backup"; + qCCritical(asset_backup) << " Error:" << zip.getZipError(); backup.corruptedBackup = true; _allBackupsLoadedSuccessfully = false; return; @@ -115,8 +128,8 @@ void AssetsBackupHandler::loadBackup(QuaZip& zip) { QuaZipFile zipFile { &zip }; if (!zipFile.open(QFile::ReadOnly)) { - qCCritical(backup_supervisor) << "Could not open backup file:" << zip.getZipName(); - qCCritical(backup_supervisor) << " Error:" << zip.getZipError(); + qCCritical(asset_backup) << "Could not unzip backup file for load:" << zip.getZipName(); + qCCritical(asset_backup) << " Error:" << zip.getZipError(); backup.corruptedBackup = true; _allBackupsLoadedSuccessfully = false; return; @@ -125,8 +138,8 @@ void AssetsBackupHandler::loadBackup(QuaZip& zip) { QJsonParseError error; auto document = QJsonDocument::fromJson(zipFile.readAll(), &error); if (document.isNull() || !document.isObject()) { - qCCritical(backup_supervisor) << "Could not parse backup file to JSON object:" << zip.getZipName(); - qCCritical(backup_supervisor) << " Error:" << error.errorString(); + qCCritical(asset_backup) << "Could not parse backup file to JSON object for load:" << zip.getZipName(); + qCCritical(asset_backup) << " Error:" << error.errorString(); backup.corruptedBackup = true; _allBackupsLoadedSuccessfully = false; return; @@ -138,33 +151,37 @@ void AssetsBackupHandler::loadBackup(QuaZip& zip) { const auto& assetHash = it.value().toString(); if (!AssetUtils::isValidHash(assetHash)) { - qCCritical(backup_supervisor) << "Corrupted mapping in backup file" << zip.getZipName() << ":" << it.key(); + qCCritical(asset_backup) << "Corrupted mapping in loading backup file" << zip.getZipName() << ":" << it.key(); backup.corruptedBackup = true; _allBackupsLoadedSuccessfully = false; - return; + continue; } backup.mappings[assetPath] = assetHash; _assetsInBackups.insert(assetHash); } + checkForMissingAssets(); + checkForAssetsToDelete(); return; } void AssetsBackupHandler::createBackup(QuaZip& zip) { + Q_ASSERT(QThread::currentThread() == thread()); + if (operationInProgress()) { - qCWarning(backup_supervisor) << "There is already an operation in progress."; + qCWarning(asset_backup) << "There is already an operation in progress."; return; } if (_lastMappingsRefresh == 0) { - qCWarning(backup_supervisor) << "Current mappings not yet loaded."; + qCWarning(asset_backup) << "Current mappings not yet loaded."; return; } static constexpr quint64 MAX_REFRESH_TIME = 15 * 60 * 1000 * 1000; if (usecTimestampNow() - _lastMappingsRefresh > MAX_REFRESH_TIME) { - qCWarning(backup_supervisor) << "Backing up asset mappings that might be stale."; + qCWarning(asset_backup) << "Backing up asset mappings that might be stale."; } AssetServerBackup backup; @@ -180,43 +197,65 @@ void AssetsBackupHandler::createBackup(QuaZip& zip) { QuaZipFile zipFile { &zip }; if (!zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo(MAPPINGS_FILE))) { - qCDebug(backup_supervisor) << "Could not open zip file:" << zipFile.getZipError(); + qCDebug(asset_backup) << "Could not open zip file:" << zipFile.getZipError(); return; } zipFile.write(document.toJson()); zipFile.close(); if (zipFile.getZipError() != UNZ_OK) { - qCDebug(backup_supervisor) << "Could not close zip file: " << zipFile.getZipError(); + qCDebug(asset_backup) << "Could not close zip file: " << zipFile.getZipError(); return; } _backups.push_back(backup); } void AssetsBackupHandler::recoverBackup(QuaZip& zip) { + Q_ASSERT(QThread::currentThread() == thread()); + if (operationInProgress()) { - qCWarning(backup_supervisor) << "There is already a backup/restore in progress."; + qCWarning(asset_backup) << "There is already a backup/restore in progress."; return; } if (_lastMappingsRefresh == 0) { - qCWarning(backup_supervisor) << "Current mappings not yet loaded."; + qCWarning(asset_backup) << "Current mappings not yet loaded."; return; } static constexpr quint64 MAX_REFRESH_TIME = 15 * 60 * 1000 * 1000; if (usecTimestampNow() - _lastMappingsRefresh > MAX_REFRESH_TIME) { - qCWarning(backup_supervisor) << "Current asset mappings that might be stale."; + qCWarning(asset_backup) << "Current asset mappings that might be stale."; } - startOperation(); - auto it = find_if(begin(_backups), end(_backups), [&](const std::vector::value_type& value) { return value.filePath == zip.getZipName(); }); if (it == end(_backups)) { - qCDebug(backup_supervisor) << "Could not find backup" << zip.getZipName() << "to restore."; - stopOperation(); - return; + qCDebug(asset_backup) << "Could not find backup" << zip.getZipName() << "to restore."; + + loadBackup(zip); + + QuaZipDir zipDir { &zip, ZIP_ASSETS_FOLDER }; + + auto assetNames = zipDir.entryList(QDir::Files); + for (const auto& asset : assetNames) { + if (AssetUtils::isValidHash(asset)) { + if (!zip.setCurrentFile(MAPPINGS_FILE)) { + qCCritical(asset_backup) << "Failed to find" << asset << "while recovering backup"; + qCCritical(asset_backup) << " Error:" << zip.getZipError(); + continue; + } + + QuaZipFile zipFile { &zip }; + if (!zipFile.open(QFile::ReadOnly)) { + qCCritical(asset_backup) << "Could not unzip asset file:" << asset; + qCCritical(asset_backup) << " Error:" << zip.getZipError(); + continue; + } + + writeAssetFile(asset, zipFile.readAll()); + } + } } const auto& newMappings = it->mappings; @@ -226,8 +265,10 @@ void AssetsBackupHandler::recoverBackup(QuaZip& zip) { } void AssetsBackupHandler::deleteBackup(QuaZip& zip) { + Q_ASSERT(QThread::currentThread() == thread()); + if (operationInProgress()) { - qCWarning(backup_supervisor) << "There is a backup/restore in progress."; + qCWarning(asset_backup) << "There is a backup/restore in progress."; return; } @@ -235,7 +276,7 @@ void AssetsBackupHandler::deleteBackup(QuaZip& zip) { return value.filePath == zip.getZipName(); }); if (it == end(_backups)) { - qCDebug(backup_supervisor) << "Could not find backup" << zip.getZipName() << "to delete."; + qCDebug(asset_backup) << "Could not find backup" << zip.getZipName() << "to delete."; return; } @@ -244,8 +285,10 @@ void AssetsBackupHandler::deleteBackup(QuaZip& zip) { } void AssetsBackupHandler::consolidateBackup(QuaZip& zip) { + Q_ASSERT(QThread::currentThread() == thread()); + if (operationInProgress()) { - qCWarning(backup_supervisor) << "There is a backup/restore in progress."; + qCWarning(asset_backup) << "There is a backup/restore in progress."; return; } QFileInfo zipInfo(zip.getZipName()); @@ -255,7 +298,7 @@ void AssetsBackupHandler::consolidateBackup(QuaZip& zip) { return info.fileName() == zipInfo.fileName(); }); if (it == end(_backups)) { - qCDebug(backup_supervisor) << "Could not find backup" << zip.getZipName() << "to consolidate."; + qCDebug(asset_backup) << "Could not find backup" << zip.getZipName() << "to consolidate."; return; } @@ -265,20 +308,19 @@ void AssetsBackupHandler::consolidateBackup(QuaZip& zip) { QDir assetsDir { _assetsDirectory }; QFile file { assetsDir.filePath(hash) }; if (!file.open(QFile::ReadOnly)) { - qCCritical(backup_supervisor) << "Could not open asset file" << file.fileName(); + qCCritical(asset_backup) << "Could not open asset file" << file.fileName(); continue; } QuaZipFile zipFile { &zip }; - static const QString ZIP_ASSETS_FOLDER = "files/"; - if (!zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo(ZIP_ASSETS_FOLDER + hash))) { - qCDebug(backup_supervisor) << "Could not open zip file:" << zipFile.getZipError(); + if (!zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo(ZIP_ASSETS_FOLDER + "/" + hash))) { + qCDebug(asset_backup) << "Could not open zip file:" << zipFile.getZipError(); continue; } zipFile.write(file.readAll()); zipFile.close(); if (zipFile.getZipError() != UNZ_OK) { - qCDebug(backup_supervisor) << "Could not close zip file: " << zipFile.getZipError(); + qCDebug(asset_backup) << "Could not close zip file: " << zipFile.getZipError(); continue; } } @@ -300,8 +342,8 @@ void AssetsBackupHandler::refreshMappings() { downloadMissingFiles(_currentMappings); } else { - qCCritical(backup_supervisor) << "Could not refresh asset server mappings."; - qCCritical(backup_supervisor) << " Error:" << request->getErrorString(); + qCCritical(asset_backup) << "Could not refresh asset server mappings."; + qCCritical(asset_backup) << " Error:" << request->getErrorString(); } request->deleteLater(); @@ -341,14 +383,14 @@ void AssetsBackupHandler::downloadNextMissingFile() { QObject::connect(assetRequest, &AssetRequest::finished, this, [this](AssetRequest* request) { if (request->getError() == AssetRequest::NoError) { - qCDebug(backup_supervisor) << "Backing up asset" << request->getHash(); + qCDebug(asset_backup) << "Backing up asset" << request->getHash(); bool success = writeAssetFile(request->getHash(), request->getData()); if (!success) { - qCCritical(backup_supervisor) << "Failed to write asset file" << request->getHash(); + qCCritical(asset_backup) << "Failed to write asset file" << request->getHash(); } } else { - qCCritical(backup_supervisor) << "Failed to backup asset" << request->getHash(); + qCCritical(asset_backup) << "Failed to backup asset" << request->getHash(); } _assetsLeftToRequest.erase(request->getHash()); @@ -364,13 +406,13 @@ bool AssetsBackupHandler::writeAssetFile(const AssetUtils::AssetHash& hash, cons QDir assetsDir { _assetsDirectory }; QFile file { assetsDir.filePath(hash) }; if (!file.open(QFile::WriteOnly)) { - qCCritical(backup_supervisor) << "Could not open backup file" << file.fileName(); + qCCritical(asset_backup) << "Could not open asset file for write:" << file.fileName(); return false; } auto bytesWritten = file.write(data); if (bytesWritten != data.size()) { - qCCritical(backup_supervisor) << "Could not write data to file" << file.fileName(); + qCCritical(asset_backup) << "Could not write data to file" << file.fileName(); file.remove(); return false; } @@ -410,9 +452,9 @@ void AssetsBackupHandler::computeServerStateDifference(const AssetUtils::Mapping } } - qCDebug(backup_supervisor) << "Mappings to set:" << _mappingsLeftToSet.size(); - qCDebug(backup_supervisor) << "Mappings to del:" << _mappingsLeftToDelete.size(); - qCDebug(backup_supervisor) << "Assets to upload:" << _assetsLeftToUpload.size(); + qCDebug(asset_backup) << "Mappings to set:" << _mappingsLeftToSet.size(); + qCDebug(asset_backup) << "Mappings to del:" << _mappingsLeftToDelete.size(); + qCDebug(asset_backup) << "Assets to upload:" << _assetsLeftToUpload.size(); } void AssetsBackupHandler::restoreAllAssets() { @@ -420,6 +462,8 @@ void AssetsBackupHandler::restoreAllAssets() { } void AssetsBackupHandler::restoreNextAsset() { + startOperation(); + if (_assetsLeftToUpload.empty()) { updateMappings(); return; @@ -435,8 +479,8 @@ void AssetsBackupHandler::restoreNextAsset() { QObject::connect(request, &AssetUpload::finished, this, [this](AssetUpload* request) { if (request->getError() != AssetUpload::NoError) { - qCCritical(backup_supervisor) << "Failed to restore asset:" << request->getFilename(); - qCCritical(backup_supervisor) << " Error:" << request->getErrorString(); + qCCritical(asset_backup) << "Failed to restore asset:" << request->getFilename(); + qCCritical(asset_backup) << " Error:" << request->getErrorString(); } restoreNextAsset(); @@ -453,8 +497,8 @@ void AssetsBackupHandler::updateMappings() { auto request = assetClient->createSetMappingRequest(mapping.first, mapping.second); QObject::connect(request, &SetMappingRequest::finished, this, [this](SetMappingRequest* request) { if (request->getError() != MappingRequest::NoError) { - qCCritical(backup_supervisor) << "Failed to set mapping:" << request->getPath(); - qCCritical(backup_supervisor) << " Error:" << request->getErrorString(); + qCCritical(asset_backup) << "Failed to set mapping:" << request->getPath(); + qCCritical(asset_backup) << " Error:" << request->getErrorString(); } if (--_mappingRequestsInFlight == 0) { @@ -472,8 +516,8 @@ void AssetsBackupHandler::updateMappings() { auto request = assetClient->createDeleteMappingsRequest(_mappingsLeftToDelete); QObject::connect(request, &DeleteMappingsRequest::finished, this, [this](DeleteMappingsRequest* request) { if (request->getError() != MappingRequest::NoError) { - qCCritical(backup_supervisor) << "Failed to delete mappings"; - qCCritical(backup_supervisor) << " Error:" << request->getErrorString(); + qCCritical(asset_backup) << "Failed to delete mappings"; + qCCritical(asset_backup) << " Error:" << request->getErrorString(); } if (--_mappingRequestsInFlight == 0) { diff --git a/domain-server/src/AssetsBackupHandler.h b/domain-server/src/AssetsBackupHandler.h index b78206b7b1..2ef454998e 100644 --- a/domain-server/src/AssetsBackupHandler.h +++ b/domain-server/src/AssetsBackupHandler.h @@ -40,6 +40,7 @@ public: bool operationInProgress() const { return _operationInProgress; } private: + void setupRefreshTimer(); void refreshMappings(); void refreshAssetsInBackups();