From 9e99c5c744ea7621b37e4c0b7e36d84906ea82f4 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 15 Feb 2018 11:49:05 -0800 Subject: [PATCH 01/13] Add restart of ES during backup recovery --- domain-server/src/DomainServer.cpp | 2 ++ domain-server/src/EntitiesBackupHandler.cpp | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 599f09ae94..2f8d8f6d03 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -1785,6 +1785,8 @@ QString DomainServer::getEntitiesReplacementFilePath() { void DomainServer::processOctreeDataRequestMessage(QSharedPointer message) { qDebug() << "Got request for octree data from " << message->getSenderSockAddr(); + maybeHandleReplacementEntityFile(); + bool remoteHasExistingData { false }; QUuid id; int version; diff --git a/domain-server/src/EntitiesBackupHandler.cpp b/domain-server/src/EntitiesBackupHandler.cpp index 6ad00d01c8..deb92ee0f6 100644 --- a/domain-server/src/EntitiesBackupHandler.cpp +++ b/domain-server/src/EntitiesBackupHandler.cpp @@ -16,6 +16,7 @@ #include #include +#include #include EntitiesBackupHandler::EntitiesBackupHandler(QString entitiesFilePath, QString entitiesReplacementFilePath) : @@ -71,5 +72,13 @@ void EntitiesBackupHandler::recoverBackup(QuaZip& zip) { if (entitiesFile.open(QIODevice::WriteOnly)) { entitiesFile.write(data.toGzippedByteArray()); + entitiesFile.close(); + + auto nodeList = DependencyManager::get(); + nodeList->eachMatchingNode([](const SharedNodePointer& otherNode) -> bool { + return otherNode->getType() == NodeType::EntityServer; + }, [nodeList](const SharedNodePointer& otherNode) { + QMetaObject::invokeMethod(nodeList.data(), "killNodeWithUUID", Q_ARG(const QUuid&, otherNode->getUUID())); + }); } } From dd0b8a0c2fd3a22a82857c06e6027c56735222be Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 15 Feb 2018 14:17:32 -0800 Subject: [PATCH 02/13] Add backup download API to DS --- .../src/DomainContentBackupManager.cpp | 78 +++++++++++++------ .../src/DomainContentBackupManager.h | 2 +- domain-server/src/DomainServer.cpp | 22 ++++++ 3 files changed, 76 insertions(+), 26 deletions(-) diff --git a/domain-server/src/DomainContentBackupManager.cpp b/domain-server/src/DomainContentBackupManager.cpp index c68ff0c6ea..f56c41dacd 100644 --- a/domain-server/src/DomainContentBackupManager.cpp +++ b/domain-server/src/DomainContentBackupManager.cpp @@ -383,32 +383,60 @@ void DomainContentBackupManager::backup() { } } -void DomainContentBackupManager::consolidate(QString fileName) { - QDir backupDir { _backupDirectory }; - if (backupDir.exists()) { - auto filePath = backupDir.absoluteFilePath(fileName); - - auto copyFilePath = QDir::tempPath() + "/" + fileName; - - auto copySuccess = QFile::copy(filePath, copyFilePath); - if (!copySuccess) { - qCritical() << "Failed to create full backup."; - return; - } - - QuaZip zip(copyFilePath); - if (!zip.open(QuaZip::mdAdd)) { - qCritical() << "Could not open backup archive:" << filePath; - qCritical() << " ERROR:" << zip.getZipError(); - return; - } - - for (auto& handler : _backupHandlers) { - handler->consolidateBackup(zip); - } - - zip.close(); +void DomainContentBackupManager::consolidateBackup(MiniPromise::Promise promise, QString fileName) { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "consolidateBackup", Q_ARG(MiniPromise::Promise, promise), + Q_ARG(const QString&, fileName)); + return; } + + QDir backupDir { _backupDirectory }; + if (!backupDir.exists()) { + qCritical() << "Backup directory does not exist, bailing consolidation of backup"; + promise->resolve({ { "success", false } }); + return; + } + + auto filePath = backupDir.absoluteFilePath(fileName); + + auto copyFilePath = QDir::tempPath() + "/" + fileName; + + { + QFile copyFile(copyFilePath); + copyFile.remove(); + copyFile.close(); + } + auto copySuccess = QFile::copy(filePath, copyFilePath); + if (!copySuccess) { + qCritical() << "Failed to create copy of backup."; + promise->resolve({ { "success", false } }); + return; + } + + QuaZip zip(copyFilePath); + if (!zip.open(QuaZip::mdAdd)) { + qCritical() << "Could not open backup archive:" << filePath; + qCritical() << " ERROR:" << zip.getZipError(); + promise->resolve({ { "success", false } }); + return; + } + + for (auto& handler : _backupHandlers) { + handler->consolidateBackup(zip); + } + + zip.close(); + + if (zip.getZipError() != UNZ_OK) { + qCritical() << "Failed to consolidate backup: " << zip.getZipError(); + promise->resolve({ { "success", false } }); + return; + } + + promise->resolve({ + { "success", true }, + { "backupFilePath", copyFilePath } + }); } void DomainContentBackupManager::createManualBackup(MiniPromise::Promise promise, const QString& name) { diff --git a/domain-server/src/DomainContentBackupManager.h b/domain-server/src/DomainContentBackupManager.h index 1e1b2360a8..9ec7eb9950 100644 --- a/domain-server/src/DomainContentBackupManager.h +++ b/domain-server/src/DomainContentBackupManager.h @@ -62,6 +62,7 @@ public slots: void createManualBackup(MiniPromise::Promise promise, const QString& name); void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName); void deleteBackup(MiniPromise::Promise promise, const QString& backupName); + void consolidateBackup(MiniPromise::Promise promise, QString fileName); signals: void loadCompleted(); @@ -73,7 +74,6 @@ protected: void load(); void backup(); - void consolidate(QString fileName); void removeOldBackupVersions(const BackupRule& rule); bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime); int64_t getMostRecentBackupTimeInSecs(const QString& format); diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 2f8d8f6d03..b91e12a9cf 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -2149,6 +2149,28 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url QJsonDocument docJSON(rootJSON); connection->respond(HTTPConnection::StatusCode200, docJSON.toJson(), JSON_MIME_TYPE.toUtf8()); + return true; + } else if (url.path().startsWith(URI_API_BACKUPS_ID)) { + auto id = url.path().mid(QString(URI_API_BACKUPS_ID).length()); + auto deferred = makePromise("consolidateBackup"); + deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) { + QJsonObject rootJSON; + auto success = result["success"].toBool(); + if (success) { + auto path = result["backupFilePath"].toString(); + auto file { std::unique_ptr(new QFile(path)) }; + if (file->open(QIODevice::ReadOnly)) { + connection->respond(HTTPConnection::StatusCode200, std::move(file)); + } else { + qCritical(domain_server) << "Unable to load consolidated backup at:" << path << result; + connection->respond(HTTPConnection::StatusCode500, "Error opening backup"); + } + } else { + connection->respond(HTTPConnection::StatusCode400); + } + }); + _contentManager->consolidateBackup(deferred, id); + return true; } else if (url.path() == URI_RESTART) { connection->respond(HTTPConnection::StatusCode200); From 2942a53a1d51b488d7e8818e180c25abb335f0a2 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 15 Feb 2018 15:55:14 -0800 Subject: [PATCH 03/13] Add recovery mode and full backup information to DS --- .../src/DomainContentBackupManager.cpp | 91 +++++++++++++++++-- .../src/DomainContentBackupManager.h | 14 +-- domain-server/src/DomainServer.cpp | 28 ++---- 3 files changed, 96 insertions(+), 37 deletions(-) diff --git a/domain-server/src/DomainContentBackupManager.cpp b/domain-server/src/DomainContentBackupManager.cpp index f56c41dacd..54ea7e23d5 100644 --- a/domain-server/src/DomainContentBackupManager.cpp +++ b/domain-server/src/DomainContentBackupManager.cpp @@ -147,7 +147,24 @@ bool DomainContentBackupManager::process() { if (sinceLastSave > intervalToCheck) { _lastCheck = now; - backup(); + if (_isRecovering) { + bool anyHandlerIsRecovering { false }; + for (auto& handler : _backupHandlers) { + bool handlerIsRecovering { false }; + float progress { 0.0f }; + //std::tie = handler->getRecoveryStatus(); + if (handlerIsRecovering) { + anyHandlerIsRecovering = true; + emit recoveryCompleted(); + break; + } + } + _isRecovering = anyHandlerIsRecovering; + } + + if (!_isRecovering) { + backup(); + } } } @@ -213,6 +230,13 @@ void DomainContentBackupManager::deleteBackup(MiniPromise::Promise promise, cons return; } + if (_isRecovering && backupName == _recoveryFilename) { + promise->resolve({ + { "success", false } + }); + return; + } + QDir backupDir { _backupDirectory }; QFile backupFile { backupDir.filePath(backupName) }; auto success = backupFile.remove(); @@ -222,6 +246,13 @@ void DomainContentBackupManager::deleteBackup(MiniPromise::Promise promise, cons } void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise, const QString& backupName) { + if (_isRecovering) { + promise->resolve({ + { "success", false } + }); + return; + }; + if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "recoverFromBackup", Q_ARG(MiniPromise::Promise, promise), Q_ARG(const QString&, backupName)); @@ -239,11 +270,12 @@ void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise, qWarning() << "Failed to unzip file: " << backupName; success = false; } else { + _isRecovering = true; for (auto& handler : _backupHandlers) { handler->recoverBackup(zip); } - qDebug() << "Successfully recovered from " << backupName; + qDebug() << "Successfully started recovering from " << backupName; success = true; } backupFile.close(); @@ -257,8 +289,11 @@ void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise, }); } -std::vector DomainContentBackupManager::getAllBackups() { - std::vector backups; +void DomainContentBackupManager::getAllBackupInformation(MiniPromise::Promise promise) { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "getAllBackupInformation", Q_ARG(MiniPromise::Promise, promise)); + return; + } QDir backupDir { _backupDirectory }; auto matchingFiles = @@ -269,6 +304,8 @@ std::vector DomainContentBackupManager::getAllBackups() { QString dateTimeFormat = "(" + DATETIME_FORMAT_RE + ")"; QRegExp backupNameFormat { prefixFormat + nameFormat + "-" + dateTimeFormat + "\\.zip" }; + QVariantList backups; + for (const auto& fileInfo : matchingFiles) { auto fileName = fileInfo.fileName(); if (backupNameFormat.exactMatch(fileName)) { @@ -280,12 +317,50 @@ std::vector DomainContentBackupManager::getAllBackups() { continue; } - BackupItemInfo backup { fileInfo.fileName(), name, fileInfo.absoluteFilePath(), createdAt, type == MANUAL_BACKUP_PREFIX }; - backups.push_back(backup); + bool isAvailable { true }; + float availabilityProgress { 0.0f }; + for (auto& handler : _backupHandlers) { + bool handlerIsAvailable { false }; + float progress { 0.0f }; + //std::tie = handler->isAvailable(); + //isAvailable = isAvailable && !handlerIsAvailable); + //availabilityProgress += progress / _backupHandlers.size(); + } + + backups.push_back(QVariantMap({ + { "id", fileInfo.fileName() }, + { "name", name }, + { "createdAtMillis", createdAt.toMSecsSinceEpoch() }, + { "isAvailable", isAvailable }, + { "availabilityProgress", availabilityProgress }, + { "isManualBackup", type == MANUAL_BACKUP_PREFIX } + })); } } - return backups; + float recoveryProgress = 0.0f; + bool isRecovering = _isRecovering.load(); + if (_isRecovering) { + for (auto& handler : _backupHandlers) { + bool handlerIsRecovering { false }; + float progress { 0.0f }; + //std::tie = handler->getRecoveryStatus(); + recoveryProgress += progress / _backupHandlers.size(); + } + } + + QVariantMap status { + { "isRecovering", isRecovering }, + { "recoveringBackupId", _recoveryFilename }, + { "recoveryProgress", recoveryProgress } + }; + + QVariantMap info { + { "backups", backups }, + { "status", status } + }; + + promise->resolve(info); } void DomainContentBackupManager::removeOldBackupVersions(const BackupRule& rule) { @@ -433,6 +508,8 @@ void DomainContentBackupManager::consolidateBackup(MiniPromise::Promise promise, return; } + qDebug() << "copyFilePath" << copyFilePath; + promise->resolve({ { "success", true }, { "backupFilePath", copyFilePath } diff --git a/domain-server/src/DomainContentBackupManager.h b/domain-server/src/DomainContentBackupManager.h index 9ec7eb9950..790dff0fb4 100644 --- a/domain-server/src/DomainContentBackupManager.h +++ b/domain-server/src/DomainContentBackupManager.h @@ -24,14 +24,6 @@ #include -struct BackupItemInfo { - QString id; - QString name; - QString absolutePath; - QDateTime createdAt; - bool isManualBackup; -}; - class DomainContentBackupManager : public GenericThread { Q_OBJECT public: @@ -52,13 +44,13 @@ public: bool debugTimestampNow = false); void addBackupHandler(BackupHandlerPointer handler); - std::vector getAllBackups(); void aboutToFinish(); /// call this to inform the persist thread that the owner is about to finish to support final persist void replaceData(QByteArray data); public slots: + void getAllBackupInformation(MiniPromise::Promise promise); void createManualBackup(MiniPromise::Promise promise, const QString& name); void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName); void deleteBackup(MiniPromise::Promise promise, const QString& backupName); @@ -66,6 +58,7 @@ public slots: signals: void loadCompleted(); + void recoveryCompleted(); protected: /// Implements generic processing behavior for this thread. @@ -86,6 +79,9 @@ private: std::vector _backupHandlers; int _persistInterval { 0 }; + std::atomic _isRecovering { false }; + QString _recoveryFilename { }; + int64_t _lastCheck { 0 }; std::vector _backupRules; }; diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index b91e12a9cf..fe145b341b 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -307,10 +307,7 @@ DomainServer::DomainServer(int argc, char* argv[]) : _contentManager->initialize(true); - qDebug() << "Existing backups:"; - for (auto& backup : _contentManager->getAllBackups()) { - qDebug() << " Backup: " << backup.name << backup.createdAt; - } + connect(_contentManager.get(), &DomainContentBackupManager::recoveryCompleted, this, &DomainServer::restart); } void DomainServer::parseCommandLine() { @@ -2131,24 +2128,13 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url return true; } else if (url.path() == URI_API_BACKUPS) { - QJsonObject rootJSON; - QJsonArray backupsJSON; + auto deferred = makePromise("getAllBackupInformation"); + deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) { + QJsonDocument docJSON(QJsonObject::fromVariantMap(result)); - auto backups = _contentManager->getAllBackups(); - - for (const auto& backup : backups) { - QJsonObject obj; - obj["id"] = backup.id; - obj["name"] = backup.name; - obj["createdAtMillis"] = backup.createdAt.toMSecsSinceEpoch(); - obj["isManualBackup"] = backup.isManualBackup; - backupsJSON.push_back(obj); - } - - rootJSON["backups"] = backupsJSON; - QJsonDocument docJSON(rootJSON); - - connection->respond(HTTPConnection::StatusCode200, docJSON.toJson(), JSON_MIME_TYPE.toUtf8()); + connection->respond(HTTPConnection::StatusCode200, docJSON.toJson(), JSON_MIME_TYPE.toUtf8()); + }); + _contentManager->getAllBackupInformation(deferred); return true; } else if (url.path().startsWith(URI_API_BACKUPS_ID)) { auto id = url.path().mid(QString(URI_API_BACKUPS_ID).length()); From 1120b12b8cb758578e16912961744a81a12b1880 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 15 Feb 2018 15:58:39 -0800 Subject: [PATCH 04/13] Fix argument to isAvailable --- domain-server/src/DomainContentBackupManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domain-server/src/DomainContentBackupManager.cpp b/domain-server/src/DomainContentBackupManager.cpp index 54ea7e23d5..eccc3e904d 100644 --- a/domain-server/src/DomainContentBackupManager.cpp +++ b/domain-server/src/DomainContentBackupManager.cpp @@ -322,7 +322,7 @@ void DomainContentBackupManager::getAllBackupInformation(MiniPromise::Promise pr for (auto& handler : _backupHandlers) { bool handlerIsAvailable { false }; float progress { 0.0f }; - //std::tie = handler->isAvailable(); + //std::tie = handler->isAvailable(fileInfo.absoluteFilePath()); //isAvailable = isAvailable && !handlerIsAvailable); //availabilityProgress += progress / _backupHandlers.size(); } From a7ca5398990a9328856f1a1aca75e4397a78db3a Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 14 Feb 2018 16:48:37 -0800 Subject: [PATCH 05/13] Simplify BackupHandler pattern --- domain-server/src/BackupHandler.h | 1 - 1 file changed, 1 deletion(-) diff --git a/domain-server/src/BackupHandler.h b/domain-server/src/BackupHandler.h index 8599dafb29..960dde9b45 100644 --- a/domain-server/src/BackupHandler.h +++ b/domain-server/src/BackupHandler.h @@ -26,7 +26,6 @@ public: virtual void deleteBackup(QuaZip& zip) = 0; virtual void consolidateBackup(QuaZip& zip) = 0; }; - using BackupHandlerPointer = std::unique_ptr; #endif /* hifi_BackupHandler_h */ From b76e1b9750973416aa2a316e601e2c36057b829e Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 15 Feb 2018 15:46:35 -0800 Subject: [PATCH 06/13] Add backup status getters --- domain-server/src/AssetsBackupHandler.cpp | 51 +++++++++++++++++++---- domain-server/src/AssetsBackupHandler.h | 21 +++++----- domain-server/src/BackupHandler.h | 5 +++ domain-server/src/EntitiesBackupHandler.h | 13 +++--- 4 files changed, 66 insertions(+), 24 deletions(-) diff --git a/domain-server/src/AssetsBackupHandler.cpp b/domain-server/src/AssetsBackupHandler.cpp index ae9cb58343..e683c626ea 100644 --- a/domain-server/src/AssetsBackupHandler.cpp +++ b/domain-server/src/AssetsBackupHandler.cpp @@ -111,6 +111,42 @@ void AssetsBackupHandler::checkForAssetsToDelete() { } } + +std::pair AssetsBackupHandler::isAvailable(QString filePath) { + auto it = find_if(begin(_backups), end(_backups), [&](const std::vector::value_type& value) { + return value.filePath == filePath; + }); + if (it == end(_backups)) { + return { true, 1.0f }; + } + + float progress = (float)it->mappings.size(); + for (const auto& mapping : it->mappings) { + if (_assetsLeftToRequest.find(mapping.second) != end(_assetsLeftToRequest)) { + progress -= 1.0f; + } + } + progress /= (float)it->mappings.size(); + + return { false, progress }; +} + +std::pair AssetsBackupHandler::getRecoveryStatus() { + if (_assetsLeftToUpload.empty() && + _mappingsLeftToSet.empty() && + _mappingsLeftToDelete.empty() && + _mappingRequestsInFlight == 0) { + return { false, 1.0f }; + } + + float progress = (float)_numRestoreOperations; + progress -= (float)_assetsLeftToUpload.size(); + progress -= (float)_mappingRequestsInFlight; + progress /= (float)_numRestoreOperations; + + return { true, progress }; +} + void AssetsBackupHandler::loadBackup(QuaZip& zip) { Q_ASSERT(QThread::currentThread() == thread()); @@ -451,6 +487,11 @@ void AssetsBackupHandler::computeServerStateDifference(const AssetUtils::Mapping } } + _numRestoreOperations = _assetsLeftToUpload.size() + _mappingsLeftToSet.size(); + if (!_mappingsLeftToDelete.empty()) { + ++_numRestoreOperations; + } + qCDebug(asset_backup) << "Mappings to set:" << _mappingsLeftToSet.size(); qCDebug(asset_backup) << "Mappings to del:" << _mappingsLeftToDelete.size(); qCDebug(asset_backup) << "Assets to upload:" << _assetsLeftToUpload.size(); @@ -461,8 +502,6 @@ void AssetsBackupHandler::restoreAllAssets() { } void AssetsBackupHandler::restoreNextAsset() { - startOperation(); - if (_assetsLeftToUpload.empty()) { updateMappings(); return; @@ -500,9 +539,7 @@ void AssetsBackupHandler::updateMappings() { qCCritical(asset_backup) << " Error:" << request->getErrorString(); } - if (--_mappingRequestsInFlight == 0) { - stopOperation(); - } + --_mappingRequestsInFlight; request->deleteLater(); }); @@ -519,9 +556,7 @@ void AssetsBackupHandler::updateMappings() { qCCritical(asset_backup) << " Error:" << request->getErrorString(); } - if (--_mappingRequestsInFlight == 0) { - stopOperation(); - } + --_mappingRequestsInFlight; request->deleteLater(); }); diff --git a/domain-server/src/AssetsBackupHandler.h b/domain-server/src/AssetsBackupHandler.h index 2ef454998e..1421ddd400 100644 --- a/domain-server/src/AssetsBackupHandler.h +++ b/domain-server/src/AssetsBackupHandler.h @@ -31,13 +31,16 @@ class AssetsBackupHandler : public QObject, public BackupHandlerInterface { public: AssetsBackupHandler(const QString& backupDirectory); - void loadBackup(QuaZip& zip); - void createBackup(QuaZip& zip); - void recoverBackup(QuaZip& zip); - void deleteBackup(QuaZip& zip); - void consolidateBackup(QuaZip& zip); + std::pair isAvailable(QString filePath) override; + std::pair getRecoveryStatus() override; - bool operationInProgress() const { return _operationInProgress; } + void loadBackup(QuaZip& zip) override; + void createBackup(QuaZip& zip) override; + void recoverBackup(QuaZip& zip) override; + void deleteBackup(QuaZip& zip) override; + void consolidateBackup(QuaZip& zip) override; + + bool operationInProgress() { return getRecoveryStatus().first; } private: void setupRefreshTimer(); @@ -48,9 +51,6 @@ private: void checkForMissingAssets(); void checkForAssetsToDelete(); - void startOperation() { _operationInProgress = true; } - void stopOperation() { _operationInProgress = false; } - void downloadMissingFiles(const AssetUtils::Mappings& mappings); void downloadNextMissingFile(); bool writeAssetFile(const AssetUtils::AssetHash& hash, const QByteArray& data); @@ -73,8 +73,6 @@ private: bool corruptedBackup; }; - bool _operationInProgress { false }; - // Internal storage for backups on disk bool _allBackupsLoadedSuccessfully { false }; std::vector _backups; @@ -89,6 +87,7 @@ private: std::vector> _mappingsLeftToSet; AssetUtils::AssetPathList _mappingsLeftToDelete; int _mappingRequestsInFlight { 0 }; + int _numRestoreOperations { 0 }; // Used to compute a restore progress. }; #endif /* hifi_AssetsBackupHandler_h */ diff --git a/domain-server/src/BackupHandler.h b/domain-server/src/BackupHandler.h index 960dde9b45..d513820000 100644 --- a/domain-server/src/BackupHandler.h +++ b/domain-server/src/BackupHandler.h @@ -20,6 +20,11 @@ class BackupHandlerInterface { public: virtual ~BackupHandlerInterface() = default; + virtual std::pair isAvailable(QString filePath) = 0; + + // Returns whether a recovery is ongoing and a progress between 0 and 1 if one is. + virtual std::pair getRecoveryStatus() = 0; + virtual void loadBackup(QuaZip& zip) = 0; virtual void createBackup(QuaZip& zip) = 0; virtual void recoverBackup(QuaZip& zip) = 0; diff --git a/domain-server/src/EntitiesBackupHandler.h b/domain-server/src/EntitiesBackupHandler.h index 6f66483a87..4cff7b6a33 100644 --- a/domain-server/src/EntitiesBackupHandler.h +++ b/domain-server/src/EntitiesBackupHandler.h @@ -20,19 +20,22 @@ class EntitiesBackupHandler : public BackupHandlerInterface { public: EntitiesBackupHandler(QString entitiesFilePath, QString entitiesReplacementFilePath); - void loadBackup(QuaZip& zip) {} + std::pair isAvailable(QString filePath) override { return { true, 1.0f }; } + std::pair getRecoveryStatus() override { return { false, 1.0f }; } + + void loadBackup(QuaZip& zip) override {} // Create a skeleton backup - void createBackup(QuaZip& zip); + void createBackup(QuaZip& zip) override; // Recover from a full backup - void recoverBackup(QuaZip& zip); + void recoverBackup(QuaZip& zip) override; // Delete a skeleton backup - void deleteBackup(QuaZip& zip) {} + void deleteBackup(QuaZip& zip) override {} // Create a full backup - void consolidateBackup(QuaZip& zip) {} + void consolidateBackup(QuaZip& zip) override {} private: QString _entitiesFilePath; From 57410e4f1cc017c5aa23220fa96f6a445899d62a Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 15 Feb 2018 16:22:06 -0800 Subject: [PATCH 07/13] Remove ES restart after restore --- domain-server/src/EntitiesBackupHandler.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/domain-server/src/EntitiesBackupHandler.cpp b/domain-server/src/EntitiesBackupHandler.cpp index deb92ee0f6..6ad00d01c8 100644 --- a/domain-server/src/EntitiesBackupHandler.cpp +++ b/domain-server/src/EntitiesBackupHandler.cpp @@ -16,7 +16,6 @@ #include #include -#include #include EntitiesBackupHandler::EntitiesBackupHandler(QString entitiesFilePath, QString entitiesReplacementFilePath) : @@ -72,13 +71,5 @@ void EntitiesBackupHandler::recoverBackup(QuaZip& zip) { if (entitiesFile.open(QIODevice::WriteOnly)) { entitiesFile.write(data.toGzippedByteArray()); - entitiesFile.close(); - - auto nodeList = DependencyManager::get(); - nodeList->eachMatchingNode([](const SharedNodePointer& otherNode) -> bool { - return otherNode->getType() == NodeType::EntityServer; - }, [nodeList](const SharedNodePointer& otherNode) { - QMetaObject::invokeMethod(nodeList.data(), "killNodeWithUUID", Q_ARG(const QUuid&, otherNode->getUUID())); - }); } } From 771e4cd9f4d5e8324813f41011717ea423327525 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 15 Feb 2018 16:45:29 -0800 Subject: [PATCH 08/13] Hook up status and progress --- .../src/DomainContentBackupManager.cpp | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/domain-server/src/DomainContentBackupManager.cpp b/domain-server/src/DomainContentBackupManager.cpp index eccc3e904d..37a8ecfce2 100644 --- a/domain-server/src/DomainContentBackupManager.cpp +++ b/domain-server/src/DomainContentBackupManager.cpp @@ -148,18 +148,15 @@ bool DomainContentBackupManager::process() { if (sinceLastSave > intervalToCheck) { _lastCheck = now; if (_isRecovering) { - bool anyHandlerIsRecovering { false }; - for (auto& handler : _backupHandlers) { - bool handlerIsRecovering { false }; - float progress { 0.0f }; - //std::tie = handler->getRecoveryStatus(); - if (handlerIsRecovering) { - anyHandlerIsRecovering = true; - emit recoveryCompleted(); - break; - } + using Value = std::vector::value_type; + bool isStillRecovering = std::any_of(begin(_backupHandlers), end(_backupHandlers), [](const Value& handler) { + return handler->getRecoveryStatus().first; + }); + + if (!isStillRecovering) { + _isRecovering = false; + emit recoveryCompleted(); } - _isRecovering = anyHandlerIsRecovering; } if (!_isRecovering) { @@ -320,11 +317,11 @@ void DomainContentBackupManager::getAllBackupInformation(MiniPromise::Promise pr bool isAvailable { true }; float availabilityProgress { 0.0f }; for (auto& handler : _backupHandlers) { - bool handlerIsAvailable { false }; + bool handlerIsAvailable { true }; float progress { 0.0f }; - //std::tie = handler->isAvailable(fileInfo.absoluteFilePath()); - //isAvailable = isAvailable && !handlerIsAvailable); - //availabilityProgress += progress / _backupHandlers.size(); + std::tie(handlerIsAvailable, progress) = handler->isAvailable(fileInfo.absoluteFilePath()); + isAvailable &= handlerIsAvailable; + availabilityProgress += progress / _backupHandlers.size(); } backups.push_back(QVariantMap({ @@ -342,9 +339,7 @@ void DomainContentBackupManager::getAllBackupInformation(MiniPromise::Promise pr bool isRecovering = _isRecovering.load(); if (_isRecovering) { for (auto& handler : _backupHandlers) { - bool handlerIsRecovering { false }; - float progress { 0.0f }; - //std::tie = handler->getRecoveryStatus(); + float progress = handler->getRecoveryStatus().second; recoveryProgress += progress / _backupHandlers.size(); } } From cae3e0a9dcceff2e182a012a6381743ca85905ba Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 15 Feb 2018 16:59:17 -0800 Subject: [PATCH 09/13] Add status func to ContentSettingsBackupHandler --- domain-server/src/ContentSettingsBackupHandler.h | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/domain-server/src/ContentSettingsBackupHandler.h b/domain-server/src/ContentSettingsBackupHandler.h index 932b7c0c3f..8a81392513 100644 --- a/domain-server/src/ContentSettingsBackupHandler.h +++ b/domain-server/src/ContentSettingsBackupHandler.h @@ -19,15 +19,18 @@ class ContentSettingsBackupHandler : public BackupHandlerInterface { public: ContentSettingsBackupHandler(DomainServerSettingsManager& domainServerSettingsManager); - void loadBackup(QuaZip& zip) {}; + std::pair isAvailable(QString filePath) override { return { true, 1.0f }; } + std::pair getRecoveryStatus() override { return { false, 1.0f }; } - void createBackup(QuaZip& zip); + void loadBackup(QuaZip& zip) override {} - void recoverBackup(QuaZip& zip); + void createBackup(QuaZip& zip) override; - void deleteBackup(QuaZip& zip) {}; + void recoverBackup(QuaZip& zip) override; - void consolidateBackup(QuaZip& zip) {}; + void deleteBackup(QuaZip& zip) override {} + + void consolidateBackup(QuaZip& zip) override {} private: DomainServerSettingsManager& _settingsManager; }; From 2b85634a21abec80f8f689013c2696afb19e2dc9 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 15 Feb 2018 17:03:55 -0800 Subject: [PATCH 10/13] Fix build error --- domain-server/src/BackupHandler.h | 2 ++ domain-server/src/EntitiesBackupHandler.h | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/domain-server/src/BackupHandler.h b/domain-server/src/BackupHandler.h index d513820000..1bd40cd9e4 100644 --- a/domain-server/src/BackupHandler.h +++ b/domain-server/src/BackupHandler.h @@ -14,6 +14,8 @@ #include +#include + class QuaZip; class BackupHandlerInterface { diff --git a/domain-server/src/EntitiesBackupHandler.h b/domain-server/src/EntitiesBackupHandler.h index 4cff7b6a33..1a6110f1cd 100644 --- a/domain-server/src/EntitiesBackupHandler.h +++ b/domain-server/src/EntitiesBackupHandler.h @@ -12,8 +12,6 @@ #ifndef hifi_EntitiesBackupHandler_h #define hifi_EntitiesBackupHandler_h -#include - #include "BackupHandler.h" class EntitiesBackupHandler : public BackupHandlerInterface { From 697f0c443cb0f3606ca2facc53dc35bd5ca98ae0 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 15 Feb 2018 17:43:02 -0800 Subject: [PATCH 11/13] Fix warning --- domain-server/src/AssetsBackupHandler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domain-server/src/AssetsBackupHandler.cpp b/domain-server/src/AssetsBackupHandler.cpp index e683c626ea..db39f2a731 100644 --- a/domain-server/src/AssetsBackupHandler.cpp +++ b/domain-server/src/AssetsBackupHandler.cpp @@ -487,7 +487,7 @@ void AssetsBackupHandler::computeServerStateDifference(const AssetUtils::Mapping } } - _numRestoreOperations = _assetsLeftToUpload.size() + _mappingsLeftToSet.size(); + _numRestoreOperations = (int)_assetsLeftToUpload.size() + (int)_mappingsLeftToSet.size(); if (!_mappingsLeftToDelete.empty()) { ++_numRestoreOperations; } From f6e9d2c6dd05358d3053795f81705c43bbde02b3 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 15 Feb 2018 18:16:30 -0800 Subject: [PATCH 12/13] Fix race condition in Asset Server --- assignment-client/src/assets/AssetServer.cpp | 28 ++++++++++++++++---- assignment-client/src/assets/AssetServer.h | 5 +++- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 0be557bccd..4c6cba2e74 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -416,9 +416,9 @@ void AssetServer::completeSetup() { PathUtils::removeTemporaryApplicationDirs(); PathUtils::removeTemporaryApplicationDirs("Oven"); - // We're fully setup, remove the request queueing and replay all requests + qCDebug(asset_server) << "Overriding temporary queuing packet handler."; + // We're fully setup, override the request queueing handler and replay all requests auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); - packetReceiver.unregisterListener(this); packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet"); packetReceiver.registerListener(PacketType::AssetGetInfo, this, "handleAssetGetInfo"); packetReceiver.registerListener(PacketType::AssetUpload, this, "handleAssetUpload"); @@ -428,11 +428,30 @@ void AssetServer::completeSetup() { } void AssetServer::queueRequests(QSharedPointer packet, SharedNodePointer senderNode) { + qCDebug(asset_server) << "Queuing requests until fully setup"; + + QMutexLocker lock { &_queuedRequestsMutex }; _queuedRequests.push_back({ packet, senderNode }); + + // If we've stopped queueing but the callback was already in flight, + // then replay it immediately. + if (!_isQueueingRequests) { + lock.unlock(); + replayRequests(); + } } void AssetServer::replayRequests() { - for (const auto& request : _queuedRequests) { + RequestQueue queue; + { + QMutexLocker lock { &_queuedRequestsMutex }; + qSwap(queue, _queuedRequests); + _isQueueingRequests = false; + } + + qCDebug(asset_server) << "Replaying" << queue.size() << "requests."; + + for (const auto& request : queue) { switch (request.first->getType()) { case PacketType::AssetGet: handleAssetGet(request.first, request.second); @@ -447,11 +466,10 @@ void AssetServer::replayRequests() { handleAssetMappingOperation(request.first, request.second); break; default: - qWarning() << "Unknown queued request type:" << request.first->getType(); + qCWarning(asset_server) << "Unknown queued request type:" << request.first->getType(); break; } } - _queuedRequests.clear(); } void AssetServer::cleanupUnmappedFiles() { diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index b8aac800ed..c85fb89175 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -123,7 +123,10 @@ private: QHash> _pendingBakes; QThreadPool _bakingTaskPool; - QVector, SharedNodePointer>> _queuedRequests; + QMutex _queuedRequestsMutex; + bool _isQueueingRequests { true }; + using RequestQueue = QVector, SharedNodePointer>>; + RequestQueue _queuedRequests; bool _wasColorTextureCompressionEnabled { false }; bool _wasGrayscaleTextureCompressionEnabled { false }; From b30f98d5414759a1eae88505812cb481bb813844 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 15 Feb 2018 18:20:14 -0800 Subject: [PATCH 13/13] CR --- domain-server/src/DomainContentBackupManager.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/domain-server/src/DomainContentBackupManager.cpp b/domain-server/src/DomainContentBackupManager.cpp index 37a8ecfce2..5eb4e7627f 100644 --- a/domain-server/src/DomainContentBackupManager.cpp +++ b/domain-server/src/DomainContentBackupManager.cpp @@ -503,8 +503,6 @@ void DomainContentBackupManager::consolidateBackup(MiniPromise::Promise promise, return; } - qDebug() << "copyFilePath" << copyFilePath; - promise->resolve({ { "success", true }, { "backupFilePath", copyFilePath }