From cb9327e03020b7cd6965c73cf7db2177eb46e7d9 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 10 Jan 2018 13:09:22 -0800 Subject: [PATCH] Add entity file sync and domain content backups --- .clang-format | 10 +- assignment-client/CMakeLists.txt | 1 + assignment-client/src/Agent.cpp | 1 - .../src/entities/EntityServer.cpp | 2 - assignment-client/src/entities/EntityServer.h | 13 +- assignment-client/src/octree/OctreeServer.cpp | 201 ++++++------ assignment-client/src/octree/OctreeServer.h | 21 +- .../src/scripts/EntityScriptServer.cpp | 2 +- domain-server/CMakeLists.txt | 12 +- .../src/DomainContentBackupManager.cpp | 303 ++++++++++++++++++ .../src/DomainContentBackupManager.h | 88 +++++ domain-server/src/DomainServer.cpp | 213 +++++++++++- domain-server/src/DomainServer.h | 28 +- interface/src/Application.cpp | 8 +- interface/src/ui/DomainConnectionModel.cpp | 2 +- libraries/entities/src/EntityTree.cpp | 10 + libraries/image/CMakeLists.txt | 3 +- libraries/networking/src/LimitedNodeList.cpp | 1 + libraries/networking/src/ThreadedAssignment.h | 6 +- libraries/networking/src/udt/PacketHeaders.h | 7 + libraries/networking/src/udt/Socket.cpp | 4 +- libraries/octree/CMakeLists.txt | 1 + libraries/octree/src/Octree.cpp | 54 +++- libraries/octree/src/Octree.h | 14 +- libraries/octree/src/OctreePersistThread.cpp | 102 +++--- libraries/octree/src/OctreePersistThread.h | 11 +- libraries/octree/src/OctreeUtils.cpp | 77 +++++ libraries/octree/src/OctreeUtils.h | 23 ++ libraries/shared/src/SharedUtil.cpp | 6 +- libraries/shared/src/SharedUtil.h | 1 + 30 files changed, 1026 insertions(+), 199 deletions(-) create mode 100644 domain-server/src/DomainContentBackupManager.cpp create mode 100644 domain-server/src/DomainContentBackupManager.h diff --git a/.clang-format b/.clang-format index f000a27017..507b1eb232 100644 --- a/.clang-format +++ b/.clang-format @@ -1,12 +1,12 @@ Language: Cpp Standard: Cpp11 -BasedOnStyle: "Chromium" +BasedOnStyle: "Chromium" ColumnLimit: 128 IndentWidth: 4 UseTab: Never BreakBeforeBraces: Custom -BraceWrapping: +BraceWrapping: AfterEnum: true AfterClass: false AfterControlStatement: false @@ -21,11 +21,11 @@ BraceWrapping: AccessModifierOffset: -4 -AllowShortFunctionsOnASingleLine: InlineOnly -BreakConstructorInitializers: BeforeColon +AllowShortFunctionsOnASingleLine: InlineOnly +BreakConstructorInitializers: BeforeColon BreakConstructorInitializersBeforeComma: true IndentCaseLabels: true -ReflowComments: false +ReflowComments: false Cpp11BracedListStyle: false ContinuationIndentWidth: 4 ConstructorInitializerAllOnOneLineOrOnePerLine: false diff --git a/assignment-client/CMakeLists.txt b/assignment-client/CMakeLists.txt index c73e8e1d34..3de4c5fd3f 100644 --- a/assignment-client/CMakeLists.txt +++ b/assignment-client/CMakeLists.txt @@ -6,6 +6,7 @@ setup_hifi_project(Core Gui Network Script Quick WebSockets) if (APPLE) set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "@executable_path/../Frameworks") endif () +set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "/testing/") setup_memory_debugger() diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index a42b78a6fa..10b8d44545 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -340,7 +340,6 @@ void Agent::scriptRequestFinished() { request->deleteLater(); } - void Agent::executeScript() { _scriptEngine = scriptEngineFactory(ScriptEngine::AGENT_SCRIPT, _scriptContents, _payload); diff --git a/assignment-client/src/entities/EntityServer.cpp b/assignment-client/src/entities/EntityServer.cpp index f72832f902..e394884dc2 100644 --- a/assignment-client/src/entities/EntityServer.cpp +++ b/assignment-client/src/entities/EntityServer.cpp @@ -116,7 +116,6 @@ void EntityServer::beforeRun() { void EntityServer::entityCreated(const EntityItem& newEntity, const SharedNodePointer& senderNode) { } - // EntityServer will use the "special packets" to send list of recently deleted entities bool EntityServer::hasSpecialPacketsToSend(const SharedNodePointer& node) { bool shouldSendDeletedEntities = false; @@ -277,7 +276,6 @@ int EntityServer::sendSpecialPackets(const SharedNodePointer& node, OctreeQueryN return totalBytes; } - void EntityServer::pruneDeletedEntities() { EntityTreePointer tree = std::static_pointer_cast(_tree); if (tree->hasAnyDeletedEntities()) { diff --git a/assignment-client/src/entities/EntityServer.h b/assignment-client/src/entities/EntityServer.h index 05404b28c8..4d3f1ee89f 100644 --- a/assignment-client/src/entities/EntityServer.h +++ b/assignment-client/src/entities/EntityServer.h @@ -30,7 +30,6 @@ struct ViewerSendingStats { class SimpleEntitySimulation; using SimpleEntitySimulationPointer = std::shared_ptr; - class EntityServer : public OctreeServer, public NewlyCreatedEntityHook { Q_OBJECT public: @@ -38,7 +37,7 @@ public: ~EntityServer(); // Subclasses must implement these methods - virtual std::unique_ptr createOctreeQueryNode() override ; + virtual std::unique_ptr createOctreeQueryNode() override; virtual char getMyNodeType() const override { return NodeType::EntityServer; } virtual PacketType getMyQueryMessageType() const override { return PacketType::EntityQuery; } virtual const char* getMyServerName() const override { return MODEL_SERVER_NAME; } @@ -82,12 +81,12 @@ private: QReadWriteLock _viewerSendingStatsLock; QMap> _viewerSendingStats; - static const int DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 45 * 60 * 1000; // 45m - static const int DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 60 * 60 * 1000; // 1h - int _MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS; // 45m - int _MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS; // 1h + static const int DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 45 * 60 * 1000; // 45m + static const int DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 60 * 60 * 1000; // 1h + int _MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS; // 45m + int _MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS; // 1h QTimer _dynamicDomainVerificationTimer; void startDynamicDomainVerification(); }; -#endif // hifi_EntityServer_h +#endif // hifi_EntityServer_h diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp index 42494ea7ee..e78f9f108b 100644 --- a/assignment-client/src/octree/OctreeServer.cpp +++ b/assignment-client/src/octree/OctreeServer.cpp @@ -33,6 +33,10 @@ #include #include +#include + +Q_LOGGING_CATEGORY(octree_server, "hifi.octree-server") + int OctreeServer::_clientCount = 0; const int MOVING_AVERAGE_SAMPLE_COUNTS = 1000; @@ -84,6 +88,8 @@ int OctreeServer::_longProcessWait = 0; int OctreeServer::_shortProcessWait = 0; int OctreeServer::_noProcessWait = 0; +static const QString PERSIST_FILE_DOWNLOAD_PATH = "/models.json.gz"; + void OctreeServer::resetSendingStats() { _averageLoopTime.reset(); @@ -202,7 +208,6 @@ void OctreeServer::trackPacketSendingTime(float time) { } } - void OctreeServer::trackProcessWaitTime(float time) { const float MAX_SHORT_TIME = 10.0f; const float MAX_LONG_TIME = 100.0f; @@ -283,8 +288,6 @@ void OctreeServer::initHTTPManager(int port) { _httpManager = new HTTPManager(QHostAddress::AnyIPv4, port, documentRoot, this, this); } -const QString PERSIST_FILE_DOWNLOAD_PATH = "/models.json.gz"; - bool OctreeServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler) { #ifdef FORCE_CRASH @@ -922,87 +925,6 @@ void OctreeServer::handleOctreeDataNackPacket(QSharedPointer me } } -void OctreeServer::handleOctreeFileReplacement(QSharedPointer message) { - if (!_isFinished && !_isShuttingDown) { - // these messages are only allowed to come from the domain server, so make sure that is the case - auto nodeList = DependencyManager::get(); - if (message->getSenderSockAddr() == nodeList->getDomainHandler().getSockAddr()) { - // it's far cleaner to load up the new content upon server startup - // so here we just store a special file at our persist path - // and then force a stop of the server so that it can pick it up when it relaunches - if (!_persistAbsoluteFilePath.isEmpty()) { - replaceContentFromMessageData(message->getMessage()); - } else { - qDebug() << "Cannot perform octree file replacement since current persist file path is not yet known"; - } - } else { - qDebug() << "Received an octree file replacement that was not from our domain server - refusing to process"; - } - } -} - -// Message->getMessage() contains a QByteArray representation of the URL to download from -void OctreeServer::handleOctreeFileReplacementFromURL(QSharedPointer message) { - qInfo() << "Received request to replace content from a url"; - if (!_isFinished && !_isShuttingDown) { - // This call comes from Interface, so we skip our domain server check - // but confirm that we have permissions to replace content sets - if (DependencyManager::get()->getThisNodeCanReplaceContent()) { - if (!_persistAbsoluteFilePath.isEmpty()) { - // Convert message data into our URL - QString url(message->getMessage()); - QUrl modelsURL = QUrl(url, QUrl::StrictMode); - QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); - QNetworkRequest request(modelsURL); - QNetworkReply* reply = networkAccessManager.get(request); - connect(reply, &QNetworkReply::finished, [this, reply, modelsURL]() { - QNetworkReply::NetworkError networkError = reply->error(); - if (networkError == QNetworkReply::NoError) { - QByteArray contents = reply->readAll(); - replaceContentFromMessageData(contents); - } else { - qDebug() << "Error downloading JSON from specified file"; - } - }); - } else { - qDebug() << "Cannot perform octree file replacement since current persist file path is not yet known"; - } - } - } -} - -void OctreeServer::replaceContentFromMessageData(QByteArray content) { - //Assume we have compressed data - auto compressedOctree = content; - QByteArray jsonOctree; - - bool wasCompressed = gunzip(compressedOctree, jsonOctree); - if (!wasCompressed) { - // the source was not compressed, assume we were sent regular JSON data - jsonOctree = compressedOctree; - } - // check the JSON data to verify it is an object - if (QJsonDocument::fromJson(jsonOctree).isObject()) { - if (!wasCompressed) { - // source was not compressed, we compress it before we write it locally - gzip(jsonOctree, compressedOctree); - } - // write the compressed octree data to a special file - auto replacementFilePath = _persistAbsoluteFilePath.append(OctreePersistThread::REPLACEMENT_FILE_EXTENSION); - QFile replacementFile(replacementFilePath); - if (replacementFile.open(QIODevice::WriteOnly) && replacementFile.write(compressedOctree) != -1) { - // we've now written our replacement file, time to take the server down so it can - // process it when it comes back up - qInfo() << "Wrote octree replacement file to" << replacementFilePath << "- stopping server"; - setFinished(true); - } else { - qWarning() << "Could not write replacement octree data to file - refusing to process"; - } - } else { - qDebug() << "Received replacement octree file that is invalid - refusing to process"; - } -} - bool OctreeServer::readOptionBool(const QString& optionName, const QJsonObject& settingsSectionObject, bool& result) { result = false; // assume it doesn't exist bool optionAvailable = false; @@ -1119,7 +1041,19 @@ void OctreeServer::readConfiguration() { _persistFilePath = getMyDefaultPersistFilename(); } + // If persist filename does not exist, let's see if there is one beside the application binary + // If there is, let's copy it over to our target persist directory + QDir persistPath { _persistFilePath }; + _persistAbsoluteFilePath = persistPath.absolutePath(); + + if (persistPath.isRelative()) { + // if the domain settings passed us a relative path, make an absolute path that is relative to the + // default data directory + _persistAbsoluteFilePath = QDir(PathUtils::getAppDataFilePath("entities/")).absoluteFilePath(_persistFilePath); + } + qDebug() << "persistFilePath=" << _persistFilePath; + qDebug() << "persisAbsoluteFilePath=" << _persistAbsoluteFilePath; _persistAsFileType = "json.gz"; @@ -1200,20 +1134,90 @@ void OctreeServer::run() { } void OctreeServer::domainSettingsRequestComplete() { + if (_state != OctreeServerState::WaitingForDomainSettings) { + qCWarning(octree_server) << "Received domain settings after they have already been received"; + return; + } + + auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); + packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket"); + packetReceiver.registerListener(PacketType::OctreeDataNack, this, "handleOctreeDataNackPacket"); + + packetReceiver.registerListener(PacketType::OctreeDataFileReply, this, "handleOctreeDataFileReply"); + + qDebug(octree_server) << "Received domain settings"; + + readConfiguration(); + + _state = OctreeServerState::WaitingForOctreeDataNegotation; + + auto nodeList = DependencyManager::get(); + const DomainHandler& domainHandler = nodeList->getDomainHandler(); + + auto packet = NLPacket::create(PacketType::OctreeDataFileRequest, -1, true, false); + + OctreeUtils::RawOctreeData data; + qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath; + if (OctreeUtils::readOctreeDataInfoFromFile(_persistAbsoluteFilePath, &data)) { + qCDebug(octree_server) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")"; + packet->writePrimitive(true); + auto id = data.id.toRfc4122(); + packet->write(id); + packet->writePrimitive(data.version); + } else { + qCWarning(octree_server) << "No octree data found"; + packet->writePrimitive(false); + } + + qCDebug(octree_server) << "Sending request for octree data to DS"; + nodeList->sendPacket(std::move(packet), domainHandler.getSockAddr()); +} + +void OctreeServer::handleOctreeDataFileReply(QSharedPointer message) { + bool includesNewData; + message->readPrimitive(&includesNewData); + QByteArray replaceData; + if (includesNewData) { + replaceData = message->readAll(); + qDebug() << "Got reply to octree data file request, new data sent"; + } else { + qDebug() << "Got reply to octree data file request, current entity data is sufficient"; + + OctreeUtils::RawOctreeData data; + qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath; + if (OctreeUtils::readOctreeDataInfoFromFile(_persistAbsoluteFilePath, &data)) { + if (data.id.isNull()) { + qCDebug(octree_server) << "Current octree data has a null id, updating"; + data.id = QUuid::createUuid(); + data.version = 0; + + QFile file(_persistAbsoluteFilePath); + if (file.open(QIODevice::WriteOnly)) { + auto entityData = data.toByteArray(); + file.write(entityData); + file.close(); + } else { + qCDebug(octree_server) << "Failed to update octree data"; + } + } + } + } + beginRunning(replaceData); +} + +void OctreeServer::beginRunning(QByteArray replaceData) { + if (_state == OctreeServerState::Running) { + qCWarning(octree_server) << "Server is already running"; + return; + } + + _state = OctreeServerState::Running; auto nodeList = DependencyManager::get(); // we need to ask the DS about agents so we can ping/reply with them nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer }); - auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); - packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket"); - packetReceiver.registerListener(PacketType::OctreeDataNack, this, "handleOctreeDataNackPacket"); - packetReceiver.registerListener(PacketType::OctreeFileReplacement, this, "handleOctreeFileReplacement"); - packetReceiver.registerListener(PacketType::OctreeFileReplacementFromUrl, this, "handleOctreeFileReplacementFromURL"); - - readConfiguration(); - beforeRun(); // after payload has been processed connect(nodeList.data(), SIGNAL(nodeAdded(SharedNodePointer)), SLOT(nodeAdded(SharedNodePointer))); @@ -1233,17 +1237,6 @@ void OctreeServer::domainSettingsRequestComplete() { // if we want Persistence, set up the local file and persist thread if (_wantPersist) { - // If persist filename does not exist, let's see if there is one beside the application binary - // If there is, let's copy it over to our target persist directory - QDir persistPath { _persistFilePath }; - _persistAbsoluteFilePath = persistPath.absolutePath(); - - if (persistPath.isRelative()) { - // if the domain settings passed us a relative path, make an absolute path that is relative to the - // default data directory - _persistAbsoluteFilePath = QDir(PathUtils::getAppDataFilePath("entities/")).absoluteFilePath(_persistFilePath); - } - static const QString ENTITY_PERSIST_EXTENSION = ".json.gz"; // force the persist file to end with .json.gz @@ -1328,7 +1321,7 @@ void OctreeServer::domainSettingsRequestComplete() { // now set up PersistThread _persistThread = new OctreePersistThread(_tree, _persistAbsoluteFilePath, _backupDirectoryPath, _persistInterval, - _wantBackup, _settings, _debugTimestampNow, _persistAsFileType); + _wantBackup, _settings, _debugTimestampNow, _persistAsFileType, replaceData); _persistThread->initialize(true); } diff --git a/assignment-client/src/octree/OctreeServer.h b/assignment-client/src/octree/OctreeServer.h index 0eba914064..6f77920ee0 100644 --- a/assignment-client/src/octree/OctreeServer.h +++ b/assignment-client/src/octree/OctreeServer.h @@ -27,8 +27,18 @@ #include "OctreeServerConsts.h" #include "OctreeInboundPacketProcessor.h" +#include + +Q_DECLARE_LOGGING_CATEGORY(octree_server) + const int DEFAULT_PACKETS_PER_INTERVAL = 2000; // some 120,000 packets per second total +enum class OctreeServerState { + WaitingForDomainSettings, + WaitingForOctreeDataNegotation, + Running +}; + /// Handles assignments of type OctreeServer - sending octrees to various clients. class OctreeServer : public ThreadedAssignment, public HTTPRequestHandler { Q_OBJECT @@ -36,6 +46,8 @@ public: OctreeServer(ReceivedMessage& message); ~OctreeServer(); + OctreeServerState _state { OctreeServerState::WaitingForDomainSettings }; + /// allows setting of run arguments void setArguments(int argc, char** argv); @@ -137,8 +149,9 @@ private slots: void domainSettingsRequestComplete(); void handleOctreeQueryPacket(QSharedPointer message, SharedNodePointer senderNode); void handleOctreeDataNackPacket(QSharedPointer message, SharedNodePointer senderNode); - void handleOctreeFileReplacement(QSharedPointer message); - void handleOctreeFileReplacementFromURL(QSharedPointer message); + //void handleOctreeFileReplacement(QSharedPointer message); + //void handleOctreeFileReplacementFromURL(QSharedPointer message); + void handleOctreeDataFileReply(QSharedPointer message); void removeSendThread(); protected: @@ -159,11 +172,13 @@ protected: QString getFileLoadTime(); QString getConfiguration(); QString getStatusLink(); + + void beginRunning(QByteArray replaceData); UniqueSendThread createSendThread(const SharedNodePointer& node); virtual UniqueSendThread newSendThread(const SharedNodePointer& node); - void replaceContentFromMessageData(QByteArray content); + //void replaceContentFromMessageData(QByteArray content); int _argc; const char** _argv; diff --git a/assignment-client/src/scripts/EntityScriptServer.cpp b/assignment-client/src/scripts/EntityScriptServer.cpp index b4a6b3af93..60cb1e349b 100644 --- a/assignment-client/src/scripts/EntityScriptServer.cpp +++ b/assignment-client/src/scripts/EntityScriptServer.cpp @@ -178,7 +178,7 @@ void EntityScriptServer::updateEntityPPS() { int numRunningScripts = _entitiesScriptEngine->getNumRunningEntityScripts(); int pps; if (std::numeric_limits::max() / _entityPPSPerScript < numRunningScripts) { - qWarning() << QString("Integer multiplaction would overflow, clamping to maxint: %1 * %2").arg(numRunningScripts).arg(_entityPPSPerScript); + qWarning() << QString("Integer multiplication would overflow, clamping to maxint: %1 * %2").arg(numRunningScripts).arg(_entityPPSPerScript); pps = std::numeric_limits::max(); pps = std::min(_maxEntityPPS, pps); } else { diff --git a/domain-server/CMakeLists.txt b/domain-server/CMakeLists.txt index c1e275e4d3..0e958b9537 100644 --- a/domain-server/CMakeLists.txt +++ b/domain-server/CMakeLists.txt @@ -22,7 +22,17 @@ setup_memory_debugger() symlink_or_copy_directory_beside_target(${_SHOULD_SYMLINK_RESOURCES} "${CMAKE_CURRENT_SOURCE_DIR}/resources" "resources") # link the shared hifi libraries -link_hifi_libraries(embedded-webserver networking shared avatars) +link_hifi_libraries(embedded-webserver networking shared avatars octree) + +add_dependency_external_projects(quazip) + +find_package(QuaZip REQUIRED) +target_include_directories(${TARGET_NAME} SYSTEM PUBLIC ${QUAZIP_INCLUDE_DIRS}) +target_link_libraries(${TARGET_NAME} ${QUAZIP_LIBRARIES}) + +if (WIN32) + add_paths_to_fixup_libs(${QUAZIP_DLL_PATH}) +endif () # find OpenSSL find_package(OpenSSL REQUIRED) diff --git a/domain-server/src/DomainContentBackupManager.cpp b/domain-server/src/DomainContentBackupManager.cpp new file mode 100644 index 0000000000..0eca10f8af --- /dev/null +++ b/domain-server/src/DomainContentBackupManager.cpp @@ -0,0 +1,303 @@ +// +// DomainContentBackupManager.cpp +// libraries/octree/src +// +// Created by Brad Hefta-Gaub on 8/21/13. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "DomainServer.h" +#include "DomainContentBackupManager.h" +const int DomainContentBackupManager::DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds + +// Backup format looks like: daily_backup-TIMESTAMP.zip +const static QString DATETIME_FORMAT { "yyyy-MM-dd_HH-mm-ss" }; +const static QString DATETIME_FORMAT_RE("\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}"); + +void DomainContentBackupManager::addCreateBackupHandler(CreateBackupHandler handler) { + _backupHandlers.push_back(handler); +} + +DomainContentBackupManager::DomainContentBackupManager(const QString& backupDirectory, + const QJsonObject& settings, + int persistInterval, + bool debugTimestampNow) + : _backupDirectory(backupDirectory), + _persistInterval(persistInterval), + _initialLoadComplete(false), + _loadTimeUSecs(0), + _lastCheck(0), + _debugTimestampNow(debugTimestampNow), + _lastTimeDebug(0) { + parseSettings(settings); +} + +void DomainContentBackupManager::parseSettings(const QJsonObject& settings) { + qDebug() << settings << settings["backups"] << settings["backups"].isArray(); + if (settings["backups"].isArray()) { + const QJsonArray& backupRules = settings["backups"].toArray(); + qCDebug(domain_server) << "BACKUP RULES:"; + + for (const QJsonValue& value : backupRules) { + QJsonObject obj = value.toObject(); + + int interval = 0; + int count = 0; + + QJsonValue intervalVal = obj["backupInterval"]; + if (intervalVal.isString()) { + interval = intervalVal.toString().toInt(); + } else { + interval = intervalVal.toInt(); + } + + QJsonValue countVal = obj["maxBackupVersions"]; + if (countVal.isString()) { + count = countVal.toString().toInt(); + } else { + count = countVal.toInt(); + } + + auto name = obj["Name"].toString(); + auto format = obj["format"].toString(); + format = name.replace(" ", "_").toLower() + "-"; + + qCDebug(domain_server) << " Name:" << name; + qCDebug(domain_server) << " format:" << format; + qCDebug(domain_server) << " interval:" << interval; + qCDebug(domain_server) << " count:" << count; + + BackupRule newRule = { name, interval, format, count, 0 }; + + newRule.lastBackupSeconds = getMostRecentBackupTimeInSecs(format); + + if (newRule.lastBackupSeconds > 0) { + auto now = QDateTime::currentSecsSinceEpoch(); + auto sinceLastBackup = now - newRule.lastBackupSeconds; + qCDebug(domain_server).noquote() << " lastBackup:" << formatSecTime(sinceLastBackup) << "ago"; + } else { + qCDebug(domain_server) << " lastBackup: NEVER"; + } + + _backupRules << newRule; + } + } else { + qCDebug(domain_server) << "BACKUP RULES: NONE"; + } +} + +int64_t DomainContentBackupManager::getMostRecentBackupTimeInSecs(const QString& format) { + int64_t mostRecentBackupInSecs = 0; + + QString mostRecentBackupFileName; + QDateTime mostRecentBackupTime; + + bool recentBackup = getMostRecentBackup(format, mostRecentBackupFileName, mostRecentBackupTime); + + if (recentBackup) { + mostRecentBackupInSecs = mostRecentBackupTime.toSecsSinceEpoch(); + } + + return mostRecentBackupInSecs; +} + +bool DomainContentBackupManager::process() { + if (isStillRunning()) { + constexpr int64_t MSECS_TO_USECS = 1000; + constexpr int64_t USECS_TO_SLEEP = 10 * MSECS_TO_USECS; // every 10ms + std::this_thread::sleep_for(std::chrono::microseconds(USECS_TO_SLEEP)); + + int64_t now = usecTimestampNow(); + int64_t sinceLastSave = now - _lastCheck; + int64_t intervalToCheck = _persistInterval * MSECS_TO_USECS; + + if (sinceLastSave > intervalToCheck) { + _lastCheck = now; + persist(); + } + } + + // if we were asked to debugTimestampNow do that now... + if (_debugTimestampNow) { + + quint64 now = usecTimestampNow(); + quint64 sinceLastDebug = now - _lastTimeDebug; + quint64 DEBUG_TIMESTAMP_INTERVAL = 600000000; // every 10 minutes + + if (sinceLastDebug > DEBUG_TIMESTAMP_INTERVAL) { + _lastTimeDebug = usecTimestampNow(true); // ask for debug output + } + } + + return isStillRunning(); +} + +void DomainContentBackupManager::aboutToFinish() { + qCDebug(domain_server) << "Persist thread about to finish..."; + persist(); +} + +void DomainContentBackupManager::persist() { + QDir backupDir { _backupDirectory }; + backupDir.mkpath("."); + + // create our "lock" file to indicate we're saving. + QString lockFileName = _backupDirectory + "/running.lock"; + + std::ofstream lockFile(qPrintable(lockFileName), std::ios::out | std::ios::binary); + if (lockFile.is_open()) { + backup(); + + lockFile.close(); + remove(qPrintable(lockFileName)); + } +} + +bool DomainContentBackupManager::getMostRecentBackup(const QString& format, + QString& mostRecentBackupFileName, + QDateTime& mostRecentBackupTime) { + QRegExp formatRE { QRegExp::escape(format) + "(" + DATETIME_FORMAT_RE + ")" + "\\.zip" }; + + QStringList filters; + filters << format + "*.zip"; + + bool bestBackupFound = false; + QString bestBackupFile; + QDateTime bestBackupFileTime; + + // Iterate over all of the backup files in the persist location + QDirIterator dirIterator(_backupDirectory, filters, QDir::Files | QDir::NoSymLinks, QDirIterator::NoIteratorFlags); + while (dirIterator.hasNext()) { + dirIterator.next(); + auto fileName = dirIterator.fileInfo().fileName(); + + if (formatRE.exactMatch(fileName)) { + auto datetime = formatRE.cap(1); + auto createdAt = QDateTime::fromString(datetime, DATETIME_FORMAT); + + if (!createdAt.isValid()) { + qDebug() << "Skipping backup with invalid timestamp: " << datetime; + continue; + } + + qDebug() << "Checking " << dirIterator.fileInfo().filePath(); + + // Based on last modified date, track the most recently modified file as the best backup + if (createdAt > bestBackupFileTime) { + bestBackupFound = true; + bestBackupFile = dirIterator.filePath(); + bestBackupFileTime = createdAt; + } + } else { + qDebug() << "NO match: " << fileName << formatRE; + } + } + + // If we found a backup then return the results + if (bestBackupFound) { + mostRecentBackupFileName = bestBackupFile; + mostRecentBackupTime = bestBackupFileTime; + } + return bestBackupFound; +} + +void DomainContentBackupManager::removeOldBackupVersions(const BackupRule& rule) { + QDir backupDir { _backupDirectory }; + if (backupDir.exists() && rule.maxBackupVersions > 0) { + qCDebug(domain_server) << "Rolling old backup versions for rule" << rule.name << "..."; + + auto matchingFiles = + backupDir.entryInfoList({ rule.extensionFormat + "*.zip" }, QDir::Files | QDir::NoSymLinks, QDir::Name); + + int backupsToDelete = matchingFiles.length() - rule.maxBackupVersions; + for (int i = 0; i < backupsToDelete; ++i) { + auto fileInfo = matchingFiles[i].absoluteFilePath(); + QFile backupFile(fileInfo); + if (backupFile.remove()) { + qCDebug(domain_server) << "Removed old backup: " << backupFile.fileName(); + } else { + qCDebug(domain_server) << "Failed to remove old backup: " << backupFile.fileName(); + } + } + + qCDebug(domain_server) << "Done rolling old backup versions..."; + } else { + qCDebug(domain_server) << "Rolling backups for rule" << rule.name << "." + << " Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]." + << " No need to roll backups..."; + } +} + +void DomainContentBackupManager::backup() { + auto nowDateTime = QDateTime::currentDateTime(); + auto nowSeconds = nowDateTime.toSecsSinceEpoch(); + + for (BackupRule& rule : _backupRules) { + auto secondsSinceLastBackup = nowSeconds - rule.lastBackupSeconds; + + qCDebug(domain_server) << "Checking [" << rule.name << "] - Time since last backup [" << secondsSinceLastBackup + << "] " + << "compared to backup interval [" << rule.intervalSeconds << "]..."; + + if (secondsSinceLastBackup > rule.intervalSeconds) { + qCDebug(domain_server) << "Time since last backup [" << secondsSinceLastBackup << "] for rule [" << rule.name + << "] exceeds backup interval [" << rule.intervalSeconds << "] doing backup now..."; + + auto timestamp = QDateTime::currentDateTime().toString(DATETIME_FORMAT); + auto fileName = "backup-" + rule.extensionFormat + timestamp + ".zip"; + auto zip = new QuaZip(_backupDirectory + "/" + fileName); + zip->open(QuaZip::mdAdd); + + for (auto& handler : _backupHandlers) { + handler(zip); + } + + zip->close(); + + qDebug() << "Created backup: " << fileName; + + removeOldBackupVersions(rule); + + if (rule.maxBackupVersions > 0) { + // Execute backup + auto result = true; + if (result) { + qCDebug(domain_server) << "DONE backing up persist file..."; + rule.lastBackupSeconds = nowSeconds; + } else { + qCDebug(domain_server) << "ERROR in backing up persist file..."; + perror("ERROR in backing up persist file"); + } + } else { + qCDebug(domain_server) << "This backup rule" << rule.name << " has Max Rolled Backup Versions less than 1 [" + << rule.maxBackupVersions << "]." + << " There are no backups to be done..."; + } + } else { + qCDebug(domain_server) << "Backup not needed for this rule [" << rule.name << "]..."; + } + } +} diff --git a/domain-server/src/DomainContentBackupManager.h b/domain-server/src/DomainContentBackupManager.h new file mode 100644 index 0000000000..20408fe486 --- /dev/null +++ b/domain-server/src/DomainContentBackupManager.h @@ -0,0 +1,88 @@ +// +// DomainContentBackupManager.h +// libraries/octree/src +// +// Created by Brad Hefta-Gaub on 8/21/13. +// Copyright 2013 High Fidelity, Inc. +// +// +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_DomainContentBackupManager_h +#define hifi_DomainContentBackupManager_h + +#include +#include +#include + +#include +#include + +#include + +using BackupResult = std::vector; +using CreateBackupHandler = std::function; +using RecoverBackupHandler = std::function; + +class DomainContentBackupManager : public GenericThread { + Q_OBJECT +public: + class BackupRule { + public: + QString name; + int intervalSeconds; + QString extensionFormat; + int maxBackupVersions; + qint64 lastBackupSeconds; + }; + + static const int DEFAULT_PERSIST_INTERVAL; + + DomainContentBackupManager(const QString& rootBackupDirectory, + const QJsonObject& settings, + int persistInterval = DEFAULT_PERSIST_INTERVAL, + bool debugTimestampNow = false); + + void addCreateBackupHandler(CreateBackupHandler handler); + bool isInitialLoadComplete() const { return _initialLoadComplete; } + int64_t getLoadElapsedTime() const { return _loadTimeUSecs; } + + void aboutToFinish(); /// call this to inform the persist thread that the owner is about to finish to support final persist + + void replaceData(QByteArray data); + +signals: + void loadCompleted(); + +protected: + /// Implements generic processing behavior for this thread. + bool process() override; + + void persist(); + void backup(); + void removeOldBackupVersions(const BackupRule& rule); + bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime); + int64_t getMostRecentBackupTimeInSecs(const QString& format); + void parseSettings(const QJsonObject& settings); + +private: + QString _backupDirectory; + std::vector _backupHandlers; + int _persistInterval; + bool _initialLoadComplete; + + int64_t _loadTimeUSecs; + + time_t _lastPersistTime; + int64_t _lastCheck; + bool _wantBackup{ true }; + QVector _backupRules; + + bool _debugTimestampNow; + int64_t _lastTimeDebug; +}; + +#endif // hifi_DomainContentBackupManager_h diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 68a36195d9..e083710d35 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -47,7 +48,14 @@ #include "DomainServerNodeData.h" #include "NodeConnectionData.h" +#include + +#include + +Q_LOGGING_CATEGORY(domain_server, "hifi.domain_server") + const QString ACCESS_TOKEN_KEY_PATH = "metaverse.access_token"; +const QString DomainServer::REPLACEMENT_FILE_EXTENSION = ".replace"; int const DomainServer::EXIT_CODE_REBOOT = 234923; @@ -280,6 +288,30 @@ DomainServer::DomainServer(int argc, char* argv[]) : qDebug() << "Ignoring subnet in whitelist, invalid ip portion: " << subnet; } } + + qDebug() << "Starting persist thread"; + if (QDir(getEntitiesDirPath()).mkpath(".")) { + qCDebug(domain_server) << "Created entities data directory"; + } + maybeHandleReplacementEntityFile(); + auto entitiesFilePath = getEntitiesFilePath(); + _contentManager.reset(new DomainContentBackupManager(getContentBackupDir(), _settingsManager.responseObjectForType("6")["entity_server_settings"].toObject())); + _contentManager->addCreateBackupHandler([entitiesFilePath](QuaZip* zip) { + qDebug() << "Creating a backup from handler"; + + QFile entitiesFile { entitiesFilePath }; + + if (entitiesFile.open(QIODevice::ReadOnly)) { + QuaZipFile zipFile { zip }; + zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo("models.json.gz", entitiesFilePath)); + zipFile.write(entitiesFile.readAll()); + zipFile.close(); + if (zipFile.getZipError() != UNZ_OK) { + qDebug() << "Failed to write entities file to backup:" << zipFile.getZipError(); + } + } + }); + _contentManager->initialize(true); } void DomainServer::parseCommandLine() { @@ -352,6 +384,11 @@ DomainServer::~DomainServer() { // destroy the LimitedNodeList before the DomainServer QCoreApplication is down DependencyManager::destroy(); + + if (_contentManager) { + _contentManager->aboutToFinish(); + _contentManager->terminating(); + } } void DomainServer::queuedQuit(QString quitMessage, int exitCode) { @@ -691,6 +728,12 @@ void DomainServer::setupNodeListAndAssignments() { packetReceiver.registerListener(PacketType::ICEServerHeartbeatDenied, this, "processICEServerHeartbeatDenialPacket"); packetReceiver.registerListener(PacketType::ICEServerHeartbeatACK, this, "processICEServerHeartbeatACK"); + packetReceiver.registerListener(PacketType::OctreeDataFileRequest, this, "processOctreeDataRequestMessage"); + packetReceiver.registerListener(PacketType::OctreeDataPersist, this, "processOctreeDataPersistMessage"); + + packetReceiver.registerListener(PacketType::OctreeFileReplacement, this, "handleOctreeFileReplacementRequest"); + packetReceiver.registerListener(PacketType::OctreeFileReplacementFromUrl, this, "handleOctreeFileReplacementFromURLRequest"); + // set a custom packetVersionMatch as the verify packet operator for the udt::Socket nodeList->setPacketFilterOperator(&DomainServer::isPacketVerified); @@ -1605,6 +1648,7 @@ void DomainServer::sendHeartbeatToIceServer() { qWarning() << "Waiting for keypair generation to complete before sending ICE heartbeat."; if (!limitedNodeList->getSessionUUID().isNull()) { + qDebug() << "generating keypair"; accountManager->generateNewDomainKeypair(limitedNodeList->getSessionUUID()); } else { qWarning() << "Attempting to send ICE server heartbeat with no domain ID. This is not supported"; @@ -1695,10 +1739,88 @@ void DomainServer::sendHeartbeatToIceServer() { } else { qDebug() << "Not sending ice-server heartbeat since there is no selected ice-server."; qDebug() << "Waiting for" << _iceServerAddr << "host lookup response"; - } } +void DomainServer::processOctreeDataPersistMessage(QSharedPointer message) { + qDebug() << "Received octree data persist message"; + auto data = message->readAll(); + auto filePath = getEntitiesFilePath(); + + QFile f(filePath); + if (f.open(QIODevice::WriteOnly)) { + f.write(data); + OctreeUtils::RawOctreeData octreeData; + if (OctreeUtils::readOctreeDataInfoFromData(data, &octreeData)) { + qCDebug(domain_server) << "Wrote new entiteis file" << octreeData.id << octreeData.version; + } else { + qCDebug(domain_server) << "Failed to read new octree data info"; + } + } else { + qCDebug(domain_server) << "Failed to write new entities file"; + } +} + +QString DomainServer::getContentBackupDir() { + return PathUtils::getAppDataFilePath("backup"); +} + +QString DomainServer::getEntitiesDirPath() { + return PathUtils::getAppDataFilePath("entities"); +} + +QString DomainServer::getEntitiesFilePath() { + return PathUtils::getAppDataFilePath("entities/models.json.gz"); +} + +QString DomainServer::getEntitiesReplacementFilePath() { + return getEntitiesFilePath().append(REPLACEMENT_FILE_EXTENSION); +} + +void DomainServer::processOctreeDataRequestMessage(QSharedPointer message) { + qDebug() << "Got request for octree data from " << message->getSenderSockAddr(); + + bool remoteHasExistingData { false }; + QUuid id; + int version; + message->readPrimitive(&remoteHasExistingData); + if (remoteHasExistingData) { + auto idData = message->read(16); + id = QUuid::fromRfc4122(idData); + message->readPrimitive(&version); + qCDebug(domain_server) << "Entity server does have existing data: ID(" << id << ") DataVersion(" << version << ")"; + } else { + qCDebug(domain_server) << "Entity server does not have existing data"; + } + auto entityFilePath = getEntitiesFilePath(); + + //QFile file(entityFilePath); + auto reply = NLPacketList::create(PacketType::OctreeDataFileReply, QByteArray(), true, true); + OctreeUtils::RawOctreeData data; + if (OctreeUtils::readOctreeDataInfoFromFile(entityFilePath, &data)) { + if (data.id == id && data.version <= version) { + qCDebug(domain_server) << "ES has sufficient octree data, not sending data"; + reply->writePrimitive(false); + } else { + qCDebug(domain_server) << "Sending newer octree data to ES"; + QFile file(entityFilePath); + if (file.open(QIODevice::ReadOnly)) { + reply->writePrimitive(true); + reply->write(file.readAll()); + } else { + qCDebug(domain_server) << "Unable to load entity file"; + reply->writePrimitive(false); + } + } + } else { + qCDebug(domain_server) << "Domain server does not have valid octree data"; + reply->writePrimitive(false); + } + + auto nodeList = DependencyManager::get(); + nodeList->sendPacketList(std::move(reply), message->getSenderSockAddr()); +} + void DomainServer::processNodeJSONStatsPacket(QSharedPointer packetList, SharedNodePointer sendingNode) { auto nodeData = static_cast(sendingNode->getLinkedData()); if (nodeData) { @@ -3105,9 +3227,64 @@ void DomainServer::setupGroupCacheRefresh() { } } +void DomainServer::maybeHandleReplacementEntityFile() { + QFile replacementFile(getEntitiesReplacementFilePath()); + if (replacementFile.exists()) { + qCDebug(domain_server) << "Replacing existing entity date with replacement file"; + QFile currentFile(getEntitiesFilePath()); + if (currentFile.exists()) { + if (currentFile.remove()) { + qCDebug(domain_server) << "Removed existing entity file"; + } else { + qCWarning(domain_server) << "Failled to remove existing entity file"; + } + } + if (replacementFile.rename(getEntitiesFilePath())) { + qCDebug(domain_server) << "Successfully updated entities data file with replacement file"; + } else { + qCWarning(domain_server) << "Failed to update entities data file with replacement file"; + } + } +} + void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) { // enumerate the nodes and find any octree type servers with active sockets + //Assume we have compressed data + auto compressedOctree = octreeFile; + QByteArray jsonOctree; + + bool wasCompressed = gunzip(compressedOctree, jsonOctree); + if (!wasCompressed) { + // the source was not compressed, assume we were sent regular JSON data + jsonOctree = compressedOctree; + } + + OctreeUtils::RawOctreeData data; + if (OctreeUtils::readOctreeDataInfoFromData(jsonOctree, &data)) { + data.id = QUuid::createUuid(); + data.version = 0; + + gzip(data.toByteArray(), compressedOctree); + + // write the compressed octree data to a special file + auto replacementFilePath = getEntitiesReplacementFilePath(); + QFile replacementFile(replacementFilePath); + if (replacementFile.open(QIODevice::WriteOnly) && replacementFile.write(compressedOctree) != -1) { + // we've now written our replacement file, time to take the server down so it can + // process it when it comes back up + qInfo() << "Wrote octree replacement file to" << replacementFilePath << "- stopping server"; + + QMetaObject::invokeMethod(this, "restart", Qt::QueuedConnection); + } else { + qWarning() << "Could not write replacement octree data to file - refusing to process"; + } + } else { + qDebug() << "Received replacement octree file that is invalid - refusing to process"; + } + + + return; auto limitedNodeList = DependencyManager::get(); limitedNodeList->eachMatchingNode([](const SharedNodePointer& node) { return node->getType() == NodeType::EntityServer && node->getActiveSocket(); @@ -3121,3 +3298,37 @@ void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) { limitedNodeList->sendPacketList(std::move(octreeFilePacketList), *octreeNode); }); } + +void DomainServer::handleOctreeFileReplacementFromURLRequest(QSharedPointer message) { + qInfo() << "Received request to replace content from a url"; + auto node = DependencyManager::get()->findNodeWithAddr(message->getSenderSockAddr()); + if (node) { + qDebug() << "Found node: " << node->getCanReplaceContent(); + } + if (node->getCanReplaceContent()) { + // Convert message data into our URL + QString url(message->getMessage()); + QUrl modelsURL = QUrl(url, QUrl::StrictMode); + QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); + QNetworkRequest request(modelsURL); + QNetworkReply* reply = networkAccessManager.get(request); + connect(reply, &QNetworkReply::finished, [this, reply, modelsURL]() { + QNetworkReply::NetworkError networkError = reply->error(); + if (networkError == QNetworkReply::NoError) { + handleOctreeFileReplacement(reply->readAll()); + } else { + qDebug() << "Error downloading JSON from specified file: " << modelsURL; + } + }); + } +} + + + + +void DomainServer::handleOctreeFileReplacementRequest(QSharedPointer message) { + auto node = DependencyManager::get()->nodeWithUUID(message->getSourceID()); + if (node->getCanReplaceContent()) { + handleOctreeFileReplacement(message->readAll()); + } +} diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index c7d779b394..ee0350665e 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -32,9 +32,14 @@ #include "DomainServerSettingsManager.h" #include "DomainServerWebSessionData.h" #include "WalletTransaction.h" +#include "DomainContentBackupManager.h" #include "PendingAssignedNodeData.h" +#include + +Q_DECLARE_LOGGING_CATEGORY(domain_server) + typedef QSharedPointer SharedAssignmentPointer; typedef QMultiHash TransactionHash; @@ -65,6 +70,8 @@ public: bool handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler = false) override; bool handleHTTPSRequest(HTTPSConnection* connection, const QUrl& url, bool skipSubHandler = false) override; + static const QString REPLACEMENT_FILE_EXTENSION; + public slots: /// Called by NodeList to inform us a node has been added void nodeAdded(SharedNodePointer node); @@ -84,6 +91,13 @@ private slots: void processICEServerHeartbeatDenialPacket(QSharedPointer message); void processICEServerHeartbeatACK(QSharedPointer message); + void handleOctreeFileReplacementFromURLRequest(QSharedPointer message); + void handleOctreeFileReplacementRequest(QSharedPointer message); + void handleOctreeFileReplacement(QByteArray octreeFile); + + void processOctreeDataRequestMessage(QSharedPointer message); + void processOctreeDataPersistMessage(QSharedPointer message); + void setupPendingAssignmentCredits(); void sendPendingTransactionsToServer(); @@ -91,8 +105,7 @@ private slots: void sendHeartbeatToMetaverse() { sendHeartbeatToMetaverse(QString()); } void sendHeartbeatToIceServer(); - void handleConnectedNode(SharedNodePointer newNode); - + void handleConnectedNode(SharedNodePointer newNode); void handleTempDomainSuccess(QNetworkReply& requestReply); void handleTempDomainError(QNetworkReply& requestReply); @@ -109,8 +122,6 @@ private slots: void handleSuccessfulICEServerAddressUpdate(QNetworkReply& requestReply); void handleFailedICEServerAddressUpdate(QNetworkReply& requestReply); - void handleOctreeFileReplacement(QByteArray octreeFile); - void updateReplicatedNodes(); void updateDownstreamNodes(); void updateUpstreamNodes(); @@ -127,6 +138,13 @@ private: const QUuid& getID(); void parseCommandLine(); + QString getContentBackupDir(); + QString getEntitiesDirPath(); + QString getEntitiesFilePath(); + QString getEntitiesReplacementFilePath(); + + void maybeHandleReplacementEntityFile(); + void setupNodeListAndAssignments(); bool optionallySetupOAuth(); bool optionallyReadX509KeyAndCertificate(); @@ -252,6 +270,8 @@ private: bool _sendICEServerAddressToMetaverseAPIInProgress { false }; bool _sendICEServerAddressToMetaverseAPIRedo { false }; + std::unique_ptr _contentManager { nullptr }; + QHash> _pendingOAuthConnections; QThread _assetClientThread; diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 5a340f471e..c031c0e8d4 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -6240,13 +6240,15 @@ bool Application::askToReplaceDomainContent(const QString& url) { // Given confirmation, send request to domain server to replace content qCDebug(interfaceapp) << "Attempting to replace domain content: " << url; QByteArray urlData(url.toUtf8()); - auto limitedNodeList = DependencyManager::get(); + auto limitedNodeList = DependencyManager::get(); + const auto& domainHandler = limitedNodeList->getDomainHandler(); limitedNodeList->eachMatchingNode([](const SharedNodePointer& node) { return node->getType() == NodeType::EntityServer && node->getActiveSocket(); - }, [&urlData, limitedNodeList](const SharedNodePointer& octreeNode) { + }, [&urlData, limitedNodeList, &domainHandler](const SharedNodePointer& octreeNode) { auto octreeFilePacket = NLPacket::create(PacketType::OctreeFileReplacementFromUrl, urlData.size(), true); octreeFilePacket->write(urlData); - limitedNodeList->sendPacket(std::move(octreeFilePacket), *octreeNode); + qDebug() << "WRiting url data: " << urlData; + limitedNodeList->sendPacket(std::move(octreeFilePacket), domainHandler.getSockAddr()); }); auto addressManager = DependencyManager::get(); addressManager->handleLookupString(DOMAIN_SPAWNING_POINT); diff --git a/interface/src/ui/DomainConnectionModel.cpp b/interface/src/ui/DomainConnectionModel.cpp index b9e4c1348e..83aa18420c 100644 --- a/interface/src/ui/DomainConnectionModel.cpp +++ b/interface/src/ui/DomainConnectionModel.cpp @@ -98,4 +98,4 @@ void DomainConnectionModel::refresh() { //inform view that we want refresh data beginResetModel(); endResetModel(); -} \ No newline at end of file +} diff --git a/libraries/entities/src/EntityTree.cpp b/libraries/entities/src/EntityTree.cpp index 60bcc85575..4f96a6d072 100644 --- a/libraries/entities/src/EntityTree.cpp +++ b/libraries/entities/src/EntityTree.cpp @@ -2244,6 +2244,8 @@ bool EntityTree::writeToMap(QVariantMap& entityDescription, OctreeElementPointer if (! entityDescription.contains("Entities")) { entityDescription["Entities"] = QVariantList(); } + entityDescription["DataVersion"] = ++_persistDataVersion; + entityDescription["Id"] = _persistID; QScriptEngine scriptEngine; RecurseOctreeToMapOperator theOperator(entityDescription, element, &scriptEngine, skipDefaultValues, skipThoseWithBadParents, _myAvatar); @@ -2256,6 +2258,14 @@ bool EntityTree::readFromMap(QVariantMap& map) { int contentVersion = map["Version"].toInt(); bool needsConversion = (contentVersion < (int)EntityVersion::ZoneLightInheritModes); + if (map.contains("Id")) { + _persistID = map["Id"].toUuid(); + } + + if (map.contains("DataVersion")) { + _persistDataVersion = map["DataVersion"].toInt(); + } + // map will have a top-level list keyed as "Entities". This will be extracted // and iterated over. Each member of this list is converted to a QVariantMap, then // to a QScriptValue, and then to EntityItemProperties. These properties are used diff --git a/libraries/image/CMakeLists.txt b/libraries/image/CMakeLists.txt index 442fa714b3..e6a1856327 100644 --- a/libraries/image/CMakeLists.txt +++ b/libraries/image/CMakeLists.txt @@ -5,7 +5,8 @@ link_hifi_libraries(shared gpu) if (NOT ANDROID) add_dependency_external_projects(nvtt) find_package(NVTT REQUIRED) + target_include_directories(${TARGET_NAME} PRIVATE ${NVTT_INCLUDE_DIRS}) target_link_libraries(${TARGET_NAME} ${NVTT_LIBRARIES}) add_paths_to_fixup_libs(${NVTT_DLL_PATH}) -endif() \ No newline at end of file +endif() diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 2343695914..3516fe948a 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -326,6 +326,7 @@ bool LimitedNodeList::packetSourceAndHashMatchAndTrackBandwidth(const udt::Packe static QMultiMap hashDebugSuppressMap; if (!hashDebugSuppressMap.contains(sourceID, headerType)) { + qCDebug(networking) << packetHeaderHash << expectedHash; qCDebug(networking) << "Packet hash mismatch on" << headerType << "- Sender" << sourceID; hashDebugSuppressMap.insert(sourceID, headerType); diff --git a/libraries/networking/src/ThreadedAssignment.h b/libraries/networking/src/ThreadedAssignment.h index 8b35acaac5..007e41a543 100644 --- a/libraries/networking/src/ThreadedAssignment.h +++ b/libraries/networking/src/ThreadedAssignment.h @@ -18,8 +18,6 @@ #include "Assignment.h" -using DownstreamNodeFoundCallback = std::function; - class ThreadedAssignment : public Assignment { Q_OBJECT public: @@ -47,10 +45,10 @@ protected: QTimer _domainServerTimer; QTimer _statsTimer; int _numQueuedCheckIns { 0 }; - + protected slots: void domainSettingsRequestFailed(); - + private slots: void checkInWithDomainServerOrExit(); }; diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 5757cea496..7cd02608a1 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -126,6 +126,11 @@ public: EntityScriptCallMethod, ChallengeOwnershipRequest, ChallengeOwnershipReply, + + OctreeDataFileRequest, + OctreeDataFileReply, + OctreeDataPersist, + NUM_PACKET_TYPE }; @@ -165,6 +170,8 @@ public: << PacketTypeEnum::Value::DomainConnectionDenied << PacketTypeEnum::Value::DomainServerPathQuery << PacketTypeEnum::Value::DomainServerPathResponse << PacketTypeEnum::Value::DomainServerAddedNode << PacketTypeEnum::Value::DomainServerConnectionToken << PacketTypeEnum::Value::DomainSettingsRequest + << PacketTypeEnum::Value::OctreeDataFileRequest << PacketTypeEnum::Value::OctreeDataFileReply + << PacketTypeEnum::Value::OctreeDataPersist << PacketTypeEnum::Value::OctreeFileReplacementFromUrl << PacketTypeEnum::Value::DomainSettings << PacketTypeEnum::Value::ICEServerPeerInformation << PacketTypeEnum::Value::ICEServerQuery << PacketTypeEnum::Value::ICEServerHeartbeat << PacketTypeEnum::Value::ICEServerHeartbeatACK << PacketTypeEnum::Value::ICEPing diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 55643985c8..019ae07c16 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -328,7 +328,7 @@ void Socket::checkForReadyReadBackup() { void Socket::readPendingDatagrams() { int packetSizeWithHeader = -1; - while ((packetSizeWithHeader = _udpSocket.pendingDatagramSize()) != -1) { + while ((packetSizeWithHeader = _udpSocket.pendingDatagramSize()) > 0) { // we're reading a packet so re-start the readyRead backup timer _readyReadBackupTimer->start(); @@ -517,7 +517,7 @@ void Socket::handleSocketError(QAbstractSocket::SocketError socketError) { static QString repeatedMessage = LogHandler::getInstance().addRepeatedMessageRegex(SOCKET_REGEX); - qCDebug(networking) << "udt::Socket error - " << socketError; + qCDebug(networking) << "udt::Socket error - " << socketError << _udpSocket.errorString(); } void Socket::handleStateChanged(QAbstractSocket::SocketState socketState) { diff --git a/libraries/octree/CMakeLists.txt b/libraries/octree/CMakeLists.txt index bea036add3..228779dbba 100644 --- a/libraries/octree/CMakeLists.txt +++ b/libraries/octree/CMakeLists.txt @@ -1,3 +1,4 @@ set(TARGET_NAME octree) +include_directories(system /usr/local/Cellar/qt5/5.9.1/include) setup_hifi_library() link_hifi_libraries(shared networking) diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp index c63ff2f560..23352a548c 100644 --- a/libraries/octree/src/Octree.cpp +++ b/libraries/octree/src/Octree.cpp @@ -1757,6 +1757,19 @@ bool Octree::readJSONFromStream(uint64_t streamLength, QDataStream& inputStream, QVariant asVariant = asDocument.toVariant(); QVariantMap asMap = asVariant.toMap(); bool success = readFromMap(asMap); + /* + if (success) { + if (asMap.contains("DataVersion") && asMap.contains("Id")) { + bool versionOk; + auto dataVersion = asMap["DataVersion"].toLongLong(&versionOk); + if (versionOk) { + auto id = asMap["Id"].toUuid(); + _persistDataVersion = dataVersion; + _persistID = id; + } + } + } + */ delete[] rawData; return success; } @@ -1778,11 +1791,9 @@ bool Octree::writeToFile(const char* fileName, const OctreeElementPointer& eleme return success; } -bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& element, bool doGzip) { +bool Octree::toJSON(QJsonDocument* doc, const OctreeElementPointer& element) { QVariantMap entityDescription; - qCDebug(octree, "Saving JSON SVO to file %s...", fileName); - OctreeElementPointer top; if (element) { top = element; @@ -1802,17 +1813,35 @@ bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& e return false; } - // convert the QVariantMap to JSON - QByteArray jsonData = QJsonDocument::fromVariant(entityDescription).toJson(); - QByteArray jsonDataForFile; + *doc = QJsonDocument::fromVariant(entityDescription); + return true; +} - if (doGzip) { - if (!gzip(jsonData, jsonDataForFile, -1)) { - qCritical("unable to gzip data while saving to json."); - return false; - } +bool Octree::toGzippedJSON(QByteArray* data, const OctreeElementPointer& element) { + QJsonDocument doc; + if (!toJSON(&doc, element)) { + qCritical("Failed to convert Entities to QVariantMap while converting to json."); + return false; + } + + QByteArray jsonData = doc.toJson(); + + if (!gzip(jsonData, *data, -1)) { + qCritical("Unable to gzip data while saving to json."); + return false; } else { - jsonDataForFile = jsonData; + qDebug() <<"Did gzip!"; + } + + return true; +} + +bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& element, bool doGzip) { + qCDebug(octree, "Saving JSON SVO to file %s...", fileName); + + QByteArray jsonDataForFile; + if (!toGzippedJSON(&jsonDataForFile)) { + return false; } QFile persistFile(fileName); @@ -1823,6 +1852,7 @@ bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& e qCritical("Could not write to JSON description of entities."); } + return success; } diff --git a/libraries/octree/src/Octree.h b/libraries/octree/src/Octree.h index 1648cb0f47..1b9495717b 100644 --- a/libraries/octree/src/Octree.h +++ b/libraries/octree/src/Octree.h @@ -283,8 +283,10 @@ public: void loadOctreeFile(const char* fileName); // Octree exporters - bool writeToFile(const char* filename, const OctreeElementPointer& element = NULL, QString persistAsFileType = "json.gz"); - bool writeToJSONFile(const char* filename, const OctreeElementPointer& element = NULL, bool doGzip = false); + bool toJSON(QJsonDocument* doc, const OctreeElementPointer& element = nullptr); + bool toGzippedJSON(QByteArray* data, const OctreeElementPointer& element = nullptr); + bool writeToFile(const char* filename, const OctreeElementPointer& element = nullptr, QString persistAsFileType = "json.gz"); + bool writeToJSONFile(const char* filename, const OctreeElementPointer& element = nullptr, bool doGzip = false); virtual bool writeToMap(QVariantMap& entityDescription, OctreeElementPointer element, bool skipDefaultValues, bool skipThoseWithBadParents) = 0; @@ -326,6 +328,11 @@ public: virtual void dumpTree() { } virtual void pruneTree() { } + void setEntityVersionInfo(QUuid id, int64_t dataVersion) { + _persistID = id; + _persistDataVersion = dataVersion; + } + virtual void resetEditStats() { } virtual quint64 getAverageDecodeTime() const { return 0; } virtual quint64 getAverageLookupTime() const { return 0; } @@ -359,6 +366,9 @@ protected: OctreeElementPointer _rootElement = nullptr; + QUuid _persistID { QUuid::createUuid() }; + int _persistDataVersion { 0 }; + bool _isDirty; bool _shouldReaverage; bool _stopImport; diff --git a/libraries/octree/src/OctreePersistThread.cpp b/libraries/octree/src/OctreePersistThread.cpp index ea6bd28fc4..9c9a4d40db 100644 --- a/libraries/octree/src/OctreePersistThread.cpp +++ b/libraries/octree/src/OctreePersistThread.cpp @@ -31,18 +31,19 @@ #include "OctreeLogging.h" #include "OctreePersistThread.h" +#include "OctreeUtils.h" const int OctreePersistThread::DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds -const QString OctreePersistThread::REPLACEMENT_FILE_EXTENSION = ".replace"; OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& filename, const QString& backupDirectory, int persistInterval, bool wantBackup, const QJsonObject& settings, bool debugTimestampNow, - QString persistAsFileType) : + QString persistAsFileType, const QByteArray& replacementData) : _tree(tree), _filename(filename), _backupDirectory(backupDirectory), _persistInterval(persistInterval), _initialLoadComplete(false), + _replacementData(replacementData), _loadTimeUSecs(0), _lastCheck(0), _wantBackup(wantBackup), @@ -52,6 +53,7 @@ OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& file { parseSettings(settings); + // in case the persist filename has an extension that doesn't match the file type QString sansExt = fileNameWithoutExtension(_filename, PERSIST_EXTENSIONS); _filename = sansExt + "." + _persistAsFileType; @@ -132,51 +134,56 @@ quint64 OctreePersistThread::getMostRecentBackupTimeInUsecs(const QString& forma return mostRecentBackupInUsecs; } -void OctreePersistThread::possiblyReplaceContent() { - // before we load the normal file, check if there's a pending replacement file - auto replacementFileName = _filename + REPLACEMENT_FILE_EXTENSION; +void OctreePersistThread::replaceData(QByteArray data) { + backupCurrentFile(); - QFile replacementFile { replacementFileName }; - if (replacementFile.exists()) { - // we have a replacement file to process - qDebug() << "Replacing models file with" << replacementFileName; - - // first take the current models file and move it to a different filename, appended with the timestamp - QFile currentFile { _filename }; - if (currentFile.exists()) { - static const QString FILENAME_TIMESTAMP_FORMAT = "yyyyMMdd-hhmmss"; - auto backupFileName = _filename + ".backup." + QDateTime::currentDateTime().toString(FILENAME_TIMESTAMP_FORMAT); - - if (currentFile.rename(backupFileName)) { - qDebug() << "Moved previous models file to" << backupFileName; - } else { - qWarning() << "Could not backup previous models file to" << backupFileName << "- removing replacement models file"; - - if (!replacementFile.remove()) { - qWarning() << "Could not remove replacement models file from" << replacementFileName - << "- replacement will be re-attempted on next server restart"; - return; - } - } - } - - // rename the replacement file to match what the persist thread is just about to read - if (!replacementFile.rename(_filename)) { - qWarning() << "Could not replace models file with" << replacementFileName << "- starting with empty models file"; - } + QFile currentFile { _filename }; + if (currentFile.open(QIODevice::WriteOnly)) { + currentFile.write(data); + qDebug() << "Wrote replacement data"; + } else { + qWarning() << "Failed to write replacement data"; } } +// Return true if current file is backed up successfully or doesn't exist. +bool OctreePersistThread::backupCurrentFile() { + // first take the current models file and move it to a different filename, appended with the timestamp + QFile currentFile { _filename }; + if (currentFile.exists()) { + static const QString FILENAME_TIMESTAMP_FORMAT = "yyyyMMdd-hhmmss"; + auto backupFileName = _filename + ".backup." + QDateTime::currentDateTime().toString(FILENAME_TIMESTAMP_FORMAT); + + if (currentFile.rename(backupFileName)) { + qDebug() << "Moved previous models file to" << backupFileName; + return true; + } else { + qWarning() << "Could not backup previous models file to" << backupFileName << "- removing replacement models file"; + return false; + } + } + return true; +} bool OctreePersistThread::process() { if (!_initialLoadComplete) { - possiblyReplaceContent(); - quint64 loadStarted = usecTimestampNow(); qCDebug(octree) << "loading Octrees from file: " << _filename << "..."; - bool persistantFileRead; + if (_replacementData.isNull()) { + sendLatestEntityDataToDS(); + } else { + replaceData(_replacementData); + _replacementData.clear(); + } + + OctreeUtils::RawOctreeData data; + if (OctreeUtils::readOctreeDataInfoFromFile(_filename, &data)) { + _tree->setEntityVersionInfo(data.id, data.version); + } + + bool persistentFileRead; _tree->withWriteLock([&] { PerformanceWarning warn(true, "Loading Octree File", true); @@ -199,7 +206,7 @@ bool OctreePersistThread::process() { qCDebug(octree) << "Loading Octree... lock file removed:" << lockFileName; } - persistantFileRead = _tree->readFromFile(qPrintable(_filename.toLocal8Bit())); + persistentFileRead = _tree->readFromFile(qPrintable(_filename.toLocal8Bit())); _tree->pruneTree(); }); @@ -207,7 +214,7 @@ bool OctreePersistThread::process() { _loadTimeUSecs = loadDone - loadStarted; _tree->clearDirtyBit(); // the tree is clean since we just loaded it - qCDebug(octree, "DONE loading Octrees from file... fileRead=%s", debug::valueOf(persistantFileRead)); + qCDebug(octree, "DONE loading Octrees from file... fileRead=%s", debug::valueOf(persistentFileRead)); unsigned long nodeCount = OctreeElement::getNodeCount(); unsigned long internalNodeCount = OctreeElement::getInternalNodeCount(); @@ -272,7 +279,6 @@ bool OctreePersistThread::process() { return isStillRunning(); // keep running till they terminate us } - void OctreePersistThread::aboutToFinish() { qCDebug(octree) << "Persist thread about to finish..."; persist(); @@ -319,6 +325,23 @@ void OctreePersistThread::persist() { remove(qPrintable(lockFileName)); qCDebug(octree) << "saving Octree lock file removed:" << lockFileName; } + + sendLatestEntityDataToDS(); + } +} + +void OctreePersistThread::sendLatestEntityDataToDS() { + qDebug() << "Sending latest entity data to DS"; + auto nodeList = DependencyManager::get(); + const DomainHandler& domainHandler = nodeList->getDomainHandler(); + + QByteArray data; + if (_tree->toGzippedJSON(&data)) { + auto message = NLPacketList::create(PacketType::OctreeDataPersist, QByteArray(), true, true); + message->write(data); + nodeList->sendPacketList(std::move(message), domainHandler.getSockAddr()); + } else { + qCWarning(octree) << "Failed to persist octree to DS"; } } @@ -453,7 +476,6 @@ void OctreePersistThread::rollOldBackupVersions(const BackupRule& rule) { } } - void OctreePersistThread::backup() { qCDebug(octree) << "backup operation wantBackup:" << _wantBackup; if (_wantBackup) { diff --git a/libraries/octree/src/OctreePersistThread.h b/libraries/octree/src/OctreePersistThread.h index 2441223467..3fdad2c3f7 100644 --- a/libraries/octree/src/OctreePersistThread.h +++ b/libraries/octree/src/OctreePersistThread.h @@ -18,7 +18,6 @@ #include #include "Octree.h" -/// Generalized threaded processor for handling received inbound packets. class OctreePersistThread : public GenericThread { Q_OBJECT public: @@ -32,11 +31,11 @@ public: }; static const int DEFAULT_PERSIST_INTERVAL; - static const QString REPLACEMENT_FILE_EXTENSION; OctreePersistThread(OctreePointer tree, const QString& filename, const QString& backupDirectory, int persistInterval = DEFAULT_PERSIST_INTERVAL, bool wantBackup = false, - const QJsonObject& settings = QJsonObject(), bool debugTimestampNow = false, QString persistAsFileType="json.gz"); + const QJsonObject& settings = QJsonObject(), bool debugTimestampNow = false, + QString persistAsFileType="json.gz", const QByteArray& replacementData = QByteArray()); bool isInitialLoadComplete() const { return _initialLoadComplete; } quint64 getLoadElapsedTime() const { return _loadTimeUSecs; } @@ -61,7 +60,10 @@ protected: bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime); quint64 getMostRecentBackupTimeInUsecs(const QString& format); void parseSettings(const QJsonObject& settings); - void possiblyReplaceContent(); + bool backupCurrentFile(); + + void replaceData(QByteArray data); + void sendLatestEntityDataToDS(); private: OctreePointer _tree; @@ -69,6 +71,7 @@ private: QString _backupDirectory; int _persistInterval; bool _initialLoadComplete; + QByteArray _replacementData; quint64 _loadTimeUSecs; diff --git a/libraries/octree/src/OctreeUtils.cpp b/libraries/octree/src/OctreeUtils.cpp index ca15324d4e..d8925a10ca 100644 --- a/libraries/octree/src/OctreeUtils.cpp +++ b/libraries/octree/src/OctreeUtils.cpp @@ -16,7 +16,11 @@ #include #include +#include +#include +#include +#include float calculateRenderAccuracy(const glm::vec3& position, const AABox& bounds, @@ -75,3 +79,76 @@ float getOrthographicAccuracySize(float octreeSizeScale, int boundaryLevelAdjust const float smallestSize = 0.01f; return (smallestSize * MAX_VISIBILITY_DISTANCE_FOR_UNIT_ELEMENT) / boundaryDistanceForRenderLevel(boundaryLevelAdjust, octreeSizeScale); } + +bool OctreeUtils::readOctreeFile(QString path, QJsonDocument* doc) { + QFile file(path); + if (!file.open(QIODevice::ReadOnly)) { + qCritical() << "Cannot open json file for reading: " << path; + return false; + } + + QByteArray data = file.readAll(); + QByteArray jsonData; + + if (path.endsWith(".json.gz")) { + if (!gunzip(data, jsonData)) { + qCritical() << "json File not in gzip format: " << path; + return false; + } + } else { + jsonData = data; + } + + *doc = QJsonDocument::fromJson(jsonData); + return !doc->isNull(); +} + +bool readOctreeDataInfoFromJSON(QJsonObject root, OctreeUtils::RawOctreeData* octreeData) { + if (root.contains("Id") && root.contains("DataVersion")) { + octreeData->id = root["Id"].toVariant().toUuid(); + octreeData->version = root["DataVersion"].toInt(); + } + if (root.contains("Entities")) { + octreeData->octreeData = root["Entities"].toArray(); + } + return true; +} + +bool OctreeUtils::readOctreeDataInfoFromData(QByteArray data, OctreeUtils::RawOctreeData* octreeData) { + QByteArray jsonData; + if (gunzip(data, jsonData)) { + data = jsonData; + } + + auto doc = QJsonDocument::fromJson(data); + if (doc.isNull()) { + return false; + } + + auto root = doc.object(); + return readOctreeDataInfoFromJSON(root, octreeData); +} + +bool OctreeUtils::readOctreeDataInfoFromFile(QString path, OctreeUtils::RawOctreeData* octreeData) { + QJsonDocument doc; + if (!OctreeUtils::readOctreeFile(path, &doc)) { + return false; + } + + auto root = doc.object(); + return readOctreeDataInfoFromJSON(root, octreeData); +} + +QByteArray OctreeUtils::RawOctreeData::toByteArray() { + QJsonObject obj { + { "DataVersion", QJsonValue((qint64)version) }, + { "Id", QJsonValue(id.toString()) }, + { "Version", QJsonValue(5) }, + { "Entities", octreeData } + }; + + QJsonDocument doc; + doc.setObject(obj); + + return doc.toJson(); +} diff --git a/libraries/octree/src/OctreeUtils.h b/libraries/octree/src/OctreeUtils.h index 0f87bb6f68..e5c7b617cd 100644 --- a/libraries/octree/src/OctreeUtils.h +++ b/libraries/octree/src/OctreeUtils.h @@ -14,7 +14,30 @@ #include "OctreeConstants.h" +#include +#include + class AABox; +class QJsonDocument; + +namespace OctreeUtils { + +// RawOctreeData is an intermediate format between JSON and a fully deserialized Octree. +class RawOctreeData { +public: + QUuid id { QUuid() }; + int64_t version { -1 }; + + QJsonArray octreeData; + + QByteArray toByteArray(); +}; + +bool readOctreeFile(QString path, QJsonDocument* doc); +bool readOctreeDataInfoFromData(QByteArray data, RawOctreeData* octreeData); +bool readOctreeDataInfoFromFile(QString path, RawOctreeData* octreeData); + +} /// renderAccuracy represents a floating point "visibility" of an object based on it's view from the camera. At a simple /// level it returns 0.0f for things that are so small for the current settings that they could not be visible. diff --git a/libraries/shared/src/SharedUtil.cpp b/libraries/shared/src/SharedUtil.cpp index 8e5c30711c..f46d0768c1 100644 --- a/libraries/shared/src/SharedUtil.cpp +++ b/libraries/shared/src/SharedUtil.cpp @@ -105,7 +105,7 @@ void usecTimestampNowForceClockSkew(qint64 clockSkew) { ::usecTimestampNowAdjust = clockSkew; } -static qint64 TIME_REFERENCE = 0; // in usec +static std::atomic TIME_REFERENCE { 0 }; // in usec static std::once_flag usecTimestampNowIsInitialized; static QElapsedTimer timestampTimer; @@ -771,6 +771,10 @@ QString formatUsecTime(double usecs) { return formatUsecTime(usecs); } +QString formatSecTime(qint64 secs) { + return formatUsecTime(secs * 1000000); +} + QString formatSecondsElapsed(float seconds) { QString result; diff --git a/libraries/shared/src/SharedUtil.h b/libraries/shared/src/SharedUtil.h index 5a1e48d9c0..7f9fb026f8 100644 --- a/libraries/shared/src/SharedUtil.h +++ b/libraries/shared/src/SharedUtil.h @@ -216,6 +216,7 @@ QString formatUsecTime(float usecs); QString formatUsecTime(double usecs); QString formatUsecTime(quint64 usecs); QString formatUsecTime(qint64 usecs); +QString formatSecTime(qint64 secs); QString formatSecondsElapsed(float seconds); bool similarStrings(const QString& stringA, const QString& stringB);