mirror of
https://github.com/overte-org/overte.git
synced 2025-04-05 21:12:25 +02:00
Add entity file sync and domain content backups
This commit is contained in:
Ryan Huffman
Atlante45
parent
c62f68264f
commit
cb9327e030
30 changed files with 1026 additions and 199 deletions
|
|
@ -1,12 +1,12 @@
|
|||
Language: Cpp
|
||||
Standard: Cpp11
|
||||
BasedOnStyle: "Chromium"
|
||||
BasedOnStyle: "Chromium"
|
||||
ColumnLimit: 128
|
||||
IndentWidth: 4
|
||||
UseTab: Never
|
||||
|
||||
BreakBeforeBraces: Custom
|
||||
BraceWrapping:
|
||||
BraceWrapping:
|
||||
AfterEnum: true
|
||||
AfterClass: false
|
||||
AfterControlStatement: false
|
||||
|
|
@ -21,11 +21,11 @@ BraceWrapping:
|
|||
|
||||
|
||||
AccessModifierOffset: -4
|
||||
AllowShortFunctionsOnASingleLine: InlineOnly
|
||||
BreakConstructorInitializers: BeforeColon
|
||||
AllowShortFunctionsOnASingleLine: InlineOnly
|
||||
BreakConstructorInitializers: BeforeColon
|
||||
BreakConstructorInitializersBeforeComma: true
|
||||
IndentCaseLabels: true
|
||||
ReflowComments: false
|
||||
ReflowComments: false
|
||||
Cpp11BracedListStyle: false
|
||||
ContinuationIndentWidth: 4
|
||||
ConstructorInitializerAllOnOneLineOrOnePerLine: false
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ setup_hifi_project(Core Gui Network Script Quick WebSockets)
|
|||
if (APPLE)
|
||||
set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "@executable_path/../Frameworks")
|
||||
endif ()
|
||||
set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "/testing/")
|
||||
|
||||
setup_memory_debugger()
|
||||
|
||||
|
|
|
|||
|
|
@ -340,7 +340,6 @@ void Agent::scriptRequestFinished() {
|
|||
request->deleteLater();
|
||||
}
|
||||
|
||||
|
||||
void Agent::executeScript() {
|
||||
_scriptEngine = scriptEngineFactory(ScriptEngine::AGENT_SCRIPT, _scriptContents, _payload);
|
||||
|
||||
|
|
|
|||
|
|
@ -116,7 +116,6 @@ void EntityServer::beforeRun() {
|
|||
void EntityServer::entityCreated(const EntityItem& newEntity, const SharedNodePointer& senderNode) {
|
||||
}
|
||||
|
||||
|
||||
// EntityServer will use the "special packets" to send list of recently deleted entities
|
||||
bool EntityServer::hasSpecialPacketsToSend(const SharedNodePointer& node) {
|
||||
bool shouldSendDeletedEntities = false;
|
||||
|
|
@ -277,7 +276,6 @@ int EntityServer::sendSpecialPackets(const SharedNodePointer& node, OctreeQueryN
|
|||
return totalBytes;
|
||||
}
|
||||
|
||||
|
||||
void EntityServer::pruneDeletedEntities() {
|
||||
EntityTreePointer tree = std::static_pointer_cast<EntityTree>(_tree);
|
||||
if (tree->hasAnyDeletedEntities()) {
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ struct ViewerSendingStats {
|
|||
class SimpleEntitySimulation;
|
||||
using SimpleEntitySimulationPointer = std::shared_ptr<SimpleEntitySimulation>;
|
||||
|
||||
|
||||
class EntityServer : public OctreeServer, public NewlyCreatedEntityHook {
|
||||
Q_OBJECT
|
||||
public:
|
||||
|
|
@ -38,7 +37,7 @@ public:
|
|||
~EntityServer();
|
||||
|
||||
// Subclasses must implement these methods
|
||||
virtual std::unique_ptr<OctreeQueryNode> createOctreeQueryNode() override ;
|
||||
virtual std::unique_ptr<OctreeQueryNode> createOctreeQueryNode() override;
|
||||
virtual char getMyNodeType() const override { return NodeType::EntityServer; }
|
||||
virtual PacketType getMyQueryMessageType() const override { return PacketType::EntityQuery; }
|
||||
virtual const char* getMyServerName() const override { return MODEL_SERVER_NAME; }
|
||||
|
|
@ -82,12 +81,12 @@ private:
|
|||
QReadWriteLock _viewerSendingStatsLock;
|
||||
QMap<QUuid, QMap<QUuid, ViewerSendingStats>> _viewerSendingStats;
|
||||
|
||||
static const int DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 45 * 60 * 1000; // 45m
|
||||
static const int DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 60 * 60 * 1000; // 1h
|
||||
int _MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS; // 45m
|
||||
int _MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS; // 1h
|
||||
static const int DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 45 * 60 * 1000; // 45m
|
||||
static const int DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 60 * 60 * 1000; // 1h
|
||||
int _MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS; // 45m
|
||||
int _MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS; // 1h
|
||||
QTimer _dynamicDomainVerificationTimer;
|
||||
void startDynamicDomainVerification();
|
||||
};
|
||||
|
||||
#endif // hifi_EntityServer_h
|
||||
#endif // hifi_EntityServer_h
|
||||
|
|
|
|||
|
|
@ -33,6 +33,10 @@
|
|||
#include <PathUtils.h>
|
||||
#include <QtCore/QDir>
|
||||
|
||||
#include <OctreeUtils.h>
|
||||
|
||||
Q_LOGGING_CATEGORY(octree_server, "hifi.octree-server")
|
||||
|
||||
int OctreeServer::_clientCount = 0;
|
||||
const int MOVING_AVERAGE_SAMPLE_COUNTS = 1000;
|
||||
|
||||
|
|
@ -84,6 +88,8 @@ int OctreeServer::_longProcessWait = 0;
|
|||
int OctreeServer::_shortProcessWait = 0;
|
||||
int OctreeServer::_noProcessWait = 0;
|
||||
|
||||
static const QString PERSIST_FILE_DOWNLOAD_PATH = "/models.json.gz";
|
||||
|
||||
|
||||
void OctreeServer::resetSendingStats() {
|
||||
_averageLoopTime.reset();
|
||||
|
|
@ -202,7 +208,6 @@ void OctreeServer::trackPacketSendingTime(float time) {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
void OctreeServer::trackProcessWaitTime(float time) {
|
||||
const float MAX_SHORT_TIME = 10.0f;
|
||||
const float MAX_LONG_TIME = 100.0f;
|
||||
|
|
@ -283,8 +288,6 @@ void OctreeServer::initHTTPManager(int port) {
|
|||
_httpManager = new HTTPManager(QHostAddress::AnyIPv4, port, documentRoot, this, this);
|
||||
}
|
||||
|
||||
const QString PERSIST_FILE_DOWNLOAD_PATH = "/models.json.gz";
|
||||
|
||||
bool OctreeServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler) {
|
||||
|
||||
#ifdef FORCE_CRASH
|
||||
|
|
@ -922,87 +925,6 @@ void OctreeServer::handleOctreeDataNackPacket(QSharedPointer<ReceivedMessage> me
|
|||
}
|
||||
}
|
||||
|
||||
void OctreeServer::handleOctreeFileReplacement(QSharedPointer<ReceivedMessage> message) {
|
||||
if (!_isFinished && !_isShuttingDown) {
|
||||
// these messages are only allowed to come from the domain server, so make sure that is the case
|
||||
auto nodeList = DependencyManager::get<NodeList>();
|
||||
if (message->getSenderSockAddr() == nodeList->getDomainHandler().getSockAddr()) {
|
||||
// it's far cleaner to load up the new content upon server startup
|
||||
// so here we just store a special file at our persist path
|
||||
// and then force a stop of the server so that it can pick it up when it relaunches
|
||||
if (!_persistAbsoluteFilePath.isEmpty()) {
|
||||
replaceContentFromMessageData(message->getMessage());
|
||||
} else {
|
||||
qDebug() << "Cannot perform octree file replacement since current persist file path is not yet known";
|
||||
}
|
||||
} else {
|
||||
qDebug() << "Received an octree file replacement that was not from our domain server - refusing to process";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Message->getMessage() contains a QByteArray representation of the URL to download from
|
||||
void OctreeServer::handleOctreeFileReplacementFromURL(QSharedPointer<ReceivedMessage> message) {
|
||||
qInfo() << "Received request to replace content from a url";
|
||||
if (!_isFinished && !_isShuttingDown) {
|
||||
// This call comes from Interface, so we skip our domain server check
|
||||
// but confirm that we have permissions to replace content sets
|
||||
if (DependencyManager::get<NodeList>()->getThisNodeCanReplaceContent()) {
|
||||
if (!_persistAbsoluteFilePath.isEmpty()) {
|
||||
// Convert message data into our URL
|
||||
QString url(message->getMessage());
|
||||
QUrl modelsURL = QUrl(url, QUrl::StrictMode);
|
||||
QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
|
||||
QNetworkRequest request(modelsURL);
|
||||
QNetworkReply* reply = networkAccessManager.get(request);
|
||||
connect(reply, &QNetworkReply::finished, [this, reply, modelsURL]() {
|
||||
QNetworkReply::NetworkError networkError = reply->error();
|
||||
if (networkError == QNetworkReply::NoError) {
|
||||
QByteArray contents = reply->readAll();
|
||||
replaceContentFromMessageData(contents);
|
||||
} else {
|
||||
qDebug() << "Error downloading JSON from specified file";
|
||||
}
|
||||
});
|
||||
} else {
|
||||
qDebug() << "Cannot perform octree file replacement since current persist file path is not yet known";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void OctreeServer::replaceContentFromMessageData(QByteArray content) {
|
||||
//Assume we have compressed data
|
||||
auto compressedOctree = content;
|
||||
QByteArray jsonOctree;
|
||||
|
||||
bool wasCompressed = gunzip(compressedOctree, jsonOctree);
|
||||
if (!wasCompressed) {
|
||||
// the source was not compressed, assume we were sent regular JSON data
|
||||
jsonOctree = compressedOctree;
|
||||
}
|
||||
// check the JSON data to verify it is an object
|
||||
if (QJsonDocument::fromJson(jsonOctree).isObject()) {
|
||||
if (!wasCompressed) {
|
||||
// source was not compressed, we compress it before we write it locally
|
||||
gzip(jsonOctree, compressedOctree);
|
||||
}
|
||||
// write the compressed octree data to a special file
|
||||
auto replacementFilePath = _persistAbsoluteFilePath.append(OctreePersistThread::REPLACEMENT_FILE_EXTENSION);
|
||||
QFile replacementFile(replacementFilePath);
|
||||
if (replacementFile.open(QIODevice::WriteOnly) && replacementFile.write(compressedOctree) != -1) {
|
||||
// we've now written our replacement file, time to take the server down so it can
|
||||
// process it when it comes back up
|
||||
qInfo() << "Wrote octree replacement file to" << replacementFilePath << "- stopping server";
|
||||
setFinished(true);
|
||||
} else {
|
||||
qWarning() << "Could not write replacement octree data to file - refusing to process";
|
||||
}
|
||||
} else {
|
||||
qDebug() << "Received replacement octree file that is invalid - refusing to process";
|
||||
}
|
||||
}
|
||||
|
||||
bool OctreeServer::readOptionBool(const QString& optionName, const QJsonObject& settingsSectionObject, bool& result) {
|
||||
result = false; // assume it doesn't exist
|
||||
bool optionAvailable = false;
|
||||
|
|
@ -1119,7 +1041,19 @@ void OctreeServer::readConfiguration() {
|
|||
_persistFilePath = getMyDefaultPersistFilename();
|
||||
}
|
||||
|
||||
// If persist filename does not exist, let's see if there is one beside the application binary
|
||||
// If there is, let's copy it over to our target persist directory
|
||||
QDir persistPath { _persistFilePath };
|
||||
_persistAbsoluteFilePath = persistPath.absolutePath();
|
||||
|
||||
if (persistPath.isRelative()) {
|
||||
// if the domain settings passed us a relative path, make an absolute path that is relative to the
|
||||
// default data directory
|
||||
_persistAbsoluteFilePath = QDir(PathUtils::getAppDataFilePath("entities/")).absoluteFilePath(_persistFilePath);
|
||||
}
|
||||
|
||||
qDebug() << "persistFilePath=" << _persistFilePath;
|
||||
qDebug() << "persisAbsoluteFilePath=" << _persistAbsoluteFilePath;
|
||||
|
||||
_persistAsFileType = "json.gz";
|
||||
|
||||
|
|
@ -1200,20 +1134,90 @@ void OctreeServer::run() {
|
|||
}
|
||||
|
||||
void OctreeServer::domainSettingsRequestComplete() {
|
||||
if (_state != OctreeServerState::WaitingForDomainSettings) {
|
||||
qCWarning(octree_server) << "Received domain settings after they have already been received";
|
||||
return;
|
||||
}
|
||||
|
||||
auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
|
||||
packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket");
|
||||
packetReceiver.registerListener(PacketType::OctreeDataNack, this, "handleOctreeDataNackPacket");
|
||||
|
||||
packetReceiver.registerListener(PacketType::OctreeDataFileReply, this, "handleOctreeDataFileReply");
|
||||
|
||||
qDebug(octree_server) << "Received domain settings";
|
||||
|
||||
readConfiguration();
|
||||
|
||||
_state = OctreeServerState::WaitingForOctreeDataNegotation;
|
||||
|
||||
auto nodeList = DependencyManager::get<NodeList>();
|
||||
const DomainHandler& domainHandler = nodeList->getDomainHandler();
|
||||
|
||||
auto packet = NLPacket::create(PacketType::OctreeDataFileRequest, -1, true, false);
|
||||
|
||||
OctreeUtils::RawOctreeData data;
|
||||
qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath;
|
||||
if (OctreeUtils::readOctreeDataInfoFromFile(_persistAbsoluteFilePath, &data)) {
|
||||
qCDebug(octree_server) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")";
|
||||
packet->writePrimitive(true);
|
||||
auto id = data.id.toRfc4122();
|
||||
packet->write(id);
|
||||
packet->writePrimitive(data.version);
|
||||
} else {
|
||||
qCWarning(octree_server) << "No octree data found";
|
||||
packet->writePrimitive(false);
|
||||
}
|
||||
|
||||
qCDebug(octree_server) << "Sending request for octree data to DS";
|
||||
nodeList->sendPacket(std::move(packet), domainHandler.getSockAddr());
|
||||
}
|
||||
|
||||
void OctreeServer::handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message) {
|
||||
bool includesNewData;
|
||||
message->readPrimitive(&includesNewData);
|
||||
QByteArray replaceData;
|
||||
if (includesNewData) {
|
||||
replaceData = message->readAll();
|
||||
qDebug() << "Got reply to octree data file request, new data sent";
|
||||
} else {
|
||||
qDebug() << "Got reply to octree data file request, current entity data is sufficient";
|
||||
|
||||
OctreeUtils::RawOctreeData data;
|
||||
qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath;
|
||||
if (OctreeUtils::readOctreeDataInfoFromFile(_persistAbsoluteFilePath, &data)) {
|
||||
if (data.id.isNull()) {
|
||||
qCDebug(octree_server) << "Current octree data has a null id, updating";
|
||||
data.id = QUuid::createUuid();
|
||||
data.version = 0;
|
||||
|
||||
QFile file(_persistAbsoluteFilePath);
|
||||
if (file.open(QIODevice::WriteOnly)) {
|
||||
auto entityData = data.toByteArray();
|
||||
file.write(entityData);
|
||||
file.close();
|
||||
} else {
|
||||
qCDebug(octree_server) << "Failed to update octree data";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
beginRunning(replaceData);
|
||||
}
|
||||
|
||||
void OctreeServer::beginRunning(QByteArray replaceData) {
|
||||
if (_state == OctreeServerState::Running) {
|
||||
qCWarning(octree_server) << "Server is already running";
|
||||
return;
|
||||
}
|
||||
|
||||
_state = OctreeServerState::Running;
|
||||
|
||||
auto nodeList = DependencyManager::get<NodeList>();
|
||||
|
||||
// we need to ask the DS about agents so we can ping/reply with them
|
||||
nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer });
|
||||
|
||||
auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
|
||||
packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket");
|
||||
packetReceiver.registerListener(PacketType::OctreeDataNack, this, "handleOctreeDataNackPacket");
|
||||
packetReceiver.registerListener(PacketType::OctreeFileReplacement, this, "handleOctreeFileReplacement");
|
||||
packetReceiver.registerListener(PacketType::OctreeFileReplacementFromUrl, this, "handleOctreeFileReplacementFromURL");
|
||||
|
||||
readConfiguration();
|
||||
|
||||
beforeRun(); // after payload has been processed
|
||||
|
||||
connect(nodeList.data(), SIGNAL(nodeAdded(SharedNodePointer)), SLOT(nodeAdded(SharedNodePointer)));
|
||||
|
|
@ -1233,17 +1237,6 @@ void OctreeServer::domainSettingsRequestComplete() {
|
|||
|
||||
// if we want Persistence, set up the local file and persist thread
|
||||
if (_wantPersist) {
|
||||
// If persist filename does not exist, let's see if there is one beside the application binary
|
||||
// If there is, let's copy it over to our target persist directory
|
||||
QDir persistPath { _persistFilePath };
|
||||
_persistAbsoluteFilePath = persistPath.absolutePath();
|
||||
|
||||
if (persistPath.isRelative()) {
|
||||
// if the domain settings passed us a relative path, make an absolute path that is relative to the
|
||||
// default data directory
|
||||
_persistAbsoluteFilePath = QDir(PathUtils::getAppDataFilePath("entities/")).absoluteFilePath(_persistFilePath);
|
||||
}
|
||||
|
||||
static const QString ENTITY_PERSIST_EXTENSION = ".json.gz";
|
||||
|
||||
// force the persist file to end with .json.gz
|
||||
|
|
@ -1328,7 +1321,7 @@ void OctreeServer::domainSettingsRequestComplete() {
|
|||
|
||||
// now set up PersistThread
|
||||
_persistThread = new OctreePersistThread(_tree, _persistAbsoluteFilePath, _backupDirectoryPath, _persistInterval,
|
||||
_wantBackup, _settings, _debugTimestampNow, _persistAsFileType);
|
||||
_wantBackup, _settings, _debugTimestampNow, _persistAsFileType, replaceData);
|
||||
_persistThread->initialize(true);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -27,8 +27,18 @@
|
|||
#include "OctreeServerConsts.h"
|
||||
#include "OctreeInboundPacketProcessor.h"
|
||||
|
||||
#include <QLoggingCategory>
|
||||
|
||||
Q_DECLARE_LOGGING_CATEGORY(octree_server)
|
||||
|
||||
const int DEFAULT_PACKETS_PER_INTERVAL = 2000; // some 120,000 packets per second total
|
||||
|
||||
enum class OctreeServerState {
|
||||
WaitingForDomainSettings,
|
||||
WaitingForOctreeDataNegotation,
|
||||
Running
|
||||
};
|
||||
|
||||
/// Handles assignments of type OctreeServer - sending octrees to various clients.
|
||||
class OctreeServer : public ThreadedAssignment, public HTTPRequestHandler {
|
||||
Q_OBJECT
|
||||
|
|
@ -36,6 +46,8 @@ public:
|
|||
OctreeServer(ReceivedMessage& message);
|
||||
~OctreeServer();
|
||||
|
||||
OctreeServerState _state { OctreeServerState::WaitingForDomainSettings };
|
||||
|
||||
/// allows setting of run arguments
|
||||
void setArguments(int argc, char** argv);
|
||||
|
||||
|
|
@ -137,8 +149,9 @@ private slots:
|
|||
void domainSettingsRequestComplete();
|
||||
void handleOctreeQueryPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
|
||||
void handleOctreeDataNackPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
|
||||
void handleOctreeFileReplacement(QSharedPointer<ReceivedMessage> message);
|
||||
void handleOctreeFileReplacementFromURL(QSharedPointer<ReceivedMessage> message);
|
||||
//void handleOctreeFileReplacement(QSharedPointer<ReceivedMessage> message);
|
||||
//void handleOctreeFileReplacementFromURL(QSharedPointer<ReceivedMessage> message);
|
||||
void handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message);
|
||||
void removeSendThread();
|
||||
|
||||
protected:
|
||||
|
|
@ -159,11 +172,13 @@ protected:
|
|||
QString getFileLoadTime();
|
||||
QString getConfiguration();
|
||||
QString getStatusLink();
|
||||
|
||||
void beginRunning(QByteArray replaceData);
|
||||
|
||||
UniqueSendThread createSendThread(const SharedNodePointer& node);
|
||||
virtual UniqueSendThread newSendThread(const SharedNodePointer& node);
|
||||
|
||||
void replaceContentFromMessageData(QByteArray content);
|
||||
//void replaceContentFromMessageData(QByteArray content);
|
||||
|
||||
int _argc;
|
||||
const char** _argv;
|
||||
|
|
|
|||
|
|
@ -178,7 +178,7 @@ void EntityScriptServer::updateEntityPPS() {
|
|||
int numRunningScripts = _entitiesScriptEngine->getNumRunningEntityScripts();
|
||||
int pps;
|
||||
if (std::numeric_limits<int>::max() / _entityPPSPerScript < numRunningScripts) {
|
||||
qWarning() << QString("Integer multiplaction would overflow, clamping to maxint: %1 * %2").arg(numRunningScripts).arg(_entityPPSPerScript);
|
||||
qWarning() << QString("Integer multiplication would overflow, clamping to maxint: %1 * %2").arg(numRunningScripts).arg(_entityPPSPerScript);
|
||||
pps = std::numeric_limits<int>::max();
|
||||
pps = std::min(_maxEntityPPS, pps);
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -22,7 +22,17 @@ setup_memory_debugger()
|
|||
symlink_or_copy_directory_beside_target(${_SHOULD_SYMLINK_RESOURCES} "${CMAKE_CURRENT_SOURCE_DIR}/resources" "resources")
|
||||
|
||||
# link the shared hifi libraries
|
||||
link_hifi_libraries(embedded-webserver networking shared avatars)
|
||||
link_hifi_libraries(embedded-webserver networking shared avatars octree)
|
||||
|
||||
add_dependency_external_projects(quazip)
|
||||
|
||||
find_package(QuaZip REQUIRED)
|
||||
target_include_directories(${TARGET_NAME} SYSTEM PUBLIC ${QUAZIP_INCLUDE_DIRS})
|
||||
target_link_libraries(${TARGET_NAME} ${QUAZIP_LIBRARIES})
|
||||
|
||||
if (WIN32)
|
||||
add_paths_to_fixup_libs(${QUAZIP_DLL_PATH})
|
||||
endif ()
|
||||
|
||||
# find OpenSSL
|
||||
find_package(OpenSSL REQUIRED)
|
||||
|
|
|
|||
303
domain-server/src/DomainContentBackupManager.cpp
Normal file
303
domain-server/src/DomainContentBackupManager.cpp
Normal file
|
|
@ -0,0 +1,303 @@
|
|||
//
|
||||
// DomainContentBackupManager.cpp
|
||||
// libraries/octree/src
|
||||
//
|
||||
// Created by Brad Hefta-Gaub on 8/21/13.
|
||||
// Copyright 2013 High Fidelity, Inc.
|
||||
//
|
||||
// Distributed under the Apache License, Version 2.0.
|
||||
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
|
||||
//
|
||||
|
||||
#include <chrono>
|
||||
#include <thread>
|
||||
|
||||
#include <cstdio>
|
||||
#include <fstream>
|
||||
#include <time.h>
|
||||
|
||||
#include <QDateTime>
|
||||
#include <QDebug>
|
||||
#include <QDir>
|
||||
#include <QDirIterator>
|
||||
#include <QFile>
|
||||
#include <QJsonArray>
|
||||
#include <QJsonObject>
|
||||
#include <QJsonDocument>
|
||||
|
||||
#include <NumericalConstants.h>
|
||||
#include <PerfStat.h>
|
||||
#include <PathUtils.h>
|
||||
|
||||
#include "DomainServer.h"
|
||||
#include "DomainContentBackupManager.h"
|
||||
const int DomainContentBackupManager::DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
|
||||
|
||||
// Backup format looks like: daily_backup-TIMESTAMP.zip
|
||||
const static QString DATETIME_FORMAT { "yyyy-MM-dd_HH-mm-ss" };
|
||||
const static QString DATETIME_FORMAT_RE("\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}");
|
||||
|
||||
void DomainContentBackupManager::addCreateBackupHandler(CreateBackupHandler handler) {
|
||||
_backupHandlers.push_back(handler);
|
||||
}
|
||||
|
||||
DomainContentBackupManager::DomainContentBackupManager(const QString& backupDirectory,
|
||||
const QJsonObject& settings,
|
||||
int persistInterval,
|
||||
bool debugTimestampNow)
|
||||
: _backupDirectory(backupDirectory),
|
||||
_persistInterval(persistInterval),
|
||||
_initialLoadComplete(false),
|
||||
_loadTimeUSecs(0),
|
||||
_lastCheck(0),
|
||||
_debugTimestampNow(debugTimestampNow),
|
||||
_lastTimeDebug(0) {
|
||||
parseSettings(settings);
|
||||
}
|
||||
|
||||
void DomainContentBackupManager::parseSettings(const QJsonObject& settings) {
|
||||
qDebug() << settings << settings["backups"] << settings["backups"].isArray();
|
||||
if (settings["backups"].isArray()) {
|
||||
const QJsonArray& backupRules = settings["backups"].toArray();
|
||||
qCDebug(domain_server) << "BACKUP RULES:";
|
||||
|
||||
for (const QJsonValue& value : backupRules) {
|
||||
QJsonObject obj = value.toObject();
|
||||
|
||||
int interval = 0;
|
||||
int count = 0;
|
||||
|
||||
QJsonValue intervalVal = obj["backupInterval"];
|
||||
if (intervalVal.isString()) {
|
||||
interval = intervalVal.toString().toInt();
|
||||
} else {
|
||||
interval = intervalVal.toInt();
|
||||
}
|
||||
|
||||
QJsonValue countVal = obj["maxBackupVersions"];
|
||||
if (countVal.isString()) {
|
||||
count = countVal.toString().toInt();
|
||||
} else {
|
||||
count = countVal.toInt();
|
||||
}
|
||||
|
||||
auto name = obj["Name"].toString();
|
||||
auto format = obj["format"].toString();
|
||||
format = name.replace(" ", "_").toLower() + "-";
|
||||
|
||||
qCDebug(domain_server) << " Name:" << name;
|
||||
qCDebug(domain_server) << " format:" << format;
|
||||
qCDebug(domain_server) << " interval:" << interval;
|
||||
qCDebug(domain_server) << " count:" << count;
|
||||
|
||||
BackupRule newRule = { name, interval, format, count, 0 };
|
||||
|
||||
newRule.lastBackupSeconds = getMostRecentBackupTimeInSecs(format);
|
||||
|
||||
if (newRule.lastBackupSeconds > 0) {
|
||||
auto now = QDateTime::currentSecsSinceEpoch();
|
||||
auto sinceLastBackup = now - newRule.lastBackupSeconds;
|
||||
qCDebug(domain_server).noquote() << " lastBackup:" << formatSecTime(sinceLastBackup) << "ago";
|
||||
} else {
|
||||
qCDebug(domain_server) << " lastBackup: NEVER";
|
||||
}
|
||||
|
||||
_backupRules << newRule;
|
||||
}
|
||||
} else {
|
||||
qCDebug(domain_server) << "BACKUP RULES: NONE";
|
||||
}
|
||||
}
|
||||
|
||||
int64_t DomainContentBackupManager::getMostRecentBackupTimeInSecs(const QString& format) {
|
||||
int64_t mostRecentBackupInSecs = 0;
|
||||
|
||||
QString mostRecentBackupFileName;
|
||||
QDateTime mostRecentBackupTime;
|
||||
|
||||
bool recentBackup = getMostRecentBackup(format, mostRecentBackupFileName, mostRecentBackupTime);
|
||||
|
||||
if (recentBackup) {
|
||||
mostRecentBackupInSecs = mostRecentBackupTime.toSecsSinceEpoch();
|
||||
}
|
||||
|
||||
return mostRecentBackupInSecs;
|
||||
}
|
||||
|
||||
bool DomainContentBackupManager::process() {
|
||||
if (isStillRunning()) {
|
||||
constexpr int64_t MSECS_TO_USECS = 1000;
|
||||
constexpr int64_t USECS_TO_SLEEP = 10 * MSECS_TO_USECS; // every 10ms
|
||||
std::this_thread::sleep_for(std::chrono::microseconds(USECS_TO_SLEEP));
|
||||
|
||||
int64_t now = usecTimestampNow();
|
||||
int64_t sinceLastSave = now - _lastCheck;
|
||||
int64_t intervalToCheck = _persistInterval * MSECS_TO_USECS;
|
||||
|
||||
if (sinceLastSave > intervalToCheck) {
|
||||
_lastCheck = now;
|
||||
persist();
|
||||
}
|
||||
}
|
||||
|
||||
// if we were asked to debugTimestampNow do that now...
|
||||
if (_debugTimestampNow) {
|
||||
|
||||
quint64 now = usecTimestampNow();
|
||||
quint64 sinceLastDebug = now - _lastTimeDebug;
|
||||
quint64 DEBUG_TIMESTAMP_INTERVAL = 600000000; // every 10 minutes
|
||||
|
||||
if (sinceLastDebug > DEBUG_TIMESTAMP_INTERVAL) {
|
||||
_lastTimeDebug = usecTimestampNow(true); // ask for debug output
|
||||
}
|
||||
}
|
||||
|
||||
return isStillRunning();
|
||||
}
|
||||
|
||||
void DomainContentBackupManager::aboutToFinish() {
|
||||
qCDebug(domain_server) << "Persist thread about to finish...";
|
||||
persist();
|
||||
}
|
||||
|
||||
void DomainContentBackupManager::persist() {
|
||||
QDir backupDir { _backupDirectory };
|
||||
backupDir.mkpath(".");
|
||||
|
||||
// create our "lock" file to indicate we're saving.
|
||||
QString lockFileName = _backupDirectory + "/running.lock";
|
||||
|
||||
std::ofstream lockFile(qPrintable(lockFileName), std::ios::out | std::ios::binary);
|
||||
if (lockFile.is_open()) {
|
||||
backup();
|
||||
|
||||
lockFile.close();
|
||||
remove(qPrintable(lockFileName));
|
||||
}
|
||||
}
|
||||
|
||||
bool DomainContentBackupManager::getMostRecentBackup(const QString& format,
|
||||
QString& mostRecentBackupFileName,
|
||||
QDateTime& mostRecentBackupTime) {
|
||||
QRegExp formatRE { QRegExp::escape(format) + "(" + DATETIME_FORMAT_RE + ")" + "\\.zip" };
|
||||
|
||||
QStringList filters;
|
||||
filters << format + "*.zip";
|
||||
|
||||
bool bestBackupFound = false;
|
||||
QString bestBackupFile;
|
||||
QDateTime bestBackupFileTime;
|
||||
|
||||
// Iterate over all of the backup files in the persist location
|
||||
QDirIterator dirIterator(_backupDirectory, filters, QDir::Files | QDir::NoSymLinks, QDirIterator::NoIteratorFlags);
|
||||
while (dirIterator.hasNext()) {
|
||||
dirIterator.next();
|
||||
auto fileName = dirIterator.fileInfo().fileName();
|
||||
|
||||
if (formatRE.exactMatch(fileName)) {
|
||||
auto datetime = formatRE.cap(1);
|
||||
auto createdAt = QDateTime::fromString(datetime, DATETIME_FORMAT);
|
||||
|
||||
if (!createdAt.isValid()) {
|
||||
qDebug() << "Skipping backup with invalid timestamp: " << datetime;
|
||||
continue;
|
||||
}
|
||||
|
||||
qDebug() << "Checking " << dirIterator.fileInfo().filePath();
|
||||
|
||||
// Based on last modified date, track the most recently modified file as the best backup
|
||||
if (createdAt > bestBackupFileTime) {
|
||||
bestBackupFound = true;
|
||||
bestBackupFile = dirIterator.filePath();
|
||||
bestBackupFileTime = createdAt;
|
||||
}
|
||||
} else {
|
||||
qDebug() << "NO match: " << fileName << formatRE;
|
||||
}
|
||||
}
|
||||
|
||||
// If we found a backup then return the results
|
||||
if (bestBackupFound) {
|
||||
mostRecentBackupFileName = bestBackupFile;
|
||||
mostRecentBackupTime = bestBackupFileTime;
|
||||
}
|
||||
return bestBackupFound;
|
||||
}
|
||||
|
||||
void DomainContentBackupManager::removeOldBackupVersions(const BackupRule& rule) {
|
||||
QDir backupDir { _backupDirectory };
|
||||
if (backupDir.exists() && rule.maxBackupVersions > 0) {
|
||||
qCDebug(domain_server) << "Rolling old backup versions for rule" << rule.name << "...";
|
||||
|
||||
auto matchingFiles =
|
||||
backupDir.entryInfoList({ rule.extensionFormat + "*.zip" }, QDir::Files | QDir::NoSymLinks, QDir::Name);
|
||||
|
||||
int backupsToDelete = matchingFiles.length() - rule.maxBackupVersions;
|
||||
for (int i = 0; i < backupsToDelete; ++i) {
|
||||
auto fileInfo = matchingFiles[i].absoluteFilePath();
|
||||
QFile backupFile(fileInfo);
|
||||
if (backupFile.remove()) {
|
||||
qCDebug(domain_server) << "Removed old backup: " << backupFile.fileName();
|
||||
} else {
|
||||
qCDebug(domain_server) << "Failed to remove old backup: " << backupFile.fileName();
|
||||
}
|
||||
}
|
||||
|
||||
qCDebug(domain_server) << "Done rolling old backup versions...";
|
||||
} else {
|
||||
qCDebug(domain_server) << "Rolling backups for rule" << rule.name << "."
|
||||
<< " Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]."
|
||||
<< " No need to roll backups...";
|
||||
}
|
||||
}
|
||||
|
||||
void DomainContentBackupManager::backup() {
|
||||
auto nowDateTime = QDateTime::currentDateTime();
|
||||
auto nowSeconds = nowDateTime.toSecsSinceEpoch();
|
||||
|
||||
for (BackupRule& rule : _backupRules) {
|
||||
auto secondsSinceLastBackup = nowSeconds - rule.lastBackupSeconds;
|
||||
|
||||
qCDebug(domain_server) << "Checking [" << rule.name << "] - Time since last backup [" << secondsSinceLastBackup
|
||||
<< "] "
|
||||
<< "compared to backup interval [" << rule.intervalSeconds << "]...";
|
||||
|
||||
if (secondsSinceLastBackup > rule.intervalSeconds) {
|
||||
qCDebug(domain_server) << "Time since last backup [" << secondsSinceLastBackup << "] for rule [" << rule.name
|
||||
<< "] exceeds backup interval [" << rule.intervalSeconds << "] doing backup now...";
|
||||
|
||||
auto timestamp = QDateTime::currentDateTime().toString(DATETIME_FORMAT);
|
||||
auto fileName = "backup-" + rule.extensionFormat + timestamp + ".zip";
|
||||
auto zip = new QuaZip(_backupDirectory + "/" + fileName);
|
||||
zip->open(QuaZip::mdAdd);
|
||||
|
||||
for (auto& handler : _backupHandlers) {
|
||||
handler(zip);
|
||||
}
|
||||
|
||||
zip->close();
|
||||
|
||||
qDebug() << "Created backup: " << fileName;
|
||||
|
||||
removeOldBackupVersions(rule);
|
||||
|
||||
if (rule.maxBackupVersions > 0) {
|
||||
// Execute backup
|
||||
auto result = true;
|
||||
if (result) {
|
||||
qCDebug(domain_server) << "DONE backing up persist file...";
|
||||
rule.lastBackupSeconds = nowSeconds;
|
||||
} else {
|
||||
qCDebug(domain_server) << "ERROR in backing up persist file...";
|
||||
perror("ERROR in backing up persist file");
|
||||
}
|
||||
} else {
|
||||
qCDebug(domain_server) << "This backup rule" << rule.name << " has Max Rolled Backup Versions less than 1 ["
|
||||
<< rule.maxBackupVersions << "]."
|
||||
<< " There are no backups to be done...";
|
||||
}
|
||||
} else {
|
||||
qCDebug(domain_server) << "Backup not needed for this rule [" << rule.name << "]...";
|
||||
}
|
||||
}
|
||||
}
|
||||
88
domain-server/src/DomainContentBackupManager.h
Normal file
88
domain-server/src/DomainContentBackupManager.h
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
//
|
||||
// DomainContentBackupManager.h
|
||||
// libraries/octree/src
|
||||
//
|
||||
// Created by Brad Hefta-Gaub on 8/21/13.
|
||||
// Copyright 2013 High Fidelity, Inc.
|
||||
//
|
||||
//
|
||||
//
|
||||
// Distributed under the Apache License, Version 2.0.
|
||||
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
|
||||
//
|
||||
|
||||
#ifndef hifi_DomainContentBackupManager_h
|
||||
#define hifi_DomainContentBackupManager_h
|
||||
|
||||
#include <QString>
|
||||
#include <GenericThread.h>
|
||||
#include <QVector>
|
||||
|
||||
#include <quazip5/quazip.h>
|
||||
#include <quazip5/quazipfile.h>
|
||||
|
||||
#include <functional>
|
||||
|
||||
using BackupResult = std::vector<QString>;
|
||||
using CreateBackupHandler = std::function<void(QuaZip* quazip)>;
|
||||
using RecoverBackupHandler = std::function<void()>;
|
||||
|
||||
class DomainContentBackupManager : public GenericThread {
|
||||
Q_OBJECT
|
||||
public:
|
||||
class BackupRule {
|
||||
public:
|
||||
QString name;
|
||||
int intervalSeconds;
|
||||
QString extensionFormat;
|
||||
int maxBackupVersions;
|
||||
qint64 lastBackupSeconds;
|
||||
};
|
||||
|
||||
static const int DEFAULT_PERSIST_INTERVAL;
|
||||
|
||||
DomainContentBackupManager(const QString& rootBackupDirectory,
|
||||
const QJsonObject& settings,
|
||||
int persistInterval = DEFAULT_PERSIST_INTERVAL,
|
||||
bool debugTimestampNow = false);
|
||||
|
||||
void addCreateBackupHandler(CreateBackupHandler handler);
|
||||
bool isInitialLoadComplete() const { return _initialLoadComplete; }
|
||||
int64_t getLoadElapsedTime() const { return _loadTimeUSecs; }
|
||||
|
||||
void aboutToFinish(); /// call this to inform the persist thread that the owner is about to finish to support final persist
|
||||
|
||||
void replaceData(QByteArray data);
|
||||
|
||||
signals:
|
||||
void loadCompleted();
|
||||
|
||||
protected:
|
||||
/// Implements generic processing behavior for this thread.
|
||||
bool process() override;
|
||||
|
||||
void persist();
|
||||
void backup();
|
||||
void removeOldBackupVersions(const BackupRule& rule);
|
||||
bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime);
|
||||
int64_t getMostRecentBackupTimeInSecs(const QString& format);
|
||||
void parseSettings(const QJsonObject& settings);
|
||||
|
||||
private:
|
||||
QString _backupDirectory;
|
||||
std::vector<CreateBackupHandler> _backupHandlers;
|
||||
int _persistInterval;
|
||||
bool _initialLoadComplete;
|
||||
|
||||
int64_t _loadTimeUSecs;
|
||||
|
||||
time_t _lastPersistTime;
|
||||
int64_t _lastCheck;
|
||||
bool _wantBackup{ true };
|
||||
QVector<BackupRule> _backupRules;
|
||||
|
||||
bool _debugTimestampNow;
|
||||
int64_t _lastTimeDebug;
|
||||
};
|
||||
|
||||
#endif // hifi_DomainContentBackupManager_h
|
||||
|
|
@ -24,6 +24,7 @@
|
|||
#include <QTimer>
|
||||
#include <QUrlQuery>
|
||||
#include <QCommandLineParser>
|
||||
#include <QUuid>
|
||||
|
||||
#include <AccountManager.h>
|
||||
#include <AssetClient.h>
|
||||
|
|
@ -47,7 +48,14 @@
|
|||
#include "DomainServerNodeData.h"
|
||||
#include "NodeConnectionData.h"
|
||||
|
||||
#include <Gzip.h>
|
||||
|
||||
#include <OctreeUtils.h>
|
||||
|
||||
Q_LOGGING_CATEGORY(domain_server, "hifi.domain_server")
|
||||
|
||||
const QString ACCESS_TOKEN_KEY_PATH = "metaverse.access_token";
|
||||
const QString DomainServer::REPLACEMENT_FILE_EXTENSION = ".replace";
|
||||
|
||||
int const DomainServer::EXIT_CODE_REBOOT = 234923;
|
||||
|
||||
|
|
@ -280,6 +288,30 @@ DomainServer::DomainServer(int argc, char* argv[]) :
|
|||
qDebug() << "Ignoring subnet in whitelist, invalid ip portion: " << subnet;
|
||||
}
|
||||
}
|
||||
|
||||
qDebug() << "Starting persist thread";
|
||||
if (QDir(getEntitiesDirPath()).mkpath(".")) {
|
||||
qCDebug(domain_server) << "Created entities data directory";
|
||||
}
|
||||
maybeHandleReplacementEntityFile();
|
||||
auto entitiesFilePath = getEntitiesFilePath();
|
||||
_contentManager.reset(new DomainContentBackupManager(getContentBackupDir(), _settingsManager.responseObjectForType("6")["entity_server_settings"].toObject()));
|
||||
_contentManager->addCreateBackupHandler([entitiesFilePath](QuaZip* zip) {
|
||||
qDebug() << "Creating a backup from handler";
|
||||
|
||||
QFile entitiesFile { entitiesFilePath };
|
||||
|
||||
if (entitiesFile.open(QIODevice::ReadOnly)) {
|
||||
QuaZipFile zipFile { zip };
|
||||
zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo("models.json.gz", entitiesFilePath));
|
||||
zipFile.write(entitiesFile.readAll());
|
||||
zipFile.close();
|
||||
if (zipFile.getZipError() != UNZ_OK) {
|
||||
qDebug() << "Failed to write entities file to backup:" << zipFile.getZipError();
|
||||
}
|
||||
}
|
||||
});
|
||||
_contentManager->initialize(true);
|
||||
}
|
||||
|
||||
void DomainServer::parseCommandLine() {
|
||||
|
|
@ -352,6 +384,11 @@ DomainServer::~DomainServer() {
|
|||
|
||||
// destroy the LimitedNodeList before the DomainServer QCoreApplication is down
|
||||
DependencyManager::destroy<LimitedNodeList>();
|
||||
|
||||
if (_contentManager) {
|
||||
_contentManager->aboutToFinish();
|
||||
_contentManager->terminating();
|
||||
}
|
||||
}
|
||||
|
||||
void DomainServer::queuedQuit(QString quitMessage, int exitCode) {
|
||||
|
|
@ -691,6 +728,12 @@ void DomainServer::setupNodeListAndAssignments() {
|
|||
packetReceiver.registerListener(PacketType::ICEServerHeartbeatDenied, this, "processICEServerHeartbeatDenialPacket");
|
||||
packetReceiver.registerListener(PacketType::ICEServerHeartbeatACK, this, "processICEServerHeartbeatACK");
|
||||
|
||||
packetReceiver.registerListener(PacketType::OctreeDataFileRequest, this, "processOctreeDataRequestMessage");
|
||||
packetReceiver.registerListener(PacketType::OctreeDataPersist, this, "processOctreeDataPersistMessage");
|
||||
|
||||
packetReceiver.registerListener(PacketType::OctreeFileReplacement, this, "handleOctreeFileReplacementRequest");
|
||||
packetReceiver.registerListener(PacketType::OctreeFileReplacementFromUrl, this, "handleOctreeFileReplacementFromURLRequest");
|
||||
|
||||
// set a custom packetVersionMatch as the verify packet operator for the udt::Socket
|
||||
nodeList->setPacketFilterOperator(&DomainServer::isPacketVerified);
|
||||
|
||||
|
|
@ -1605,6 +1648,7 @@ void DomainServer::sendHeartbeatToIceServer() {
|
|||
qWarning() << "Waiting for keypair generation to complete before sending ICE heartbeat.";
|
||||
|
||||
if (!limitedNodeList->getSessionUUID().isNull()) {
|
||||
qDebug() << "generating keypair";
|
||||
accountManager->generateNewDomainKeypair(limitedNodeList->getSessionUUID());
|
||||
} else {
|
||||
qWarning() << "Attempting to send ICE server heartbeat with no domain ID. This is not supported";
|
||||
|
|
@ -1695,10 +1739,88 @@ void DomainServer::sendHeartbeatToIceServer() {
|
|||
} else {
|
||||
qDebug() << "Not sending ice-server heartbeat since there is no selected ice-server.";
|
||||
qDebug() << "Waiting for" << _iceServerAddr << "host lookup response";
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
void DomainServer::processOctreeDataPersistMessage(QSharedPointer<ReceivedMessage> message) {
|
||||
qDebug() << "Received octree data persist message";
|
||||
auto data = message->readAll();
|
||||
auto filePath = getEntitiesFilePath();
|
||||
|
||||
QFile f(filePath);
|
||||
if (f.open(QIODevice::WriteOnly)) {
|
||||
f.write(data);
|
||||
OctreeUtils::RawOctreeData octreeData;
|
||||
if (OctreeUtils::readOctreeDataInfoFromData(data, &octreeData)) {
|
||||
qCDebug(domain_server) << "Wrote new entiteis file" << octreeData.id << octreeData.version;
|
||||
} else {
|
||||
qCDebug(domain_server) << "Failed to read new octree data info";
|
||||
}
|
||||
} else {
|
||||
qCDebug(domain_server) << "Failed to write new entities file";
|
||||
}
|
||||
}
|
||||
|
||||
QString DomainServer::getContentBackupDir() {
|
||||
return PathUtils::getAppDataFilePath("backup");
|
||||
}
|
||||
|
||||
QString DomainServer::getEntitiesDirPath() {
|
||||
return PathUtils::getAppDataFilePath("entities");
|
||||
}
|
||||
|
||||
QString DomainServer::getEntitiesFilePath() {
|
||||
return PathUtils::getAppDataFilePath("entities/models.json.gz");
|
||||
}
|
||||
|
||||
QString DomainServer::getEntitiesReplacementFilePath() {
|
||||
return getEntitiesFilePath().append(REPLACEMENT_FILE_EXTENSION);
|
||||
}
|
||||
|
||||
void DomainServer::processOctreeDataRequestMessage(QSharedPointer<ReceivedMessage> message) {
|
||||
qDebug() << "Got request for octree data from " << message->getSenderSockAddr();
|
||||
|
||||
bool remoteHasExistingData { false };
|
||||
QUuid id;
|
||||
int version;
|
||||
message->readPrimitive(&remoteHasExistingData);
|
||||
if (remoteHasExistingData) {
|
||||
auto idData = message->read(16);
|
||||
id = QUuid::fromRfc4122(idData);
|
||||
message->readPrimitive(&version);
|
||||
qCDebug(domain_server) << "Entity server does have existing data: ID(" << id << ") DataVersion(" << version << ")";
|
||||
} else {
|
||||
qCDebug(domain_server) << "Entity server does not have existing data";
|
||||
}
|
||||
auto entityFilePath = getEntitiesFilePath();
|
||||
|
||||
//QFile file(entityFilePath);
|
||||
auto reply = NLPacketList::create(PacketType::OctreeDataFileReply, QByteArray(), true, true);
|
||||
OctreeUtils::RawOctreeData data;
|
||||
if (OctreeUtils::readOctreeDataInfoFromFile(entityFilePath, &data)) {
|
||||
if (data.id == id && data.version <= version) {
|
||||
qCDebug(domain_server) << "ES has sufficient octree data, not sending data";
|
||||
reply->writePrimitive(false);
|
||||
} else {
|
||||
qCDebug(domain_server) << "Sending newer octree data to ES";
|
||||
QFile file(entityFilePath);
|
||||
if (file.open(QIODevice::ReadOnly)) {
|
||||
reply->writePrimitive(true);
|
||||
reply->write(file.readAll());
|
||||
} else {
|
||||
qCDebug(domain_server) << "Unable to load entity file";
|
||||
reply->writePrimitive(false);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
qCDebug(domain_server) << "Domain server does not have valid octree data";
|
||||
reply->writePrimitive(false);
|
||||
}
|
||||
|
||||
auto nodeList = DependencyManager::get<LimitedNodeList>();
|
||||
nodeList->sendPacketList(std::move(reply), message->getSenderSockAddr());
|
||||
}
|
||||
|
||||
void DomainServer::processNodeJSONStatsPacket(QSharedPointer<ReceivedMessage> packetList, SharedNodePointer sendingNode) {
|
||||
auto nodeData = static_cast<DomainServerNodeData*>(sendingNode->getLinkedData());
|
||||
if (nodeData) {
|
||||
|
|
@ -3105,9 +3227,64 @@ void DomainServer::setupGroupCacheRefresh() {
|
|||
}
|
||||
}
|
||||
|
||||
void DomainServer::maybeHandleReplacementEntityFile() {
|
||||
QFile replacementFile(getEntitiesReplacementFilePath());
|
||||
if (replacementFile.exists()) {
|
||||
qCDebug(domain_server) << "Replacing existing entity date with replacement file";
|
||||
QFile currentFile(getEntitiesFilePath());
|
||||
if (currentFile.exists()) {
|
||||
if (currentFile.remove()) {
|
||||
qCDebug(domain_server) << "Removed existing entity file";
|
||||
} else {
|
||||
qCWarning(domain_server) << "Failled to remove existing entity file";
|
||||
}
|
||||
}
|
||||
if (replacementFile.rename(getEntitiesFilePath())) {
|
||||
qCDebug(domain_server) << "Successfully updated entities data file with replacement file";
|
||||
} else {
|
||||
qCWarning(domain_server) << "Failed to update entities data file with replacement file";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) {
|
||||
// enumerate the nodes and find any octree type servers with active sockets
|
||||
|
||||
//Assume we have compressed data
|
||||
auto compressedOctree = octreeFile;
|
||||
QByteArray jsonOctree;
|
||||
|
||||
bool wasCompressed = gunzip(compressedOctree, jsonOctree);
|
||||
if (!wasCompressed) {
|
||||
// the source was not compressed, assume we were sent regular JSON data
|
||||
jsonOctree = compressedOctree;
|
||||
}
|
||||
|
||||
OctreeUtils::RawOctreeData data;
|
||||
if (OctreeUtils::readOctreeDataInfoFromData(jsonOctree, &data)) {
|
||||
data.id = QUuid::createUuid();
|
||||
data.version = 0;
|
||||
|
||||
gzip(data.toByteArray(), compressedOctree);
|
||||
|
||||
// write the compressed octree data to a special file
|
||||
auto replacementFilePath = getEntitiesReplacementFilePath();
|
||||
QFile replacementFile(replacementFilePath);
|
||||
if (replacementFile.open(QIODevice::WriteOnly) && replacementFile.write(compressedOctree) != -1) {
|
||||
// we've now written our replacement file, time to take the server down so it can
|
||||
// process it when it comes back up
|
||||
qInfo() << "Wrote octree replacement file to" << replacementFilePath << "- stopping server";
|
||||
|
||||
QMetaObject::invokeMethod(this, "restart", Qt::QueuedConnection);
|
||||
} else {
|
||||
qWarning() << "Could not write replacement octree data to file - refusing to process";
|
||||
}
|
||||
} else {
|
||||
qDebug() << "Received replacement octree file that is invalid - refusing to process";
|
||||
}
|
||||
|
||||
|
||||
return;
|
||||
auto limitedNodeList = DependencyManager::get<LimitedNodeList>();
|
||||
limitedNodeList->eachMatchingNode([](const SharedNodePointer& node) {
|
||||
return node->getType() == NodeType::EntityServer && node->getActiveSocket();
|
||||
|
|
@ -3121,3 +3298,37 @@ void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) {
|
|||
limitedNodeList->sendPacketList(std::move(octreeFilePacketList), *octreeNode);
|
||||
});
|
||||
}
|
||||
|
||||
void DomainServer::handleOctreeFileReplacementFromURLRequest(QSharedPointer<ReceivedMessage> message) {
|
||||
qInfo() << "Received request to replace content from a url";
|
||||
auto node = DependencyManager::get<LimitedNodeList>()->findNodeWithAddr(message->getSenderSockAddr());
|
||||
if (node) {
|
||||
qDebug() << "Found node: " << node->getCanReplaceContent();
|
||||
}
|
||||
if (node->getCanReplaceContent()) {
|
||||
// Convert message data into our URL
|
||||
QString url(message->getMessage());
|
||||
QUrl modelsURL = QUrl(url, QUrl::StrictMode);
|
||||
QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
|
||||
QNetworkRequest request(modelsURL);
|
||||
QNetworkReply* reply = networkAccessManager.get(request);
|
||||
connect(reply, &QNetworkReply::finished, [this, reply, modelsURL]() {
|
||||
QNetworkReply::NetworkError networkError = reply->error();
|
||||
if (networkError == QNetworkReply::NoError) {
|
||||
handleOctreeFileReplacement(reply->readAll());
|
||||
} else {
|
||||
qDebug() << "Error downloading JSON from specified file: " << modelsURL;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
void DomainServer::handleOctreeFileReplacementRequest(QSharedPointer<ReceivedMessage> message) {
|
||||
auto node = DependencyManager::get<NodeList>()->nodeWithUUID(message->getSourceID());
|
||||
if (node->getCanReplaceContent()) {
|
||||
handleOctreeFileReplacement(message->readAll());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,9 +32,14 @@
|
|||
#include "DomainServerSettingsManager.h"
|
||||
#include "DomainServerWebSessionData.h"
|
||||
#include "WalletTransaction.h"
|
||||
#include "DomainContentBackupManager.h"
|
||||
|
||||
#include "PendingAssignedNodeData.h"
|
||||
|
||||
#include <QLoggingCategory>
|
||||
|
||||
Q_DECLARE_LOGGING_CATEGORY(domain_server)
|
||||
|
||||
typedef QSharedPointer<Assignment> SharedAssignmentPointer;
|
||||
typedef QMultiHash<QUuid, WalletTransaction*> TransactionHash;
|
||||
|
||||
|
|
@ -65,6 +70,8 @@ public:
|
|||
bool handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler = false) override;
|
||||
bool handleHTTPSRequest(HTTPSConnection* connection, const QUrl& url, bool skipSubHandler = false) override;
|
||||
|
||||
static const QString REPLACEMENT_FILE_EXTENSION;
|
||||
|
||||
public slots:
|
||||
/// Called by NodeList to inform us a node has been added
|
||||
void nodeAdded(SharedNodePointer node);
|
||||
|
|
@ -84,6 +91,13 @@ private slots:
|
|||
void processICEServerHeartbeatDenialPacket(QSharedPointer<ReceivedMessage> message);
|
||||
void processICEServerHeartbeatACK(QSharedPointer<ReceivedMessage> message);
|
||||
|
||||
void handleOctreeFileReplacementFromURLRequest(QSharedPointer<ReceivedMessage> message);
|
||||
void handleOctreeFileReplacementRequest(QSharedPointer<ReceivedMessage> message);
|
||||
void handleOctreeFileReplacement(QByteArray octreeFile);
|
||||
|
||||
void processOctreeDataRequestMessage(QSharedPointer<ReceivedMessage> message);
|
||||
void processOctreeDataPersistMessage(QSharedPointer<ReceivedMessage> message);
|
||||
|
||||
void setupPendingAssignmentCredits();
|
||||
void sendPendingTransactionsToServer();
|
||||
|
||||
|
|
@ -91,8 +105,7 @@ private slots:
|
|||
void sendHeartbeatToMetaverse() { sendHeartbeatToMetaverse(QString()); }
|
||||
void sendHeartbeatToIceServer();
|
||||
|
||||
void handleConnectedNode(SharedNodePointer newNode);
|
||||
|
||||
void handleConnectedNode(SharedNodePointer newNode);
|
||||
void handleTempDomainSuccess(QNetworkReply& requestReply);
|
||||
void handleTempDomainError(QNetworkReply& requestReply);
|
||||
|
||||
|
|
@ -109,8 +122,6 @@ private slots:
|
|||
void handleSuccessfulICEServerAddressUpdate(QNetworkReply& requestReply);
|
||||
void handleFailedICEServerAddressUpdate(QNetworkReply& requestReply);
|
||||
|
||||
void handleOctreeFileReplacement(QByteArray octreeFile);
|
||||
|
||||
void updateReplicatedNodes();
|
||||
void updateDownstreamNodes();
|
||||
void updateUpstreamNodes();
|
||||
|
|
@ -127,6 +138,13 @@ private:
|
|||
const QUuid& getID();
|
||||
void parseCommandLine();
|
||||
|
||||
QString getContentBackupDir();
|
||||
QString getEntitiesDirPath();
|
||||
QString getEntitiesFilePath();
|
||||
QString getEntitiesReplacementFilePath();
|
||||
|
||||
void maybeHandleReplacementEntityFile();
|
||||
|
||||
void setupNodeListAndAssignments();
|
||||
bool optionallySetupOAuth();
|
||||
bool optionallyReadX509KeyAndCertificate();
|
||||
|
|
@ -252,6 +270,8 @@ private:
|
|||
bool _sendICEServerAddressToMetaverseAPIInProgress { false };
|
||||
bool _sendICEServerAddressToMetaverseAPIRedo { false };
|
||||
|
||||
std::unique_ptr<DomainContentBackupManager> _contentManager { nullptr };
|
||||
|
||||
QHash<QUuid, QPointer<HTTPSConnection>> _pendingOAuthConnections;
|
||||
|
||||
QThread _assetClientThread;
|
||||
|
|
|
|||
|
|
@ -6240,13 +6240,15 @@ bool Application::askToReplaceDomainContent(const QString& url) {
|
|||
// Given confirmation, send request to domain server to replace content
|
||||
qCDebug(interfaceapp) << "Attempting to replace domain content: " << url;
|
||||
QByteArray urlData(url.toUtf8());
|
||||
auto limitedNodeList = DependencyManager::get<LimitedNodeList>();
|
||||
auto limitedNodeList = DependencyManager::get<NodeList>();
|
||||
const auto& domainHandler = limitedNodeList->getDomainHandler();
|
||||
limitedNodeList->eachMatchingNode([](const SharedNodePointer& node) {
|
||||
return node->getType() == NodeType::EntityServer && node->getActiveSocket();
|
||||
}, [&urlData, limitedNodeList](const SharedNodePointer& octreeNode) {
|
||||
}, [&urlData, limitedNodeList, &domainHandler](const SharedNodePointer& octreeNode) {
|
||||
auto octreeFilePacket = NLPacket::create(PacketType::OctreeFileReplacementFromUrl, urlData.size(), true);
|
||||
octreeFilePacket->write(urlData);
|
||||
limitedNodeList->sendPacket(std::move(octreeFilePacket), *octreeNode);
|
||||
qDebug() << "WRiting url data: " << urlData;
|
||||
limitedNodeList->sendPacket(std::move(octreeFilePacket), domainHandler.getSockAddr());
|
||||
});
|
||||
auto addressManager = DependencyManager::get<AddressManager>();
|
||||
addressManager->handleLookupString(DOMAIN_SPAWNING_POINT);
|
||||
|
|
|
|||
|
|
@ -98,4 +98,4 @@ void DomainConnectionModel::refresh() {
|
|||
//inform view that we want refresh data
|
||||
beginResetModel();
|
||||
endResetModel();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2244,6 +2244,8 @@ bool EntityTree::writeToMap(QVariantMap& entityDescription, OctreeElementPointer
|
|||
if (! entityDescription.contains("Entities")) {
|
||||
entityDescription["Entities"] = QVariantList();
|
||||
}
|
||||
entityDescription["DataVersion"] = ++_persistDataVersion;
|
||||
entityDescription["Id"] = _persistID;
|
||||
QScriptEngine scriptEngine;
|
||||
RecurseOctreeToMapOperator theOperator(entityDescription, element, &scriptEngine, skipDefaultValues,
|
||||
skipThoseWithBadParents, _myAvatar);
|
||||
|
|
@ -2256,6 +2258,14 @@ bool EntityTree::readFromMap(QVariantMap& map) {
|
|||
int contentVersion = map["Version"].toInt();
|
||||
bool needsConversion = (contentVersion < (int)EntityVersion::ZoneLightInheritModes);
|
||||
|
||||
if (map.contains("Id")) {
|
||||
_persistID = map["Id"].toUuid();
|
||||
}
|
||||
|
||||
if (map.contains("DataVersion")) {
|
||||
_persistDataVersion = map["DataVersion"].toInt();
|
||||
}
|
||||
|
||||
// map will have a top-level list keyed as "Entities". This will be extracted
|
||||
// and iterated over. Each member of this list is converted to a QVariantMap, then
|
||||
// to a QScriptValue, and then to EntityItemProperties. These properties are used
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@ link_hifi_libraries(shared gpu)
|
|||
if (NOT ANDROID)
|
||||
add_dependency_external_projects(nvtt)
|
||||
find_package(NVTT REQUIRED)
|
||||
|
||||
target_include_directories(${TARGET_NAME} PRIVATE ${NVTT_INCLUDE_DIRS})
|
||||
target_link_libraries(${TARGET_NAME} ${NVTT_LIBRARIES})
|
||||
add_paths_to_fixup_libs(${NVTT_DLL_PATH})
|
||||
endif()
|
||||
endif()
|
||||
|
|
|
|||
|
|
@ -326,6 +326,7 @@ bool LimitedNodeList::packetSourceAndHashMatchAndTrackBandwidth(const udt::Packe
|
|||
static QMultiMap<QUuid, PacketType> hashDebugSuppressMap;
|
||||
|
||||
if (!hashDebugSuppressMap.contains(sourceID, headerType)) {
|
||||
qCDebug(networking) << packetHeaderHash << expectedHash;
|
||||
qCDebug(networking) << "Packet hash mismatch on" << headerType << "- Sender" << sourceID;
|
||||
|
||||
hashDebugSuppressMap.insert(sourceID, headerType);
|
||||
|
|
|
|||
|
|
@ -18,8 +18,6 @@
|
|||
|
||||
#include "Assignment.h"
|
||||
|
||||
using DownstreamNodeFoundCallback = std::function<void(Node& downstreamNode)>;
|
||||
|
||||
class ThreadedAssignment : public Assignment {
|
||||
Q_OBJECT
|
||||
public:
|
||||
|
|
@ -47,10 +45,10 @@ protected:
|
|||
QTimer _domainServerTimer;
|
||||
QTimer _statsTimer;
|
||||
int _numQueuedCheckIns { 0 };
|
||||
|
||||
|
||||
protected slots:
|
||||
void domainSettingsRequestFailed();
|
||||
|
||||
|
||||
private slots:
|
||||
void checkInWithDomainServerOrExit();
|
||||
};
|
||||
|
|
|
|||
|
|
@ -126,6 +126,11 @@ public:
|
|||
EntityScriptCallMethod,
|
||||
ChallengeOwnershipRequest,
|
||||
ChallengeOwnershipReply,
|
||||
|
||||
OctreeDataFileRequest,
|
||||
OctreeDataFileReply,
|
||||
OctreeDataPersist,
|
||||
|
||||
NUM_PACKET_TYPE
|
||||
};
|
||||
|
||||
|
|
@ -165,6 +170,8 @@ public:
|
|||
<< PacketTypeEnum::Value::DomainConnectionDenied << PacketTypeEnum::Value::DomainServerPathQuery
|
||||
<< PacketTypeEnum::Value::DomainServerPathResponse << PacketTypeEnum::Value::DomainServerAddedNode
|
||||
<< PacketTypeEnum::Value::DomainServerConnectionToken << PacketTypeEnum::Value::DomainSettingsRequest
|
||||
<< PacketTypeEnum::Value::OctreeDataFileRequest << PacketTypeEnum::Value::OctreeDataFileReply
|
||||
<< PacketTypeEnum::Value::OctreeDataPersist << PacketTypeEnum::Value::OctreeFileReplacementFromUrl
|
||||
<< PacketTypeEnum::Value::DomainSettings << PacketTypeEnum::Value::ICEServerPeerInformation
|
||||
<< PacketTypeEnum::Value::ICEServerQuery << PacketTypeEnum::Value::ICEServerHeartbeat
|
||||
<< PacketTypeEnum::Value::ICEServerHeartbeatACK << PacketTypeEnum::Value::ICEPing
|
||||
|
|
|
|||
|
|
@ -328,7 +328,7 @@ void Socket::checkForReadyReadBackup() {
|
|||
void Socket::readPendingDatagrams() {
|
||||
int packetSizeWithHeader = -1;
|
||||
|
||||
while ((packetSizeWithHeader = _udpSocket.pendingDatagramSize()) != -1) {
|
||||
while ((packetSizeWithHeader = _udpSocket.pendingDatagramSize()) > 0) {
|
||||
|
||||
// we're reading a packet so re-start the readyRead backup timer
|
||||
_readyReadBackupTimer->start();
|
||||
|
|
@ -517,7 +517,7 @@ void Socket::handleSocketError(QAbstractSocket::SocketError socketError) {
|
|||
static QString repeatedMessage
|
||||
= LogHandler::getInstance().addRepeatedMessageRegex(SOCKET_REGEX);
|
||||
|
||||
qCDebug(networking) << "udt::Socket error - " << socketError;
|
||||
qCDebug(networking) << "udt::Socket error - " << socketError << _udpSocket.errorString();
|
||||
}
|
||||
|
||||
void Socket::handleStateChanged(QAbstractSocket::SocketState socketState) {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
set(TARGET_NAME octree)
|
||||
include_directories(system /usr/local/Cellar/qt5/5.9.1/include)
|
||||
setup_hifi_library()
|
||||
link_hifi_libraries(shared networking)
|
||||
|
|
|
|||
|
|
@ -1757,6 +1757,19 @@ bool Octree::readJSONFromStream(uint64_t streamLength, QDataStream& inputStream,
|
|||
QVariant asVariant = asDocument.toVariant();
|
||||
QVariantMap asMap = asVariant.toMap();
|
||||
bool success = readFromMap(asMap);
|
||||
/*
|
||||
if (success) {
|
||||
if (asMap.contains("DataVersion") && asMap.contains("Id")) {
|
||||
bool versionOk;
|
||||
auto dataVersion = asMap["DataVersion"].toLongLong(&versionOk);
|
||||
if (versionOk) {
|
||||
auto id = asMap["Id"].toUuid();
|
||||
_persistDataVersion = dataVersion;
|
||||
_persistID = id;
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
delete[] rawData;
|
||||
return success;
|
||||
}
|
||||
|
|
@ -1778,11 +1791,9 @@ bool Octree::writeToFile(const char* fileName, const OctreeElementPointer& eleme
|
|||
return success;
|
||||
}
|
||||
|
||||
bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& element, bool doGzip) {
|
||||
bool Octree::toJSON(QJsonDocument* doc, const OctreeElementPointer& element) {
|
||||
QVariantMap entityDescription;
|
||||
|
||||
qCDebug(octree, "Saving JSON SVO to file %s...", fileName);
|
||||
|
||||
OctreeElementPointer top;
|
||||
if (element) {
|
||||
top = element;
|
||||
|
|
@ -1802,17 +1813,35 @@ bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& e
|
|||
return false;
|
||||
}
|
||||
|
||||
// convert the QVariantMap to JSON
|
||||
QByteArray jsonData = QJsonDocument::fromVariant(entityDescription).toJson();
|
||||
QByteArray jsonDataForFile;
|
||||
*doc = QJsonDocument::fromVariant(entityDescription);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (doGzip) {
|
||||
if (!gzip(jsonData, jsonDataForFile, -1)) {
|
||||
qCritical("unable to gzip data while saving to json.");
|
||||
return false;
|
||||
}
|
||||
bool Octree::toGzippedJSON(QByteArray* data, const OctreeElementPointer& element) {
|
||||
QJsonDocument doc;
|
||||
if (!toJSON(&doc, element)) {
|
||||
qCritical("Failed to convert Entities to QVariantMap while converting to json.");
|
||||
return false;
|
||||
}
|
||||
|
||||
QByteArray jsonData = doc.toJson();
|
||||
|
||||
if (!gzip(jsonData, *data, -1)) {
|
||||
qCritical("Unable to gzip data while saving to json.");
|
||||
return false;
|
||||
} else {
|
||||
jsonDataForFile = jsonData;
|
||||
qDebug() <<"Did gzip!";
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& element, bool doGzip) {
|
||||
qCDebug(octree, "Saving JSON SVO to file %s...", fileName);
|
||||
|
||||
QByteArray jsonDataForFile;
|
||||
if (!toGzippedJSON(&jsonDataForFile)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
QFile persistFile(fileName);
|
||||
|
|
@ -1823,6 +1852,7 @@ bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& e
|
|||
qCritical("Could not write to JSON description of entities.");
|
||||
}
|
||||
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -283,8 +283,10 @@ public:
|
|||
void loadOctreeFile(const char* fileName);
|
||||
|
||||
// Octree exporters
|
||||
bool writeToFile(const char* filename, const OctreeElementPointer& element = NULL, QString persistAsFileType = "json.gz");
|
||||
bool writeToJSONFile(const char* filename, const OctreeElementPointer& element = NULL, bool doGzip = false);
|
||||
bool toJSON(QJsonDocument* doc, const OctreeElementPointer& element = nullptr);
|
||||
bool toGzippedJSON(QByteArray* data, const OctreeElementPointer& element = nullptr);
|
||||
bool writeToFile(const char* filename, const OctreeElementPointer& element = nullptr, QString persistAsFileType = "json.gz");
|
||||
bool writeToJSONFile(const char* filename, const OctreeElementPointer& element = nullptr, bool doGzip = false);
|
||||
virtual bool writeToMap(QVariantMap& entityDescription, OctreeElementPointer element, bool skipDefaultValues,
|
||||
bool skipThoseWithBadParents) = 0;
|
||||
|
||||
|
|
@ -326,6 +328,11 @@ public:
|
|||
virtual void dumpTree() { }
|
||||
virtual void pruneTree() { }
|
||||
|
||||
void setEntityVersionInfo(QUuid id, int64_t dataVersion) {
|
||||
_persistID = id;
|
||||
_persistDataVersion = dataVersion;
|
||||
}
|
||||
|
||||
virtual void resetEditStats() { }
|
||||
virtual quint64 getAverageDecodeTime() const { return 0; }
|
||||
virtual quint64 getAverageLookupTime() const { return 0; }
|
||||
|
|
@ -359,6 +366,9 @@ protected:
|
|||
|
||||
OctreeElementPointer _rootElement = nullptr;
|
||||
|
||||
QUuid _persistID { QUuid::createUuid() };
|
||||
int _persistDataVersion { 0 };
|
||||
|
||||
bool _isDirty;
|
||||
bool _shouldReaverage;
|
||||
bool _stopImport;
|
||||
|
|
|
|||
|
|
@ -31,18 +31,19 @@
|
|||
|
||||
#include "OctreeLogging.h"
|
||||
#include "OctreePersistThread.h"
|
||||
#include "OctreeUtils.h"
|
||||
|
||||
const int OctreePersistThread::DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
|
||||
const QString OctreePersistThread::REPLACEMENT_FILE_EXTENSION = ".replace";
|
||||
|
||||
OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& filename, const QString& backupDirectory, int persistInterval,
|
||||
bool wantBackup, const QJsonObject& settings, bool debugTimestampNow,
|
||||
QString persistAsFileType) :
|
||||
QString persistAsFileType, const QByteArray& replacementData) :
|
||||
_tree(tree),
|
||||
_filename(filename),
|
||||
_backupDirectory(backupDirectory),
|
||||
_persistInterval(persistInterval),
|
||||
_initialLoadComplete(false),
|
||||
_replacementData(replacementData),
|
||||
_loadTimeUSecs(0),
|
||||
_lastCheck(0),
|
||||
_wantBackup(wantBackup),
|
||||
|
|
@ -52,6 +53,7 @@ OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& file
|
|||
{
|
||||
parseSettings(settings);
|
||||
|
||||
|
||||
// in case the persist filename has an extension that doesn't match the file type
|
||||
QString sansExt = fileNameWithoutExtension(_filename, PERSIST_EXTENSIONS);
|
||||
_filename = sansExt + "." + _persistAsFileType;
|
||||
|
|
@ -132,51 +134,56 @@ quint64 OctreePersistThread::getMostRecentBackupTimeInUsecs(const QString& forma
|
|||
return mostRecentBackupInUsecs;
|
||||
}
|
||||
|
||||
void OctreePersistThread::possiblyReplaceContent() {
|
||||
// before we load the normal file, check if there's a pending replacement file
|
||||
auto replacementFileName = _filename + REPLACEMENT_FILE_EXTENSION;
|
||||
void OctreePersistThread::replaceData(QByteArray data) {
|
||||
backupCurrentFile();
|
||||
|
||||
QFile replacementFile { replacementFileName };
|
||||
if (replacementFile.exists()) {
|
||||
// we have a replacement file to process
|
||||
qDebug() << "Replacing models file with" << replacementFileName;
|
||||
|
||||
// first take the current models file and move it to a different filename, appended with the timestamp
|
||||
QFile currentFile { _filename };
|
||||
if (currentFile.exists()) {
|
||||
static const QString FILENAME_TIMESTAMP_FORMAT = "yyyyMMdd-hhmmss";
|
||||
auto backupFileName = _filename + ".backup." + QDateTime::currentDateTime().toString(FILENAME_TIMESTAMP_FORMAT);
|
||||
|
||||
if (currentFile.rename(backupFileName)) {
|
||||
qDebug() << "Moved previous models file to" << backupFileName;
|
||||
} else {
|
||||
qWarning() << "Could not backup previous models file to" << backupFileName << "- removing replacement models file";
|
||||
|
||||
if (!replacementFile.remove()) {
|
||||
qWarning() << "Could not remove replacement models file from" << replacementFileName
|
||||
<< "- replacement will be re-attempted on next server restart";
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rename the replacement file to match what the persist thread is just about to read
|
||||
if (!replacementFile.rename(_filename)) {
|
||||
qWarning() << "Could not replace models file with" << replacementFileName << "- starting with empty models file";
|
||||
}
|
||||
QFile currentFile { _filename };
|
||||
if (currentFile.open(QIODevice::WriteOnly)) {
|
||||
currentFile.write(data);
|
||||
qDebug() << "Wrote replacement data";
|
||||
} else {
|
||||
qWarning() << "Failed to write replacement data";
|
||||
}
|
||||
}
|
||||
|
||||
// Return true if current file is backed up successfully or doesn't exist.
|
||||
bool OctreePersistThread::backupCurrentFile() {
|
||||
// first take the current models file and move it to a different filename, appended with the timestamp
|
||||
QFile currentFile { _filename };
|
||||
if (currentFile.exists()) {
|
||||
static const QString FILENAME_TIMESTAMP_FORMAT = "yyyyMMdd-hhmmss";
|
||||
auto backupFileName = _filename + ".backup." + QDateTime::currentDateTime().toString(FILENAME_TIMESTAMP_FORMAT);
|
||||
|
||||
if (currentFile.rename(backupFileName)) {
|
||||
qDebug() << "Moved previous models file to" << backupFileName;
|
||||
return true;
|
||||
} else {
|
||||
qWarning() << "Could not backup previous models file to" << backupFileName << "- removing replacement models file";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool OctreePersistThread::process() {
|
||||
|
||||
if (!_initialLoadComplete) {
|
||||
possiblyReplaceContent();
|
||||
|
||||
quint64 loadStarted = usecTimestampNow();
|
||||
qCDebug(octree) << "loading Octrees from file: " << _filename << "...";
|
||||
|
||||
bool persistantFileRead;
|
||||
if (_replacementData.isNull()) {
|
||||
sendLatestEntityDataToDS();
|
||||
} else {
|
||||
replaceData(_replacementData);
|
||||
_replacementData.clear();
|
||||
}
|
||||
|
||||
OctreeUtils::RawOctreeData data;
|
||||
if (OctreeUtils::readOctreeDataInfoFromFile(_filename, &data)) {
|
||||
_tree->setEntityVersionInfo(data.id, data.version);
|
||||
}
|
||||
|
||||
bool persistentFileRead;
|
||||
|
||||
_tree->withWriteLock([&] {
|
||||
PerformanceWarning warn(true, "Loading Octree File", true);
|
||||
|
|
@ -199,7 +206,7 @@ bool OctreePersistThread::process() {
|
|||
qCDebug(octree) << "Loading Octree... lock file removed:" << lockFileName;
|
||||
}
|
||||
|
||||
persistantFileRead = _tree->readFromFile(qPrintable(_filename.toLocal8Bit()));
|
||||
persistentFileRead = _tree->readFromFile(qPrintable(_filename.toLocal8Bit()));
|
||||
_tree->pruneTree();
|
||||
});
|
||||
|
||||
|
|
@ -207,7 +214,7 @@ bool OctreePersistThread::process() {
|
|||
_loadTimeUSecs = loadDone - loadStarted;
|
||||
|
||||
_tree->clearDirtyBit(); // the tree is clean since we just loaded it
|
||||
qCDebug(octree, "DONE loading Octrees from file... fileRead=%s", debug::valueOf(persistantFileRead));
|
||||
qCDebug(octree, "DONE loading Octrees from file... fileRead=%s", debug::valueOf(persistentFileRead));
|
||||
|
||||
unsigned long nodeCount = OctreeElement::getNodeCount();
|
||||
unsigned long internalNodeCount = OctreeElement::getInternalNodeCount();
|
||||
|
|
@ -272,7 +279,6 @@ bool OctreePersistThread::process() {
|
|||
return isStillRunning(); // keep running till they terminate us
|
||||
}
|
||||
|
||||
|
||||
void OctreePersistThread::aboutToFinish() {
|
||||
qCDebug(octree) << "Persist thread about to finish...";
|
||||
persist();
|
||||
|
|
@ -319,6 +325,23 @@ void OctreePersistThread::persist() {
|
|||
remove(qPrintable(lockFileName));
|
||||
qCDebug(octree) << "saving Octree lock file removed:" << lockFileName;
|
||||
}
|
||||
|
||||
sendLatestEntityDataToDS();
|
||||
}
|
||||
}
|
||||
|
||||
void OctreePersistThread::sendLatestEntityDataToDS() {
|
||||
qDebug() << "Sending latest entity data to DS";
|
||||
auto nodeList = DependencyManager::get<NodeList>();
|
||||
const DomainHandler& domainHandler = nodeList->getDomainHandler();
|
||||
|
||||
QByteArray data;
|
||||
if (_tree->toGzippedJSON(&data)) {
|
||||
auto message = NLPacketList::create(PacketType::OctreeDataPersist, QByteArray(), true, true);
|
||||
message->write(data);
|
||||
nodeList->sendPacketList(std::move(message), domainHandler.getSockAddr());
|
||||
} else {
|
||||
qCWarning(octree) << "Failed to persist octree to DS";
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -453,7 +476,6 @@ void OctreePersistThread::rollOldBackupVersions(const BackupRule& rule) {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
void OctreePersistThread::backup() {
|
||||
qCDebug(octree) << "backup operation wantBackup:" << _wantBackup;
|
||||
if (_wantBackup) {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@
|
|||
#include <GenericThread.h>
|
||||
#include "Octree.h"
|
||||
|
||||
/// Generalized threaded processor for handling received inbound packets.
|
||||
class OctreePersistThread : public GenericThread {
|
||||
Q_OBJECT
|
||||
public:
|
||||
|
|
@ -32,11 +31,11 @@ public:
|
|||
};
|
||||
|
||||
static const int DEFAULT_PERSIST_INTERVAL;
|
||||
static const QString REPLACEMENT_FILE_EXTENSION;
|
||||
|
||||
OctreePersistThread(OctreePointer tree, const QString& filename, const QString& backupDirectory,
|
||||
int persistInterval = DEFAULT_PERSIST_INTERVAL, bool wantBackup = false,
|
||||
const QJsonObject& settings = QJsonObject(), bool debugTimestampNow = false, QString persistAsFileType="json.gz");
|
||||
const QJsonObject& settings = QJsonObject(), bool debugTimestampNow = false,
|
||||
QString persistAsFileType="json.gz", const QByteArray& replacementData = QByteArray());
|
||||
|
||||
bool isInitialLoadComplete() const { return _initialLoadComplete; }
|
||||
quint64 getLoadElapsedTime() const { return _loadTimeUSecs; }
|
||||
|
|
@ -61,7 +60,10 @@ protected:
|
|||
bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime);
|
||||
quint64 getMostRecentBackupTimeInUsecs(const QString& format);
|
||||
void parseSettings(const QJsonObject& settings);
|
||||
void possiblyReplaceContent();
|
||||
bool backupCurrentFile();
|
||||
|
||||
void replaceData(QByteArray data);
|
||||
void sendLatestEntityDataToDS();
|
||||
|
||||
private:
|
||||
OctreePointer _tree;
|
||||
|
|
@ -69,6 +71,7 @@ private:
|
|||
QString _backupDirectory;
|
||||
int _persistInterval;
|
||||
bool _initialLoadComplete;
|
||||
QByteArray _replacementData;
|
||||
|
||||
quint64 _loadTimeUSecs;
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,11 @@
|
|||
#include <glm/glm.hpp>
|
||||
|
||||
#include <AABox.h>
|
||||
#include <Gzip.h>
|
||||
|
||||
#include <QJsonObject>
|
||||
#include <QJsonDocument>
|
||||
#include <QFile>
|
||||
|
||||
float calculateRenderAccuracy(const glm::vec3& position,
|
||||
const AABox& bounds,
|
||||
|
|
@ -75,3 +79,76 @@ float getOrthographicAccuracySize(float octreeSizeScale, int boundaryLevelAdjust
|
|||
const float smallestSize = 0.01f;
|
||||
return (smallestSize * MAX_VISIBILITY_DISTANCE_FOR_UNIT_ELEMENT) / boundaryDistanceForRenderLevel(boundaryLevelAdjust, octreeSizeScale);
|
||||
}
|
||||
|
||||
bool OctreeUtils::readOctreeFile(QString path, QJsonDocument* doc) {
|
||||
QFile file(path);
|
||||
if (!file.open(QIODevice::ReadOnly)) {
|
||||
qCritical() << "Cannot open json file for reading: " << path;
|
||||
return false;
|
||||
}
|
||||
|
||||
QByteArray data = file.readAll();
|
||||
QByteArray jsonData;
|
||||
|
||||
if (path.endsWith(".json.gz")) {
|
||||
if (!gunzip(data, jsonData)) {
|
||||
qCritical() << "json File not in gzip format: " << path;
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
jsonData = data;
|
||||
}
|
||||
|
||||
*doc = QJsonDocument::fromJson(jsonData);
|
||||
return !doc->isNull();
|
||||
}
|
||||
|
||||
bool readOctreeDataInfoFromJSON(QJsonObject root, OctreeUtils::RawOctreeData* octreeData) {
|
||||
if (root.contains("Id") && root.contains("DataVersion")) {
|
||||
octreeData->id = root["Id"].toVariant().toUuid();
|
||||
octreeData->version = root["DataVersion"].toInt();
|
||||
}
|
||||
if (root.contains("Entities")) {
|
||||
octreeData->octreeData = root["Entities"].toArray();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool OctreeUtils::readOctreeDataInfoFromData(QByteArray data, OctreeUtils::RawOctreeData* octreeData) {
|
||||
QByteArray jsonData;
|
||||
if (gunzip(data, jsonData)) {
|
||||
data = jsonData;
|
||||
}
|
||||
|
||||
auto doc = QJsonDocument::fromJson(data);
|
||||
if (doc.isNull()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
auto root = doc.object();
|
||||
return readOctreeDataInfoFromJSON(root, octreeData);
|
||||
}
|
||||
|
||||
bool OctreeUtils::readOctreeDataInfoFromFile(QString path, OctreeUtils::RawOctreeData* octreeData) {
|
||||
QJsonDocument doc;
|
||||
if (!OctreeUtils::readOctreeFile(path, &doc)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
auto root = doc.object();
|
||||
return readOctreeDataInfoFromJSON(root, octreeData);
|
||||
}
|
||||
|
||||
QByteArray OctreeUtils::RawOctreeData::toByteArray() {
|
||||
QJsonObject obj {
|
||||
{ "DataVersion", QJsonValue((qint64)version) },
|
||||
{ "Id", QJsonValue(id.toString()) },
|
||||
{ "Version", QJsonValue(5) },
|
||||
{ "Entities", octreeData }
|
||||
};
|
||||
|
||||
QJsonDocument doc;
|
||||
doc.setObject(obj);
|
||||
|
||||
return doc.toJson();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,30 @@
|
|||
|
||||
#include "OctreeConstants.h"
|
||||
|
||||
#include <QUuid>
|
||||
#include <QJsonArray>
|
||||
|
||||
class AABox;
|
||||
class QJsonDocument;
|
||||
|
||||
namespace OctreeUtils {
|
||||
|
||||
// RawOctreeData is an intermediate format between JSON and a fully deserialized Octree.
|
||||
class RawOctreeData {
|
||||
public:
|
||||
QUuid id { QUuid() };
|
||||
int64_t version { -1 };
|
||||
|
||||
QJsonArray octreeData;
|
||||
|
||||
QByteArray toByteArray();
|
||||
};
|
||||
|
||||
bool readOctreeFile(QString path, QJsonDocument* doc);
|
||||
bool readOctreeDataInfoFromData(QByteArray data, RawOctreeData* octreeData);
|
||||
bool readOctreeDataInfoFromFile(QString path, RawOctreeData* octreeData);
|
||||
|
||||
}
|
||||
|
||||
/// renderAccuracy represents a floating point "visibility" of an object based on it's view from the camera. At a simple
|
||||
/// level it returns 0.0f for things that are so small for the current settings that they could not be visible.
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ void usecTimestampNowForceClockSkew(qint64 clockSkew) {
|
|||
::usecTimestampNowAdjust = clockSkew;
|
||||
}
|
||||
|
||||
static qint64 TIME_REFERENCE = 0; // in usec
|
||||
static std::atomic<qint64> TIME_REFERENCE { 0 }; // in usec
|
||||
static std::once_flag usecTimestampNowIsInitialized;
|
||||
static QElapsedTimer timestampTimer;
|
||||
|
||||
|
|
@ -771,6 +771,10 @@ QString formatUsecTime(double usecs) {
|
|||
return formatUsecTime<double>(usecs);
|
||||
}
|
||||
|
||||
QString formatSecTime(qint64 secs) {
|
||||
return formatUsecTime(secs * 1000000);
|
||||
}
|
||||
|
||||
|
||||
QString formatSecondsElapsed(float seconds) {
|
||||
QString result;
|
||||
|
|
|
|||
|
|
@ -216,6 +216,7 @@ QString formatUsecTime(float usecs);
|
|||
QString formatUsecTime(double usecs);
|
||||
QString formatUsecTime(quint64 usecs);
|
||||
QString formatUsecTime(qint64 usecs);
|
||||
QString formatSecTime(qint64 secs);
|
||||
|
||||
QString formatSecondsElapsed(float seconds);
|
||||
bool similarStrings(const QString& stringA, const QString& stringB);
|
||||
|
|
|
|||
Loading…
Reference in a new issue