mirror of
https://github.com/overte-org/overte.git
synced 2025-04-07 19:12:28 +02:00
Merge branch 'master' into snoretoast
This commit is contained in:
SimonWalton-HiFi
GitHub
commit
4807f9d8eb
10 changed files with 320 additions and 635 deletions
|
|
@ -396,21 +396,26 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node)
|
|||
quint64 end = usecTimestampNow();
|
||||
_stats.toByteArrayElapsedTime += (end - start);
|
||||
|
||||
static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID);
|
||||
if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) {
|
||||
qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data";
|
||||
auto maxAvatarDataBytes = avatarPacketList->getMaxSegmentSize() - NUM_BYTES_RFC4122_UUID;
|
||||
if (bytes.size() > maxAvatarDataBytes) {
|
||||
qCWarning(avatars) << "otherAvatar.toByteArray() for" << otherNode->getUUID()
|
||||
<< "resulted in very large buffer of" << bytes.size() << "bytes - dropping facial data";
|
||||
|
||||
dropFaceTracking = true; // first try dropping the facial data
|
||||
bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther,
|
||||
hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther);
|
||||
|
||||
if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) {
|
||||
qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData";
|
||||
if (bytes.size() > maxAvatarDataBytes) {
|
||||
qCWarning(avatars) << "otherAvatar.toByteArray() for" << otherNode->getUUID()
|
||||
<< "without facial data resulted in very large buffer of" << bytes.size()
|
||||
<< "bytes - reducing to MinimumData";
|
||||
bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther,
|
||||
hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther);
|
||||
|
||||
if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) {
|
||||
qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!";
|
||||
if (bytes.size() > maxAvatarDataBytes) {
|
||||
qCWarning(avatars) << "otherAvatar.toByteArray() for" << otherNode->getUUID()
|
||||
<< "MinimumData resulted in very large buffer of" << bytes.size()
|
||||
<< "bytes - refusing to send avatar";
|
||||
includeThisAvatar = false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -231,18 +231,19 @@ void OctreeServer::trackProcessWaitTime(float time) {
|
|||
OctreeServer::OctreeServer(ReceivedMessage& message) :
|
||||
ThreadedAssignment(message),
|
||||
_argc(0),
|
||||
_argv(NULL),
|
||||
_parsedArgV(NULL),
|
||||
_argv(nullptr),
|
||||
_parsedArgV(nullptr),
|
||||
_httpManager(nullptr),
|
||||
_statusPort(0),
|
||||
_packetsPerClientPerInterval(10),
|
||||
_packetsTotalPerInterval(DEFAULT_PACKETS_PER_INTERVAL),
|
||||
_tree(NULL),
|
||||
_tree(nullptr),
|
||||
_wantPersist(true),
|
||||
_debugSending(false),
|
||||
_debugReceiving(false),
|
||||
_verboseDebug(false),
|
||||
_octreeInboundPacketProcessor(NULL),
|
||||
_persistThread(NULL),
|
||||
_octreeInboundPacketProcessor(nullptr),
|
||||
_persistManager(nullptr),
|
||||
_started(time(0)),
|
||||
_startedUSecs(usecTimestampNow())
|
||||
{
|
||||
|
|
@ -265,11 +266,8 @@ OctreeServer::~OctreeServer() {
|
|||
_octreeInboundPacketProcessor->deleteLater();
|
||||
}
|
||||
|
||||
if (_persistThread) {
|
||||
_persistThread->terminating();
|
||||
_persistThread->terminate();
|
||||
_persistThread->deleteLater();
|
||||
}
|
||||
qDebug() << "Waiting for persist thread to come down";
|
||||
_persistThread.wait();
|
||||
|
||||
// cleanup our tree here...
|
||||
qDebug() << qPrintable(_safeServerName) << "server START cleaning up octree... [" << this << "]";
|
||||
|
|
@ -1052,19 +1050,13 @@ void OctreeServer::readConfiguration() {
|
|||
_persistAsFileType = "json.gz";
|
||||
|
||||
_persistInterval = OctreePersistThread::DEFAULT_PERSIST_INTERVAL;
|
||||
readOptionInt(QString("persistInterval"), settingsSectionObject, _persistInterval);
|
||||
qDebug() << "persistInterval=" << _persistInterval;
|
||||
|
||||
bool noBackup;
|
||||
readOptionBool(QString("NoBackup"), settingsSectionObject, noBackup);
|
||||
_wantBackup = !noBackup;
|
||||
qDebug() << "wantBackup=" << _wantBackup;
|
||||
|
||||
if (!readOptionString("backupDirectoryPath", settingsSectionObject, _backupDirectoryPath)) {
|
||||
_backupDirectoryPath = "";
|
||||
int result { -1 };
|
||||
readOptionInt(QString("persistInterval"), settingsSectionObject, result);
|
||||
if (result != -1) {
|
||||
_persistInterval = std::chrono::milliseconds(result);
|
||||
}
|
||||
|
||||
qDebug() << "backupDirectoryPath=" << _backupDirectoryPath;
|
||||
qDebug() << "persistInterval=" << _persistInterval.count();
|
||||
|
||||
readOptionBool(QString("persistFileDownload"), settingsSectionObject, _persistFileDownload);
|
||||
qDebug() << "persistFileDownload=" << _persistFileDownload;
|
||||
|
|
@ -1128,111 +1120,14 @@ void OctreeServer::run() {
|
|||
}
|
||||
|
||||
void OctreeServer::domainSettingsRequestComplete() {
|
||||
if (_state != OctreeServerState::WaitingForDomainSettings) {
|
||||
qCWarning(octree_server) << "Received domain settings after they have already been received";
|
||||
return;
|
||||
}
|
||||
|
||||
auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
|
||||
packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket");
|
||||
packetReceiver.registerListener(PacketType::OctreeDataNack, this, "handleOctreeDataNackPacket");
|
||||
|
||||
packetReceiver.registerListener(PacketType::OctreeDataFileReply, this, "handleOctreeDataFileReply");
|
||||
packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket");
|
||||
|
||||
qDebug(octree_server) << "Received domain settings";
|
||||
|
||||
readConfiguration();
|
||||
|
||||
_state = OctreeServerState::WaitingForOctreeDataNegotation;
|
||||
|
||||
auto nodeList = DependencyManager::get<NodeList>();
|
||||
const DomainHandler& domainHandler = nodeList->getDomainHandler();
|
||||
|
||||
auto packet = NLPacket::create(PacketType::OctreeDataFileRequest, -1, true, false);
|
||||
|
||||
OctreeUtils::RawOctreeData data;
|
||||
qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath;
|
||||
if (data.readOctreeDataInfoFromFile(_persistAbsoluteFilePath)) {
|
||||
qCDebug(octree_server) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")";
|
||||
packet->writePrimitive(true);
|
||||
auto id = data.id.toRfc4122();
|
||||
packet->write(id);
|
||||
packet->writePrimitive(data.version);
|
||||
} else {
|
||||
qCWarning(octree_server) << "No octree data found";
|
||||
packet->writePrimitive(false);
|
||||
}
|
||||
|
||||
qCDebug(octree_server) << "Sending request for octree data to DS";
|
||||
nodeList->sendPacket(std::move(packet), domainHandler.getSockAddr());
|
||||
}
|
||||
|
||||
void OctreeServer::handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message) {
|
||||
if (_state != OctreeServerState::WaitingForOctreeDataNegotation) {
|
||||
qCWarning(octree_server) << "Server received ocree data file reply but is not currently negotiating.";
|
||||
return;
|
||||
}
|
||||
|
||||
bool includesNewData;
|
||||
message->readPrimitive(&includesNewData);
|
||||
QByteArray replaceData;
|
||||
if (includesNewData) {
|
||||
replaceData = message->readAll();
|
||||
qDebug() << "Got reply to octree data file request, new data sent";
|
||||
} else {
|
||||
qDebug() << "Got reply to octree data file request, current entity data is sufficient";
|
||||
|
||||
OctreeUtils::RawEntityData data;
|
||||
qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath;
|
||||
if (data.readOctreeDataInfoFromFile(_persistAbsoluteFilePath)) {
|
||||
if (data.id.isNull()) {
|
||||
qCDebug(octree_server) << "Current octree data has a null id, updating";
|
||||
data.resetIdAndVersion();
|
||||
|
||||
QFile file(_persistAbsoluteFilePath);
|
||||
if (file.open(QIODevice::WriteOnly)) {
|
||||
auto entityData = data.toGzippedByteArray();
|
||||
file.write(entityData);
|
||||
file.close();
|
||||
} else {
|
||||
qCDebug(octree_server) << "Failed to update octree data";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_state = OctreeServerState::Running;
|
||||
beginRunning(replaceData);
|
||||
}
|
||||
|
||||
void OctreeServer::beginRunning(QByteArray replaceData) {
|
||||
if (_state != OctreeServerState::Running) {
|
||||
qCWarning(octree_server) << "Server is not running";
|
||||
return;
|
||||
}
|
||||
|
||||
auto nodeList = DependencyManager::get<NodeList>();
|
||||
|
||||
// we need to ask the DS about agents so we can ping/reply with them
|
||||
nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer });
|
||||
|
||||
beforeRun(); // after payload has been processed
|
||||
|
||||
connect(nodeList.data(), SIGNAL(nodeAdded(SharedNodePointer)), SLOT(nodeAdded(SharedNodePointer)));
|
||||
connect(nodeList.data(), SIGNAL(nodeKilled(SharedNodePointer)), SLOT(nodeKilled(SharedNodePointer)));
|
||||
|
||||
#ifndef WIN32
|
||||
setvbuf(stdout, NULL, _IOLBF, 0);
|
||||
#endif
|
||||
|
||||
nodeList->linkedDataCreateCallback = [this](Node* node) {
|
||||
auto queryNodeData = createOctreeQueryNode();
|
||||
queryNodeData->init();
|
||||
node->setLinkedData(std::move(queryNodeData));
|
||||
};
|
||||
|
||||
srand((unsigned)time(0));
|
||||
|
||||
// if we want Persistence, set up the local file and persist thread
|
||||
if (_wantPersist) {
|
||||
static const QString ENTITY_PERSIST_EXTENSION = ".json.gz";
|
||||
|
|
@ -1288,40 +1183,40 @@ void OctreeServer::beginRunning(QByteArray replaceData) {
|
|||
}
|
||||
|
||||
auto persistFileDirectory = QFileInfo(_persistAbsoluteFilePath).absolutePath();
|
||||
if (_backupDirectoryPath.isEmpty()) {
|
||||
// Use the persist file's directory to store backups
|
||||
_backupDirectoryPath = persistFileDirectory;
|
||||
} else {
|
||||
// The backup directory has been set.
|
||||
// If relative, make it relative to the entities directory in the application data directory
|
||||
// If absolute, no resolution is necessary
|
||||
QDir backupDirectory { _backupDirectoryPath };
|
||||
QString absoluteBackupDirectory;
|
||||
if (backupDirectory.isRelative()) {
|
||||
absoluteBackupDirectory = QDir(PathUtils::getAppDataFilePath("entities/")).absoluteFilePath(_backupDirectoryPath);
|
||||
absoluteBackupDirectory = QDir(absoluteBackupDirectory).absolutePath();
|
||||
} else {
|
||||
absoluteBackupDirectory = backupDirectory.absolutePath();
|
||||
}
|
||||
backupDirectory = QDir(absoluteBackupDirectory);
|
||||
if (!backupDirectory.exists()) {
|
||||
if (backupDirectory.mkpath(".")) {
|
||||
qDebug() << "Created backup directory";
|
||||
} else {
|
||||
qDebug() << "ERROR creating backup directory, using persist file directory";
|
||||
_backupDirectoryPath = persistFileDirectory;
|
||||
}
|
||||
} else {
|
||||
_backupDirectoryPath = absoluteBackupDirectory;
|
||||
}
|
||||
}
|
||||
qDebug() << "Backups will be stored in: " << _backupDirectoryPath;
|
||||
|
||||
// now set up PersistThread
|
||||
_persistThread = new OctreePersistThread(_tree, _persistAbsoluteFilePath, _backupDirectoryPath, _persistInterval,
|
||||
_wantBackup, _settings, _debugTimestampNow, _persistAsFileType, replaceData);
|
||||
_persistThread->initialize(true);
|
||||
_persistManager = new OctreePersistThread(_tree, _persistAbsoluteFilePath, _persistInterval, _debugTimestampNow,
|
||||
_persistAsFileType);
|
||||
_persistManager->moveToThread(&_persistThread);
|
||||
connect(&_persistThread, &QThread::finished, _persistManager, &QObject::deleteLater);
|
||||
connect(&_persistThread, &QThread::started, _persistManager, &OctreePersistThread::start);
|
||||
connect(_persistManager, &OctreePersistThread::loadCompleted, this, [this]() {
|
||||
beginRunning();
|
||||
});
|
||||
_persistThread.start();
|
||||
} else {
|
||||
beginRunning();
|
||||
}
|
||||
}
|
||||
|
||||
void OctreeServer::beginRunning() {
|
||||
auto nodeList = DependencyManager::get<NodeList>();
|
||||
|
||||
// we need to ask the DS about agents so we can ping/reply with them
|
||||
nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer });
|
||||
|
||||
beforeRun(); // after payload has been processed
|
||||
|
||||
connect(nodeList.data(), &NodeList::nodeAdded, this, &OctreeServer::nodeAdded);
|
||||
connect(nodeList.data(), &NodeList::nodeKilled, this, &OctreeServer::nodeKilled);
|
||||
|
||||
nodeList->linkedDataCreateCallback = [this](Node* node) {
|
||||
auto queryNodeData = createOctreeQueryNode();
|
||||
queryNodeData->init();
|
||||
node->setLinkedData(std::move(queryNodeData));
|
||||
};
|
||||
|
||||
srand((unsigned)time(0));
|
||||
|
||||
// set up our OctreeServerPacketProcessor
|
||||
_octreeInboundPacketProcessor = new OctreeInboundPacketProcessor(this);
|
||||
|
|
@ -1384,7 +1279,7 @@ void OctreeServer::aboutToFinish() {
|
|||
|
||||
qDebug() << qPrintable(_safeServerName) << "inform Octree Inbound Packet Processor that we are shutting down...";
|
||||
|
||||
// we're going down - set the NodeList linkedDataCallback to NULL so we do not create any more OctreeQueryNode objects.
|
||||
// we're going down - set the NodeList linkedDataCallback to nullptr so we do not create any more OctreeQueryNode objects.
|
||||
// This ensures that we don't get any more newly connecting nodes
|
||||
DependencyManager::get<NodeList>()->linkedDataCreateCallback = nullptr;
|
||||
|
||||
|
|
@ -1402,9 +1297,8 @@ void OctreeServer::aboutToFinish() {
|
|||
// which waits on the thread to be done before returning
|
||||
_sendThreads.clear(); // Cleans up all the send threads.
|
||||
|
||||
if (_persistThread) {
|
||||
_persistThread->aboutToFinish();
|
||||
_persistThread->terminating();
|
||||
if (_persistManager) {
|
||||
_persistThread.quit();
|
||||
}
|
||||
|
||||
qDebug() << qPrintable(_safeServerName) << "server ENDING about to finish...";
|
||||
|
|
|
|||
|
|
@ -33,12 +33,6 @@ Q_DECLARE_LOGGING_CATEGORY(octree_server)
|
|||
|
||||
const int DEFAULT_PACKETS_PER_INTERVAL = 2000; // some 120,000 packets per second total
|
||||
|
||||
enum class OctreeServerState {
|
||||
WaitingForDomainSettings,
|
||||
WaitingForOctreeDataNegotation,
|
||||
Running
|
||||
};
|
||||
|
||||
/// Handles assignments of type OctreeServer - sending octrees to various clients.
|
||||
class OctreeServer : public ThreadedAssignment, public HTTPRequestHandler {
|
||||
Q_OBJECT
|
||||
|
|
@ -46,8 +40,6 @@ public:
|
|||
OctreeServer(ReceivedMessage& message);
|
||||
~OctreeServer();
|
||||
|
||||
OctreeServerState _state { OctreeServerState::WaitingForDomainSettings };
|
||||
|
||||
/// allows setting of run arguments
|
||||
void setArguments(int argc, char** argv);
|
||||
|
||||
|
|
@ -68,12 +60,12 @@ public:
|
|||
static void clientConnected() { _clientCount++; }
|
||||
static void clientDisconnected() { _clientCount--; }
|
||||
|
||||
bool isInitialLoadComplete() const { return (_persistThread) ? _persistThread->isInitialLoadComplete() : true; }
|
||||
bool isPersistEnabled() const { return (_persistThread) ? true : false; }
|
||||
quint64 getLoadElapsedTime() const { return (_persistThread) ? _persistThread->getLoadElapsedTime() : 0; }
|
||||
QString getPersistFilename() const { return (_persistThread) ? _persistThread->getPersistFilename() : ""; }
|
||||
QString getPersistFileMimeType() const { return (_persistThread) ? _persistThread->getPersistFileMimeType() : "text/plain"; }
|
||||
QByteArray getPersistFileContents() const { return (_persistThread) ? _persistThread->getPersistFileContents() : QByteArray(); }
|
||||
bool isInitialLoadComplete() const { return (_persistManager) ? _persistManager->isInitialLoadComplete() : true; }
|
||||
bool isPersistEnabled() const { return (_persistManager) ? true : false; }
|
||||
quint64 getLoadElapsedTime() const { return (_persistManager) ? _persistManager->getLoadElapsedTime() : 0; }
|
||||
QString getPersistFilename() const { return (_persistManager) ? _persistManager->getPersistFilename() : ""; }
|
||||
QString getPersistFileMimeType() const { return (_persistManager) ? _persistManager->getPersistFileMimeType() : "text/plain"; }
|
||||
QByteArray getPersistFileContents() const { return (_persistManager) ? _persistManager->getPersistFileContents() : QByteArray(); }
|
||||
|
||||
// Subclasses must implement these methods
|
||||
virtual std::unique_ptr<OctreeQueryNode> createOctreeQueryNode() = 0;
|
||||
|
|
@ -149,7 +141,6 @@ private slots:
|
|||
void domainSettingsRequestComplete();
|
||||
void handleOctreeQueryPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
|
||||
void handleOctreeDataNackPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
|
||||
void handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message);
|
||||
void removeSendThread();
|
||||
|
||||
protected:
|
||||
|
|
@ -171,7 +162,7 @@ protected:
|
|||
QString getConfiguration();
|
||||
QString getStatusLink();
|
||||
|
||||
void beginRunning(QByteArray replaceData);
|
||||
void beginRunning();
|
||||
|
||||
UniqueSendThread createSendThread(const SharedNodePointer& node);
|
||||
virtual UniqueSendThread newSendThread(const SharedNodePointer& node) = 0;
|
||||
|
|
@ -190,7 +181,6 @@ protected:
|
|||
QString _persistFilePath;
|
||||
QString _persistAbsoluteFilePath;
|
||||
QString _persistAsFileType;
|
||||
QString _backupDirectoryPath;
|
||||
int _packetsPerClientPerInterval;
|
||||
int _packetsTotalPerInterval;
|
||||
OctreePointer _tree; // this IS a reaveraging tree
|
||||
|
|
@ -200,13 +190,11 @@ protected:
|
|||
bool _debugTimestampNow;
|
||||
bool _verboseDebug;
|
||||
OctreeInboundPacketProcessor* _octreeInboundPacketProcessor;
|
||||
OctreePersistThread* _persistThread;
|
||||
OctreePersistThread* _persistManager;
|
||||
QThread _persistThread;
|
||||
|
||||
int _persistInterval;
|
||||
bool _wantBackup;
|
||||
std::chrono::milliseconds _persistInterval;
|
||||
bool _persistFileDownload;
|
||||
QString _backupExtensionFormat;
|
||||
int _backupInterval;
|
||||
int _maxBackupVersions;
|
||||
|
||||
time_t _started;
|
||||
|
|
|
|||
|
|
@ -44,6 +44,19 @@ size_t std::hash<EntityItemID>::operator()(const EntityItemID& id) const { retur
|
|||
std::function<bool()> EntityTreeRenderer::_entitiesShouldFadeFunction;
|
||||
std::function<bool()> EntityTreeRenderer::_renderDebugHullsOperator = [] { return false; };
|
||||
|
||||
QString resolveScriptURL(const QString& scriptUrl) {
|
||||
auto normalizedScriptUrl = DependencyManager::get<ResourceManager>()->normalizeURL(scriptUrl);
|
||||
QUrl url { normalizedScriptUrl };
|
||||
if (url.isLocalFile()) {
|
||||
// Outside of the ScriptEngine, /~/ resolves to the /resources directory.
|
||||
// Inside of the ScriptEngine, /~/ resolves to the /scripts directory.
|
||||
// Here we expand local paths in case they are /~/ paths, so they aren't
|
||||
// incorrectly recognized as being located in /scripts when utilized in ScriptEngine.
|
||||
return PathUtils::expandToLocalDataAbsolutePath(url).toString();
|
||||
}
|
||||
return normalizedScriptUrl;
|
||||
}
|
||||
|
||||
EntityTreeRenderer::EntityTreeRenderer(bool wantScripts, AbstractViewStateInterface* viewState,
|
||||
AbstractScriptingServicesInterface* scriptingServices) :
|
||||
_wantScripts(wantScripts),
|
||||
|
|
@ -221,7 +234,7 @@ void EntityTreeRenderer::reloadEntityScripts() {
|
|||
const auto& renderer = entry.second;
|
||||
const auto& entity = renderer->getEntity();
|
||||
if (!entity->getScript().isEmpty()) {
|
||||
_entitiesScriptEngine->loadEntityScript(entity->getEntityItemID(), entity->getScript(), true);
|
||||
_entitiesScriptEngine->loadEntityScript(entity->getEntityItemID(), resolveScriptURL(entity->getScript()), true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -914,8 +927,7 @@ void EntityTreeRenderer::checkAndCallPreload(const EntityItemID& entityID, bool
|
|||
entity->scriptHasUnloaded();
|
||||
}
|
||||
if (shouldLoad) {
|
||||
scriptUrl = DependencyManager::get<ResourceManager>()->normalizeURL(scriptUrl);
|
||||
_entitiesScriptEngine->loadEntityScript(entityID, scriptUrl, reload);
|
||||
_entitiesScriptEngine->loadEntityScript(entityID, resolveScriptURL(scriptUrl), reload);
|
||||
entity->scriptHasPreloaded();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@
|
|||
#include <QJsonObject>
|
||||
#include <QJsonArray>
|
||||
#include <QFileInfo>
|
||||
#include <QSaveFile>
|
||||
#include <QString>
|
||||
#include <QRegularExpression>
|
||||
#include <QRegularExpressionMatch>
|
||||
|
|
@ -972,12 +973,19 @@ bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& e
|
|||
return false;
|
||||
}
|
||||
|
||||
QFile persistFile(fileName);
|
||||
QSaveFile persistFile(fileName);
|
||||
bool success = false;
|
||||
if (persistFile.open(QIODevice::WriteOnly)) {
|
||||
success = persistFile.write(jsonDataForFile) != -1;
|
||||
if (persistFile.write(jsonDataForFile) != -1) {
|
||||
success = persistFile.commit();
|
||||
if (!success) {
|
||||
qCritical() << "Failed to commit to JSON save file:" << persistFile.errorString();
|
||||
}
|
||||
} else {
|
||||
qCritical("Failed to write to JSON file.");
|
||||
}
|
||||
} else {
|
||||
qCritical("Could not write to JSON description of entities.");
|
||||
qCritical("Failed to open JSON file for writing.");
|
||||
}
|
||||
|
||||
return success;
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@
|
|||
#include <QJsonArray>
|
||||
#include <QJsonObject>
|
||||
#include <QJsonDocument>
|
||||
#include <QRegExp>
|
||||
|
||||
#include <NumericalConstants.h>
|
||||
#include <PerfStat.h>
|
||||
|
|
@ -35,32 +36,152 @@
|
|||
#include "OctreeUtils.h"
|
||||
#include "OctreeDataUtils.h"
|
||||
|
||||
const int OctreePersistThread::DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
|
||||
constexpr std::chrono::seconds OctreePersistThread::DEFAULT_PERSIST_INTERVAL { 30 };
|
||||
constexpr std::chrono::milliseconds TIME_BETWEEN_PROCESSING { 10 };
|
||||
|
||||
OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& filename, const QString& backupDirectory, int persistInterval,
|
||||
bool wantBackup, const QJsonObject& settings, bool debugTimestampNow,
|
||||
QString persistAsFileType, const QByteArray& replacementData) :
|
||||
constexpr int MAX_OCTREE_REPLACEMENT_BACKUP_FILES_COUNT { 20 };
|
||||
constexpr int64_t MAX_OCTREE_REPLACEMENT_BACKUP_FILES_SIZE_BYTES { 50 * 1000 * 1000 };
|
||||
|
||||
OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& filename, std::chrono::milliseconds persistInterval,
|
||||
bool debugTimestampNow, QString persistAsFileType) :
|
||||
_tree(tree),
|
||||
_filename(filename),
|
||||
_backupDirectory(backupDirectory),
|
||||
_persistInterval(persistInterval),
|
||||
_lastPersistCheck(std::chrono::steady_clock::now()),
|
||||
_initialLoadComplete(false),
|
||||
_replacementData(replacementData),
|
||||
_loadTimeUSecs(0),
|
||||
_lastCheck(0),
|
||||
_wantBackup(wantBackup),
|
||||
_debugTimestampNow(debugTimestampNow),
|
||||
_lastTimeDebug(0),
|
||||
_persistAsFileType(persistAsFileType)
|
||||
{
|
||||
parseSettings(settings);
|
||||
|
||||
|
||||
// in case the persist filename has an extension that doesn't match the file type
|
||||
QString sansExt = fileNameWithoutExtension(_filename, PERSIST_EXTENSIONS);
|
||||
_filename = sansExt + "." + _persistAsFileType;
|
||||
}
|
||||
|
||||
void OctreePersistThread::start() {
|
||||
cleanupOldReplacementBackups();
|
||||
|
||||
auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
|
||||
packetReceiver.registerListener(PacketType::OctreeDataFileReply, this, "handleOctreeDataFileReply");
|
||||
|
||||
auto nodeList = DependencyManager::get<NodeList>();
|
||||
const DomainHandler& domainHandler = nodeList->getDomainHandler();
|
||||
|
||||
auto packet = NLPacket::create(PacketType::OctreeDataFileRequest, -1, true, false);
|
||||
|
||||
OctreeUtils::RawOctreeData data;
|
||||
qCDebug(octree) << "Reading octree data from" << _filename;
|
||||
if (data.readOctreeDataInfoFromFile(_filename)) {
|
||||
qCDebug(octree) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")";
|
||||
packet->writePrimitive(true);
|
||||
auto id = data.id.toRfc4122();
|
||||
packet->write(id);
|
||||
packet->writePrimitive(data.version);
|
||||
} else {
|
||||
qCWarning(octree) << "No octree data found";
|
||||
packet->writePrimitive(false);
|
||||
}
|
||||
|
||||
qCDebug(octree) << "Sending OctreeDataFileRequest to DS";
|
||||
nodeList->sendPacket(std::move(packet), domainHandler.getSockAddr());
|
||||
}
|
||||
|
||||
void OctreePersistThread::handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message) {
|
||||
if (_initialLoadComplete) {
|
||||
qCWarning(octree) << "Received OctreeDataFileReply after initial load had completed";
|
||||
return;
|
||||
}
|
||||
|
||||
bool includesNewData;
|
||||
message->readPrimitive(&includesNewData);
|
||||
QByteArray replacementData;
|
||||
OctreeUtils::RawOctreeData data;
|
||||
bool hasValidOctreeData { false };
|
||||
if (includesNewData) {
|
||||
replacementData = message->readAll();
|
||||
replaceData(replacementData);
|
||||
hasValidOctreeData = data.readOctreeDataInfoFromFile(_filename);
|
||||
qDebug() << "Got OctreeDataFileReply, new data sent";
|
||||
} else {
|
||||
qDebug() << "Got OctreeDataFileReply, current entity data is sufficient";
|
||||
|
||||
OctreeUtils::RawEntityData data;
|
||||
qCDebug(octree) << "Reading octree data from" << _filename;
|
||||
if (data.readOctreeDataInfoFromFile(_filename)) {
|
||||
hasValidOctreeData = true;
|
||||
if (data.id.isNull()) {
|
||||
qCDebug(octree) << "Current octree data has a null id, updating";
|
||||
data.resetIdAndVersion();
|
||||
|
||||
QFile file(_filename);
|
||||
if (file.open(QIODevice::WriteOnly)) {
|
||||
auto entityData = data.toGzippedByteArray();
|
||||
file.write(entityData);
|
||||
file.close();
|
||||
} else {
|
||||
qCDebug(octree) << "Failed to update octree data";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
quint64 loadStarted = usecTimestampNow();
|
||||
qCDebug(octree) << "loading Octrees from file: " << _filename << "...";
|
||||
|
||||
if (hasValidOctreeData) {
|
||||
qDebug() << "Setting entity version info to: " << data.id << data.version;
|
||||
_tree->setOctreeVersionInfo(data.id, data.version);
|
||||
}
|
||||
|
||||
bool persistentFileRead;
|
||||
|
||||
_tree->withWriteLock([&] {
|
||||
PerformanceWarning warn(true, "Loading Octree File", true);
|
||||
|
||||
persistentFileRead = _tree->readFromFile(_filename.toLocal8Bit().constData());
|
||||
_tree->pruneTree();
|
||||
});
|
||||
|
||||
quint64 loadDone = usecTimestampNow();
|
||||
_loadTimeUSecs = loadDone - loadStarted;
|
||||
|
||||
_tree->clearDirtyBit(); // the tree is clean since we just loaded it
|
||||
qCDebug(octree, "DONE loading Octrees from file... fileRead=%s", debug::valueOf(persistentFileRead));
|
||||
|
||||
unsigned long nodeCount = OctreeElement::getNodeCount();
|
||||
unsigned long internalNodeCount = OctreeElement::getInternalNodeCount();
|
||||
unsigned long leafNodeCount = OctreeElement::getLeafNodeCount();
|
||||
qCDebug(octree, "Nodes after loading scene %lu nodes %lu internal %lu leaves", nodeCount, internalNodeCount, leafNodeCount);
|
||||
|
||||
bool wantDebug = false;
|
||||
if (wantDebug) {
|
||||
double usecPerGet = (double)OctreeElement::getGetChildAtIndexTime()
|
||||
/ (double)OctreeElement::getGetChildAtIndexCalls();
|
||||
qCDebug(octree) << "getChildAtIndexCalls=" << OctreeElement::getGetChildAtIndexCalls()
|
||||
<< " getChildAtIndexTime=" << OctreeElement::getGetChildAtIndexTime() << " perGet=" << usecPerGet;
|
||||
|
||||
double usecPerSet = (double)OctreeElement::getSetChildAtIndexTime()
|
||||
/ (double)OctreeElement::getSetChildAtIndexCalls();
|
||||
qCDebug(octree) << "setChildAtIndexCalls=" << OctreeElement::getSetChildAtIndexCalls()
|
||||
<< " setChildAtIndexTime=" << OctreeElement::getSetChildAtIndexTime() << " perSet=" << usecPerSet;
|
||||
}
|
||||
|
||||
_initialLoadComplete = true;
|
||||
|
||||
// Since we just loaded the persistent file, we can consider ourselves as having just persisted
|
||||
_lastPersistCheck = std::chrono::steady_clock::now();
|
||||
|
||||
if (replacementData.isNull()) {
|
||||
sendLatestEntityDataToDS();
|
||||
}
|
||||
|
||||
QTimer::singleShot(TIME_BETWEEN_PROCESSING.count(), this, &OctreePersistThread::process);
|
||||
|
||||
emit loadCompleted();
|
||||
}
|
||||
|
||||
|
||||
QString OctreePersistThread::getPersistFileMimeType() const {
|
||||
if (_persistAsFileType == "json") {
|
||||
return "application/json";
|
||||
|
|
@ -70,72 +191,6 @@ QString OctreePersistThread::getPersistFileMimeType() const {
|
|||
return "";
|
||||
}
|
||||
|
||||
void OctreePersistThread::parseSettings(const QJsonObject& settings) {
|
||||
if (settings["backups"].isArray()) {
|
||||
const QJsonArray& backupRules = settings["backups"].toArray();
|
||||
qCDebug(octree) << "BACKUP RULES:";
|
||||
|
||||
foreach (const QJsonValue& value, backupRules) {
|
||||
|
||||
QJsonObject obj = value.toObject();
|
||||
|
||||
int interval = 0;
|
||||
int count = 0;
|
||||
|
||||
QJsonValue intervalVal = obj["backupInterval"];
|
||||
if (intervalVal.isString()) {
|
||||
interval = intervalVal.toString().toInt();
|
||||
} else {
|
||||
interval = intervalVal.toInt();
|
||||
}
|
||||
|
||||
QJsonValue countVal = obj["maxBackupVersions"];
|
||||
if (countVal.isString()) {
|
||||
count = countVal.toString().toInt();
|
||||
} else {
|
||||
count = countVal.toInt();
|
||||
}
|
||||
|
||||
qCDebug(octree) << " Name:" << obj["Name"].toString();
|
||||
qCDebug(octree) << " format:" << obj["format"].toString();
|
||||
qCDebug(octree) << " interval:" << interval;
|
||||
qCDebug(octree) << " count:" << count;
|
||||
|
||||
BackupRule newRule = { obj["Name"].toString(), interval, obj["format"].toString(), count, 0};
|
||||
|
||||
newRule.lastBackup = getMostRecentBackupTimeInUsecs(obj["format"].toString());
|
||||
|
||||
if (newRule.lastBackup > 0) {
|
||||
quint64 now = usecTimestampNow();
|
||||
quint64 sinceLastBackup = now - newRule.lastBackup;
|
||||
qCDebug(octree) << " lastBackup:" << qPrintable(formatUsecTime(sinceLastBackup)) << "ago";
|
||||
} else {
|
||||
qCDebug(octree) << " lastBackup: NEVER";
|
||||
}
|
||||
|
||||
_backupRules << newRule;
|
||||
}
|
||||
} else {
|
||||
qCDebug(octree) << "BACKUP RULES: NONE";
|
||||
}
|
||||
}
|
||||
|
||||
quint64 OctreePersistThread::getMostRecentBackupTimeInUsecs(const QString& format) {
|
||||
|
||||
quint64 mostRecentBackupInUsecs = 0;
|
||||
|
||||
QString mostRecentBackupFileName;
|
||||
QDateTime mostRecentBackupTime;
|
||||
|
||||
bool recentBackup = getMostRecentBackup(format, mostRecentBackupFileName, mostRecentBackupTime);
|
||||
|
||||
if (recentBackup) {
|
||||
mostRecentBackupInUsecs = mostRecentBackupTime.toMSecsSinceEpoch() * USECS_PER_MSEC;
|
||||
}
|
||||
|
||||
return mostRecentBackupInUsecs;
|
||||
}
|
||||
|
||||
void OctreePersistThread::replaceData(QByteArray data) {
|
||||
backupCurrentFile();
|
||||
|
||||
|
|
@ -167,123 +222,24 @@ bool OctreePersistThread::backupCurrentFile() {
|
|||
return true;
|
||||
}
|
||||
|
||||
bool OctreePersistThread::process() {
|
||||
void OctreePersistThread::process() {
|
||||
_tree->update();
|
||||
|
||||
if (!_initialLoadComplete) {
|
||||
quint64 loadStarted = usecTimestampNow();
|
||||
qCDebug(octree) << "loading Octrees from file: " << _filename << "...";
|
||||
auto now = std::chrono::steady_clock::now();
|
||||
auto timeSinceLastPersist = now - _lastPersistCheck;
|
||||
|
||||
if (!_replacementData.isNull()) {
|
||||
replaceData(_replacementData);
|
||||
}
|
||||
|
||||
OctreeUtils::RawOctreeData data;
|
||||
if (data.readOctreeDataInfoFromFile(_filename)) {
|
||||
qDebug() << "Setting entity version info to: " << data.id << data.dataVersion;
|
||||
_tree->setOctreeVersionInfo(data.id, data.dataVersion);
|
||||
}
|
||||
|
||||
bool persistentFileRead;
|
||||
|
||||
_tree->withWriteLock([&] {
|
||||
PerformanceWarning warn(true, "Loading Octree File", true);
|
||||
|
||||
// First check to make sure "lock" file doesn't exist. If it does exist, then
|
||||
// our last save crashed during the save, and we want to load our most recent backup.
|
||||
QString lockFileName = _filename + ".lock";
|
||||
std::ifstream lockFile(qPrintable(lockFileName), std::ios::in | std::ios::binary | std::ios::ate);
|
||||
if (lockFile.is_open()) {
|
||||
qCDebug(octree) << "WARNING: Octree lock file detected at startup:" << lockFileName;
|
||||
|
||||
lockFile.close();
|
||||
qCDebug(octree) << "Removing lock file:" << lockFileName;
|
||||
remove(qPrintable(lockFileName));
|
||||
qCDebug(octree) << "Lock file removed:" << lockFileName;
|
||||
}
|
||||
|
||||
persistentFileRead = _tree->readFromFile(qPrintable(_filename.toLocal8Bit()));
|
||||
_tree->pruneTree();
|
||||
});
|
||||
|
||||
quint64 loadDone = usecTimestampNow();
|
||||
_loadTimeUSecs = loadDone - loadStarted;
|
||||
|
||||
_tree->clearDirtyBit(); // the tree is clean since we just loaded it
|
||||
qCDebug(octree, "DONE loading Octrees from file... fileRead=%s", debug::valueOf(persistentFileRead));
|
||||
|
||||
unsigned long nodeCount = OctreeElement::getNodeCount();
|
||||
unsigned long internalNodeCount = OctreeElement::getInternalNodeCount();
|
||||
unsigned long leafNodeCount = OctreeElement::getLeafNodeCount();
|
||||
qCDebug(octree, "Nodes after loading scene %lu nodes %lu internal %lu leaves", nodeCount, internalNodeCount, leafNodeCount);
|
||||
|
||||
bool wantDebug = false;
|
||||
if (wantDebug) {
|
||||
double usecPerGet = (double)OctreeElement::getGetChildAtIndexTime()
|
||||
/ (double)OctreeElement::getGetChildAtIndexCalls();
|
||||
qCDebug(octree) << "getChildAtIndexCalls=" << OctreeElement::getGetChildAtIndexCalls()
|
||||
<< " getChildAtIndexTime=" << OctreeElement::getGetChildAtIndexTime() << " perGet=" << usecPerGet;
|
||||
|
||||
double usecPerSet = (double)OctreeElement::getSetChildAtIndexTime()
|
||||
/ (double)OctreeElement::getSetChildAtIndexCalls();
|
||||
qCDebug(octree) << "setChildAtIndexCalls=" << OctreeElement::getSetChildAtIndexCalls()
|
||||
<< " setChildAtIndexTime=" << OctreeElement::getSetChildAtIndexTime() << " perSet=" << usecPerSet;
|
||||
}
|
||||
|
||||
_initialLoadComplete = true;
|
||||
|
||||
// Since we just loaded the persistent file, we can consider ourselves as having "just checked" for persistance.
|
||||
_lastCheck = usecTimestampNow(); // we just loaded, no need to save again
|
||||
|
||||
// This last persist time is not really used until the file is actually persisted. It is only
|
||||
// used in formatting the backup filename in cases of non-rolling backup names. However, we don't
|
||||
// want an uninitialized value for this, so we set it to the current time (startup of the server)
|
||||
time(&_lastPersistTime);
|
||||
|
||||
if (_replacementData.isNull()) {
|
||||
sendLatestEntityDataToDS();
|
||||
}
|
||||
_replacementData.clear();
|
||||
|
||||
emit loadCompleted();
|
||||
if (timeSinceLastPersist > _persistInterval) {
|
||||
_lastPersistCheck = now;
|
||||
persist();
|
||||
}
|
||||
|
||||
if (isStillRunning()) {
|
||||
quint64 MSECS_TO_USECS = 1000;
|
||||
quint64 USECS_TO_SLEEP = 10 * MSECS_TO_USECS; // every 10ms
|
||||
std::this_thread::sleep_for(std::chrono::microseconds(USECS_TO_SLEEP));
|
||||
|
||||
// do our updates then check to save...
|
||||
_tree->update();
|
||||
|
||||
quint64 now = usecTimestampNow();
|
||||
quint64 sinceLastSave = now - _lastCheck;
|
||||
quint64 intervalToCheck = _persistInterval * MSECS_TO_USECS;
|
||||
|
||||
if (sinceLastSave > intervalToCheck) {
|
||||
_lastCheck = now;
|
||||
persist();
|
||||
}
|
||||
}
|
||||
|
||||
// if we were asked to debugTimestampNow do that now...
|
||||
if (_debugTimestampNow) {
|
||||
quint64 now = usecTimestampNow();
|
||||
quint64 sinceLastDebug = now - _lastTimeDebug;
|
||||
quint64 DEBUG_TIMESTAMP_INTERVAL = 600000000; // every 10 minutes
|
||||
|
||||
if (sinceLastDebug > DEBUG_TIMESTAMP_INTERVAL) {
|
||||
_lastTimeDebug = usecTimestampNow(true); // ask for debug output
|
||||
}
|
||||
|
||||
}
|
||||
return isStillRunning(); // keep running till they terminate us
|
||||
QTimer::singleShot(TIME_BETWEEN_PROCESSING.count(), this, &OctreePersistThread::process);
|
||||
}
|
||||
|
||||
void OctreePersistThread::aboutToFinish() {
|
||||
qCDebug(octree) << "Persist thread about to finish...";
|
||||
persist();
|
||||
qCDebug(octree) << "Persist thread done with about to finish...";
|
||||
_stopThread = true;
|
||||
}
|
||||
|
||||
QByteArray OctreePersistThread::getPersistFileContents() const {
|
||||
|
|
@ -295,6 +251,36 @@ QByteArray OctreePersistThread::getPersistFileContents() const {
|
|||
return fileContents;
|
||||
}
|
||||
|
||||
void OctreePersistThread::cleanupOldReplacementBackups() {
|
||||
QRegExp filenameRegex { ".*\\.backup\\.\\d{8}-\\d{6}$" };
|
||||
QFileInfo persistFile { _filename };
|
||||
QDir backupDir { persistFile.absolutePath() };
|
||||
backupDir.setSorting(QDir::SortFlag::Time);
|
||||
backupDir.setFilter(QDir::Filter::Files);
|
||||
qDebug() << "Scanning backups for cleanup:" << backupDir.absolutePath();
|
||||
|
||||
int count = 0;
|
||||
int64_t totalSize = 0;
|
||||
for (auto fileInfo : backupDir.entryInfoList()) {
|
||||
auto absPath = fileInfo.absoluteFilePath();
|
||||
qDebug() << " Found:" << absPath;
|
||||
if (filenameRegex.exactMatch(absPath)) {
|
||||
if (count >= MAX_OCTREE_REPLACEMENT_BACKUP_FILES_COUNT || totalSize > MAX_OCTREE_REPLACEMENT_BACKUP_FILES_SIZE_BYTES) {
|
||||
qDebug() << " Removing:" << absPath;
|
||||
QFile backup(absPath);
|
||||
if (backup.remove()) {
|
||||
qDebug() << " Removed backup:" << absPath;
|
||||
} else {
|
||||
qWarning() << " Failed to remove backup:" << absPath;
|
||||
}
|
||||
}
|
||||
totalSize += fileInfo.size();
|
||||
count++;
|
||||
}
|
||||
}
|
||||
qDebug() << "Found" << count << "backups";
|
||||
}
|
||||
|
||||
void OctreePersistThread::persist() {
|
||||
if (_tree->isDirty() && _initialLoadComplete) {
|
||||
|
||||
|
|
@ -304,27 +290,14 @@ void OctreePersistThread::persist() {
|
|||
qCDebug(octree) << "DONE pruning Octree before saving...";
|
||||
});
|
||||
|
||||
qCDebug(octree) << "persist operation calling backup...";
|
||||
backup(); // handle backup if requested
|
||||
qCDebug(octree) << "persist operation DONE with backup...";
|
||||
|
||||
_tree->incrementPersistDataVersion();
|
||||
|
||||
// create our "lock" file to indicate we're saving.
|
||||
QString lockFileName = _filename + ".lock";
|
||||
std::ofstream lockFile(qPrintable(lockFileName), std::ios::out|std::ios::binary);
|
||||
if(lockFile.is_open()) {
|
||||
qCDebug(octree) << "saving Octree lock file created at:" << lockFileName;
|
||||
|
||||
_tree->writeToFile(qPrintable(_filename), NULL, _persistAsFileType);
|
||||
time(&_lastPersistTime);
|
||||
qCDebug(octree) << "Saving Octree data to:" << _filename;
|
||||
if (_tree->writeToFile(_filename.toLocal8Bit().constData(), nullptr, _persistAsFileType)) {
|
||||
_tree->clearDirtyBit(); // tree is clean after saving
|
||||
qCDebug(octree) << "DONE saving Octree to file...";
|
||||
|
||||
lockFile.close();
|
||||
qCDebug(octree) << "saving Octree lock file closed:" << lockFileName;
|
||||
remove(qPrintable(lockFileName));
|
||||
qCDebug(octree) << "saving Octree lock file removed:" << lockFileName;
|
||||
qCDebug(octree) << "DONE persisting Octree data to" << _filename;
|
||||
} else {
|
||||
qCWarning(octree) << "Failed to persist Octree data to" << _filename;
|
||||
}
|
||||
|
||||
sendLatestEntityDataToDS();
|
||||
|
|
@ -345,197 +318,3 @@ void OctreePersistThread::sendLatestEntityDataToDS() {
|
|||
qCWarning(octree) << "Failed to persist octree to DS";
|
||||
}
|
||||
}
|
||||
|
||||
void OctreePersistThread::restoreFromMostRecentBackup() {
|
||||
qCDebug(octree) << "Restoring from most recent backup...";
|
||||
|
||||
QString mostRecentBackupFileName;
|
||||
QDateTime mostRecentBackupTime;
|
||||
|
||||
bool recentBackup = getMostRecentBackup(QString(""), mostRecentBackupFileName, mostRecentBackupTime);
|
||||
|
||||
// If we found a backup file, restore from that file.
|
||||
if (recentBackup) {
|
||||
qCDebug(octree) << "BEST backup file:" << mostRecentBackupFileName << " last modified:" << mostRecentBackupTime.toString();
|
||||
|
||||
qCDebug(octree) << "Removing old file:" << _filename;
|
||||
remove(qPrintable(_filename));
|
||||
|
||||
qCDebug(octree) << "Restoring backup file " << mostRecentBackupFileName << "...";
|
||||
bool result = QFile::copy(mostRecentBackupFileName, _filename);
|
||||
if (result) {
|
||||
qCDebug(octree) << "DONE restoring backup file " << mostRecentBackupFileName << "to" << _filename << "...";
|
||||
} else {
|
||||
qCDebug(octree) << "ERROR while restoring backup file " << mostRecentBackupFileName << "to" << _filename << "...";
|
||||
perror("ERROR while restoring backup file");
|
||||
}
|
||||
} else {
|
||||
qCDebug(octree) << "NO BEST backup file found.";
|
||||
}
|
||||
}
|
||||
|
||||
bool OctreePersistThread::getMostRecentBackup(const QString& format,
|
||||
QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime) {
|
||||
|
||||
// Based on our backup file name, determine the path and file name pattern for backup files
|
||||
QFileInfo persistFileInfo(_filename);
|
||||
QString path = _backupDirectory;
|
||||
QString fileNamePart = persistFileInfo.fileName();
|
||||
|
||||
QStringList filters;
|
||||
|
||||
if (format.isEmpty()) {
|
||||
// Create a file filter that will find all backup files of this extension format
|
||||
foreach(const BackupRule& rule, _backupRules) {
|
||||
QString backupExtension = rule.extensionFormat;
|
||||
backupExtension.replace(QRegExp("%."), "*");
|
||||
QString backupFileNamePart = fileNamePart + backupExtension;
|
||||
filters << backupFileNamePart;
|
||||
}
|
||||
} else {
|
||||
QString backupExtension = format;
|
||||
backupExtension.replace(QRegExp("%."), "*");
|
||||
QString backupFileNamePart = fileNamePart + backupExtension;
|
||||
filters << backupFileNamePart;
|
||||
}
|
||||
|
||||
bool bestBackupFound = false;
|
||||
QString bestBackupFile;
|
||||
QDateTime bestBackupFileTime;
|
||||
|
||||
// Iterate over all of the backup files in the persist location
|
||||
QDirIterator dirIterator(path, filters, QDir::Files|QDir::NoSymLinks, QDirIterator::NoIteratorFlags);
|
||||
while(dirIterator.hasNext()) {
|
||||
|
||||
dirIterator.next();
|
||||
QDateTime lastModified = dirIterator.fileInfo().lastModified();
|
||||
|
||||
// Based on last modified date, track the most recently modified file as the best backup
|
||||
if (lastModified > bestBackupFileTime) {
|
||||
bestBackupFound = true;
|
||||
bestBackupFile = dirIterator.filePath();
|
||||
bestBackupFileTime = lastModified;
|
||||
}
|
||||
}
|
||||
|
||||
// If we found a backup then return the results
|
||||
if (bestBackupFound) {
|
||||
mostRecentBackupFileName = bestBackupFile;
|
||||
mostRecentBackupTime = bestBackupFileTime;
|
||||
}
|
||||
return bestBackupFound;
|
||||
}
|
||||
|
||||
void OctreePersistThread::rollOldBackupVersions(const BackupRule& rule) {
|
||||
|
||||
if (rule.extensionFormat.contains("%N")) {
|
||||
if (rule.maxBackupVersions > 0) {
|
||||
qCDebug(octree) << "Rolling old backup versions for rule" << rule.name << "...";
|
||||
|
||||
QString backupFileName = _backupDirectory + "/" + QUrl(_filename).fileName();
|
||||
|
||||
// Delete maximum rolling file because rename() fails on Windows if target exists
|
||||
QString backupMaxExtensionN = rule.extensionFormat;
|
||||
backupMaxExtensionN.replace(QString("%N"), QString::number(rule.maxBackupVersions));
|
||||
QString backupMaxFilenameN = backupFileName + backupMaxExtensionN;
|
||||
QFile backupMaxFileN(backupMaxFilenameN);
|
||||
if (backupMaxFileN.exists()) {
|
||||
int result = remove(qPrintable(backupMaxFilenameN));
|
||||
if (result != 0) {
|
||||
qCDebug(octree) << "ERROR deleting old backup file " << backupMaxFilenameN;
|
||||
}
|
||||
}
|
||||
|
||||
for(int n = rule.maxBackupVersions - 1; n > 0; n--) {
|
||||
QString backupExtensionN = rule.extensionFormat;
|
||||
QString backupExtensionNplusOne = rule.extensionFormat;
|
||||
backupExtensionN.replace(QString("%N"), QString::number(n));
|
||||
backupExtensionNplusOne.replace(QString("%N"), QString::number(n+1));
|
||||
|
||||
QString backupFilenameN = findMostRecentFileExtension(backupFileName, PERSIST_EXTENSIONS) + backupExtensionN;
|
||||
QString backupFilenameNplusOne = backupFileName + backupExtensionNplusOne;
|
||||
|
||||
QFile backupFileN(backupFilenameN);
|
||||
|
||||
if (backupFileN.exists()) {
|
||||
qCDebug(octree) << "rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
|
||||
int result = rename(qPrintable(backupFilenameN), qPrintable(backupFilenameNplusOne));
|
||||
if (result == 0) {
|
||||
qCDebug(octree) << "DONE rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
|
||||
} else {
|
||||
qCDebug(octree) << "ERROR in rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
|
||||
perror("ERROR in rolling backup file");
|
||||
}
|
||||
}
|
||||
}
|
||||
qCDebug(octree) << "Done rolling old backup versions...";
|
||||
} else {
|
||||
qCDebug(octree) << "Rolling backups for rule" << rule.name << "."
|
||||
<< " Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]."
|
||||
<< " No need to roll backups...";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void OctreePersistThread::backup() {
|
||||
qCDebug(octree) << "backup operation wantBackup:" << _wantBackup;
|
||||
if (_wantBackup) {
|
||||
quint64 now = usecTimestampNow();
|
||||
|
||||
for(int i = 0; i < _backupRules.count(); i++) {
|
||||
BackupRule& rule = _backupRules[i];
|
||||
|
||||
quint64 sinceLastBackup = now - rule.lastBackup;
|
||||
quint64 SECS_TO_USECS = 1000 * 1000;
|
||||
quint64 intervalToBackup = rule.interval * SECS_TO_USECS;
|
||||
|
||||
qCDebug(octree) << "Checking [" << rule.name << "] - Time since last backup [" << sinceLastBackup << "] " <<
|
||||
"compared to backup interval [" << intervalToBackup << "]...";
|
||||
|
||||
if (sinceLastBackup > intervalToBackup) {
|
||||
qCDebug(octree) << "Time since last backup [" << sinceLastBackup << "] for rule [" << rule.name
|
||||
<< "] exceeds backup interval [" << intervalToBackup << "] doing backup now...";
|
||||
|
||||
struct tm* localTime = localtime(&_lastPersistTime);
|
||||
|
||||
QString backupFileName = _backupDirectory + "/" + QUrl(_filename).fileName();
|
||||
|
||||
// check to see if they asked for version rolling format
|
||||
if (rule.extensionFormat.contains("%N")) {
|
||||
rollOldBackupVersions(rule); // rename all the old backup files accordingly
|
||||
QString backupExtension = rule.extensionFormat;
|
||||
backupExtension.replace(QString("%N"), QString("1"));
|
||||
backupFileName += backupExtension;
|
||||
} else {
|
||||
char backupExtension[256];
|
||||
strftime(backupExtension, sizeof(backupExtension), qPrintable(rule.extensionFormat), localTime);
|
||||
backupFileName += backupExtension;
|
||||
}
|
||||
|
||||
if (rule.maxBackupVersions > 0) {
|
||||
QFile persistFile(_filename);
|
||||
if (persistFile.exists()) {
|
||||
qCDebug(octree) << "backing up persist file " << _filename << "to" << backupFileName << "...";
|
||||
bool result = QFile::copy(_filename, backupFileName);
|
||||
if (result) {
|
||||
qCDebug(octree) << "DONE backing up persist file...";
|
||||
rule.lastBackup = now; // only record successful backup in this case.
|
||||
} else {
|
||||
qCDebug(octree) << "ERROR in backing up persist file...";
|
||||
perror("ERROR in backing up persist file");
|
||||
}
|
||||
} else {
|
||||
qCDebug(octree) << "persist file " << _filename << " does not exist. " <<
|
||||
"nothing to backup for this rule ["<< rule.name << "]...";
|
||||
}
|
||||
} else {
|
||||
qCDebug(octree) << "This backup rule" << rule.name
|
||||
<< " has Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]."
|
||||
<< " There are no backups to be done...";
|
||||
}
|
||||
} else {
|
||||
qCDebug(octree) << "Backup not needed for this rule ["<< rule.name << "]...";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
#include <GenericThread.h>
|
||||
#include "Octree.h"
|
||||
|
||||
class OctreePersistThread : public GenericThread {
|
||||
class OctreePersistThread : public QObject {
|
||||
Q_OBJECT
|
||||
public:
|
||||
class BackupRule {
|
||||
|
|
@ -30,37 +30,37 @@ public:
|
|||
quint64 lastBackup;
|
||||
};
|
||||
|
||||
static const int DEFAULT_PERSIST_INTERVAL;
|
||||
static const std::chrono::seconds DEFAULT_PERSIST_INTERVAL;
|
||||
|
||||
OctreePersistThread(OctreePointer tree, const QString& filename, const QString& backupDirectory,
|
||||
int persistInterval = DEFAULT_PERSIST_INTERVAL, bool wantBackup = false,
|
||||
const QJsonObject& settings = QJsonObject(), bool debugTimestampNow = false,
|
||||
QString persistAsFileType = "json.gz", const QByteArray& replacementData = QByteArray());
|
||||
OctreePersistThread(OctreePointer tree,
|
||||
const QString& filename,
|
||||
std::chrono::milliseconds persistInterval = DEFAULT_PERSIST_INTERVAL,
|
||||
bool debugTimestampNow = false,
|
||||
QString persistAsFileType = "json.gz");
|
||||
|
||||
bool isInitialLoadComplete() const { return _initialLoadComplete; }
|
||||
quint64 getLoadElapsedTime() const { return _loadTimeUSecs; }
|
||||
|
||||
void aboutToFinish(); /// call this to inform the persist thread that the owner is about to finish to support final persist
|
||||
|
||||
QString getPersistFilename() const { return _filename; }
|
||||
QString getPersistFileMimeType() const;
|
||||
QByteArray getPersistFileContents() const;
|
||||
|
||||
void aboutToFinish(); /// call this to inform the persist thread that the owner is about to finish to support final persist
|
||||
|
||||
public slots:
|
||||
void start();
|
||||
|
||||
signals:
|
||||
void loadCompleted();
|
||||
|
||||
protected:
|
||||
/// Implements generic processing behavior for this thread.
|
||||
virtual bool process() override;
|
||||
protected slots:
|
||||
void process();
|
||||
void handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message);
|
||||
|
||||
protected:
|
||||
void persist();
|
||||
void backup();
|
||||
void rollOldBackupVersions(const BackupRule& rule);
|
||||
void restoreFromMostRecentBackup();
|
||||
bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime);
|
||||
quint64 getMostRecentBackupTimeInUsecs(const QString& format);
|
||||
void parseSettings(const QJsonObject& settings);
|
||||
bool backupCurrentFile();
|
||||
void cleanupOldReplacementBackups();
|
||||
|
||||
void replaceData(QByteArray data);
|
||||
void sendLatestEntityDataToDS();
|
||||
|
|
@ -68,18 +68,12 @@ protected:
|
|||
private:
|
||||
OctreePointer _tree;
|
||||
QString _filename;
|
||||
QString _backupDirectory;
|
||||
int _persistInterval;
|
||||
std::chrono::milliseconds _persistInterval;
|
||||
std::chrono::steady_clock::time_point _lastPersistCheck;
|
||||
bool _initialLoadComplete;
|
||||
QByteArray _replacementData;
|
||||
|
||||
quint64 _loadTimeUSecs;
|
||||
|
||||
time_t _lastPersistTime;
|
||||
quint64 _lastCheck;
|
||||
bool _wantBackup;
|
||||
QVector<BackupRule> _backupRules;
|
||||
|
||||
bool _debugTimestampNow;
|
||||
quint64 _lastTimeDebug;
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@
|
|||
"cheerio": "^0.19.0",
|
||||
"electron-log": "1.1.1",
|
||||
"extend": "^3.0.0",
|
||||
"fs-extra": "^1.0.0",
|
||||
"fs-extra": "^6.0.0",
|
||||
"node-notifier": "^5.2.1",
|
||||
"os-homedir": "^1.0.1",
|
||||
"request": "^2.85.0",
|
||||
|
|
|
|||
|
|
@ -115,17 +115,43 @@ const UPDATER_LOCK_FULL_PATH = getRootHifiDataDirectory() + "/" + UPDATER_LOCK_F
|
|||
|
||||
// Configure log
|
||||
global.log = require('electron-log');
|
||||
const logFile = getApplicationDataDirectory(true) + '/log.txt';
|
||||
const oldLogFile = path.join(getApplicationDataDirectory(), '/log.txt');
|
||||
const logFile = path.join(getApplicationDataDirectory(true), '/log.txt');
|
||||
if (oldLogFile != logFile && fs.existsSync(oldLogFile)) {
|
||||
if (!fs.existsSync(oldLogFile)) {
|
||||
fs.moveSync(oldLogFile, logFile);
|
||||
} else {
|
||||
fs.remove(oldLogFile);
|
||||
}
|
||||
}
|
||||
fs.ensureFileSync(logFile); // Ensure file exists
|
||||
log.transports.file.maxSize = 5 * 1024 * 1024;
|
||||
log.transports.file.file = logFile;
|
||||
|
||||
log.debug("build info", buildInfo);
|
||||
log.debug("Root hifi directory is: ", getRootHifiDataDirectory());
|
||||
log.debug("App Data directory:", getApplicationDataDirectory());
|
||||
fs.ensureDirSync(getApplicationDataDirectory());
|
||||
|
||||
var oldLogPath = path.join(getApplicationDataDirectory(), '/logs');
|
||||
var logPath = path.join(getApplicationDataDirectory(true), '/logs');
|
||||
if (oldLogPath != logPath && fs.existsSync(oldLogPath)) {
|
||||
if (!fs.existsSync(oldLogPath)) {
|
||||
fs.moveSync(oldLogPath, logPath);
|
||||
} else {
|
||||
fs.remove(oldLogPath);
|
||||
}
|
||||
}
|
||||
fs.ensureDirSync(logPath);
|
||||
log.debug("Log directory:", logPath);
|
||||
|
||||
const configPath = path.join(getApplicationDataDirectory(), 'config.json');
|
||||
var userConfig = new Config();
|
||||
userConfig.load(configPath);
|
||||
|
||||
|
||||
const ipcMain = electron.ipcMain;
|
||||
|
||||
|
||||
var isShuttingDown = false;
|
||||
function shutdown() {
|
||||
log.debug("Normal shutdown (isShuttingDown: " + isShuttingDown + ")");
|
||||
|
|
@ -232,27 +258,6 @@ function deleteOldFiles(directoryPath, maxAgeInSeconds, filenameRegex) {
|
|||
}
|
||||
}
|
||||
|
||||
var oldLogPath = path.join(getApplicationDataDirectory(), '/logs');
|
||||
var logPath = path.join(getApplicationDataDirectory(true), '/logs');
|
||||
|
||||
if (oldLogPath != logPath) {
|
||||
console.log("Migrating old logs from " + oldLogPath + " to " + logPath);
|
||||
fs.copy(oldLogPath, logPath, err => {
|
||||
if (err) {
|
||||
console.error(err);
|
||||
} else {
|
||||
console.log('success!');
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
log.debug("Log directory:", logPath);
|
||||
log.debug("Data directory:", getRootHifiDataDirectory());
|
||||
|
||||
const configPath = path.join(getApplicationDataDirectory(), 'config.json');
|
||||
var userConfig = new Config();
|
||||
userConfig.load(configPath);
|
||||
|
||||
app.setAppUserModelId(buildInfo.appUserModelId);
|
||||
|
||||
// print out uncaught exceptions in the console
|
||||
|
|
|
|||
|
|
@ -205,7 +205,7 @@ Rectangle {
|
|||
// Spectator Camera Preview
|
||||
Hifi.ResourceImageItem {
|
||||
id: spectatorCameraPreview;
|
||||
visible: masterSwitch.checked;
|
||||
visible: masterSwitch.checked && !root.processing360Snapshot;
|
||||
url: showCameraView.checked || !HMD.active ? "resource://spectatorCameraFrame" : "resource://hmdPreviewFrame";
|
||||
ready: masterSwitch.checked;
|
||||
mirrorVertically: true;
|
||||
|
|
|
|||
Loading…
Reference in a new issue