Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-20 18:23:54 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #3798 from ZappoMan/persistThreadImprovements

Browse source 
Persist thread improvements/Backup support
This commit is contained in:
Clément Brisset 2014-11-14 17:25:17 -08:00
commit c80dff064f
6 changed files with 222 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
assignment-client/src/octree/OctreeServer.cpp
View file

@ -1020,7 +1020,7 @@ void OctreeServer::readConfiguration() {
bool noPersist;
readOptionBool(QString("NoPersist"), settingsSectionObject, noPersist);
_wantPersist = !noPersist;
qDebug("wantPersist=%s", debug::valueOf(_wantPersist));
qDebug() << "wantPersist=" << _wantPersist;
if (_wantPersist) {
QString persistFilename;
@ -1029,6 +1029,26 @@ void OctreeServer::readConfiguration() {
}
strcpy(_persistFilename, qPrintable(persistFilename));
qDebug("persistFilename=%s", _persistFilename);
_persistInterval = OctreePersistThread::DEFAULT_PERSIST_INTERVAL;
readOptionInt(QString("persistInterval"), settingsSectionObject, _persistInterval);
qDebug() << "persistInterval=" << _persistInterval;
bool noBackup;
readOptionBool(QString("NoBackup"), settingsSectionObject, noBackup);
_wantBackup = !noBackup;
qDebug() << "wantBackup=" << _wantBackup;
if (_wantBackup) {
_backupExtensionFormat = OctreePersistThread::DEFAULT_BACKUP_EXTENSION_FORMAT;
readOptionString(QString("backupExtensionFormat"), settingsSectionObject, _backupExtensionFormat);
qDebug() << "backupExtensionFormat=" << _backupExtensionFormat;
_backupInterval = OctreePersistThread::DEFAULT_BACKUP_INTERVAL;
readOptionInt(QString("backupInterval"), settingsSectionObject, _backupInterval);
qDebug() << "backupInterval=" << _backupInterval;
}
} else {
qDebug("persistFilename= DISABLED");
}
@ -1112,7 +1132,8 @@ void OctreeServer::run() {
if (_wantPersist) {
// now set up PersistThread
_persistThread = new OctreePersistThread(_tree, _persistFilename);
_persistThread = new OctreePersistThread(_tree, _persistFilename, _persistInterval,
_wantBackup, _backupInterval, _backupExtensionFormat);
if (_persistThread) {
_persistThread->initialize(true);
}
@ -1197,6 +1218,9 @@ void OctreeServer::aboutToFinish() {
qDebug() << qPrintable(_safeServerName) << "server about to finish while node still connected node:" << *node;
forceNodeShutdown(node);
}
if (_persistThread) {
_persistThread->aboutToFinish();
}
qDebug() << qPrintable(_safeServerName) << "server ENDING about to finish...";
}

5
assignment-client/src/octree/OctreeServer.h
View file

@ -167,6 +167,11 @@ protected:
JurisdictionSender* _jurisdictionSender;
OctreeInboundPacketProcessor* _octreeInboundPacketProcessor;
OctreePersistThread* _persistThread;
int _persistInterval;
bool _wantBackup;
QString _backupExtensionFormat;
int _backupInterval;
static OctreeServer* _instance;

93
domain-server/resources/describe-settings.json
View file

@ -276,6 +276,52 @@
"label": "Entity Server Settings",
"assignment-types": [6],
"settings": [
{
"name": "persistFilename",
"label": "Persistant Filename",
"help": "the filename for your entities",
"placeholder": "resources/models.svo",
"default": "resources/models.svo",
"advanced": true
},
{
"name": "persistInterval",
"label": "Persist Interval",
"help": "Interval between persist checks in msecs.",
"placeholder": "30000",
"default": "30000",
"advanced": true
},
{
"name": "NoPersist",
"type": "checkbox",
"help": "Don't persist your entities to a file.",
"default": false,
"advanced": true
},
{
"name": "backupExtensionFormat",
"label": "Backup File Extension Format:",
"help": "Format used to create the extension for the backup of your persisted entities.",
"placeholder": ".backup.%Y-%m-%d.%H:%M:%S.%z",
"default": ".backup.%Y-%m-%d.%H:%M:%S.%z",
"advanced": true
},
{
"name": "backupInterval",
"label": "Backup Interval",
"help": "Interval between backup checks in msecs.",
"placeholder": "1800000",
"default": "1800000",
"advanced": true
},
{
"name": "NoBackup",
"type": "checkbox",
"help": "Don't regularly backup your persisted entities to a backup file.",
"default": false,
"advanced": true
},
{
"name": "statusHost",
"label": "Status Hostname",
@ -330,7 +376,52 @@
"label": "Voxel Server Settings",
"assignment-types": [3],
"settings": [
{
"name": "persistFilename",
"label": "Persistant Filename",
"help": "the filename for your voxels",
"placeholder": "resources/voxels.svo",
"default": "resources/voxels.svo",
"advanced": true
},
{
"name": "persistInterval",
"label": "Persist Interval",
"help": "Interval between persist checks in msecs.",
"placeholder": "30000",
"default": "30000",
"advanced": true
},
{
"name": "NoPersist",
"type": "checkbox",
"help": "Don't persist your voxels to a file.",
"default": false,
"advanced": true
},
{
"name": "backupExtensionFormat",
"label": "Backup File Extension Format:",
"help": "Format used to create the extension for the backup of your persisted voxels.",
"placeholder": ".backup.%Y-%m-%d.%H:%M:%S.%z",
"default": ".backup.%Y-%m-%d.%H:%M:%S.%z",
"advanced": true
},
{
"name": "backupInterval",
"label": "Backup Interval",
"help": "Interval between backup checks in msecs.",
"placeholder": "1800000",
"default": "1800000",
"advanced": true
},
{
"name": "NoBackup",
"type": "checkbox",
"help": "Don't regularly backup your persisted voxels to a backup file.",
"default": false,
"advanced": true
},
{
"name": "statusHost",
"label": "Status Hostname",

2
libraries/entities/src/EntityTree.cpp
View file

@ -1089,7 +1089,6 @@ bool DebugOperator::preRecursion(OctreeElement* element) {
}
void EntityTree::dumpTree() {
// First, look for the existing entity in the tree..
DebugOperator theOperator;
recurseTreeWithOperator(&theOperator);
}
@ -1107,7 +1106,6 @@ bool PruneOperator::postRecursion(OctreeElement* element) {
}
void EntityTree::pruneTree() {
// First, look for the existing entity in the tree..
PruneOperator theOperator;
recurseTreeWithOperator(&theOperator);
}

104
libraries/octree/src/OctreePersistThread.cpp
View file

@ -9,18 +9,31 @@
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include <time.h>
#include <QDebug>
#include <PerfStat.h>
#include <SharedUtil.h>
#include "OctreePersistThread.h"
OctreePersistThread::OctreePersistThread(Octree* tree, const QString& filename, int persistInterval) :
const int OctreePersistThread::DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
const int OctreePersistThread::DEFAULT_BACKUP_INTERVAL = 1000 * 60 * 30; // every 30 minutes
const QString OctreePersistThread::DEFAULT_BACKUP_EXTENSION_FORMAT(".backup.%Y-%m-%d.%H:%M:%S.%z");
OctreePersistThread::OctreePersistThread(Octree* tree, const QString& filename, int persistInterval,
bool wantBackup, int backupInterval, const QString& backupExtensionFormat) :
_tree(tree),
_filename(filename),
_backupExtensionFormat(backupExtensionFormat),
_persistInterval(persistInterval),
_backupInterval(backupInterval),
_initialLoadComplete(false),
_loadTimeUSecs(0)
_loadTimeUSecs(0),
_lastCheck(0),
_lastBackup(0),
_wantBackup(wantBackup)
{
}
@ -51,16 +64,22 @@ bool OctreePersistThread::process() {
unsigned long leafNodeCount = OctreeElement::getLeafNodeCount();
qDebug("Nodes after loading scene %lu nodes %lu internal %lu leaves", nodeCount, internalNodeCount, leafNodeCount);
double usecPerGet = (double)OctreeElement::getGetChildAtIndexTime() / (double)OctreeElement::getGetChildAtIndexCalls();
qDebug() << "getChildAtIndexCalls=" << OctreeElement::getGetChildAtIndexCalls()
<< " getChildAtIndexTime=" << OctreeElement::getGetChildAtIndexTime() << " perGet=" << usecPerGet;
bool wantDebug = false;
if (wantDebug) {
double usecPerGet = (double)OctreeElement::getGetChildAtIndexTime()
/ (double)OctreeElement::getGetChildAtIndexCalls();
qDebug() << "getChildAtIndexCalls=" << OctreeElement::getGetChildAtIndexCalls()
<< " getChildAtIndexTime=" << OctreeElement::getGetChildAtIndexTime() << " perGet=" << usecPerGet;
double usecPerSet = (double)OctreeElement::getSetChildAtIndexTime() / (double)OctreeElement::getSetChildAtIndexCalls();
qDebug() << "setChildAtIndexCalls=" << OctreeElement::getSetChildAtIndexCalls()
<< " setChildAtIndexTime=" << OctreeElement::getSetChildAtIndexTime() << " perset=" << usecPerSet;
double usecPerSet = (double)OctreeElement::getSetChildAtIndexTime()
/ (double)OctreeElement::getSetChildAtIndexCalls();
qDebug() << "setChildAtIndexCalls=" << OctreeElement::getSetChildAtIndexCalls()
<< " setChildAtIndexTime=" << OctreeElement::getSetChildAtIndexTime() << " perSet=" << usecPerSet;
}
_initialLoadComplete = true;
_lastCheck = usecTimestampNow(); // we just loaded, no need to save again
_lastBackup = _lastCheck = usecTimestampNow(); // we just loaded, no need to save again
time(&_lastPersistTime);
emit loadCompleted();
}
@ -78,19 +97,62 @@ bool OctreePersistThread::process() {
quint64 intervalToCheck = _persistInterval * MSECS_TO_USECS;
if (sinceLastSave > intervalToCheck) {
// check the dirty bit and persist here...
_lastCheck = usecTimestampNow();
if (_tree->isDirty()) {
qDebug() << "pruning Octree before saving...";
_tree->pruneTree();
qDebug() << "DONE pruning Octree before saving...";
qDebug() << "saving Octree to file " << _filename << "...";
_tree->writeToSVOFile(_filename.toLocal8Bit().constData());
_tree->clearDirtyBit(); // tree is clean after saving
qDebug("DONE saving Octree to file...");
}
_lastCheck = now;
persist();
}
}
return isStillRunning(); // keep running till they terminate us
}
void OctreePersistThread::aboutToFinish() {
qDebug() << "Persist thread about to finish...";
persist();
qDebug() << "Persist thread done with about to finish...";
}
void OctreePersistThread::persist() {
if (_tree->isDirty()) {
_tree->lockForWrite();
{
qDebug() << "pruning Octree before saving...";
_tree->pruneTree();
qDebug() << "DONE pruning Octree before saving...";
}
_tree->unlock();
backup(); // handle backup if requested
qDebug() << "saving Octree to file " << _filename << "...";
_tree->writeToSVOFile(qPrintable(_filename));
time(&_lastPersistTime);
_tree->clearDirtyBit(); // tree is clean after saving
qDebug() << "DONE saving Octree to file...";
}
}
void OctreePersistThread::backup() {
if (_wantBackup) {
quint64 now = usecTimestampNow();
quint64 sinceLastBackup = now - _lastBackup;
quint64 MSECS_TO_USECS = 1000;
quint64 intervalToBackup = _backupInterval * MSECS_TO_USECS;
if (sinceLastBackup > intervalToBackup) {
struct tm* localTime = localtime(&_lastPersistTime);
char backupExtension[256];
strftime(backupExtension, sizeof(backupExtension), qPrintable(_backupExtensionFormat), localTime);
QString backupFileName = _filename + backupExtension;
qDebug() << "backing up persist file " << _filename << "to" << backupFileName << "...";
int result = rename(qPrintable(_filename), qPrintable(backupFileName));
if (result == 0) {
qDebug() << "DONE backing up persist file...";
} else {
qDebug() << "ERROR in backing up persist file...";
}
}
}
}

18
libraries/octree/src/OctreePersistThread.h
View file

@ -22,27 +22,41 @@
class OctreePersistThread : public GenericThread {
Q_OBJECT
public:
static const int DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
static const int DEFAULT_PERSIST_INTERVAL;
static const int DEFAULT_BACKUP_INTERVAL;
static const QString DEFAULT_BACKUP_EXTENSION_FORMAT;
OctreePersistThread(Octree* tree, const QString& filename, int persistInterval = DEFAULT_PERSIST_INTERVAL);
OctreePersistThread(Octree* tree, const QString& filename, int persistInterval = DEFAULT_PERSIST_INTERVAL,
bool wantBackup = false, int backupInterval = DEFAULT_BACKUP_INTERVAL,
const QString& backupExtensionFormat = DEFAULT_BACKUP_EXTENSION_FORMAT);
bool isInitialLoadComplete() const { return _initialLoadComplete; }
quint64 getLoadElapsedTime() const { return _loadTimeUSecs; }
void aboutToFinish(); /// call this to inform the persist thread that the owner is about to finish to support final persist
signals:
void loadCompleted();
protected:
/// Implements generic processing behavior for this thread.
virtual bool process();
void persist();
void backup();
private:
Octree* _tree;
QString _filename;
QString _backupExtensionFormat;
int _persistInterval;
int _backupInterval;
bool _initialLoadComplete;
quint64 _loadTimeUSecs;
quint64 _lastCheck;
quint64 _lastBackup;
bool _wantBackup;
time_t _lastPersistTime;
};
#endif // hifi_OctreePersistThread_h