Mirrors/overte-lubosz
3
0
Fork 0
mirror of https://github.com/lubosz/overte.git synced 2025-04-16 09:29:16 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

add support for multiple backup rules

Browse source
This commit is contained in:
ZappoMan 2015-01-13 13:41:32 -08:00
parent ea3a5b7fcd
commit 60aec8ac00
5 changed files with 200 additions and 147 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
assignment-client/src/octree/OctreeServer.cpp
View file

@ -9,6 +9,7 @@
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include <QJsonArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QTimer>
@ -978,6 +979,7 @@ void OctreeServer::readConfiguration() {
const QJsonObject& settingsObject = domainHandler.getSettingsObject();
QString settingsKey = getMyDomainSettingsKey();
QJsonObject settingsSectionObject = settingsObject[settingsKey].toObject();
_settings = settingsSectionObject; // keep this for later
if (!readOptionString(QString("statusHost"), settingsSectionObject, _statusHost) || _statusHost.isEmpty()) {
_statusHost = getLocalAddress().toString();
@ -1042,20 +1044,8 @@ void OctreeServer::readConfiguration() {
_wantBackup = !noBackup;
qDebug() << "wantBackup=" << _wantBackup;
if (_wantBackup) {
_backupExtensionFormat = OctreePersistThread::DEFAULT_BACKUP_EXTENSION_FORMAT;
readOptionString(QString("backupExtensionFormat"), settingsSectionObject, _backupExtensionFormat);
qDebug() << "backupExtensionFormat=" << _backupExtensionFormat;
_backupInterval = OctreePersistThread::DEFAULT_BACKUP_INTERVAL;
readOptionInt(QString("backupInterval"), settingsSectionObject, _backupInterval);
qDebug() << "backupInterval=" << _backupInterval;
_maxBackupVersions = OctreePersistThread::DEFAULT_MAX_BACKUP_VERSIONS;
readOptionInt(QString("maxBackupVersions"), settingsSectionObject, _maxBackupVersions);
qDebug() << "maxBackupVersions=" << _maxBackupVersions;
}
//qDebug() << "settingsSectionObject:" << settingsSectionObject;
} else {
qDebug("persistFilename= DISABLED");
}
@ -1140,8 +1130,7 @@ void OctreeServer::run() {
// now set up PersistThread
_persistThread = new OctreePersistThread(_tree, _persistFilename, _persistInterval,
_wantBackup, _backupInterval, _backupExtensionFormat,
_maxBackupVersions, _debugTimestampNow);
_wantBackup, _settings, _debugTimestampNow);
if (_persistThread) {
_persistThread->initialize(true);
}

1
assignment-client/src/octree/OctreeServer.h
View file

@ -150,6 +150,7 @@ protected:
int _argc;
const char** _argv;
char** _parsedArgV;
QJsonObject _settings;
HTTPManager* _httpManager;
int _statusPort;

88
domain-server/resources/describe-settings.json
View file

@ -308,70 +308,86 @@
"label": "Persistant Filename",
"help": "the filename for your entities",
"placeholder": "resources/models.svo",
"default": "resources/models.svo",
"advanced": true
"default": "resources/models.svo"
},
{
"name": "persistInterval",
"label": "Persist Interval",
"help": "Interval between persist checks in msecs.",
"placeholder": "30000",
"default": "30000",
"advanced": true
"default": "30000"
},
{
"name": "backups",
"type": "table",
"label": "Backup Rules",
"help": "In this table you can define a set of rules for how frequently to backup copies of your content file.",
"numbered": false,
"default": [
{"Name":"Half Hourly Rolling","backupInterval":1800,"format":".backup.halfhourly.%N","maxBackupVersions":5},
{"Name":"Daily Rolling","backupInterval":86400,"format":".backup.daily.%N","maxBackupVersions":7},
{"Name":"Weekly Rolling","backupInterval":604800,"format":".backup.weekly.%N","maxBackupVersions":4},
{"Name":"Thirty Day Rolling","backupInterval":2592000,"format":".backup.thirtyday.%N","maxBackupVersions":12}
],
"columns": [
{
"name": "Name",
"label": "Name",
"can_set": true,
"placeholder": "Example",
"default": "Example"
},
{
"name": "format",
"label": "Rule Format",
"can_set": true,
"help": "Format used to create the extension for the backup of your persisted entities. Use a format with %N to get rolling. Or use date formatting like %Y-%m-%d.%H:%M:%S.%z",
"placeholder": ".backup.example.%N",
"default": ".backup.example.%N"
},
{
"name": "backupInterval",
"label": "Backup Interval",
"help": "Interval between backup checks in seconds.",
"placeholder": 1800,
"default": 1800,
"can_set": true
},
{
"name": "maxBackupVersions",
"label": "Max Rolled Backup Versions",
"help": "If your backup extension format uses 'rolling', how many versions do you want us to keep?",
"placeholder": 5,
"default": 5,
"can_set": true
}
]
},
{
"name": "NoPersist",
"type": "checkbox",
"help": "Don't persist your entities to a file.",
"default": false,
"advanced": true
},
{
"name": "backupExtensionFormat",
"label": "Backup File Extension Format:",
"help": "Format used to create the extension for the backup of your persisted entities. Use a format with %N to get rolling. Or use date formatting like %Y-%m-%d.%H:%M:%S.%z",
"placeholder": ".backup.%N",
"default": ".backup.%N",
"advanced": true
},
{
"name": "backupInterval",
"label": "Backup Interval",
"help": "Interval between backup checks in msecs.",
"placeholder": "1800000",
"default": "1800000",
"advanced": true
},
{
"name": "maxBackupVersions",
"label": "Max Rolled Backup Versions",
"help": "If your backup extension format uses 'rolling', how many versions do you want us to keep?",
"placeholder": "5",
"default": "5",
"advanced": true
"default": false
},
{
"name": "NoBackup",
"type": "checkbox",
"help": "Don't regularly backup your persisted entities to a backup file.",
"default": false,
"advanced": true
"default": false
},
{
"name": "statusHost",
"label": "Status Hostname",
"help": "host name or IP address of the server for accessing the status page",
"placeholder": "",
"default": "",
"advanced": true
"default": ""
},
{
"name": "statusPort",
"label": "Status Port",
"help": "port of the server for accessing the status page",
"placeholder": "",
"default": "",
"advanced": true
"default": ""
},
{
"name": "verboseDebug",

204
libraries/octree/src/OctreePersistThread.cpp
View file

@ -18,6 +18,8 @@
#include <QDir>
#include <QDirIterator>
#include <QFile>
#include <QJsonArray>
#include <QJsonObject>
#include <PerfStat.h>
#include <SharedUtil.h>
@ -25,38 +27,68 @@
#include "OctreePersistThread.h"
const int OctreePersistThread::DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
const int OctreePersistThread::DEFAULT_BACKUP_INTERVAL = 1000 * 60 * 30; // every 30 minutes
const QString OctreePersistThread::DEFAULT_BACKUP_EXTENSION_FORMAT(".backup.%N");
const int OctreePersistThread::DEFAULT_MAX_BACKUP_VERSIONS = 5;
OctreePersistThread::OctreePersistThread(Octree* tree, const QString& filename, int persistInterval,
bool wantBackup, int backupInterval, const QString& backupExtensionFormat,
int maxBackupVersions, bool debugTimestampNow) :
bool wantBackup, const QJsonObject& settings, bool debugTimestampNow) :
_tree(tree),
_filename(filename),
_backupExtensionFormat(backupExtensionFormat),
_maxBackupVersions(maxBackupVersions),
_persistInterval(persistInterval),
_backupInterval(backupInterval),
_initialLoadComplete(false),
_loadTimeUSecs(0),
_lastCheck(0),
_lastBackup(0),
_wantBackup(wantBackup),
_debugTimestampNow(debugTimestampNow),
_lastTimeDebug(0)
{
parseSettings(settings);
}
quint64 OctreePersistThread::getMostRecentBackupTimeInUsecs() {
void OctreePersistThread::parseSettings(const QJsonObject& settings) {
qDebug() << "settings[backups]:" << settings["backups"];
if (settings["backups"].isArray()) {
const QJsonArray& backupRules = settings["backups"].toArray();
qDebug() << "BACKUP RULES:" << backupRules;
foreach (const QJsonValue& value, backupRules) {
QJsonObject obj = value.toObject();
qDebug() << " Name:" << obj["Name"].toString();
qDebug() << " format:" << obj["format"].toString();
qDebug() << " interval:" << obj["backupInterval"].toInt();
qDebug() << " count:" << obj["maxBackupVersions"].toInt();
BackupRule newRule = { obj["Name"].toString(), obj["backupInterval"].toInt(),
obj["format"].toString(), obj["maxBackupVersions"].toInt(), 0};
newRule.lastBackup = getMostRecentBackupTimeInUsecs(obj["format"].toString());
if (newRule.lastBackup > 0) {
quint64 now = usecTimestampNow();
quint64 sinceLastBackup = now - newRule.lastBackup;
qDebug() << " now:" << now;
qDebug() << "newRule.lastBackup:" << newRule.lastBackup;
qDebug() << " sinceLastBackup:" << sinceLastBackup;
qDebug() << " lastBackup:" << qPrintable(formatUsecTime(sinceLastBackup)) << "ago";
} else {
qDebug() << " lastBackup: NEVER";
}
_backupRules << newRule;
}
} else {
qDebug() << "BACKUP RULES: NONE";
}
}
quint64 OctreePersistThread::getMostRecentBackupTimeInUsecs(const QString& format) {
quint64 mostRecentBackupInUsecs = 0;
QString mostRecentBackupFileName;
QDateTime mostRecentBackupTime;
bool recentBackup = getMostRecentBackup(mostRecentBackupFileName, mostRecentBackupTime);
bool recentBackup = getMostRecentBackup(format, mostRecentBackupFileName, mostRecentBackupTime);
if (recentBackup) {
mostRecentBackupInUsecs = mostRecentBackupTime.toMSecsSinceEpoch() * USECS_PER_MSEC;
@ -130,12 +162,6 @@ bool OctreePersistThread::process() {
// Since we just loaded the persistent file, we can consider ourselves as having "just checked" for persistance.
_lastCheck = usecTimestampNow(); // we just loaded, no need to save again
// The last backup time, should be the timestamp for most recent backup file.
_lastBackup = getMostRecentBackupTimeInUsecs();
qDebug() << "Last Check:" << qPrintable(formatUsecTime(usecTimestampNow() - _lastCheck)) << "ago...";
qDebug() << "Last Backup:" << qPrintable(formatUsecTime(usecTimestampNow() - _lastBackup)) << "ago...";
// This last persist time is not really used until the file is actually persisted. It is only
// used in formatting the backup filename in cases of non-rolling backup names. However, we don't
// want an uninitialized value for this, so we set it to the current time (startup of the server)
@ -226,7 +252,7 @@ void OctreePersistThread::restoreFromMostRecentBackup() {
QString mostRecentBackupFileName;
QDateTime mostRecentBackupTime;
bool recentBackup = getMostRecentBackup(mostRecentBackupFileName, mostRecentBackupTime);
bool recentBackup = getMostRecentBackup(QString(""), mostRecentBackupFileName, mostRecentBackupTime);
// If we found a backup file, restore from that file.
if (recentBackup) {
@ -247,27 +273,31 @@ void OctreePersistThread::restoreFromMostRecentBackup() {
}
}
bool OctreePersistThread::getMostRecentBackup(QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime) {
bool OctreePersistThread::getMostRecentBackup(const QString& format,
QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime) {
// Based on our backup file name, determine the path and file name pattern for backup files
QFileInfo persistFileInfo(_filename);
QString path = persistFileInfo.path();
QString fileNamePart = persistFileInfo.fileName();
// Create a file filter that will find all backup files of this extension format
QString backupExtension = _backupExtensionFormat;
if (_backupExtensionFormat.contains("%N")) {
backupExtension.replace(QString("%N"), "*");
QStringList filters;
if (format.isEmpty()) {
// Create a file filter that will find all backup files of this extension format
foreach(const BackupRule& rule, _backupRules) {
QString backupExtension = rule.extensionFormat;
backupExtension.replace(QRegExp("%."), "*");
QString backupFileNamePart = fileNamePart + backupExtension;
filters << backupFileNamePart;
}
} else {
qDebug() << "This backup extension format does not yet support restoring from most recent backup...";
return false; // exit early, unable to restore from backup
QString backupExtension = format;
backupExtension.replace(QRegExp("%."), "*");
QString backupFileNamePart = fileNamePart + backupExtension;
filters << backupFileNamePart;
}
QString backupFileNamePart = fileNamePart + backupExtension;
QStringList filters;
filters << backupFileNamePart;
bool bestBackupFound = false;
QString bestBackupFile;
QDateTime bestBackupFileTime;
@ -295,74 +325,88 @@ bool OctreePersistThread::getMostRecentBackup(QString& mostRecentBackupFileName,
return bestBackupFound;
}
void OctreePersistThread::rollOldBackupVersions() {
if (!_backupExtensionFormat.contains("%N")) {
return; // this backup extension format doesn't support rolling
}
void OctreePersistThread::rollOldBackupVersions(const BackupRule& rule) {
qDebug() << "Rolling old backup versions...";
for(int n = _maxBackupVersions - 1; n > 0; n--) {
QString backupExtensionN = _backupExtensionFormat;
QString backupExtensionNplusOne = _backupExtensionFormat;
backupExtensionN.replace(QString("%N"), QString::number(n));
backupExtensionNplusOne.replace(QString("%N"), QString::number(n+1));
QString backupFilenameN = _filename + backupExtensionN;
QString backupFilenameNplusOne = _filename + backupExtensionNplusOne;
if (rule.extensionFormat.contains("%N")) {
qDebug() << "Rolling old backup versions for rule" << rule.name << "...";
for(int n = rule.maxBackupVersions - 1; n > 0; n--) {
QString backupExtensionN = rule.extensionFormat;
QString backupExtensionNplusOne = rule.extensionFormat;
backupExtensionN.replace(QString("%N"), QString::number(n));
backupExtensionNplusOne.replace(QString("%N"), QString::number(n+1));
QString backupFilenameN = _filename + backupExtensionN;
QString backupFilenameNplusOne = _filename + backupExtensionNplusOne;
QFile backupFileN(backupFilenameN);
QFile backupFileN(backupFilenameN);
if (backupFileN.exists()) {
qDebug() << "rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
int result = rename(qPrintable(backupFilenameN), qPrintable(backupFilenameNplusOne));
if (result == 0) {
qDebug() << "DONE rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
} else {
qDebug() << "ERROR in rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
if (backupFileN.exists()) {
qDebug() << "rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
int result = rename(qPrintable(backupFilenameN), qPrintable(backupFilenameNplusOne));
if (result == 0) {
qDebug() << "DONE rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
} else {
qDebug() << "ERROR in rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
}
}
}
qDebug() << "Done rolling old backup versions...";
}
qDebug() << "Done rolling old backup versions...";
}
void OctreePersistThread::backup() {
if (_wantBackup) {
quint64 now = usecTimestampNow();
quint64 sinceLastBackup = now - _lastBackup;
quint64 MSECS_TO_USECS = 1000;
quint64 intervalToBackup = _backupInterval * MSECS_TO_USECS;
qDebug() << "OctreePersistThread::backup() - now:" << now;
if (sinceLastBackup > intervalToBackup) {
qDebug() << "Time since last backup [" << sinceLastBackup << "] exceeds backup interval ["
<< intervalToBackup << "] doing backup now...";
// TODO: add a loop over all backup rules, we need to keep track of the last backup for each rule
// because we need to know when each backup rule has "elapsed"
for(int i = 0; i < _backupRules.count(); i++) {
BackupRule& rule = _backupRules[i];
struct tm* localTime = localtime(&_lastPersistTime);
quint64 sinceLastBackup = now - rule.lastBackup;
QString backupFileName;
qDebug() << " now:" << now;
qDebug() << "newRule.lastBackup:" << rule.lastBackup;
qDebug() << " sinceLastBackup:" << sinceLastBackup;
qDebug() << " lastBackup:" << qPrintable(formatUsecTime(sinceLastBackup)) << "ago";
quint64 SECS_TO_USECS = 1000 * 1000;
quint64 intervalToBackup = rule.interval * SECS_TO_USECS;
if (sinceLastBackup > intervalToBackup) {
qDebug() << "Time since last backup [" << sinceLastBackup << "] for rule [" << rule.name
<< "] exceeds backup interval [" << intervalToBackup << "] doing backup now...";
struct tm* localTime = localtime(&_lastPersistTime);
QString backupFileName;
// check to see if they asked for version rolling format
if (_backupExtensionFormat.contains("%N")) {
rollOldBackupVersions(); // rename all the old backup files accordingly
QString backupExtension = _backupExtensionFormat;
backupExtension.replace(QString("%N"), QString("1"));
backupFileName = _filename + backupExtension;
} else {
char backupExtension[256];
strftime(backupExtension, sizeof(backupExtension), qPrintable(_backupExtensionFormat), localTime);
backupFileName = _filename + backupExtension;
}
// check to see if they asked for version rolling format
if (rule.extensionFormat.contains("%N")) {
rollOldBackupVersions(rule); // rename all the old backup files accordingly
QString backupExtension = rule.extensionFormat;
backupExtension.replace(QString("%N"), QString("1"));
backupFileName = _filename + backupExtension;
} else {
char backupExtension[256];
strftime(backupExtension, sizeof(backupExtension), qPrintable(rule.extensionFormat), localTime);
backupFileName = _filename + backupExtension;
}
qDebug() << "backing up persist file " << _filename << "to" << backupFileName << "...";
int result = rename(qPrintable(_filename), qPrintable(backupFileName));
if (result == 0) {
qDebug() << "DONE backing up persist file...";
} else {
qDebug() << "ERROR in backing up persist file...";
}
qDebug() << "backing up persist file " << _filename << "to" << backupFileName << "...";
bool result = QFile::copy(_filename, backupFileName);
if (result) {
qDebug() << "DONE backing up persist file...";
} else {
qDebug() << "ERROR in backing up persist file...";
}
_lastBackup = now;
rule.lastBackup = now;
}
}
}
}

33
libraries/octree/src/OctreePersistThread.h
View file

@ -22,15 +22,19 @@
class OctreePersistThread : public GenericThread {
Q_OBJECT
public:
class BackupRule {
public:
QString name;
int interval;
QString extensionFormat;
int maxBackupVersions;
quint64 lastBackup;
};
static const int DEFAULT_PERSIST_INTERVAL;
static const int DEFAULT_BACKUP_INTERVAL;
static const QString DEFAULT_BACKUP_EXTENSION_FORMAT;
static const int DEFAULT_MAX_BACKUP_VERSIONS;
OctreePersistThread(Octree* tree, const QString& filename, int persistInterval = DEFAULT_PERSIST_INTERVAL,
bool wantBackup = false, int backupInterval = DEFAULT_BACKUP_INTERVAL,
const QString& backupExtensionFormat = DEFAULT_BACKUP_EXTENSION_FORMAT,
int maxBackupVersions = DEFAULT_MAX_BACKUP_VERSIONS,
bool wantBackup = false, const QJsonObject& settings = QJsonObject(),
bool debugTimestampNow = false);
bool isInitialLoadComplete() const { return _initialLoadComplete; }
@ -47,25 +51,24 @@ protected:
void persist();
void backup();
void rollOldBackupVersions();
void rollOldBackupVersions(const BackupRule& rule);
void restoreFromMostRecentBackup();
bool getMostRecentBackup(QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime);
quint64 getMostRecentBackupTimeInUsecs();
bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime);
quint64 getMostRecentBackupTimeInUsecs(const QString& format);
void parseSettings(const QJsonObject& settings);
private:
Octree* _tree;
QString _filename;
QString _backupExtensionFormat;
int _maxBackupVersions;
int _persistInterval;
int _backupInterval;
bool _initialLoadComplete;
quint64 _loadTimeUSecs;
quint64 _lastCheck;
quint64 _lastBackup;
bool _wantBackup;
time_t _lastPersistTime;
quint64 _lastCheck;
bool _wantBackup;
QVector<BackupRule> _backupRules;
bool _debugTimestampNow;
quint64 _lastTimeDebug;