diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp index cb206f626e..3ac094ff7a 100644 --- a/assignment-client/src/octree/OctreeServer.cpp +++ b/assignment-client/src/octree/OctreeServer.cpp @@ -9,6 +9,7 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +#include #include #include #include @@ -978,6 +979,7 @@ void OctreeServer::readConfiguration() { const QJsonObject& settingsObject = domainHandler.getSettingsObject(); QString settingsKey = getMyDomainSettingsKey(); QJsonObject settingsSectionObject = settingsObject[settingsKey].toObject(); + _settings = settingsSectionObject; // keep this for later if (!readOptionString(QString("statusHost"), settingsSectionObject, _statusHost) || _statusHost.isEmpty()) { _statusHost = getLocalAddress().toString(); @@ -1042,20 +1044,8 @@ void OctreeServer::readConfiguration() { _wantBackup = !noBackup; qDebug() << "wantBackup=" << _wantBackup; - if (_wantBackup) { - _backupExtensionFormat = OctreePersistThread::DEFAULT_BACKUP_EXTENSION_FORMAT; - readOptionString(QString("backupExtensionFormat"), settingsSectionObject, _backupExtensionFormat); - qDebug() << "backupExtensionFormat=" << _backupExtensionFormat; - - _backupInterval = OctreePersistThread::DEFAULT_BACKUP_INTERVAL; - readOptionInt(QString("backupInterval"), settingsSectionObject, _backupInterval); - qDebug() << "backupInterval=" << _backupInterval; - - _maxBackupVersions = OctreePersistThread::DEFAULT_MAX_BACKUP_VERSIONS; - readOptionInt(QString("maxBackupVersions"), settingsSectionObject, _maxBackupVersions); - qDebug() << "maxBackupVersions=" << _maxBackupVersions; - } - + //qDebug() << "settingsSectionObject:" << settingsSectionObject; + } else { qDebug("persistFilename= DISABLED"); } @@ -1140,8 +1130,7 @@ void OctreeServer::run() { // now set up PersistThread _persistThread = new OctreePersistThread(_tree, _persistFilename, _persistInterval, - _wantBackup, _backupInterval, _backupExtensionFormat, - _maxBackupVersions, _debugTimestampNow); + _wantBackup, _settings, _debugTimestampNow); if (_persistThread) { _persistThread->initialize(true); } diff --git a/assignment-client/src/octree/OctreeServer.h b/assignment-client/src/octree/OctreeServer.h index a467a8a650..0f90c2941e 100644 --- a/assignment-client/src/octree/OctreeServer.h +++ b/assignment-client/src/octree/OctreeServer.h @@ -150,6 +150,7 @@ protected: int _argc; const char** _argv; char** _parsedArgV; + QJsonObject _settings; HTTPManager* _httpManager; int _statusPort; diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index 3435f49b71..3ee62634c8 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -308,70 +308,86 @@ "label": "Persistant Filename", "help": "the filename for your entities", "placeholder": "resources/models.svo", - "default": "resources/models.svo", - "advanced": true + "default": "resources/models.svo" }, { "name": "persistInterval", "label": "Persist Interval", "help": "Interval between persist checks in msecs.", "placeholder": "30000", - "default": "30000", - "advanced": true + "default": "30000" + }, + { + "name": "backups", + "type": "table", + "label": "Backup Rules", + "help": "In this table you can define a set of rules for how frequently to backup copies of your content file.", + "numbered": false, + "default": [ + {"Name":"Half Hourly Rolling","backupInterval":1800,"format":".backup.halfhourly.%N","maxBackupVersions":5}, + {"Name":"Daily Rolling","backupInterval":86400,"format":".backup.daily.%N","maxBackupVersions":7}, + {"Name":"Weekly Rolling","backupInterval":604800,"format":".backup.weekly.%N","maxBackupVersions":4}, + {"Name":"Thirty Day Rolling","backupInterval":2592000,"format":".backup.thirtyday.%N","maxBackupVersions":12} + ], + "columns": [ + { + "name": "Name", + "label": "Name", + "can_set": true, + "placeholder": "Example", + "default": "Example" + }, + { + "name": "format", + "label": "Rule Format", + "can_set": true, + "help": "Format used to create the extension for the backup of your persisted entities. Use a format with %N to get rolling. Or use date formatting like %Y-%m-%d.%H:%M:%S.%z", + "placeholder": ".backup.example.%N", + "default": ".backup.example.%N" + }, + { + "name": "backupInterval", + "label": "Backup Interval", + "help": "Interval between backup checks in seconds.", + "placeholder": 1800, + "default": 1800, + "can_set": true + }, + { + "name": "maxBackupVersions", + "label": "Max Rolled Backup Versions", + "help": "If your backup extension format uses 'rolling', how many versions do you want us to keep?", + "placeholder": 5, + "default": 5, + "can_set": true + } + ] }, { "name": "NoPersist", "type": "checkbox", "help": "Don't persist your entities to a file.", - "default": false, - "advanced": true - }, - { - "name": "backupExtensionFormat", - "label": "Backup File Extension Format:", - "help": "Format used to create the extension for the backup of your persisted entities. Use a format with %N to get rolling. Or use date formatting like %Y-%m-%d.%H:%M:%S.%z", - "placeholder": ".backup.%N", - "default": ".backup.%N", - "advanced": true - }, - { - "name": "backupInterval", - "label": "Backup Interval", - "help": "Interval between backup checks in msecs.", - "placeholder": "1800000", - "default": "1800000", - "advanced": true - }, - { - "name": "maxBackupVersions", - "label": "Max Rolled Backup Versions", - "help": "If your backup extension format uses 'rolling', how many versions do you want us to keep?", - "placeholder": "5", - "default": "5", - "advanced": true + "default": false }, { "name": "NoBackup", "type": "checkbox", "help": "Don't regularly backup your persisted entities to a backup file.", - "default": false, - "advanced": true + "default": false }, { "name": "statusHost", "label": "Status Hostname", "help": "host name or IP address of the server for accessing the status page", "placeholder": "", - "default": "", - "advanced": true + "default": "" }, { "name": "statusPort", "label": "Status Port", "help": "port of the server for accessing the status page", "placeholder": "", - "default": "", - "advanced": true + "default": "" }, { "name": "verboseDebug", diff --git a/libraries/octree/src/OctreePersistThread.cpp b/libraries/octree/src/OctreePersistThread.cpp index 79469ade1f..9b54a3ab25 100644 --- a/libraries/octree/src/OctreePersistThread.cpp +++ b/libraries/octree/src/OctreePersistThread.cpp @@ -18,6 +18,8 @@ #include #include #include +#include +#include #include #include @@ -25,38 +27,68 @@ #include "OctreePersistThread.h" const int OctreePersistThread::DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds -const int OctreePersistThread::DEFAULT_BACKUP_INTERVAL = 1000 * 60 * 30; // every 30 minutes -const QString OctreePersistThread::DEFAULT_BACKUP_EXTENSION_FORMAT(".backup.%N"); -const int OctreePersistThread::DEFAULT_MAX_BACKUP_VERSIONS = 5; - OctreePersistThread::OctreePersistThread(Octree* tree, const QString& filename, int persistInterval, - bool wantBackup, int backupInterval, const QString& backupExtensionFormat, - int maxBackupVersions, bool debugTimestampNow) : + bool wantBackup, const QJsonObject& settings, bool debugTimestampNow) : _tree(tree), _filename(filename), - _backupExtensionFormat(backupExtensionFormat), - _maxBackupVersions(maxBackupVersions), _persistInterval(persistInterval), - _backupInterval(backupInterval), _initialLoadComplete(false), _loadTimeUSecs(0), _lastCheck(0), - _lastBackup(0), _wantBackup(wantBackup), _debugTimestampNow(debugTimestampNow), _lastTimeDebug(0) { + parseSettings(settings); } -quint64 OctreePersistThread::getMostRecentBackupTimeInUsecs() { +void OctreePersistThread::parseSettings(const QJsonObject& settings) { + qDebug() << "settings[backups]:" << settings["backups"]; + + if (settings["backups"].isArray()) { + const QJsonArray& backupRules = settings["backups"].toArray(); + qDebug() << "BACKUP RULES:" << backupRules; + + foreach (const QJsonValue& value, backupRules) { + QJsonObject obj = value.toObject(); + qDebug() << " Name:" << obj["Name"].toString(); + qDebug() << " format:" << obj["format"].toString(); + qDebug() << " interval:" << obj["backupInterval"].toInt(); + qDebug() << " count:" << obj["maxBackupVersions"].toInt(); + + BackupRule newRule = { obj["Name"].toString(), obj["backupInterval"].toInt(), + obj["format"].toString(), obj["maxBackupVersions"].toInt(), 0}; + + newRule.lastBackup = getMostRecentBackupTimeInUsecs(obj["format"].toString()); + + if (newRule.lastBackup > 0) { + quint64 now = usecTimestampNow(); + quint64 sinceLastBackup = now - newRule.lastBackup; + qDebug() << " now:" << now; + qDebug() << "newRule.lastBackup:" << newRule.lastBackup; + qDebug() << " sinceLastBackup:" << sinceLastBackup; + + qDebug() << " lastBackup:" << qPrintable(formatUsecTime(sinceLastBackup)) << "ago"; + } else { + qDebug() << " lastBackup: NEVER"; + } + + _backupRules << newRule; + } + } else { + qDebug() << "BACKUP RULES: NONE"; + } +} + +quint64 OctreePersistThread::getMostRecentBackupTimeInUsecs(const QString& format) { quint64 mostRecentBackupInUsecs = 0; QString mostRecentBackupFileName; QDateTime mostRecentBackupTime; - bool recentBackup = getMostRecentBackup(mostRecentBackupFileName, mostRecentBackupTime); + bool recentBackup = getMostRecentBackup(format, mostRecentBackupFileName, mostRecentBackupTime); if (recentBackup) { mostRecentBackupInUsecs = mostRecentBackupTime.toMSecsSinceEpoch() * USECS_PER_MSEC; @@ -130,12 +162,6 @@ bool OctreePersistThread::process() { // Since we just loaded the persistent file, we can consider ourselves as having "just checked" for persistance. _lastCheck = usecTimestampNow(); // we just loaded, no need to save again - // The last backup time, should be the timestamp for most recent backup file. - _lastBackup = getMostRecentBackupTimeInUsecs(); - - qDebug() << "Last Check:" << qPrintable(formatUsecTime(usecTimestampNow() - _lastCheck)) << "ago..."; - qDebug() << "Last Backup:" << qPrintable(formatUsecTime(usecTimestampNow() - _lastBackup)) << "ago..."; - // This last persist time is not really used until the file is actually persisted. It is only // used in formatting the backup filename in cases of non-rolling backup names. However, we don't // want an uninitialized value for this, so we set it to the current time (startup of the server) @@ -226,7 +252,7 @@ void OctreePersistThread::restoreFromMostRecentBackup() { QString mostRecentBackupFileName; QDateTime mostRecentBackupTime; - bool recentBackup = getMostRecentBackup(mostRecentBackupFileName, mostRecentBackupTime); + bool recentBackup = getMostRecentBackup(QString(""), mostRecentBackupFileName, mostRecentBackupTime); // If we found a backup file, restore from that file. if (recentBackup) { @@ -247,27 +273,31 @@ void OctreePersistThread::restoreFromMostRecentBackup() { } } -bool OctreePersistThread::getMostRecentBackup(QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime) { +bool OctreePersistThread::getMostRecentBackup(const QString& format, + QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime) { // Based on our backup file name, determine the path and file name pattern for backup files QFileInfo persistFileInfo(_filename); QString path = persistFileInfo.path(); QString fileNamePart = persistFileInfo.fileName(); - // Create a file filter that will find all backup files of this extension format - QString backupExtension = _backupExtensionFormat; - - if (_backupExtensionFormat.contains("%N")) { - backupExtension.replace(QString("%N"), "*"); + QStringList filters; + + if (format.isEmpty()) { + // Create a file filter that will find all backup files of this extension format + foreach(const BackupRule& rule, _backupRules) { + QString backupExtension = rule.extensionFormat; + backupExtension.replace(QRegExp("%."), "*"); + QString backupFileNamePart = fileNamePart + backupExtension; + filters << backupFileNamePart; + } } else { - qDebug() << "This backup extension format does not yet support restoring from most recent backup..."; - return false; // exit early, unable to restore from backup + QString backupExtension = format; + backupExtension.replace(QRegExp("%."), "*"); + QString backupFileNamePart = fileNamePart + backupExtension; + filters << backupFileNamePart; } - QString backupFileNamePart = fileNamePart + backupExtension; - QStringList filters; - filters << backupFileNamePart; - bool bestBackupFound = false; QString bestBackupFile; QDateTime bestBackupFileTime; @@ -295,74 +325,88 @@ bool OctreePersistThread::getMostRecentBackup(QString& mostRecentBackupFileName, return bestBackupFound; } -void OctreePersistThread::rollOldBackupVersions() { - if (!_backupExtensionFormat.contains("%N")) { - return; // this backup extension format doesn't support rolling - } +void OctreePersistThread::rollOldBackupVersions(const BackupRule& rule) { - qDebug() << "Rolling old backup versions..."; - for(int n = _maxBackupVersions - 1; n > 0; n--) { - QString backupExtensionN = _backupExtensionFormat; - QString backupExtensionNplusOne = _backupExtensionFormat; - backupExtensionN.replace(QString("%N"), QString::number(n)); - backupExtensionNplusOne.replace(QString("%N"), QString::number(n+1)); - - QString backupFilenameN = _filename + backupExtensionN; - QString backupFilenameNplusOne = _filename + backupExtensionNplusOne; + if (rule.extensionFormat.contains("%N")) { + qDebug() << "Rolling old backup versions for rule" << rule.name << "..."; + for(int n = rule.maxBackupVersions - 1; n > 0; n--) { + QString backupExtensionN = rule.extensionFormat; + QString backupExtensionNplusOne = rule.extensionFormat; + backupExtensionN.replace(QString("%N"), QString::number(n)); + backupExtensionNplusOne.replace(QString("%N"), QString::number(n+1)); + + QString backupFilenameN = _filename + backupExtensionN; + QString backupFilenameNplusOne = _filename + backupExtensionNplusOne; - QFile backupFileN(backupFilenameN); + QFile backupFileN(backupFilenameN); - if (backupFileN.exists()) { - qDebug() << "rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "..."; - int result = rename(qPrintable(backupFilenameN), qPrintable(backupFilenameNplusOne)); - if (result == 0) { - qDebug() << "DONE rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "..."; - } else { - qDebug() << "ERROR in rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "..."; + if (backupFileN.exists()) { + qDebug() << "rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "..."; + int result = rename(qPrintable(backupFilenameN), qPrintable(backupFilenameNplusOne)); + if (result == 0) { + qDebug() << "DONE rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "..."; + } else { + qDebug() << "ERROR in rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "..."; + } } } + qDebug() << "Done rolling old backup versions..."; } - qDebug() << "Done rolling old backup versions..."; } void OctreePersistThread::backup() { if (_wantBackup) { quint64 now = usecTimestampNow(); - quint64 sinceLastBackup = now - _lastBackup; - quint64 MSECS_TO_USECS = 1000; - quint64 intervalToBackup = _backupInterval * MSECS_TO_USECS; + qDebug() << "OctreePersistThread::backup() - now:" << now; - if (sinceLastBackup > intervalToBackup) { - qDebug() << "Time since last backup [" << sinceLastBackup << "] exceeds backup interval [" - << intervalToBackup << "] doing backup now..."; + // TODO: add a loop over all backup rules, we need to keep track of the last backup for each rule + // because we need to know when each backup rule has "elapsed" + for(int i = 0; i < _backupRules.count(); i++) { + BackupRule& rule = _backupRules[i]; - struct tm* localTime = localtime(&_lastPersistTime); + quint64 sinceLastBackup = now - rule.lastBackup; - QString backupFileName; + qDebug() << " now:" << now; + qDebug() << "newRule.lastBackup:" << rule.lastBackup; + qDebug() << " sinceLastBackup:" << sinceLastBackup; + + qDebug() << " lastBackup:" << qPrintable(formatUsecTime(sinceLastBackup)) << "ago"; + + quint64 SECS_TO_USECS = 1000 * 1000; + quint64 intervalToBackup = rule.interval * SECS_TO_USECS; + + if (sinceLastBackup > intervalToBackup) { + qDebug() << "Time since last backup [" << sinceLastBackup << "] for rule [" << rule.name + << "] exceeds backup interval [" << intervalToBackup << "] doing backup now..."; + + struct tm* localTime = localtime(&_lastPersistTime); + + QString backupFileName; - // check to see if they asked for version rolling format - if (_backupExtensionFormat.contains("%N")) { - rollOldBackupVersions(); // rename all the old backup files accordingly - QString backupExtension = _backupExtensionFormat; - backupExtension.replace(QString("%N"), QString("1")); - backupFileName = _filename + backupExtension; - } else { - char backupExtension[256]; - strftime(backupExtension, sizeof(backupExtension), qPrintable(_backupExtensionFormat), localTime); - backupFileName = _filename + backupExtension; - } + // check to see if they asked for version rolling format + if (rule.extensionFormat.contains("%N")) { + rollOldBackupVersions(rule); // rename all the old backup files accordingly + QString backupExtension = rule.extensionFormat; + backupExtension.replace(QString("%N"), QString("1")); + backupFileName = _filename + backupExtension; + } else { + char backupExtension[256]; + strftime(backupExtension, sizeof(backupExtension), qPrintable(rule.extensionFormat), localTime); + backupFileName = _filename + backupExtension; + } - qDebug() << "backing up persist file " << _filename << "to" << backupFileName << "..."; - int result = rename(qPrintable(_filename), qPrintable(backupFileName)); - if (result == 0) { - qDebug() << "DONE backing up persist file..."; - } else { - qDebug() << "ERROR in backing up persist file..."; - } + qDebug() << "backing up persist file " << _filename << "to" << backupFileName << "..."; + bool result = QFile::copy(_filename, backupFileName); + if (result) { + qDebug() << "DONE backing up persist file..."; + } else { + qDebug() << "ERROR in backing up persist file..."; + } - _lastBackup = now; + rule.lastBackup = now; + } } } } diff --git a/libraries/octree/src/OctreePersistThread.h b/libraries/octree/src/OctreePersistThread.h index 6f08b63197..374de79f0a 100644 --- a/libraries/octree/src/OctreePersistThread.h +++ b/libraries/octree/src/OctreePersistThread.h @@ -22,15 +22,19 @@ class OctreePersistThread : public GenericThread { Q_OBJECT public: + class BackupRule { + public: + QString name; + int interval; + QString extensionFormat; + int maxBackupVersions; + quint64 lastBackup; + }; + static const int DEFAULT_PERSIST_INTERVAL; - static const int DEFAULT_BACKUP_INTERVAL; - static const QString DEFAULT_BACKUP_EXTENSION_FORMAT; - static const int DEFAULT_MAX_BACKUP_VERSIONS; OctreePersistThread(Octree* tree, const QString& filename, int persistInterval = DEFAULT_PERSIST_INTERVAL, - bool wantBackup = false, int backupInterval = DEFAULT_BACKUP_INTERVAL, - const QString& backupExtensionFormat = DEFAULT_BACKUP_EXTENSION_FORMAT, - int maxBackupVersions = DEFAULT_MAX_BACKUP_VERSIONS, + bool wantBackup = false, const QJsonObject& settings = QJsonObject(), bool debugTimestampNow = false); bool isInitialLoadComplete() const { return _initialLoadComplete; } @@ -47,25 +51,24 @@ protected: void persist(); void backup(); - void rollOldBackupVersions(); + void rollOldBackupVersions(const BackupRule& rule); void restoreFromMostRecentBackup(); - bool getMostRecentBackup(QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime); - quint64 getMostRecentBackupTimeInUsecs(); + bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime); + quint64 getMostRecentBackupTimeInUsecs(const QString& format); + void parseSettings(const QJsonObject& settings); private: Octree* _tree; QString _filename; - QString _backupExtensionFormat; - int _maxBackupVersions; int _persistInterval; - int _backupInterval; bool _initialLoadComplete; quint64 _loadTimeUSecs; - quint64 _lastCheck; - quint64 _lastBackup; - bool _wantBackup; + time_t _lastPersistTime; + quint64 _lastCheck; + bool _wantBackup; + QVector _backupRules; bool _debugTimestampNow; quint64 _lastTimeDebug;