overte/libraries/octree/src/OctreePersistThread.cpp
Ryan Huffman 010356709d Remove lock file recovery in entity server
* We no longer make backups at the ES, so there is no backup to
  recover from if we crash during persist to disk.
  * When loading the most recent backup we look for files that match our
  backup rules. If we have no backup rules we end up grabbing
  everything.
  * Because we grab everything, we are at risk of grabbing the lock file
  and treating it as a backup. In the case where we crash on persist, we
  end up replacing the models.json.gz file with the lock file (!!).

This commit makes a small change to not do the recovery on startup. In a
future update we will remove the backup-related code on the ES
altogether.
2018-04-05 11:52:31 -07:00

540 lines
22 KiB
C++

//
// OctreePersistThread.cpp
// libraries/octree/src
//
// Created by Brad Hefta-Gaub on 8/21/13.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include <chrono>
#include <thread>
#include <cstdio>
#include <fstream>
#include <time.h>
#include <QDateTime>
#include <QDebug>
#include <QDir>
#include <QDirIterator>
#include <QFile>
#include <QJsonArray>
#include <QJsonObject>
#include <QJsonDocument>
#include <NumericalConstants.h>
#include <PerfStat.h>
#include <PathUtils.h>
#include "OctreeLogging.h"
#include "OctreePersistThread.h"
#include "OctreeUtils.h"
#include "OctreeDataUtils.h"
const int OctreePersistThread::DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& filename, const QString& backupDirectory, int persistInterval,
bool wantBackup, const QJsonObject& settings, bool debugTimestampNow,
QString persistAsFileType, const QByteArray& replacementData) :
_tree(tree),
_filename(filename),
_backupDirectory(backupDirectory),
_persistInterval(persistInterval),
_initialLoadComplete(false),
_replacementData(replacementData),
_loadTimeUSecs(0),
_lastCheck(0),
_wantBackup(wantBackup),
_debugTimestampNow(debugTimestampNow),
_lastTimeDebug(0),
_persistAsFileType(persistAsFileType)
{
parseSettings(settings);
// in case the persist filename has an extension that doesn't match the file type
QString sansExt = fileNameWithoutExtension(_filename, PERSIST_EXTENSIONS);
_filename = sansExt + "." + _persistAsFileType;
}
QString OctreePersistThread::getPersistFileMimeType() const {
if (_persistAsFileType == "json") {
return "application/json";
} if (_persistAsFileType == "json.gz") {
return "application/zip";
}
return "";
}
void OctreePersistThread::parseSettings(const QJsonObject& settings) {
if (settings["backups"].isArray()) {
const QJsonArray& backupRules = settings["backups"].toArray();
qCDebug(octree) << "BACKUP RULES:";
foreach (const QJsonValue& value, backupRules) {
QJsonObject obj = value.toObject();
int interval = 0;
int count = 0;
QJsonValue intervalVal = obj["backupInterval"];
if (intervalVal.isString()) {
interval = intervalVal.toString().toInt();
} else {
interval = intervalVal.toInt();
}
QJsonValue countVal = obj["maxBackupVersions"];
if (countVal.isString()) {
count = countVal.toString().toInt();
} else {
count = countVal.toInt();
}
qCDebug(octree) << " Name:" << obj["Name"].toString();
qCDebug(octree) << " format:" << obj["format"].toString();
qCDebug(octree) << " interval:" << interval;
qCDebug(octree) << " count:" << count;
BackupRule newRule = { obj["Name"].toString(), interval, obj["format"].toString(), count, 0};
newRule.lastBackup = getMostRecentBackupTimeInUsecs(obj["format"].toString());
if (newRule.lastBackup > 0) {
quint64 now = usecTimestampNow();
quint64 sinceLastBackup = now - newRule.lastBackup;
qCDebug(octree) << " lastBackup:" << qPrintable(formatUsecTime(sinceLastBackup)) << "ago";
} else {
qCDebug(octree) << " lastBackup: NEVER";
}
_backupRules << newRule;
}
} else {
qCDebug(octree) << "BACKUP RULES: NONE";
}
}
quint64 OctreePersistThread::getMostRecentBackupTimeInUsecs(const QString& format) {
quint64 mostRecentBackupInUsecs = 0;
QString mostRecentBackupFileName;
QDateTime mostRecentBackupTime;
bool recentBackup = getMostRecentBackup(format, mostRecentBackupFileName, mostRecentBackupTime);
if (recentBackup) {
mostRecentBackupInUsecs = mostRecentBackupTime.toMSecsSinceEpoch() * USECS_PER_MSEC;
}
return mostRecentBackupInUsecs;
}
void OctreePersistThread::replaceData(QByteArray data) {
backupCurrentFile();
QFile currentFile { _filename };
if (currentFile.open(QIODevice::WriteOnly)) {
currentFile.write(data);
qDebug() << "Wrote replacement data";
} else {
qWarning() << "Failed to write replacement data";
}
}
// Return true if current file is backed up successfully or doesn't exist.
bool OctreePersistThread::backupCurrentFile() {
// first take the current models file and move it to a different filename, appended with the timestamp
QFile currentFile { _filename };
if (currentFile.exists()) {
static const QString FILENAME_TIMESTAMP_FORMAT = "yyyyMMdd-hhmmss";
auto backupFileName = _filename + ".backup." + QDateTime::currentDateTime().toString(FILENAME_TIMESTAMP_FORMAT);
if (currentFile.rename(backupFileName)) {
qDebug() << "Moved previous models file to" << backupFileName;
return true;
} else {
qWarning() << "Could not backup previous models file to" << backupFileName << "- removing replacement models file";
return false;
}
}
return true;
}
bool OctreePersistThread::process() {
if (!_initialLoadComplete) {
quint64 loadStarted = usecTimestampNow();
qCDebug(octree) << "loading Octrees from file: " << _filename << "...";
if (!_replacementData.isNull()) {
replaceData(_replacementData);
}
OctreeUtils::RawOctreeData data;
if (data.readOctreeDataInfoFromFile(_filename)) {
qDebug() << "Setting entity version info to: " << data.id << data.version;
_tree->setOctreeVersionInfo(data.id, data.version);
}
bool persistentFileRead;
_tree->withWriteLock([&] {
PerformanceWarning warn(true, "Loading Octree File", true);
// First check to make sure "lock" file doesn't exist. If it does exist, then
// our last save crashed during the save, and we want to load our most recent backup.
QString lockFileName = _filename + ".lock";
std::ifstream lockFile(qPrintable(lockFileName), std::ios::in | std::ios::binary | std::ios::ate);
if (lockFile.is_open()) {
qCDebug(octree) << "WARNING: Octree lock file detected at startup:" << lockFileName;
lockFile.close();
qCDebug(octree) << "Removing lock file:" << lockFileName;
remove(qPrintable(lockFileName));
qCDebug(octree) << "Lock file removed:" << lockFileName;
}
persistentFileRead = _tree->readFromFile(qPrintable(_filename.toLocal8Bit()));
_tree->pruneTree();
});
quint64 loadDone = usecTimestampNow();
_loadTimeUSecs = loadDone - loadStarted;
_tree->clearDirtyBit(); // the tree is clean since we just loaded it
qCDebug(octree, "DONE loading Octrees from file... fileRead=%s", debug::valueOf(persistentFileRead));
unsigned long nodeCount = OctreeElement::getNodeCount();
unsigned long internalNodeCount = OctreeElement::getInternalNodeCount();
unsigned long leafNodeCount = OctreeElement::getLeafNodeCount();
qCDebug(octree, "Nodes after loading scene %lu nodes %lu internal %lu leaves", nodeCount, internalNodeCount, leafNodeCount);
bool wantDebug = false;
if (wantDebug) {
double usecPerGet = (double)OctreeElement::getGetChildAtIndexTime()
/ (double)OctreeElement::getGetChildAtIndexCalls();
qCDebug(octree) << "getChildAtIndexCalls=" << OctreeElement::getGetChildAtIndexCalls()
<< " getChildAtIndexTime=" << OctreeElement::getGetChildAtIndexTime() << " perGet=" << usecPerGet;
double usecPerSet = (double)OctreeElement::getSetChildAtIndexTime()
/ (double)OctreeElement::getSetChildAtIndexCalls();
qCDebug(octree) << "setChildAtIndexCalls=" << OctreeElement::getSetChildAtIndexCalls()
<< " setChildAtIndexTime=" << OctreeElement::getSetChildAtIndexTime() << " perSet=" << usecPerSet;
}
_initialLoadComplete = true;
// Since we just loaded the persistent file, we can consider ourselves as having "just checked" for persistance.
_lastCheck = usecTimestampNow(); // we just loaded, no need to save again
// This last persist time is not really used until the file is actually persisted. It is only
// used in formatting the backup filename in cases of non-rolling backup names. However, we don't
// want an uninitialized value for this, so we set it to the current time (startup of the server)
time(&_lastPersistTime);
if (_replacementData.isNull()) {
sendLatestEntityDataToDS();
}
_replacementData.clear();
emit loadCompleted();
}
if (isStillRunning()) {
quint64 MSECS_TO_USECS = 1000;
quint64 USECS_TO_SLEEP = 10 * MSECS_TO_USECS; // every 10ms
std::this_thread::sleep_for(std::chrono::microseconds(USECS_TO_SLEEP));
// do our updates then check to save...
_tree->update();
quint64 now = usecTimestampNow();
quint64 sinceLastSave = now - _lastCheck;
quint64 intervalToCheck = _persistInterval * MSECS_TO_USECS;
if (sinceLastSave > intervalToCheck) {
_lastCheck = now;
persist();
}
}
// if we were asked to debugTimestampNow do that now...
if (_debugTimestampNow) {
quint64 now = usecTimestampNow();
quint64 sinceLastDebug = now - _lastTimeDebug;
quint64 DEBUG_TIMESTAMP_INTERVAL = 600000000; // every 10 minutes
if (sinceLastDebug > DEBUG_TIMESTAMP_INTERVAL) {
_lastTimeDebug = usecTimestampNow(true); // ask for debug output
}
}
return isStillRunning(); // keep running till they terminate us
}
void OctreePersistThread::aboutToFinish() {
qCDebug(octree) << "Persist thread about to finish...";
persist();
qCDebug(octree) << "Persist thread done with about to finish...";
_stopThread = true;
}
QByteArray OctreePersistThread::getPersistFileContents() const {
QByteArray fileContents;
QFile file(_filename);
if (file.open(QIODevice::ReadOnly)) {
fileContents = file.readAll();
}
return fileContents;
}
void OctreePersistThread::persist() {
if (_tree->isDirty() && _initialLoadComplete) {
_tree->withWriteLock([&] {
qCDebug(octree) << "pruning Octree before saving...";
_tree->pruneTree();
qCDebug(octree) << "DONE pruning Octree before saving...";
});
qCDebug(octree) << "persist operation calling backup...";
backup(); // handle backup if requested
qCDebug(octree) << "persist operation DONE with backup...";
_tree->incrementPersistDataVersion();
// create our "lock" file to indicate we're saving.
QString lockFileName = _filename + ".lock";
std::ofstream lockFile(qPrintable(lockFileName), std::ios::out|std::ios::binary);
if(lockFile.is_open()) {
qCDebug(octree) << "saving Octree lock file created at:" << lockFileName;
_tree->writeToFile(qPrintable(_filename), NULL, _persistAsFileType);
time(&_lastPersistTime);
_tree->clearDirtyBit(); // tree is clean after saving
qCDebug(octree) << "DONE saving Octree to file...";
lockFile.close();
qCDebug(octree) << "saving Octree lock file closed:" << lockFileName;
remove(qPrintable(lockFileName));
qCDebug(octree) << "saving Octree lock file removed:" << lockFileName;
}
sendLatestEntityDataToDS();
}
}
void OctreePersistThread::sendLatestEntityDataToDS() {
qDebug() << "Sending latest entity data to DS";
auto nodeList = DependencyManager::get<NodeList>();
const DomainHandler& domainHandler = nodeList->getDomainHandler();
QByteArray data;
if (_tree->toJSON(&data, nullptr, true)) {
auto message = NLPacketList::create(PacketType::OctreeDataPersist, QByteArray(), true, true);
message->write(data);
nodeList->sendPacketList(std::move(message), domainHandler.getSockAddr());
} else {
qCWarning(octree) << "Failed to persist octree to DS";
}
}
void OctreePersistThread::restoreFromMostRecentBackup() {
qCDebug(octree) << "Restoring from most recent backup...";
QString mostRecentBackupFileName;
QDateTime mostRecentBackupTime;
bool recentBackup = getMostRecentBackup(QString(""), mostRecentBackupFileName, mostRecentBackupTime);
// If we found a backup file, restore from that file.
if (recentBackup) {
qCDebug(octree) << "BEST backup file:" << mostRecentBackupFileName << " last modified:" << mostRecentBackupTime.toString();
qCDebug(octree) << "Removing old file:" << _filename;
remove(qPrintable(_filename));
qCDebug(octree) << "Restoring backup file " << mostRecentBackupFileName << "...";
bool result = QFile::copy(mostRecentBackupFileName, _filename);
if (result) {
qCDebug(octree) << "DONE restoring backup file " << mostRecentBackupFileName << "to" << _filename << "...";
} else {
qCDebug(octree) << "ERROR while restoring backup file " << mostRecentBackupFileName << "to" << _filename << "...";
perror("ERROR while restoring backup file");
}
} else {
qCDebug(octree) << "NO BEST backup file found.";
}
}
bool OctreePersistThread::getMostRecentBackup(const QString& format,
QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime) {
// Based on our backup file name, determine the path and file name pattern for backup files
QFileInfo persistFileInfo(_filename);
QString path = _backupDirectory;
QString fileNamePart = persistFileInfo.fileName();
QStringList filters;
if (format.isEmpty()) {
// Create a file filter that will find all backup files of this extension format
foreach(const BackupRule& rule, _backupRules) {
QString backupExtension = rule.extensionFormat;
backupExtension.replace(QRegExp("%."), "*");
QString backupFileNamePart = fileNamePart + backupExtension;
filters << backupFileNamePart;
}
} else {
QString backupExtension = format;
backupExtension.replace(QRegExp("%."), "*");
QString backupFileNamePart = fileNamePart + backupExtension;
filters << backupFileNamePart;
}
bool bestBackupFound = false;
QString bestBackupFile;
QDateTime bestBackupFileTime;
// Iterate over all of the backup files in the persist location
QDirIterator dirIterator(path, filters, QDir::Files|QDir::NoSymLinks, QDirIterator::NoIteratorFlags);
while(dirIterator.hasNext()) {
dirIterator.next();
QDateTime lastModified = dirIterator.fileInfo().lastModified();
// Based on last modified date, track the most recently modified file as the best backup
if (lastModified > bestBackupFileTime) {
bestBackupFound = true;
bestBackupFile = dirIterator.filePath();
bestBackupFileTime = lastModified;
}
}
// If we found a backup then return the results
if (bestBackupFound) {
mostRecentBackupFileName = bestBackupFile;
mostRecentBackupTime = bestBackupFileTime;
}
return bestBackupFound;
}
void OctreePersistThread::rollOldBackupVersions(const BackupRule& rule) {
if (rule.extensionFormat.contains("%N")) {
if (rule.maxBackupVersions > 0) {
qCDebug(octree) << "Rolling old backup versions for rule" << rule.name << "...";
QString backupFileName = _backupDirectory + "/" + QUrl(_filename).fileName();
// Delete maximum rolling file because rename() fails on Windows if target exists
QString backupMaxExtensionN = rule.extensionFormat;
backupMaxExtensionN.replace(QString("%N"), QString::number(rule.maxBackupVersions));
QString backupMaxFilenameN = backupFileName + backupMaxExtensionN;
QFile backupMaxFileN(backupMaxFilenameN);
if (backupMaxFileN.exists()) {
int result = remove(qPrintable(backupMaxFilenameN));
if (result != 0) {
qCDebug(octree) << "ERROR deleting old backup file " << backupMaxFilenameN;
}
}
for(int n = rule.maxBackupVersions - 1; n > 0; n--) {
QString backupExtensionN = rule.extensionFormat;
QString backupExtensionNplusOne = rule.extensionFormat;
backupExtensionN.replace(QString("%N"), QString::number(n));
backupExtensionNplusOne.replace(QString("%N"), QString::number(n+1));
QString backupFilenameN = findMostRecentFileExtension(backupFileName, PERSIST_EXTENSIONS) + backupExtensionN;
QString backupFilenameNplusOne = backupFileName + backupExtensionNplusOne;
QFile backupFileN(backupFilenameN);
if (backupFileN.exists()) {
qCDebug(octree) << "rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
int result = rename(qPrintable(backupFilenameN), qPrintable(backupFilenameNplusOne));
if (result == 0) {
qCDebug(octree) << "DONE rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
} else {
qCDebug(octree) << "ERROR in rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
perror("ERROR in rolling backup file");
}
}
}
qCDebug(octree) << "Done rolling old backup versions...";
} else {
qCDebug(octree) << "Rolling backups for rule" << rule.name << "."
<< " Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]."
<< " No need to roll backups...";
}
}
}
void OctreePersistThread::backup() {
qCDebug(octree) << "backup operation wantBackup:" << _wantBackup;
if (_wantBackup) {
quint64 now = usecTimestampNow();
for(int i = 0; i < _backupRules.count(); i++) {
BackupRule& rule = _backupRules[i];
quint64 sinceLastBackup = now - rule.lastBackup;
quint64 SECS_TO_USECS = 1000 * 1000;
quint64 intervalToBackup = rule.interval * SECS_TO_USECS;
qCDebug(octree) << "Checking [" << rule.name << "] - Time since last backup [" << sinceLastBackup << "] " <<
"compared to backup interval [" << intervalToBackup << "]...";
if (sinceLastBackup > intervalToBackup) {
qCDebug(octree) << "Time since last backup [" << sinceLastBackup << "] for rule [" << rule.name
<< "] exceeds backup interval [" << intervalToBackup << "] doing backup now...";
struct tm* localTime = localtime(&_lastPersistTime);
QString backupFileName = _backupDirectory + "/" + QUrl(_filename).fileName();
// check to see if they asked for version rolling format
if (rule.extensionFormat.contains("%N")) {
rollOldBackupVersions(rule); // rename all the old backup files accordingly
QString backupExtension = rule.extensionFormat;
backupExtension.replace(QString("%N"), QString("1"));
backupFileName += backupExtension;
} else {
char backupExtension[256];
strftime(backupExtension, sizeof(backupExtension), qPrintable(rule.extensionFormat), localTime);
backupFileName += backupExtension;
}
if (rule.maxBackupVersions > 0) {
QFile persistFile(_filename);
if (persistFile.exists()) {
qCDebug(octree) << "backing up persist file " << _filename << "to" << backupFileName << "...";
bool result = QFile::copy(_filename, backupFileName);
if (result) {
qCDebug(octree) << "DONE backing up persist file...";
rule.lastBackup = now; // only record successful backup in this case.
} else {
qCDebug(octree) << "ERROR in backing up persist file...";
perror("ERROR in backing up persist file");
}
} else {
qCDebug(octree) << "persist file " << _filename << " does not exist. " <<
"nothing to backup for this rule ["<< rule.name << "]...";
}
} else {
qCDebug(octree) << "This backup rule" << rule.name
<< " has Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]."
<< " There are no backups to be done...";
}
} else {
qCDebug(octree) << "Backup not needed for this rule ["<< rule.name << "]...";
}
}
}
}