Monitor traffic from nodes and report if not heard from in over 2 sec

Also, remove spammy backup logging.
This commit is contained in:
Roxanne Skelly 2019-06-04 11:16:36 -07:00
parent c568e7c6da
commit 73e3ad7e4c
4 changed files with 21 additions and 24 deletions

View file

@ -276,7 +276,6 @@ void AssetsBackupHandler::createBackup(const QString& backupName, QuaZip& zip) {
return;
}
_backups.emplace_back(backupName, mappings, false);
qDebug() << "Created asset backup:" << backupName;
}
void AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {

View file

@ -466,31 +466,20 @@ void DomainContentBackupManager::getAllBackupsAndStatus(MiniPromise::Promise pro
void DomainContentBackupManager::removeOldBackupVersions(const BackupRule& rule) {
QDir backupDir { _backupDirectory };
if (backupDir.exists() && rule.maxBackupVersions > 0) {
qCDebug(domain_server) << "Rolling old backup versions for rule" << rule.name;
auto matchingFiles =
backupDir.entryInfoList({ AUTOMATIC_BACKUP_PREFIX + rule.extensionFormat + "*.zip" }, QDir::Files | QDir::NoSymLinks, QDir::Name);
int backupsToDelete = matchingFiles.length() - rule.maxBackupVersions;
if (backupsToDelete <= 0) {
qCDebug(domain_server) << "Found" << matchingFiles.length() << "backups, no backups need to be deleted";
} else {
qCDebug(domain_server) << "Found" << matchingFiles.length() << "backups, deleting " << backupsToDelete << "backup(s)";
if (backupsToDelete > 0) {
for (int i = 0; i < backupsToDelete; ++i) {
auto fileInfo = matchingFiles[i].absoluteFilePath();
QFile backupFile(fileInfo);
if (backupFile.remove()) {
qCDebug(domain_server) << "Removed old backup: " << backupFile.fileName();
} else {
if (!backupFile.remove()) {
qCDebug(domain_server) << "Failed to remove old backup: " << backupFile.fileName();
}
}
qCDebug(domain_server) << "Done removing old backup versions";
}
} else {
qCDebug(domain_server) << "Rolling backups for rule" << rule.name << "."
<< " Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]."
<< " No need to roll backups";
}
}
@ -501,13 +490,7 @@ void DomainContentBackupManager::backup() {
for (BackupRule& rule : _backupRules) {
auto secondsSinceLastBackup = nowSeconds - rule.lastBackupSeconds;
qCDebug(domain_server) << "Checking [" << rule.name << "] - Time since last backup [" << secondsSinceLastBackup
<< "] "
<< "compared to backup interval [" << rule.intervalSeconds << "]...";
if (secondsSinceLastBackup > rule.intervalSeconds) {
qCDebug(domain_server) << "Time since last backup [" << secondsSinceLastBackup << "] for rule [" << rule.name
<< "] exceeds backup interval [" << rule.intervalSeconds << "] doing backup now...";
bool success;
QString path;
@ -517,13 +500,9 @@ void DomainContentBackupManager::backup() {
continue;
}
qDebug() << "Created backup: " << path;
rule.lastBackupSeconds = nowSeconds;
removeOldBackupVersions(rule);
} else {
qCDebug(domain_server) << "Backup not needed for this rule [" << rule.name << "]...";
}
}
}

View file

@ -322,6 +322,11 @@ DomainServer::DomainServer(int argc, char* argv[]) :
_contentManager->initialize(true);
connect(_contentManager.get(), &DomainContentBackupManager::recoveryCompleted, this, &DomainServer::restart);
static const int NODE_PING_MONITOR_INTERVAL_MSECS = 1 * MSECS_PER_SECOND;
_nodePingMonitorTimer = new QTimer{ this };
connect(_nodePingMonitorTimer, &QTimer::timeout, this, &DomainServer::nodePingMonitor);
_nodePingMonitorTimer->start(NODE_PING_MONITOR_INTERVAL_MSECS);
}
void DomainServer::parseCommandLine(int argc, char* argv[]) {
@ -1722,6 +1727,18 @@ void DomainServer::sendHeartbeatToIceServer() {
}
}
void DomainServer::nodePingMonitor() {
auto nodeList = DependencyManager::get<LimitedNodeList>();
quint64 now = usecTimestampNow();
nodeList->eachNode([now](const SharedNodePointer& node) {
quint64 lastHeard = now - node->getLastHeardMicrostamp();
if (lastHeard > 2 * USECS_PER_SECOND) {
qCDebug(domain_server) << "Haven't heard from " << node->getPublicSocket() << " in " << lastHeard / USECS_PER_MSEC << " msec";
}
});
}
void DomainServer::processOctreeDataPersistMessage(QSharedPointer<ReceivedMessage> message) {
qDebug() << "Received octree data persist message";
auto data = message->readAll();

View file

@ -110,6 +110,7 @@ private slots:
void performIPAddressUpdate(const HifiSockAddr& newPublicSockAddr);
void sendHeartbeatToMetaverse() { sendHeartbeatToMetaverse(QString()); }
void sendHeartbeatToIceServer();
void nodePingMonitor();
void handleConnectedNode(SharedNodePointer newNode, quint64 requestReceiveTime);
void handleTempDomainSuccess(QNetworkReply* requestReply);
@ -257,6 +258,7 @@ private:
QTimer* _iceHeartbeatTimer { nullptr };
QTimer* _metaverseHeartbeatTimer { nullptr };
QTimer* _metaverseGroupCacheTimer { nullptr };
QTimer* _nodePingMonitorTimer { nullptr };
QList<QHostAddress> _iceServerAddresses;
QSet<QHostAddress> _failedIceServerAddresses;