From 73e3ad7e4ce03c74bbd671feae70f55b86b2fd2e Mon Sep 17 00:00:00 2001 From: Roxanne Skelly Date: Tue, 4 Jun 2019 11:16:36 -0700 Subject: [PATCH] Monitor traffic from nodes and report if not heard from in over 2 sec Also, remove spammy backup logging. --- domain-server/src/AssetsBackupHandler.cpp | 1 - .../src/DomainContentBackupManager.cpp | 25 ++----------------- domain-server/src/DomainServer.cpp | 17 +++++++++++++ domain-server/src/DomainServer.h | 2 ++ 4 files changed, 21 insertions(+), 24 deletions(-) diff --git a/domain-server/src/AssetsBackupHandler.cpp b/domain-server/src/AssetsBackupHandler.cpp index 6c879aca70..0f63817d80 100644 --- a/domain-server/src/AssetsBackupHandler.cpp +++ b/domain-server/src/AssetsBackupHandler.cpp @@ -276,7 +276,6 @@ void AssetsBackupHandler::createBackup(const QString& backupName, QuaZip& zip) { return; } _backups.emplace_back(backupName, mappings, false); - qDebug() << "Created asset backup:" << backupName; } void AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) { diff --git a/domain-server/src/DomainContentBackupManager.cpp b/domain-server/src/DomainContentBackupManager.cpp index d81de8b212..52bc7d679f 100644 --- a/domain-server/src/DomainContentBackupManager.cpp +++ b/domain-server/src/DomainContentBackupManager.cpp @@ -466,31 +466,20 @@ void DomainContentBackupManager::getAllBackupsAndStatus(MiniPromise::Promise pro void DomainContentBackupManager::removeOldBackupVersions(const BackupRule& rule) { QDir backupDir { _backupDirectory }; if (backupDir.exists() && rule.maxBackupVersions > 0) { - qCDebug(domain_server) << "Rolling old backup versions for rule" << rule.name; auto matchingFiles = backupDir.entryInfoList({ AUTOMATIC_BACKUP_PREFIX + rule.extensionFormat + "*.zip" }, QDir::Files | QDir::NoSymLinks, QDir::Name); int backupsToDelete = matchingFiles.length() - rule.maxBackupVersions; - if (backupsToDelete <= 0) { - qCDebug(domain_server) << "Found" << matchingFiles.length() << "backups, no backups need to be deleted"; - } else { - qCDebug(domain_server) << "Found" << matchingFiles.length() << "backups, deleting " << backupsToDelete << "backup(s)"; + if (backupsToDelete > 0) { for (int i = 0; i < backupsToDelete; ++i) { auto fileInfo = matchingFiles[i].absoluteFilePath(); QFile backupFile(fileInfo); - if (backupFile.remove()) { - qCDebug(domain_server) << "Removed old backup: " << backupFile.fileName(); - } else { + if (!backupFile.remove()) { qCDebug(domain_server) << "Failed to remove old backup: " << backupFile.fileName(); } } - qCDebug(domain_server) << "Done removing old backup versions"; } - } else { - qCDebug(domain_server) << "Rolling backups for rule" << rule.name << "." - << " Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]." - << " No need to roll backups"; } } @@ -501,13 +490,7 @@ void DomainContentBackupManager::backup() { for (BackupRule& rule : _backupRules) { auto secondsSinceLastBackup = nowSeconds - rule.lastBackupSeconds; - qCDebug(domain_server) << "Checking [" << rule.name << "] - Time since last backup [" << secondsSinceLastBackup - << "] " - << "compared to backup interval [" << rule.intervalSeconds << "]..."; - if (secondsSinceLastBackup > rule.intervalSeconds) { - qCDebug(domain_server) << "Time since last backup [" << secondsSinceLastBackup << "] for rule [" << rule.name - << "] exceeds backup interval [" << rule.intervalSeconds << "] doing backup now..."; bool success; QString path; @@ -517,13 +500,9 @@ void DomainContentBackupManager::backup() { continue; } - qDebug() << "Created backup: " << path; - rule.lastBackupSeconds = nowSeconds; removeOldBackupVersions(rule); - } else { - qCDebug(domain_server) << "Backup not needed for this rule [" << rule.name << "]..."; } } } diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 1941ef19b6..e71ea0971c 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -322,6 +322,11 @@ DomainServer::DomainServer(int argc, char* argv[]) : _contentManager->initialize(true); connect(_contentManager.get(), &DomainContentBackupManager::recoveryCompleted, this, &DomainServer::restart); + +static const int NODE_PING_MONITOR_INTERVAL_MSECS = 1 * MSECS_PER_SECOND; + _nodePingMonitorTimer = new QTimer{ this }; + connect(_nodePingMonitorTimer, &QTimer::timeout, this, &DomainServer::nodePingMonitor); + _nodePingMonitorTimer->start(NODE_PING_MONITOR_INTERVAL_MSECS); } void DomainServer::parseCommandLine(int argc, char* argv[]) { @@ -1722,6 +1727,18 @@ void DomainServer::sendHeartbeatToIceServer() { } } +void DomainServer::nodePingMonitor() { + auto nodeList = DependencyManager::get(); + quint64 now = usecTimestampNow(); + + nodeList->eachNode([now](const SharedNodePointer& node) { + quint64 lastHeard = now - node->getLastHeardMicrostamp(); + if (lastHeard > 2 * USECS_PER_SECOND) { + qCDebug(domain_server) << "Haven't heard from " << node->getPublicSocket() << " in " << lastHeard / USECS_PER_MSEC << " msec"; + } + }); +} + void DomainServer::processOctreeDataPersistMessage(QSharedPointer message) { qDebug() << "Received octree data persist message"; auto data = message->readAll(); diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index 704650e594..36ebe5a234 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -110,6 +110,7 @@ private slots: void performIPAddressUpdate(const HifiSockAddr& newPublicSockAddr); void sendHeartbeatToMetaverse() { sendHeartbeatToMetaverse(QString()); } void sendHeartbeatToIceServer(); + void nodePingMonitor(); void handleConnectedNode(SharedNodePointer newNode, quint64 requestReceiveTime); void handleTempDomainSuccess(QNetworkReply* requestReply); @@ -257,6 +258,7 @@ private: QTimer* _iceHeartbeatTimer { nullptr }; QTimer* _metaverseHeartbeatTimer { nullptr }; QTimer* _metaverseGroupCacheTimer { nullptr }; + QTimer* _nodePingMonitorTimer { nullptr }; QList _iceServerAddresses; QSet _failedIceServerAddresses;