Merge pull request #15664 from roxanneskelly/domaincheckin

BUGZ-516: last-heard-from monitoring on domain server
This commit is contained in:
Shannon Romano 2019-06-04 14:48:51 -07:00 committed by GitHub
commit 0cfc090655
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 32 additions and 38 deletions

View file

@ -276,7 +276,6 @@ void AssetsBackupHandler::createBackup(const QString& backupName, QuaZip& zip) {
return;
}
_backups.emplace_back(backupName, mappings, false);
qDebug() << "Created asset backup:" << backupName;
}
void AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {

View file

@ -466,31 +466,20 @@ void DomainContentBackupManager::getAllBackupsAndStatus(MiniPromise::Promise pro
void DomainContentBackupManager::removeOldBackupVersions(const BackupRule& rule) {
QDir backupDir { _backupDirectory };
if (backupDir.exists() && rule.maxBackupVersions > 0) {
qCDebug(domain_server) << "Rolling old backup versions for rule" << rule.name;
auto matchingFiles =
backupDir.entryInfoList({ AUTOMATIC_BACKUP_PREFIX + rule.extensionFormat + "*.zip" }, QDir::Files | QDir::NoSymLinks, QDir::Name);
int backupsToDelete = matchingFiles.length() - rule.maxBackupVersions;
if (backupsToDelete <= 0) {
qCDebug(domain_server) << "Found" << matchingFiles.length() << "backups, no backups need to be deleted";
} else {
qCDebug(domain_server) << "Found" << matchingFiles.length() << "backups, deleting " << backupsToDelete << "backup(s)";
if (backupsToDelete > 0) {
for (int i = 0; i < backupsToDelete; ++i) {
auto fileInfo = matchingFiles[i].absoluteFilePath();
QFile backupFile(fileInfo);
if (backupFile.remove()) {
qCDebug(domain_server) << "Removed old backup: " << backupFile.fileName();
} else {
if (!backupFile.remove()) {
qCDebug(domain_server) << "Failed to remove old backup: " << backupFile.fileName();
}
}
qCDebug(domain_server) << "Done removing old backup versions";
}
} else {
qCDebug(domain_server) << "Rolling backups for rule" << rule.name << "."
<< " Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]."
<< " No need to roll backups";
}
}
@ -501,13 +490,7 @@ void DomainContentBackupManager::backup() {
for (BackupRule& rule : _backupRules) {
auto secondsSinceLastBackup = nowSeconds - rule.lastBackupSeconds;
qCDebug(domain_server) << "Checking [" << rule.name << "] - Time since last backup [" << secondsSinceLastBackup
<< "] "
<< "compared to backup interval [" << rule.intervalSeconds << "]...";
if (secondsSinceLastBackup > rule.intervalSeconds) {
qCDebug(domain_server) << "Time since last backup [" << secondsSinceLastBackup << "] for rule [" << rule.name
<< "] exceeds backup interval [" << rule.intervalSeconds << "] doing backup now...";
bool success;
QString path;
@ -517,13 +500,9 @@ void DomainContentBackupManager::backup() {
continue;
}
qDebug() << "Created backup: " << path;
rule.lastBackupSeconds = nowSeconds;
removeOldBackupVersions(rule);
} else {
qCDebug(domain_server) << "Backup not needed for this rule [" << rule.name << "]...";
}
}
}

View file

@ -322,6 +322,11 @@ DomainServer::DomainServer(int argc, char* argv[]) :
_contentManager->initialize(true);
connect(_contentManager.get(), &DomainContentBackupManager::recoveryCompleted, this, &DomainServer::restart);
static const int NODE_PING_MONITOR_INTERVAL_MSECS = 1 * MSECS_PER_SECOND;
_nodePingMonitorTimer = new QTimer{ this };
connect(_nodePingMonitorTimer, &QTimer::timeout, this, &DomainServer::nodePingMonitor);
_nodePingMonitorTimer->start(NODE_PING_MONITOR_INTERVAL_MSECS);
}
void DomainServer::parseCommandLine(int argc, char* argv[]) {
@ -1722,6 +1727,18 @@ void DomainServer::sendHeartbeatToIceServer() {
}
}
void DomainServer::nodePingMonitor() {
auto nodeList = DependencyManager::get<LimitedNodeList>();
quint64 now = usecTimestampNow();
nodeList->eachNode([now](const SharedNodePointer& node) {
quint64 lastHeard = now - node->getLastHeardMicrostamp();
if (lastHeard > 2 * USECS_PER_SECOND) {
qCDebug(domain_server) << "Haven't heard from " << node->getPublicSocket() << " in " << lastHeard / USECS_PER_MSEC << " msec";
}
});
}
void DomainServer::processOctreeDataPersistMessage(QSharedPointer<ReceivedMessage> message) {
qDebug() << "Received octree data persist message";
auto data = message->readAll();

View file

@ -110,6 +110,7 @@ private slots:
void performIPAddressUpdate(const HifiSockAddr& newPublicSockAddr);
void sendHeartbeatToMetaverse() { sendHeartbeatToMetaverse(QString()); }
void sendHeartbeatToIceServer();
void nodePingMonitor();
void handleConnectedNode(SharedNodePointer newNode, quint64 requestReceiveTime);
void handleTempDomainSuccess(QNetworkReply* requestReply);
@ -257,6 +258,7 @@ private:
QTimer* _iceHeartbeatTimer { nullptr };
QTimer* _metaverseHeartbeatTimer { nullptr };
QTimer* _metaverseGroupCacheTimer { nullptr };
QTimer* _nodePingMonitorTimer { nullptr };
QList<QHostAddress> _iceServerAddresses;
QSet<QHostAddress> _failedIceServerAddresses;

View file

@ -89,7 +89,7 @@ SequenceNumberStats::ArrivalInfo SequenceNumberStats::sequenceNumberReceived(qui
} else if (absGap > MAX_REASONABLE_SEQUENCE_GAP) {
arrivalInfo._status = Unreasonable;
HIFI_FCDEBUG(networking(), "unreasonable sequence number:" << incoming << "previous:" << _lastReceivedSequence);
qCDebug(networking) << "unreasonable sequence number:" << incoming << "previous:" << _lastReceivedSequence;
_stats._unreasonable++;
@ -153,7 +153,7 @@ SequenceNumberStats::ArrivalInfo SequenceNumberStats::sequenceNumberReceived(qui
arrivalInfo._status = Unreasonable;
HIFI_FCDEBUG(networking(), "unreasonable sequence number:" << incoming << "(possible duplicate)");
qCDebug(networking) << "unreasonable sequence number:" << incoming << "(possible duplicate)";
_stats._unreasonable++;

View file

@ -162,16 +162,16 @@ qint64 Socket::writePacket(std::unique_ptr<Packet> packet, const HifiSockAddr& s
}
qint64 Socket::writePacketList(std::unique_ptr<PacketList> packetList, const HifiSockAddr& sockAddr) {
if (packetList->getNumPackets() == 0) {
qCWarning(networking) << "Trying to send packet list with 0 packets, bailing.";
return 0;
}
if (packetList->isReliable()) {
// hand this packetList off to writeReliablePacketList
// because Qt can't invoke with the unique_ptr we have to release it here and re-construct in writeReliablePacketList
if (packetList->getNumPackets() == 0) {
qCWarning(networking) << "Trying to send packet list with 0 packets, bailing.";
return 0;
}
if (QThread::currentThread() != thread()) {
auto ptr = packetList.release();
QMetaObject::invokeMethod(this, "writeReliablePacketList", Qt::AutoConnection,
@ -189,7 +189,6 @@ qint64 Socket::writePacketList(std::unique_ptr<PacketList> packetList, const Hif
while (!packetList->_packets.empty()) {
totalBytesSent += writePacket(packetList->takeFront<Packet>(), sockAddr);
}
return totalBytesSent;
}
@ -228,7 +227,7 @@ qint64 Socket::writeDatagram(const QByteArray& datagram, const HifiSockAddr& soc
if (bytesWritten < 0) {
// when saturating a link this isn't an uncommon message - suppress it so it doesn't bomb the debug
HIFI_FCDEBUG(networking(), "Socket::writeDatagram" << _udpSocket.error());
qCDebug(networking) << "Socket::writeDatagram : " << sockAddr << " " << _udpSocket.error();
}
return bytesWritten;
@ -245,7 +244,7 @@ Connection* Socket::findOrCreateConnection(const HifiSockAddr& sockAddr, bool fi
#ifdef UDT_CONNECTION_DEBUG
qCDebug(networking) << "Socket::findOrCreateConnection refusing to create connection for" << sockAddr
<< "due to connection creation filter";
#endif
#endif // UDT_CONNECTION_DEBUG
return nullptr;
} else {
auto congestionControl = _ccFactory->create();
@ -259,9 +258,7 @@ Connection* Socket::findOrCreateConnection(const HifiSockAddr& sockAddr, bool fi
QObject::connect(connection.get(), &Connection::receiverHandshakeRequestComplete,
this, &Socket::clientHandshakeRequestComplete);
#ifdef UDT_CONNECTION_DEBUG
qCDebug(networking) << "Creating new connection to" << sockAddr;
#endif
it = _connectionsHash.insert(it, std::make_pair(sockAddr, std::move(connection)));
}
@ -493,7 +490,7 @@ std::vector<HifiSockAddr> Socket::getConnectionSockAddrs() {
}
void Socket::handleSocketError(QAbstractSocket::SocketError socketError) {
HIFI_FCDEBUG(networking(), "udt::Socket error - " << socketError);
qCDebug(networking) << "udt::Socket error - " << socketError;
}
void Socket::handleStateChanged(QAbstractSocket::SocketState socketState) {