From 8a3672f3c5c15b722c4010e65b024189ef97a122 Mon Sep 17 00:00:00 2001 From: Roxanne Skelly Date: Tue, 11 Jun 2019 12:41:45 -0700 Subject: [PATCH 1/7] Checkpoint MTBF uptime reporting --- domain-server/src/DomainGatekeeper.cpp | 22 ++++++++++--------- domain-server/src/NodeConnectionData.cpp | 4 ++++ domain-server/src/NodeConnectionData.h | 2 ++ libraries/networking/src/LimitedNodeList.cpp | 1 + libraries/networking/src/LimitedNodeList.h | 8 +++++++ libraries/networking/src/NodeList.cpp | 14 +++++++++++- .../networking/src/udt/PacketHeaders.cpp | 2 +- libraries/networking/src/udt/PacketHeaders.h | 3 ++- libraries/networking/src/udt/Socket.cpp | 16 ++++++++++---- 9 files changed, 55 insertions(+), 17 deletions(-) diff --git a/domain-server/src/DomainGatekeeper.cpp b/domain-server/src/DomainGatekeeper.cpp index f5705a570b..289d583719 100644 --- a/domain-server/src/DomainGatekeeper.cpp +++ b/domain-server/src/DomainGatekeeper.cpp @@ -57,7 +57,7 @@ void DomainGatekeeper::processConnectRequestPacket(QSharedPointergetSize() == 0) { return; } - + QDataStream packetStream(message->getMessage()); // read a NodeConnectionData object from the packet so we can pass around this data while we're inspecting it @@ -88,11 +88,10 @@ void DomainGatekeeper::processConnectRequestPacket(QSharedPointersecond); } else if (!STATICALLY_ASSIGNED_NODES.contains(nodeConnection.nodeType)) { - QString username; QByteArray usernameSignature; if (message->getBytesLeftToRead() > 0) { @@ -122,9 +121,13 @@ void DomainGatekeeper::processConnectRequestPacket(QSharedPointersetNodeInterestSet(safeInterestSet); nodeData->setPlaceName(nodeConnection.placeName); - qDebug() << "Allowed connection from node" << uuidStringWithoutCurlyBraces(node->getUUID()) - << "on" << message->getSenderSockAddr() << "with MAC" << nodeConnection.hardwareAddress - << "and machine fingerprint" << nodeConnection.machineFingerprint; + qDebug() << "Allowed connection from node" << uuidStringWithoutCurlyBraces(node->getUUID()) + << "on" << message->getSenderSockAddr() + << "with MAC" << nodeConnection.hardwareAddress + << "and machine fingerprint" << nodeConnection.machineFingerprint + << "user" << username + << "reason" << nodeConnection.connectReason + << "previous connection uptime" << nodeConnection.previousConnectionUpTime/USECS_PER_MSEC << "msec"; // signal that we just connected a node so the DomainServer can get it a list // and broadcast its presence right away @@ -468,7 +471,7 @@ SharedNodePointer DomainGatekeeper::processAgentConnectRequest(const NodeConnect if (node->getPublicSocket() == nodeConnection.publicSockAddr && node->getLocalSocket() == nodeConnection.localSockAddr) { // we have a node that already has these exact sockets // this can occur if a node is failing to connect to the domain - + // remove the old node before adding the new node qDebug() << "Deleting existing connection from same sockaddr: " << node->getUUID(); existingNodeID = node->getUUID(); @@ -842,7 +845,7 @@ void DomainGatekeeper::processICEPingPacket(QSharedPointer mess // before we respond to this ICE ping packet, make sure we have a peer in the list that matches QUuid icePeerID = QUuid::fromRfc4122({ message->getRawMessage(), NUM_BYTES_RFC4122_UUID }); - + if (_icePeers.contains(icePeerID)) { auto pingReplyPacket = limitedNodeList->constructICEPingReplyPacket(*message, limitedNodeList->getSessionUUID()); @@ -882,7 +885,6 @@ void DomainGatekeeper::getGroupMemberships(const QString& username) { QJsonArray groupIDs = QJsonArray::fromStringList(groupIDSet.toList()); json["groups"] = groupIDs; - // if we've already asked, wait for the answer before asking again QString lowerUsername = username.toLower(); if (_inFlightGroupMembershipsRequests.contains(lowerUsername)) { @@ -969,7 +971,7 @@ void DomainGatekeeper::getDomainOwnerFriendsList() { QNetworkAccessManager::GetOperation, callbackParams, QByteArray(), NULL, QVariantMap()); } - + } void DomainGatekeeper::getDomainOwnerFriendsListJSONCallback(QNetworkReply* requestReply) { diff --git a/domain-server/src/NodeConnectionData.cpp b/domain-server/src/NodeConnectionData.cpp index b3ea005bd1..b4aaacd749 100644 --- a/domain-server/src/NodeConnectionData.cpp +++ b/domain-server/src/NodeConnectionData.cpp @@ -35,6 +35,10 @@ NodeConnectionData NodeConnectionData::fromDataStream(QDataStream& dataStream, c // now the machine fingerprint dataStream >> newHeader.machineFingerprint; + + dataStream >> newHeader.connectReason; + + dataStream >> newHeader.previousConnectionUpTime; } dataStream >> newHeader.lastPingTimestamp; diff --git a/domain-server/src/NodeConnectionData.h b/domain-server/src/NodeConnectionData.h index 43661f9caf..23eceb0dca 100644 --- a/domain-server/src/NodeConnectionData.h +++ b/domain-server/src/NodeConnectionData.h @@ -31,6 +31,8 @@ public: QString placeName; QString hardwareAddress; QUuid machineFingerprint; + quint32 connectReason; + quint64 previousConnectionUpTime; QByteArray protocolVersion; }; diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 48f08d6d2e..9f4eb39013 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -632,6 +632,7 @@ void LimitedNodeList::processKillNode(ReceivedMessage& message) { } void LimitedNodeList::handleNodeKill(const SharedNodePointer& node, ConnectionID nextConnectionID) { + _nodeDisconnectTimestamp = usecTimestampNow(); qCDebug(networking) << "Killed" << *node; node->stopPingTimer(); emit nodeKilled(node); diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index f7ea0ec2ad..42fb5311b1 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -337,6 +337,12 @@ public: NodeType::EntityScriptServer }; + enum DomainConnectReason : quint32 { + START = 0, + RECONNECT + }; + Q_ENUM(DomainConnectReason); + public slots: void reset(); void eraseAllNodes(); @@ -461,6 +467,8 @@ protected: } std::unordered_map _connectionIDs; + quint64 _nodeConnectTimestamp { 0 }; + quint64 _nodeDisconnectTimestamp { 0 }; private slots: void flagTimeForConnectionStep(ConnectionStep connectionStep, quint64 timestamp); diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 0e6b5503d7..c352b9f5ea 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -296,6 +296,8 @@ void NodeList::addSetOfNodeTypesToNodeInterestSet(const NodeSet& setOfNodeTypes) void NodeList::sendDomainServerCheckIn() { + int outstandingCheckins = _domainHandler.getCheckInPacketsSinceLastReply(); + // On ThreadedAssignments (assignment clients), this function // is called by the server check-in timer thread // not the NodeList thread. Calling it on the NodeList thread @@ -414,6 +416,16 @@ void NodeList::sendDomainServerCheckIn() { // now add the machine fingerprint auto accountManager = DependencyManager::get(); packetStream << FingerprintUtils::getMachineFingerprint(); + + packetStream << ((outstandingCheckins >= MAX_SILENT_DOMAIN_SERVER_CHECK_INS) ? RECONNECT : START); + + if (_nodeDisconnectTimestamp < _nodeConnectTimestamp) { + _nodeDisconnectTimestamp = usecTimestampNow(); + } + quint64 previousConnectionUptime = _nodeConnectTimestamp ? _nodeDisconnectTimestamp - _nodeConnectTimestamp : 0; + + packetStream << previousConnectionUptime; + } packetStream << quint64(duration_cast(system_clock::now().time_since_epoch()).count()); @@ -439,7 +451,6 @@ void NodeList::sendDomainServerCheckIn() { // Send duplicate check-ins in the exponentially increasing sequence 1, 1, 2, 4, ... static const int MAX_CHECKINS_TOGETHER = 20; static const int REBIND_CHECKIN_COUNT = 2; - int outstandingCheckins = _domainHandler.getCheckInPacketsSinceLastReply(); if (outstandingCheckins > REBIND_CHECKIN_COUNT) { _nodeSocket.rebind(); @@ -626,6 +637,7 @@ void NodeList::processDomainServerConnectionTokenPacket(QSharedPointerreadWithoutCopy(NUM_BYTES_RFC4122_UUID))); _domainHandler.clearPendingCheckins(); + _nodeConnectTimestamp = usecTimestampNow(); sendDomainServerCheckIn(); } diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index 566e1e4946..7ebaf5224f 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -72,7 +72,7 @@ PacketVersion versionForPacketType(PacketType packetType) { return static_cast(DomainConnectionDeniedVersion::IncludesExtraInfo); case PacketType::DomainConnectRequest: - return static_cast(DomainConnectRequestVersion::HasTimestamp); + return static_cast(DomainConnectRequestVersion::HasReason); case PacketType::DomainServerAddedNode: return static_cast(DomainServerAddedNodeVersion::PermissionsGrid); diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 903c1f4c93..5baf5448dd 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -345,7 +345,8 @@ enum class DomainConnectRequestVersion : PacketVersion { HasMACAddress, HasMachineFingerprint, AlwaysHasMachineFingerprint, - HasTimestamp + HasTimestamp, + HasReason }; enum class DomainConnectionDeniedVersion : PacketVersion { diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 406c2ff213..350c592c61 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -59,10 +59,13 @@ void Socket::bind(const QHostAddress& address, quint16 port) { auto sd = _udpSocket.socketDescriptor(); int val = IP_PMTUDISC_DONT; setsockopt(sd, IPPROTO_IP, IP_MTU_DISCOVER, &val, sizeof(val)); -#elif defined(Q_OS_WINDOWS) +#elif defined(Q_OS_WIN) auto sd = _udpSocket.socketDescriptor(); int val = 0; // false - setsockopt(sd, IPPROTO_IP, IP_DONTFRAGMENT, &val, sizeof(val)); + if (setsockopt(sd, IPPROTO_IP, IP_DONTFRAGMENT, (const char*)&val, sizeof(val))) { + auto err = WSAGetLastError(); + qCWarning(networking) << "Socket::bind Cannot setsockopt IP_DONTFRAGMENT" << err; + } #endif } } @@ -232,14 +235,17 @@ qint64 Socket::writeDatagram(const QByteArray& datagram, const HifiSockAddr& soc } qint64 bytesWritten = _udpSocket.writeDatagram(datagram, sockAddr.getAddress(), sockAddr.getPort()); + + if (bytesWritten < 0) { - qCDebug(networking) << "udt::writeDatagram (" << _udpSocket.state() << ") error - " << _udpSocket.error() << "(" << _udpSocket.errorString() << ")"; #ifdef WIN32 int wsaError = WSAGetLastError(); qCDebug(networking) << "windows socket error " << wsaError; #endif + qCDebug(networking) << "udt::writeDatagram (" << _udpSocket.state() << ") error - " << _udpSocket.error() << "(" << _udpSocket.errorString() << ")"; + #ifdef DEBUG_EVENT_QUEUE int nodeListQueueSize = ::hifi::qt::getEventQueueSize(thread()); qCDebug(networking) << "Networking queue size - " << nodeListQueueSize; @@ -506,11 +512,13 @@ std::vector Socket::getConnectionSockAddrs() { } void Socket::handleSocketError(QAbstractSocket::SocketError socketError) { - qCDebug(networking) << "udt::Socket (" << _udpSocket.state() << ") error - " << socketError << "(" << _udpSocket.errorString() << ")"; + #ifdef WIN32 int wsaError = WSAGetLastError(); qCDebug(networking) << "windows socket error " << wsaError; #endif + + qCDebug(networking) << "udt::Socket (" << _udpSocket.state() << ") error - " << socketError << "(" << _udpSocket.errorString() << ")"; #ifdef DEBUG_EVENT_QUEUE int nodeListQueueSize = ::hifi::qt::getEventQueueSize(thread()); qCDebug(networking) << "Networking queue size - " << nodeListQueueSize; From ec91f4dae4f5141802d95480d0e9a9746cfbbb91 Mon Sep 17 00:00:00 2001 From: Roxanne Skelly Date: Tue, 11 Jun 2019 14:37:46 -0700 Subject: [PATCH 2/7] More checkpoint MTBF logging --- domain-server/src/DomainGatekeeper.cpp | 2 +- domain-server/src/DomainServer.cpp | 7 ++++--- domain-server/src/DomainServer.h | 2 +- domain-server/src/NodeConnectionData.cpp | 2 +- domain-server/src/NodeConnectionData.h | 2 +- libraries/networking/src/NodeList.cpp | 8 +++++++- 6 files changed, 15 insertions(+), 8 deletions(-) diff --git a/domain-server/src/DomainGatekeeper.cpp b/domain-server/src/DomainGatekeeper.cpp index 289d583719..4645a25ca6 100644 --- a/domain-server/src/DomainGatekeeper.cpp +++ b/domain-server/src/DomainGatekeeper.cpp @@ -126,7 +126,7 @@ void DomainGatekeeper::processConnectRequestPacket(QSharedPointer mess // client-side send time of last connect/domain list request nodeData->setLastDomainCheckinTimestamp(nodeRequestData.lastPingTimestamp); - sendDomainListToNode(sendingNode, message->getFirstPacketReceiveTime(), message->getSenderSockAddr()); + sendDomainListToNode(sendingNode, message->getFirstPacketReceiveTime(), message->getSenderSockAddr(), false); } bool DomainServer::isInInterestSet(const SharedNodePointer& nodeA, const SharedNodePointer& nodeB) { @@ -1145,7 +1145,7 @@ void DomainServer::handleConnectedNode(SharedNodePointer newNode, quint64 reques DomainServerNodeData* nodeData = static_cast(newNode->getLinkedData()); // reply back to the user with a PacketType::DomainList - sendDomainListToNode(newNode, requestReceiveTime, nodeData->getSendingSockAddr()); + sendDomainListToNode(newNode, requestReceiveTime, nodeData->getSendingSockAddr(), true); // if this node is a user (unassigned Agent), signal if (newNode->getType() == NodeType::Agent && !nodeData->wasAssigned()) { @@ -1161,7 +1161,7 @@ void DomainServer::handleConnectedNode(SharedNodePointer newNode, quint64 reques broadcastNewNode(newNode); } -void DomainServer::sendDomainListToNode(const SharedNodePointer& node, quint64 requestPacketReceiveTime, const HifiSockAddr &senderSockAddr) { +void DomainServer::sendDomainListToNode(const SharedNodePointer& node, quint64 requestPacketReceiveTime, const HifiSockAddr &senderSockAddr, bool newConnection) { const int NUM_DOMAIN_LIST_EXTENDED_HEADER_BYTES = NUM_BYTES_RFC4122_UUID + NLPacket::NUM_BYTES_LOCALID + NUM_BYTES_RFC4122_UUID + NLPacket::NUM_BYTES_LOCALID + 4; @@ -1181,6 +1181,7 @@ void DomainServer::sendDomainListToNode(const SharedNodePointer& node, quint64 r extendedHeaderStream << nodeData->getLastDomainCheckinTimestamp(); extendedHeaderStream << quint64(duration_cast(system_clock::now().time_since_epoch()).count()); extendedHeaderStream << quint64(duration_cast(p_high_resolution_clock::now().time_since_epoch()).count()) - requestPacketReceiveTime; + extendedHeaderStream << newConnection; auto domainListPackets = NLPacketList::create(PacketType::DomainList, extendedHeader); // always send the node their own UUID back diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index 36ebe5a234..54b7fbe466 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -173,7 +173,7 @@ private: void handleKillNode(SharedNodePointer nodeToKill); void broadcastNodeDisconnect(const SharedNodePointer& disconnnectedNode); - void sendDomainListToNode(const SharedNodePointer& node, quint64 requestPacketReceiveTime, const HifiSockAddr& senderSockAddr); + void sendDomainListToNode(const SharedNodePointer& node, quint64 requestPacketReceiveTime, const HifiSockAddr& senderSockAddr, bool newConnection); bool isInInterestSet(const SharedNodePointer& nodeA, const SharedNodePointer& nodeB); diff --git a/domain-server/src/NodeConnectionData.cpp b/domain-server/src/NodeConnectionData.cpp index b4aaacd749..c99437d185 100644 --- a/domain-server/src/NodeConnectionData.cpp +++ b/domain-server/src/NodeConnectionData.cpp @@ -36,7 +36,7 @@ NodeConnectionData NodeConnectionData::fromDataStream(QDataStream& dataStream, c // now the machine fingerprint dataStream >> newHeader.machineFingerprint; - dataStream >> newHeader.connectReason; + dataStream >> newHeader.wasSilentDomainDisconnect; dataStream >> newHeader.previousConnectionUpTime; } diff --git a/domain-server/src/NodeConnectionData.h b/domain-server/src/NodeConnectionData.h index 23eceb0dca..96a9c0d392 100644 --- a/domain-server/src/NodeConnectionData.h +++ b/domain-server/src/NodeConnectionData.h @@ -31,7 +31,7 @@ public: QString placeName; QString hardwareAddress; QUuid machineFingerprint; - quint32 connectReason; + quint32 wasSilentDomainDisconnect; quint64 previousConnectionUpTime; QByteArray protocolVersion; diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index c352b9f5ea..dc1bd701b8 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -637,7 +637,6 @@ void NodeList::processDomainServerConnectionTokenPacket(QSharedPointerreadWithoutCopy(NUM_BYTES_RFC4122_UUID))); _domainHandler.clearPendingCheckins(); - _nodeConnectTimestamp = usecTimestampNow(); sendDomainServerCheckIn(); } @@ -678,6 +677,13 @@ void NodeList::processDomainServerList(QSharedPointer message) quint64 domainServerCheckinProcessingTime; packetStream >> domainServerCheckinProcessingTime; + bool newConnection; + packetStream >> newConnection; + + if (newConnection) { + _nodeConnectTimestamp = usecTimestampNow(); + } + qint64 pingLagTime = (now - qint64(connectRequestTimestamp)) / qint64(USECS_PER_MSEC); qint64 domainServerRequestLag = (qint64(domainServerPingSendTime - domainServerCheckinProcessingTime) - qint64(connectRequestTimestamp)) / qint64(USECS_PER_MSEC);; From 071a779930321b853fbdbe5b40395a0435530925 Mon Sep 17 00:00:00 2001 From: Roxanne Skelly Date: Tue, 11 Jun 2019 14:38:39 -0700 Subject: [PATCH 3/7] convert enum reason for disconnect to boolean --- libraries/networking/src/LimitedNodeList.h | 7 +------ libraries/networking/src/NodeList.cpp | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 42fb5311b1..4df6df621f 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -337,12 +337,6 @@ public: NodeType::EntityScriptServer }; - enum DomainConnectReason : quint32 { - START = 0, - RECONNECT - }; - Q_ENUM(DomainConnectReason); - public slots: void reset(); void eraseAllNodes(); @@ -469,6 +463,7 @@ protected: std::unordered_map _connectionIDs; quint64 _nodeConnectTimestamp { 0 }; quint64 _nodeDisconnectTimestamp { 0 }; + bool _wasSilentDomainDisconnect { false }; private slots: void flagTimeForConnectionStep(ConnectionStep connectionStep, quint64 timestamp); diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index c352b9f5ea..fa3c40b138 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -113,6 +113,9 @@ NodeList::NodeList(char newOwnerType, int socketListenPort, int dtlsListenPort) connect(&_domainHandler, SIGNAL(connectedToDomain(QUrl)), &_keepAlivePingTimer, SLOT(start())); connect(&_domainHandler, &DomainHandler::disconnectedFromDomain, &_keepAlivePingTimer, &QTimer::stop); + connect(&_domainHandler, &DomainHandler::limitOfSilentDomainCheckInsReached, + this, [this]() {_wasSilentDomainDisconnect = true; }); + // set our sockAddrBelongsToDomainOrNode method as the connection creation filter for the udt::Socket using std::placeholders::_1; _nodeSocket.setConnectionCreationFilterOperator(std::bind(&NodeList::sockAddrBelongsToDomainOrNode, this, _1)); @@ -296,8 +299,6 @@ void NodeList::addSetOfNodeTypesToNodeInterestSet(const NodeSet& setOfNodeTypes) void NodeList::sendDomainServerCheckIn() { - int outstandingCheckins = _domainHandler.getCheckInPacketsSinceLastReply(); - // On ThreadedAssignments (assignment clients), this function // is called by the server check-in timer thread // not the NodeList thread. Calling it on the NodeList thread @@ -417,7 +418,7 @@ void NodeList::sendDomainServerCheckIn() { auto accountManager = DependencyManager::get(); packetStream << FingerprintUtils::getMachineFingerprint(); - packetStream << ((outstandingCheckins >= MAX_SILENT_DOMAIN_SERVER_CHECK_INS) ? RECONNECT : START); + packetStream << quint32(_wasSilentDomainDisconnect ? 1 : 0); if (_nodeDisconnectTimestamp < _nodeConnectTimestamp) { _nodeDisconnectTimestamp = usecTimestampNow(); @@ -451,7 +452,7 @@ void NodeList::sendDomainServerCheckIn() { // Send duplicate check-ins in the exponentially increasing sequence 1, 1, 2, 4, ... static const int MAX_CHECKINS_TOGETHER = 20; static const int REBIND_CHECKIN_COUNT = 2; - + int outstandingCheckins = _domainHandler.getCheckInPacketsSinceLastReply(); if (outstandingCheckins > REBIND_CHECKIN_COUNT) { _nodeSocket.rebind(); } @@ -637,7 +638,6 @@ void NodeList::processDomainServerConnectionTokenPacket(QSharedPointerreadWithoutCopy(NUM_BYTES_RFC4122_UUID))); _domainHandler.clearPendingCheckins(); - _nodeConnectTimestamp = usecTimestampNow(); sendDomainServerCheckIn(); } @@ -678,6 +678,14 @@ void NodeList::processDomainServerList(QSharedPointer message) quint64 domainServerCheckinProcessingTime; packetStream >> domainServerCheckinProcessingTime; + bool newConnection; + packetStream >> newConnection; + + if (newConnection) { + _nodeConnectTimestamp = usecTimestampNow(); + _wasSilentDomainDisconnect = false; + } + qint64 pingLagTime = (now - qint64(connectRequestTimestamp)) / qint64(USECS_PER_MSEC); qint64 domainServerRequestLag = (qint64(domainServerPingSendTime - domainServerCheckinProcessingTime) - qint64(connectRequestTimestamp)) / qint64(USECS_PER_MSEC);; From 3c9edb90d87747ba1f520df7d63a7cb276844b6a Mon Sep 17 00:00:00 2001 From: Roxanne Skelly Date: Tue, 11 Jun 2019 15:35:50 -0700 Subject: [PATCH 4/7] Log formatting fix --- domain-server/src/DomainGatekeeper.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domain-server/src/DomainGatekeeper.cpp b/domain-server/src/DomainGatekeeper.cpp index 4645a25ca6..1b4b78c81e 100644 --- a/domain-server/src/DomainGatekeeper.cpp +++ b/domain-server/src/DomainGatekeeper.cpp @@ -126,7 +126,7 @@ void DomainGatekeeper::processConnectRequestPacket(QSharedPointer Date: Tue, 11 Jun 2019 16:00:49 -0700 Subject: [PATCH 5/7] Update packet headers / add connect 'reason' --- domain-server/src/DomainGatekeeper.cpp | 2 +- domain-server/src/NodeConnectionData.cpp | 2 +- domain-server/src/NodeConnectionData.h | 2 +- libraries/networking/src/LimitedNodeList.h | 168 +++++++++++------- libraries/networking/src/NodeList.cpp | 7 +- .../networking/src/udt/PacketHeaders.cpp | 2 +- libraries/networking/src/udt/PacketHeaders.h | 3 +- 7 files changed, 108 insertions(+), 78 deletions(-) diff --git a/domain-server/src/DomainGatekeeper.cpp b/domain-server/src/DomainGatekeeper.cpp index 1b4b78c81e..fee6069c7a 100644 --- a/domain-server/src/DomainGatekeeper.cpp +++ b/domain-server/src/DomainGatekeeper.cpp @@ -126,7 +126,7 @@ void DomainGatekeeper::processConnectRequestPacket(QSharedPointer> newHeader.machineFingerprint; - dataStream >> newHeader.wasSilentDomainDisconnect; + dataStream >> newHeader.connectReason; dataStream >> newHeader.previousConnectionUpTime; } diff --git a/domain-server/src/NodeConnectionData.h b/domain-server/src/NodeConnectionData.h index 96a9c0d392..23eceb0dca 100644 --- a/domain-server/src/NodeConnectionData.h +++ b/domain-server/src/NodeConnectionData.h @@ -31,7 +31,7 @@ public: QString placeName; QString hardwareAddress; QUuid machineFingerprint; - quint32 wasSilentDomainDisconnect; + quint32 connectReason; quint64 previousConnectionUpTime; QByteArray protocolVersion; diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 4df6df621f..fc93435599 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -20,7 +20,7 @@ #include #ifndef _WIN32 -#include // not on windows, not needed for mac or windows +#include // not on windows, not needed for mac or windows #endif #include @@ -51,7 +51,7 @@ const int INVALID_PORT = -1; const quint64 NODE_SILENCE_THRESHOLD_MSECS = 10 * 1000; -static const size_t DEFAULT_MAX_CONNECTION_RATE { std::numeric_limits::max() }; +static const size_t DEFAULT_MAX_CONNECTION_RATE{ std::numeric_limits::max() }; const char DEFAULT_ASSIGNMENT_SERVER_HOSTNAME[] = "localhost"; @@ -67,26 +67,26 @@ const QHostAddress DEFAULT_ASSIGNMENT_CLIENT_MONITOR_HOSTNAME = QHostAddress::Lo const QString USERNAME_UUID_REPLACEMENT_STATS_KEY = "$username"; using ConnectionID = int64_t; -const ConnectionID NULL_CONNECTION_ID { -1 }; -const ConnectionID INITIAL_CONNECTION_ID { 0 }; +const ConnectionID NULL_CONNECTION_ID{ -1 }; +const ConnectionID INITIAL_CONNECTION_ID{ 0 }; typedef std::pair UUIDNodePair; typedef tbb::concurrent_unordered_map NodeHash; typedef quint8 PingType_t; namespace PingType { - const PingType_t Agnostic = 0; - const PingType_t Local = 1; - const PingType_t Public = 2; - const PingType_t Symmetric = 3; -} +const PingType_t Agnostic = 0; +const PingType_t Local = 1; +const PingType_t Public = 2; +const PingType_t Symmetric = 3; +} // namespace PingType class LimitedNodeList : public QObject, public Dependency { Q_OBJECT SINGLETON_DEPENDENCY public: - - enum ConnectionStep { + enum ConnectionStep + { LookupAddress = 1, HandleAddress, SendSTUNRequest, @@ -108,6 +108,14 @@ public: }; Q_ENUM(ConnectionStep); + + enum ConnectReason : quint32 + { + Connect = 0, + SilentDomainDisconnect + }; + Q_ENUM(ConnectReason); + QUuid getSessionUUID() const; void setSessionUUID(const QUuid& sessionUUID); Node::LocalID getSessionLocalID() const; @@ -117,12 +125,18 @@ public: bool isAllowedEditor() const { return _permissions.can(NodePermissions::Permission::canAdjustLocks); } bool getThisNodeCanRez() const { return _permissions.can(NodePermissions::Permission::canRezPermanentEntities); } bool getThisNodeCanRezTmp() const { return _permissions.can(NodePermissions::Permission::canRezTemporaryEntities); } - bool getThisNodeCanRezCertified() const { return _permissions.can(NodePermissions::Permission::canRezPermanentCertifiedEntities); } - bool getThisNodeCanRezTmpCertified() const { return _permissions.can(NodePermissions::Permission::canRezTemporaryCertifiedEntities); } + bool getThisNodeCanRezCertified() const { + return _permissions.can(NodePermissions::Permission::canRezPermanentCertifiedEntities); + } + bool getThisNodeCanRezTmpCertified() const { + return _permissions.can(NodePermissions::Permission::canRezTemporaryCertifiedEntities); + } bool getThisNodeCanWriteAssets() const { return _permissions.can(NodePermissions::Permission::canWriteToAssetServer); } bool getThisNodeCanKick() const { return _permissions.can(NodePermissions::Permission::canKick); } bool getThisNodeCanReplaceContent() const { return _permissions.can(NodePermissions::Permission::canReplaceDomainContent); } - bool getThisNodeCanGetAndSetPrivateUserData() const { return _permissions.can(NodePermissions::Permission::canGetAndSetPrivateUserData); } + bool getThisNodeCanGetAndSetPrivateUserData() const { + return _permissions.can(NodePermissions::Permission::canGetAndSetPrivateUserData); + } quint16 getSocketLocalPort() const { return _nodeSocket.localPort(); } Q_INVOKABLE void setSocketLocalPort(quint16 socketLocalPort); @@ -132,9 +146,18 @@ public: PacketReceiver& getPacketReceiver() { return *_packetReceiver; } virtual bool isDomainServer() const { return true; } - virtual QUuid getDomainUUID() const { assert(false); return QUuid(); } - virtual Node::LocalID getDomainLocalID() const { assert(false); return Node::NULL_LOCAL_ID; } - virtual HifiSockAddr getDomainSockAddr() const { assert(false); return HifiSockAddr(); } + virtual QUuid getDomainUUID() const { + assert(false); + return QUuid(); + } + virtual Node::LocalID getDomainLocalID() const { + assert(false); + return Node::NULL_LOCAL_ID; + } + virtual HifiSockAddr getDomainSockAddr() const { + assert(false); + return HifiSockAddr(); + } // use sendUnreliablePacket to send an unreliable packet (that you do not need to move) // either to a node (via its active socket) or to a manual sockaddr @@ -148,8 +171,9 @@ public: // use sendUnreliableUnorderedPacketList to unreliably send separate packets from the packet list // either to a node's active socket or to a manual sockaddr qint64 sendUnreliableUnorderedPacketList(NLPacketList& packetList, const Node& destinationNode); - qint64 sendUnreliableUnorderedPacketList(NLPacketList& packetList, const HifiSockAddr& sockAddr, - HMACAuth* hmacAuth = nullptr); + qint64 sendUnreliableUnorderedPacketList(NLPacketList& packetList, + const HifiSockAddr& sockAddr, + HMACAuth* hmacAuth = nullptr); // use sendPacketList to send reliable packet lists (ordered or unordered) to a node's active socket // or to a manual sock addr @@ -158,15 +182,22 @@ public: std::function linkedDataCreateCallback; - size_t size() const { QReadLocker readLock(&_nodeMutex); return _nodeHash.size(); } + size_t size() const { + QReadLocker readLock(&_nodeMutex); + return _nodeHash.size(); + } SharedNodePointer nodeWithUUID(const QUuid& nodeUUID); SharedNodePointer nodeWithLocalID(Node::LocalID localID) const; - SharedNodePointer addOrUpdateNode(const QUuid& uuid, NodeType_t nodeType, - const HifiSockAddr& publicSocket, const HifiSockAddr& localSocket, - Node::LocalID localID = Node::NULL_LOCAL_ID, bool isReplicated = false, - bool isUpstream = false, const QUuid& connectionSecret = QUuid(), + SharedNodePointer addOrUpdateNode(const QUuid& uuid, + NodeType_t nodeType, + const HifiSockAddr& publicSocket, + const HifiSockAddr& localSocket, + Node::LocalID localID = Node::NULL_LOCAL_ID, + bool isReplicated = false, + bool isUpstream = false, + const QUuid& connectionSecret = QUuid(), const NodePermissions& permissions = DEFAULT_AGENT_PERMISSIONS); static bool parseSTUNResponse(udt::BasePacket* packet, QHostAddress& newPublicAddress, uint16_t& newPublicPort); @@ -201,7 +232,7 @@ public: // Use this for nested loops instead of taking nested read locks! // This allows multiple threads (i.e. a thread pool) to share a lock // without deadlocking when a dying node attempts to acquire a write lock - template + template void nestedEach(NestedNodeLambda functor, int* lockWaitOut = nullptr, int* nodeTransformOut = nullptr, @@ -221,9 +252,8 @@ public: // so reserve enough memory for the current size // and then back insert all the nodes found nodes.reserve(_nodeHash.size()); - std::transform(_nodeHash.cbegin(), _nodeHash.cend(), std::back_inserter(nodes), [&](const NodeHash::value_type& it) { - return it.second; - }); + std::transform(_nodeHash.cbegin(), _nodeHash.cend(), std::back_inserter(nodes), + [&](const NodeHash::value_type& it) { return it.second; }); endTransform = usecTimestampNow(); if (nodeTransformOut) { @@ -238,7 +268,7 @@ public: } } - template + template void eachNode(NodeLambda functor) { QReadLocker readLock(&_nodeMutex); @@ -247,7 +277,7 @@ public: } } - template + template void eachMatchingNode(PredLambda predicate, NodeLambda functor) { QReadLocker readLock(&_nodeMutex); @@ -258,7 +288,7 @@ public: } } - template + template void eachNodeBreakable(BreakableNodeLambda functor) { QReadLocker readLock(&_nodeMutex); @@ -269,7 +299,7 @@ public: } } - template + template SharedNodePointer nodeMatchingPredicate(const PredLambda predicate) { QReadLocker readLock(&_nodeMutex); @@ -285,7 +315,7 @@ public: // This is unsafe because it does not take a lock // Must only be called when you know that a read lock on the node mutex is held // and will be held for the duration of your iteration - template + template void unsafeEachNode(NodeLambda functor) { for (NodeHash::const_iterator it = _nodeHash.cbegin(); it != _nodeHash.cend(); ++it) { functor(it->second); @@ -295,15 +325,19 @@ public: void putLocalPortIntoSharedMemory(const QString key, QObject* parent, quint16 localPort); bool getLocalServerPortFromSharedMemory(const QString key, quint16& localPort); - const QMap getLastConnectionTimes() const - { QReadLocker readLock(&_connectionTimeLock); return _lastConnectionTimes; } + const QMap getLastConnectionTimes() const { + QReadLocker readLock(&_connectionTimeLock); + return _lastConnectionTimes; + } void flagTimeForConnectionStep(ConnectionStep connectionStep); udt::Socket::StatsVector sampleStatsForAllConnections() { return _nodeSocket.sampleStatsForAllConnections(); } void setConnectionMaxBandwidth(int maxBandwidth) { _nodeSocket.setConnectionMaxBandwidth(maxBandwidth); } - void setPacketFilterOperator(udt::PacketFilterOperator filterOperator) { _nodeSocket.setPacketFilterOperator(filterOperator); } + void setPacketFilterOperator(udt::PacketFilterOperator filterOperator) { + _nodeSocket.setPacketFilterOperator(filterOperator); + } bool packetVersionMatch(const udt::Packet& packet); bool isPacketVerifiedWithSource(const udt::Packet& packet, Node* sourceNode = nullptr); @@ -328,14 +362,9 @@ public: float getInboundKbps() const { return _inboundKbps; } float getOutboundKbps() const { return _outboundKbps; } - const std::set SOLO_NODE_TYPES = { - NodeType::AvatarMixer, - NodeType::AudioMixer, - NodeType::AssetServer, - NodeType::EntityServer, - NodeType::MessagesMixer, - NodeType::EntityScriptServer - }; + const std::set SOLO_NODE_TYPES = + { NodeType::AvatarMixer, NodeType::AudioMixer, NodeType::AssetServer, + NodeType::EntityServer, NodeType::MessagesMixer, NodeType::EntityScriptServer }; public slots: void reset(); @@ -399,11 +428,10 @@ protected: }; LimitedNodeList(int socketListenPort = INVALID_PORT, int dtlsListenPort = INVALID_PORT); - LimitedNodeList(LimitedNodeList const&) = delete; // Don't implement, needed to avoid copies of singleton - void operator=(LimitedNodeList const&) = delete; // Don't implement, needed to avoid copies of singleton + LimitedNodeList(LimitedNodeList const&) = delete; // Don't implement, needed to avoid copies of singleton + void operator=(LimitedNodeList const&) = delete; // Don't implement, needed to avoid copies of singleton - qint64 sendPacket(std::unique_ptr packet, const Node& destinationNode, - const HifiSockAddr& overridenSockAddr); + qint64 sendPacket(std::unique_ptr packet, const Node& destinationNode, const HifiSockAddr& overridenSockAddr); void fillPacketHeader(const NLPacket& packet, HMACAuth* hmacAuth = nullptr); void setLocalSocket(const HifiSockAddr& sockAddr); @@ -415,7 +443,9 @@ protected: void stopInitialSTUNUpdate(bool success); - void sendPacketToIceServer(PacketType packetType, const HifiSockAddr& iceServerSockAddr, const QUuid& clientID, + void sendPacketToIceServer(PacketType packetType, + const HifiSockAddr& iceServerSockAddr, + const QUuid& clientID, const QUuid& peerRequestID = QUuid()); bool sockAddrBelongsToNode(const HifiSockAddr& sockAddr); @@ -426,14 +456,14 @@ protected: bool isDelayedNode(QUuid nodeUUID); NodeHash _nodeHash; - mutable QReadWriteLock _nodeMutex { QReadWriteLock::Recursive }; + mutable QReadWriteLock _nodeMutex{ QReadWriteLock::Recursive }; udt::Socket _nodeSocket; - QUdpSocket* _dtlsSocket { nullptr }; + QUdpSocket* _dtlsSocket{ nullptr }; HifiSockAddr _localSockAddr; HifiSockAddr _publicSockAddr; - HifiSockAddr _stunSockAddr { STUN_SERVER_HOSTNAME, STUN_SERVER_PORT }; - bool _hasTCPCheckedLocalSocket { false }; - bool _useAuthentication { true }; + HifiSockAddr _stunSockAddr{ STUN_SERVER_HOSTNAME, STUN_SERVER_PORT }; + bool _hasTCPCheckedLocalSocket{ false }; + bool _useAuthentication{ true }; PacketReceiver* _packetReceiver; @@ -446,11 +476,11 @@ protected: quint64 _firstSTUNTime = 0; quint64 _publicSocketUpdateTime = 0; - mutable QReadWriteLock _connectionTimeLock { }; + mutable QReadWriteLock _connectionTimeLock{}; QMap _lastConnectionTimes; bool _areConnectionTimesComplete = false; - template + template void eachNodeHashIterator(IteratorLambda functor) { QWriteLocker writeLock(&_nodeMutex); NodeHash::iterator it = _nodeHash.begin(); @@ -461,31 +491,31 @@ protected: } std::unordered_map _connectionIDs; - quint64 _nodeConnectTimestamp { 0 }; - quint64 _nodeDisconnectTimestamp { 0 }; - bool _wasSilentDomainDisconnect { false }; + quint64 _nodeConnectTimestamp{ 0 }; + quint64 _nodeDisconnectTimestamp{ 0 }; + ConnectReason _connectReason { Connect }; private slots: void flagTimeForConnectionStep(ConnectionStep connectionStep, quint64 timestamp); void possiblyTimeoutSTUNAddressLookup(); - void addSTUNHandlerToUnfiltered(); // called once STUN socket known + void addSTUNHandlerToUnfiltered(); // called once STUN socket known private: mutable QReadWriteLock _sessionUUIDLock; QUuid _sessionUUID; using LocalIDMapping = tbb::concurrent_unordered_map; LocalIDMapping _localIDMap; - Node::LocalID _sessionLocalID { 0 }; - bool _flagTimeForConnectionStep { false }; // only keep track in interface + Node::LocalID _sessionLocalID{ 0 }; + bool _flagTimeForConnectionStep{ false }; // only keep track in interface - size_t _maxConnectionRate { DEFAULT_MAX_CONNECTION_RATE }; - size_t _nodesAddedInCurrentTimeSlice { 0 }; + size_t _maxConnectionRate{ DEFAULT_MAX_CONNECTION_RATE }; + size_t _nodesAddedInCurrentTimeSlice{ 0 }; std::vector _delayedNodeAdds; - int _inboundPPS { 0 }; - int _outboundPPS { 0 }; - float _inboundKbps { 0.0f }; - float _outboundKbps { 0.0f }; + int _inboundPPS{ 0 }; + int _outboundPPS{ 0 }; + float _inboundKbps{ 0.0f }; + float _outboundKbps{ 0.0f }; }; -#endif // hifi_LimitedNodeList_h +#endif // hifi_LimitedNodeList_h diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index fa3c40b138..1f933920eb 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -113,8 +113,7 @@ NodeList::NodeList(char newOwnerType, int socketListenPort, int dtlsListenPort) connect(&_domainHandler, SIGNAL(connectedToDomain(QUrl)), &_keepAlivePingTimer, SLOT(start())); connect(&_domainHandler, &DomainHandler::disconnectedFromDomain, &_keepAlivePingTimer, &QTimer::stop); - connect(&_domainHandler, &DomainHandler::limitOfSilentDomainCheckInsReached, - this, [this]() {_wasSilentDomainDisconnect = true; }); + connect(&_domainHandler, &DomainHandler::limitOfSilentDomainCheckInsReached, this, [this]() { _connectReason = LimitedNodeList::SilentDomainDisconnect; }); // set our sockAddrBelongsToDomainOrNode method as the connection creation filter for the udt::Socket using std::placeholders::_1; @@ -418,7 +417,7 @@ void NodeList::sendDomainServerCheckIn() { auto accountManager = DependencyManager::get(); packetStream << FingerprintUtils::getMachineFingerprint(); - packetStream << quint32(_wasSilentDomainDisconnect ? 1 : 0); + packetStream << _connectReason; if (_nodeDisconnectTimestamp < _nodeConnectTimestamp) { _nodeDisconnectTimestamp = usecTimestampNow(); @@ -683,7 +682,7 @@ void NodeList::processDomainServerList(QSharedPointer message) if (newConnection) { _nodeConnectTimestamp = usecTimestampNow(); - _wasSilentDomainDisconnect = false; + _connectReason = Connect; } qint64 pingLagTime = (now - qint64(connectRequestTimestamp)) / qint64(USECS_PER_MSEC); diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index 7ebaf5224f..30066b68e8 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -27,7 +27,7 @@ PacketVersion versionForPacketType(PacketType packetType) { case PacketType::StunResponse: return 17; case PacketType::DomainList: - return static_cast(DomainListVersion::HasTimestamp); + return static_cast(DomainListVersion::HasConnectReason); case PacketType::EntityAdd: case PacketType::EntityClone: case PacketType::EntityEdit: diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 5baf5448dd..93a5d4e2b4 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -366,7 +366,8 @@ enum class DomainListVersion : PacketVersion { GetUsernameFromUUIDSupport, GetMachineFingerprintFromUUIDSupport, AuthenticationOptional, - HasTimestamp + HasTimestamp, + HasConnectReason }; enum class AudioVersion : PacketVersion { From 6e6bd18baad192fec6828ad9eb3900e22ab220c3 Mon Sep 17 00:00:00 2001 From: Roxanne Skelly Date: Wed, 12 Jun 2019 11:43:43 -0700 Subject: [PATCH 6/7] Undo Visual Studio mis-formatting --- domain-server/src/DomainGatekeeper.cpp | 2 +- libraries/networking/src/LimitedNodeList.h | 157 +++++++++------------ 2 files changed, 68 insertions(+), 91 deletions(-) diff --git a/domain-server/src/DomainGatekeeper.cpp b/domain-server/src/DomainGatekeeper.cpp index fee6069c7a..709c5810e2 100644 --- a/domain-server/src/DomainGatekeeper.cpp +++ b/domain-server/src/DomainGatekeeper.cpp @@ -126,7 +126,7 @@ void DomainGatekeeper::processConnectRequestPacket(QSharedPointer #ifndef _WIN32 -#include // not on windows, not needed for mac or windows +#include // not on windows, not needed for mac or windows #endif #include @@ -51,7 +51,7 @@ const int INVALID_PORT = -1; const quint64 NODE_SILENCE_THRESHOLD_MSECS = 10 * 1000; -static const size_t DEFAULT_MAX_CONNECTION_RATE{ std::numeric_limits::max() }; +static const size_t DEFAULT_MAX_CONNECTION_RATE { std::numeric_limits::max() }; const char DEFAULT_ASSIGNMENT_SERVER_HOSTNAME[] = "localhost"; @@ -67,26 +67,26 @@ const QHostAddress DEFAULT_ASSIGNMENT_CLIENT_MONITOR_HOSTNAME = QHostAddress::Lo const QString USERNAME_UUID_REPLACEMENT_STATS_KEY = "$username"; using ConnectionID = int64_t; -const ConnectionID NULL_CONNECTION_ID{ -1 }; -const ConnectionID INITIAL_CONNECTION_ID{ 0 }; +const ConnectionID NULL_CONNECTION_ID { -1 }; +const ConnectionID INITIAL_CONNECTION_ID { 0 }; typedef std::pair UUIDNodePair; typedef tbb::concurrent_unordered_map NodeHash; typedef quint8 PingType_t; namespace PingType { -const PingType_t Agnostic = 0; -const PingType_t Local = 1; -const PingType_t Public = 2; -const PingType_t Symmetric = 3; -} // namespace PingType + const PingType_t Agnostic = 0; + const PingType_t Local = 1; + const PingType_t Public = 2; + const PingType_t Symmetric = 3; +} class LimitedNodeList : public QObject, public Dependency { Q_OBJECT SINGLETON_DEPENDENCY public: - enum ConnectionStep - { + + enum ConnectionStep { LookupAddress = 1, HandleAddress, SendSTUNRequest, @@ -109,8 +109,7 @@ public: Q_ENUM(ConnectionStep); - enum ConnectReason : quint32 - { + enum ConnectReason : quint32 { Connect = 0, SilentDomainDisconnect }; @@ -125,18 +124,12 @@ public: bool isAllowedEditor() const { return _permissions.can(NodePermissions::Permission::canAdjustLocks); } bool getThisNodeCanRez() const { return _permissions.can(NodePermissions::Permission::canRezPermanentEntities); } bool getThisNodeCanRezTmp() const { return _permissions.can(NodePermissions::Permission::canRezTemporaryEntities); } - bool getThisNodeCanRezCertified() const { - return _permissions.can(NodePermissions::Permission::canRezPermanentCertifiedEntities); - } - bool getThisNodeCanRezTmpCertified() const { - return _permissions.can(NodePermissions::Permission::canRezTemporaryCertifiedEntities); - } + bool getThisNodeCanRezCertified() const { return _permissions.can(NodePermissions::Permission::canRezPermanentCertifiedEntities); } + bool getThisNodeCanRezTmpCertified() const { return _permissions.can(NodePermissions::Permission::canRezTemporaryCertifiedEntities); } bool getThisNodeCanWriteAssets() const { return _permissions.can(NodePermissions::Permission::canWriteToAssetServer); } bool getThisNodeCanKick() const { return _permissions.can(NodePermissions::Permission::canKick); } bool getThisNodeCanReplaceContent() const { return _permissions.can(NodePermissions::Permission::canReplaceDomainContent); } - bool getThisNodeCanGetAndSetPrivateUserData() const { - return _permissions.can(NodePermissions::Permission::canGetAndSetPrivateUserData); - } + bool getThisNodeCanGetAndSetPrivateUserData() const { return _permissions.can(NodePermissions::Permission::canGetAndSetPrivateUserData); } quint16 getSocketLocalPort() const { return _nodeSocket.localPort(); } Q_INVOKABLE void setSocketLocalPort(quint16 socketLocalPort); @@ -146,18 +139,9 @@ public: PacketReceiver& getPacketReceiver() { return *_packetReceiver; } virtual bool isDomainServer() const { return true; } - virtual QUuid getDomainUUID() const { - assert(false); - return QUuid(); - } - virtual Node::LocalID getDomainLocalID() const { - assert(false); - return Node::NULL_LOCAL_ID; - } - virtual HifiSockAddr getDomainSockAddr() const { - assert(false); - return HifiSockAddr(); - } + virtual QUuid getDomainUUID() const { assert(false); return QUuid(); } + virtual Node::LocalID getDomainLocalID() const { assert(false); return Node::NULL_LOCAL_ID; } + virtual HifiSockAddr getDomainSockAddr() const { assert(false); return HifiSockAddr(); } // use sendUnreliablePacket to send an unreliable packet (that you do not need to move) // either to a node (via its active socket) or to a manual sockaddr @@ -171,9 +155,8 @@ public: // use sendUnreliableUnorderedPacketList to unreliably send separate packets from the packet list // either to a node's active socket or to a manual sockaddr qint64 sendUnreliableUnorderedPacketList(NLPacketList& packetList, const Node& destinationNode); - qint64 sendUnreliableUnorderedPacketList(NLPacketList& packetList, - const HifiSockAddr& sockAddr, - HMACAuth* hmacAuth = nullptr); + qint64 sendUnreliableUnorderedPacketList(NLPacketList& packetList, const HifiSockAddr& sockAddr, + HMACAuth* hmacAuth = nullptr); // use sendPacketList to send reliable packet lists (ordered or unordered) to a node's active socket // or to a manual sock addr @@ -182,22 +165,15 @@ public: std::function linkedDataCreateCallback; - size_t size() const { - QReadLocker readLock(&_nodeMutex); - return _nodeHash.size(); - } + size_t size() const { QReadLocker readLock(&_nodeMutex); return _nodeHash.size(); } SharedNodePointer nodeWithUUID(const QUuid& nodeUUID); SharedNodePointer nodeWithLocalID(Node::LocalID localID) const; - SharedNodePointer addOrUpdateNode(const QUuid& uuid, - NodeType_t nodeType, - const HifiSockAddr& publicSocket, - const HifiSockAddr& localSocket, - Node::LocalID localID = Node::NULL_LOCAL_ID, - bool isReplicated = false, - bool isUpstream = false, - const QUuid& connectionSecret = QUuid(), + SharedNodePointer addOrUpdateNode(const QUuid& uuid, NodeType_t nodeType, + const HifiSockAddr& publicSocket, const HifiSockAddr& localSocket, + Node::LocalID localID = Node::NULL_LOCAL_ID, bool isReplicated = false, + bool isUpstream = false, const QUuid& connectionSecret = QUuid(), const NodePermissions& permissions = DEFAULT_AGENT_PERMISSIONS); static bool parseSTUNResponse(udt::BasePacket* packet, QHostAddress& newPublicAddress, uint16_t& newPublicPort); @@ -232,7 +208,7 @@ public: // Use this for nested loops instead of taking nested read locks! // This allows multiple threads (i.e. a thread pool) to share a lock // without deadlocking when a dying node attempts to acquire a write lock - template + template void nestedEach(NestedNodeLambda functor, int* lockWaitOut = nullptr, int* nodeTransformOut = nullptr, @@ -252,8 +228,9 @@ public: // so reserve enough memory for the current size // and then back insert all the nodes found nodes.reserve(_nodeHash.size()); - std::transform(_nodeHash.cbegin(), _nodeHash.cend(), std::back_inserter(nodes), - [&](const NodeHash::value_type& it) { return it.second; }); + std::transform(_nodeHash.cbegin(), _nodeHash.cend(), std::back_inserter(nodes), [&](const NodeHash::value_type& it) { + return it.second; + }); endTransform = usecTimestampNow(); if (nodeTransformOut) { @@ -268,7 +245,7 @@ public: } } - template + template void eachNode(NodeLambda functor) { QReadLocker readLock(&_nodeMutex); @@ -277,7 +254,7 @@ public: } } - template + template void eachMatchingNode(PredLambda predicate, NodeLambda functor) { QReadLocker readLock(&_nodeMutex); @@ -288,7 +265,7 @@ public: } } - template + template void eachNodeBreakable(BreakableNodeLambda functor) { QReadLocker readLock(&_nodeMutex); @@ -299,7 +276,7 @@ public: } } - template + template SharedNodePointer nodeMatchingPredicate(const PredLambda predicate) { QReadLocker readLock(&_nodeMutex); @@ -315,7 +292,7 @@ public: // This is unsafe because it does not take a lock // Must only be called when you know that a read lock on the node mutex is held // and will be held for the duration of your iteration - template + template void unsafeEachNode(NodeLambda functor) { for (NodeHash::const_iterator it = _nodeHash.cbegin(); it != _nodeHash.cend(); ++it) { functor(it->second); @@ -325,19 +302,15 @@ public: void putLocalPortIntoSharedMemory(const QString key, QObject* parent, quint16 localPort); bool getLocalServerPortFromSharedMemory(const QString key, quint16& localPort); - const QMap getLastConnectionTimes() const { - QReadLocker readLock(&_connectionTimeLock); - return _lastConnectionTimes; - } + const QMap getLastConnectionTimes() const + { QReadLocker readLock(&_connectionTimeLock); return _lastConnectionTimes; } void flagTimeForConnectionStep(ConnectionStep connectionStep); udt::Socket::StatsVector sampleStatsForAllConnections() { return _nodeSocket.sampleStatsForAllConnections(); } void setConnectionMaxBandwidth(int maxBandwidth) { _nodeSocket.setConnectionMaxBandwidth(maxBandwidth); } - void setPacketFilterOperator(udt::PacketFilterOperator filterOperator) { - _nodeSocket.setPacketFilterOperator(filterOperator); - } + void setPacketFilterOperator(udt::PacketFilterOperator filterOperator) { _nodeSocket.setPacketFilterOperator(filterOperator); } bool packetVersionMatch(const udt::Packet& packet); bool isPacketVerifiedWithSource(const udt::Packet& packet, Node* sourceNode = nullptr); @@ -362,9 +335,14 @@ public: float getInboundKbps() const { return _inboundKbps; } float getOutboundKbps() const { return _outboundKbps; } - const std::set SOLO_NODE_TYPES = - { NodeType::AvatarMixer, NodeType::AudioMixer, NodeType::AssetServer, - NodeType::EntityServer, NodeType::MessagesMixer, NodeType::EntityScriptServer }; + const std::set SOLO_NODE_TYPES = { + NodeType::AvatarMixer, + NodeType::AudioMixer, + NodeType::AssetServer, + NodeType::EntityServer, + NodeType::MessagesMixer, + NodeType::EntityScriptServer + }; public slots: void reset(); @@ -428,10 +406,11 @@ protected: }; LimitedNodeList(int socketListenPort = INVALID_PORT, int dtlsListenPort = INVALID_PORT); - LimitedNodeList(LimitedNodeList const&) = delete; // Don't implement, needed to avoid copies of singleton - void operator=(LimitedNodeList const&) = delete; // Don't implement, needed to avoid copies of singleton + LimitedNodeList(LimitedNodeList const&) = delete; // Don't implement, needed to avoid copies of singleton + void operator=(LimitedNodeList const&) = delete; // Don't implement, needed to avoid copies of singleton - qint64 sendPacket(std::unique_ptr packet, const Node& destinationNode, const HifiSockAddr& overridenSockAddr); + qint64 sendPacket(std::unique_ptr packet, const Node& destinationNode, + const HifiSockAddr& overridenSockAddr); void fillPacketHeader(const NLPacket& packet, HMACAuth* hmacAuth = nullptr); void setLocalSocket(const HifiSockAddr& sockAddr); @@ -443,9 +422,7 @@ protected: void stopInitialSTUNUpdate(bool success); - void sendPacketToIceServer(PacketType packetType, - const HifiSockAddr& iceServerSockAddr, - const QUuid& clientID, + void sendPacketToIceServer(PacketType packetType, const HifiSockAddr& iceServerSockAddr, const QUuid& clientID, const QUuid& peerRequestID = QUuid()); bool sockAddrBelongsToNode(const HifiSockAddr& sockAddr); @@ -456,14 +433,14 @@ protected: bool isDelayedNode(QUuid nodeUUID); NodeHash _nodeHash; - mutable QReadWriteLock _nodeMutex{ QReadWriteLock::Recursive }; + mutable QReadWriteLock _nodeMutex { QReadWriteLock::Recursive }; udt::Socket _nodeSocket; - QUdpSocket* _dtlsSocket{ nullptr }; + QUdpSocket* _dtlsSocket { nullptr }; HifiSockAddr _localSockAddr; HifiSockAddr _publicSockAddr; - HifiSockAddr _stunSockAddr{ STUN_SERVER_HOSTNAME, STUN_SERVER_PORT }; - bool _hasTCPCheckedLocalSocket{ false }; - bool _useAuthentication{ true }; + HifiSockAddr _stunSockAddr { STUN_SERVER_HOSTNAME, STUN_SERVER_PORT }; + bool _hasTCPCheckedLocalSocket { false }; + bool _useAuthentication { true }; PacketReceiver* _packetReceiver; @@ -476,11 +453,11 @@ protected: quint64 _firstSTUNTime = 0; quint64 _publicSocketUpdateTime = 0; - mutable QReadWriteLock _connectionTimeLock{}; + mutable QReadWriteLock _connectionTimeLock { }; QMap _lastConnectionTimes; bool _areConnectionTimesComplete = false; - template + template void eachNodeHashIterator(IteratorLambda functor) { QWriteLocker writeLock(&_nodeMutex); NodeHash::iterator it = _nodeHash.begin(); @@ -498,24 +475,24 @@ protected: private slots: void flagTimeForConnectionStep(ConnectionStep connectionStep, quint64 timestamp); void possiblyTimeoutSTUNAddressLookup(); - void addSTUNHandlerToUnfiltered(); // called once STUN socket known + void addSTUNHandlerToUnfiltered(); // called once STUN socket known private: mutable QReadWriteLock _sessionUUIDLock; QUuid _sessionUUID; using LocalIDMapping = tbb::concurrent_unordered_map; LocalIDMapping _localIDMap; - Node::LocalID _sessionLocalID{ 0 }; - bool _flagTimeForConnectionStep{ false }; // only keep track in interface + Node::LocalID _sessionLocalID { 0 }; + bool _flagTimeForConnectionStep { false }; // only keep track in interface - size_t _maxConnectionRate{ DEFAULT_MAX_CONNECTION_RATE }; - size_t _nodesAddedInCurrentTimeSlice{ 0 }; + size_t _maxConnectionRate { DEFAULT_MAX_CONNECTION_RATE }; + size_t _nodesAddedInCurrentTimeSlice { 0 }; std::vector _delayedNodeAdds; - int _inboundPPS{ 0 }; - int _outboundPPS{ 0 }; - float _inboundKbps{ 0.0f }; - float _outboundKbps{ 0.0f }; + int _inboundPPS { 0 }; + int _outboundPPS { 0 }; + float _inboundKbps { 0.0f }; + float _outboundKbps { 0.0f }; }; -#endif // hifi_LimitedNodeList_h +#endif // hifi_LimitedNodeList_h From ca0c9e236bd8f1d41589369574e7de7b47f68eca Mon Sep 17 00:00:00 2001 From: Roxanne Skelly Date: Wed, 12 Jun 2019 11:58:26 -0700 Subject: [PATCH 7/7] Cleanup --- libraries/networking/src/NodeList.cpp | 1 + libraries/networking/src/udt/Socket.cpp | 10 +++------- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 1f933920eb..86b33bbe20 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -452,6 +452,7 @@ void NodeList::sendDomainServerCheckIn() { static const int MAX_CHECKINS_TOGETHER = 20; static const int REBIND_CHECKIN_COUNT = 2; int outstandingCheckins = _domainHandler.getCheckInPacketsSinceLastReply(); + if (outstandingCheckins > REBIND_CHECKIN_COUNT) { _nodeSocket.rebind(); } diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index b8e2784017..9871935853 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -62,9 +62,9 @@ void Socket::bind(const QHostAddress& address, quint16 port) { #elif defined(Q_OS_WIN) auto sd = _udpSocket.socketDescriptor(); int val = 0; // false - if (setsockopt(sd, IPPROTO_IP, IP_DONTFRAGMENT, (const char*)&val, sizeof(val))) { - auto err = WSAGetLastError(); - qCWarning(networking) << "Socket::bind Cannot setsockopt IP_DONTFRAGMENT" << err; + if (setsockopt(sd, IPPROTO_IP, IP_DONTFRAGMENT, (const char *)&val, sizeof(val))) { + auto wsaErr = WSAGetLastError(); + qCWarning(networking) << "Socket::bind Cannot setsockopt IP_DONTFRAGMENT" << wsaErr; } #endif } @@ -234,7 +234,6 @@ qint64 Socket::writeDatagram(const QByteArray& datagram, const HifiSockAddr& soc return -1; } qint64 bytesWritten = _udpSocket.writeDatagram(datagram, sockAddr.getAddress(), sockAddr.getPort()); - int pending = _udpSocket.bytesToWrite(); if (bytesWritten < 0 || pending) { int wsaError = 0; @@ -243,9 +242,6 @@ qint64 Socket::writeDatagram(const QByteArray& datagram, const HifiSockAddr& soc #endif qCDebug(networking) << "udt::writeDatagram (" << _udpSocket.state() << ") error - " << wsaError << _udpSocket.error() << "(" << _udpSocket.errorString() << ")" << (pending ? "pending bytes:" : "pending:") << pending; - - qCDebug(networking) << "udt::writeDatagram (" << _udpSocket.state() << ") error - " << _udpSocket.error() << "(" << _udpSocket.errorString() << ")"; - #ifdef DEBUG_EVENT_QUEUE int nodeListQueueSize = ::hifi::qt::getEventQueueSize(thread()); qCDebug(networking) << "Networking queue size - " << nodeListQueueSize;