Merge commit '16eb3444c11c6513ff0d8a1d73a50c9856bf432b'

assignment-client/src/avatars/AvatarMixer.cpp

@@ -703,8 +703,11 @@ void AvatarMixer::handleRadiusIgnoreRequestPacket(QSharedPointer<ReceivedMessage
 }
 
 void AvatarMixer::sendStatsPacket() {
-    auto start = usecTimestampNow();
+    if (!_numTightLoopFrames) {
+        return;
+    }
 
+    auto start = usecTimestampNow();
 
     QJsonObject statsObject;
 

libraries/networking/src/NodeList.cpp

@@ -290,41 +290,47 @@ void NodeList::addSetOfNodeTypesToNodeInterestSet(const NodeSet& setOfNodeTypes)
 
 void NodeList::sendDomainServerCheckIn() {
 
+    // This function is called by the server check-in timer thread
+    // not the NodeList thread.  Calling it on the NodeList thread
+    // resulted in starvation of the server check-in function.
+    // be VERY CAREFUL modifying this code as members of NodeList
+    // may be called by multiple threads.
+
     if (!_sendDomainServerCheckInEnabled) {
         qCDebug(networking) << "Refusing to send a domain-server check in while it is disabled.";
         return;
     }
 
-    if (thread() != QThread::currentThread()) {
-        QMetaObject::invokeMethod(this, "sendDomainServerCheckIn", Qt::QueuedConnection);
-        return;
-    }
-
     if (_isShuttingDown) {
         qCDebug(networking) << "Refusing to send a domain-server check in while shutting down.";
         return;
     }
 
-    if (_publicSockAddr.isNull()) {
+    auto publicSockAddr = _publicSockAddr;
+    auto domainHandlerIp = _domainHandler.getIP();
+
+    if (publicSockAddr.isNull()) {
         // we don't know our public socket and we need to send it to the domain server
         qCDebug(networking) << "Waiting for inital public socket from STUN. Will not send domain-server check in.";
-    } else if (_domainHandler.getIP().isNull() && _domainHandler.requiresICE()) {
+    } else if (domainHandlerIp.isNull() && _domainHandler.requiresICE()) {
         qCDebug(networking) << "Waiting for ICE discovered domain-server socket. Will not send domain-server check in.";
         handleICEConnectionToDomainServer();
         // let the domain handler know we are due to send a checkin packet
-    } else if (!_domainHandler.getIP().isNull() && !_domainHandler.checkInPacketTimeout()) {
+    } else if (!domainHandlerIp.isNull() && !_domainHandler.checkInPacketTimeout()) {
-
+        bool domainIsConnected = _domainHandler.isConnected();
-        PacketType domainPacketType = !_domainHandler.isConnected()
+        HifiSockAddr domainSockAddr = _domainHandler.getSockAddr();
+        PacketType domainPacketType = !domainIsConnected
             ? PacketType::DomainConnectRequest : PacketType::DomainListRequest;
 
-        if (!_domainHandler.isConnected()) {
+        if (!domainIsConnected) {
-            qCDebug(networking) << "Sending connect request to domain-server at" << _domainHandler.getHostname();
+            auto hostname = _domainHandler.getHostname();
+            qCDebug(networking) << "Sending connect request to domain-server at" << hostname;
 
             // is this our localhost domain-server?
             // if so we need to make sure we have an up-to-date local port in case it restarted
 
-            if (_domainHandler.getSockAddr().getAddress() == QHostAddress::LocalHost
+            if (domainSockAddr.getAddress() == QHostAddress::LocalHost
-                || _domainHandler.getHostname() == "localhost") {
+                || hostname == "localhost") {
 
                 quint16 domainPort = DEFAULT_DOMAIN_SERVER_PORT;
                 getLocalServerPortFromSharedMemory(DOMAIN_SERVER_LOCAL_PORT_SMEM_KEY, domainPort);
@@ -337,7 +343,7 @@ void NodeList::sendDomainServerCheckIn() {
         auto accountManager = DependencyManager::get<AccountManager>();
         const QUuid& connectionToken = _domainHandler.getConnectionToken();
 
-        bool requiresUsernameSignature = !_domainHandler.isConnected() && !connectionToken.isNull();
+        bool requiresUsernameSignature = !domainIsConnected && !connectionToken.isNull();
 
         if (requiresUsernameSignature && !accountManager->getAccountInfo().hasPrivateKey()) {
             qWarning() << "A keypair is required to present a username signature to the domain-server"
@@ -352,6 +358,7 @@ void NodeList::sendDomainServerCheckIn() {
 
         QDataStream packetStream(domainPacket.get());
 
+        HifiSockAddr localSockAddr = _localSockAddr;
         if (domainPacketType == PacketType::DomainConnectRequest) {
 
 #if (PR_BUILD || DEV_BUILD)
@@ -360,13 +367,9 @@ void NodeList::sendDomainServerCheckIn() {
             }
 #endif
 
-            QUuid connectUUID;
+            QUuid connectUUID = _domainHandler.getAssignmentUUID();
 
-            if (!_domainHandler.getAssignmentUUID().isNull()) {
+            if (connectUUID.isNull() && _domainHandler.requiresICE()) {
-                // this is a connect request and we're an assigned node
-                // so set our packetUUID as the assignment UUID
-                connectUUID = _domainHandler.getAssignmentUUID();
-            } else if (_domainHandler.requiresICE()) {
                 // this is a connect request and we're an interface client
                 // that used ice to discover the DS
                 // so send our ICE client UUID with the connect request
@@ -382,10 +385,9 @@ void NodeList::sendDomainServerCheckIn() {
 
             // if possible, include the MAC address for the current interface in our connect request
             QString hardwareAddress;
-
             for (auto networkInterface : QNetworkInterface::allInterfaces()) {
                 for (auto interfaceAddress : networkInterface.addressEntries()) {
-                    if (interfaceAddress.ip() == _localSockAddr.getAddress()) {
+                    if (interfaceAddress.ip() == localSockAddr.getAddress()) {
                         // this is the interface whose local IP matches what we've detected the current IP to be
                         hardwareAddress = networkInterface.hardwareAddress();
 
@@ -409,10 +411,10 @@ void NodeList::sendDomainServerCheckIn() {
 
         // pack our data to send to the domain-server including
         // the hostname information (so the domain-server can see which place name we came in on)
-        packetStream << _ownerType.load() << _publicSockAddr << _localSockAddr << _nodeTypesOfInterest.toList();
+        packetStream << _ownerType.load() << publicSockAddr << localSockAddr << _nodeTypesOfInterest.toList();
         packetStream << DependencyManager::get<AddressManager>()->getPlaceName();
 
-        if (!_domainHandler.isConnected()) {
+        if (!domainIsConnected) {
             DataServerAccountInfo& accountInfo = accountManager->getAccountInfo();
             packetStream << accountInfo.getUsername();
 
@@ -432,9 +434,9 @@ void NodeList::sendDomainServerCheckIn() {
         checkinCount = std::min(checkinCount, MAX_CHECKINS_TOGETHER);
         for (int i = 1; i < checkinCount; ++i) {
             auto packetCopy = domainPacket->createCopy(*domainPacket);
-            sendPacket(std::move(packetCopy), _domainHandler.getSockAddr());
+            sendPacket(std::move(packetCopy), domainSockAddr);
         }
-        sendPacket(std::move(domainPacket), _domainHandler.getSockAddr());
+        sendPacket(std::move(domainPacket), domainSockAddr);
         
     }
 }

libraries/networking/src/ThreadedAssignment.cpp

@@ -102,6 +102,11 @@ void ThreadedAssignment::addPacketStatsAndSendStatsPacket(QJsonObject statsObjec
 
     statsObject["io_stats"] = ioStats;
 
+    QJsonObject assignmentStats;
+    assignmentStats["numQueuedCheckIns"] = _numQueuedCheckIns;
+
+    statsObject["assignmentStats"] = assignmentStats;
+
     nodeList->sendStatsToDomainServer(statsObject);
 }
 
@@ -119,7 +124,10 @@ void ThreadedAssignment::checkInWithDomainServerOrExit() {
         stop();
     } else {
         auto nodeList = DependencyManager::get<NodeList>();
-        QMetaObject::invokeMethod(nodeList.data(), "sendDomainServerCheckIn");
+        // Call sendDomainServerCheckIn directly instead of putting it on
+        // the event queue.  Under high load, the event queue can back up
+        // longer than the total timeout period and cause a restart
+        nodeList->sendDomainServerCheckIn();
 
         // increase the number of queued check ins
         _numQueuedCheckIns++;