diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 2ffa42cb82..6fb6ab00e7 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -27,6 +28,7 @@ #include "ControlPacket.h" #include "Packet.h" #include "PacketList.h" +#include "../UserActivityLogger.h" #include "Socket.h" using namespace udt; @@ -328,7 +330,33 @@ void SendQueue::run() { nextPacketTimestamp += std::chrono::microseconds(nextPacketDelta); // sleep as long as we need until next packet send, if we can - const auto timeToSleep = duration_cast(nextPacketTimestamp - p_high_resolution_clock::now()); + auto now = p_high_resolution_clock::now(); + auto timeToSleep = duration_cast(nextPacketTimestamp - now); + + // we're seeing SendQueues sleep for a long period of time here, + // which can lock the NodeList if it's attempting to clear connections + // for now we guard this by capping the time this thread and sleep for + + const int MAX_SEND_QUEUE_SLEEP_USECS = 5000000; + if (timeToSleep.count() > MAX_SEND_QUEUE_SLEEP_USECS) { + // alright, we're in a weird state + // we want to know why this is happening so we can implement a better fix than this guard + // send some details up to the API (if the user allows us) that indicate how we could such a large timeToSleep + static const QString SEND_QUEUE_LONG_SLEEP_ACTION = "sendqueue-sleep"; + + // setup a json object with the details we want + QJsonObject longSleepObject; + longSleepObject["timeToSleep"] = timeToSleep.count(); + longSleepObject["packetSendPeriod"] = _packetSendPeriod.load(); + longSleepObject["nextPacketDelta"] = nextPacketDelta; + longSleepObject["nextPacketTimestamp"] = nextPacketTimestamp.time_since_epoch().count(); + longSleepObject["then"] = now.time_since_epoch().count(); + + // hopefully send this event using the user activity logger + UserActivityLogger::getInstance().logAction(SEND_QUEUE_LONG_SLEEP_ACTION, longSleepObject); + + timeToSleep = std::chrono::microseconds(MAX_SEND_QUEUE_SLEEP_USECS); + } std::this_thread::sleep_for(timeToSleep); }