From 0043a84234c615553fa5b6d9bb39daec32332412 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 25 Mar 2016 10:22:14 -0700 Subject: [PATCH] guard the SendQueue sleep and send a user activity when weird --- libraries/networking/src/udt/SendQueue.cpp | 30 +++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 2ffa42cb82..6fb6ab00e7 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -27,6 +28,7 @@ #include "ControlPacket.h" #include "Packet.h" #include "PacketList.h" +#include "../UserActivityLogger.h" #include "Socket.h" using namespace udt; @@ -328,7 +330,33 @@ void SendQueue::run() { nextPacketTimestamp += std::chrono::microseconds(nextPacketDelta); // sleep as long as we need until next packet send, if we can - const auto timeToSleep = duration_cast(nextPacketTimestamp - p_high_resolution_clock::now()); + auto now = p_high_resolution_clock::now(); + auto timeToSleep = duration_cast(nextPacketTimestamp - now); + + // we're seeing SendQueues sleep for a long period of time here, + // which can lock the NodeList if it's attempting to clear connections + // for now we guard this by capping the time this thread and sleep for + + const int MAX_SEND_QUEUE_SLEEP_USECS = 5000000; + if (timeToSleep.count() > MAX_SEND_QUEUE_SLEEP_USECS) { + // alright, we're in a weird state + // we want to know why this is happening so we can implement a better fix than this guard + // send some details up to the API (if the user allows us) that indicate how we could such a large timeToSleep + static const QString SEND_QUEUE_LONG_SLEEP_ACTION = "sendqueue-sleep"; + + // setup a json object with the details we want + QJsonObject longSleepObject; + longSleepObject["timeToSleep"] = timeToSleep.count(); + longSleepObject["packetSendPeriod"] = _packetSendPeriod.load(); + longSleepObject["nextPacketDelta"] = nextPacketDelta; + longSleepObject["nextPacketTimestamp"] = nextPacketTimestamp.time_since_epoch().count(); + longSleepObject["then"] = now.time_since_epoch().count(); + + // hopefully send this event using the user activity logger + UserActivityLogger::getInstance().logAction(SEND_QUEUE_LONG_SLEEP_ACTION, longSleepObject); + + timeToSleep = std::chrono::microseconds(MAX_SEND_QUEUE_SLEEP_USECS); + } std::this_thread::sleep_for(timeToSleep); }