Merge pull request #11995 from birarda/bug/fix-connection-reset

make udt::Connection live as long as node connection
This commit is contained in:
Howard Stearns 2017-12-19 09:43:54 -08:00 committed by GitHub
commit f8cef6631b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 82 additions and 142 deletions

View file

@ -11,6 +11,8 @@
#include "Connection.h"
#include <random>
#include <QtCore/QThread>
#include <NumericalConstants.h>
@ -60,6 +62,15 @@ Connection::Connection(Socket* parentSocket, HifiSockAddr destination, std::uniq
_ack2Packet = ControlPacket::create(ControlPacket::ACK2, ACK2_PAYLOAD_BYTES);
_lossReport = ControlPacket::create(ControlPacket::NAK, NAK_PACKET_PAYLOAD_BYTES);
_handshakeACK = ControlPacket::create(ControlPacket::HandshakeACK, HANDSHAKE_ACK_PAYLOAD_BYTES);
// setup psuedo-random number generation shared by all connections
static std::random_device rd;
static std::mt19937 generator(rd());
static std::uniform_int_distribution<> distribution(0, SequenceNumber::MAX);
// randomize the intial sequence number
_initialSequenceNumber = SequenceNumber(distribution(generator));
}
Connection::~Connection() {
@ -79,11 +90,11 @@ void Connection::stopSendQueue() {
// tell the send queue to stop and be deleted
sendQueue->stop();
_lastMessageNumber = sendQueue->getCurrentMessageNumber();
sendQueue->deleteLater();
// since we're stopping the send queue we should consider our handshake ACK not receieved
_hasReceivedHandshakeACK = false;
// wait on the send queue thread so we know the send queue is gone
sendQueueThread->quit();
sendQueueThread->wait();
@ -101,13 +112,19 @@ void Connection::setMaxBandwidth(int maxBandwidth) {
SendQueue& Connection::getSendQueue() {
if (!_sendQueue) {
// we may have a sequence number from the previous inactive queue - re-use that so that the
// receiver is getting the sequence numbers it expects (given that the connection must still be active)
// Lasily create send queue
_sendQueue = SendQueue::create(_parentSocket, _destination);
_lastReceivedACK = _sendQueue->getCurrentSequenceNumber();
if (!_hasReceivedHandshakeACK) {
// First time creating a send queue for this connection
_sendQueue = SendQueue::create(_parentSocket, _destination, _initialSequenceNumber - 1, _lastMessageNumber, _hasReceivedHandshakeACK);
_lastReceivedACK = _sendQueue->getCurrentSequenceNumber();
} else {
// Connection already has a handshake from a previous send queue
_sendQueue = SendQueue::create(_parentSocket, _destination, _lastReceivedACK, _lastMessageNumber, _hasReceivedHandshakeACK);
}
#ifdef UDT_CONNECTION_DEBUG
qCDebug(networking) << "Created SendQueue for connection to" << _destination;
@ -142,14 +159,6 @@ void Connection::queueInactive() {
#ifdef UDT_CONNECTION_DEBUG
qCDebug(networking) << "Connection to" << _destination << "has stopped its SendQueue.";
#endif
if (!_hasReceivedHandshake || !_isReceivingData) {
#ifdef UDT_CONNECTION_DEBUG
qCDebug(networking) << "Connection SendQueue to" << _destination << "stopped and no data is being received - stopping connection.";
#endif
deactivate();
}
}
void Connection::queueTimeout() {
@ -184,11 +193,16 @@ void Connection::queueReceivedMessagePacket(std::unique_ptr<Packet> packet) {
while (pendingMessage.hasAvailablePackets()) {
auto packet = pendingMessage.removeNextPacket();
_parentSocket->messageReceived(std::move(packet));
}
if (pendingMessage.isComplete()) {
_pendingReceivedMessages.erase(messageNumber);
auto packetPosition = packet->getPacketPosition();
_parentSocket->messageReceived(std::move(packet));
// if this was the last or only packet, then we can remove the pending message from our hash
if (packetPosition == Packet::PacketPosition::LAST ||
packetPosition == Packet::PacketPosition::ONLY) {
_pendingReceivedMessages.erase(messageNumber);
}
}
}
@ -208,19 +222,6 @@ void Connection::sync() {
&& duration_cast<seconds>(sincePacketReceive).count() >= MIN_SECONDS_BEFORE_EXPIRY ) {
// the receive side of this connection is expired
_isReceivingData = false;
// if we don't have a send queue that means the whole connection has expired and we can emit our signal
// otherwise we'll wait for it to also timeout before cleaning up
if (!_sendQueue) {
#ifdef UDT_CONNECTION_DEBUG
qCDebug(networking) << "Connection to" << _destination << "no longer receiving any data and there is currently no send queue - stopping connection.";
#endif
deactivate();
return;
}
}
// reset the number of light ACKs or non SYN ACKs during this sync interval
@ -242,26 +243,6 @@ void Connection::sync() {
sendTimeoutNAK();
}
}
} else if (!_sendQueue) {
// we haven't received a packet and we're not sending
// this most likely means we were started erroneously
// check the start time for this connection and auto expire it after 5 seconds of not receiving or sending any data
static const int CONNECTION_NOT_USED_EXPIRY_SECONDS = 5;
auto secondsSinceStart = duration_cast<seconds>(p_high_resolution_clock::now() - _connectionStart).count();
if (secondsSinceStart >= CONNECTION_NOT_USED_EXPIRY_SECONDS) {
// it's been CONNECTION_NOT_USED_EXPIRY_SECONDS and nothing has actually happened with this connection
// consider it inactive and emit our inactivity signal
#ifdef UDT_CONNECTION_DEBUG
qCDebug(networking) << "Connection to" << _destination << "did not receive or send any data in last"
<< CONNECTION_NOT_USED_EXPIRY_SECONDS << "seconds - stopping connection.";
#endif
deactivate();
return;
}
}
}
@ -444,7 +425,6 @@ void Connection::sendHandshakeRequest() {
}
bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, int packetSize, int payloadSize) {
if (!_hasReceivedHandshake) {
// Refuse to process any packets until we've received the handshake
// Send handshake request to re-request a handshake
@ -536,7 +516,7 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in
} else {
_stats.recordReceivedPackets(payloadSize, packetSize);
}
return !wasDuplicate;
}
@ -827,11 +807,13 @@ void Connection::processHandshakeACK(ControlPacketPointer controlPacket) {
SequenceNumber initialSequenceNumber;
controlPacket->readPrimitive(&initialSequenceNumber);
// hand off this handshake ACK to the send queue so it knows it can start sending
getSendQueue().handshakeACK(initialSequenceNumber);
// indicate that handshake ACK was received
_hasReceivedHandshakeACK = true;
if (initialSequenceNumber == _initialSequenceNumber) {
// hand off this handshake ACK to the send queue so it knows it can start sending
getSendQueue().handshakeACK();
// indicate that handshake ACK was received
_hasReceivedHandshakeACK = true;
}
}
}

View file

@ -37,7 +37,6 @@ class Socket;
class PendingReceivedMessage {
public:
void enqueuePacket(std::unique_ptr<Packet> packet);
bool isComplete() const { return _hasLastPacket && _numPackets == _packets.size(); }
bool hasAvailablePackets() const;
std::unique_ptr<Packet> removeNextPacket();
@ -72,8 +71,6 @@ public:
void queueReceivedMessagePacket(std::unique_ptr<Packet> packet);
ConnectionStats::Stats sampleStats() { return _stats.sample(); }
bool isActive() const { return _isActive; }
HifiSockAddr getDestination() const { return _destination; }
@ -83,7 +80,6 @@ public:
signals:
void packetSent();
void connectionInactive(const HifiSockAddr& sockAddr);
void receiverHandshakeRequestComplete(const HifiSockAddr& sockAddr);
private slots:
@ -112,8 +108,6 @@ private:
void resetReceiveState();
void resetRTT();
void deactivate() { _isActive = false; emit connectionInactive(_destination); }
SendQueue& getSendQueue();
SequenceNumber nextACK() const;
void updateRTT(int rtt);
@ -138,9 +132,11 @@ private:
p_high_resolution_clock::time_point _lastReceiveTime; // holds the last time we received anything from sender
bool _isReceivingData { false }; // flag used for expiry of receipt portion of connection
bool _isActive { true }; // flag used for inactivity of connection
SequenceNumber _initialReceiveSequenceNumber; // Randomized by peer SendQueue on creation, identifies connection during re-connect requests
SequenceNumber _initialSequenceNumber; // Randomized on Connection creation, identifies connection during re-connect requests
SequenceNumber _initialReceiveSequenceNumber; // Randomized by peer Connection on creation, identifies connection during re-connect requests
MessageNumber _lastMessageNumber { 0 };
LossList _lossList; // List of all missing packets
SequenceNumber _lastReceivedSequenceNumber; // The largest sequence number received from the peer

View file

@ -15,7 +15,7 @@
using namespace udt;
PacketQueue::PacketQueue() {
PacketQueue::PacketQueue(MessageNumber messageNumber) : _currentMessageNumber(messageNumber) {
_channels.emplace_back(new std::list<PacketPointer>());
}

View file

@ -34,7 +34,7 @@ class PacketQueue {
using Channels = std::vector<Channel>;
public:
PacketQueue();
PacketQueue(MessageNumber messageNumber = 0);
void queuePacket(PacketPointer packet);
void queuePacketList(PacketListPointer packetList);
@ -42,6 +42,8 @@ public:
PacketPointer takePacket();
Mutex& getLock() { return _packetsLock; }
MessageNumber getCurrentMessageNumber() const { return _currentMessageNumber; }
private:
MessageNumber getNextMessageNumber();

View file

@ -12,7 +12,6 @@
#include "SendQueue.h"
#include <algorithm>
#include <random>
#include <thread>
#include <QtCore/QCoreApplication>
@ -62,10 +61,12 @@ private:
Mutex2& _mutex2;
};
std::unique_ptr<SendQueue> SendQueue::create(Socket* socket, HifiSockAddr destination) {
std::unique_ptr<SendQueue> SendQueue::create(Socket* socket, HifiSockAddr destination, SequenceNumber currentSequenceNumber,
MessageNumber currentMessageNumber, bool hasReceivedHandshakeACK) {
Q_ASSERT_X(socket, "SendQueue::create", "Must be called with a valid Socket*");
auto queue = std::unique_ptr<SendQueue>(new SendQueue(socket, destination));
auto queue = std::unique_ptr<SendQueue>(new SendQueue(socket, destination, currentSequenceNumber,
currentMessageNumber, hasReceivedHandshakeACK));
// Setup queue private thread
QThread* thread = new QThread;
@ -84,25 +85,18 @@ std::unique_ptr<SendQueue> SendQueue::create(Socket* socket, HifiSockAddr destin
return queue;
}
SendQueue::SendQueue(Socket* socket, HifiSockAddr dest) :
SendQueue::SendQueue(Socket* socket, HifiSockAddr dest, SequenceNumber currentSequenceNumber,
MessageNumber currentMessageNumber, bool hasReceivedHandshakeACK) :
_packets(currentMessageNumber),
_socket(socket),
_destination(dest)
{
// setup psuedo-random number generation for all instances of SendQueue
static std::random_device rd;
static std::mt19937 generator(rd());
static std::uniform_int_distribution<> distribution(0, SequenceNumber::MAX);
// randomize the intial sequence number
_initialSequenceNumber = SequenceNumber(distribution(generator));
// set our member variables from randomized initial number
_currentSequenceNumber = _initialSequenceNumber - 1;
// set our member variables from current sequence number
_currentSequenceNumber = currentSequenceNumber;
_atomicCurrentSequenceNumber = uint32_t(_currentSequenceNumber);
_lastACKSequenceNumber = uint32_t(_currentSequenceNumber) - 1;
_lastACKSequenceNumber = uint32_t(_currentSequenceNumber);
// default the last receiver response to the current time
_lastReceiverResponse = QDateTime::currentMSecsSinceEpoch();
_hasReceivedHandshakeACK = hasReceivedHandshakeACK;
}
SendQueue::~SendQueue() {
@ -114,8 +108,8 @@ void SendQueue::queuePacket(std::unique_ptr<Packet> packet) {
// call notify_one on the condition_variable_any in case the send thread is sleeping waiting for packets
_emptyCondition.notify_one();
if (!this->thread()->isRunning() && _state == State::NotStarted) {
this->thread()->start();
if (!thread()->isRunning() && _state == State::NotStarted) {
thread()->start();
}
}
@ -125,8 +119,8 @@ void SendQueue::queuePacketList(std::unique_ptr<PacketList> packetList) {
// call notify_one on the condition_variable_any in case the send thread is sleeping waiting for packets
_emptyCondition.notify_one();
if (!this->thread()->isRunning() && _state == State::NotStarted) {
this->thread()->start();
if (!thread()->isRunning() && _state == State::NotStarted) {
thread()->start();
}
}
@ -144,9 +138,6 @@ int SendQueue::sendPacket(const Packet& packet) {
}
void SendQueue::ack(SequenceNumber ack) {
// this is a response from the client, re-set our timeout expiry and our last response time
_lastReceiverResponse = QDateTime::currentMSecsSinceEpoch();
if (_lastACKSequenceNumber == (uint32_t) ack) {
return;
}
@ -173,10 +164,7 @@ void SendQueue::ack(SequenceNumber ack) {
_emptyCondition.notify_one();
}
void SendQueue::nak(SequenceNumber start, SequenceNumber end) {
// this is a response from the client, re-set our timeout expiry
_lastReceiverResponse = QDateTime::currentMSecsSinceEpoch();
void SendQueue::nak(SequenceNumber start, SequenceNumber end) {
{
std::lock_guard<std::mutex> nakLocker(_naksLock);
_naks.insert(start, end);
@ -197,9 +185,6 @@ void SendQueue::fastRetransmit(udt::SequenceNumber ack) {
}
void SendQueue::overrideNAKListFromPacket(ControlPacket& packet) {
// this is a response from the client, re-set our timeout expiry
_lastReceiverResponse = QDateTime::currentMSecsSinceEpoch();
{
std::lock_guard<std::mutex> nakLocker(_naksLock);
_naks.clear();
@ -225,8 +210,11 @@ void SendQueue::sendHandshake() {
std::unique_lock<std::mutex> handshakeLock { _handshakeMutex };
if (!_hasReceivedHandshakeACK) {
// we haven't received a handshake ACK from the client, send another now
// if the handshake hasn't been completed, then the initial sequence number
// should be the current sequence number + 1
SequenceNumber initialSequenceNumber = _currentSequenceNumber + 1;
auto handshakePacket = ControlPacket::create(ControlPacket::Handshake, sizeof(SequenceNumber));
handshakePacket->writePrimitive(_initialSequenceNumber);
handshakePacket->writePrimitive(initialSequenceNumber);
_socket->writeBasePacket(*handshakePacket, _destination);
// we wait for the ACK or the re-send interval to expire
@ -235,18 +223,14 @@ void SendQueue::sendHandshake() {
}
}
void SendQueue::handshakeACK(SequenceNumber initialSequenceNumber) {
if (initialSequenceNumber == _initialSequenceNumber) {
{
std::lock_guard<std::mutex> locker { _handshakeMutex };
_hasReceivedHandshakeACK = true;
}
_lastReceiverResponse = QDateTime::currentMSecsSinceEpoch();
// Notify on the handshake ACK condition
_handshakeACKCondition.notify_one();
void SendQueue::handshakeACK() {
{
std::lock_guard<std::mutex> locker { _handshakeMutex };
_hasReceivedHandshakeACK = true;
}
// Notify on the handshake ACK condition
_handshakeACKCondition.notify_one();
}
SequenceNumber SendQueue::getNextSequenceNumber() {
@ -540,28 +524,6 @@ bool SendQueue::maybeResendPacket() {
bool SendQueue::isInactive(bool attemptedToSendPacket) {
// check for connection timeout first
// that will be the case if we have had 16 timeouts since hearing back from the client, and it has been
// at least 5 seconds
static const int NUM_TIMEOUTS_BEFORE_INACTIVE = 16;
static const int MIN_MS_BEFORE_INACTIVE = 5 * 1000;
auto sinceLastResponse = (QDateTime::currentMSecsSinceEpoch() - _lastReceiverResponse);
if (sinceLastResponse > 0 &&
sinceLastResponse >= int64_t(NUM_TIMEOUTS_BEFORE_INACTIVE * (_estimatedTimeout / USECS_PER_MSEC)) &&
sinceLastResponse > MIN_MS_BEFORE_INACTIVE) {
// If the flow window has been full for over CONSIDER_INACTIVE_AFTER,
// then signal the queue is inactive and return so it can be cleaned up
#ifdef UDT_CONNECTION_DEBUG
qCDebug(networking) << "SendQueue to" << _destination << "reached" << NUM_TIMEOUTS_BEFORE_INACTIVE << "timeouts"
<< "and" << MIN_MS_BEFORE_INACTIVE << "milliseconds before receiving any ACK/NAK and is now inactive. Stopping.";
#endif
deactivate();
return true;
}
if (!attemptedToSendPacket) {
// During our processing above we didn't send any packets

View file

@ -50,7 +50,9 @@ public:
Stopped
};
static std::unique_ptr<SendQueue> create(Socket* socket, HifiSockAddr destination);
static std::unique_ptr<SendQueue> create(Socket* socket, HifiSockAddr destination,
SequenceNumber currentSequenceNumber, MessageNumber currentMessageNumber,
bool hasReceivedHandshakeACK);
virtual ~SendQueue();
@ -58,6 +60,7 @@ public:
void queuePacketList(std::unique_ptr<PacketList> packetList);
SequenceNumber getCurrentSequenceNumber() const { return SequenceNumber(_atomicCurrentSequenceNumber); }
MessageNumber getCurrentMessageNumber() const { return _packets.getCurrentMessageNumber(); }
void setFlowWindowSize(int flowWindowSize) { _flowWindowSize = flowWindowSize; }
@ -76,7 +79,7 @@ public slots:
void nak(SequenceNumber start, SequenceNumber end);
void fastRetransmit(SequenceNumber ack);
void overrideNAKListFromPacket(ControlPacket& packet);
void handshakeACK(SequenceNumber initialSequenceNumber);
void handshakeACK();
signals:
void packetSent(int wireSize, int payloadSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint);
@ -91,7 +94,8 @@ private slots:
void run();
private:
SendQueue(Socket* socket, HifiSockAddr dest);
SendQueue(Socket* socket, HifiSockAddr dest, SequenceNumber currentSequenceNumber,
MessageNumber currentMessageNumber, bool hasReceivedHandshakeACK);
SendQueue(SendQueue& other) = delete;
SendQueue(SendQueue&& other) = delete;
@ -115,8 +119,6 @@ private:
Socket* _socket { nullptr }; // Socket to send packet on
HifiSockAddr _destination; // Destination addr
SequenceNumber _initialSequenceNumber; // Randomized on SendQueue creation, identifies connection during re-connect requests
std::atomic<uint32_t> _lastACKSequenceNumber { 0 }; // Last ACKed sequence number
@ -128,7 +130,6 @@ private:
std::atomic<int> _estimatedTimeout { 0 }; // Estimated timeout, set from CC
std::atomic<int> _syncInterval { udt::DEFAULT_SYN_INTERVAL_USECS }; // Sync interval, set from CC
std::atomic<int64_t> _lastReceiverResponse { 0 }; // Timestamp for the last time we got new data from the receiver (ACK/NAK)
std::atomic<int> _flowWindowSize { 0 }; // Flow control window size (number of packets that can be on wire) - set from CC

View file

@ -257,9 +257,6 @@ Connection* Socket::findOrCreateConnection(const HifiSockAddr& sockAddr) {
congestionControl->setMaxBandwidth(_maxBandwidth);
auto connection = std::unique_ptr<Connection>(new Connection(this, sockAddr, std::move(congestionControl)));
// we queue the connection to cleanup connection in case it asks for it during its own rate control sync
QObject::connect(connection.get(), &Connection::connectionInactive, this, &Socket::cleanupConnection);
// allow higher-level classes to find out when connections have completed a handshake
QObject::connect(connection.get(), &Connection::receiverHandshakeRequestComplete,
this, &Socket::clientHandshakeRequestComplete);