diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index f42049f107..2f57523f79 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -11,6 +11,8 @@ #include "Connection.h" +#include + #include #include @@ -60,6 +62,15 @@ Connection::Connection(Socket* parentSocket, HifiSockAddr destination, std::uniq _ack2Packet = ControlPacket::create(ControlPacket::ACK2, ACK2_PAYLOAD_BYTES); _lossReport = ControlPacket::create(ControlPacket::NAK, NAK_PACKET_PAYLOAD_BYTES); _handshakeACK = ControlPacket::create(ControlPacket::HandshakeACK, HANDSHAKE_ACK_PAYLOAD_BYTES); + + + // setup psuedo-random number generation shared by all connections + static std::random_device rd; + static std::mt19937 generator(rd()); + static std::uniform_int_distribution<> distribution(0, SequenceNumber::MAX); + + // randomize the intial sequence number + _initialSequenceNumber = SequenceNumber(distribution(generator)); } Connection::~Connection() { @@ -79,11 +90,11 @@ void Connection::stopSendQueue() { // tell the send queue to stop and be deleted sendQueue->stop(); + + _lastMessageNumber = sendQueue->getCurrentMessageNumber(); + sendQueue->deleteLater(); - // since we're stopping the send queue we should consider our handshake ACK not receieved - _hasReceivedHandshakeACK = false; - // wait on the send queue thread so we know the send queue is gone sendQueueThread->quit(); sendQueueThread->wait(); @@ -101,13 +112,19 @@ void Connection::setMaxBandwidth(int maxBandwidth) { SendQueue& Connection::getSendQueue() { if (!_sendQueue) { - // we may have a sequence number from the previous inactive queue - re-use that so that the // receiver is getting the sequence numbers it expects (given that the connection must still be active) // Lasily create send queue - _sendQueue = SendQueue::create(_parentSocket, _destination); - _lastReceivedACK = _sendQueue->getCurrentSequenceNumber(); + + if (!_hasReceivedHandshakeACK) { + // First time creating a send queue for this connection + _sendQueue = SendQueue::create(_parentSocket, _destination, _initialSequenceNumber - 1, _lastMessageNumber, _hasReceivedHandshakeACK); + _lastReceivedACK = _sendQueue->getCurrentSequenceNumber(); + } else { + // Connection already has a handshake from a previous send queue + _sendQueue = SendQueue::create(_parentSocket, _destination, _lastReceivedACK, _lastMessageNumber, _hasReceivedHandshakeACK); + } #ifdef UDT_CONNECTION_DEBUG qCDebug(networking) << "Created SendQueue for connection to" << _destination; @@ -142,14 +159,6 @@ void Connection::queueInactive() { #ifdef UDT_CONNECTION_DEBUG qCDebug(networking) << "Connection to" << _destination << "has stopped its SendQueue."; #endif - - if (!_hasReceivedHandshake || !_isReceivingData) { -#ifdef UDT_CONNECTION_DEBUG - qCDebug(networking) << "Connection SendQueue to" << _destination << "stopped and no data is being received - stopping connection."; -#endif - - deactivate(); - } } void Connection::queueTimeout() { @@ -184,11 +193,16 @@ void Connection::queueReceivedMessagePacket(std::unique_ptr packet) { while (pendingMessage.hasAvailablePackets()) { auto packet = pendingMessage.removeNextPacket(); - _parentSocket->messageReceived(std::move(packet)); - } - if (pendingMessage.isComplete()) { - _pendingReceivedMessages.erase(messageNumber); + auto packetPosition = packet->getPacketPosition(); + + _parentSocket->messageReceived(std::move(packet)); + + // if this was the last or only packet, then we can remove the pending message from our hash + if (packetPosition == Packet::PacketPosition::LAST || + packetPosition == Packet::PacketPosition::ONLY) { + _pendingReceivedMessages.erase(messageNumber); + } } } @@ -208,19 +222,6 @@ void Connection::sync() { && duration_cast(sincePacketReceive).count() >= MIN_SECONDS_BEFORE_EXPIRY ) { // the receive side of this connection is expired _isReceivingData = false; - - // if we don't have a send queue that means the whole connection has expired and we can emit our signal - // otherwise we'll wait for it to also timeout before cleaning up - if (!_sendQueue) { - -#ifdef UDT_CONNECTION_DEBUG - qCDebug(networking) << "Connection to" << _destination << "no longer receiving any data and there is currently no send queue - stopping connection."; -#endif - - deactivate(); - - return; - } } // reset the number of light ACKs or non SYN ACKs during this sync interval @@ -242,26 +243,6 @@ void Connection::sync() { sendTimeoutNAK(); } } - } else if (!_sendQueue) { - // we haven't received a packet and we're not sending - // this most likely means we were started erroneously - // check the start time for this connection and auto expire it after 5 seconds of not receiving or sending any data - static const int CONNECTION_NOT_USED_EXPIRY_SECONDS = 5; - auto secondsSinceStart = duration_cast(p_high_resolution_clock::now() - _connectionStart).count(); - - if (secondsSinceStart >= CONNECTION_NOT_USED_EXPIRY_SECONDS) { - // it's been CONNECTION_NOT_USED_EXPIRY_SECONDS and nothing has actually happened with this connection - // consider it inactive and emit our inactivity signal - -#ifdef UDT_CONNECTION_DEBUG - qCDebug(networking) << "Connection to" << _destination << "did not receive or send any data in last" - << CONNECTION_NOT_USED_EXPIRY_SECONDS << "seconds - stopping connection."; -#endif - - deactivate(); - - return; - } } } @@ -444,7 +425,6 @@ void Connection::sendHandshakeRequest() { } bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, int packetSize, int payloadSize) { - if (!_hasReceivedHandshake) { // Refuse to process any packets until we've received the handshake // Send handshake request to re-request a handshake @@ -536,7 +516,7 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in } else { _stats.recordReceivedPackets(payloadSize, packetSize); } - + return !wasDuplicate; } @@ -827,11 +807,13 @@ void Connection::processHandshakeACK(ControlPacketPointer controlPacket) { SequenceNumber initialSequenceNumber; controlPacket->readPrimitive(&initialSequenceNumber); - // hand off this handshake ACK to the send queue so it knows it can start sending - getSendQueue().handshakeACK(initialSequenceNumber); - - // indicate that handshake ACK was received - _hasReceivedHandshakeACK = true; + if (initialSequenceNumber == _initialSequenceNumber) { + // hand off this handshake ACK to the send queue so it knows it can start sending + getSendQueue().handshakeACK(); + + // indicate that handshake ACK was received + _hasReceivedHandshakeACK = true; + } } } diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index c134081dde..0017eb204a 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -37,7 +37,6 @@ class Socket; class PendingReceivedMessage { public: void enqueuePacket(std::unique_ptr packet); - bool isComplete() const { return _hasLastPacket && _numPackets == _packets.size(); } bool hasAvailablePackets() const; std::unique_ptr removeNextPacket(); @@ -72,8 +71,6 @@ public: void queueReceivedMessagePacket(std::unique_ptr packet); ConnectionStats::Stats sampleStats() { return _stats.sample(); } - - bool isActive() const { return _isActive; } HifiSockAddr getDestination() const { return _destination; } @@ -83,7 +80,6 @@ public: signals: void packetSent(); - void connectionInactive(const HifiSockAddr& sockAddr); void receiverHandshakeRequestComplete(const HifiSockAddr& sockAddr); private slots: @@ -112,8 +108,6 @@ private: void resetReceiveState(); void resetRTT(); - void deactivate() { _isActive = false; emit connectionInactive(_destination); } - SendQueue& getSendQueue(); SequenceNumber nextACK() const; void updateRTT(int rtt); @@ -138,9 +132,11 @@ private: p_high_resolution_clock::time_point _lastReceiveTime; // holds the last time we received anything from sender bool _isReceivingData { false }; // flag used for expiry of receipt portion of connection - bool _isActive { true }; // flag used for inactivity of connection - SequenceNumber _initialReceiveSequenceNumber; // Randomized by peer SendQueue on creation, identifies connection during re-connect requests + SequenceNumber _initialSequenceNumber; // Randomized on Connection creation, identifies connection during re-connect requests + SequenceNumber _initialReceiveSequenceNumber; // Randomized by peer Connection on creation, identifies connection during re-connect requests + + MessageNumber _lastMessageNumber { 0 }; LossList _lossList; // List of all missing packets SequenceNumber _lastReceivedSequenceNumber; // The largest sequence number received from the peer diff --git a/libraries/networking/src/udt/PacketQueue.cpp b/libraries/networking/src/udt/PacketQueue.cpp index 9560f2f187..0560855ecb 100644 --- a/libraries/networking/src/udt/PacketQueue.cpp +++ b/libraries/networking/src/udt/PacketQueue.cpp @@ -15,7 +15,7 @@ using namespace udt; -PacketQueue::PacketQueue() { +PacketQueue::PacketQueue(MessageNumber messageNumber) : _currentMessageNumber(messageNumber) { _channels.emplace_back(new std::list()); } diff --git a/libraries/networking/src/udt/PacketQueue.h b/libraries/networking/src/udt/PacketQueue.h index 2b3d3a4b5b..bc4c5e3432 100644 --- a/libraries/networking/src/udt/PacketQueue.h +++ b/libraries/networking/src/udt/PacketQueue.h @@ -34,7 +34,7 @@ class PacketQueue { using Channels = std::vector; public: - PacketQueue(); + PacketQueue(MessageNumber messageNumber = 0); void queuePacket(PacketPointer packet); void queuePacketList(PacketListPointer packetList); @@ -42,6 +42,8 @@ public: PacketPointer takePacket(); Mutex& getLock() { return _packetsLock; } + + MessageNumber getCurrentMessageNumber() const { return _currentMessageNumber; } private: MessageNumber getNextMessageNumber(); diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 0c029751aa..b62624aab9 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -12,7 +12,6 @@ #include "SendQueue.h" #include -#include #include #include @@ -62,10 +61,12 @@ private: Mutex2& _mutex2; }; -std::unique_ptr SendQueue::create(Socket* socket, HifiSockAddr destination) { +std::unique_ptr SendQueue::create(Socket* socket, HifiSockAddr destination, SequenceNumber currentSequenceNumber, + MessageNumber currentMessageNumber, bool hasReceivedHandshakeACK) { Q_ASSERT_X(socket, "SendQueue::create", "Must be called with a valid Socket*"); - auto queue = std::unique_ptr(new SendQueue(socket, destination)); + auto queue = std::unique_ptr(new SendQueue(socket, destination, currentSequenceNumber, + currentMessageNumber, hasReceivedHandshakeACK)); // Setup queue private thread QThread* thread = new QThread; @@ -84,25 +85,18 @@ std::unique_ptr SendQueue::create(Socket* socket, HifiSockAddr destin return queue; } -SendQueue::SendQueue(Socket* socket, HifiSockAddr dest) : +SendQueue::SendQueue(Socket* socket, HifiSockAddr dest, SequenceNumber currentSequenceNumber, + MessageNumber currentMessageNumber, bool hasReceivedHandshakeACK) : + _packets(currentMessageNumber), _socket(socket), _destination(dest) { - // setup psuedo-random number generation for all instances of SendQueue - static std::random_device rd; - static std::mt19937 generator(rd()); - static std::uniform_int_distribution<> distribution(0, SequenceNumber::MAX); - - // randomize the intial sequence number - _initialSequenceNumber = SequenceNumber(distribution(generator)); - - // set our member variables from randomized initial number - _currentSequenceNumber = _initialSequenceNumber - 1; + // set our member variables from current sequence number + _currentSequenceNumber = currentSequenceNumber; _atomicCurrentSequenceNumber = uint32_t(_currentSequenceNumber); - _lastACKSequenceNumber = uint32_t(_currentSequenceNumber) - 1; + _lastACKSequenceNumber = uint32_t(_currentSequenceNumber); - // default the last receiver response to the current time - _lastReceiverResponse = QDateTime::currentMSecsSinceEpoch(); + _hasReceivedHandshakeACK = hasReceivedHandshakeACK; } SendQueue::~SendQueue() { @@ -114,8 +108,8 @@ void SendQueue::queuePacket(std::unique_ptr packet) { // call notify_one on the condition_variable_any in case the send thread is sleeping waiting for packets _emptyCondition.notify_one(); - if (!this->thread()->isRunning() && _state == State::NotStarted) { - this->thread()->start(); + if (!thread()->isRunning() && _state == State::NotStarted) { + thread()->start(); } } @@ -125,8 +119,8 @@ void SendQueue::queuePacketList(std::unique_ptr packetList) { // call notify_one on the condition_variable_any in case the send thread is sleeping waiting for packets _emptyCondition.notify_one(); - if (!this->thread()->isRunning() && _state == State::NotStarted) { - this->thread()->start(); + if (!thread()->isRunning() && _state == State::NotStarted) { + thread()->start(); } } @@ -144,9 +138,6 @@ int SendQueue::sendPacket(const Packet& packet) { } void SendQueue::ack(SequenceNumber ack) { - // this is a response from the client, re-set our timeout expiry and our last response time - _lastReceiverResponse = QDateTime::currentMSecsSinceEpoch(); - if (_lastACKSequenceNumber == (uint32_t) ack) { return; } @@ -173,10 +164,7 @@ void SendQueue::ack(SequenceNumber ack) { _emptyCondition.notify_one(); } -void SendQueue::nak(SequenceNumber start, SequenceNumber end) { - // this is a response from the client, re-set our timeout expiry - _lastReceiverResponse = QDateTime::currentMSecsSinceEpoch(); - +void SendQueue::nak(SequenceNumber start, SequenceNumber end) { { std::lock_guard nakLocker(_naksLock); _naks.insert(start, end); @@ -197,9 +185,6 @@ void SendQueue::fastRetransmit(udt::SequenceNumber ack) { } void SendQueue::overrideNAKListFromPacket(ControlPacket& packet) { - // this is a response from the client, re-set our timeout expiry - _lastReceiverResponse = QDateTime::currentMSecsSinceEpoch(); - { std::lock_guard nakLocker(_naksLock); _naks.clear(); @@ -225,8 +210,11 @@ void SendQueue::sendHandshake() { std::unique_lock handshakeLock { _handshakeMutex }; if (!_hasReceivedHandshakeACK) { // we haven't received a handshake ACK from the client, send another now + // if the handshake hasn't been completed, then the initial sequence number + // should be the current sequence number + 1 + SequenceNumber initialSequenceNumber = _currentSequenceNumber + 1; auto handshakePacket = ControlPacket::create(ControlPacket::Handshake, sizeof(SequenceNumber)); - handshakePacket->writePrimitive(_initialSequenceNumber); + handshakePacket->writePrimitive(initialSequenceNumber); _socket->writeBasePacket(*handshakePacket, _destination); // we wait for the ACK or the re-send interval to expire @@ -235,18 +223,14 @@ void SendQueue::sendHandshake() { } } -void SendQueue::handshakeACK(SequenceNumber initialSequenceNumber) { - if (initialSequenceNumber == _initialSequenceNumber) { - { - std::lock_guard locker { _handshakeMutex }; - _hasReceivedHandshakeACK = true; - } - - _lastReceiverResponse = QDateTime::currentMSecsSinceEpoch(); - - // Notify on the handshake ACK condition - _handshakeACKCondition.notify_one(); +void SendQueue::handshakeACK() { + { + std::lock_guard locker { _handshakeMutex }; + _hasReceivedHandshakeACK = true; } + + // Notify on the handshake ACK condition + _handshakeACKCondition.notify_one(); } SequenceNumber SendQueue::getNextSequenceNumber() { @@ -540,28 +524,6 @@ bool SendQueue::maybeResendPacket() { bool SendQueue::isInactive(bool attemptedToSendPacket) { // check for connection timeout first - // that will be the case if we have had 16 timeouts since hearing back from the client, and it has been - // at least 5 seconds - static const int NUM_TIMEOUTS_BEFORE_INACTIVE = 16; - static const int MIN_MS_BEFORE_INACTIVE = 5 * 1000; - - auto sinceLastResponse = (QDateTime::currentMSecsSinceEpoch() - _lastReceiverResponse); - - if (sinceLastResponse > 0 && - sinceLastResponse >= int64_t(NUM_TIMEOUTS_BEFORE_INACTIVE * (_estimatedTimeout / USECS_PER_MSEC)) && - sinceLastResponse > MIN_MS_BEFORE_INACTIVE) { - // If the flow window has been full for over CONSIDER_INACTIVE_AFTER, - // then signal the queue is inactive and return so it can be cleaned up - -#ifdef UDT_CONNECTION_DEBUG - qCDebug(networking) << "SendQueue to" << _destination << "reached" << NUM_TIMEOUTS_BEFORE_INACTIVE << "timeouts" - << "and" << MIN_MS_BEFORE_INACTIVE << "milliseconds before receiving any ACK/NAK and is now inactive. Stopping."; -#endif - - deactivate(); - return true; - } - if (!attemptedToSendPacket) { // During our processing above we didn't send any packets diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 484afcb88e..a11aacdb91 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -50,7 +50,9 @@ public: Stopped }; - static std::unique_ptr create(Socket* socket, HifiSockAddr destination); + static std::unique_ptr create(Socket* socket, HifiSockAddr destination, + SequenceNumber currentSequenceNumber, MessageNumber currentMessageNumber, + bool hasReceivedHandshakeACK); virtual ~SendQueue(); @@ -58,6 +60,7 @@ public: void queuePacketList(std::unique_ptr packetList); SequenceNumber getCurrentSequenceNumber() const { return SequenceNumber(_atomicCurrentSequenceNumber); } + MessageNumber getCurrentMessageNumber() const { return _packets.getCurrentMessageNumber(); } void setFlowWindowSize(int flowWindowSize) { _flowWindowSize = flowWindowSize; } @@ -76,7 +79,7 @@ public slots: void nak(SequenceNumber start, SequenceNumber end); void fastRetransmit(SequenceNumber ack); void overrideNAKListFromPacket(ControlPacket& packet); - void handshakeACK(SequenceNumber initialSequenceNumber); + void handshakeACK(); signals: void packetSent(int wireSize, int payloadSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint); @@ -91,7 +94,8 @@ private slots: void run(); private: - SendQueue(Socket* socket, HifiSockAddr dest); + SendQueue(Socket* socket, HifiSockAddr dest, SequenceNumber currentSequenceNumber, + MessageNumber currentMessageNumber, bool hasReceivedHandshakeACK); SendQueue(SendQueue& other) = delete; SendQueue(SendQueue&& other) = delete; @@ -115,8 +119,6 @@ private: Socket* _socket { nullptr }; // Socket to send packet on HifiSockAddr _destination; // Destination addr - - SequenceNumber _initialSequenceNumber; // Randomized on SendQueue creation, identifies connection during re-connect requests std::atomic _lastACKSequenceNumber { 0 }; // Last ACKed sequence number @@ -128,7 +130,6 @@ private: std::atomic _estimatedTimeout { 0 }; // Estimated timeout, set from CC std::atomic _syncInterval { udt::DEFAULT_SYN_INTERVAL_USECS }; // Sync interval, set from CC - std::atomic _lastReceiverResponse { 0 }; // Timestamp for the last time we got new data from the receiver (ACK/NAK) std::atomic _flowWindowSize { 0 }; // Flow control window size (number of packets that can be on wire) - set from CC diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index a3374a0f47..55643985c8 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -257,9 +257,6 @@ Connection* Socket::findOrCreateConnection(const HifiSockAddr& sockAddr) { congestionControl->setMaxBandwidth(_maxBandwidth); auto connection = std::unique_ptr(new Connection(this, sockAddr, std::move(congestionControl))); - // we queue the connection to cleanup connection in case it asks for it during its own rate control sync - QObject::connect(connection.get(), &Connection::connectionInactive, this, &Socket::cleanupConnection); - // allow higher-level classes to find out when connections have completed a handshake QObject::connect(connection.get(), &Connection::receiverHandshakeRequestComplete, this, &Socket::clientHandshakeRequestComplete);