diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 5826bfa11c..e5940668e7 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -45,7 +45,7 @@ DefaultCC::DefaultCC() : { _mss = udt::MAX_PACKET_SIZE_WITH_UDP_HEADER; - _congestionWindowSize = 16.0; + _congestionWindowSize = 16; setPacketSendPeriod(1.0); } @@ -218,7 +218,7 @@ void DefaultCC::stopSlowStart() { // If no receiving rate is observed, we have to compute the sending // rate according to the current window size, and decrease it // using the method below. - setPacketSendPeriod(_congestionWindowSize / (_rtt + synInterval())); + setPacketSendPeriod(double(_congestionWindowSize) / (_rtt + synInterval())); } } diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 3ab69efe52..c833b83664 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -43,6 +43,7 @@ public: virtual void onACK(SequenceNumber ackNum) {} virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) {} virtual void onTimeout() {} + virtual void onPacketSent(int packetSize, SequenceNumber seqNum) {} protected: void setAckInterval(int ackInterval) { _ackInterval = ackInterval; } void setRTO(int rto) { _userDefinedRTO = true; _rto = rto; } @@ -57,11 +58,11 @@ protected: void setPacketSendPeriod(double newSendPeriod); // call this internally to ensure send period doesn't go past max bandwidth double _packetSendPeriod { 1.0 }; // Packet sending period, in microseconds - double _congestionWindowSize { 16.0 }; // Congestion window size, in packets + uint64_t _congestionWindowSize { 16 }; // Congestion window size, in packets int _bandwidth { 0 }; // estimated bandwidth, packets per second std::atomic _maxBandwidth { -1 }; // Maximum desired bandwidth, bits per second - double _maxCongestionWindowSize { 0.0 }; // maximum cwnd size, in packets + uint64_t _maxCongestionWindowSize { 0 }; // maximum cwnd size, in packets int _mss { 0 }; // Maximum Packet Size, including all packet headers SequenceNumber _sendCurrSeqNum; // current maximum seq num sent out diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 53cacaeeb4..448290116f 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -111,13 +111,13 @@ SendQueue& Connection::getSendQueue() { #ifdef UDT_CONNECTION_DEBUG qCDebug(networking) << "Created SendQueue for connection to" << _destination; #endif - - QObject::connect(_sendQueue.get(), &SendQueue::packetSent, this, &Connection::packetSent); + QObject::connect(_sendQueue.get(), &SendQueue::packetSent, this, &Connection::recordSentPackets); QObject::connect(_sendQueue.get(), &SendQueue::packetRetransmitted, this, &Connection::recordRetransmission); QObject::connect(_sendQueue.get(), &SendQueue::queueInactive, this, &Connection::queueInactive); QObject::connect(_sendQueue.get(), &SendQueue::timeout, this, &Connection::queueTimeout); QObject::connect(_sendQueue.get(), &SendQueue::shortCircuitLoss, this, &Connection::queueShortCircuitLoss); + // set defaults on the send queue from our congestion control object and estimatedTimeout() _sendQueue->setPacketSendPeriod(_congestionControl->_packetSendPeriod); @@ -260,12 +260,16 @@ void Connection::sync() { } } -void Connection::recordSentPackets(int dataSize, int payloadSize) { +void Connection::recordSentPackets(int dataSize, int payloadSize, SequenceNumber seqNum) { _stats.recordSentPackets(payloadSize, dataSize); + + _congestionControl->onPacketSent(dataSize, seqNum); } -void Connection::recordRetransmission() { +void Connection::recordRetransmission(int packetSize, SequenceNumber seqNum) { _stats.record(ConnectionStats::Stats::Retransmission); + + _congestionControl->onPacketSent(packetSize, seqNum); } void Connection::sendACK(bool wasCausedBySyncTimeout) { diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index a18a23f160..4f4ece5059 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -87,8 +87,8 @@ signals: void receiverHandshakeRequestComplete(const HifiSockAddr& sockAddr); private slots: - void recordSentPackets(int payload, int total); - void recordRetransmission(); + void recordSentPackets(int dataSize, int payloadSize, SequenceNumber seqNum); + void recordRetransmission(int packetSize, SequenceNumber sequenceNumber); void queueInactive(); void queueTimeout(); void queueShortCircuitLoss(quint32 sequenceNumber); diff --git a/libraries/networking/src/udt/Constants.h b/libraries/networking/src/udt/Constants.h index 3186571f9b..243fa4edda 100644 --- a/libraries/networking/src/udt/Constants.h +++ b/libraries/networking/src/udt/Constants.h @@ -17,6 +17,7 @@ #include "SequenceNumber.h" namespace udt { + static const int UDP_IPV4_HEADER_SIZE = 28; static const int MAX_PACKET_SIZE_WITH_UDP_HEADER = 1492; static const int MAX_PACKET_SIZE = MAX_PACKET_SIZE_WITH_UDP_HEADER - UDP_IPV4_HEADER_SIZE; diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 65619e2b50..98c8450c30 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -242,7 +242,7 @@ bool SendQueue::sendNewPacketAndAddToSentList(std::unique_ptr newPacket, newPacket->writeSequenceNumber(sequenceNumber); // Save packet/payload size before we move it - auto packetSize = newPacket->getDataSize(); + auto packetSize = newPacket->getWireSize(); auto payloadSize = newPacket->getPayloadSize(); auto bytesWritten = sendPacket(*newPacket); @@ -255,8 +255,8 @@ bool SendQueue::sendNewPacketAndAddToSentList(std::unique_ptr newPacket, entry.second.swap(newPacket); } Q_ASSERT_X(!newPacket, "SendQueue::sendNewPacketAndAddToSentList()", "Overriden packet in sent list"); - - emit packetSent(packetSize, payloadSize); + + emit packetSent(packetSize, payloadSize, sequenceNumber); if (bytesWritten < 0) { // this is a short-circuit loss - we failed to put this packet on the wire @@ -492,7 +492,7 @@ bool SendQueue::maybeResendPacket() { sentLocker.unlock(); } - emit packetRetransmitted(); + emit packetRetransmitted(resendPacket.getWireSize(), it->first); // Signal that we did resend a packet return true; diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 21f6141c3c..d874e4bbd7 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -74,8 +74,8 @@ public slots: void handshakeACK(SequenceNumber initialSequenceNumber); signals: - void packetSent(int dataSize, int payloadSize); - void packetRetransmitted(); + void packetSent(int dataSize, int payloadSize, SequenceNumber seqNum); + void packetRetransmitted(int dataSize, SequenceNumber seqNum); void queueInactive(); diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 3f2664daea..e57edfdf97 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -22,7 +22,7 @@ #include #include "../HifiSockAddr.h" -#include "CongestionControl.h" +#include "TCPVegasCC.h" #include "Connection.h" #include "TCPRenoCC.h" @@ -131,8 +131,8 @@ private: QTimer* _synTimer { nullptr }; int _maxBandwidth { -1 }; - - std::unique_ptr _ccFactory { new CongestionControlFactory() }; + + std::unique_ptr _ccFactory { new CongestionControlFactory() }; friend UDTTest; }; diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp new file mode 100644 index 0000000000..b911b01dae --- /dev/null +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -0,0 +1,123 @@ +// +// TCPVegasCC.cpp +// libraries/networking/src/udt +// +// Created by Stephen Birarda on 2016-09-20. +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "TCPVegasCC.h" + +#include + +using namespace udt; +using namespace std::chrono; + +void TCPVegasCC::onACK(SequenceNumber ackNum) { + auto it = _sentPacketTimes.find(ackNum); + + if (it != _sentPacketTimes.end()) { + + // calculate the RTT (time now - time ACK sent) + auto now = p_high_resolution_clock::now(); + int lastRTT = duration_cast(now - it->second.first).count(); + + if (lastRTT < 0) { + Q_ASSERT_X(false, "TCPVegasCC::onACK", "calculated an RTT that is not > 0"); + return; + } else if (lastRTT == 0) { + // we do not allow a 0 microsecond RTT + lastRTT = 1; + } + + // keep track of the lowest RTT during connection + _baseRTT = std::min(_baseRTT, lastRTT); + + // find the min RTT during the last RTT + _currentMinRTT = std::min(_currentMinRTT, lastRTT); + + ++_numRTT; + + if (ackNum > _lastRTTMaxSeqNum) { + performCongestionAvoidance(ackNum); + } + + } else { + Q_ASSERT_X(false, + "TCPVegasCC::onACK", + "called with a sequence number that has not been sent"); + } +} + +void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { + static int VEGAS_MIN_RTT_FOR_CALC = 3; + + static uint64_t VEGAS_ALPHA_SEGMENTS = 2; + static uint64_t VEGAS_BETA_SEGMENTS = 4; + static uint64_t VEGAS_GAMMA_SEGMENTS = 1; + + if (_numRTT < VEGAS_MIN_RTT_FOR_CALC) { + // Vegas calculations are only done if there are enough RTT samples to be + // pretty sure that at least one sample did not come from a delayed ACK. + // If that is the case, we fallback to the Reno behaviour + TCPRenoCC::performCongestionAvoidance(ack); + } else { + // There are enough RTT samples, use the Vegas algorithm to see if we should + // increase or decrease the congestion window size, and by how much + + // Grab the minimum RTT seen during the last RTT + // Taking the min avoids the effects of delayed ACKs + // (though congestion may be noticed a bit later) + int rtt = _currentMinRTT; + + uint64_t expectedWindowSize = _congestionWindowSize * _baseRTT / rtt; + uint64_t diff = _congestionWindowSize * (rtt - _baseRTT) / _baseRTT; + + if (diff > VEGAS_GAMMA_SEGMENTS && isInSlowStart()) { + // we're going too fast, slow down and switch to congestion avoidance + + // the congestion window should match the actual rate + _congestionWindowSize = std::min(_congestionWindowSize, expectedWindowSize + 1); + adjustSlowStartThreshold(); + + } else if (isInSlowStart()) { + // slow start + performSlowStart(ack); + } else { + // figure out where the congestion window should be + if (diff > VEGAS_BETA_SEGMENTS) { + // the old congestion window was too fast (difference > beta) + // so reduce it to slow down + --_congestionWindowSize; + adjustSlowStartThreshold(); + } else if (diff < VEGAS_ALPHA_SEGMENTS) { + // there aren't enough packets on the wire, add more to the congestion window + ++_congestionWindowSize; + } else { + // sending rate seems good, no congestion window adjustment + } + } + + // we never allow the congestion window to be smaller than two packets + static uint64_t VEGAS_CW_MIN_PACKETS = 2; + _congestionWindowSize = std::min(_congestionWindowSize, VEGAS_CW_MIN_PACKETS); + +// _slowStartThreshold = currentSlowStartThreshold(); + + } + + _lastRTTMaxSeqNum = _sendCurrSeqNum; + + // reset our state for the next RTT + _currentMinRTT = std::numeric_limits::max(); + _numRTT = 0; +} + + +void TCPVegasCC::onPacketSent(int packetSize, SequenceNumber seqNum) { + _sentPacketTimes[seqNum] = { p_high_resolution_clock::now(), packetSize }; +} + diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h new file mode 100644 index 0000000000..7b15f1c8da --- /dev/null +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -0,0 +1,59 @@ +// +// TCPVegasCC.h +// libraries/networking/src/udt +// +// Created by Stephen Birarda on 2016-09-20. +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#pragma once + +#ifndef hifi_TCPVegasCC_h +#define hifi_TCPVegasCC_h + +#include + +#include "CongestionControl.h" +#include "Constants.h" +#include "TCPRenoCC.h" + +namespace udt { + + +class TCPVegasCC : public TCPRenoCC { +public: + TCPVegasCC() {}; + +public: + virtual void onACK(SequenceNumber ackNum) override; + virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) override {}; + virtual void onTimeout() override {}; + virtual void onPacketSent(int packetSize, SequenceNumber seqNum) override; + +protected: + virtual void performCongestionAvoidance(SequenceNumber ack) override; +private: + void adjustSlowStartThreshold() + { _slowStartThreshold = std::min(_slowStartThreshold, (uint32_t) _congestionWindowSize - 1); } + + using TimeSizePair = std::pair; + using PacketTimeList = std::map; + PacketTimeList _sentPacketTimes; // Map of sequence numbers to sent time + + int _currentMinRTT { 0x7FFFFFFF }; // Current RTT, in microseconds + int _baseRTT { 0x7FFFFFFF }; // Lowest RTT during connection, in microseconds + int _numRTT { 0 }; // Number of RTT collected during last RTT + + SequenceNumber _lastRTTMaxSeqNum; // Highest sent sequence number at time of last congestion window adjustment +}; + +} + + + + + +#endif // hifi_TCPVegasCC_h