From bad96d54ea954025a443da2942385a9fba8812cc Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 20 Sep 2016 17:29:31 -0700 Subject: [PATCH 01/65] TCP congestion control first draft --- libraries/networking/src/udt/Socket.h | 3 +- libraries/networking/src/udt/TCPRenoCC.cpp | 84 ++++++++++++++++++++++ libraries/networking/src/udt/TCPRenoCC.h | 40 +++++++++++ tests/render-perf/src/main.cpp | 2 + tools/udt-test/src/UDTTest.cpp | 2 +- 5 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 libraries/networking/src/udt/TCPRenoCC.cpp create mode 100644 libraries/networking/src/udt/TCPRenoCC.h diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 59c3ec9cde..3f2664daea 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -24,6 +24,7 @@ #include "../HifiSockAddr.h" #include "CongestionControl.h" #include "Connection.h" +#include "TCPRenoCC.h" //#define UDT_CONNECTION_DEBUG @@ -131,7 +132,7 @@ private: int _maxBandwidth { -1 }; - std::unique_ptr _ccFactory { new CongestionControlFactory() }; + std::unique_ptr _ccFactory { new CongestionControlFactory() }; friend UDTTest; }; diff --git a/libraries/networking/src/udt/TCPRenoCC.cpp b/libraries/networking/src/udt/TCPRenoCC.cpp new file mode 100644 index 0000000000..978b862d4a --- /dev/null +++ b/libraries/networking/src/udt/TCPRenoCC.cpp @@ -0,0 +1,84 @@ +// +// TCPRenoCC.cpp +// libraries/networking/src/udt +// +// Created by Clement on 9/20/16. +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "TCPRenoCC.h" + + +using namespace udt; + +void TCPRenoCC::init() { + _slowStart = true; + _issThreshold = 83333; + + _packetSendPeriod = 0.0; + _congestionWindowSize = 2.0; + + setAckInterval(2); + setRTO(1000000); +} + +void TCPRenoCC::onACK(SequenceNumber ackNum) { + if (ackNum == _lastACK) { + if (3 == ++_duplicateAckCount) { + duplicateACKAction(); + } else if (_duplicateAckCount > 3) { + _congestionWindowSize += 1.0; + } else { + ackAction(); + } + } else { + if (_duplicateAckCount >= 3) { + _congestionWindowSize = _issThreshold; + } + + _lastACK = ackNum; + _duplicateAckCount = 1; + + ackAction(); + } +} + +void TCPRenoCC::onTimeout() { + _issThreshold = seqlen(_lastACK, _sendCurrSeqNum) / 2; + if (_issThreshold < 2) { + _issThreshold = 2; + } + + _slowStart = true; + _congestionWindowSize = 2.0; +} + +void TCPRenoCC::ackAction() { + if (_slowStart) { + _congestionWindowSize += 1.0; + + if (_congestionWindowSize >= _issThreshold) { + _slowStart = false; + } + } else { + _congestionWindowSize += 1.0 / _congestionWindowSize; + } +} + +void TCPRenoCC::duplicateACKAction() { + _slowStart = false; + + _issThreshold = seqlen(_lastACK, _sendCurrSeqNum) / 2; + if (_issThreshold < 2) { + _issThreshold = 2; + } + + _congestionWindowSize = _issThreshold + 3; +} + +void TCPRenoCC::setInitialSendSequenceNumber(SequenceNumber seqNum) { + _lastACK = seqNum - 1; +} diff --git a/libraries/networking/src/udt/TCPRenoCC.h b/libraries/networking/src/udt/TCPRenoCC.h new file mode 100644 index 0000000000..79b7b6f559 --- /dev/null +++ b/libraries/networking/src/udt/TCPRenoCC.h @@ -0,0 +1,40 @@ +// +// TCPRenoCC.h +// libraries/networking/src/udt +// +// Created by Clement on 9/20/16. +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_TCPRenoCC_h +#define hifi_TCPRenoCC_h + +#include "CongestionControl.h" + +namespace udt { + +class TCPRenoCC : public CongestionControl { +public: + virtual void init(); + virtual void onACK(SequenceNumber ackNum); + virtual void onTimeout(); + +protected: + virtual void ackAction(); + virtual void duplicateACKAction(); + virtual void setInitialSendSequenceNumber(SequenceNumber seqNum); + + int _issThreshold; + bool _slowStart; + + int _duplicateAckCount; + SequenceNumber _lastACK; +}; + +} + + +#endif // hifi_TCPRenoCC_h diff --git a/tests/render-perf/src/main.cpp b/tests/render-perf/src/main.cpp index 8abb9f54e5..3df36c0edf 100644 --- a/tests/render-perf/src/main.cpp +++ b/tests/render-perf/src/main.cpp @@ -18,10 +18,12 @@ #include #include #include +#include #include #include #include + #include #include #include diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index af94d2294b..633251d6c6 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -25,7 +25,7 @@ const QCommandLineOption TARGET_OPTION { "IP:PORT or HOSTNAME:PORT" }; const QCommandLineOption PACKET_SIZE { - "packet-size", "size for sent packets in bytes (defaults to " + QString(MAX_PACKET_SIZE) + ")", "bytes", + "packet-size", "size for sent packets in bytes (defaults to " + QString(udt::MAX_PACKET_SIZE) + ")", "bytes", QString(udt::MAX_PACKET_SIZE) }; const QCommandLineOption MIN_PACKET_SIZE { From 59bcd724ea952070844a76ddb15ae33ed4f7b691 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 23 Sep 2016 15:10:42 -0700 Subject: [PATCH 02/65] More Reno work --- libraries/networking/src/udt/TCPRenoCC.cpp | 109 +++++++++++---------- libraries/networking/src/udt/TCPRenoCC.h | 26 +++-- 2 files changed, 77 insertions(+), 58 deletions(-) diff --git a/libraries/networking/src/udt/TCPRenoCC.cpp b/libraries/networking/src/udt/TCPRenoCC.cpp index 978b862d4a..7f83601354 100644 --- a/libraries/networking/src/udt/TCPRenoCC.cpp +++ b/libraries/networking/src/udt/TCPRenoCC.cpp @@ -11,13 +11,11 @@ #include "TCPRenoCC.h" +#include using namespace udt; void TCPRenoCC::init() { - _slowStart = true; - _issThreshold = 83333; - _packetSendPeriod = 0.0; _congestionWindowSize = 2.0; @@ -26,59 +24,70 @@ void TCPRenoCC::init() { } void TCPRenoCC::onACK(SequenceNumber ackNum) { - if (ackNum == _lastACK) { - if (3 == ++_duplicateAckCount) { - duplicateACKAction(); - } else if (_duplicateAckCount > 3) { - _congestionWindowSize += 1.0; - } else { - ackAction(); + int numAcked = seqlen(_lastACK, ackNum); + + performCongestionAvoidance(ackNum, numAcked); + + _lastACK = ackNum; +} + +bool TCPRenoCC::isInSlowStart() { + return _sendCongestionWindowSize < _sendSlowStartThreshold; +} + +int TCPRenoCC::slowStart(int numAcked) { + int congestionWindow = std::min(_sendCongestionWindowSize + numAcked, _sendSlowStartThreshold); + numAcked -= congestionWindow - _sendCongestionWindowSize; + _sendCongestionWindowSize = std::min(congestionWindow, _sendCongestionWindowClamp); + + return numAcked; +} + +int TCPRenoCC::slowStartThreshold() { + // Slow start threshold is half the congestion window (min 2) + return std::max(_sendCongestionWindowSize >> 1, 2); +} + +bool TCPRenoCC::isCongestionWindowLimited() { + if (isInSlowStart()) { + return _sendCongestionWindowSize < 2 * _maxPacketsOut; + } + + return _isCongestionWindowLimited; +} + +void TCPRenoCC::performCongestionAvoidance(SequenceNumber ack, int numAcked) { + if (!isCongestionWindowLimited()) { + return; + } + + // In "safe" area, increase. + if (isInSlowStart()) { + numAcked = slowStart(numAcked); + if (!numAcked) { + return; } - } else { - if (_duplicateAckCount >= 3) { - _congestionWindowSize = _issThreshold; - } - - _lastACK = ackNum; - _duplicateAckCount = 1; - - ackAction(); - } -} - -void TCPRenoCC::onTimeout() { - _issThreshold = seqlen(_lastACK, _sendCurrSeqNum) / 2; - if (_issThreshold < 2) { - _issThreshold = 2; } - _slowStart = true; - _congestionWindowSize = 2.0; + // In dangerous area, increase slowly. + performCongestionAvoidanceAI(_sendCongestionWindowSize, numAcked); } -void TCPRenoCC::ackAction() { - if (_slowStart) { - _congestionWindowSize += 1.0; - - if (_congestionWindowSize >= _issThreshold) { - _slowStart = false; - } - } else { - _congestionWindowSize += 1.0 / _congestionWindowSize; - } -} - -void TCPRenoCC::duplicateACKAction() { - _slowStart = false; - - _issThreshold = seqlen(_lastACK, _sendCurrSeqNum) / 2; - if (_issThreshold < 2) { - _issThreshold = 2; +void TCPRenoCC::performCongestionAvoidanceAI(int sendCongestionWindowSize, int numAcked) { + // If credits accumulated at a higher w, apply them gently now. + if (_sendCongestionWindowCount >= sendCongestionWindowSize) { + _sendCongestionWindowCount = 0; + _sendCongestionWindowSize++; } - _congestionWindowSize = _issThreshold + 3; + _sendCongestionWindowCount += numAcked; + if (_sendCongestionWindowCount >= sendCongestionWindowSize) { + int delta = _sendCongestionWindowCount / sendCongestionWindowSize; + + _sendCongestionWindowCount -= delta * sendCongestionWindowSize; + _sendCongestionWindowCount += delta; + } + _sendCongestionWindowCount = std::min(_sendCongestionWindowCount, _sendCongestionWindowClamp); + } -void TCPRenoCC::setInitialSendSequenceNumber(SequenceNumber seqNum) { - _lastACK = seqNum - 1; -} diff --git a/libraries/networking/src/udt/TCPRenoCC.h b/libraries/networking/src/udt/TCPRenoCC.h index 79b7b6f559..71c32b3a48 100644 --- a/libraries/networking/src/udt/TCPRenoCC.h +++ b/libraries/networking/src/udt/TCPRenoCC.h @@ -20,17 +20,27 @@ class TCPRenoCC : public CongestionControl { public: virtual void init(); virtual void onACK(SequenceNumber ackNum); - virtual void onTimeout(); protected: - virtual void ackAction(); - virtual void duplicateACKAction(); - virtual void setInitialSendSequenceNumber(SequenceNumber seqNum); + virtual void setInitialSendSequenceNumber(SequenceNumber seqNum) { _lastACK = seqNum - 1; } + + bool isInSlowStart(); + int slowStart(int numAcked); + int slowStartThreshold(); + + bool isCongestionWindowLimited(); + void performCongestionAvoidanceAI(int sendCongestionWindowSize, int numAcked); + + virtual void performCongestionAvoidance(SequenceNumber ack, int numAcked); + + + int _sendSlowStartThreshold; + int _sendCongestionWindowSize; + int _sendCongestionWindowCount; + int _sendCongestionWindowClamp = ~0; + int _maxPacketsOut; + bool _isCongestionWindowLimited; - int _issThreshold; - bool _slowStart; - - int _duplicateAckCount; SequenceNumber _lastACK; }; From 8304bb81d17c747aee63d577a415f2ae43a169c8 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 21 Sep 2016 12:53:56 -0700 Subject: [PATCH 03/65] create the base of the TCPVegasCC class --- .../networking/src/udt/CongestionControl.cpp | 4 +- .../networking/src/udt/CongestionControl.h | 5 +- libraries/networking/src/udt/Connection.cpp | 12 +- libraries/networking/src/udt/Connection.h | 4 +- libraries/networking/src/udt/Constants.h | 1 + libraries/networking/src/udt/SendQueue.cpp | 8 +- libraries/networking/src/udt/SendQueue.h | 4 +- libraries/networking/src/udt/Socket.h | 6 +- libraries/networking/src/udt/TCPVegasCC.cpp | 123 ++++++++++++++++++ libraries/networking/src/udt/TCPVegasCC.h | 59 +++++++++ 10 files changed, 207 insertions(+), 19 deletions(-) create mode 100644 libraries/networking/src/udt/TCPVegasCC.cpp create mode 100644 libraries/networking/src/udt/TCPVegasCC.h diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 5826bfa11c..e5940668e7 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -45,7 +45,7 @@ DefaultCC::DefaultCC() : { _mss = udt::MAX_PACKET_SIZE_WITH_UDP_HEADER; - _congestionWindowSize = 16.0; + _congestionWindowSize = 16; setPacketSendPeriod(1.0); } @@ -218,7 +218,7 @@ void DefaultCC::stopSlowStart() { // If no receiving rate is observed, we have to compute the sending // rate according to the current window size, and decrease it // using the method below. - setPacketSendPeriod(_congestionWindowSize / (_rtt + synInterval())); + setPacketSendPeriod(double(_congestionWindowSize) / (_rtt + synInterval())); } } diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 3ab69efe52..c833b83664 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -43,6 +43,7 @@ public: virtual void onACK(SequenceNumber ackNum) {} virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) {} virtual void onTimeout() {} + virtual void onPacketSent(int packetSize, SequenceNumber seqNum) {} protected: void setAckInterval(int ackInterval) { _ackInterval = ackInterval; } void setRTO(int rto) { _userDefinedRTO = true; _rto = rto; } @@ -57,11 +58,11 @@ protected: void setPacketSendPeriod(double newSendPeriod); // call this internally to ensure send period doesn't go past max bandwidth double _packetSendPeriod { 1.0 }; // Packet sending period, in microseconds - double _congestionWindowSize { 16.0 }; // Congestion window size, in packets + uint64_t _congestionWindowSize { 16 }; // Congestion window size, in packets int _bandwidth { 0 }; // estimated bandwidth, packets per second std::atomic _maxBandwidth { -1 }; // Maximum desired bandwidth, bits per second - double _maxCongestionWindowSize { 0.0 }; // maximum cwnd size, in packets + uint64_t _maxCongestionWindowSize { 0 }; // maximum cwnd size, in packets int _mss { 0 }; // Maximum Packet Size, including all packet headers SequenceNumber _sendCurrSeqNum; // current maximum seq num sent out diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 53cacaeeb4..448290116f 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -111,13 +111,13 @@ SendQueue& Connection::getSendQueue() { #ifdef UDT_CONNECTION_DEBUG qCDebug(networking) << "Created SendQueue for connection to" << _destination; #endif - - QObject::connect(_sendQueue.get(), &SendQueue::packetSent, this, &Connection::packetSent); + QObject::connect(_sendQueue.get(), &SendQueue::packetSent, this, &Connection::recordSentPackets); QObject::connect(_sendQueue.get(), &SendQueue::packetRetransmitted, this, &Connection::recordRetransmission); QObject::connect(_sendQueue.get(), &SendQueue::queueInactive, this, &Connection::queueInactive); QObject::connect(_sendQueue.get(), &SendQueue::timeout, this, &Connection::queueTimeout); QObject::connect(_sendQueue.get(), &SendQueue::shortCircuitLoss, this, &Connection::queueShortCircuitLoss); + // set defaults on the send queue from our congestion control object and estimatedTimeout() _sendQueue->setPacketSendPeriod(_congestionControl->_packetSendPeriod); @@ -260,12 +260,16 @@ void Connection::sync() { } } -void Connection::recordSentPackets(int dataSize, int payloadSize) { +void Connection::recordSentPackets(int dataSize, int payloadSize, SequenceNumber seqNum) { _stats.recordSentPackets(payloadSize, dataSize); + + _congestionControl->onPacketSent(dataSize, seqNum); } -void Connection::recordRetransmission() { +void Connection::recordRetransmission(int packetSize, SequenceNumber seqNum) { _stats.record(ConnectionStats::Stats::Retransmission); + + _congestionControl->onPacketSent(packetSize, seqNum); } void Connection::sendACK(bool wasCausedBySyncTimeout) { diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index a18a23f160..4f4ece5059 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -87,8 +87,8 @@ signals: void receiverHandshakeRequestComplete(const HifiSockAddr& sockAddr); private slots: - void recordSentPackets(int payload, int total); - void recordRetransmission(); + void recordSentPackets(int dataSize, int payloadSize, SequenceNumber seqNum); + void recordRetransmission(int packetSize, SequenceNumber sequenceNumber); void queueInactive(); void queueTimeout(); void queueShortCircuitLoss(quint32 sequenceNumber); diff --git a/libraries/networking/src/udt/Constants.h b/libraries/networking/src/udt/Constants.h index 3186571f9b..243fa4edda 100644 --- a/libraries/networking/src/udt/Constants.h +++ b/libraries/networking/src/udt/Constants.h @@ -17,6 +17,7 @@ #include "SequenceNumber.h" namespace udt { + static const int UDP_IPV4_HEADER_SIZE = 28; static const int MAX_PACKET_SIZE_WITH_UDP_HEADER = 1492; static const int MAX_PACKET_SIZE = MAX_PACKET_SIZE_WITH_UDP_HEADER - UDP_IPV4_HEADER_SIZE; diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 65619e2b50..98c8450c30 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -242,7 +242,7 @@ bool SendQueue::sendNewPacketAndAddToSentList(std::unique_ptr newPacket, newPacket->writeSequenceNumber(sequenceNumber); // Save packet/payload size before we move it - auto packetSize = newPacket->getDataSize(); + auto packetSize = newPacket->getWireSize(); auto payloadSize = newPacket->getPayloadSize(); auto bytesWritten = sendPacket(*newPacket); @@ -255,8 +255,8 @@ bool SendQueue::sendNewPacketAndAddToSentList(std::unique_ptr newPacket, entry.second.swap(newPacket); } Q_ASSERT_X(!newPacket, "SendQueue::sendNewPacketAndAddToSentList()", "Overriden packet in sent list"); - - emit packetSent(packetSize, payloadSize); + + emit packetSent(packetSize, payloadSize, sequenceNumber); if (bytesWritten < 0) { // this is a short-circuit loss - we failed to put this packet on the wire @@ -492,7 +492,7 @@ bool SendQueue::maybeResendPacket() { sentLocker.unlock(); } - emit packetRetransmitted(); + emit packetRetransmitted(resendPacket.getWireSize(), it->first); // Signal that we did resend a packet return true; diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 21f6141c3c..d874e4bbd7 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -74,8 +74,8 @@ public slots: void handshakeACK(SequenceNumber initialSequenceNumber); signals: - void packetSent(int dataSize, int payloadSize); - void packetRetransmitted(); + void packetSent(int dataSize, int payloadSize, SequenceNumber seqNum); + void packetRetransmitted(int dataSize, SequenceNumber seqNum); void queueInactive(); diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 3f2664daea..e57edfdf97 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -22,7 +22,7 @@ #include #include "../HifiSockAddr.h" -#include "CongestionControl.h" +#include "TCPVegasCC.h" #include "Connection.h" #include "TCPRenoCC.h" @@ -131,8 +131,8 @@ private: QTimer* _synTimer { nullptr }; int _maxBandwidth { -1 }; - - std::unique_ptr _ccFactory { new CongestionControlFactory() }; + + std::unique_ptr _ccFactory { new CongestionControlFactory() }; friend UDTTest; }; diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp new file mode 100644 index 0000000000..b911b01dae --- /dev/null +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -0,0 +1,123 @@ +// +// TCPVegasCC.cpp +// libraries/networking/src/udt +// +// Created by Stephen Birarda on 2016-09-20. +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "TCPVegasCC.h" + +#include + +using namespace udt; +using namespace std::chrono; + +void TCPVegasCC::onACK(SequenceNumber ackNum) { + auto it = _sentPacketTimes.find(ackNum); + + if (it != _sentPacketTimes.end()) { + + // calculate the RTT (time now - time ACK sent) + auto now = p_high_resolution_clock::now(); + int lastRTT = duration_cast(now - it->second.first).count(); + + if (lastRTT < 0) { + Q_ASSERT_X(false, "TCPVegasCC::onACK", "calculated an RTT that is not > 0"); + return; + } else if (lastRTT == 0) { + // we do not allow a 0 microsecond RTT + lastRTT = 1; + } + + // keep track of the lowest RTT during connection + _baseRTT = std::min(_baseRTT, lastRTT); + + // find the min RTT during the last RTT + _currentMinRTT = std::min(_currentMinRTT, lastRTT); + + ++_numRTT; + + if (ackNum > _lastRTTMaxSeqNum) { + performCongestionAvoidance(ackNum); + } + + } else { + Q_ASSERT_X(false, + "TCPVegasCC::onACK", + "called with a sequence number that has not been sent"); + } +} + +void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { + static int VEGAS_MIN_RTT_FOR_CALC = 3; + + static uint64_t VEGAS_ALPHA_SEGMENTS = 2; + static uint64_t VEGAS_BETA_SEGMENTS = 4; + static uint64_t VEGAS_GAMMA_SEGMENTS = 1; + + if (_numRTT < VEGAS_MIN_RTT_FOR_CALC) { + // Vegas calculations are only done if there are enough RTT samples to be + // pretty sure that at least one sample did not come from a delayed ACK. + // If that is the case, we fallback to the Reno behaviour + TCPRenoCC::performCongestionAvoidance(ack); + } else { + // There are enough RTT samples, use the Vegas algorithm to see if we should + // increase or decrease the congestion window size, and by how much + + // Grab the minimum RTT seen during the last RTT + // Taking the min avoids the effects of delayed ACKs + // (though congestion may be noticed a bit later) + int rtt = _currentMinRTT; + + uint64_t expectedWindowSize = _congestionWindowSize * _baseRTT / rtt; + uint64_t diff = _congestionWindowSize * (rtt - _baseRTT) / _baseRTT; + + if (diff > VEGAS_GAMMA_SEGMENTS && isInSlowStart()) { + // we're going too fast, slow down and switch to congestion avoidance + + // the congestion window should match the actual rate + _congestionWindowSize = std::min(_congestionWindowSize, expectedWindowSize + 1); + adjustSlowStartThreshold(); + + } else if (isInSlowStart()) { + // slow start + performSlowStart(ack); + } else { + // figure out where the congestion window should be + if (diff > VEGAS_BETA_SEGMENTS) { + // the old congestion window was too fast (difference > beta) + // so reduce it to slow down + --_congestionWindowSize; + adjustSlowStartThreshold(); + } else if (diff < VEGAS_ALPHA_SEGMENTS) { + // there aren't enough packets on the wire, add more to the congestion window + ++_congestionWindowSize; + } else { + // sending rate seems good, no congestion window adjustment + } + } + + // we never allow the congestion window to be smaller than two packets + static uint64_t VEGAS_CW_MIN_PACKETS = 2; + _congestionWindowSize = std::min(_congestionWindowSize, VEGAS_CW_MIN_PACKETS); + +// _slowStartThreshold = currentSlowStartThreshold(); + + } + + _lastRTTMaxSeqNum = _sendCurrSeqNum; + + // reset our state for the next RTT + _currentMinRTT = std::numeric_limits::max(); + _numRTT = 0; +} + + +void TCPVegasCC::onPacketSent(int packetSize, SequenceNumber seqNum) { + _sentPacketTimes[seqNum] = { p_high_resolution_clock::now(), packetSize }; +} + diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h new file mode 100644 index 0000000000..7b15f1c8da --- /dev/null +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -0,0 +1,59 @@ +// +// TCPVegasCC.h +// libraries/networking/src/udt +// +// Created by Stephen Birarda on 2016-09-20. +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#pragma once + +#ifndef hifi_TCPVegasCC_h +#define hifi_TCPVegasCC_h + +#include + +#include "CongestionControl.h" +#include "Constants.h" +#include "TCPRenoCC.h" + +namespace udt { + + +class TCPVegasCC : public TCPRenoCC { +public: + TCPVegasCC() {}; + +public: + virtual void onACK(SequenceNumber ackNum) override; + virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) override {}; + virtual void onTimeout() override {}; + virtual void onPacketSent(int packetSize, SequenceNumber seqNum) override; + +protected: + virtual void performCongestionAvoidance(SequenceNumber ack) override; +private: + void adjustSlowStartThreshold() + { _slowStartThreshold = std::min(_slowStartThreshold, (uint32_t) _congestionWindowSize - 1); } + + using TimeSizePair = std::pair; + using PacketTimeList = std::map; + PacketTimeList _sentPacketTimes; // Map of sequence numbers to sent time + + int _currentMinRTT { 0x7FFFFFFF }; // Current RTT, in microseconds + int _baseRTT { 0x7FFFFFFF }; // Lowest RTT during connection, in microseconds + int _numRTT { 0 }; // Number of RTT collected during last RTT + + SequenceNumber _lastRTTMaxSeqNum; // Highest sent sequence number at time of last congestion window adjustment +}; + +} + + + + + +#endif // hifi_TCPVegasCC_h From eb180b15d8753225d1e07dfd2f3d9b87ad0fceaf Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 11:02:16 -0700 Subject: [PATCH 04/65] only raise ssthresh if not reducing cwnd --- libraries/networking/src/udt/TCPVegasCC.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index b911b01dae..6b158f8467 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -76,6 +76,8 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { uint64_t expectedWindowSize = _congestionWindowSize * _baseRTT / rtt; uint64_t diff = _congestionWindowSize * (rtt - _baseRTT) / _baseRTT; + bool inWindowReduction = false; + if (diff > VEGAS_GAMMA_SEGMENTS && isInSlowStart()) { // we're going too fast, slow down and switch to congestion avoidance @@ -83,6 +85,7 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { _congestionWindowSize = std::min(_congestionWindowSize, expectedWindowSize + 1); adjustSlowStartThreshold(); + inWindowReduction = true; } else if (isInSlowStart()) { // slow start performSlowStart(ack); @@ -93,6 +96,8 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { // so reduce it to slow down --_congestionWindowSize; adjustSlowStartThreshold(); + + inWindowReduction = true; } else if (diff < VEGAS_ALPHA_SEGMENTS) { // there aren't enough packets on the wire, add more to the congestion window ++_congestionWindowSize; @@ -101,12 +106,13 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { } } - // we never allow the congestion window to be smaller than two packets - static uint64_t VEGAS_CW_MIN_PACKETS = 2; - _congestionWindowSize = std::min(_congestionWindowSize, VEGAS_CW_MIN_PACKETS); - -// _slowStartThreshold = currentSlowStartThreshold(); + if (!inWindowReduction) { + // we didn't reduce the congestion window size, so it is now time to raise the slow start threshold + // we never allow the congestion window to be smaller than two packets + static uint64_t VEGAS_CW_MIN_PACKETS = 2; + _congestionWindowSize = std::min(_congestionWindowSize, VEGAS_CW_MIN_PACKETS); + } } _lastRTTMaxSeqNum = _sendCurrSeqNum; From 8c9b46c1e053022209810deb14ddba31ab42e2b1 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 11:05:44 -0700 Subject: [PATCH 05/65] fix method signatures, adjust ssthresh increases --- libraries/networking/src/udt/TCPVegasCC.cpp | 35 ++++++++++++--------- libraries/networking/src/udt/TCPVegasCC.h | 2 +- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 6b158f8467..4e716c4d1c 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -42,9 +42,12 @@ void TCPVegasCC::onACK(SequenceNumber ackNum) { ++_numRTT; if (ackNum > _lastRTTMaxSeqNum) { - performCongestionAvoidance(ackNum); + int numACK = seqlen(_lastACK, ackNum); + performCongestionAvoidance(ackNum, numACK); } + _lastACK = ackNum; + } else { Q_ASSERT_X(false, "TCPVegasCC::onACK", @@ -52,7 +55,7 @@ void TCPVegasCC::onACK(SequenceNumber ackNum) { } } -void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { +void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack, int numACK) { static int VEGAS_MIN_RTT_FOR_CALC = 3; static uint64_t VEGAS_ALPHA_SEGMENTS = 2; @@ -63,7 +66,8 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { // Vegas calculations are only done if there are enough RTT samples to be // pretty sure that at least one sample did not come from a delayed ACK. // If that is the case, we fallback to the Reno behaviour - TCPRenoCC::performCongestionAvoidance(ack); + + TCPRenoCC::performCongestionAvoidance(ack, numACK); } else { // There are enough RTT samples, use the Vegas algorithm to see if we should // increase or decrease the congestion window size, and by how much @@ -106,20 +110,23 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { } } - if (!inWindowReduction) { - // we didn't reduce the congestion window size, so it is now time to raise the slow start threshold + // we never allow the congestion window to be smaller than two packets + static uint64_t VEGAS_CW_MIN_PACKETS = 2; + _congestionWindowSize = std::min(_congestionWindowSize, VEGAS_CW_MIN_PACKETS); - // we never allow the congestion window to be smaller than two packets - static uint64_t VEGAS_CW_MIN_PACKETS = 2; - _congestionWindowSize = std::min(_congestionWindowSize, VEGAS_CW_MIN_PACKETS); + if (!inWindowReduction && _congestionWindowSize > _slowStartThreshold) { + // if we didn't just reduce the congestion window size and the + // the congestion window is greater than the slow start threshold + // we raise the slow start threshold half the distance to the congestion window + _slowStartThreshold = (_congestionWindowSize >> 1) + (_congestionWindowSize >> 2); } + + _lastRTTMaxSeqNum = _sendCurrSeqNum; + + // reset our state for the next RTT + _currentMinRTT = std::numeric_limits::max(); + _numRTT = 0; } - - _lastRTTMaxSeqNum = _sendCurrSeqNum; - - // reset our state for the next RTT - _currentMinRTT = std::numeric_limits::max(); - _numRTT = 0; } diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index 7b15f1c8da..634262423a 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -34,7 +34,7 @@ public: virtual void onPacketSent(int packetSize, SequenceNumber seqNum) override; protected: - virtual void performCongestionAvoidance(SequenceNumber ack) override; + virtual void performCongestionAvoidance(SequenceNumber ack, int numACK) override; private: void adjustSlowStartThreshold() { _slowStartThreshold = std::min(_slowStartThreshold, (uint32_t) _congestionWindowSize - 1); } From 69ca0d2675ad43eeed3fcc64231aee299e736baf Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 11:29:52 -0700 Subject: [PATCH 06/65] point TCPVegasCC at correct TCPRenoCC base --- libraries/networking/src/udt/TCPRenoCC.cpp | 4 ++-- libraries/networking/src/udt/TCPRenoCC.h | 2 +- libraries/networking/src/udt/TCPVegasCC.cpp | 10 +++++----- libraries/networking/src/udt/TCPVegasCC.h | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/libraries/networking/src/udt/TCPRenoCC.cpp b/libraries/networking/src/udt/TCPRenoCC.cpp index 7f83601354..85f120bd50 100644 --- a/libraries/networking/src/udt/TCPRenoCC.cpp +++ b/libraries/networking/src/udt/TCPRenoCC.cpp @@ -35,7 +35,7 @@ bool TCPRenoCC::isInSlowStart() { return _sendCongestionWindowSize < _sendSlowStartThreshold; } -int TCPRenoCC::slowStart(int numAcked) { +int TCPRenoCC::performSlowStart(int numAcked) { int congestionWindow = std::min(_sendCongestionWindowSize + numAcked, _sendSlowStartThreshold); numAcked -= congestionWindow - _sendCongestionWindowSize; _sendCongestionWindowSize = std::min(congestionWindow, _sendCongestionWindowClamp); @@ -63,7 +63,7 @@ void TCPRenoCC::performCongestionAvoidance(SequenceNumber ack, int numAcked) { // In "safe" area, increase. if (isInSlowStart()) { - numAcked = slowStart(numAcked); + numAcked = performSlowStart(numAcked); if (!numAcked) { return; } diff --git a/libraries/networking/src/udt/TCPRenoCC.h b/libraries/networking/src/udt/TCPRenoCC.h index 71c32b3a48..da54dc9a6b 100644 --- a/libraries/networking/src/udt/TCPRenoCC.h +++ b/libraries/networking/src/udt/TCPRenoCC.h @@ -25,7 +25,7 @@ protected: virtual void setInitialSendSequenceNumber(SequenceNumber seqNum) { _lastACK = seqNum - 1; } bool isInSlowStart(); - int slowStart(int numAcked); + int performSlowStart(int numAcked); int slowStartThreshold(); bool isCongestionWindowLimited(); diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 4e716c4d1c..b76fe91b43 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -55,7 +55,7 @@ void TCPVegasCC::onACK(SequenceNumber ackNum) { } } -void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack, int numACK) { +void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack, int numAcked) { static int VEGAS_MIN_RTT_FOR_CALC = 3; static uint64_t VEGAS_ALPHA_SEGMENTS = 2; @@ -67,7 +67,7 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack, int numACK) // pretty sure that at least one sample did not come from a delayed ACK. // If that is the case, we fallback to the Reno behaviour - TCPRenoCC::performCongestionAvoidance(ack, numACK); + TCPRenoCC::performCongestionAvoidance(ack, numAcked); } else { // There are enough RTT samples, use the Vegas algorithm to see if we should // increase or decrease the congestion window size, and by how much @@ -92,7 +92,7 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack, int numACK) inWindowReduction = true; } else if (isInSlowStart()) { // slow start - performSlowStart(ack); + performSlowStart(numAcked); } else { // figure out where the congestion window should be if (diff > VEGAS_BETA_SEGMENTS) { @@ -114,11 +114,11 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack, int numACK) static uint64_t VEGAS_CW_MIN_PACKETS = 2; _congestionWindowSize = std::min(_congestionWindowSize, VEGAS_CW_MIN_PACKETS); - if (!inWindowReduction && _congestionWindowSize > _slowStartThreshold) { + if (!inWindowReduction && _congestionWindowSize > _sendSlowStartThreshold) { // if we didn't just reduce the congestion window size and the // the congestion window is greater than the slow start threshold // we raise the slow start threshold half the distance to the congestion window - _slowStartThreshold = (_congestionWindowSize >> 1) + (_congestionWindowSize >> 2); + _sendSlowStartThreshold = (_congestionWindowSize >> 1) + (_congestionWindowSize >> 2); } _lastRTTMaxSeqNum = _sendCurrSeqNum; diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index 634262423a..112a53338a 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -34,10 +34,10 @@ public: virtual void onPacketSent(int packetSize, SequenceNumber seqNum) override; protected: - virtual void performCongestionAvoidance(SequenceNumber ack, int numACK) override; + virtual void performCongestionAvoidance(SequenceNumber ack, int numAcked) override; private: void adjustSlowStartThreshold() - { _slowStartThreshold = std::min(_slowStartThreshold, (uint32_t) _congestionWindowSize - 1); } + { _sendSlowStartThreshold = std::min(_sendSlowStartThreshold, (int) _congestionWindowSize - 1); } using TimeSizePair = std::pair; using PacketTimeList = std::map; From b232a44c8dbea7f8dffdabab33b0e9a596b38f4d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 11:31:22 -0700 Subject: [PATCH 07/65] don't send sync ACK when interval is == 1 --- libraries/networking/src/udt/Connection.cpp | 8 +++++--- libraries/networking/src/udt/TCPRenoCC.cpp | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 448290116f..01e5477429 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -223,9 +223,11 @@ void Connection::sync() { // reset the number of light ACKs or non SYN ACKs during this sync interval _lightACKsDuringSYN = 1; _acksDuringSYN = 1; - - // we send out a periodic ACK every rate control interval - sendACK(); + + if (_congestionControl->_ackInterval > 1) { + // we send out a periodic ACK every rate control interval + sendACK(); + } if (_lossList.getLength() > 0) { // check if we need to re-transmit a loss list diff --git a/libraries/networking/src/udt/TCPRenoCC.cpp b/libraries/networking/src/udt/TCPRenoCC.cpp index 85f120bd50..1fb8d36e19 100644 --- a/libraries/networking/src/udt/TCPRenoCC.cpp +++ b/libraries/networking/src/udt/TCPRenoCC.cpp @@ -19,7 +19,7 @@ void TCPRenoCC::init() { _packetSendPeriod = 0.0; _congestionWindowSize = 2.0; - setAckInterval(2); + setAckInterval(1); setRTO(1000000); } From 5e637c31defa15dc1587705dec1c609182d669fb Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 11:32:41 -0700 Subject: [PATCH 08/65] rename congestion window size var in TCPRenoCC --- libraries/networking/src/udt/TCPRenoCC.cpp | 16 ++++++++-------- libraries/networking/src/udt/TCPRenoCC.h | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/libraries/networking/src/udt/TCPRenoCC.cpp b/libraries/networking/src/udt/TCPRenoCC.cpp index 1fb8d36e19..116da2dab1 100644 --- a/libraries/networking/src/udt/TCPRenoCC.cpp +++ b/libraries/networking/src/udt/TCPRenoCC.cpp @@ -32,25 +32,25 @@ void TCPRenoCC::onACK(SequenceNumber ackNum) { } bool TCPRenoCC::isInSlowStart() { - return _sendCongestionWindowSize < _sendSlowStartThreshold; + return _congestionWindowSize < _sendSlowStartThreshold; } int TCPRenoCC::performSlowStart(int numAcked) { - int congestionWindow = std::min(_sendCongestionWindowSize + numAcked, _sendSlowStartThreshold); - numAcked -= congestionWindow - _sendCongestionWindowSize; - _sendCongestionWindowSize = std::min(congestionWindow, _sendCongestionWindowClamp); + int congestionWindow = std::min(_congestionWindowSize + numAcked, _sendSlowStartThreshold); + numAcked -= congestionWindow - _congestionWindowSize; + _congestionWindowSize = std::min(congestionWindow, _sendCongestionWindowClamp); return numAcked; } int TCPRenoCC::slowStartThreshold() { // Slow start threshold is half the congestion window (min 2) - return std::max(_sendCongestionWindowSize >> 1, 2); + return std::max(_congestionWindowSize >> 1, 2); } bool TCPRenoCC::isCongestionWindowLimited() { if (isInSlowStart()) { - return _sendCongestionWindowSize < 2 * _maxPacketsOut; + return _congestionWindowSize < 2 * _maxPacketsOut; } return _isCongestionWindowLimited; @@ -70,14 +70,14 @@ void TCPRenoCC::performCongestionAvoidance(SequenceNumber ack, int numAcked) { } // In dangerous area, increase slowly. - performCongestionAvoidanceAI(_sendCongestionWindowSize, numAcked); + performCongestionAvoidanceAI(_congestionWindowSize, numAcked); } void TCPRenoCC::performCongestionAvoidanceAI(int sendCongestionWindowSize, int numAcked) { // If credits accumulated at a higher w, apply them gently now. if (_sendCongestionWindowCount >= sendCongestionWindowSize) { _sendCongestionWindowCount = 0; - _sendCongestionWindowSize++; + _congestionWindowSize++; } _sendCongestionWindowCount += numAcked; diff --git a/libraries/networking/src/udt/TCPRenoCC.h b/libraries/networking/src/udt/TCPRenoCC.h index da54dc9a6b..28da21f316 100644 --- a/libraries/networking/src/udt/TCPRenoCC.h +++ b/libraries/networking/src/udt/TCPRenoCC.h @@ -35,7 +35,6 @@ protected: int _sendSlowStartThreshold; - int _sendCongestionWindowSize; int _sendCongestionWindowCount; int _sendCongestionWindowClamp = ~0; int _maxPacketsOut; From 8e05a2893a20f2395624728dbb47f554615622cd Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 11:35:01 -0700 Subject: [PATCH 09/65] change congestion window size to an int --- libraries/networking/src/udt/CongestionControl.h | 4 ++-- libraries/networking/src/udt/TCPVegasCC.cpp | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index c833b83664..202ab98e7e 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -58,11 +58,11 @@ protected: void setPacketSendPeriod(double newSendPeriod); // call this internally to ensure send period doesn't go past max bandwidth double _packetSendPeriod { 1.0 }; // Packet sending period, in microseconds - uint64_t _congestionWindowSize { 16 }; // Congestion window size, in packets + int _congestionWindowSize { 16 }; // Congestion window size, in packets int _bandwidth { 0 }; // estimated bandwidth, packets per second std::atomic _maxBandwidth { -1 }; // Maximum desired bandwidth, bits per second - uint64_t _maxCongestionWindowSize { 0 }; // maximum cwnd size, in packets + int _maxCongestionWindowSize { 0 }; // maximum cwnd size, in packets int _mss { 0 }; // Maximum Packet Size, including all packet headers SequenceNumber _sendCurrSeqNum; // current maximum seq num sent out diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index b76fe91b43..af3c266be9 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -58,9 +58,9 @@ void TCPVegasCC::onACK(SequenceNumber ackNum) { void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack, int numAcked) { static int VEGAS_MIN_RTT_FOR_CALC = 3; - static uint64_t VEGAS_ALPHA_SEGMENTS = 2; - static uint64_t VEGAS_BETA_SEGMENTS = 4; - static uint64_t VEGAS_GAMMA_SEGMENTS = 1; + static int VEGAS_ALPHA_SEGMENTS = 2; + static int VEGAS_BETA_SEGMENTS = 4; + static int VEGAS_GAMMA_SEGMENTS = 1; if (_numRTT < VEGAS_MIN_RTT_FOR_CALC) { // Vegas calculations are only done if there are enough RTT samples to be @@ -77,8 +77,8 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack, int numAcke // (though congestion may be noticed a bit later) int rtt = _currentMinRTT; - uint64_t expectedWindowSize = _congestionWindowSize * _baseRTT / rtt; - uint64_t diff = _congestionWindowSize * (rtt - _baseRTT) / _baseRTT; + int expectedWindowSize = _congestionWindowSize * _baseRTT / rtt; + int diff = _congestionWindowSize * (rtt - _baseRTT) / _baseRTT; bool inWindowReduction = false; @@ -111,7 +111,7 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack, int numAcke } // we never allow the congestion window to be smaller than two packets - static uint64_t VEGAS_CW_MIN_PACKETS = 2; + static int VEGAS_CW_MIN_PACKETS = 2; _congestionWindowSize = std::min(_congestionWindowSize, VEGAS_CW_MIN_PACKETS); if (!inWindowReduction && _congestionWindowSize > _sendSlowStartThreshold) { From 0954d5bf93cb9d1ed3bfe2b852bb200a447bb575 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 11:37:04 -0700 Subject: [PATCH 10/65] cleanup sent packet time hash after RTT calc --- libraries/networking/src/udt/TCPVegasCC.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index af3c266be9..dfaee140ec 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -48,6 +48,9 @@ void TCPVegasCC::onACK(SequenceNumber ackNum) { _lastACK = ackNum; + // remove this sent packet time from the hash + _sentPacketTimes.erase(it); + } else { Q_ASSERT_X(false, "TCPVegasCC::onACK", From eee617e39c02c26facd30ac7a323673cd549c7fa Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 26 Sep 2016 11:49:53 -0700 Subject: [PATCH 11/65] Remove congestion window size clamp --- libraries/networking/src/udt/TCPRenoCC.cpp | 4 +--- libraries/networking/src/udt/TCPRenoCC.h | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/libraries/networking/src/udt/TCPRenoCC.cpp b/libraries/networking/src/udt/TCPRenoCC.cpp index 116da2dab1..7c56e9bc93 100644 --- a/libraries/networking/src/udt/TCPRenoCC.cpp +++ b/libraries/networking/src/udt/TCPRenoCC.cpp @@ -38,7 +38,7 @@ bool TCPRenoCC::isInSlowStart() { int TCPRenoCC::performSlowStart(int numAcked) { int congestionWindow = std::min(_congestionWindowSize + numAcked, _sendSlowStartThreshold); numAcked -= congestionWindow - _congestionWindowSize; - _congestionWindowSize = std::min(congestionWindow, _sendCongestionWindowClamp); + _congestionWindowSize = congestionWindow; return numAcked; } @@ -87,7 +87,5 @@ void TCPRenoCC::performCongestionAvoidanceAI(int sendCongestionWindowSize, int n _sendCongestionWindowCount -= delta * sendCongestionWindowSize; _sendCongestionWindowCount += delta; } - _sendCongestionWindowCount = std::min(_sendCongestionWindowCount, _sendCongestionWindowClamp); - } diff --git a/libraries/networking/src/udt/TCPRenoCC.h b/libraries/networking/src/udt/TCPRenoCC.h index 28da21f316..f07f42ebbf 100644 --- a/libraries/networking/src/udt/TCPRenoCC.h +++ b/libraries/networking/src/udt/TCPRenoCC.h @@ -36,7 +36,6 @@ protected: int _sendSlowStartThreshold; int _sendCongestionWindowCount; - int _sendCongestionWindowClamp = ~0; int _maxPacketsOut; bool _isCongestionWindowLimited; From 75d4b156f15eecd4baa7ff07cc6ae0a833d5b9b6 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 26 Sep 2016 13:57:51 -0700 Subject: [PATCH 12/65] Added shouldNAK to CC class --- libraries/networking/src/udt/CongestionControl.h | 2 ++ libraries/networking/src/udt/Connection.cpp | 4 ++-- libraries/networking/src/udt/TCPRenoCC.h | 8 ++++---- libraries/networking/src/udt/TCPVegasCC.cpp | 10 +++++----- libraries/networking/src/udt/TCPVegasCC.h | 2 ++ 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 202ab98e7e..133602fa5e 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -44,6 +44,8 @@ public: virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) {} virtual void onTimeout() {} virtual void onPacketSent(int packetSize, SequenceNumber seqNum) {} + + virtual bool shouldNAK() { return true; } protected: void setAckInterval(int ackInterval) { _ackInterval = ackInterval; } void setRTO(int rto) { _userDefinedRTO = true; _rto = rto; } diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 01e5477429..1559e6211c 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -475,7 +475,7 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in _receiveWindow.onPacketArrival(); // If this is not the next sequence number, report loss - if (sequenceNumber > _lastReceivedSequenceNumber + 1) { + if (_congestionControl->shouldNAK() && sequenceNumber > _lastReceivedSequenceNumber + 1) { if (_lastReceivedSequenceNumber + 1 == sequenceNumber - 1) { _lossList.append(_lastReceivedSequenceNumber + 1); } else { @@ -505,7 +505,7 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in if (sequenceNumber > _lastReceivedSequenceNumber) { // Update largest recieved sequence number _lastReceivedSequenceNumber = sequenceNumber; - } else { + } else if (_congestionControl->shouldNAK()) { // Otherwise, it could be a resend, try and remove it from the loss list wasDuplicate = !_lossList.remove(sequenceNumber); } diff --git a/libraries/networking/src/udt/TCPRenoCC.h b/libraries/networking/src/udt/TCPRenoCC.h index f07f42ebbf..3e7872ebe9 100644 --- a/libraries/networking/src/udt/TCPRenoCC.h +++ b/libraries/networking/src/udt/TCPRenoCC.h @@ -34,10 +34,10 @@ protected: virtual void performCongestionAvoidance(SequenceNumber ack, int numAcked); - int _sendSlowStartThreshold; - int _sendCongestionWindowCount; - int _maxPacketsOut; - bool _isCongestionWindowLimited; + int _sendSlowStartThreshold { std::numeric_limits::max() }; + int _sendCongestionWindowCount { 0 }; + int _maxPacketsOut { 0 }; + bool _isCongestionWindowLimited { true }; SequenceNumber _lastACK; }; diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index dfaee140ec..d005475b38 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -59,11 +59,11 @@ void TCPVegasCC::onACK(SequenceNumber ackNum) { } void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack, int numAcked) { - static int VEGAS_MIN_RTT_FOR_CALC = 3; + static const int VEGAS_MIN_RTT_FOR_CALC = 3; - static int VEGAS_ALPHA_SEGMENTS = 2; - static int VEGAS_BETA_SEGMENTS = 4; - static int VEGAS_GAMMA_SEGMENTS = 1; + static const int VEGAS_ALPHA_SEGMENTS = 2; + static const int VEGAS_BETA_SEGMENTS = 4; + static const int VEGAS_GAMMA_SEGMENTS = 1; if (_numRTT < VEGAS_MIN_RTT_FOR_CALC) { // Vegas calculations are only done if there are enough RTT samples to be @@ -121,7 +121,7 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack, int numAcke // if we didn't just reduce the congestion window size and the // the congestion window is greater than the slow start threshold // we raise the slow start threshold half the distance to the congestion window - _sendSlowStartThreshold = (_congestionWindowSize >> 1) + (_congestionWindowSize >> 2); + _sendSlowStartThreshold = (_congestionWindowSize >> 1) + (_congestionWindowSize >> 2); } _lastRTTMaxSeqNum = _sendCurrSeqNum; diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index 112a53338a..61a7afd953 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -32,6 +32,8 @@ public: virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) override {}; virtual void onTimeout() override {}; virtual void onPacketSent(int packetSize, SequenceNumber seqNum) override; + + virtual bool shouldNAK() { return false; } protected: virtual void performCongestionAvoidance(SequenceNumber ack, int numAcked) override; From ecbba3d4cd7d00e3104cbb932c918a651a7ad785 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 13:30:34 -0700 Subject: [PATCH 13/65] send time point with packet sent signal --- libraries/networking/src/udt/CongestionControl.h | 3 ++- libraries/networking/src/udt/Connection.cpp | 9 +++++---- libraries/networking/src/udt/Connection.h | 4 ++-- libraries/networking/src/udt/SendQueue.cpp | 4 ++-- libraries/networking/src/udt/SendQueue.h | 4 ++-- libraries/networking/src/udt/TCPVegasCC.cpp | 16 ++++++++++++++-- libraries/networking/src/udt/TCPVegasCC.h | 3 ++- 7 files changed, 29 insertions(+), 14 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 133602fa5e..04c54e87c7 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -43,9 +43,10 @@ public: virtual void onACK(SequenceNumber ackNum) {} virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) {} virtual void onTimeout() {} - virtual void onPacketSent(int packetSize, SequenceNumber seqNum) {} virtual bool shouldNAK() { return true; } + + virtual void onPacketSent(int packetSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) {} protected: void setAckInterval(int ackInterval) { _ackInterval = ackInterval; } void setRTO(int rto) { _userDefinedRTO = true; _rto = rto; } diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 1559e6211c..890e989385 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -262,16 +262,17 @@ void Connection::sync() { } } -void Connection::recordSentPackets(int dataSize, int payloadSize, SequenceNumber seqNum) { +void Connection::recordSentPackets(int dataSize, int payloadSize, + SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) { _stats.recordSentPackets(payloadSize, dataSize); - _congestionControl->onPacketSent(dataSize, seqNum); + _congestionControl->onPacketSent(dataSize, seqNum, timePoint); } -void Connection::recordRetransmission(int packetSize, SequenceNumber seqNum) { +void Connection::recordRetransmission(int packetSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) { _stats.record(ConnectionStats::Stats::Retransmission); - _congestionControl->onPacketSent(packetSize, seqNum); + _congestionControl->onPacketSent(packetSize, seqNum, timePoint); } void Connection::sendACK(bool wasCausedBySyncTimeout) { diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 4f4ece5059..8df6d5c7a0 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -87,8 +87,8 @@ signals: void receiverHandshakeRequestComplete(const HifiSockAddr& sockAddr); private slots: - void recordSentPackets(int dataSize, int payloadSize, SequenceNumber seqNum); - void recordRetransmission(int packetSize, SequenceNumber sequenceNumber); + void recordSentPackets(int dataSize, int payloadSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint); + void recordRetransmission(int packetSize, SequenceNumber sequenceNumber, p_high_resolution_clock::time_point timePoint); void queueInactive(); void queueTimeout(); void queueShortCircuitLoss(quint32 sequenceNumber); diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 98c8450c30..5b82f8547a 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -256,7 +256,7 @@ bool SendQueue::sendNewPacketAndAddToSentList(std::unique_ptr newPacket, } Q_ASSERT_X(!newPacket, "SendQueue::sendNewPacketAndAddToSentList()", "Overriden packet in sent list"); - emit packetSent(packetSize, payloadSize, sequenceNumber); + emit packetSent(packetSize, payloadSize, sequenceNumber, p_high_resolution_clock::now()); if (bytesWritten < 0) { // this is a short-circuit loss - we failed to put this packet on the wire @@ -492,7 +492,7 @@ bool SendQueue::maybeResendPacket() { sentLocker.unlock(); } - emit packetRetransmitted(resendPacket.getWireSize(), it->first); + emit packetRetransmitted(resendPacket.getWireSize(), it->first, p_high_resolution_clock::now()); // Signal that we did resend a packet return true; diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index d874e4bbd7..8dd62bcad0 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -74,8 +74,8 @@ public slots: void handshakeACK(SequenceNumber initialSequenceNumber); signals: - void packetSent(int dataSize, int payloadSize, SequenceNumber seqNum); - void packetRetransmitted(int dataSize, SequenceNumber seqNum); + void packetSent(int dataSize, int payloadSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint); + void packetRetransmitted(int dataSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint); void queueInactive(); diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index d005475b38..277273c672 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -11,6 +11,7 @@ #include "TCPVegasCC.h" +#include #include using namespace udt; @@ -65,6 +66,10 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack, int numAcke static const int VEGAS_BETA_SEGMENTS = 4; static const int VEGAS_GAMMA_SEGMENTS = 1; + qDebug() << "============"; + qDebug() << "PCA" << _numRTT; + qDebug() << "CWS:" << _congestionWindowSize << "SS:" << _sendSlowStartThreshold; + if (_numRTT < VEGAS_MIN_RTT_FOR_CALC) { // Vegas calculations are only done if there are enough RTT samples to be // pretty sure that at least one sample did not come from a delayed ACK. @@ -83,6 +88,10 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack, int numAcke int expectedWindowSize = _congestionWindowSize * _baseRTT / rtt; int diff = _congestionWindowSize * (rtt - _baseRTT) / _baseRTT; + qDebug() << "BRTT:" << _baseRTT << "CRTT:" << _currentMinRTT; + qDebug() << "EWS" << expectedWindowSize; + qDebug() << "D:" << diff; + bool inWindowReduction = false; if (diff > VEGAS_GAMMA_SEGMENTS && isInSlowStart()) { @@ -130,10 +139,13 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack, int numAcke _currentMinRTT = std::numeric_limits::max(); _numRTT = 0; } + + qDebug() << "CW:" << _congestionWindowSize << "SS:" << _sendSlowStartThreshold; + qDebug() << "============"; } -void TCPVegasCC::onPacketSent(int packetSize, SequenceNumber seqNum) { - _sentPacketTimes[seqNum] = { p_high_resolution_clock::now(), packetSize }; +void TCPVegasCC::onPacketSent(int packetSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) { + _sentPacketTimes[seqNum] = { timePoint, packetSize }; } diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index 61a7afd953..a4d66b35ee 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -31,9 +31,10 @@ public: virtual void onACK(SequenceNumber ackNum) override; virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) override {}; virtual void onTimeout() override {}; - virtual void onPacketSent(int packetSize, SequenceNumber seqNum) override; virtual bool shouldNAK() { return false; } + + virtual void onPacketSent(int packetSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) override; protected: virtual void performCongestionAvoidance(SequenceNumber ack, int numAcked) override; From 0d2e83e0fca5720ce695c728944fa7a468720e54 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 14:58:38 -0700 Subject: [PATCH 14/65] cleanup according to Vegas paper, add fast retransmit --- .../networking/src/udt/CongestionControl.cpp | 10 +- .../networking/src/udt/CongestionControl.h | 4 +- libraries/networking/src/udt/Connection.cpp | 5 +- libraries/networking/src/udt/SendQueue.cpp | 10 ++ libraries/networking/src/udt/SendQueue.h | 1 + libraries/networking/src/udt/Socket.h | 1 - libraries/networking/src/udt/TCPRenoCC.cpp | 91 ---------- libraries/networking/src/udt/TCPRenoCC.h | 48 ----- libraries/networking/src/udt/TCPVegasCC.cpp | 169 ++++++++++-------- libraries/networking/src/udt/TCPVegasCC.h | 27 ++- 10 files changed, 134 insertions(+), 232 deletions(-) delete mode 100644 libraries/networking/src/udt/TCPRenoCC.cpp delete mode 100644 libraries/networking/src/udt/TCPRenoCC.h diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index e5940668e7..ee0425fc89 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -49,7 +49,7 @@ DefaultCC::DefaultCC() : setPacketSendPeriod(1.0); } -void DefaultCC::onACK(SequenceNumber ackNum) { +bool DefaultCC::onACK(SequenceNumber ackNum) { double increase = 0; // Note from UDT original code: @@ -61,7 +61,7 @@ void DefaultCC::onACK(SequenceNumber ackNum) { // we will only adjust once per sync interval so check that it has been at least that long now auto now = p_high_resolution_clock::now(); if (duration_cast(now - _lastRCTime).count() < synInterval()) { - return; + return false; } // our last rate increase time is now @@ -93,13 +93,13 @@ void DefaultCC::onACK(SequenceNumber ackNum) { // during slow start we perform no rate increases if (_slowStart) { - return; + return false; } // if loss has happened since the last rate increase we do not perform another increase if (_loss) { _loss = false; - return; + return false; } double capacitySpeedDelta = (_bandwidth - USECS_PER_SECOND / _packetSendPeriod); @@ -132,6 +132,8 @@ void DefaultCC::onACK(SequenceNumber ackNum) { } setPacketSendPeriod((_packetSendPeriod * synInterval()) / (_packetSendPeriod * increase + synInterval())); + + return false; } void DefaultCC::onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) { diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 04c54e87c7..491956bb2b 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -40,7 +40,7 @@ public: void setMaxBandwidth(int maxBandwidth); virtual void init() {} - virtual void onACK(SequenceNumber ackNum) {} + virtual bool onACK(SequenceNumber ackNum) { return false; } virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) {} virtual void onTimeout() {} @@ -106,7 +106,7 @@ public: DefaultCC(); public: - virtual void onACK(SequenceNumber ackNum) override; + virtual bool onACK(SequenceNumber ackNum) override; virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) override; virtual void onTimeout() override; diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 890e989385..86e7876ef0 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -686,7 +686,10 @@ void Connection::processACK(ControlPacketPointer controlPacket) { // give this ACK to the congestion control and update the send queue parameters updateCongestionControlAndSendQueue([this, ack](){ - _congestionControl->onACK(ack); + if (_congestionControl->onACK(ack)) { + // the congestion control has told us it needs a fast re-transmit of ack + 1, add that now + _sendQueue->fastRetransmit(ack + 1); + } }); _stats.record(ConnectionStats::Stats::ProcessedACK); diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 5b82f8547a..f98567656e 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -180,6 +180,16 @@ void SendQueue::nak(SequenceNumber start, SequenceNumber end) { _emptyCondition.notify_one(); } +void SendQueue::fastRetransmit(udt::SequenceNumber ack) { + { + std::lock_guard nakLocker(_naksLock); + _naks.insert(ack, ack); + } + + // call notify_one on the condition_variable_any in case the send thread is sleeping waiting for losses to re-send + _emptyCondition.notify_one(); +} + void SendQueue::overrideNAKListFromPacket(ControlPacket& packet) { // this is a response from the client, re-set our timeout expiry _lastReceiverResponse = QDateTime::currentMSecsSinceEpoch(); diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 8dd62bcad0..c7110d4044 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -70,6 +70,7 @@ public slots: void ack(SequenceNumber ack); void nak(SequenceNumber start, SequenceNumber end); + void fastRetransmit(SequenceNumber ack); void overrideNAKListFromPacket(ControlPacket& packet); void handshakeACK(SequenceNumber initialSequenceNumber); diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index e57edfdf97..cced0c53df 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -24,7 +24,6 @@ #include "../HifiSockAddr.h" #include "TCPVegasCC.h" #include "Connection.h" -#include "TCPRenoCC.h" //#define UDT_CONNECTION_DEBUG diff --git a/libraries/networking/src/udt/TCPRenoCC.cpp b/libraries/networking/src/udt/TCPRenoCC.cpp deleted file mode 100644 index 7c56e9bc93..0000000000 --- a/libraries/networking/src/udt/TCPRenoCC.cpp +++ /dev/null @@ -1,91 +0,0 @@ -// -// TCPRenoCC.cpp -// libraries/networking/src/udt -// -// Created by Clement on 9/20/16. -// Copyright 2016 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "TCPRenoCC.h" - -#include - -using namespace udt; - -void TCPRenoCC::init() { - _packetSendPeriod = 0.0; - _congestionWindowSize = 2.0; - - setAckInterval(1); - setRTO(1000000); -} - -void TCPRenoCC::onACK(SequenceNumber ackNum) { - int numAcked = seqlen(_lastACK, ackNum); - - performCongestionAvoidance(ackNum, numAcked); - - _lastACK = ackNum; -} - -bool TCPRenoCC::isInSlowStart() { - return _congestionWindowSize < _sendSlowStartThreshold; -} - -int TCPRenoCC::performSlowStart(int numAcked) { - int congestionWindow = std::min(_congestionWindowSize + numAcked, _sendSlowStartThreshold); - numAcked -= congestionWindow - _congestionWindowSize; - _congestionWindowSize = congestionWindow; - - return numAcked; -} - -int TCPRenoCC::slowStartThreshold() { - // Slow start threshold is half the congestion window (min 2) - return std::max(_congestionWindowSize >> 1, 2); -} - -bool TCPRenoCC::isCongestionWindowLimited() { - if (isInSlowStart()) { - return _congestionWindowSize < 2 * _maxPacketsOut; - } - - return _isCongestionWindowLimited; -} - -void TCPRenoCC::performCongestionAvoidance(SequenceNumber ack, int numAcked) { - if (!isCongestionWindowLimited()) { - return; - } - - // In "safe" area, increase. - if (isInSlowStart()) { - numAcked = performSlowStart(numAcked); - if (!numAcked) { - return; - } - } - - // In dangerous area, increase slowly. - performCongestionAvoidanceAI(_congestionWindowSize, numAcked); -} - -void TCPRenoCC::performCongestionAvoidanceAI(int sendCongestionWindowSize, int numAcked) { - // If credits accumulated at a higher w, apply them gently now. - if (_sendCongestionWindowCount >= sendCongestionWindowSize) { - _sendCongestionWindowCount = 0; - _congestionWindowSize++; - } - - _sendCongestionWindowCount += numAcked; - if (_sendCongestionWindowCount >= sendCongestionWindowSize) { - int delta = _sendCongestionWindowCount / sendCongestionWindowSize; - - _sendCongestionWindowCount -= delta * sendCongestionWindowSize; - _sendCongestionWindowCount += delta; - } -} - diff --git a/libraries/networking/src/udt/TCPRenoCC.h b/libraries/networking/src/udt/TCPRenoCC.h deleted file mode 100644 index 3e7872ebe9..0000000000 --- a/libraries/networking/src/udt/TCPRenoCC.h +++ /dev/null @@ -1,48 +0,0 @@ -// -// TCPRenoCC.h -// libraries/networking/src/udt -// -// Created by Clement on 9/20/16. -// Copyright 2016 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_TCPRenoCC_h -#define hifi_TCPRenoCC_h - -#include "CongestionControl.h" - -namespace udt { - -class TCPRenoCC : public CongestionControl { -public: - virtual void init(); - virtual void onACK(SequenceNumber ackNum); - -protected: - virtual void setInitialSendSequenceNumber(SequenceNumber seqNum) { _lastACK = seqNum - 1; } - - bool isInSlowStart(); - int performSlowStart(int numAcked); - int slowStartThreshold(); - - bool isCongestionWindowLimited(); - void performCongestionAvoidanceAI(int sendCongestionWindowSize, int numAcked); - - virtual void performCongestionAvoidance(SequenceNumber ack, int numAcked); - - - int _sendSlowStartThreshold { std::numeric_limits::max() }; - int _sendCongestionWindowCount { 0 }; - int _maxPacketsOut { 0 }; - bool _isCongestionWindowLimited { true }; - - SequenceNumber _lastACK; -}; - -} - - -#endif // hifi_TCPRenoCC_h diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 277273c672..1fae58aa0a 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -17,8 +17,15 @@ using namespace udt; using namespace std::chrono; -void TCPVegasCC::onACK(SequenceNumber ackNum) { - auto it = _sentPacketTimes.find(ackNum); +TCPVegasCC::TCPVegasCC() { + _packetSendPeriod = 0.0; + _congestionWindowSize = 2; + + setAckInterval(1); // TCP sends an ACK for every packet received +} + +bool TCPVegasCC::onACK(SequenceNumber ack) { + auto it = _sentPacketTimes.find(ack); if (it != _sentPacketTimes.end()) { @@ -28,27 +35,29 @@ void TCPVegasCC::onACK(SequenceNumber ackNum) { if (lastRTT < 0) { Q_ASSERT_X(false, "TCPVegasCC::onACK", "calculated an RTT that is not > 0"); - return; + return false; } else if (lastRTT == 0) { // we do not allow a 0 microsecond RTT lastRTT = 1; } + static const int RTT_ESTIMATION_ALPHA_NUMERATOR = 8; + static const int RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR = 4; + + _ewmaRTT = (_ewmaRTT * (RTT_ESTIMATION_ALPHA_NUMERATOR - 1) + lastRTT) / RTT_ESTIMATION_ALPHA_NUMERATOR; + _rttVariance = (_rttVariance * (RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR - 1) + + abs(lastRTT - _ewmaRTT)) / RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR; + // keep track of the lowest RTT during connection _baseRTT = std::min(_baseRTT, lastRTT); // find the min RTT during the last RTT _currentMinRTT = std::min(_currentMinRTT, lastRTT); - ++_numRTT; - - if (ackNum > _lastRTTMaxSeqNum) { - int numACK = seqlen(_lastACK, ackNum); - performCongestionAvoidance(ackNum, numACK); + if (ack >= _lastAdjustmentNextSendAck) { + performCongestionAvoidance(ack); } - _lastACK = ackNum; - // remove this sent packet time from the hash _sentPacketTimes.erase(it); @@ -57,90 +66,98 @@ void TCPVegasCC::onACK(SequenceNumber ackNum) { "TCPVegasCC::onACK", "called with a sequence number that has not been sent"); } + + + // we check if we need a fast re-transmit if this is a duplicate ACK or if this is the first or second ACK + // after a previous fast re-transmit + ++_numACKSinceFastRetransmit; + + if (ack == _lastAck || _numACKSinceFastRetransmit < 3) { + // we may need to re-send ackNum + 1 if it has been more than our estimated timeout since it was sent + + auto it = _sentPacketTimes.find(ack + 1); + if (it != _sentPacketTimes.end()) { + auto estimatedTimeout = _ewmaRTT + _rttVariance * 4; + + auto now = p_high_resolution_clock::now(); + auto sinceSend = duration_cast(now - it->second.first).count(); + + if (sinceSend >= estimatedTimeout) { + // we've detected we need a fast re-transmit, send that back to the connection + _numACKSinceFastRetransmit = 0; + return true; + } + } + } + + // ACK processed, no fast re-transmit required + return false; } -void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack, int numAcked) { - static const int VEGAS_MIN_RTT_FOR_CALC = 3; - - static const int VEGAS_ALPHA_SEGMENTS = 2; - static const int VEGAS_BETA_SEGMENTS = 4; - static const int VEGAS_GAMMA_SEGMENTS = 1; +void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { + static int VEGAS_ALPHA_SEGMENTS = 2; + static int VEGAS_BETA_SEGMENTS = 4; + static int VEGAS_GAMMA_SEGMENTS = 1; qDebug() << "============"; qDebug() << "PCA" << _numRTT; - qDebug() << "CWS:" << _congestionWindowSize << "SS:" << _sendSlowStartThreshold; + qDebug() << "CWS:" << _congestionWindowSize << "SS:" << _slowStart; - if (_numRTT < VEGAS_MIN_RTT_FOR_CALC) { - // Vegas calculations are only done if there are enough RTT samples to be - // pretty sure that at least one sample did not come from a delayed ACK. - // If that is the case, we fallback to the Reno behaviour + // Use the Vegas algorithm to see if we should + // increase or decrease the congestion window size, and by how much - TCPRenoCC::performCongestionAvoidance(ack, numAcked); - } else { - // There are enough RTT samples, use the Vegas algorithm to see if we should - // increase or decrease the congestion window size, and by how much + // Grab the minimum RTT seen during the last RTT + // Taking the min avoids the effects of delayed ACKs + // (though congestion may be noticed a bit later) + int rtt = _currentMinRTT; - // Grab the minimum RTT seen during the last RTT - // Taking the min avoids the effects of delayed ACKs - // (though congestion may be noticed a bit later) - int rtt = _currentMinRTT; + int expectedWindowSize = _congestionWindowSize * _baseRTT / rtt; + int windowSizeDiff = _congestionWindowSize * (rtt - _baseRTT) / _baseRTT; - int expectedWindowSize = _congestionWindowSize * _baseRTT / rtt; - int diff = _congestionWindowSize * (rtt - _baseRTT) / _baseRTT; + qDebug() << "BRTT:" << _baseRTT << "CRTT:" << _currentMinRTT; + qDebug() << "D:" << windowSizeDiff; - qDebug() << "BRTT:" << _baseRTT << "CRTT:" << _currentMinRTT; - qDebug() << "EWS" << expectedWindowSize; - qDebug() << "D:" << diff; - bool inWindowReduction = false; + if (_slowStart) { + if (windowSizeDiff > VEGAS_GAMMA_SEGMENTS) { + // we're going too fast - this breaks us out of slow start and we switch to linear increase/decrease + _slowStart = false; - if (diff > VEGAS_GAMMA_SEGMENTS && isInSlowStart()) { - // we're going too fast, slow down and switch to congestion avoidance - - // the congestion window should match the actual rate + // drop the congestion window size to the expected size, if smaller _congestionWindowSize = std::min(_congestionWindowSize, expectedWindowSize + 1); - adjustSlowStartThreshold(); + + } else if (++_slowStartOddAdjust & 1) { + // we're in slow start and not going too fast + // this means that once every second RTT we perform exponential congestion window growth + _congestionWindowSize *= 2; + } + } else { + // this is the normal linear increase/decrease of the Vegas algorithm + // to figure out where the congestion window should be + if (windowSizeDiff > VEGAS_BETA_SEGMENTS) { + // the old congestion window was too fast (difference > beta) + // so reduce it to slow down + --_congestionWindowSize; - inWindowReduction = true; - } else if (isInSlowStart()) { - // slow start - performSlowStart(numAcked); + } else if (windowSizeDiff < VEGAS_ALPHA_SEGMENTS) { + // there aren't enough packets on the wire, add more to the congestion window + ++_congestionWindowSize; } else { - // figure out where the congestion window should be - if (diff > VEGAS_BETA_SEGMENTS) { - // the old congestion window was too fast (difference > beta) - // so reduce it to slow down - --_congestionWindowSize; - adjustSlowStartThreshold(); - - inWindowReduction = true; - } else if (diff < VEGAS_ALPHA_SEGMENTS) { - // there aren't enough packets on the wire, add more to the congestion window - ++_congestionWindowSize; - } else { - // sending rate seems good, no congestion window adjustment - } + // sending rate seems good, no congestion window adjustment } - - // we never allow the congestion window to be smaller than two packets - static int VEGAS_CW_MIN_PACKETS = 2; - _congestionWindowSize = std::min(_congestionWindowSize, VEGAS_CW_MIN_PACKETS); - - if (!inWindowReduction && _congestionWindowSize > _sendSlowStartThreshold) { - // if we didn't just reduce the congestion window size and the - // the congestion window is greater than the slow start threshold - // we raise the slow start threshold half the distance to the congestion window - _sendSlowStartThreshold = (_congestionWindowSize >> 1) + (_congestionWindowSize >> 2); - } - - _lastRTTMaxSeqNum = _sendCurrSeqNum; - - // reset our state for the next RTT - _currentMinRTT = std::numeric_limits::max(); - _numRTT = 0; } - qDebug() << "CW:" << _congestionWindowSize << "SS:" << _sendSlowStartThreshold; + // we never allow the congestion window to be smaller than two packets + static int VEGAS_CW_MIN_PACKETS = 2; + _congestionWindowSize = std::min(_congestionWindowSize, VEGAS_CW_MIN_PACKETS); + + // mark this as the last adjustment time + _lastAdjustmentNextSendAck = _sendCurrSeqNum; + + // reset our state for the next RTT + _currentMinRTT = std::numeric_limits::max(); + + qDebug() << "CW:" << _congestionWindowSize << "SS:" << _slowStart; qDebug() << "============"; } diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index a4d66b35ee..21a8ecafdd 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -18,39 +18,48 @@ #include "CongestionControl.h" #include "Constants.h" -#include "TCPRenoCC.h" namespace udt { -class TCPVegasCC : public TCPRenoCC { +class TCPVegasCC : public CongestionControl { public: - TCPVegasCC() {}; + TCPVegasCC(); public: - virtual void onACK(SequenceNumber ackNum) override; + virtual bool onACK(SequenceNumber ackNum) override; virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) override {}; virtual void onTimeout() override {}; - virtual bool shouldNAK() { return false; } + virtual bool shouldNAK() override { return false; } virtual void onPacketSent(int packetSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) override; protected: - virtual void performCongestionAvoidance(SequenceNumber ack, int numAcked) override; + virtual void performCongestionAvoidance(SequenceNumber ack); + virtual void setInitialSendSequenceNumber(SequenceNumber seqNum) override { _lastAck = seqNum - 1; } private: - void adjustSlowStartThreshold() - { _sendSlowStartThreshold = std::min(_sendSlowStartThreshold, (int) _congestionWindowSize - 1); } using TimeSizePair = std::pair; using PacketTimeList = std::map; PacketTimeList _sentPacketTimes; // Map of sequence numbers to sent time + SequenceNumber _lastAdjustmentNextSendAck; // Sequence number of next packet to be sent at time of last adjustment + + bool _slowStart { false }; // Marker for slow start phase + + SequenceNumber _lastAck; // Sequence number of last packet that was ACKed + + int _numACKSinceFastRetransmit { 0 }; // Number of ACKs received since last fast re-transmit + int _currentMinRTT { 0x7FFFFFFF }; // Current RTT, in microseconds int _baseRTT { 0x7FFFFFFF }; // Lowest RTT during connection, in microseconds int _numRTT { 0 }; // Number of RTT collected during last RTT + int _ewmaRTT { 0 }; // Exponential weighted moving average RTT + int _rttVariance { 0 }; // Variance in collected RTT values + + int _slowStartOddAdjust { 0 }; // Marker for every window adjustment every other RTT in slow-start - SequenceNumber _lastRTTMaxSeqNum; // Highest sent sequence number at time of last congestion window adjustment }; } From 7cc1f3261e318f6d8ecf4d8a465f0893a34f712b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 15:02:27 -0700 Subject: [PATCH 15/65] use last adjustment time to adjust every RTT --- libraries/networking/src/udt/TCPVegasCC.cpp | 5 +++-- libraries/networking/src/udt/TCPVegasCC.h | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 1fae58aa0a..c06bcf67ca 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -54,7 +54,8 @@ bool TCPVegasCC::onACK(SequenceNumber ack) { // find the min RTT during the last RTT _currentMinRTT = std::min(_currentMinRTT, lastRTT); - if (ack >= _lastAdjustmentNextSendAck) { + auto sinceLastAdjustment = duration_cast(now - _lastAdjustmentTime).count(); + if (sinceLastAdjustment >= _ewmaRTT) { performCongestionAvoidance(ack); } @@ -152,7 +153,7 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { _congestionWindowSize = std::min(_congestionWindowSize, VEGAS_CW_MIN_PACKETS); // mark this as the last adjustment time - _lastAdjustmentNextSendAck = _sendCurrSeqNum; + _lastAdjustmentTime = p_high_resolution_clock::now(); // reset our state for the next RTT _currentMinRTT = std::numeric_limits::max(); diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index 21a8ecafdd..a1145ddd3d 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -44,7 +44,7 @@ private: using PacketTimeList = std::map; PacketTimeList _sentPacketTimes; // Map of sequence numbers to sent time - SequenceNumber _lastAdjustmentNextSendAck; // Sequence number of next packet to be sent at time of last adjustment + p_high_resolution_clock::time_point _lastAdjustmentTime; // Time of last congestion control adjustment bool _slowStart { false }; // Marker for slow start phase From 96ef1fa483c015e92d7850565bfa32d5a52cc6ee Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 15:06:55 -0700 Subject: [PATCH 16/65] add FRT debug --- libraries/networking/src/udt/TCPVegasCC.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index c06bcf67ca..b9654aa0f0 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -75,6 +75,7 @@ bool TCPVegasCC::onACK(SequenceNumber ack) { if (ack == _lastAck || _numACKSinceFastRetransmit < 3) { // we may need to re-send ackNum + 1 if it has been more than our estimated timeout since it was sent + qDebug() << "FRT:" << (uint32_t) ack << (uint32_t) _lastAck << _numACKSinceFastRetransmit; auto it = _sentPacketTimes.find(ack + 1); if (it != _sentPacketTimes.end()) { From c794c4af1a3aa7a928978d068bc5e4777283da16 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 15:09:01 -0700 Subject: [PATCH 17/65] adjust EWMA RTT start values in TCPVegasCC --- libraries/networking/src/udt/TCPVegasCC.cpp | 8 +++++--- libraries/networking/src/udt/TCPVegasCC.h | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index b9654aa0f0..39680d1059 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -113,18 +113,20 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { // (though congestion may be noticed a bit later) int rtt = _currentMinRTT; - int expectedWindowSize = _congestionWindowSize * _baseRTT / rtt; int windowSizeDiff = _congestionWindowSize * (rtt - _baseRTT) / _baseRTT; - qDebug() << "BRTT:" << _baseRTT << "CRTT:" << _currentMinRTT; + qDebug() << "BRTT:" << _baseRTT << "CRTT:" << _currentMinRTT << "ERTT:" << _ewmaRTT; qDebug() << "D:" << windowSizeDiff; - if (_slowStart) { if (windowSizeDiff > VEGAS_GAMMA_SEGMENTS) { // we're going too fast - this breaks us out of slow start and we switch to linear increase/decrease _slowStart = false; + int expectedWindowSize = _congestionWindowSize * _baseRTT / rtt; + + qDebug() << "EWS:" << expectedWindowSize; + // drop the congestion window size to the expected size, if smaller _congestionWindowSize = std::min(_congestionWindowSize, expectedWindowSize + 1); diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index a1145ddd3d..d7d884fc4b 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -55,8 +55,8 @@ private: int _currentMinRTT { 0x7FFFFFFF }; // Current RTT, in microseconds int _baseRTT { 0x7FFFFFFF }; // Lowest RTT during connection, in microseconds int _numRTT { 0 }; // Number of RTT collected during last RTT - int _ewmaRTT { 0 }; // Exponential weighted moving average RTT - int _rttVariance { 0 }; // Variance in collected RTT values + int _ewmaRTT { 100000 }; // Exponential weighted moving average RTT + int _rttVariance { 50000 }; // Variance in collected RTT values int _slowStartOddAdjust { 0 }; // Marker for every window adjustment every other RTT in slow-start From 3ecfd6af673113f50407523123b16ea858f79b41 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 15:15:51 -0700 Subject: [PATCH 18/65] added debug for fast re-transmit --- libraries/networking/src/udt/SendQueue.cpp | 2 ++ libraries/networking/src/udt/TCPVegasCC.cpp | 1 + 2 files changed, 3 insertions(+) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index f98567656e..52d45a9d3c 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -186,6 +186,8 @@ void SendQueue::fastRetransmit(udt::SequenceNumber ack) { _naks.insert(ack, ack); } + qDebug() << "Added" << (uint32_t) ack << "for fast retransmit"; + // call notify_one on the condition_variable_any in case the send thread is sleeping waiting for losses to re-send _emptyCondition.notify_one(); } diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 39680d1059..1736c737e1 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -85,6 +85,7 @@ bool TCPVegasCC::onACK(SequenceNumber ack) { auto sinceSend = duration_cast(now - it->second.first).count(); if (sinceSend >= estimatedTimeout) { + qDebug() << "FRT needed:" << sinceSend << estimatedTimeout; // we've detected we need a fast re-transmit, send that back to the connection _numACKSinceFastRetransmit = 0; return true; From 7731b407bd9825ce578494319f93dff53a059ad2 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 15:21:52 -0700 Subject: [PATCH 19/65] only attempt SQ sleep with > 0 PSP --- libraries/networking/src/udt/SendQueue.cpp | 80 +++++++++++----------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 52d45a9d3c..c152a212c9 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -340,57 +340,59 @@ void SendQueue::run() { return; } - // push the next packet timestamp forwards by the current packet send period - auto nextPacketDelta = (newPacketCount == 2 ? 2 : 1) * _packetSendPeriod; - nextPacketTimestamp += std::chrono::microseconds(nextPacketDelta); + if (_packetSendPeriod > 0) { + // push the next packet timestamp forwards by the current packet send period + auto nextPacketDelta = (newPacketCount == 2 ? 2 : 1) * _packetSendPeriod; + nextPacketTimestamp += std::chrono::microseconds(nextPacketDelta); - // sleep as long as we need for next packet send, if we can - auto now = p_high_resolution_clock::now(); + // sleep as long as we need for next packet send, if we can + auto now = p_high_resolution_clock::now(); - auto timeToSleep = duration_cast(nextPacketTimestamp - now); + auto timeToSleep = duration_cast(nextPacketTimestamp - now); - // we use nextPacketTimestamp so that we don't fall behind, not to force long sleeps - // we'll never allow nextPacketTimestamp to force us to sleep for more than nextPacketDelta - // so cap it to that value - if (timeToSleep > std::chrono::microseconds(nextPacketDelta)) { - // reset the nextPacketTimestamp so that it is correct next time we come around - nextPacketTimestamp = now + std::chrono::microseconds(nextPacketDelta); + // we use nextPacketTimestamp so that we don't fall behind, not to force long sleeps + // we'll never allow nextPacketTimestamp to force us to sleep for more than nextPacketDelta + // so cap it to that value + if (timeToSleep > std::chrono::microseconds(nextPacketDelta)) { + // reset the nextPacketTimestamp so that it is correct next time we come around + nextPacketTimestamp = now + std::chrono::microseconds(nextPacketDelta); - timeToSleep = std::chrono::microseconds(nextPacketDelta); - } + timeToSleep = std::chrono::microseconds(nextPacketDelta); + } - // we're seeing SendQueues sleep for a long period of time here, - // which can lock the NodeList if it's attempting to clear connections - // for now we guard this by capping the time this thread and sleep for + // we're seeing SendQueues sleep for a long period of time here, + // which can lock the NodeList if it's attempting to clear connections + // for now we guard this by capping the time this thread and sleep for - const microseconds MAX_SEND_QUEUE_SLEEP_USECS { 2000000 }; - if (timeToSleep > MAX_SEND_QUEUE_SLEEP_USECS) { - qWarning() << "udt::SendQueue wanted to sleep for" << timeToSleep.count() << "microseconds"; - qWarning() << "Capping sleep to" << MAX_SEND_QUEUE_SLEEP_USECS.count(); - qWarning() << "PSP:" << _packetSendPeriod << "NPD:" << nextPacketDelta + const microseconds MAX_SEND_QUEUE_SLEEP_USECS { 2000000 }; + if (timeToSleep > MAX_SEND_QUEUE_SLEEP_USECS) { + qWarning() << "udt::SendQueue wanted to sleep for" << timeToSleep.count() << "microseconds"; + qWarning() << "Capping sleep to" << MAX_SEND_QUEUE_SLEEP_USECS.count(); + qWarning() << "PSP:" << _packetSendPeriod << "NPD:" << nextPacketDelta << "NPT:" << nextPacketTimestamp.time_since_epoch().count() << "NOW:" << now.time_since_epoch().count(); - // alright, we're in a weird state - // we want to know why this is happening so we can implement a better fix than this guard - // send some details up to the API (if the user allows us) that indicate how we could such a large timeToSleep - static const QString SEND_QUEUE_LONG_SLEEP_ACTION = "sendqueue-sleep"; + // alright, we're in a weird state + // we want to know why this is happening so we can implement a better fix than this guard + // send some details up to the API (if the user allows us) that indicate how we could such a large timeToSleep + static const QString SEND_QUEUE_LONG_SLEEP_ACTION = "sendqueue-sleep"; - // setup a json object with the details we want - QJsonObject longSleepObject; - longSleepObject["timeToSleep"] = qint64(timeToSleep.count()); - longSleepObject["packetSendPeriod"] = _packetSendPeriod.load(); - longSleepObject["nextPacketDelta"] = nextPacketDelta; - longSleepObject["nextPacketTimestamp"] = qint64(nextPacketTimestamp.time_since_epoch().count()); - longSleepObject["then"] = qint64(now.time_since_epoch().count()); + // setup a json object with the details we want + QJsonObject longSleepObject; + longSleepObject["timeToSleep"] = qint64(timeToSleep.count()); + longSleepObject["packetSendPeriod"] = _packetSendPeriod.load(); + longSleepObject["nextPacketDelta"] = nextPacketDelta; + longSleepObject["nextPacketTimestamp"] = qint64(nextPacketTimestamp.time_since_epoch().count()); + longSleepObject["then"] = qint64(now.time_since_epoch().count()); - // hopefully send this event using the user activity logger - UserActivityLogger::getInstance().logAction(SEND_QUEUE_LONG_SLEEP_ACTION, longSleepObject); - - timeToSleep = MAX_SEND_QUEUE_SLEEP_USECS; + // hopefully send this event using the user activity logger + UserActivityLogger::getInstance().logAction(SEND_QUEUE_LONG_SLEEP_ACTION, longSleepObject); + + timeToSleep = MAX_SEND_QUEUE_SLEEP_USECS; + } + + std::this_thread::sleep_for(timeToSleep); } - - std::this_thread::sleep_for(timeToSleep); } } From f143e9bf3f4fc771779b478d0a94f7924706362a Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 26 Sep 2016 15:33:46 -0700 Subject: [PATCH 20/65] Register SequenceNumber and time_point --- libraries/networking/src/udt/SequenceNumber.cpp | 8 ++++++++ libraries/shared/src/PortableHighResolutionClock.h | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/libraries/networking/src/udt/SequenceNumber.cpp b/libraries/networking/src/udt/SequenceNumber.cpp index 3e94c35d44..a518beb172 100644 --- a/libraries/networking/src/udt/SequenceNumber.cpp +++ b/libraries/networking/src/udt/SequenceNumber.cpp @@ -11,6 +11,14 @@ #include "SequenceNumber.h" +#include + +using namespace udt; + +Q_DECLARE_METATYPE(SequenceNumber); + +static const int sequenceNumberMetaTypeID = qRegisterMetaType(); + int udt::seqlen(const SequenceNumber& seq1, const SequenceNumber& seq2) { return (seq1._value <= seq2._value) ? (seq2._value - seq1._value + 1) : (seq2._value - seq1._value + SequenceNumber::MAX + 2); diff --git a/libraries/shared/src/PortableHighResolutionClock.h b/libraries/shared/src/PortableHighResolutionClock.h index ec1b068e66..383c6cdf3e 100644 --- a/libraries/shared/src/PortableHighResolutionClock.h +++ b/libraries/shared/src/PortableHighResolutionClock.h @@ -21,6 +21,8 @@ #include +#include + #if defined(_MSC_VER) && _MSC_VER < 1900 #define WIN32_LEAN_AND_MEAN @@ -47,5 +49,8 @@ using p_high_resolution_clock = std::chrono::high_resolution_clock; #endif +Q_DECLARE_METATYPE(p_high_resolution_clock::time_point); + +static const int timePointMetaTypeID = qRegisterMetaType(); #endif // hifi_PortableHighResolutionClock_h From f0dcc54a18653e01c5f78320087863700acdaeea Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 26 Sep 2016 15:38:49 -0700 Subject: [PATCH 21/65] Start in slow start --- libraries/networking/src/udt/TCPVegasCC.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index d7d884fc4b..995e8c56c1 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -46,7 +46,7 @@ private: p_high_resolution_clock::time_point _lastAdjustmentTime; // Time of last congestion control adjustment - bool _slowStart { false }; // Marker for slow start phase + bool _slowStart { true }; // Marker for slow start phase SequenceNumber _lastAck; // Sequence number of last packet that was ACKed From 995f383edbac9e5a8aa1cccf6ba99465614ae536 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 15:40:11 -0700 Subject: [PATCH 22/65] default EWMA RTT and variance to first value --- libraries/networking/src/udt/TCPVegasCC.cpp | 17 ++++++++++++----- libraries/networking/src/udt/TCPVegasCC.h | 4 ++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 1736c737e1..288fcad430 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -41,12 +41,19 @@ bool TCPVegasCC::onACK(SequenceNumber ack) { lastRTT = 1; } - static const int RTT_ESTIMATION_ALPHA_NUMERATOR = 8; - static const int RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR = 4; + if (_ewmaRTT == -1) { + _ewmaRTT = lastRTT; + _rttVariance = lastRTT / 2; + } else { + static const int RTT_ESTIMATION_ALPHA_NUMERATOR = 8; + static const int RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR = 4; + + _ewmaRTT = (_ewmaRTT * (RTT_ESTIMATION_ALPHA_NUMERATOR - 1) + lastRTT) / RTT_ESTIMATION_ALPHA_NUMERATOR; + _rttVariance = (_rttVariance * (RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR - 1) + + abs(lastRTT - _ewmaRTT)) / RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR; + } + - _ewmaRTT = (_ewmaRTT * (RTT_ESTIMATION_ALPHA_NUMERATOR - 1) + lastRTT) / RTT_ESTIMATION_ALPHA_NUMERATOR; - _rttVariance = (_rttVariance * (RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR - 1) - + abs(lastRTT - _ewmaRTT)) / RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR; // keep track of the lowest RTT during connection _baseRTT = std::min(_baseRTT, lastRTT); diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index 995e8c56c1..f579654dcf 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -55,8 +55,8 @@ private: int _currentMinRTT { 0x7FFFFFFF }; // Current RTT, in microseconds int _baseRTT { 0x7FFFFFFF }; // Lowest RTT during connection, in microseconds int _numRTT { 0 }; // Number of RTT collected during last RTT - int _ewmaRTT { 100000 }; // Exponential weighted moving average RTT - int _rttVariance { 50000 }; // Variance in collected RTT values + int _ewmaRTT { -1 }; // Exponential weighted moving average RTT + int _rttVariance { 0 }; // Variance in collected RTT values int _slowStartOddAdjust { 0 }; // Marker for every window adjustment every other RTT in slow-start From c0219d9ddf5200df59a8e3b2950ad038f6dd444b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 15:42:46 -0700 Subject: [PATCH 23/65] remove numRTT from TCPVegasCC --- libraries/networking/src/udt/TCPVegasCC.cpp | 1 - libraries/networking/src/udt/TCPVegasCC.h | 1 - 2 files changed, 2 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 288fcad430..469cf9b431 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -110,7 +110,6 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { static int VEGAS_GAMMA_SEGMENTS = 1; qDebug() << "============"; - qDebug() << "PCA" << _numRTT; qDebug() << "CWS:" << _congestionWindowSize << "SS:" << _slowStart; // Use the Vegas algorithm to see if we should diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index f579654dcf..c3eda245be 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -54,7 +54,6 @@ private: int _currentMinRTT { 0x7FFFFFFF }; // Current RTT, in microseconds int _baseRTT { 0x7FFFFFFF }; // Lowest RTT during connection, in microseconds - int _numRTT { 0 }; // Number of RTT collected during last RTT int _ewmaRTT { -1 }; // Exponential weighted moving average RTT int _rttVariance { 0 }; // Variance in collected RTT values From 456bcbca67e1b16abe31290fe85a4390bf3424b7 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 15:44:11 -0700 Subject: [PATCH 24/65] use max and not min for CW clamp --- libraries/networking/src/udt/TCPVegasCC.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 469cf9b431..cbec63765a 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -160,7 +160,7 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { // we never allow the congestion window to be smaller than two packets static int VEGAS_CW_MIN_PACKETS = 2; - _congestionWindowSize = std::min(_congestionWindowSize, VEGAS_CW_MIN_PACKETS); + _congestionWindowSize = std::max(_congestionWindowSize, VEGAS_CW_MIN_PACKETS); // mark this as the last adjustment time _lastAdjustmentTime = p_high_resolution_clock::now(); From 5ffbd2d11eb93b553512cb3899ab4dd8bca6c353 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 15:52:16 -0700 Subject: [PATCH 25/65] start num ACKs since FRT to 3 --- libraries/networking/src/udt/TCPVegasCC.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index c3eda245be..021c4398bc 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -50,7 +50,7 @@ private: SequenceNumber _lastAck; // Sequence number of last packet that was ACKed - int _numACKSinceFastRetransmit { 0 }; // Number of ACKs received since last fast re-transmit + int _numACKSinceFastRetransmit { 3 }; // Number of ACKs received since last fast re-transmit int _currentMinRTT { 0x7FFFFFFF }; // Current RTT, in microseconds int _baseRTT { 0x7FFFFFFF }; // Lowest RTT during connection, in microseconds From 9760fdc906f2193fc40333c40a0e88256906a07d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Sep 2016 15:55:23 -0700 Subject: [PATCH 26/65] use MAX_PACKETS_IN_FLIGHT as receiver max flow window --- libraries/networking/src/udt/Connection.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 86e7876ef0..b0ce22d456 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -315,8 +315,8 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { // pack the available buffer size, in packets // in our implementation we have no hard limit on receive buffer size, send the default value - _ackPacket->writePrimitive((int32_t) udt::CONNECTION_RECEIVE_BUFFER_SIZE_PACKETS); - + _ackPacket->writePrimitive((int32_t) udt::MAX_PACKETS_IN_FLIGHT); + if (wasCausedBySyncTimeout) { // grab the up to date packet receive speed and estimated bandwidth int32_t packetReceiveSpeed = _receiveWindow.getPacketReceiveSpeed(); From 3d0f449afb06e1176489f45528f2fd2d668f2876 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 26 Sep 2016 17:17:41 -0700 Subject: [PATCH 27/65] Ignore ack syn when using vegas --- libraries/networking/src/udt/Connection.cpp | 8 ++++++-- tools/udt-test/src/UDTTest.cpp | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index b0ce22d456..72433ef868 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -111,7 +111,8 @@ SendQueue& Connection::getSendQueue() { #ifdef UDT_CONNECTION_DEBUG qCDebug(networking) << "Created SendQueue for connection to" << _destination; #endif - + + QObject::connect(_sendQueue.get(), &SendQueue::packetSent, this, &Connection::packetSent); QObject::connect(_sendQueue.get(), &SendQueue::packetSent, this, &Connection::recordSentPackets); QObject::connect(_sendQueue.get(), &SendQueue::packetRetransmitted, this, &Connection::recordRetransmission); QObject::connect(_sendQueue.get(), &SendQueue::queueInactive, this, &Connection::queueInactive); @@ -515,7 +516,10 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in ++_packetsSinceACK; // check if we need to send an ACK, according to CC params - if (_congestionControl->_ackInterval > 0 && _packetsSinceACK >= _congestionControl->_ackInterval * _acksDuringSYN) { + if (_congestionControl->_ackInterval == 1) { + // Using TCP Vegas + sendACK(true); + } else if (_congestionControl->_ackInterval > 0 && _packetsSinceACK >= _congestionControl->_ackInterval * _acksDuringSYN) { _acksDuringSYN++; sendACK(false); } else if (_congestionControl->_lightACKInterval > 0 diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 633251d6c6..6161dbfdbc 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -25,7 +25,7 @@ const QCommandLineOption TARGET_OPTION { "IP:PORT or HOSTNAME:PORT" }; const QCommandLineOption PACKET_SIZE { - "packet-size", "size for sent packets in bytes (defaults to " + QString(udt::MAX_PACKET_SIZE) + ")", "bytes", + "packet-size", "size for sent packets in bytes (defaults to " + QString::number(udt::MAX_PACKET_SIZE) + ")", "bytes", QString(udt::MAX_PACKET_SIZE) }; const QCommandLineOption MIN_PACKET_SIZE { From b0192df04fc4c08860f72cde9dfd5bf034570cf9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 27 Sep 2016 10:29:08 -0700 Subject: [PATCH 28/65] allow congestion control to disable ACK2 --- libraries/networking/src/udt/CongestionControl.h | 1 + libraries/networking/src/udt/Connection.cpp | 3 ++- libraries/networking/src/udt/TCPVegasCC.h | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 491956bb2b..5183c25413 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -45,6 +45,7 @@ public: virtual void onTimeout() {} virtual bool shouldNAK() { return true; } + virtual bool shouldACK2() { return true; } virtual void onPacketSent(int packetSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) {} protected: diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 72433ef868..c3dc7dd68a 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -609,7 +609,8 @@ void Connection::processACK(ControlPacketPointer controlPacket) { microseconds sinceLastACK2 = duration_cast(currentTime - lastACK2SendTime); - if (sinceLastACK2.count() >= _synInterval || currentACKSubSequenceNumber == _lastSentACK2) { + if (_congestionControl->shouldACK2() + && (sinceLastACK2.count() >= _synInterval || currentACKSubSequenceNumber == _lastSentACK2)) { // Send ACK2 packet sendACK2(currentACKSubSequenceNumber); diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index 021c4398bc..c13d590095 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -32,6 +32,7 @@ public: virtual void onTimeout() override {}; virtual bool shouldNAK() override { return false; } + virtual bool shouldACK2() override { return false; } virtual void onPacketSent(int packetSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) override; From 6a3dd1588752f0c982e37bf72662457b130534b0 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 27 Sep 2016 10:38:13 -0700 Subject: [PATCH 29/65] emit packetSent signal immediately after send --- libraries/networking/src/udt/SendQueue.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index c152a212c9..f36167ddf2 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -258,6 +258,8 @@ bool SendQueue::sendNewPacketAndAddToSentList(std::unique_ptr newPacket, auto payloadSize = newPacket->getPayloadSize(); auto bytesWritten = sendPacket(*newPacket); + + emit packetSent(packetSize, payloadSize, sequenceNumber, p_high_resolution_clock::now()); { // Insert the packet we have just sent in the sent list @@ -267,8 +269,6 @@ bool SendQueue::sendNewPacketAndAddToSentList(std::unique_ptr newPacket, entry.second.swap(newPacket); } Q_ASSERT_X(!newPacket, "SendQueue::sendNewPacketAndAddToSentList()", "Overriden packet in sent list"); - - emit packetSent(packetSize, payloadSize, sequenceNumber, p_high_resolution_clock::now()); if (bytesWritten < 0) { // this is a short-circuit loss - we failed to put this packet on the wire From afc59eea198eefbc1274f610f1347b4fee87a8a2 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 27 Sep 2016 10:46:27 -0700 Subject: [PATCH 30/65] grab earliest possible receive time in socket --- libraries/networking/src/udt/BasePacket.h | 7 +++++++ libraries/networking/src/udt/CongestionControl.cpp | 2 +- libraries/networking/src/udt/CongestionControl.h | 4 ++-- libraries/networking/src/udt/Connection.cpp | 4 ++-- libraries/networking/src/udt/SendQueue.cpp | 2 +- libraries/networking/src/udt/Socket.cpp | 6 ++++++ libraries/networking/src/udt/TCPVegasCC.cpp | 9 ++++----- libraries/networking/src/udt/TCPVegasCC.h | 2 +- 8 files changed, 24 insertions(+), 12 deletions(-) diff --git a/libraries/networking/src/udt/BasePacket.h b/libraries/networking/src/udt/BasePacket.h index 33b8020d3c..d9b624b595 100644 --- a/libraries/networking/src/udt/BasePacket.h +++ b/libraries/networking/src/udt/BasePacket.h @@ -18,6 +18,8 @@ #include +#include + #include "../HifiSockAddr.h" #include "Constants.h" @@ -80,6 +82,9 @@ public: qint64 writeString(const QString& string); QString readString(); + + void setReceiveTime(p_high_resolution_clock::time_point receiveTime) { _receiveTime = receiveTime; } + p_high_resolution_clock::time_point getReceiveTime() const { return _receiveTime; } template qint64 peekPrimitive(T* data); template qint64 readPrimitive(T* data); @@ -108,6 +113,8 @@ protected: qint64 _payloadSize = 0; // How much of the payload is actually used HifiSockAddr _senderSockAddr; // sender address for packet (only used on receiving end) + + p_high_resolution_clock::time_point _receiveTime; // captures the time the packet received (only used on receiving end) }; template qint64 BasePacket::peekPrimitive(T* data) { diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index ee0425fc89..7ade4f004f 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -49,7 +49,7 @@ DefaultCC::DefaultCC() : setPacketSendPeriod(1.0); } -bool DefaultCC::onACK(SequenceNumber ackNum) { +bool DefaultCC::onACK(SequenceNumber ackNum, p_high_resolution_clock::time_point receiveTime) { double increase = 0; // Note from UDT original code: diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 5183c25413..81de761632 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -40,7 +40,7 @@ public: void setMaxBandwidth(int maxBandwidth); virtual void init() {} - virtual bool onACK(SequenceNumber ackNum) { return false; } + virtual bool onACK(SequenceNumber ackNum, p_high_resolution_clock::time_point receiveTime) { return false; } virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) {} virtual void onTimeout() {} @@ -107,7 +107,7 @@ public: DefaultCC(); public: - virtual bool onACK(SequenceNumber ackNum) override; + virtual bool onACK(SequenceNumber ackNum, p_high_resolution_clock::time_point receiveTime) override; virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) override; virtual void onTimeout() override; diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index c3dc7dd68a..764511408c 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -690,8 +690,8 @@ void Connection::processACK(ControlPacketPointer controlPacket) { } // give this ACK to the congestion control and update the send queue parameters - updateCongestionControlAndSendQueue([this, ack](){ - if (_congestionControl->onACK(ack)) { + updateCongestionControlAndSendQueue([this, ack, &controlPacket](){ + if (_congestionControl->onACK(ack, controlPacket->getReceiveTime())) { // the congestion control has told us it needs a fast re-transmit of ack + 1, add that now _sendQueue->fastRetransmit(ack + 1); } diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index f36167ddf2..e8f3375ba6 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -258,7 +258,7 @@ bool SendQueue::sendNewPacketAndAddToSentList(std::unique_ptr newPacket, auto payloadSize = newPacket->getPayloadSize(); auto bytesWritten = sendPacket(*newPacket); - + emit packetSent(packetSize, payloadSize, sequenceNumber, p_high_resolution_clock::now()); { diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 03a32a5fe3..01530fe94f 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -305,12 +305,16 @@ void Socket::readPendingDatagrams() { continue; } + // grab a time point we can mark as the receive time of this packet + auto receiveTime = p_high_resolution_clock::now(); + auto it = _unfilteredHandlers.find(senderSockAddr); if (it != _unfilteredHandlers.end()) { // we have a registered unfiltered handler for this HifiSockAddr - call that and return if (it->second) { auto basePacket = BasePacket::fromReceivedPacket(std::move(buffer), packetSizeWithHeader, senderSockAddr); + basePacket->setReceiveTime(receiveTime); it->second(std::move(basePacket)); } @@ -323,6 +327,7 @@ void Socket::readPendingDatagrams() { if (isControlPacket) { // setup a control packet from the data we just read auto controlPacket = ControlPacket::fromReceivedPacket(std::move(buffer), packetSizeWithHeader, senderSockAddr); + controlPacket->setReceiveTime(receiveTime); // move this control packet to the matching connection, if there is one auto connection = findOrCreateConnection(senderSockAddr); @@ -334,6 +339,7 @@ void Socket::readPendingDatagrams() { } else { // setup a Packet from the data we just read auto packet = Packet::fromReceivedPacket(std::move(buffer), packetSizeWithHeader, senderSockAddr); + packet->setReceiveTime(receiveTime); // call our verification operator to see if this packet is verified if (!_packetFilterOperator || _packetFilterOperator(*packet)) { diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index cbec63765a..d3da7e6886 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -24,14 +24,13 @@ TCPVegasCC::TCPVegasCC() { setAckInterval(1); // TCP sends an ACK for every packet received } -bool TCPVegasCC::onACK(SequenceNumber ack) { +bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point receiveTime) { auto it = _sentPacketTimes.find(ack); if (it != _sentPacketTimes.end()) { - // calculate the RTT (time now - time ACK sent) - auto now = p_high_resolution_clock::now(); - int lastRTT = duration_cast(now - it->second.first).count(); + // calculate the RTT (receive time - time ACK sent) + int lastRTT = duration_cast(receiveTime - it->second.first).count(); if (lastRTT < 0) { Q_ASSERT_X(false, "TCPVegasCC::onACK", "calculated an RTT that is not > 0"); @@ -61,7 +60,7 @@ bool TCPVegasCC::onACK(SequenceNumber ack) { // find the min RTT during the last RTT _currentMinRTT = std::min(_currentMinRTT, lastRTT); - auto sinceLastAdjustment = duration_cast(now - _lastAdjustmentTime).count(); + auto sinceLastAdjustment = duration_cast(p_high_resolution_clock::now() - _lastAdjustmentTime).count(); if (sinceLastAdjustment >= _ewmaRTT) { performCongestionAvoidance(ack); } diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index c13d590095..7dffc18702 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -27,7 +27,7 @@ public: TCPVegasCC(); public: - virtual bool onACK(SequenceNumber ackNum) override; + virtual bool onACK(SequenceNumber ackNum, p_high_resolution_clock::time_point receiveTime) override; virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) override {}; virtual void onTimeout() override {}; From 5174891b66f8870bd49f47e455a16ec3068ffe39 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 27 Sep 2016 10:51:28 -0700 Subject: [PATCH 31/65] move receive time of packet higher in Socket --- libraries/networking/src/udt/Socket.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 01530fe94f..16a1bd1639 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -289,6 +289,9 @@ void Socket::messageFailed(Connection* connection, Packet::MessageNumber message void Socket::readPendingDatagrams() { int packetSizeWithHeader = -1; while ((packetSizeWithHeader = _udpSocket.pendingDatagramSize()) != -1) { + // grab a time point we can mark as the receive time of this packet + auto receiveTime = p_high_resolution_clock::now(); + // setup a HifiSockAddr to read into HifiSockAddr senderSockAddr; @@ -305,9 +308,6 @@ void Socket::readPendingDatagrams() { continue; } - // grab a time point we can mark as the receive time of this packet - auto receiveTime = p_high_resolution_clock::now(); - auto it = _unfilteredHandlers.find(senderSockAddr); if (it != _unfilteredHandlers.end()) { From 202641a3499255f94625e107423210d4ea398df5 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 27 Sep 2016 12:43:24 -0700 Subject: [PATCH 32/65] Debug tweaks --- libraries/networking/src/udt/TCPVegasCC.cpp | 36 ++++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index d3da7e6886..c27b82f485 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -69,9 +69,7 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r _sentPacketTimes.erase(it); } else { - Q_ASSERT_X(false, - "TCPVegasCC::onACK", - "called with a sequence number that has not been sent"); + qDebug() << "called with a sequence number that has not been sent"; } @@ -108,9 +106,6 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { static int VEGAS_BETA_SEGMENTS = 4; static int VEGAS_GAMMA_SEGMENTS = 1; - qDebug() << "============"; - qDebug() << "CWS:" << _congestionWindowSize << "SS:" << _slowStart; - // Use the Vegas algorithm to see if we should // increase or decrease the congestion window size, and by how much @@ -121,17 +116,32 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { int windowSizeDiff = _congestionWindowSize * (rtt - _baseRTT) / _baseRTT; - qDebug() << "BRTT:" << _baseRTT << "CRTT:" << _currentMinRTT << "ERTT:" << _ewmaRTT; - qDebug() << "D:" << windowSizeDiff; + static int count = 0; + bool wantDebug = false; + if (++count > 200) { + wantDebug = true; + count = 0; + } + + auto debug = qDebug(); + if (wantDebug) { + debug << " ============\n"; + debug << "CWS:" << _congestionWindowSize << "SS:" << _slowStart << "\n"; + debug << "BRTT:" << _baseRTT << "CRTT:" << _currentMinRTT << "ERTT:" << _ewmaRTT << "\n"; + debug << "D:" << windowSizeDiff << "\n"; + } if (_slowStart) { if (windowSizeDiff > VEGAS_GAMMA_SEGMENTS) { // we're going too fast - this breaks us out of slow start and we switch to linear increase/decrease _slowStart = false; - int expectedWindowSize = _congestionWindowSize * _baseRTT / rtt; + int expectedWindowSize = _congestionWindowSize * _baseRTT / rtt; + _baseRTT = std::numeric_limits::max(); - qDebug() << "EWS:" << expectedWindowSize; + if (wantDebug) { + debug << "EWS:" << expectedWindowSize; + } // drop the congestion window size to the expected size, if smaller _congestionWindowSize = std::min(_congestionWindowSize, expectedWindowSize + 1); @@ -167,8 +177,10 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { // reset our state for the next RTT _currentMinRTT = std::numeric_limits::max(); - qDebug() << "CW:" << _congestionWindowSize << "SS:" << _slowStart; - qDebug() << "============"; + if (wantDebug) { + debug << "CW:" << _congestionWindowSize << "SS:" << _slowStart << "\n"; + debug << "============"; + } } From 2c026d7b0fbe6e32d01bc91330df495e8322822c Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 27 Sep 2016 13:54:24 -0700 Subject: [PATCH 33/65] Disable packet probes --- libraries/networking/src/udt/CongestionControl.h | 1 + libraries/networking/src/udt/Connection.cpp | 1 + libraries/networking/src/udt/SendQueue.cpp | 6 +++++- libraries/networking/src/udt/SendQueue.h | 5 +++++ libraries/networking/src/udt/TCPVegasCC.cpp | 2 -- libraries/networking/src/udt/TCPVegasCC.h | 1 + 6 files changed, 13 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 81de761632..e56dd253db 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -46,6 +46,7 @@ public: virtual bool shouldNAK() { return true; } virtual bool shouldACK2() { return true; } + virtual bool shouldProbe() { return true; } virtual void onPacketSent(int packetSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) {} protected: diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 764511408c..45ecf11e50 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -125,6 +125,7 @@ SendQueue& Connection::getSendQueue() { _sendQueue->setSyncInterval(_synInterval); _sendQueue->setEstimatedTimeout(estimatedTimeout()); _sendQueue->setFlowWindowSize(std::min(_flowWindowSize, (int) _congestionControl->_congestionWindowSize)); + _sendQueue->setProbePacketEnabled(_congestionControl->shouldProbe()); // give the randomized sequence number to the congestion control object _congestionControl->setInitialSendSequenceNumber(_sendQueue->getCurrentSequenceNumber()); diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index e8f3375ba6..537f5c9215 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -396,6 +396,10 @@ void SendQueue::run() { } } +void SendQueue::setProbePacketEnabled(bool enabled) { + _shouldSendProbes = enabled; +} + int SendQueue::maybeSendNewPacket() { if (!isFlowWindowFull()) { // we didn't re-send a packet, so time to send a new one @@ -413,7 +417,7 @@ int SendQueue::maybeSendNewPacket() { std::unique_ptr secondPacket; bool shouldSendPairTail = false; - if (((uint32_t) nextNumber & 0xF) == 0) { + if (_shouldSendProbes && ((uint32_t) nextNumber & 0xF) == 0) { // the first packet is the first in a probe pair - every 16 (rightmost 16 bits = 0) packets // pull off a second packet if we can before we unlock shouldSendPairTail = true; diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index c7110d4044..e7651dd934 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -64,6 +64,8 @@ public: void setEstimatedTimeout(int estimatedTimeout) { _estimatedTimeout = estimatedTimeout; } void setSyncInterval(int syncInterval) { _syncInterval = syncInterval; } + + void setProbePacketEnabled(bool enabled); public slots: void stop(); @@ -140,6 +142,9 @@ private: std::condition_variable _handshakeACKCondition; std::condition_variable_any _emptyCondition; + + + std::atomic _shouldSendProbes { true }; }; } diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index c27b82f485..15f3863b54 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -52,8 +52,6 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r + abs(lastRTT - _ewmaRTT)) / RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR; } - - // keep track of the lowest RTT during connection _baseRTT = std::min(_baseRTT, lastRTT); diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index 7dffc18702..4f538cd5a4 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -33,6 +33,7 @@ public: virtual bool shouldNAK() override { return false; } virtual bool shouldACK2() override { return false; } + virtual bool shouldProbe() override { return false; } virtual void onPacketSent(int packetSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) override; From dbbdda3468c43bef359e1e97d5df4f664e7c6543 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 27 Sep 2016 13:57:11 -0700 Subject: [PATCH 34/65] keep track of num RTT and debug reno fallback --- libraries/networking/src/udt/TCPVegasCC.cpp | 10 ++++++++++ libraries/networking/src/udt/TCPVegasCC.h | 2 ++ 2 files changed, 12 insertions(+) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 15f3863b54..39c4c0c5da 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -52,6 +52,9 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r + abs(lastRTT - _ewmaRTT)) / RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR; } + // add 1 to the number of RTT samples + ++_numRTTs; + // keep track of the lowest RTT during connection _baseRTT = std::min(_baseRTT, lastRTT); @@ -129,6 +132,10 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { debug << "D:" << windowSizeDiff << "\n"; } + if (_numRTTs <= 2) { + qDebug() << "WE SHOULD BE USING RENO HERE" << _numRTTs; + } + if (_slowStart) { if (windowSizeDiff > VEGAS_GAMMA_SEGMENTS) { // we're going too fast - this breaks us out of slow start and we switch to linear increase/decrease @@ -175,6 +182,9 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { // reset our state for the next RTT _currentMinRTT = std::numeric_limits::max(); + // reset our count of collected RTT samples + _numRTTs = 0; + if (wantDebug) { debug << "CW:" << _congestionWindowSize << "SS:" << _slowStart << "\n"; debug << "============"; diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index 4f538cd5a4..031e4afacf 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -59,6 +59,8 @@ private: int _ewmaRTT { -1 }; // Exponential weighted moving average RTT int _rttVariance { 0 }; // Variance in collected RTT values + int _numRTTs { 0 }; // Holds the number of ACKs received during the last RTT + int _slowStartOddAdjust { 0 }; // Marker for every window adjustment every other RTT in slow-start }; From 0b2e0a4167f601868486cc25cfc50e9e6c52f628 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 27 Sep 2016 14:40:50 -0700 Subject: [PATCH 35/65] initial re-implementation of reno cong avoid --- libraries/networking/src/udt/TCPVegasCC.cpp | 121 ++++++++++++++------ libraries/networking/src/udt/TCPVegasCC.h | 6 +- 2 files changed, 88 insertions(+), 39 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 39c4c0c5da..9e502e9020 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -27,6 +27,9 @@ TCPVegasCC::TCPVegasCC() { bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point receiveTime) { auto it = _sentPacketTimes.find(ack); + auto previousAck = _lastAck; + _lastAck = ack; + if (it != _sentPacketTimes.end()) { // calculate the RTT (receive time - time ACK sent) @@ -52,8 +55,8 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r + abs(lastRTT - _ewmaRTT)) / RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR; } - // add 1 to the number of RTT samples - ++_numRTTs; + // add 1 to the number of ACKs during this RTT + ++_numACKs; // keep track of the lowest RTT during connection _baseRTT = std::min(_baseRTT, lastRTT); @@ -78,7 +81,7 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r // after a previous fast re-transmit ++_numACKSinceFastRetransmit; - if (ack == _lastAck || _numACKSinceFastRetransmit < 3) { + if (ack == previousAck || _numACKSinceFastRetransmit < 3) { // we may need to re-send ackNum + 1 if it has been more than our estimated timeout since it was sent qDebug() << "FRT:" << (uint32_t) ack << (uint32_t) _lastAck << _numACKSinceFastRetransmit; @@ -98,6 +101,8 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r } } + _lastAck = ack; + // ACK processed, no fast re-transmit required return false; } @@ -132,43 +137,43 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { debug << "D:" << windowSizeDiff << "\n"; } - if (_numRTTs <= 2) { - qDebug() << "WE SHOULD BE USING RENO HERE" << _numRTTs; - } - - if (_slowStart) { - if (windowSizeDiff > VEGAS_GAMMA_SEGMENTS) { - // we're going too fast - this breaks us out of slow start and we switch to linear increase/decrease - _slowStart = false; - - int expectedWindowSize = _congestionWindowSize * _baseRTT / rtt; - _baseRTT = std::numeric_limits::max(); - - if (wantDebug) { - debug << "EWS:" << expectedWindowSize; - } - - // drop the congestion window size to the expected size, if smaller - _congestionWindowSize = std::min(_congestionWindowSize, expectedWindowSize + 1); - - } else if (++_slowStartOddAdjust & 1) { - // we're in slow start and not going too fast - // this means that once every second RTT we perform exponential congestion window growth - _congestionWindowSize *= 2; - } + if (_numACKs <= 2) { + performRenoCongestionAvoidance(ack); } else { - // this is the normal linear increase/decrease of the Vegas algorithm - // to figure out where the congestion window should be - if (windowSizeDiff > VEGAS_BETA_SEGMENTS) { - // the old congestion window was too fast (difference > beta) - // so reduce it to slow down - --_congestionWindowSize; + if (_slowStart) { + if (windowSizeDiff > VEGAS_GAMMA_SEGMENTS) { + // we're going too fast - this breaks us out of slow start and we switch to linear increase/decrease + _slowStart = false; - } else if (windowSizeDiff < VEGAS_ALPHA_SEGMENTS) { - // there aren't enough packets on the wire, add more to the congestion window - ++_congestionWindowSize; + int expectedWindowSize = _congestionWindowSize * _baseRTT / rtt; + _baseRTT = std::numeric_limits::max(); + + if (wantDebug) { + debug << "EWS:" << expectedWindowSize; + } + + // drop the congestion window size to the expected size, if smaller + _congestionWindowSize = std::min(_congestionWindowSize, expectedWindowSize + 1); + + } else if (++_slowStartOddAdjust & 1) { + // we're in slow start and not going too fast + // this means that once every second RTT we perform exponential congestion window growth + _congestionWindowSize *= 2; + } } else { - // sending rate seems good, no congestion window adjustment + // this is the normal linear increase/decrease of the Vegas algorithm + // to figure out where the congestion window should be + if (windowSizeDiff > VEGAS_BETA_SEGMENTS) { + // the old congestion window was too fast (difference > beta) + // so reduce it to slow down + --_congestionWindowSize; + + } else if (windowSizeDiff < VEGAS_ALPHA_SEGMENTS) { + // there aren't enough packets on the wire, add more to the congestion window + ++_congestionWindowSize; + } else { + // sending rate seems good, no congestion window adjustment + } } } @@ -183,7 +188,7 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { _currentMinRTT = std::numeric_limits::max(); // reset our count of collected RTT samples - _numRTTs = 0; + _numACKs = 0; if (wantDebug) { debug << "CW:" << _congestionWindowSize << "SS:" << _slowStart << "\n"; @@ -191,6 +196,46 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { } } +bool TCPVegasCC::isCongestionWindowLimited() { + if (_slowStart) { + return true; + } else { + return seqlen(_sendCurrSeqNum, _lastAck) < _congestionWindowSize; + } +} + + +void TCPVegasCC::performRenoCongestionAvoidance(SequenceNumber ack) { + if (!isCongestionWindowLimited()) { + return; + } + + int numAcked = _numACKs; + + // In "safe" area, increase. + if (_slowStart) { + int congestionWindow = std::min(_congestionWindowSize + numAcked, udt::MAX_PACKETS_IN_FLIGHT); + numAcked -= congestionWindow - _congestionWindowSize; + } + + if (numAcked > 0) { + // In dangerous area, increase slowly. + // If credits accumulated at a higher w, apply them gently now. + if (_acksBeforeAdditiveIncrease >= _congestionWindowSize) { + _acksBeforeAdditiveIncrease = 0; + ++_congestionWindowSize; + } + + _acksBeforeAdditiveIncrease += numAcked; + if (_acksBeforeAdditiveIncrease >= _congestionWindowSize) { + int delta = _acksBeforeAdditiveIncrease / _congestionWindowSize; + + _acksBeforeAdditiveIncrease -= delta * _congestionWindowSize; + _acksBeforeAdditiveIncrease += delta; + } + } +} + void TCPVegasCC::onPacketSent(int packetSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) { _sentPacketTimes[seqNum] = { timePoint, packetSize }; diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index 031e4afacf..817126b1ca 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -41,6 +41,8 @@ protected: virtual void performCongestionAvoidance(SequenceNumber ack); virtual void setInitialSendSequenceNumber(SequenceNumber seqNum) override { _lastAck = seqNum - 1; } private: + bool isCongestionWindowLimited(); + void performRenoCongestionAvoidance(SequenceNumber ack); using TimeSizePair = std::pair; using PacketTimeList = std::map; @@ -59,7 +61,9 @@ private: int _ewmaRTT { -1 }; // Exponential weighted moving average RTT int _rttVariance { 0 }; // Variance in collected RTT values - int _numRTTs { 0 }; // Holds the number of ACKs received during the last RTT + int _numACKs { 0 }; // Holds the number of ACKs received during the last RTT + + int _acksBeforeAdditiveIncrease { 0 }; int _slowStartOddAdjust { 0 }; // Marker for every window adjustment every other RTT in slow-start From 64a0239248d0904793a817a2008f6574305ab0ab Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 27 Sep 2016 14:49:00 -0700 Subject: [PATCH 36/65] add reno fallback for fast re-transmit --- libraries/networking/src/udt/TCPVegasCC.cpp | 34 +++++++++++++++------ libraries/networking/src/udt/TCPVegasCC.h | 3 +- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 9e502e9020..5c4f121826 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -83,7 +83,6 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r if (ack == previousAck || _numACKSinceFastRetransmit < 3) { // we may need to re-send ackNum + 1 if it has been more than our estimated timeout since it was sent - qDebug() << "FRT:" << (uint32_t) ack << (uint32_t) _lastAck << _numACKSinceFastRetransmit; auto it = _sentPacketTimes.find(ack + 1); if (it != _sentPacketTimes.end()) { @@ -93,12 +92,27 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r auto sinceSend = duration_cast(now - it->second.first).count(); if (sinceSend >= estimatedTimeout) { - qDebug() << "FRT needed:" << sinceSend << estimatedTimeout; // we've detected we need a fast re-transmit, send that back to the connection _numACKSinceFastRetransmit = 0; return true; } } + + // if this is the 3rd duplicate ACK, we fallback to Reno's fast re-transmit + static const int RENO_FAST_RETRANSMIT_DUPLICATE_COUNT = 3; + if (ack == previousAck && ++_duplicateACKCount == RENO_FAST_RETRANSMIT_DUPLICATE_COUNT) { + // break out of slow start, we just hit loss + _slowStart = false; + + // reset our fast re-transmit counters + _numACKSinceFastRetransmit = 0; + _duplicateACKCount = 0; + + // return true so the caller knows we needed a fast re-transmit + return true; + } + } else { + _duplicateACKCount = 0; } _lastAck = ack; @@ -218,20 +232,22 @@ void TCPVegasCC::performRenoCongestionAvoidance(SequenceNumber ack) { numAcked -= congestionWindow - _congestionWindowSize; } + int preAIWindowSize = _congestionWindowSize; + if (numAcked > 0) { // In dangerous area, increase slowly. // If credits accumulated at a higher w, apply them gently now. - if (_acksBeforeAdditiveIncrease >= _congestionWindowSize) { - _acksBeforeAdditiveIncrease = 0; + if (_ackAICount >= preAIWindowSize) { + _ackAICount = 0; ++_congestionWindowSize; } - _acksBeforeAdditiveIncrease += numAcked; - if (_acksBeforeAdditiveIncrease >= _congestionWindowSize) { - int delta = _acksBeforeAdditiveIncrease / _congestionWindowSize; + _ackAICount += numAcked; + if (_ackAICount >= preAIWindowSize) { + int delta = _ackAICount / preAIWindowSize; - _acksBeforeAdditiveIncrease -= delta * _congestionWindowSize; - _acksBeforeAdditiveIncrease += delta; + _ackAICount -= delta * _congestionWindowSize; + _congestionWindowSize += delta; } } } diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index 817126b1ca..3be3f1df0b 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -63,7 +63,8 @@ private: int _numACKs { 0 }; // Holds the number of ACKs received during the last RTT - int _acksBeforeAdditiveIncrease { 0 }; + int _ackAICount { 0 }; // Counter for number of ACKs received for Reno additive increase + int _duplicateACKCount { 0 }; // Counter for duplicate ACKs received int _slowStartOddAdjust { 0 }; // Marker for every window adjustment every other RTT in slow-start From d05e3e1dd610d46a752b5a0f0e5cbb3b89180c06 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 27 Sep 2016 14:56:57 -0700 Subject: [PATCH 37/65] drop CWS to 2 if reno fast re-transmit is hit --- libraries/networking/src/udt/TCPVegasCC.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 5c4f121826..4d71a30a1c 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -108,6 +108,9 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r _numACKSinceFastRetransmit = 0; _duplicateACKCount = 0; + // drop the congestion window size to 2 segments + _congestionWindowSize = 2; + // return true so the caller knows we needed a fast re-transmit return true; } From b74a967c95a93f1482878b8b3c29b307358c39a6 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 27 Sep 2016 14:57:33 -0700 Subject: [PATCH 38/65] fix an additional reference to preAIWindowSize --- libraries/networking/src/udt/TCPVegasCC.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 4d71a30a1c..a227a675ae 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -249,7 +249,7 @@ void TCPVegasCC::performRenoCongestionAvoidance(SequenceNumber ack) { if (_ackAICount >= preAIWindowSize) { int delta = _ackAICount / preAIWindowSize; - _ackAICount -= delta * _congestionWindowSize; + _ackAICount -= delta * preAIWindowSize; _congestionWindowSize += delta; } } From b03bb6b73f0ac441c07a02e63459a546b9c34fe4 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 27 Sep 2016 14:58:18 -0700 Subject: [PATCH 39/65] halve the congestion window on 3rd duplicate ACK --- libraries/networking/src/udt/TCPVegasCC.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index a227a675ae..227bcadc33 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -109,7 +109,7 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r _duplicateACKCount = 0; // drop the congestion window size to 2 segments - _congestionWindowSize = 2; + _congestionWindowSize /= 2; // return true so the caller knows we needed a fast re-transmit return true; From a075bb6b1cdc8ae45c6575ac83093f9abdede60d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 27 Sep 2016 17:35:59 -0700 Subject: [PATCH 40/65] slow down new UDT debug for asset-server tests --- libraries/networking/src/udt/TCPVegasCC.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 227bcadc33..2a67fe5bed 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -72,8 +72,6 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r // remove this sent packet time from the hash _sentPacketTimes.erase(it); - } else { - qDebug() << "called with a sequence number that has not been sent"; } @@ -141,7 +139,7 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { static int count = 0; bool wantDebug = false; - if (++count > 200) { + if (++count > 500) { wantDebug = true; count = 0; } From ec51f01c591accfc0fef35c82710ad91c19da867 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 27 Sep 2016 18:02:49 -0700 Subject: [PATCH 41/65] remove debugs from TCPVegasCC --- libraries/networking/src/udt/TCPVegasCC.cpp | 28 ++------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 2a67fe5bed..34a883ccbd 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -123,8 +123,8 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r } void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { - static int VEGAS_ALPHA_SEGMENTS = 2; - static int VEGAS_BETA_SEGMENTS = 4; + static int VEGAS_ALPHA_SEGMENTS = 4; + static int VEGAS_BETA_SEGMENTS = 6; static int VEGAS_GAMMA_SEGMENTS = 1; // Use the Vegas algorithm to see if we should @@ -137,21 +137,6 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { int windowSizeDiff = _congestionWindowSize * (rtt - _baseRTT) / _baseRTT; - static int count = 0; - bool wantDebug = false; - if (++count > 500) { - wantDebug = true; - count = 0; - } - - auto debug = qDebug(); - if (wantDebug) { - debug << " ============\n"; - debug << "CWS:" << _congestionWindowSize << "SS:" << _slowStart << "\n"; - debug << "BRTT:" << _baseRTT << "CRTT:" << _currentMinRTT << "ERTT:" << _ewmaRTT << "\n"; - debug << "D:" << windowSizeDiff << "\n"; - } - if (_numACKs <= 2) { performRenoCongestionAvoidance(ack); } else { @@ -163,10 +148,6 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { int expectedWindowSize = _congestionWindowSize * _baseRTT / rtt; _baseRTT = std::numeric_limits::max(); - if (wantDebug) { - debug << "EWS:" << expectedWindowSize; - } - // drop the congestion window size to the expected size, if smaller _congestionWindowSize = std::min(_congestionWindowSize, expectedWindowSize + 1); @@ -204,11 +185,6 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { // reset our count of collected RTT samples _numACKs = 0; - - if (wantDebug) { - debug << "CW:" << _congestionWindowSize << "SS:" << _slowStart << "\n"; - debug << "============"; - } } bool TCPVegasCC::isCongestionWindowLimited() { From 54b28e646a9cca455c5da81776ad8721c8437e8f Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 27 Sep 2016 18:09:56 -0700 Subject: [PATCH 42/65] remove congestion window halving on reno dupe loss --- libraries/networking/src/udt/TCPVegasCC.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 34a883ccbd..ad5dc68c54 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -106,9 +106,6 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r _numACKSinceFastRetransmit = 0; _duplicateACKCount = 0; - // drop the congestion window size to 2 segments - _congestionWindowSize /= 2; - // return true so the caller knows we needed a fast re-transmit return true; } From af969231e7c15ce44802b38f87619751b9373a0b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 27 Sep 2016 18:11:40 -0700 Subject: [PATCH 43/65] remove some extra lines --- libraries/networking/src/udt/TCPVegasCC.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index ad5dc68c54..3248012878 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -192,7 +192,6 @@ bool TCPVegasCC::isCongestionWindowLimited() { } } - void TCPVegasCC::performRenoCongestionAvoidance(SequenceNumber ack) { if (!isCongestionWindowLimited()) { return; @@ -226,7 +225,6 @@ void TCPVegasCC::performRenoCongestionAvoidance(SequenceNumber ack) { } } - void TCPVegasCC::onPacketSent(int packetSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) { _sentPacketTimes[seqNum] = { timePoint, packetSize }; } From 6d69a0dc71d1e89a1714b87526ab71bc6ed76932 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 28 Sep 2016 10:19:33 -0700 Subject: [PATCH 44/65] push version for asset packets to mismatch for TCPVegasCC --- libraries/networking/src/udt/PacketHeaders.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index ec4e724c1b..f737700c55 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -58,8 +58,8 @@ PacketVersion versionForPacketType(PacketType packetType) { case PacketType::AssetGetInfo: case PacketType::AssetGet: case PacketType::AssetUpload: - // Removal of extension from Asset requests - return 18; + // Introduction of TCPVegasCC as default congestion control + return 19; case PacketType::NodeIgnoreRequest: return 18; // Introduction of node ignore request (which replaced an unused packet tpye) From e7c0ed035f9d974c5d906639b45d1ffa2c2f9e14 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 28 Sep 2016 10:23:18 -0700 Subject: [PATCH 45/65] remove SendQueue fast re-transmit debug --- libraries/networking/src/udt/SendQueue.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 537f5c9215..f42e2b77fb 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -186,8 +186,6 @@ void SendQueue::fastRetransmit(udt::SequenceNumber ack) { _naks.insert(ack, ack); } - qDebug() << "Added" << (uint32_t) ack << "for fast retransmit"; - // call notify_one on the condition_variable_any in case the send thread is sleeping waiting for losses to re-send _emptyCondition.notify_one(); } From fdcface8cb95679144d0c110abdb3bdd10e7a6ca Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 29 Sep 2016 11:22:05 -0700 Subject: [PATCH 46/65] use enum in PacketHeaders, coding standard fixes in Connection --- libraries/networking/src/udt/Connection.cpp | 8 ++++---- libraries/networking/src/udt/PacketHeaders.cpp | 3 +-- libraries/networking/src/udt/PacketHeaders.h | 4 ++++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 45ecf11e50..6d17670cce 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -152,13 +152,13 @@ void Connection::queueInactive() { } void Connection::queueTimeout() { - updateCongestionControlAndSendQueue([this]{ + updateCongestionControlAndSendQueue([this] { _congestionControl->onTimeout(); }); } void Connection::queueShortCircuitLoss(quint32 sequenceNumber) { - updateCongestionControlAndSendQueue([this, sequenceNumber]{ + updateCongestionControlAndSendQueue([this, sequenceNumber] { _congestionControl->onLoss(SequenceNumber { sequenceNumber }, SequenceNumber { sequenceNumber }); }); } @@ -691,7 +691,7 @@ void Connection::processACK(ControlPacketPointer controlPacket) { } // give this ACK to the congestion control and update the send queue parameters - updateCongestionControlAndSendQueue([this, ack, &controlPacket](){ + updateCongestionControlAndSendQueue([this, ack, &controlPacket] { if (_congestionControl->onACK(ack, controlPacket->getReceiveTime())) { // the congestion control has told us it needs a fast re-transmit of ack + 1, add that now _sendQueue->fastRetransmit(ack + 1); @@ -780,7 +780,7 @@ void Connection::processNAK(ControlPacketPointer controlPacket) { getSendQueue().nak(start, end); // give the loss to the congestion control object and update the send queue parameters - updateCongestionControlAndSendQueue([this, start, end](){ + updateCongestionControlAndSendQueue([this, start, end] { _congestionControl->onLoss(start, end); }); diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index f737700c55..7e788ce02b 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -58,8 +58,7 @@ PacketVersion versionForPacketType(PacketType packetType) { case PacketType::AssetGetInfo: case PacketType::AssetGet: case PacketType::AssetUpload: - // Introduction of TCPVegasCC as default congestion control - return 19; + return static_cast(AssetServerPacketVersion::VegasCongestionControl); case PacketType::NodeIgnoreRequest: return 18; // Introduction of node ignore request (which replaced an unused packet tpye) diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index aa775b9f53..844847e791 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -189,6 +189,10 @@ const PacketVersion VERSION_MODEL_ENTITIES_SUPPORT_SIMPLE_HULLS = 62; const PacketVersion VERSION_WEB_ENTITIES_SUPPORT_DPI = 63; const PacketVersion VERSION_ENTITIES_ARROW_ACTION = 64; +enum class AssetServerPacketVersion: PacketVersion { + VegasCongestionControl = 19 +}; + enum class AvatarMixerPacketVersion : PacketVersion { TranslationSupport = 17, SoftAttachmentSupport, From 0a7d799abf1135ad4bd5b03a91d20b97d29c973e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 10 Oct 2016 16:33:14 -0700 Subject: [PATCH 47/65] don't double set _lastACK in TCPVegasCC --- libraries/networking/src/udt/TCPVegasCC.cpp | 8 +++----- libraries/networking/src/udt/TCPVegasCC.h | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 3248012878..8b0750e8b1 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -27,8 +27,8 @@ TCPVegasCC::TCPVegasCC() { bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point receiveTime) { auto it = _sentPacketTimes.find(ack); - auto previousAck = _lastAck; - _lastAck = ack; + auto previousAck = _lastACK; + _lastACK = ack; if (it != _sentPacketTimes.end()) { @@ -113,8 +113,6 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r _duplicateACKCount = 0; } - _lastAck = ack; - // ACK processed, no fast re-transmit required return false; } @@ -188,7 +186,7 @@ bool TCPVegasCC::isCongestionWindowLimited() { if (_slowStart) { return true; } else { - return seqlen(_sendCurrSeqNum, _lastAck) < _congestionWindowSize; + return seqlen(_sendCurrSeqNum, _lastACK) < _congestionWindowSize; } } diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index 3be3f1df0b..50990b19e8 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -39,7 +39,7 @@ public: protected: virtual void performCongestionAvoidance(SequenceNumber ack); - virtual void setInitialSendSequenceNumber(SequenceNumber seqNum) override { _lastAck = seqNum - 1; } + virtual void setInitialSendSequenceNumber(SequenceNumber seqNum) override { _lastACK = seqNum - 1; } private: bool isCongestionWindowLimited(); void performRenoCongestionAvoidance(SequenceNumber ack); @@ -52,7 +52,7 @@ private: bool _slowStart { true }; // Marker for slow start phase - SequenceNumber _lastAck; // Sequence number of last packet that was ACKed + SequenceNumber _lastACK; // Sequence number of last packet that was ACKed int _numACKSinceFastRetransmit { 3 }; // Number of ACKs received since last fast re-transmit From d15cb6171d9b5a47f62a8b5c0ed483396d444ec9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 13 Oct 2016 17:43:37 -0700 Subject: [PATCH 48/65] fix bubble up of duplicated packet info --- libraries/networking/src/udt/Connection.cpp | 37 +++++++++++---------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 6d17670cce..2ed142f25b 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -478,29 +478,30 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in _receiveWindow.onPacketArrival(); // If this is not the next sequence number, report loss - if (_congestionControl->shouldNAK() && sequenceNumber > _lastReceivedSequenceNumber + 1) { + if (sequenceNumber > _lastReceivedSequenceNumber + 1) { if (_lastReceivedSequenceNumber + 1 == sequenceNumber - 1) { _lossList.append(_lastReceivedSequenceNumber + 1); } else { _lossList.append(_lastReceivedSequenceNumber + 1, sequenceNumber - 1); } - - // Send a NAK packet - sendNAK(sequenceNumber); - - // figure out when we should send the next loss report, if we haven't heard anything back - _nakInterval = estimatedTimeout(); - - int receivedPacketsPerSecond = _receiveWindow.getPacketReceiveSpeed(); - if (receivedPacketsPerSecond > 0) { - // the NAK interval is at least the _minNAKInterval - // but might be the time required for all lost packets to be retransmitted - _nakInterval += (int) (_lossList.getLength() * (USECS_PER_SECOND / receivedPacketsPerSecond)); - } - - // the NAK interval is at least the _minNAKInterval but might be the value calculated above, if that is larger - _nakInterval = std::max(_nakInterval, _minNAKInterval); + if (_congestionControl->shouldNAK()) { + // Send a NAK packet + sendNAK(sequenceNumber); + + // figure out when we should send the next loss report, if we haven't heard anything back + _nakInterval = estimatedTimeout(); + + int receivedPacketsPerSecond = _receiveWindow.getPacketReceiveSpeed(); + if (receivedPacketsPerSecond > 0) { + // the NAK interval is at least the _minNAKInterval + // but might be the time required for all lost packets to be retransmitted + _nakInterval += (int) (_lossList.getLength() * (USECS_PER_SECOND / receivedPacketsPerSecond)); + } + + // the NAK interval is at least the _minNAKInterval but might be the value calculated above, if that is larger + _nakInterval = std::max(_nakInterval, _minNAKInterval); + } } bool wasDuplicate = false; @@ -508,7 +509,7 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in if (sequenceNumber > _lastReceivedSequenceNumber) { // Update largest recieved sequence number _lastReceivedSequenceNumber = sequenceNumber; - } else if (_congestionControl->shouldNAK()) { + } else { // Otherwise, it could be a resend, try and remove it from the loss list wasDuplicate = !_lossList.remove(sequenceNumber); } From ef660722cd90e7a40acb1b35cc2600bddc04f74a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 13 Oct 2016 18:29:08 -0700 Subject: [PATCH 49/65] force duplicated sends to use highest RTT, not lowest --- libraries/networking/src/udt/TCPVegasCC.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 8b0750e8b1..028212cec5 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -71,7 +71,6 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r // remove this sent packet time from the hash _sentPacketTimes.erase(it); - } @@ -224,6 +223,8 @@ void TCPVegasCC::performRenoCongestionAvoidance(SequenceNumber ack) { } void TCPVegasCC::onPacketSent(int packetSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) { - _sentPacketTimes[seqNum] = { timePoint, packetSize }; + if (_sentPacketTimes.find(seqNum) == _sentPacketTimes.end()) { + _sentPacketTimes[seqNum] = { timePoint, packetSize }; + } } From 7ef596a9f0502eff1be9344a7ae0d7e9a3682cc6 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Sun, 16 Oct 2016 19:26:49 -0700 Subject: [PATCH 50/65] use ova-welcome for default hifi address --- libraries/networking/src/AddressManager.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/AddressManager.h b/libraries/networking/src/AddressManager.h index 1707132c08..f158c3f081 100644 --- a/libraries/networking/src/AddressManager.h +++ b/libraries/networking/src/AddressManager.h @@ -23,7 +23,7 @@ #include "AccountManager.h" const QString HIFI_URL_SCHEME = "hifi"; -const QString DEFAULT_HIFI_ADDRESS = "hifi://welcome"; +const QString DEFAULT_HIFI_ADDRESS = "hifi://ova-welcome"; const QString SANDBOX_HIFI_ADDRESS = "hifi://localhost"; const QString INDEX_PATH = "/"; From 9407efbbc5cbc2b20135a535870e4b34f8d46fd1 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Oct 2016 11:54:05 -0700 Subject: [PATCH 51/65] protect against an overflow in TCPVegasCC --- libraries/networking/src/udt/TCPVegasCC.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 028212cec5..5c23be52d7 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -35,12 +35,16 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r // calculate the RTT (receive time - time ACK sent) int lastRTT = duration_cast(receiveTime - it->second.first).count(); + const int MAX_RTT_SAMPLE_MICROSECONDS = 10000000; + if (lastRTT < 0) { Q_ASSERT_X(false, "TCPVegasCC::onACK", "calculated an RTT that is not > 0"); return false; } else if (lastRTT == 0) { // we do not allow a 0 microsecond RTT lastRTT = 1; + } else if (lastRTT > MAX_RTT_SAMPLE_MICROSECONDS) { + lastRTT = MAX_RTT_SAMPLE_MICROSECONDS; } if (_ewmaRTT == -1) { @@ -129,7 +133,7 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { // (though congestion may be noticed a bit later) int rtt = _currentMinRTT; - int windowSizeDiff = _congestionWindowSize * (rtt - _baseRTT) / _baseRTT; + int64_t windowSizeDiff = (int64_t) _congestionWindowSize * (rtt - _baseRTT) / _baseRTT; if (_numACKs <= 2) { performRenoCongestionAvoidance(ack); @@ -169,7 +173,11 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { // we never allow the congestion window to be smaller than two packets static int VEGAS_CW_MIN_PACKETS = 2; - _congestionWindowSize = std::max(_congestionWindowSize, VEGAS_CW_MIN_PACKETS); + if (_congestionWindowSize < VEGAS_CW_MIN_PACKETS) { + _congestionWindowSize = VEGAS_CW_MIN_PACKETS; + } else if (_congestionWindowSize > udt::MAX_PACKETS_IN_FLIGHT) { + _congestionWindowSize = udt::MAX_PACKETS_IN_FLIGHT; + } // mark this as the last adjustment time _lastAdjustmentTime = p_high_resolution_clock::now(); From 7480d1e45dabc0aa2990375a663b4eeed0f0a644 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Oct 2016 15:48:48 -0700 Subject: [PATCH 52/65] switch default hifi address back to welcome --- libraries/networking/src/AddressManager.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/AddressManager.h b/libraries/networking/src/AddressManager.h index f158c3f081..1707132c08 100644 --- a/libraries/networking/src/AddressManager.h +++ b/libraries/networking/src/AddressManager.h @@ -23,7 +23,7 @@ #include "AccountManager.h" const QString HIFI_URL_SCHEME = "hifi"; -const QString DEFAULT_HIFI_ADDRESS = "hifi://ova-welcome"; +const QString DEFAULT_HIFI_ADDRESS = "hifi://welcome"; const QString SANDBOX_HIFI_ADDRESS = "hifi://localhost"; const QString INDEX_PATH = "/"; From 3c7bcf1127d036b230291c51876358c57c46ba9f Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Oct 2016 16:08:08 -0700 Subject: [PATCH 53/65] remove the max bandwidth option from DS settings while broken --- domain-server/resources/describe-settings.json | 9 --------- 1 file changed, 9 deletions(-) diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index 8a7d03ba75..4006673cad 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -863,15 +863,6 @@ "help": "The path to the directory assets are stored in.
If this path is relative, it will be relative to the application data directory.
If you change this path you will need to manually copy any existing assets from the previous directory.", "default": "", "advanced": true - }, - { - "name": "max_bandwidth", - "type": "double", - "label": "Max Bandwidth Per User", - "help": "The maximum upstream bandwidth each user can use (in Mb/s).", - "placeholder": "10.0", - "default": "", - "advanced": true } ] }, From 41d48b12bd78fd3f45372a8c3737c9d15314e58d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Oct 2016 10:26:18 -0700 Subject: [PATCH 54/65] add a comment clearing up onACK return --- libraries/networking/src/udt/CongestionControl.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index e56dd253db..53c5cdb3de 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -40,7 +40,10 @@ public: void setMaxBandwidth(int maxBandwidth); virtual void init() {} + + // return value specifies if connection should perform a fast re-transmit of ACK + 1 (used in TCP style congestion control) virtual bool onACK(SequenceNumber ackNum, p_high_resolution_clock::time_point receiveTime) { return false; } + virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) {} virtual void onTimeout() {} From 7567cff59068b300374b6338802714359c8f5f25 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Oct 2016 10:27:14 -0700 Subject: [PATCH 55/65] fix comment for ACK interval of 1 --- libraries/networking/src/udt/Connection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 2ed142f25b..889f7b8297 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -519,7 +519,7 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in // check if we need to send an ACK, according to CC params if (_congestionControl->_ackInterval == 1) { - // Using TCP Vegas + // using a congestion control that ACKs every packet (like TCP Vegas) sendACK(true); } else if (_congestionControl->_ackInterval > 0 && _packetsSinceACK >= _congestionControl->_ackInterval * _acksDuringSYN) { _acksDuringSYN++; From 13c38760b3c7c3f0746eed48e1f5def8fbc0d171 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Oct 2016 10:36:08 -0700 Subject: [PATCH 56/65] switch to wireSize, use time only, remove double public --- libraries/networking/src/udt/CongestionControl.h | 2 +- libraries/networking/src/udt/Connection.cpp | 10 +++++----- libraries/networking/src/udt/Connection.h | 4 ++-- libraries/networking/src/udt/SendQueue.h | 4 ++-- libraries/networking/src/udt/TCPVegasCC.cpp | 8 ++++---- libraries/networking/src/udt/TCPVegasCC.h | 6 ++---- 6 files changed, 16 insertions(+), 18 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 53c5cdb3de..3a93134a5e 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -51,7 +51,7 @@ public: virtual bool shouldACK2() { return true; } virtual bool shouldProbe() { return true; } - virtual void onPacketSent(int packetSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) {} + virtual void onPacketSent(int wireSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) {} protected: void setAckInterval(int ackInterval) { _ackInterval = ackInterval; } void setRTO(int rto) { _userDefinedRTO = true; _rto = rto; } diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 889f7b8297..a190fe3ddc 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -264,17 +264,17 @@ void Connection::sync() { } } -void Connection::recordSentPackets(int dataSize, int payloadSize, +void Connection::recordSentPackets(int wireSize, int payloadSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) { - _stats.recordSentPackets(payloadSize, dataSize); + _stats.recordSentPackets(payloadSize, wireSize); - _congestionControl->onPacketSent(dataSize, seqNum, timePoint); + _congestionControl->onPacketSent(wireSize, seqNum, timePoint); } -void Connection::recordRetransmission(int packetSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) { +void Connection::recordRetransmission(int wireSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) { _stats.record(ConnectionStats::Stats::Retransmission); - _congestionControl->onPacketSent(packetSize, seqNum, timePoint); + _congestionControl->onPacketSent(wireSize, seqNum, timePoint); } void Connection::sendACK(bool wasCausedBySyncTimeout) { diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 8df6d5c7a0..f94550426e 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -87,8 +87,8 @@ signals: void receiverHandshakeRequestComplete(const HifiSockAddr& sockAddr); private slots: - void recordSentPackets(int dataSize, int payloadSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint); - void recordRetransmission(int packetSize, SequenceNumber sequenceNumber, p_high_resolution_clock::time_point timePoint); + void recordSentPackets(int wireSize, int payloadSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint); + void recordRetransmission(int wireSize, SequenceNumber sequenceNumber, p_high_resolution_clock::time_point timePoint); void queueInactive(); void queueTimeout(); void queueShortCircuitLoss(quint32 sequenceNumber); diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index e7651dd934..6e17c5b9c6 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -77,8 +77,8 @@ public slots: void handshakeACK(SequenceNumber initialSequenceNumber); signals: - void packetSent(int dataSize, int payloadSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint); - void packetRetransmitted(int dataSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint); + void packetSent(int wireSize, int payloadSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint); + void packetRetransmitted(int wireSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint); void queueInactive(); diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 5c23be52d7..923bf869ae 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -33,7 +33,7 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r if (it != _sentPacketTimes.end()) { // calculate the RTT (receive time - time ACK sent) - int lastRTT = duration_cast(receiveTime - it->second.first).count(); + int lastRTT = duration_cast(receiveTime - it->second).count(); const int MAX_RTT_SAMPLE_MICROSECONDS = 10000000; @@ -90,7 +90,7 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r auto estimatedTimeout = _ewmaRTT + _rttVariance * 4; auto now = p_high_resolution_clock::now(); - auto sinceSend = duration_cast(now - it->second.first).count(); + auto sinceSend = duration_cast(now - it->second).count(); if (sinceSend >= estimatedTimeout) { // we've detected we need a fast re-transmit, send that back to the connection @@ -230,9 +230,9 @@ void TCPVegasCC::performRenoCongestionAvoidance(SequenceNumber ack) { } } -void TCPVegasCC::onPacketSent(int packetSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) { +void TCPVegasCC::onPacketSent(int wireSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) { if (_sentPacketTimes.find(seqNum) == _sentPacketTimes.end()) { - _sentPacketTimes[seqNum] = { timePoint, packetSize }; + _sentPacketTimes[seqNum] = timePoint; } } diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index 50990b19e8..bc6bc8c6c5 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -26,7 +26,6 @@ class TCPVegasCC : public CongestionControl { public: TCPVegasCC(); -public: virtual bool onACK(SequenceNumber ackNum, p_high_resolution_clock::time_point receiveTime) override; virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) override {}; virtual void onTimeout() override {}; @@ -35,7 +34,7 @@ public: virtual bool shouldACK2() override { return false; } virtual bool shouldProbe() override { return false; } - virtual void onPacketSent(int packetSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) override; + virtual void onPacketSent(int wireSize, SequenceNumber seqNum, p_high_resolution_clock::time_point timePoint) override; protected: virtual void performCongestionAvoidance(SequenceNumber ack); @@ -44,8 +43,7 @@ private: bool isCongestionWindowLimited(); void performRenoCongestionAvoidance(SequenceNumber ack); - using TimeSizePair = std::pair; - using PacketTimeList = std::map; + using PacketTimeList = std::map; PacketTimeList _sentPacketTimes; // Map of sequence numbers to sent time p_high_resolution_clock::time_point _lastAdjustmentTime; // Time of last congestion control adjustment From 10414437961ab20bb55f3595d5a3d242aa5f419a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Oct 2016 11:20:30 -0700 Subject: [PATCH 57/65] use __FUNCTION__ and clearup zero us RTT comment --- libraries/networking/src/udt/TCPVegasCC.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 923bf869ae..b8dc4910f6 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -38,12 +38,13 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r const int MAX_RTT_SAMPLE_MICROSECONDS = 10000000; if (lastRTT < 0) { - Q_ASSERT_X(false, "TCPVegasCC::onACK", "calculated an RTT that is not > 0"); + Q_ASSERT_X(false, __FUNCTION__, "calculated an RTT that is not > 0"); return false; } else if (lastRTT == 0) { - // we do not allow a 0 microsecond RTT + // we do not allow a zero microsecond RTT (as per the UNIX kernel implementation of TCP Vegas) lastRTT = 1; } else if (lastRTT > MAX_RTT_SAMPLE_MICROSECONDS) { + // we cap the lastRTT to MAX_RTT_SAMPLE_MICROSECONDS to avoid overflows in window size calculations lastRTT = MAX_RTT_SAMPLE_MICROSECONDS; } From d5a450ad307033745e562b2bc6b2997ad591c70a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Oct 2016 11:24:08 -0700 Subject: [PATCH 58/65] add comment about Jacobson's formula for RTT estimation --- libraries/networking/src/udt/TCPVegasCC.cpp | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index b8dc4910f6..a3b6fdbdb6 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -49,15 +49,26 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r } if (_ewmaRTT == -1) { + // first RTT sample - set _ewmaRTT to the value and set the variance to half the value _ewmaRTT = lastRTT; _rttVariance = lastRTT / 2; } else { - static const int RTT_ESTIMATION_ALPHA_NUMERATOR = 8; - static const int RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR = 4; + // This updates the RTT using exponential weighted moving average + // This is the Jacobson's forumla for RTT estimation + // http://www.mathcs.emory.edu/~cheung/Courses/455/Syllabus/7-transport/Jacobson-88.pdf - _ewmaRTT = (_ewmaRTT * (RTT_ESTIMATION_ALPHA_NUMERATOR - 1) + lastRTT) / RTT_ESTIMATION_ALPHA_NUMERATOR; - _rttVariance = (_rttVariance * (RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR - 1) - + abs(lastRTT - _ewmaRTT)) / RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR; + // Estimated RTT = (1 - x)(estimatedRTT) + (x)(sampleRTT) + // (where x = 0.125 via Jacobson) + + // Deviation = (1 - x)(deviation) + x |sampleRTT - estimatedRTT| + // (where x = 0.25 via Jacobson) + + static const int RTT_ESTIMATION_ALPHA = 8; + static const int RTT_ESTIMATION_VARIANCE_ALPHA = 4; + + _ewmaRTT = (_ewmaRTT * (RTT_ESTIMATION_ALPHA - 1) + lastRTT) / RTT_ESTIMATION_ALPHA; + _rttVariance = (_rttVariance * (RTT_ESTIMATION_VARIANCE_ALPHA- 1) + + abs(lastRTT - _ewmaRTT)) / RTT_ESTIMATION_VARIANCE_ALPHA; } // add 1 to the number of ACKs during this RTT From b817e72976688b0ccdd678073de4b8ac3b41c7d9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Oct 2016 11:29:02 -0700 Subject: [PATCH 59/65] comment clarity and readability improvements --- libraries/networking/src/udt/TCPVegasCC.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index a3b6fdbdb6..ba5af2e618 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -89,11 +89,10 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r _sentPacketTimes.erase(it); } - - // we check if we need a fast re-transmit if this is a duplicate ACK or if this is the first or second ACK - // after a previous fast re-transmit ++_numACKSinceFastRetransmit; + // perform the fast re-transmit check if this is a duplicate ACK or if this is the first or second ACK + // after a previous fast re-transmit if (ack == previousAck || _numACKSinceFastRetransmit < 3) { // we may need to re-send ackNum + 1 if it has been more than our estimated timeout since it was sent @@ -113,7 +112,10 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r // if this is the 3rd duplicate ACK, we fallback to Reno's fast re-transmit static const int RENO_FAST_RETRANSMIT_DUPLICATE_COUNT = 3; - if (ack == previousAck && ++_duplicateACKCount == RENO_FAST_RETRANSMIT_DUPLICATE_COUNT) { + + ++_duplicateACKCount; + + if (ack == previousAck && _duplicateACKCount == RENO_FAST_RETRANSMIT_DUPLICATE_COUNT) { // break out of slow start, we just hit loss _slowStart = false; @@ -137,10 +139,12 @@ void TCPVegasCC::performCongestionAvoidance(udt::SequenceNumber ack) { static int VEGAS_BETA_SEGMENTS = 6; static int VEGAS_GAMMA_SEGMENTS = 1; + // http://pages.cs.wisc.edu/~akella/CS740/S08/740-Papers/BOP94.pdf // Use the Vegas algorithm to see if we should // increase or decrease the congestion window size, and by how much - // Grab the minimum RTT seen during the last RTT + // Grab the minimum RTT seen during the last RTT (since the last performed congestion avoidance) + // Taking the min avoids the effects of delayed ACKs // (though congestion may be noticed a bit later) int rtt = _currentMinRTT; From 974cccf5198545277e07a25e9fa4f3062f75fb42 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Oct 2016 11:42:21 -0700 Subject: [PATCH 60/65] cleanup initialization of min RTT member variables --- libraries/networking/src/udt/TCPVegasCC.cpp | 5 +++++ libraries/networking/src/udt/TCPVegasCC.h | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index ba5af2e618..18bff57cfe 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -22,6 +22,11 @@ TCPVegasCC::TCPVegasCC() { _congestionWindowSize = 2; setAckInterval(1); // TCP sends an ACK for every packet received + + // set our minimum RTT variables to the maximum possible value + // we can't do this as a member initializer until our VS has support for constexpr + _currentMinRTT = std::numeric_limits::max(); + _baseRTT = std::numeric_limits::max(); } bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point receiveTime) { diff --git a/libraries/networking/src/udt/TCPVegasCC.h b/libraries/networking/src/udt/TCPVegasCC.h index bc6bc8c6c5..862ea36d8f 100644 --- a/libraries/networking/src/udt/TCPVegasCC.h +++ b/libraries/networking/src/udt/TCPVegasCC.h @@ -52,14 +52,14 @@ private: SequenceNumber _lastACK; // Sequence number of last packet that was ACKed - int _numACKSinceFastRetransmit { 3 }; // Number of ACKs received since last fast re-transmit + int _numACKSinceFastRetransmit { 3 }; // Number of ACKs received since fast re-transmit, default avoids immediate re-transmit - int _currentMinRTT { 0x7FFFFFFF }; // Current RTT, in microseconds - int _baseRTT { 0x7FFFFFFF }; // Lowest RTT during connection, in microseconds + int _currentMinRTT; // Current min RTT during last RTT (since last congestion avoidance check), in microseconds + int _baseRTT; // Lowest RTT during connection, in microseconds int _ewmaRTT { -1 }; // Exponential weighted moving average RTT int _rttVariance { 0 }; // Variance in collected RTT values - int _numACKs { 0 }; // Holds the number of ACKs received during the last RTT + int _numACKs { 0 }; // Number of ACKs received during the last RTT (since last performed congestion avoidance) int _ackAICount { 0 }; // Counter for number of ACKs received for Reno additive increase int _duplicateACKCount { 0 }; // Counter for duplicate ACKs received From 71de9701e0fff848e9e6dddc1bfc111dbc36cbaf Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Oct 2016 11:58:26 -0700 Subject: [PATCH 61/65] add some additional comments to reno congestion avoidance --- libraries/networking/src/udt/TCPVegasCC.cpp | 25 ++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 18bff57cfe..757cf12fca 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -225,22 +225,37 @@ void TCPVegasCC::performRenoCongestionAvoidance(SequenceNumber ack) { int numAcked = _numACKs; - // In "safe" area, increase. if (_slowStart) { - int congestionWindow = std::min(_congestionWindowSize + numAcked, udt::MAX_PACKETS_IN_FLIGHT); - numAcked -= congestionWindow - _congestionWindowSize; + // while in slow start we grow the congestion window by the number of ACKed packets + // allowing it to grow as high as the slow start threshold + int congestionWindow = _congestionWindowSize + numAcked; + + if (_congestionWindowSize > udt::MAX_PACKETS_IN_FLIGHT) { + // we're done with slow start, set the congestion window to the slow start threshold + _congestionWindowSize = udt::MAX_PACKETS_IN_FLIGHT; + + // figure out how many left over ACKs we should apply using the regular reno congestion avoidance + numAcked -= congestionWindow - _congestionWindowSize; + } else { + _congestionWindowSize = congestionWindow; + numAcked = 0; + } } + // grab the size of the window prior to reno additive increase int preAIWindowSize = _congestionWindowSize; if (numAcked > 0) { - // In dangerous area, increase slowly. - // If credits accumulated at a higher w, apply them gently now. + // Once we are out of slow start, we use additive increase to grow the window slowly. + // We grow the congestion window by a single packet everytime the entire congestion window is sent. + + // If credits accumulated at a higher preAIWindowSize, apply them gently now. if (_ackAICount >= preAIWindowSize) { _ackAICount = 0; ++_congestionWindowSize; } + // increase the window size by (1 / window size) for every ACK received _ackAICount += numAcked; if (_ackAICount >= preAIWindowSize) { int delta = _ackAICount / preAIWindowSize; From 214d2e4c911b70fc0112157aa7631cc7c0ad1262 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Oct 2016 11:59:58 -0700 Subject: [PATCH 62/65] break slow start on vegas timeout loss --- libraries/networking/src/udt/TCPVegasCC.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 757cf12fca..454962c016 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -109,8 +109,13 @@ bool TCPVegasCC::onACK(SequenceNumber ack, p_high_resolution_clock::time_point r auto sinceSend = duration_cast(now - it->second).count(); if (sinceSend >= estimatedTimeout) { - // we've detected we need a fast re-transmit, send that back to the connection + // break out of slow start, we've decided this is loss + _slowStart = false; + + // reset the fast re-transmit counter _numACKSinceFastRetransmit = 0; + + // return true so the caller knows we needed a fast re-transmit return true; } } From ddce913095f4d893938863f1692b15a3220852ef Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Oct 2016 15:53:23 -0700 Subject: [PATCH 63/65] cleanup math for leftover ACKs --- libraries/networking/src/udt/TCPVegasCC.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 454962c016..240a84b376 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -240,7 +240,7 @@ void TCPVegasCC::performRenoCongestionAvoidance(SequenceNumber ack) { _congestionWindowSize = udt::MAX_PACKETS_IN_FLIGHT; // figure out how many left over ACKs we should apply using the regular reno congestion avoidance - numAcked -= congestionWindow - _congestionWindowSize; + numAcked = congestionWindow - udt::MAX_PACKETS_IN_FLIGHT; } else { _congestionWindowSize = congestionWindow; numAcked = 0; From 5bd6cc9fac69f910d48ad8708d2c14848411df0c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Oct 2016 16:01:03 -0700 Subject: [PATCH 64/65] fix comparison for reno slow start --- libraries/networking/src/udt/TCPVegasCC.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index 240a84b376..c811b2d2ea 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -235,7 +235,7 @@ void TCPVegasCC::performRenoCongestionAvoidance(SequenceNumber ack) { // allowing it to grow as high as the slow start threshold int congestionWindow = _congestionWindowSize + numAcked; - if (_congestionWindowSize > udt::MAX_PACKETS_IN_FLIGHT) { + if (congestionWindow > udt::MAX_PACKETS_IN_FLIGHT) { // we're done with slow start, set the congestion window to the slow start threshold _congestionWindowSize = udt::MAX_PACKETS_IN_FLIGHT; From 22e0e649b19573a6e166d627ec3156279a65a309 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Oct 2016 16:16:09 -0700 Subject: [PATCH 65/65] add a comment for ack count calculations --- libraries/networking/src/udt/TCPVegasCC.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/networking/src/udt/TCPVegasCC.cpp b/libraries/networking/src/udt/TCPVegasCC.cpp index c811b2d2ea..5738ea8421 100644 --- a/libraries/networking/src/udt/TCPVegasCC.cpp +++ b/libraries/networking/src/udt/TCPVegasCC.cpp @@ -263,6 +263,9 @@ void TCPVegasCC::performRenoCongestionAvoidance(SequenceNumber ack) { // increase the window size by (1 / window size) for every ACK received _ackAICount += numAcked; if (_ackAICount >= preAIWindowSize) { + // when _ackAICount % preAIWindowSize == 0 then _ackAICount is 0 + // when _ackAICount % preAIWindowSize != 0 then _ackAICount is _ackAICount - (_ackAICount % preAIWindowSize) + int delta = _ackAICount / preAIWindowSize; _ackAICount -= delta * preAIWindowSize;