From dce1fc68553a6269433b6b4f46d810cd7ced035a Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 23 Jul 2015 11:18:52 -0700 Subject: [PATCH 001/242] SendQueue first draft --- libraries/networking/src/udt/SendQueue.cpp | 82 ++++++++++++++++++++++ libraries/networking/src/udt/SendQueue.h | 68 ++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 libraries/networking/src/udt/SendQueue.cpp create mode 100644 libraries/networking/src/udt/SendQueue.h diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp new file mode 100644 index 0000000000..af19162bb8 --- /dev/null +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -0,0 +1,82 @@ +// +// SendQueue.cpp +// libraries/networking/src/udt +// +// Created by Clement on 7/21/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "SendQueue.h" + +#include +#include + +#include + +namespace udt { + +std::unique_ptr SendQueue::create() { + return std::unique_ptr(new SendQueue()); +} + +SendQueue::SendQueue() { + _sendTimer.reset(new QTimer(this)); + _sendTimer->setSingleShot(true); + QObject::connect(_sendTimer.get(), &QTimer::timeout, this, &SendQueue::sendNextPacket); + + _packetSendPeriod = DEFAULT_SEND_PERIOD; + _lastSendTimestamp = 0; +} + +void SendQueue::queuePacket(std::unique_ptr packet) { + QWriteLocker locker(&_packetsLock); + _packets.push_back(std::move(packet)); +} + +void SendQueue::start() { + // We need to make sure this is called on the right thread + if (thread() != QThread::currentThread()) { + QMetaObject::invokeMethod(this, "start", Qt::QueuedConnection); + } + // This will send a packet and fire the send timer + sendNextPacket(); +} + +void SendQueue::stop() { + // We need to make sure this is called on the right thread + if (thread() != QThread::currentThread()) { + QMetaObject::invokeMethod(this, "stop", Qt::QueuedConnection); + } + // Stopping the timer will stop the sending of packets + _sendTimer->stop(); +} + +void SendQueue::sendNextPacket() { + // Record timing + auto sendTime = msecTimestampNow(); // msec + _lastSendTimestamp = sendTime; + // TODO send packet + + // Insert the packet we have just sent in the sent list + _sentPackets[_nextPacket->readSequenceNumber()].swap(_nextPacket); + Q_ASSERT(!_nextPacket); // There should be no packet where we inserted + + { // Grab next packet to be sent + QWriteLocker locker(&_packetsLock); + _nextPacket.swap(_packets.front()); + _packets.pop_front(); + } + + // How long before next packet send + auto timeToSleep = (sendTime + _packetSendPeriod) - msecTimestampNow(); // msec + if (timeToSleep > 0) { + _sendTimer->start(timeToSleep); + } else { + _sendTimer->start(0); + } +} + +} diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h new file mode 100644 index 0000000000..9767cfe7aa --- /dev/null +++ b/libraries/networking/src/udt/SendQueue.h @@ -0,0 +1,68 @@ +// +// SendQueue.h +// libraries/networking/src/udt +// +// Created by Clement on 7/21/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_SendQueue_h +#define hifi_SendQueue_h + +#include +#include + +#include +#include + +#include "Packet.h" + +class QTimer; + +namespace udt { + +class SendQueue : public QObject { + Q_OBJECT + +public: + static const int DEFAULT_SEND_PERIOD = 16; // msec + + static std::unique_ptr create(); + + void queuePacket(std::unique_ptr packet); + int getQueueSize() const { return _packets.size(); } + + quint64 getLastSendTimestamp() const { return _lastSendTimestamp; } + + int getPacketSendPeriod() const { return _packetSendPeriod; } + void setPacketSendPeriod(int newPeriod) { _packetSendPeriod = newPeriod; } + +public slots: + void start(); + void stop(); + +private slots: + void sendNextPacket(); + +private: + SendQueue(); + SendQueue(SendQueue& other) = delete; + SendQueue(SendQueue&& other) = delete; + + QReadWriteLock _packetsLock; // Protects the packets to be sent list. + std::list> _packets; // List of packets to be sent + std::unique_ptr _nextPacket; + + std::unique_ptr _sendTimer; // Send timer + std::atomic _packetSendPeriod; // Interval between two packet send envent in msec + std::atomic _lastSendTimestamp; // Record last time of packet departure + + std::unordered_map> _sentPackets; // Packets waiting for ACK. +}; + +} + +#endif // hifi_SendQueue_h From c5901c3f6b2ac0a0b23129551f4ce375f9aedbd3 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 23 Jul 2015 15:00:32 -0700 Subject: [PATCH 002/242] Sequence Numbers helper class --- libraries/networking/src/udt/SeqNum.cpp | 38 +++++++++++ libraries/networking/src/udt/SeqNum.h | 84 +++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 libraries/networking/src/udt/SeqNum.cpp create mode 100644 libraries/networking/src/udt/SeqNum.h diff --git a/libraries/networking/src/udt/SeqNum.cpp b/libraries/networking/src/udt/SeqNum.cpp new file mode 100644 index 0000000000..29347a9249 --- /dev/null +++ b/libraries/networking/src/udt/SeqNum.cpp @@ -0,0 +1,38 @@ +// +// SeqNum.cpp +// +// +// Created by Clement on 7/23/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "SeqNum.h" + +namespace udt { + +int seqcmp(const SeqNum& seq1, const SeqNum& seq2) { + return (glm::abs(seq1._value - seq2._value) < SeqNum::THRESHOLD) ? (seq1._value - seq2._value) + : (seq2._value - seq1._value); +} + +int seqlen(const SeqNum& seq1, const SeqNum& seq2) { + return (seq1._value <= seq2._value) ? (seq2._value - seq1._value + 1) + : (seq2._value - seq1._value + SeqNum::MAX + 2); +} + +int seqoff(const SeqNum& seq1, const SeqNum& seq2) { + if (glm::abs(seq1._value - seq2._value) < SeqNum::THRESHOLD) { + return seq2._value - seq1._value; + } + + if (seq1._value < seq2._value) { + return seq2._value - seq1._value - SeqNum::MAX - 1; + } + + return seq2._value - seq1._value + SeqNum::MAX + 1; +} + +} diff --git a/libraries/networking/src/udt/SeqNum.h b/libraries/networking/src/udt/SeqNum.h new file mode 100644 index 0000000000..058ddcb6cc --- /dev/null +++ b/libraries/networking/src/udt/SeqNum.h @@ -0,0 +1,84 @@ +// +// SeqNum.h +// libraries/networking/src/udt +// +// Created by Clement on 7/23/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_SeqNum_h +#define hifi_SeqNum_h + +#include + +namespace udt { + +class SeqNum { +public: + // Base type of sequence numbers + using Type = uint32_t; + + // Values are for 29 bit SeqNum + static const Type THRESHOLD = 0x0FFFFFFF; // threshold for comparing sequence numbers + static const Type MAX = 0x1FFFFFFF; // maximum sequence number used in UDT + + SeqNum() : _value(0) {} + + // Only explicit conversions + explicit SeqNum(Type value) { *this = value; } + explicit operator Type() { return _value; } + + inline SeqNum& operator++() { + _value = (_value == MAX) ? 0 : ++_value; + return *this; + } + inline SeqNum& operator--() { + _value = (_value == 0) ? MAX : --_value; + return *this; + } + inline SeqNum operator++(int) { + SeqNum before = *this; + (*this)++; + return before; + } + inline SeqNum operator--(int) { + SeqNum before = *this; + (*this)--; + return before; + } + + inline SeqNum& operator=(Type value) { + _value = (value <= MAX) ? value : MAX; + return *this; + } + inline SeqNum& operator=(SeqNum& other) { + _value = other._value; + return *this; + } + inline SeqNum& operator+=(Type inc) { + _value = (_value + inc > MAX) ? _value + inc - (MAX + 1) : _value + inc; + return *this; + } + inline SeqNum& operator-=(Type dec) { + _value = (_value < dec) ? MAX - (dec - _value + 1) : _value - dec; + return *this; + } + + friend int seqcmp(const SeqNum& seq1, const SeqNum& seq2); + friend int seqlen(const SeqNum& seq1, const SeqNum& seq2); + friend int seqoff(const SeqNum& seq1, const SeqNum& seq2); + +private: + Type _value; +}; + +int seqcmp(const SeqNum& seq1, const SeqNum& seq2); +int seqlen(const SeqNum& seq1, const SeqNum& seq2); +int seqoff(const SeqNum& seq1, const SeqNum& seq2); + +} + +#endif // hifi_SeqNum_h From 82545df26615599499d252bf8f367bf95f356c23 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 23 Jul 2015 15:01:46 -0700 Subject: [PATCH 003/242] Added msecTimestampNow as usecTimestampNow narrowing --- libraries/shared/src/SharedUtil.cpp | 4 ++++ libraries/shared/src/SharedUtil.h | 1 + 2 files changed, 5 insertions(+) diff --git a/libraries/shared/src/SharedUtil.cpp b/libraries/shared/src/SharedUtil.cpp index f78c8c47e0..d73e8f446e 100644 --- a/libraries/shared/src/SharedUtil.cpp +++ b/libraries/shared/src/SharedUtil.cpp @@ -39,6 +39,10 @@ void usecTimestampNowForceClockSkew(int clockSkew) { ::usecTimestampNowAdjust = clockSkew; } +quint64 msecTimestampNow(bool wantDebug) { + return usecTimestampNow() / 1000; +} + quint64 usecTimestampNow(bool wantDebug) { static bool usecTimestampNowIsInitialized = false; static qint64 TIME_REFERENCE = 0; // in usec diff --git a/libraries/shared/src/SharedUtil.h b/libraries/shared/src/SharedUtil.h index 17a32e7910..a60c0defd0 100644 --- a/libraries/shared/src/SharedUtil.h +++ b/libraries/shared/src/SharedUtil.h @@ -64,6 +64,7 @@ inline bool operator!=(const xColor& lhs, const xColor& rhs) // Use a custom User-Agent to avoid ModSecurity filtering, e.g. by hosting providers. const QByteArray HIGH_FIDELITY_USER_AGENT = "Mozilla/5.0 (HighFidelityInterface)"; +quint64 msecTimestampNow(bool wantDebug = false); quint64 usecTimestampNow(bool wantDebug = false); void usecTimestampNowForceClockSkew(int clockSkew); From 9a51aef712e556fa9ccda96ec2c6272a84ae2476 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 23 Jul 2015 15:02:34 -0700 Subject: [PATCH 004/242] Added congestion control base classes --- .../networking/src/udt/CongestionControl.cpp | 148 ++++++++++++++++++ .../networking/src/udt/CongestionControl.h | 115 ++++++++++++++ 2 files changed, 263 insertions(+) create mode 100644 libraries/networking/src/udt/CongestionControl.cpp create mode 100644 libraries/networking/src/udt/CongestionControl.h diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp new file mode 100644 index 0000000000..75ebc0929c --- /dev/null +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -0,0 +1,148 @@ +// +// CongestionControl.cpp +// +// +// Created by Clement on 7/23/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "CongestionControl.h" + +#include + +namespace udt { + +void UdtCC::init() { + _rcInterval = _synInterval; + _lastRCTime = usecTimestampNow(); + setAckTimer(_rcInterval); + + _lastAck = _sendCurrSeqNum; + _lastDecSeq = SeqNum::MAX; + + _congestionWindowSize = 16.0; + _packetSendPeriod = 1.0; +} + +void UdtCC::onACK(SeqNum ackNum) { + int64_t B = 0; + double inc = 0; + // Note: 1/24/2012 + // The minimum increase parameter is increased from "1.0 / _mss" to 0.01 + // because the original was too small and caused sending rate to stay at low level + // for long time. + const double min_inc = 0.01; + + uint64_t currtime = usecTimestampNow(); + if (currtime - _lastRCTime < (uint64_t)_rcInterval) { + return; + } + + _lastRCTime = currtime; + + if (_slowStart) { + _congestionWindowSize += seqlen(_lastAck, ackNum); + _lastAck = ackNum; + + if (_congestionWindowSize > _maxCongestionWindowSize) { + _slowStart = false; + if (_recvieveRate > 0) { + _packetSendPeriod = 1000000.0 / _recvieveRate; + } else { + _packetSendPeriod = (_rtt + _rcInterval) / _congestionWindowSize; + } + } + } else { + _congestionWindowSize = _recvieveRate / 1000000.0 * (_rtt + _rcInterval) + 16; + } + + // During Slow Start, no rate increase + if (_slowStart) { + return; + } + + if (_loss) { + _loss = false; + return; + } + + B = (int64_t)(_bandwidth - 1000000.0 / _packetSendPeriod); + if ((_packetSendPeriod > _lastDecPeriod) && ((_bandwidth / 9) < B)) { + B = _bandwidth / 9; + } + if (B <= 0) { + inc = min_inc; + } else { + // inc = max(10 ^ ceil(log10( B * MSS * 8 ) * Beta / MSS, 1/MSS) + // Beta = 1.5 * 10^(-6) + + inc = pow(10.0, ceil(log10(B * _mss * 8.0))) * 0.0000015 / _mss; + + if (inc < min_inc) { + inc = min_inc; + } + } + + _packetSendPeriod = (_packetSendPeriod * _rcInterval) / (_packetSendPeriod * inc + _rcInterval); +} + +void UdtCC::onLoss(const std::vector& losslist) { + //Slow Start stopped, if it hasn't yet + if (_slowStart) { + _slowStart = false; + if (_recvieveRate > 0) { + // Set the sending rate to the receiving rate. + _packetSendPeriod = 1000000.0 / _recvieveRate; + return; + } + // If no receiving rate is observed, we have to compute the sending + // rate according to the current window size, and decrease it + // using the method below. + _packetSendPeriod = _congestionWindowSize / (_rtt + _rcInterval); + } + + _loss = true; + + if (seqcmp(losslist[0], _lastDecSeq) > 0) { + _lastDecPeriod = _packetSendPeriod; + _packetSendPeriod = ceil(_packetSendPeriod * 1.125); + + _avgNAKNum = (int)ceil(_avgNAKNum * 0.875 + _nakCount * 0.125); + _nakCount = 1; + _decCount = 1; + + _lastDecSeq = _sendCurrSeqNum; + + // remove global synchronization using randomization + srand((uint32_t)_lastDecSeq); + _decRandom = (int)ceil(_avgNAKNum * (double(rand()) / RAND_MAX)); + if (_decRandom < 1) + _decRandom = 1; + } else if ((_decCount ++ < 5) && (0 == (++ _nakCount % _decRandom))) { + // 0.875^5 = 0.51, rate should not be decreased by more than half within a congestion period + _packetSendPeriod = ceil(_packetSendPeriod * 1.125); + _lastDecSeq = _sendCurrSeqNum; + } +} + +void UdtCC::onTimeout() { + if (_slowStart) { + _slowStart = false; + if (_recvieveRate > 0) { + _packetSendPeriod = 1000000.0 / _recvieveRate; + } else { + _packetSendPeriod = _congestionWindowSize / (_rtt + _rcInterval); + } + } else { + /* + _lastDecPeriod = _packetSendPeriod; + _packetSendPeriod = ceil(_packetSendPeriod * 2); + _lastDecSeq = _lastAck; + */ + } +} + +} \ No newline at end of file diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h new file mode 100644 index 0000000000..fdf8803ec0 --- /dev/null +++ b/libraries/networking/src/udt/CongestionControl.h @@ -0,0 +1,115 @@ +// +// CongestionControl.h +// +// +// Created by Clement on 7/23/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_CongestionControl_h +#define hifi_CongestionControl_h + +#include + +#include "SeqNum.h" + +namespace udt { + +class Packet; + +class CongestionControl { +public: + static const int32_t DEFAULT_SYN_INTERVAL = 10000; // 10 ms + + CongestionControl() {} + virtual ~CongestionControl() {} + + virtual void init() {} + virtual void close() {} + virtual void onAck(SeqNum ackNum) {} + virtual void onLoss(const std::vector& lossList) {} + virtual void onPacketSent(const Packet& packet) {} + virtual void onPacketReceived(const Packet& packet) {} + +protected: + void setAckTimer(int syn) { _ackPeriod = (syn > _synInterval) ? _synInterval : syn; } + void setAckInterval(int interval) { _ackInterval = interval; } + void setRto(int rto) { _userDefinedRto = true; _rto = rto; } + + int32_t _synInterval = DEFAULT_SYN_INTERVAL; // UDT constant parameter, SYN + + double _packetSendPeriod = 1.0; // Packet sending period, in microseconds + double _congestionWindowSize = 16.0; // Congestion window size, in packets + + int _bandwidth = 0; // estimated bandwidth, packets per second + double _maxCongestionWindowSize = 0.0; // maximum cwnd size, in packets + + int _mss = 0; // Maximum Packet Size, including all packet headers + SeqNum _sendCurrSeqNum; // current maximum seq num sent out + int _recvieveRate = 0; // packet arrive rate at receiver side, packets per second + int _rtt = 0; // current estimated RTT, microsecond + +private: + CongestionControl(const CongestionControl& other) = delete; + CongestionControl& operator=(const CongestionControl& other) = delete; + + void setMss(int mss) { _mss = mss; } + void setMaxCongestionWindowSize(int window) { _maxCongestionWindowSize = window; } + void setBandwidth(int bandwidth) { _bandwidth = bandwidth; } + void setSndCurrSeqNum(SeqNum seqNum) { _sendCurrSeqNum = seqNum; } + void setRcvRate(int rate) { _recvieveRate = rate; } + void setRtt(int rtt) { _rtt = rtt; } + + int _ackPeriod = 0; // Periodical timer to send an ACK, in milliseconds + int _ackInterval = 0; // How many packets to send one ACK, in packets + + bool _userDefinedRto = false; // if the RTO value is defined by users + int _rto = -1; // RTO value, microseconds +}; + + +class CongestionControlVirtualFactory { +public: + virtual ~CongestionControlVirtualFactory() {} + + virtual std::unique_ptr create() = 0; +}; + +template class CongestionControlFactory: public CongestionControlVirtualFactory +{ +public: + virtual ~CongestionControlFactory() {} + + virtual std::unique_ptr create() { return std::unique_ptr(new T()); } +}; + +class UdtCC: public CongestionControl { +public: + UdtCC() {} + +public: + virtual void init(); + virtual void onACK(SeqNum ackNum); + virtual void onLoss(const std::vector& lossList); + virtual void onTimeout(); + +private: + int _rcInterval = 0; // UDT Rate control interval + uint64_t _lastRCTime = 0; // last rate increase time + bool _slowStart = true; // if in slow start phase + SeqNum _lastAck; // last ACKed seq num + bool _loss = false; // if loss happened since last rate increase + SeqNum _lastDecSeq; // max pkt seq num sent out when last decrease happened + double _lastDecPeriod = 1; // value of pktsndperiod when last decrease happened + int _nakCount = 0; // NAK counter + int _decRandom = 1; // random threshold on decrease by number of loss events + int _avgNAKNum = 0; // average number of NAKs per congestion + int _decCount = 0; // number of decreases in a congestion epoch +}; + +} + +#endif // hifi_CongestionControl_h \ No newline at end of file From d4504ae6c83cb66c29f9d810cf8a994681ff30bc Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 24 Jul 2015 12:21:20 -0700 Subject: [PATCH 005/242] Make SeqNum hash friendly --- libraries/networking/src/udt/SeqNum.h | 31 +++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/libraries/networking/src/udt/SeqNum.h b/libraries/networking/src/udt/SeqNum.h index 058ddcb6cc..ad5a6c7c21 100644 --- a/libraries/networking/src/udt/SeqNum.h +++ b/libraries/networking/src/udt/SeqNum.h @@ -12,6 +12,8 @@ #ifndef hifi_SeqNum_h #define hifi_SeqNum_h +#include + #include namespace udt { @@ -26,9 +28,11 @@ public: static const Type MAX = 0x1FFFFFFF; // maximum sequence number used in UDT SeqNum() : _value(0) {} + SeqNum(const SeqNum& other) : _value(other._value) {} // Only explicit conversions - explicit SeqNum(Type value) { *this = value; } + explicit SeqNum(char* value) { _value = (*reinterpret_cast(value)) & MAX; } + explicit SeqNum(Type value) { _value = (value <= MAX) ? value : MAX; } explicit operator Type() { return _value; } inline SeqNum& operator++() { @@ -50,11 +54,7 @@ public: return before; } - inline SeqNum& operator=(Type value) { - _value = (value <= MAX) ? value : MAX; - return *this; - } - inline SeqNum& operator=(SeqNum& other) { + inline SeqNum& operator=(const SeqNum& other) { _value = other._value; return *this; } @@ -67,18 +67,37 @@ public: return *this; } + inline bool operator==(const SeqNum& other) const { + return _value == other._value; + } + inline bool operator!=(const SeqNum& other) const { + return _value != other._value; + } + friend int seqcmp(const SeqNum& seq1, const SeqNum& seq2); friend int seqlen(const SeqNum& seq1, const SeqNum& seq2); friend int seqoff(const SeqNum& seq1, const SeqNum& seq2); private: Type _value; + + friend struct std::hash; }; int seqcmp(const SeqNum& seq1, const SeqNum& seq2); int seqlen(const SeqNum& seq1, const SeqNum& seq2); int seqoff(const SeqNum& seq1, const SeqNum& seq2); + +} +namespace std { + +template<> struct hash { + size_t operator()(const udt::SeqNum& seqNum) const { + return std::hash()(seqNum._value); + } +}; + } #endif // hifi_SeqNum_h From 797f74f3d94699759bb8f888cb32eb49165ea9f2 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 24 Jul 2015 12:27:40 -0700 Subject: [PATCH 006/242] Use SeqNum --- libraries/networking/src/Node.h | 2 +- libraries/networking/src/udt/CongestionControl.cpp | 2 +- libraries/networking/src/udt/Packet.cpp | 13 ++++--------- libraries/networking/src/udt/Packet.h | 6 ++++-- libraries/networking/src/udt/SeqNum.h | 4 ++-- libraries/networking/src/udt/Socket.h | 2 +- 6 files changed, 13 insertions(+), 16 deletions(-) diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index 178146c948..411e9982c5 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -85,7 +85,7 @@ private: bool _canAdjustLocks; bool _canRez; - std::map _lastSequenceNumbers; + std::map _lastSequenceNumbers; }; typedef QSharedPointer SharedNodePointer; diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 75ebc0929c..fbb172ecee 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -21,7 +21,7 @@ void UdtCC::init() { setAckTimer(_rcInterval); _lastAck = _sendCurrSeqNum; - _lastDecSeq = SeqNum::MAX; + _lastDecSeq = SeqNum{ SeqNum::MAX }; _congestionWindowSize = 16.0; _packetSendPeriod = 1.0; diff --git a/libraries/networking/src/udt/Packet.cpp b/libraries/networking/src/udt/Packet.cpp index 47ea6ffef3..ebfea3742e 100644 --- a/libraries/networking/src/udt/Packet.cpp +++ b/libraries/networking/src/udt/Packet.cpp @@ -65,7 +65,7 @@ Packet::Packet(qint64 size, bool isReliable, bool isPartOfMessage) : adjustPayloadStartAndCapacity(); // set the UDT header to default values - writeSequenceNumber(0); + writeSequenceNumber(SeqNum()); } Packet::Packet(std::unique_ptr data, qint64 size, const HifiSockAddr& senderSockAddr) : @@ -142,18 +142,13 @@ void Packet::readIsPartOfMessage() { void Packet::readSequenceNumber() { SequenceNumberAndBitField seqNumBitField = *reinterpret_cast(_packet.get()); - _sequenceNumber = (seqNumBitField & ~BIT_FIELD_MASK); // Remove the bit field + _sequenceNumber = SeqNum{ seqNumBitField & ~BIT_FIELD_MASK }; // Remove the bit field } -static const uint32_t MAX_SEQUENCE_NUMBER = UINT32_MAX >> BIT_FIELD_LENGTH; - -void Packet::writeSequenceNumber(SequenceNumber sequenceNumber) { - // make sure this is a sequence number <= 29 bit unsigned max (536,870,911) - Q_ASSERT(sequenceNumber <= MAX_SEQUENCE_NUMBER); - +void Packet::writeSequenceNumber(SeqNum seqNum) { // grab pointer to current SequenceNumberAndBitField SequenceNumberAndBitField* seqNumBitField = reinterpret_cast(_packet.get()); // write new value by ORing (old value & BIT_FIELD_MASK) with new seqNum - *seqNumBitField = (*seqNumBitField & BIT_FIELD_MASK) | sequenceNumber; + *seqNumBitField = (*seqNumBitField & BIT_FIELD_MASK) | (SeqNum::Type)seqNum; } diff --git a/libraries/networking/src/udt/Packet.h b/libraries/networking/src/udt/Packet.h index 39770bb022..51f66e0dab 100644 --- a/libraries/networking/src/udt/Packet.h +++ b/libraries/networking/src/udt/Packet.h @@ -18,6 +18,7 @@ #include "BasePacket.h" #include "PacketHeaders.h" +#include "SeqNum.h" namespace udt { @@ -50,7 +51,8 @@ public: virtual qint64 totalHeadersSize() const; // Cumulated size of all the headers - void writeSequenceNumber(SequenceNumber sequenceNumber); + void writeSequenceNumber(SeqNum seqNum); + SeqNum getSequenceNumber() const { return _sequenceNumber; } protected: Packet(qint64 size, bool isReliable = false, bool isPartOfMessage = false); @@ -70,7 +72,7 @@ protected: bool _isReliable { false }; bool _isPartOfMessage { false }; - SequenceNumber _sequenceNumber { 0 }; + SeqNum _sequenceNumber; }; } // namespace udt diff --git a/libraries/networking/src/udt/SeqNum.h b/libraries/networking/src/udt/SeqNum.h index ad5a6c7c21..b73f5fa23c 100644 --- a/libraries/networking/src/udt/SeqNum.h +++ b/libraries/networking/src/udt/SeqNum.h @@ -27,7 +27,7 @@ public: static const Type THRESHOLD = 0x0FFFFFFF; // threshold for comparing sequence numbers static const Type MAX = 0x1FFFFFFF; // maximum sequence number used in UDT - SeqNum() : _value(0) {} + SeqNum() = default; SeqNum(const SeqNum& other) : _value(other._value) {} // Only explicit conversions @@ -79,7 +79,7 @@ public: friend int seqoff(const SeqNum& seq1, const SeqNum& seq2); private: - Type _value; + Type _value { 0 }; friend struct std::hash; }; diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 68985133ac..8c8b4b4883 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -64,7 +64,7 @@ private: std::unordered_map _unfilteredHandlers; - std::unordered_map _packetSequenceNumbers; + std::unordered_map _packetSequenceNumbers; }; } // namespace udt From da93301ac6d39669f7d7bc0310cd0bce5316b777 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 24 Jul 2015 14:33:40 -0700 Subject: [PATCH 007/242] More work on SendQueue (Threading) --- libraries/networking/src/udt/SendQueue.cpp | 70 +++++++++++++++------- libraries/networking/src/udt/SendQueue.h | 25 +++++--- 2 files changed, 68 insertions(+), 27 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index af19162bb8..9bc306c88f 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -11,18 +11,38 @@ #include "SendQueue.h" +#include + #include #include #include +#include "Packet.h" +#include "Socket.h" + namespace udt { -std::unique_ptr SendQueue::create() { - return std::unique_ptr(new SendQueue()); +std::unique_ptr SendQueue::create(Socket* socket, HifiSockAddr dest) { + auto queue = std::unique_ptr(new SendQueue(socket, dest)); + + // Setup queue private thread + QThread* thread = new QThread(queue.get()); + thread->setObjectName("Networking: SendQueue"); // Name thread for easier debug + thread->connect(queue.get(), &QObject::destroyed, thread, &QThread::quit); // Thread auto cleanup + thread->connect(thread, &QThread::finished, thread, &QThread::deleteLater); // Thread auto cleanup + + // Move queue to private thread and start it + queue->moveToThread(thread); + thread->start(); + + return std::move(queue); } -SendQueue::SendQueue() { +SendQueue::SendQueue(Socket* socket, HifiSockAddr dest) : + _socket(socket), + _destination(dest) +{ _sendTimer.reset(new QTimer(this)); _sendTimer->setSingleShot(true); QObject::connect(_sendTimer.get(), &QTimer::timeout, this, &SendQueue::sendNextPacket); @@ -32,8 +52,13 @@ SendQueue::SendQueue() { } void SendQueue::queuePacket(std::unique_ptr packet) { - QWriteLocker locker(&_packetsLock); - _packets.push_back(std::move(packet)); + { + QWriteLocker locker(&_packetsLock); + _packets.push_back(std::move(packet)); + } + if (!_running) { + start(); + } } void SendQueue::start() { @@ -41,28 +66,37 @@ void SendQueue::start() { if (thread() != QThread::currentThread()) { QMetaObject::invokeMethod(this, "start", Qt::QueuedConnection); } + _running = true; + // This will send a packet and fire the send timer sendNextPacket(); } void SendQueue::stop() { - // We need to make sure this is called on the right thread - if (thread() != QThread::currentThread()) { - QMetaObject::invokeMethod(this, "stop", Qt::QueuedConnection); - } - // Stopping the timer will stop the sending of packets - _sendTimer->stop(); + _running = false; +} + +void SendQueue::sendPacket(const Packet& packet) { + _socket->writeDatagram(packet.getData(), packet.getDataSize(), _destination); } void SendQueue::sendNextPacket() { + if (!_running) { + return; + } + // Record timing auto sendTime = msecTimestampNow(); // msec _lastSendTimestamp = sendTime; - // TODO send packet - // Insert the packet we have just sent in the sent list - _sentPackets[_nextPacket->readSequenceNumber()].swap(_nextPacket); - Q_ASSERT(!_nextPacket); // There should be no packet where we inserted + if (_nextPacket) { + _nextPacket->writeSequenceNumber(++_currentSeqNum); + sendPacket(*_nextPacket); + + // Insert the packet we have just sent in the sent list + _sentPackets[_nextPacket->getSequenceNumber()].swap(_nextPacket); + Q_ASSERT(!_nextPacket); // There should be no packet where we inserted + } { // Grab next packet to be sent QWriteLocker locker(&_packetsLock); @@ -72,11 +106,7 @@ void SendQueue::sendNextPacket() { // How long before next packet send auto timeToSleep = (sendTime + _packetSendPeriod) - msecTimestampNow(); // msec - if (timeToSleep > 0) { - _sendTimer->start(timeToSleep); - } else { - _sendTimer->start(0); - } + _sendTimer->start(std::max((quint64)0, timeToSleep)); } } diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 9767cfe7aa..3317ead795 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -18,19 +18,24 @@ #include #include -#include "Packet.h" +#include "../HifiSockAddr.h" + +#include "SeqNum.h" class QTimer; namespace udt { +class Socket; +class Packet; + class SendQueue : public QObject { Q_OBJECT public: static const int DEFAULT_SEND_PERIOD = 16; // msec - static std::unique_ptr create(); + static std::unique_ptr create(Socket* socket, HifiSockAddr dest); void queuePacket(std::unique_ptr packet); int getQueueSize() const { return _packets.size(); } @@ -43,24 +48,30 @@ public: public slots: void start(); void stop(); + void sendPacket(const Packet& packet); private slots: void sendNextPacket(); private: - SendQueue(); + SendQueue(Socket* socket, HifiSockAddr dest); SendQueue(SendQueue& other) = delete; SendQueue(SendQueue&& other) = delete; QReadWriteLock _packetsLock; // Protects the packets to be sent list. std::list> _packets; // List of packets to be sent - std::unique_ptr _nextPacket; + std::unique_ptr _nextPacket; // Next packet to be sent + + Socket* _socket { nullptr }; // Socket to send packet on + HifiSockAddr _destination; // Destination addr + SeqNum _currentSeqNum; // Last sequence number sent out std::unique_ptr _sendTimer; // Send timer - std::atomic _packetSendPeriod; // Interval between two packet send envent in msec - std::atomic _lastSendTimestamp; // Record last time of packet departure + std::atomic _packetSendPeriod { 0 }; // Interval between two packet send envent in msec + std::atomic _lastSendTimestamp { 0 }; // Record last time of packet departure + std::atomic _running { false }; - std::unordered_map> _sentPackets; // Packets waiting for ACK. + std::unordered_map> _sentPackets; // Packets waiting for ACK. }; } From e330727e36a65ffd365cf4d46aac4b904dbc574c Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 24 Jul 2015 15:24:40 -0700 Subject: [PATCH 008/242] Added NAK resend --- libraries/networking/src/udt/SendQueue.cpp | 43 +++++++++++++++++++--- libraries/networking/src/udt/SendQueue.h | 5 +++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 9bc306c88f..68b47914c2 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -21,7 +21,7 @@ #include "Packet.h" #include "Socket.h" -namespace udt { +using namespace udt; std::unique_ptr SendQueue::create(Socket* socket, HifiSockAddr dest) { auto queue = std::unique_ptr(new SendQueue(socket, dest)); @@ -79,6 +79,15 @@ void SendQueue::stop() { void SendQueue::sendPacket(const Packet& packet) { _socket->writeDatagram(packet.getData(), packet.getDataSize(), _destination); } + +void SendQueue::ack(SeqNum ack) { + // TODO Remove acked packet from sent list +} + +void SendQueue::nak(std::list naks) { + QWriteLocker locker(&_naksLock); + _naks.splice(_naks.end(), naks); // Add naks at the end +} void SendQueue::sendNextPacket() { if (!_running) { @@ -95,10 +104,36 @@ void SendQueue::sendNextPacket() { // Insert the packet we have just sent in the sent list _sentPackets[_nextPacket->getSequenceNumber()].swap(_nextPacket); - Q_ASSERT(!_nextPacket); // There should be no packet where we inserted + Q_ASSERT_X(!_nextPacket, + "SendQueue::sendNextPacket()", "Overriden packet in sent list"); } - { // Grab next packet to be sent + bool hasResend = false; + SeqNum seqNum; + { + // Check nak list for packet to resend + QWriteLocker locker(&_naksLock); + if (!_naks.empty()) { + hasResend = true; + seqNum = _naks.front(); + _naks.pop_front(); + } + } + + // Find packet in sent list using SeqNum + if (hasResend) { + auto it = _sentPackets.find(seqNum); + Q_ASSERT_X(it != _sentPackets.end(), + "SendQueue::sendNextPacket()", "Couldn't find NAKed packet to resend"); + + if (it != _sentPackets.end()) { + it->second.swap(_nextPacket); + _sentPackets.erase(it); + } + } + + // If there is no packet to resend, grab the next one in the list + if (!_nextPacket) { QWriteLocker locker(&_packetsLock); _nextPacket.swap(_packets.front()); _packets.pop_front(); @@ -108,5 +143,3 @@ void SendQueue::sendNextPacket() { auto timeToSleep = (sendTime + _packetSendPeriod) - msecTimestampNow(); // msec _sendTimer->start(std::max((quint64)0, timeToSleep)); } - -} diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 3317ead795..56b71eec54 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -50,6 +50,9 @@ public slots: void stop(); void sendPacket(const Packet& packet); + void ack(SeqNum ack); + void nak(std::list naks); + private slots: void sendNextPacket(); @@ -71,6 +74,8 @@ private: std::atomic _lastSendTimestamp { 0 }; // Record last time of packet departure std::atomic _running { false }; + QReadWriteLock _naksLock; // Protects the naks list. + std::list _naks; // Sequence numbers of packets to resend std::unordered_map> _sentPackets; // Packets waiting for ACK. }; From f06d21691d5205508836648a31a74083ea4ab01a Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 24 Jul 2015 15:33:42 -0700 Subject: [PATCH 009/242] Added ACKs removal from sent list --- libraries/networking/src/udt/SendQueue.cpp | 13 ++++++++++++- libraries/networking/src/udt/SendQueue.h | 3 +++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 68b47914c2..9046e05f26 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -81,7 +81,16 @@ void SendQueue::sendPacket(const Packet& packet) { } void SendQueue::ack(SeqNum ack) { - // TODO Remove acked packet from sent list + if (_lastAck == ack) { + return; + } + + QWriteLocker locker(&_sentLock); + for (auto seq = _lastAck; seq != ack; ++seq) { + _sentPackets.erase(seq); + } + + _lastAck = ack; } void SendQueue::nak(std::list naks) { @@ -103,6 +112,7 @@ void SendQueue::sendNextPacket() { sendPacket(*_nextPacket); // Insert the packet we have just sent in the sent list + QWriteLocker locker(&_sentLock); _sentPackets[_nextPacket->getSequenceNumber()].swap(_nextPacket); Q_ASSERT_X(!_nextPacket, "SendQueue::sendNextPacket()", "Overriden packet in sent list"); @@ -122,6 +132,7 @@ void SendQueue::sendNextPacket() { // Find packet in sent list using SeqNum if (hasResend) { + QWriteLocker locker(&_sentLock); auto it = _sentPackets.find(seqNum); Q_ASSERT_X(it != _sentPackets.end(), "SendQueue::sendNextPacket()", "Couldn't find NAKed packet to resend"); diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 56b71eec54..fa73c14578 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -68,6 +68,7 @@ private: Socket* _socket { nullptr }; // Socket to send packet on HifiSockAddr _destination; // Destination addr SeqNum _currentSeqNum; // Last sequence number sent out + SeqNum _lastAck; // ACKed sequence number std::unique_ptr _sendTimer; // Send timer std::atomic _packetSendPeriod { 0 }; // Interval between two packet send envent in msec @@ -76,6 +77,8 @@ private: QReadWriteLock _naksLock; // Protects the naks list. std::list _naks; // Sequence numbers of packets to resend + + QReadWriteLock _sentLock; // Protects the sent packet list std::unordered_map> _sentPackets; // Packets waiting for ACK. }; From f4a9f21869886d0fca5ad1b2b4f3c3c5bd01f506 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 27 Jul 2015 11:28:48 -0700 Subject: [PATCH 010/242] Use udt instead declaring namespace in cpp --- libraries/networking/src/udt/CongestionControl.cpp | 4 +--- libraries/networking/src/udt/SeqNum.cpp | 10 +++------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index fbb172ecee..ebbf86ecd8 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -13,7 +13,7 @@ #include -namespace udt { +using namespace udt; void UdtCC::init() { _rcInterval = _synInterval; @@ -144,5 +144,3 @@ void UdtCC::onTimeout() { */ } } - -} \ No newline at end of file diff --git a/libraries/networking/src/udt/SeqNum.cpp b/libraries/networking/src/udt/SeqNum.cpp index 29347a9249..3ab4b23606 100644 --- a/libraries/networking/src/udt/SeqNum.cpp +++ b/libraries/networking/src/udt/SeqNum.cpp @@ -11,19 +11,17 @@ #include "SeqNum.h" -namespace udt { - -int seqcmp(const SeqNum& seq1, const SeqNum& seq2) { +int udt::seqcmp(const SeqNum& seq1, const SeqNum& seq2) { return (glm::abs(seq1._value - seq2._value) < SeqNum::THRESHOLD) ? (seq1._value - seq2._value) : (seq2._value - seq1._value); } -int seqlen(const SeqNum& seq1, const SeqNum& seq2) { +int udt::seqlen(const SeqNum& seq1, const SeqNum& seq2) { return (seq1._value <= seq2._value) ? (seq2._value - seq1._value + 1) : (seq2._value - seq1._value + SeqNum::MAX + 2); } -int seqoff(const SeqNum& seq1, const SeqNum& seq2) { +int udt::seqoff(const SeqNum& seq1, const SeqNum& seq2) { if (glm::abs(seq1._value - seq2._value) < SeqNum::THRESHOLD) { return seq2._value - seq1._value; } @@ -34,5 +32,3 @@ int seqoff(const SeqNum& seq1, const SeqNum& seq2) { return seq2._value - seq1._value + SeqNum::MAX + 1; } - -} From 1e766faa881f4e042a508708cc1cb9d735957db5 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 27 Jul 2015 11:29:15 -0700 Subject: [PATCH 011/242] Remove unecessary headers --- libraries/networking/src/udt/Socket.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 8c8b4b4883..882ebe5841 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -20,10 +20,10 @@ #include #include -#include "../HifiSockAddr.h" -#include "Packet.h" - namespace udt { + +class Packet; +class HifiSockAddr; using PacketFilterOperator = std::function; From 3e8bb371d67224ce73c7ca24dba45c23f2752939 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 27 Jul 2015 11:32:17 -0700 Subject: [PATCH 012/242] declare BasePacket in Socket --- libraries/networking/src/udt/Socket.h | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 882ebe5841..b97dce45fa 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -22,6 +22,7 @@ namespace udt { +class BasePacket; class Packet; class HifiSockAddr; From d0c3274b5f42fa5a191d402879ed5000fc31eddc Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 27 Jul 2015 11:36:09 -0700 Subject: [PATCH 013/242] include Packet for udt::Socket --- libraries/networking/src/udt/Socket.cpp | 1 + libraries/networking/src/udt/Socket.h | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 7eb53f5a28..8bf7f973ae 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -12,6 +12,7 @@ #include "Socket.h" #include "../NetworkLogging.h" +#include "Packet.h" using namespace udt; diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index b97dce45fa..184d8716fe 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -20,11 +20,13 @@ #include #include +#include "../HifiSockAddr.h" +#include "SeqNum.h" + namespace udt { class BasePacket; class Packet; -class HifiSockAddr; using PacketFilterOperator = std::function; From 6db401f6d347ba579db211e1c8796712076350c4 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 27 Jul 2015 11:38:49 -0700 Subject: [PATCH 014/242] Foreward declare SeqNum --- libraries/networking/src/udt/Socket.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 184d8716fe..b23903501e 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -21,12 +21,12 @@ #include #include "../HifiSockAddr.h" -#include "SeqNum.h" namespace udt { class BasePacket; class Packet; +class SeqNum; using PacketFilterOperator = std::function; From 507592b08868fe75f1ca2c7cfa4a7735e9c0c1f2 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 27 Jul 2015 11:40:05 -0700 Subject: [PATCH 015/242] add paths to CongestionControl files --- libraries/networking/src/udt/CongestionControl.cpp | 2 +- libraries/networking/src/udt/CongestionControl.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index ebbf86ecd8..4f32be0d06 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -1,6 +1,6 @@ // // CongestionControl.cpp -// +// libraries/networking/src/udt // // Created by Clement on 7/23/15. // Copyright 2015 High Fidelity, Inc. diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index fdf8803ec0..fe7f4a15fc 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -1,6 +1,6 @@ // // CongestionControl.h -// +// libraries/networking/src/udt // // Created by Clement on 7/23/15. // Copyright 2015 High Fidelity, Inc. From f520caa5fe47440bad330b3b0c800aab6b6b6cfe Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 27 Jul 2015 12:06:27 -0700 Subject: [PATCH 016/242] Connection class draft --- libraries/networking/src/udt/Connection.cpp | 31 ++++++++++++++++ libraries/networking/src/udt/Connection.h | 41 +++++++++++++++++++++ libraries/networking/src/udt/SendQueue.cpp | 1 - libraries/networking/src/udt/SendQueue.h | 7 ++-- 4 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 libraries/networking/src/udt/Connection.cpp create mode 100644 libraries/networking/src/udt/Connection.h diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp new file mode 100644 index 0000000000..c28a5f6ed0 --- /dev/null +++ b/libraries/networking/src/udt/Connection.cpp @@ -0,0 +1,31 @@ +// +// Connection.cpp +// +// +// Created by Clement on 7/27/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "Connection.h" + +#include "../HifiSockAddr.h" +#include "ControlPacket.h" +#include "Packet.h" +#include "Socket.h" + +using namespace udt; + +Connection::Connection(Socket* parentSocket, HifiSockAddr destination) { + +} + +void Connection::send(std::unique_ptr packet) { + +} + +void Connection::processControl(std::unique_ptr controlPacket) { + +} diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h new file mode 100644 index 0000000000..ea49edd1e7 --- /dev/null +++ b/libraries/networking/src/udt/Connection.h @@ -0,0 +1,41 @@ +// +// Connection.h +// +// +// Created by Clement on 7/27/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_Connection_h +#define hifi_Connection_h + +#include + +#include "SendQueue.h" + +class HifiSockAddr; + +namespace udt { + +class ControlPacket; +class Packet; +class Socket; + +class Connection { + +public: + Connection(Socket* parentSocket, HifiSockAddr destination); + + void send(std::unique_ptr packet); + void processControl(std::unique_ptr controlPacket); + +private: + std::unique_ptr _sendQueue; +}; + +} + +#endif // hifi_Connection_h diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 9046e05f26..c52621dccd 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -14,7 +14,6 @@ #include #include -#include #include diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index fa73c14578..9a46f0f119 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -15,15 +15,14 @@ #include #include -#include -#include +#include +#include +#include #include "../HifiSockAddr.h" #include "SeqNum.h" -class QTimer; - namespace udt { class Socket; From b0e7c208ac289d9a87d5bcbb704cdb51ea72ea81 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 27 Jul 2015 12:21:27 -0700 Subject: [PATCH 017/242] initial version of the sleeping ControlSender --- .../networking/src/udt/ControlSender.cpp | 40 +++++++++++++++++++ libraries/networking/src/udt/ControlSender.h | 39 ++++++++++++++++++ libraries/networking/src/udt/Socket.cpp | 35 ++++++++++++++++ libraries/networking/src/udt/Socket.h | 6 ++- 4 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 libraries/networking/src/udt/ControlSender.cpp create mode 100644 libraries/networking/src/udt/ControlSender.h diff --git a/libraries/networking/src/udt/ControlSender.cpp b/libraries/networking/src/udt/ControlSender.cpp new file mode 100644 index 0000000000..dd9948502a --- /dev/null +++ b/libraries/networking/src/udt/ControlSender.cpp @@ -0,0 +1,40 @@ +// +// ControlSender.cpp +// libraries/networking/src/udt +// +// Created by Stephen Birarda on 2015-07-27. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include +#include + +#include +#include + +#include "ControlSender.h" + +using namespace udt; + +void ControlSender::loop() { + while (!_isStopped) { + // grab the time now + auto start = std::chrono::high_resolution_clock::now(); + + // for each of the connections managed by the udt::Socket, we need to ask for the ACK value to send + + // since we're infinitely looping, give the thread a chance to process events + QCoreApplication::processEvents(); + + auto end = std::chrono::high_resolution_clock::now(); + + std::chrono::duration elapsed = end - start; + int timeToSleep = _synInterval - (int) elapsed.count(); + + // based on how much time it took us to process, let's figure out how much time we have need to sleep + std::this_thread::sleep_for(std::chrono::microseconds(timeToSleep)); + } +} diff --git a/libraries/networking/src/udt/ControlSender.h b/libraries/networking/src/udt/ControlSender.h new file mode 100644 index 0000000000..040c6a70db --- /dev/null +++ b/libraries/networking/src/udt/ControlSender.h @@ -0,0 +1,39 @@ +// +// ControlSender.h +// libraries/networking/src/udt +// +// Created by Stephen Birarda on 2015-07-27. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#pragma once + +#ifndef hifi_ControlSender_h +#define hifi_ControlSender_h + +#include + +namespace udt { + +// Handles the sending of periodic control packets for all active UDT reliable connections +// Currently the interval for all connections is the same, so one thread is sufficient to manage all +class ControlSender : public QObject { + Q_OBJECT +public: + ControlSender(QObject* parent = 0) : QObject(parent) {}; + +public slots: + void loop(); // infinitely loops and sleeps to manage rate of control packet sending + void stop() { _isStopped = true; } // stops the loop + +private: + int _synInterval = 10 * 1000; + bool _isStopped { false }; +}; + +} + +#endif // hifi_ControlSender_h diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 8bf7f973ae..57c4b75b12 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -11,7 +11,10 @@ #include "Socket.h" +#include + #include "../NetworkLogging.h" +#include "ControlSender.h" #include "Packet.h" using namespace udt; @@ -20,6 +23,38 @@ Socket::Socket(QObject* parent) : QObject(parent) { connect(&_udpSocket, &QUdpSocket::readyRead, this, &Socket::readPendingDatagrams); + + // make a QThread for the ControlSender to live on + QThread* controlThread = new QThread(this); + + // setup the ControlSender and parent it + _controlSender = new ControlSender; + + // move the ControlSender to its thread + _controlSender->moveToThread(controlThread); + + // start the ControlSender once the thread is started + connect(controlThread, &QThread::started, _controlSender, &ControlSender::loop); + + // make sure the control thread is named so we can identify it + controlThread->setObjectName("UDT Control Thread"); + + // start the control thread + controlThread->start(); +} + +Socket::~Socket() { + if (_controlSender) { + QThread* controlThread = _controlSender->thread(); + + // tell the control sender to stop and be deleted so we can wait on its thread + QMetaObject::invokeMethod(_controlSender, "stop"); + _controlSender->deleteLater(); + + // make sure the control thread goes down + controlThread->quit(); + controlThread->wait(); + } } void Socket::rebind() { diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 184d8716fe..be6ef982af 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -24,8 +24,9 @@ #include "SeqNum.h" namespace udt { - + class BasePacket; +class ControlSender; class Packet; using PacketFilterOperator = std::function; @@ -37,6 +38,7 @@ class Socket : public QObject { Q_OBJECT public: Socket(QObject* object = 0); + ~Socket(); quint16 localPort() const { return _udpSocket.localPort(); } @@ -68,6 +70,8 @@ private: std::unordered_map _unfilteredHandlers; std::unordered_map _packetSequenceNumbers; + + ControlSender* _controlSender { nullptr }; }; } // namespace udt From 594b4f4814819de64d513799f78f06ee94821822 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 27 Jul 2015 12:25:46 -0700 Subject: [PATCH 018/242] fix the shutdown of the ControlSender --- libraries/networking/src/udt/Socket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 57c4b75b12..9078ac0f6d 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -48,7 +48,7 @@ Socket::~Socket() { QThread* controlThread = _controlSender->thread(); // tell the control sender to stop and be deleted so we can wait on its thread - QMetaObject::invokeMethod(_controlSender, "stop"); + _controlSender->stop(); _controlSender->deleteLater(); // make sure the control thread goes down From a31053d4508c0b0491db89806af6d39fe7fe2c11 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 27 Jul 2015 13:02:11 -0700 Subject: [PATCH 019/242] small spacing change --- libraries/networking/src/udt/ControlSender.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/ControlSender.cpp b/libraries/networking/src/udt/ControlSender.cpp index dd9948502a..1b955d0254 100644 --- a/libraries/networking/src/udt/ControlSender.cpp +++ b/libraries/networking/src/udt/ControlSender.cpp @@ -25,7 +25,7 @@ void ControlSender::loop() { auto start = std::chrono::high_resolution_clock::now(); // for each of the connections managed by the udt::Socket, we need to ask for the ACK value to send - + // since we're infinitely looping, give the thread a chance to process events QCoreApplication::processEvents(); From 41a0f6b9809939022b48a47c4dc868e258861f9f Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 27 Jul 2015 14:18:18 -0700 Subject: [PATCH 020/242] Added +/- operator to SeqNum --- libraries/networking/src/udt/SeqNum.cpp | 4 ++-- libraries/networking/src/udt/SeqNum.h | 31 +++++++++++++++++++++---- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/udt/SeqNum.cpp b/libraries/networking/src/udt/SeqNum.cpp index 3ab4b23606..506b8bf3c9 100644 --- a/libraries/networking/src/udt/SeqNum.cpp +++ b/libraries/networking/src/udt/SeqNum.cpp @@ -13,12 +13,12 @@ int udt::seqcmp(const SeqNum& seq1, const SeqNum& seq2) { return (glm::abs(seq1._value - seq2._value) < SeqNum::THRESHOLD) ? (seq1._value - seq2._value) - : (seq2._value - seq1._value); + : (seq2._value - seq1._value); } int udt::seqlen(const SeqNum& seq1, const SeqNum& seq2) { return (seq1._value <= seq2._value) ? (seq2._value - seq1._value + 1) - : (seq2._value - seq1._value + SeqNum::MAX + 2); + : (seq2._value - seq1._value + SeqNum::MAX + 2); } int udt::seqoff(const SeqNum& seq1, const SeqNum& seq2) { diff --git a/libraries/networking/src/udt/SeqNum.h b/libraries/networking/src/udt/SeqNum.h index b73f5fa23c..a57ad01e11 100644 --- a/libraries/networking/src/udt/SeqNum.h +++ b/libraries/networking/src/udt/SeqNum.h @@ -74,6 +74,11 @@ public: return _value != other._value; } + friend SeqNum operator+(const SeqNum a, const Type& b); + friend SeqNum operator+(const Type& a, const SeqNum b); + friend SeqNum operator-(const SeqNum a, const Type& b); + friend SeqNum operator-(const Type& a, const SeqNum b); + friend int seqcmp(const SeqNum& seq1, const SeqNum& seq2); friend int seqlen(const SeqNum& seq1, const SeqNum& seq2); friend int seqoff(const SeqNum& seq1, const SeqNum& seq2); @@ -84,20 +89,36 @@ private: friend struct std::hash; }; +SeqNum operator+(SeqNum a, const SeqNum::Type& b) { + a += b; + return a; +} + +SeqNum operator+(const SeqNum::Type& a, SeqNum b) { + b += a; + return b; +} + +SeqNum operator-(SeqNum a, const SeqNum::Type& b) { + a -= b; + return a; +} + +SeqNum operator-(const SeqNum::Type& a, SeqNum b) { + b -= a; + return b; +} + int seqcmp(const SeqNum& seq1, const SeqNum& seq2); int seqlen(const SeqNum& seq1, const SeqNum& seq2); int seqoff(const SeqNum& seq1, const SeqNum& seq2); } -namespace std { - -template<> struct hash { +template<> struct std::hash { size_t operator()(const udt::SeqNum& seqNum) const { return std::hash()(seqNum._value); } }; - -} #endif // hifi_SeqNum_h From 2b29f5c1301552a2c78d8a02f734ced25f621f3d Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 27 Jul 2015 15:57:41 -0700 Subject: [PATCH 021/242] More work on Connnections --- libraries/networking/src/udt/Connection.cpp | 27 ++++++++++++++++- libraries/networking/src/udt/Connection.h | 4 +++ libraries/networking/src/udt/ControlPacket.h | 2 ++ libraries/networking/src/udt/SendQueue.cpp | 9 ++++-- libraries/networking/src/udt/SendQueue.h | 11 ++++--- libraries/networking/src/udt/SeqNum.h | 31 +++++++++++++++++--- 6 files changed, 73 insertions(+), 11 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index c28a5f6ed0..7fd643cd5c 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -23,9 +23,34 @@ Connection::Connection(Socket* parentSocket, HifiSockAddr destination) { } void Connection::send(std::unique_ptr packet) { + if (_sendQueue) { + _sendQueue->queuePacket(std::move(packet)); + } +} +void Connection::processReceivedSeqNum(SeqNum seq) { + if (udt::seqcmp(seq, _largestRecievedSeqNum + 1) > 0) { + // TODO: Add range to loss list + + // TODO: Send loss report + } + + if (seq > _largestRecievedSeqNum) { + _largestRecievedSeqNum = seq; + } else { + // TODO: Remove seq from loss list + } } void Connection::processControl(std::unique_ptr controlPacket) { - + switch (controlPacket->getType()) { + case ControlPacket::Type::ACK: + break; + case ControlPacket::Type::ACK2: + break; + case ControlPacket::Type::NAK: + break; + case ControlPacket::Type::PacketPair: + break; + } } diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index ea49edd1e7..3f3b39524b 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -30,9 +30,13 @@ public: Connection(Socket* parentSocket, HifiSockAddr destination); void send(std::unique_ptr packet); + + void processReceivedSeqNum(SeqNum seq); void processControl(std::unique_ptr controlPacket); private: + + SeqNum _largestRecievedSeqNum; std::unique_ptr _sendQueue; }; diff --git a/libraries/networking/src/udt/ControlPacket.h b/libraries/networking/src/udt/ControlPacket.h index 04357ab307..8b9ccbf073 100644 --- a/libraries/networking/src/udt/ControlPacket.h +++ b/libraries/networking/src/udt/ControlPacket.h @@ -42,6 +42,8 @@ public: static qint64 localHeaderSize(); // Current level's header size virtual qint64 totalHeadersSize() const; // Cumulated size of all the headers + Type getType() const { return _type; } + private: ControlPacket(Type type, const SequenceNumberList& sequenceNumbers); ControlPacket(quint64 timestamp); diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index c52621dccd..7561a7042d 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -28,8 +28,8 @@ std::unique_ptr SendQueue::create(Socket* socket, HifiSockAddr dest) // Setup queue private thread QThread* thread = new QThread(queue.get()); thread->setObjectName("Networking: SendQueue"); // Name thread for easier debug - thread->connect(queue.get(), &QObject::destroyed, thread, &QThread::quit); // Thread auto cleanup - thread->connect(thread, &QThread::finished, thread, &QThread::deleteLater); // Thread auto cleanup + connect(queue.get(), &QObject::destroyed, thread, &QThread::quit); // Thread auto cleanup + connect(thread, &QThread::finished, thread, &QThread::deleteLater); // Thread auto cleanup // Move queue to private thread and start it queue->moveToThread(thread); @@ -50,6 +50,11 @@ SendQueue::SendQueue(Socket* socket, HifiSockAddr dest) : _lastSendTimestamp = 0; } +SendQueue::~SendQueue() { + assert(thread() == QThread::currentThread()); + _sendTimer->stop(); +} + void SendQueue::queuePacket(std::unique_ptr packet) { { QWriteLocker locker(&_packetsLock); diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 9a46f0f119..1a1a2fc90d 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -37,7 +37,7 @@ public: static std::unique_ptr create(Socket* socket, HifiSockAddr dest); void queuePacket(std::unique_ptr packet); - int getQueueSize() const { return _packets.size(); } + int getQueueSize() const { QReadLocker locker(&_packetsLock); return _packets.size(); } quint64 getLastSendTimestamp() const { return _lastSendTimestamp; } @@ -56,11 +56,14 @@ private slots: void sendNextPacket(); private: + friend struct std::default_delete; + SendQueue(Socket* socket, HifiSockAddr dest); SendQueue(SendQueue& other) = delete; SendQueue(SendQueue&& other) = delete; + ~SendQueue(); - QReadWriteLock _packetsLock; // Protects the packets to be sent list. + mutable QReadWriteLock _packetsLock; // Protects the packets to be sent list. std::list> _packets; // List of packets to be sent std::unique_ptr _nextPacket; // Next packet to be sent @@ -74,10 +77,10 @@ private: std::atomic _lastSendTimestamp { 0 }; // Record last time of packet departure std::atomic _running { false }; - QReadWriteLock _naksLock; // Protects the naks list. + mutable QReadWriteLock _naksLock; // Protects the naks list. std::list _naks; // Sequence numbers of packets to resend - QReadWriteLock _sentLock; // Protects the sent packet list + mutable QReadWriteLock _sentLock; // Protects the sent packet list std::unordered_map> _sentPackets; // Packets waiting for ACK. }; diff --git a/libraries/networking/src/udt/SeqNum.h b/libraries/networking/src/udt/SeqNum.h index a57ad01e11..5f541b0c36 100644 --- a/libraries/networking/src/udt/SeqNum.h +++ b/libraries/networking/src/udt/SeqNum.h @@ -74,6 +74,11 @@ public: return _value != other._value; } + friend bool operator<(const SeqNum& a, const SeqNum& b); + friend bool operator>(const SeqNum& a, const SeqNum& b); + friend bool operator<=(const SeqNum& a, const SeqNum& b); + friend bool operator>=(const SeqNum& a, const SeqNum& b); + friend SeqNum operator+(const SeqNum a, const Type& b); friend SeqNum operator+(const Type& a, const SeqNum b); friend SeqNum operator-(const SeqNum a, const Type& b); @@ -89,22 +94,40 @@ private: friend struct std::hash; }; -SeqNum operator+(SeqNum a, const SeqNum::Type& b) { + +inline bool operator<(const SeqNum& a, const SeqNum& b) { + +} + +inline bool operator>(const SeqNum& a, const SeqNum& b) { + +} + +inline bool operator<=(const SeqNum& a, const SeqNum& b) { + +} + +inline bool operator>=(const SeqNum& a, const SeqNum& b) { + +} + + +inline SeqNum operator+(SeqNum a, const SeqNum::Type& b) { a += b; return a; } -SeqNum operator+(const SeqNum::Type& a, SeqNum b) { +inline SeqNum operator+(const SeqNum::Type& a, SeqNum b) { b += a; return b; } -SeqNum operator-(SeqNum a, const SeqNum::Type& b) { +inline SeqNum operator-(SeqNum a, const SeqNum::Type& b) { a -= b; return a; } -SeqNum operator-(const SeqNum::Type& a, SeqNum b) { +inline SeqNum operator-(const SeqNum::Type& a, SeqNum b) { b -= a; return b; } From 50a7ef7c2024ed07f7fbf879fa6acbc013eb99fd Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 27 Jul 2015 15:59:04 -0700 Subject: [PATCH 022/242] remove the ControlSender and perform sync in Socket --- .../networking/src/udt/ControlSender.cpp | 40 ----------------- libraries/networking/src/udt/ControlSender.h | 39 ---------------- libraries/networking/src/udt/Socket.cpp | 45 +++++++------------ libraries/networking/src/udt/Socket.h | 5 ++- 4 files changed, 19 insertions(+), 110 deletions(-) delete mode 100644 libraries/networking/src/udt/ControlSender.cpp delete mode 100644 libraries/networking/src/udt/ControlSender.h diff --git a/libraries/networking/src/udt/ControlSender.cpp b/libraries/networking/src/udt/ControlSender.cpp deleted file mode 100644 index 1b955d0254..0000000000 --- a/libraries/networking/src/udt/ControlSender.cpp +++ /dev/null @@ -1,40 +0,0 @@ -// -// ControlSender.cpp -// libraries/networking/src/udt -// -// Created by Stephen Birarda on 2015-07-27. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include -#include - -#include -#include - -#include "ControlSender.h" - -using namespace udt; - -void ControlSender::loop() { - while (!_isStopped) { - // grab the time now - auto start = std::chrono::high_resolution_clock::now(); - - // for each of the connections managed by the udt::Socket, we need to ask for the ACK value to send - - // since we're infinitely looping, give the thread a chance to process events - QCoreApplication::processEvents(); - - auto end = std::chrono::high_resolution_clock::now(); - - std::chrono::duration elapsed = end - start; - int timeToSleep = _synInterval - (int) elapsed.count(); - - // based on how much time it took us to process, let's figure out how much time we have need to sleep - std::this_thread::sleep_for(std::chrono::microseconds(timeToSleep)); - } -} diff --git a/libraries/networking/src/udt/ControlSender.h b/libraries/networking/src/udt/ControlSender.h deleted file mode 100644 index 040c6a70db..0000000000 --- a/libraries/networking/src/udt/ControlSender.h +++ /dev/null @@ -1,39 +0,0 @@ -// -// ControlSender.h -// libraries/networking/src/udt -// -// Created by Stephen Birarda on 2015-07-27. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#pragma once - -#ifndef hifi_ControlSender_h -#define hifi_ControlSender_h - -#include - -namespace udt { - -// Handles the sending of periodic control packets for all active UDT reliable connections -// Currently the interval for all connections is the same, so one thread is sufficient to manage all -class ControlSender : public QObject { - Q_OBJECT -public: - ControlSender(QObject* parent = 0) : QObject(parent) {}; - -public slots: - void loop(); // infinitely loops and sleeps to manage rate of control packet sending - void stop() { _isStopped = true; } // stops the loop - -private: - int _synInterval = 10 * 1000; - bool _isStopped { false }; -}; - -} - -#endif // hifi_ControlSender_h diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 9078ac0f6d..a110a4f649 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -24,37 +24,11 @@ Socket::Socket(QObject* parent) : { connect(&_udpSocket, &QUdpSocket::readyRead, this, &Socket::readPendingDatagrams); - // make a QThread for the ControlSender to live on - QThread* controlThread = new QThread(this); + // make sure our synchronization method is called every SYN interval + connect(&_synTimer, &QTimer::timeout, this, &Socket::rateControlSync); - // setup the ControlSender and parent it - _controlSender = new ControlSender; - - // move the ControlSender to its thread - _controlSender->moveToThread(controlThread); - - // start the ControlSender once the thread is started - connect(controlThread, &QThread::started, _controlSender, &ControlSender::loop); - - // make sure the control thread is named so we can identify it - controlThread->setObjectName("UDT Control Thread"); - - // start the control thread - controlThread->start(); -} - -Socket::~Socket() { - if (_controlSender) { - QThread* controlThread = _controlSender->thread(); - - // tell the control sender to stop and be deleted so we can wait on its thread - _controlSender->stop(); - _controlSender->deleteLater(); - - // make sure the control thread goes down - controlThread->quit(); - controlThread->wait(); - } + // start our timer for the synchronization time interval + _synTimer->start(_synInterval); } void Socket::rebind() { @@ -148,3 +122,14 @@ void Socket::readPendingDatagrams() { } } } + +void Socket::rateControlSync() { + + + + if (_synTimer.interval() != _synInterval) { + // if the _synTimer interval doesn't match the current _synInterval (changes when the CC factory is changed) + // then restart it now with the right interval + _synTimer.start(_synInterval); + } +} diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index be6ef982af..35f8991634 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -18,6 +18,7 @@ #include #include +#include #include #include "../HifiSockAddr.h" @@ -61,6 +62,7 @@ public: private slots: void readPendingDatagrams(); + void rateControlSync(); private: QUdpSocket _udpSocket { this }; @@ -71,7 +73,8 @@ private: std::unordered_map _packetSequenceNumbers; - ControlSender* _controlSender { nullptr }; + int32_t _synInterval; + QTimer _synTimer; }; } // namespace udt From 10d6520098beb0c74039184c63f5b3c24f46cd04 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 27 Jul 2015 16:00:00 -0700 Subject: [PATCH 023/242] add a TODO message to Socket::rateControlSync --- libraries/networking/src/udt/Socket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index a110a4f649..75924b791a 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -125,7 +125,7 @@ void Socket::readPendingDatagrams() { void Socket::rateControlSync() { - + // TODO: enumerate our list of connections and ask each of them to send off periodic ACK packet for rate control if (_synTimer.interval() != _synInterval) { // if the _synTimer interval doesn't match the current _synInterval (changes when the CC factory is changed) From d6487dcb81c6a2bf1f08cc6ea10889b4023b9a49 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 27 Jul 2015 16:01:07 -0700 Subject: [PATCH 024/242] Comparaison operators for seq nums --- libraries/networking/src/udt/SeqNum.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/udt/SeqNum.h b/libraries/networking/src/udt/SeqNum.h index 5f541b0c36..5ae85f3dcf 100644 --- a/libraries/networking/src/udt/SeqNum.h +++ b/libraries/networking/src/udt/SeqNum.h @@ -96,19 +96,19 @@ private: inline bool operator<(const SeqNum& a, const SeqNum& b) { - + return (glm::abs(a._value - b._value) < SeqNum::THRESHOLD) ? a._value < b._value : b._value < a._value; } inline bool operator>(const SeqNum& a, const SeqNum& b) { - + return (glm::abs(a._value - b._value) < SeqNum::THRESHOLD) ? a._value > b._value : b._value > a._value; } inline bool operator<=(const SeqNum& a, const SeqNum& b) { - + return (glm::abs(a._value - b._value) < SeqNum::THRESHOLD) ? a._value <= b._value : b._value <= a._value; } inline bool operator>=(const SeqNum& a, const SeqNum& b) { - + return (glm::abs(a._value - b._value) < SeqNum::THRESHOLD) ? a._value >= b._value : b._value >= a._value; } From d787e086327bc1e78a31eff197580936dd5553f9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 27 Jul 2015 16:10:52 -0700 Subject: [PATCH 025/242] fix for rate control sync in Socket --- libraries/networking/src/udt/Socket.cpp | 7 +++---- libraries/networking/src/udt/Socket.h | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 75924b791a..c258283c61 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -14,7 +14,6 @@ #include #include "../NetworkLogging.h" -#include "ControlSender.h" #include "Packet.h" using namespace udt; @@ -28,7 +27,7 @@ Socket::Socket(QObject* parent) : connect(&_synTimer, &QTimer::timeout, this, &Socket::rateControlSync); // start our timer for the synchronization time interval - _synTimer->start(_synInterval); + _synTimer.start(_synInterval); } void Socket::rebind() { @@ -124,8 +123,8 @@ void Socket::readPendingDatagrams() { } void Socket::rateControlSync() { - - // TODO: enumerate our list of connections and ask each of them to send off periodic ACK packet for rate control + + // TODO: enumerate our list of connections and ask each of them to send off periodic ACK packet for rate control if (_synTimer.interval() != _synInterval) { // if the _synTimer interval doesn't match the current _synInterval (changes when the CC factory is changed) diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 2f49b9caf7..4feafa6c4c 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -39,7 +39,6 @@ class Socket : public QObject { Q_OBJECT public: Socket(QObject* object = 0); - ~Socket(); quint16 localPort() const { return _udpSocket.localPort(); } @@ -73,7 +72,7 @@ private: std::unordered_map _packetSequenceNumbers; - int32_t _synInterval; + int32_t _synInterval = 10; // 10ms QTimer _synTimer; }; From 75a722f63c8eed010211d858c30fffd0e6d7280c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 27 Jul 2015 16:12:14 -0700 Subject: [PATCH 026/242] remove a couple of unused variables --- libraries/networking/src/udt/Packet.cpp | 1 - libraries/networking/src/udt/Packet.h | 2 -- 2 files changed, 3 deletions(-) diff --git a/libraries/networking/src/udt/Packet.cpp b/libraries/networking/src/udt/Packet.cpp index e9a5636172..a70c3bfbef 100644 --- a/libraries/networking/src/udt/Packet.cpp +++ b/libraries/networking/src/udt/Packet.cpp @@ -117,7 +117,6 @@ Packet& Packet::operator=(Packet&& other) { static const uint32_t CONTROL_BIT_MASK = 1 << (sizeof(Packet::SequenceNumberAndBitField) - 1); static const uint32_t RELIABILITY_BIT_MASK = 1 << (sizeof(Packet::SequenceNumberAndBitField) - 2); static const uint32_t MESSAGE_BIT_MASK = 1 << (sizeof(Packet::SequenceNumberAndBitField) - 3); -static const int BIT_FIELD_LENGTH = 3; static const uint32_t BIT_FIELD_MASK = CONTROL_BIT_MASK | RELIABILITY_BIT_MASK | MESSAGE_BIT_MASK; void Packet::readIsReliable() { diff --git a/libraries/networking/src/udt/Packet.h b/libraries/networking/src/udt/Packet.h index 905af15ded..6189be0ef1 100644 --- a/libraries/networking/src/udt/Packet.h +++ b/libraries/networking/src/udt/Packet.h @@ -33,8 +33,6 @@ public: using MessageNumber = uint32_t; using MessageNumberAndBitField = uint32_t; - static const uint32_t DEFAULT_SEQUENCE_NUMBER = 0; - static std::unique_ptr create(qint64 size = -1, bool isReliable = false, bool isPartOfMessage = false); static std::unique_ptr fromReceivedPacket(std::unique_ptr data, qint64 size, const HifiSockAddr& senderSockAddr); From 90d25156741c78d6bfcc5a63aed40c5f22a250e0 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 27 Jul 2015 16:43:49 -0700 Subject: [PATCH 027/242] add methods to send ACKs from Connection --- libraries/networking/src/udt/Connection.cpp | 46 ++++++++++++++++--- libraries/networking/src/udt/Connection.h | 9 +++- .../networking/src/udt/ControlPacket.cpp | 36 ++++++++++----- libraries/networking/src/udt/ControlPacket.h | 11 ++--- libraries/networking/src/udt/SendQueue.cpp | 4 +- libraries/networking/src/udt/SendQueue.h | 3 +- libraries/networking/src/udt/Socket.cpp | 2 +- libraries/networking/src/udt/Socket.h | 2 +- 8 files changed, 82 insertions(+), 31 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 7fd643cd5c..4f309249ff 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -28,15 +28,47 @@ void Connection::send(std::unique_ptr packet) { } } +void Connection::sendACK() { + static const int ACK_PACKET_PAYLOAD_BYTES = 8; + + // setup the ACK packet, make it static so we can re-use it + static auto ackPacket = ControlPacket::create(ControlPacket::ACK, ACK_PACKET_PAYLOAD_BYTES); + + // have the send queue send off our packet + _sendQueue->sendPacket(*ackPacket); +} + +void Connection::sendLightACK() const { + static const int LIGHT_ACK_PACKET_PAYLOAD_BYTES = 4; + + // create the light ACK packet, make it static so we can re-use it + static auto lightACKPacket = ControlPacket::create(ControlPacket::ACK, LIGHT_ACK_PACKET_PAYLOAD_BYTES); + + SeqNum nextACKNumber = nextACK(); + + if (nextACKNumber != _lastReceivedAcknowledgedACK) { + // pack in the ACK + memcpy(lightACKPacket->getPayload(), &nextACKNumber, sizeof(nextACKNumber)); + + // have the send queue send off our packet + _sendQueue->sendPacket(*lightACKPacket); + } +} + +SeqNum Connection::nextACK() const { + // TODO: check if we have a loss list + return _largestReceivedSeqNum + 1; +} + void Connection::processReceivedSeqNum(SeqNum seq) { - if (udt::seqcmp(seq, _largestRecievedSeqNum + 1) > 0) { + if (udt::seqcmp(seq, _largestReceivedSeqNum + 1) > 0) { // TODO: Add range to loss list // TODO: Send loss report } - if (seq > _largestRecievedSeqNum) { - _largestRecievedSeqNum = seq; + if (seq > _largestReceivedSeqNum) { + _largestReceivedSeqNum = seq; } else { // TODO: Remove seq from loss list } @@ -44,13 +76,13 @@ void Connection::processReceivedSeqNum(SeqNum seq) { void Connection::processControl(std::unique_ptr controlPacket) { switch (controlPacket->getType()) { - case ControlPacket::Type::ACK: + case ControlPacket::ACK: break; - case ControlPacket::Type::ACK2: + case ControlPacket::ACK2: break; - case ControlPacket::Type::NAK: + case ControlPacket::NAK: break; - case ControlPacket::Type::PacketPair: + case ControlPacket::PacketPair: break; } } diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 3f3b39524b..8bfc2a3d31 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -31,12 +31,17 @@ public: void send(std::unique_ptr packet); + void sendACK(); + void sendLightACK() const; + + SeqNum nextACK() const; + void processReceivedSeqNum(SeqNum seq); void processControl(std::unique_ptr controlPacket); private: - - SeqNum _largestRecievedSeqNum; + SeqNum _largestReceivedSeqNum; + SeqNum _lastReceivedAcknowledgedACK; // The last sent ACK that has been acknowledged via an ACK2 from the peer std::unique_ptr _sendQueue; }; diff --git a/libraries/networking/src/udt/ControlPacket.cpp b/libraries/networking/src/udt/ControlPacket.cpp index a8688f2cdf..8d3a1561fc 100644 --- a/libraries/networking/src/udt/ControlPacket.cpp +++ b/libraries/networking/src/udt/ControlPacket.cpp @@ -13,8 +13,22 @@ using namespace udt; -std::unique_ptr ControlPacket::create(Type type, const SequenceNumberList& sequenceNumbers) { - return ControlPacket::create(type, sequenceNumbers); +std::unique_ptr ControlPacket::create(Type type, qint64 size) { + + std::unique_ptr controlPacket; + + if (size == -1) { + controlPacket = ControlPacket::create(type); + } else { + // Fail with invalid size + Q_ASSERT(size >= 0); + + controlPacket = ControlPacket::create(type, size); + } + + controlPacket->open(QIODevice::ReadWrite); + + return controlPacket; } ControlPacket::ControlPacketPair ControlPacket::createPacketPair(quint64 timestamp) { @@ -32,18 +46,18 @@ qint64 ControlPacket::totalHeadersSize() const { return BasePacket::totalHeadersSize() + localHeaderSize(); } -ControlPacket::ControlPacket(Type type, const SequenceNumberList& sequenceNumbers) : - BasePacket(localHeaderSize() + (sizeof(Packet::SequenceNumber) * sequenceNumbers.size())), +ControlPacket::ControlPacket(Type type) : + BasePacket(-1), + _type(type) +{ + adjustPayloadStartAndCapacity(); +} + +ControlPacket::ControlPacket(Type type, qint64 size) : + BasePacket(localHeaderSize() + size), _type(type) { adjustPayloadStartAndCapacity(); - - open(QIODevice::ReadWrite); - - // pack in the sequence numbers - for (auto& sequenceNumber : sequenceNumbers) { - writePrimitive(sequenceNumber); - } } ControlPacket::ControlPacket(quint64 timestamp) : diff --git a/libraries/networking/src/udt/ControlPacket.h b/libraries/networking/src/udt/ControlPacket.h index 8b9ccbf073..2eab8a2192 100644 --- a/libraries/networking/src/udt/ControlPacket.h +++ b/libraries/networking/src/udt/ControlPacket.h @@ -21,23 +21,21 @@ namespace udt { -using SequenceNumberList = std::vector; - class ControlPacket : public BasePacket { Q_OBJECT public: using TypeAndSubSequenceNumber = uint32_t; using ControlPacketPair = std::pair, std::unique_ptr>; - enum class Type : uint16_t { + enum Type : uint16_t { ACK, ACK2, NAK, PacketPair }; - std::unique_ptr create(Type type, const SequenceNumberList& sequenceNumbers); - ControlPacketPair createPacketPair(quint64 timestamp); + static std::unique_ptr create(Type type, qint64 size = -1); + static ControlPacketPair createPacketPair(quint64 timestamp); static qint64 localHeaderSize(); // Current level's header size virtual qint64 totalHeadersSize() const; // Cumulated size of all the headers @@ -45,7 +43,8 @@ public: Type getType() const { return _type; } private: - ControlPacket(Type type, const SequenceNumberList& sequenceNumbers); + ControlPacket(Type type); + ControlPacket(Type type, qint64 size); ControlPacket(quint64 timestamp); ControlPacket(ControlPacket&& other); ControlPacket(const ControlPacket& other) = delete; diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 7561a7042d..65c1807d19 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -80,8 +80,8 @@ void SendQueue::stop() { _running = false; } -void SendQueue::sendPacket(const Packet& packet) { - _socket->writeDatagram(packet.getData(), packet.getDataSize(), _destination); +void SendQueue::sendPacket(const BasePacket& packet) { + _socket->writeUnreliablePacket(packet, _destination); } void SendQueue::ack(SeqNum ack) { diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 1a1a2fc90d..9c14c19ede 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -26,6 +26,7 @@ namespace udt { class Socket; +class BasePacket; class Packet; class SendQueue : public QObject { @@ -47,7 +48,7 @@ public: public slots: void start(); void stop(); - void sendPacket(const Packet& packet); + void sendPacket(const BasePacket& packet); void ack(SeqNum ack); void nak(std::list naks); diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index c258283c61..6923879201 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -66,7 +66,7 @@ void Socket::setBufferSizes(int numBytes) { } } -qint64 Socket::writeUnreliablePacket(const Packet& packet, const HifiSockAddr& sockAddr) { +qint64 Socket::writeUnreliablePacket(const BasePacket& packet, const HifiSockAddr& sockAddr) { return writeDatagram(packet.getData(), packet.getDataSize(), sockAddr); } diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 4feafa6c4c..f74ea49f0b 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -42,7 +42,7 @@ public: quint16 localPort() const { return _udpSocket.localPort(); } - qint64 writeUnreliablePacket(const Packet& packet, const HifiSockAddr& sockAddr); + qint64 writeUnreliablePacket(const BasePacket& packet, const HifiSockAddr& sockAddr); qint64 writeDatagram(const char* data, qint64 size, const HifiSockAddr& sockAddr) { return writeDatagram(QByteArray::fromRawData(data, size), sockAddr); } From f6fb421bf297c09c234b9e3dbd198b6a81a802ce Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 27 Jul 2015 17:15:26 -0700 Subject: [PATCH 028/242] write out more packing of ACK packet in Connection --- libraries/networking/src/udt/Connection.cpp | 67 ++++++++++++++++++--- libraries/networking/src/udt/Connection.h | 12 +++- 2 files changed, 68 insertions(+), 11 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 4f309249ff..ae0ea6e065 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -17,6 +17,7 @@ #include "Socket.h" using namespace udt; +using namespace std::chrono; Connection::Connection(Socket* parentSocket, HifiSockAddr destination) { @@ -28,12 +29,57 @@ void Connection::send(std::unique_ptr packet) { } } -void Connection::sendACK() { - static const int ACK_PACKET_PAYLOAD_BYTES = 8; +void Connection::sendACK(bool wasCausedBySyncTimeout) { + static const int ACK_PACKET_PAYLOAD_BYTES = sizeof(SeqNum); // setup the ACK packet, make it static so we can re-use it static auto ackPacket = ControlPacket::create(ControlPacket::ACK, ACK_PACKET_PAYLOAD_BYTES); - + + auto currentTime = high_resolution_clock::now(); + + SeqNum nextACKNumber = nextACK(); + + if (nextACKNumber <= _lastReceivedAcknowledgedACK) { + // we already got an ACK2 for this ACK we would be sending, don't bother + return; + } + + if (nextACKNumber >= _lastSentACK) { + // we have received new packets since the last sent ACK + + // update the last sent ACK + _lastSentACK = nextACKNumber; + + // remove the ACKed packets from the receive queue + + } else if (nextACKNumber == _lastSentACK) { + // We already sent this ACK, but check if we should re-send it. + // We will re-send if it has been more than RTT + (4 * RTT variance) since the last ACK + milliseconds sinceLastACK = duration_cast(currentTime - _lastACKTime); + if (sinceLastACK.count() < (_rtt + (4 * _rttVariance))) { + return; + } + } + + // reset the ACK packet so we can fill it up and have it figure out what size it is + ackPacket->reset(); + + // pack in the ACK + ackPacket->writePrimitive(nextACKNumber); + + // pack in the RTT and variance + ackPacket->writePrimitive(_rtt); + ackPacket->writePrimitive(_rttVariance); + + // pack the available buffer size - must be a minimum of 2 + + if (wasCausedBySyncTimeout) { + // pack in the receive speed and bandwidth + + // record this as the last ACK send time + _lastACKTime = high_resolution_clock::now(); + } + // have the send queue send off our packet _sendQueue->sendPacket(*ackPacket); } @@ -46,13 +92,16 @@ void Connection::sendLightACK() const { SeqNum nextACKNumber = nextACK(); - if (nextACKNumber != _lastReceivedAcknowledgedACK) { - // pack in the ACK - memcpy(lightACKPacket->getPayload(), &nextACKNumber, sizeof(nextACKNumber)); - - // have the send queue send off our packet - _sendQueue->sendPacket(*lightACKPacket); + if (nextACKNumber == _lastReceivedAcknowledgedACK) { + // we already got an ACK2 for this ACK we would be sending, don't bother + return; } + + // pack in the ACK + memcpy(lightACKPacket->getPayload(), &nextACKNumber, sizeof(nextACKNumber)); + + // have the send queue send off our packet + _sendQueue->sendPacket(*lightACKPacket); } SeqNum Connection::nextACK() const { diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 8bfc2a3d31..882de3b79f 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -12,6 +12,7 @@ #ifndef hifi_Connection_h #define hifi_Connection_h +#include #include #include "SendQueue.h" @@ -31,7 +32,7 @@ public: void send(std::unique_ptr packet); - void sendACK(); + void sendACK(bool wasCausedBySyncTimeout = true); void sendLightACK() const; SeqNum nextACK() const; @@ -40,8 +41,15 @@ public: void processControl(std::unique_ptr controlPacket); private: - SeqNum _largestReceivedSeqNum; + SeqNum _largestReceivedSeqNum; // The largest sequence number received from the peer + SeqNum _lastSentACK; // The last sent ACK SeqNum _lastReceivedAcknowledgedACK; // The last sent ACK that has been acknowledged via an ACK2 from the peer + + int32_t _rtt; // RTT, in milliseconds + int32_t _rttVariance; // RTT variance + + std::chrono::high_resolution_clock::time_point _lastACKTime; + std::unique_ptr _sendQueue; }; From 52411bb8bac7ecd18515782761a56194ddddeade Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 27 Jul 2015 17:23:39 -0700 Subject: [PATCH 029/242] pack ACK sub sequence number manually for control --- libraries/networking/src/udt/Connection.cpp | 5 ++++- libraries/networking/src/udt/Connection.h | 1 + libraries/networking/src/udt/ControlPacket.cpp | 2 +- libraries/networking/src/udt/ControlPacket.h | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index ae0ea6e065..4b249bc91e 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -64,7 +64,10 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { // reset the ACK packet so we can fill it up and have it figure out what size it is ackPacket->reset(); - // pack in the ACK + // pack in the ACK sub-sequence number + ackPacket->writePrimitive(_currentACKSubSequenceNumber++); + + // pack in the ACK number ackPacket->writePrimitive(nextACKNumber); // pack in the RTT and variance diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 882de3b79f..a9c73ce74a 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -44,6 +44,7 @@ private: SeqNum _largestReceivedSeqNum; // The largest sequence number received from the peer SeqNum _lastSentACK; // The last sent ACK SeqNum _lastReceivedAcknowledgedACK; // The last sent ACK that has been acknowledged via an ACK2 from the peer + SeqNum _currentACKSubSequenceNumber; // The current ACK sub-sequence number (used for Acknowledgment of ACKs) int32_t _rtt; // RTT, in milliseconds int32_t _rttVariance; // RTT variance diff --git a/libraries/networking/src/udt/ControlPacket.cpp b/libraries/networking/src/udt/ControlPacket.cpp index 8d3a1561fc..f62c7a6454 100644 --- a/libraries/networking/src/udt/ControlPacket.cpp +++ b/libraries/networking/src/udt/ControlPacket.cpp @@ -39,7 +39,7 @@ ControlPacket::ControlPacketPair ControlPacket::createPacketPair(quint64 timesta } qint64 ControlPacket::localHeaderSize() { - return sizeof(TypeAndSubSequenceNumber); + return sizeof(BitFieldAndControlType); } qint64 ControlPacket::totalHeadersSize() const { diff --git a/libraries/networking/src/udt/ControlPacket.h b/libraries/networking/src/udt/ControlPacket.h index 2eab8a2192..049f95cf59 100644 --- a/libraries/networking/src/udt/ControlPacket.h +++ b/libraries/networking/src/udt/ControlPacket.h @@ -24,7 +24,7 @@ namespace udt { class ControlPacket : public BasePacket { Q_OBJECT public: - using TypeAndSubSequenceNumber = uint32_t; + using BitFieldAndControlType = uint32_t; using ControlPacketPair = std::pair, std::unique_ptr>; enum Type : uint16_t { From d2c5e79ac2e0d26c1fcde50987ecc2e37f3cd5b9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 27 Jul 2015 17:38:10 -0700 Subject: [PATCH 030/242] add writing of control bit and type to ControlPacket --- .../networking/src/udt/ControlPacket.cpp | 32 +++++++++++++++---- libraries/networking/src/udt/ControlPacket.h | 5 ++- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/libraries/networking/src/udt/ControlPacket.cpp b/libraries/networking/src/udt/ControlPacket.cpp index f62c7a6454..8b863c5a21 100644 --- a/libraries/networking/src/udt/ControlPacket.cpp +++ b/libraries/networking/src/udt/ControlPacket.cpp @@ -18,17 +18,13 @@ std::unique_ptr ControlPacket::create(Type type, qint64 size) { std::unique_ptr controlPacket; if (size == -1) { - controlPacket = ControlPacket::create(type); + return ControlPacket::create(type); } else { // Fail with invalid size Q_ASSERT(size >= 0); - controlPacket = ControlPacket::create(type, size); + return ControlPacket::create(type, size); } - - controlPacket->open(QIODevice::ReadWrite); - - return controlPacket; } ControlPacket::ControlPacketPair ControlPacket::createPacketPair(quint64 timestamp) { @@ -39,7 +35,7 @@ ControlPacket::ControlPacketPair ControlPacket::createPacketPair(quint64 timesta } qint64 ControlPacket::localHeaderSize() { - return sizeof(BitFieldAndControlType); + return sizeof(ControlBitAndType); } qint64 ControlPacket::totalHeadersSize() const { @@ -51,6 +47,10 @@ ControlPacket::ControlPacket(Type type) : _type(type) { adjustPayloadStartAndCapacity(); + + open(QIODevice::ReadWrite); + + writeControlBitAndType(); } ControlPacket::ControlPacket(Type type, qint64 size) : @@ -58,6 +58,10 @@ ControlPacket::ControlPacket(Type type, qint64 size) : _type(type) { adjustPayloadStartAndCapacity(); + + open(QIODevice::ReadWrite); + + writeControlBitAndType(); } ControlPacket::ControlPacket(quint64 timestamp) : @@ -68,6 +72,8 @@ ControlPacket::ControlPacket(quint64 timestamp) : open(QIODevice::ReadWrite); + writeControlBitAndType(); + // pack in the timestamp writePrimitive(timestamp); } @@ -85,3 +91,15 @@ ControlPacket& ControlPacket::operator=(ControlPacket&& other) { return *this; } + +static const uint32_t CONTROL_BIT_MASK = 1 << (sizeof(ControlPacket::ControlBitAndType) - 1); + +void ControlPacket::writeControlBitAndType() { + ControlBitAndType* bitAndType = reinterpret_cast(_packet.get()); + + // write the control bit by OR'ing the current value with the CONTROL_BIT_MASK + *bitAndType = (*bitAndType | CONTROL_BIT_MASK); + + // write the type by OR'ing the type with the current value & CONTROL_BIT_MASK + *bitAndType = (*bitAndType & CONTROL_BIT_MASK) | (_type << sizeof((ControlPacket::Type) - 1)); +} diff --git a/libraries/networking/src/udt/ControlPacket.h b/libraries/networking/src/udt/ControlPacket.h index 049f95cf59..6d1b7eb5d3 100644 --- a/libraries/networking/src/udt/ControlPacket.h +++ b/libraries/networking/src/udt/ControlPacket.h @@ -24,7 +24,7 @@ namespace udt { class ControlPacket : public BasePacket { Q_OBJECT public: - using BitFieldAndControlType = uint32_t; + using ControlBitAndType = uint32_t; using ControlPacketPair = std::pair, std::unique_ptr>; enum Type : uint16_t { @@ -52,6 +52,9 @@ private: ControlPacket& operator=(ControlPacket&& other); ControlPacket& operator=(const ControlPacket& other) = delete; + // Header writers + void writeControlBitAndType(); + Type _type; }; From 75765d02e4ba3d073aa74758a11d163989538514 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 27 Jul 2015 17:40:32 -0700 Subject: [PATCH 031/242] correct sizeof ACK_PACKET_PAYLOAD_BYTES in Connection --- libraries/networking/src/udt/Connection.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 4b249bc91e..3753729123 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -30,7 +30,8 @@ void Connection::send(std::unique_ptr packet) { } void Connection::sendACK(bool wasCausedBySyncTimeout) { - static const int ACK_PACKET_PAYLOAD_BYTES = sizeof(SeqNum); + static const int ACK_PACKET_PAYLOAD_BYTES = sizeof(_lastSentACK) + sizeof(_currentACKSubSequenceNumber) + + sizeof(_rtt) + sizeof(_rttVariance) + sizeof(int32_t) + sizeof(int32_t); // setup the ACK packet, make it static so we can re-use it static auto ackPacket = ControlPacket::create(ControlPacket::ACK, ACK_PACKET_PAYLOAD_BYTES); From c4f88a0f537f5761eb57515c49e91cd5e158a32a Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 27 Jul 2015 17:41:15 -0700 Subject: [PATCH 032/242] Added loss list to connections --- libraries/networking/src/udt/Connection.cpp | 20 +++-- libraries/networking/src/udt/Connection.h | 2 + libraries/networking/src/udt/LossList.cpp | 99 +++++++++++++++++++++ libraries/networking/src/udt/LossList.h | 42 +++++++++ 4 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 libraries/networking/src/udt/LossList.cpp create mode 100644 libraries/networking/src/udt/LossList.h diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 7fd643cd5c..d3b521e054 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -17,32 +17,40 @@ #include "Socket.h" using namespace udt; +using namespace std; Connection::Connection(Socket* parentSocket, HifiSockAddr destination) { } -void Connection::send(std::unique_ptr packet) { +void Connection::send(unique_ptr packet) { if (_sendQueue) { - _sendQueue->queuePacket(std::move(packet)); + _sendQueue->queuePacket(move(packet)); } } void Connection::processReceivedSeqNum(SeqNum seq) { - if (udt::seqcmp(seq, _largestRecievedSeqNum + 1) > 0) { - // TODO: Add range to loss list + // If this is not the next sequence number, report loss + if (seq > _largestRecievedSeqNum + 1) { + if (_largestRecievedSeqNum + 1 == seq - 1) { + _lossList.append(_largestRecievedSeqNum + 1); + } else { + _lossList.append(_largestRecievedSeqNum + 1, seq - 1); + } // TODO: Send loss report } if (seq > _largestRecievedSeqNum) { + // Update largest recieved sequence number _largestRecievedSeqNum = seq; } else { - // TODO: Remove seq from loss list + // Otherwise, it's a resend, remove it from the loss list + _lossList.remove(seq); } } -void Connection::processControl(std::unique_ptr controlPacket) { +void Connection::processControl(unique_ptr controlPacket) { switch (controlPacket->getType()) { case ControlPacket::Type::ACK: break; diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 3f3b39524b..3b25c7f1b1 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -14,6 +14,7 @@ #include +#include "LossList.h" #include "SendQueue.h" class HifiSockAddr; @@ -36,6 +37,7 @@ public: private: + LossList _lossList; SeqNum _largestRecievedSeqNum; std::unique_ptr _sendQueue; }; diff --git a/libraries/networking/src/udt/LossList.cpp b/libraries/networking/src/udt/LossList.cpp new file mode 100644 index 0000000000..0a3eb5b4fb --- /dev/null +++ b/libraries/networking/src/udt/LossList.cpp @@ -0,0 +1,99 @@ +// +// LossList.cpp +// +// +// Created by Clement on 7/27/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "LossList.h" + +using namespace udt; +using namespace std; + +void LossList::append(SeqNum seq) { + assert(_lossList.back().second < seq); + + if (getLength() > 0 && _lossList.back().second + 1 == seq) { + ++_lossList.back().second; + } else { + _lossList.push_back(make_pair(seq, seq)); + } + _length += 1; +} + +void LossList::append(SeqNum start, SeqNum end) { + if (getLength() > 0 && _lossList.back().second + 1 == start) { + _lossList.back().second = end; + } else { + _lossList.push_back(make_pair(start, end)); + } + _length += seqlen(start, end); +} + +void LossList::remove(SeqNum seq) { + auto it = find_if(_lossList.begin(), _lossList.end(), [&seq](pair pair) { + return pair.first <= seq && seq <= pair.second; + }); + + if (it != end(_lossList)) { + if (it->first == it->second) { + _lossList.erase(it); + } else if (seq == it->first) { + ++it->first; + } else if (seq == it->second) { + --it->second; + } else { + auto temp = it->second; + it->second = seq - 1; + _lossList.insert(it, make_pair(seq + 1, temp)); + } + } +} + +void LossList::remove(SeqNum start, SeqNum end) { + // Find the first segment sharing sequence numbers + auto it = find_if(_lossList.begin(), _lossList.end(), [&start, &end](pair pair) { + return (pair.first <= start && start <= pair.second) || + (start <= pair.first && pair.first <= end); + }); + + // If we found one + if (it != _lossList.end()) { + + // While the end of the current segment is contained, either shorten it (first one only - sometimes) + // or remove it altogether since it is fully contained it the range + while (it != _lossList.end() && end >= it->second) { + if (start <= it->first) { + // Segment is contained, erase it. + it = _lossList.erase(it); + } else { + // Beginning of segment not contained, modify end of segment. + // Will only occur sometimes one the first loop + it->second = start - 1; + ++it; + } + } + + // There might be more to remove + if (it != _lossList.end() && it->first <= end) { + if (start <= it->first) { + // Truncate beginning of segment + it->first = end + 1; + } else { + // Cut it in half if the range we are removing is contained within one segment + auto temp = it->second; + it->second = start - 1; + _lossList.insert(it, make_pair(end + 1, temp)); + } + } + } +} + +SeqNum LossList::getFirstSeqNum() { + assert(getLength() > 0); + return _lossList.front().first; +} \ No newline at end of file diff --git a/libraries/networking/src/udt/LossList.h b/libraries/networking/src/udt/LossList.h new file mode 100644 index 0000000000..ee20104b4d --- /dev/null +++ b/libraries/networking/src/udt/LossList.h @@ -0,0 +1,42 @@ +// +// LossList.h +// +// +// Created by Clement on 7/27/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_LossList_h +#define hifi_LossList_h + +#include + +#include "SeqNum.h" + +namespace udt { + +class LossList { +public: + LossList() {} + + // Should always add at the end + void append(SeqNum seq); + void append(SeqNum start, SeqNum end); + + void remove(SeqNum seq); + void remove(SeqNum start, SeqNum end); + + int getLength() const { return _length; } + SeqNum getFirstSeqNum(); + +private: + std::list> _lossList; + int _length { 0 }; +}; + +} + +#endif // hifi_LossList_h \ No newline at end of file From f176c45a619c8ce97efb31c86ca954442bd1f001 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 27 Jul 2015 17:48:38 -0700 Subject: [PATCH 033/242] return an ACK2 from processControl --- libraries/networking/src/udt/Connection.cpp | 4 ++++ libraries/networking/src/udt/ControlPacket.cpp | 14 +++++++++++++- libraries/networking/src/udt/ControlPacket.h | 2 ++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 3753729123..b0c35d4678 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -132,6 +132,10 @@ void Connection::processControl(std::unique_ptr controlPacket) { case ControlPacket::ACK: break; case ControlPacket::ACK2: + // change the type of the packet to an ACK2 and send it back + controlPacket->setType(ControlPacket::ACK2); + _sendQueue->sendPacket(*controlPacket); + break; case ControlPacket::NAK: break; diff --git a/libraries/networking/src/udt/ControlPacket.cpp b/libraries/networking/src/udt/ControlPacket.cpp index 8b863c5a21..5f28b96750 100644 --- a/libraries/networking/src/udt/ControlPacket.cpp +++ b/libraries/networking/src/udt/ControlPacket.cpp @@ -94,12 +94,24 @@ ControlPacket& ControlPacket::operator=(ControlPacket&& other) { static const uint32_t CONTROL_BIT_MASK = 1 << (sizeof(ControlPacket::ControlBitAndType) - 1); +void ControlPacket::setType(udt::ControlPacket::Type type) { + _type = type; + + writeType(); +} + void ControlPacket::writeControlBitAndType() { ControlBitAndType* bitAndType = reinterpret_cast(_packet.get()); // write the control bit by OR'ing the current value with the CONTROL_BIT_MASK *bitAndType = (*bitAndType | CONTROL_BIT_MASK); - // write the type by OR'ing the type with the current value & CONTROL_BIT_MASK + writeType(); +} + +void ControlPacket::writeType() { + ControlBitAndType* bitAndType = reinterpret_cast(_packet.get()); + + // write the type by OR'ing the new type with the current value & CONTROL_BIT_MASK *bitAndType = (*bitAndType & CONTROL_BIT_MASK) | (_type << sizeof((ControlPacket::Type) - 1)); } diff --git a/libraries/networking/src/udt/ControlPacket.h b/libraries/networking/src/udt/ControlPacket.h index 6d1b7eb5d3..d17e12e182 100644 --- a/libraries/networking/src/udt/ControlPacket.h +++ b/libraries/networking/src/udt/ControlPacket.h @@ -41,6 +41,7 @@ public: virtual qint64 totalHeadersSize() const; // Cumulated size of all the headers Type getType() const { return _type; } + void setType(Type type); private: ControlPacket(Type type); @@ -54,6 +55,7 @@ private: // Header writers void writeControlBitAndType(); + void writeType(); Type _type; }; From 5833cea29deca7ebfe500660fba175fba469dad9 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 27 Jul 2015 18:51:17 -0700 Subject: [PATCH 034/242] Send out NAK packets --- libraries/networking/src/udt/Connection.cpp | 29 ++++++++++++++++----- libraries/networking/src/udt/Connection.h | 2 +- libraries/networking/src/udt/LossList.cpp | 2 +- libraries/networking/src/udt/LossList.h | 2 +- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 20d8a9cb92..b66110979b 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -90,10 +90,10 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { } void Connection::sendLightACK() const { - static const int LIGHT_ACK_PACKET_PAYLOAD_BYTES = 4; - // create the light ACK packet, make it static so we can re-use it + static const int LIGHT_ACK_PACKET_PAYLOAD_BYTES = sizeof(SeqNum); static auto lightACKPacket = ControlPacket::create(ControlPacket::ACK, LIGHT_ACK_PACKET_PAYLOAD_BYTES); + lightACKPacket->reset(); // We need to reset it every time. SeqNum nextACKNumber = nextACK(); @@ -103,15 +103,18 @@ void Connection::sendLightACK() const { } // pack in the ACK - memcpy(lightACKPacket->getPayload(), &nextACKNumber, sizeof(nextACKNumber)); + lightACKPacket->writePrimitive(nextACKNumber); - // have the send queue send off our packet + // have the send queue send off our packet immediately _sendQueue->sendPacket(*lightACKPacket); } SeqNum Connection::nextACK() const { - // TODO: check if we have a loss list - return _largestReceivedSeqNum + 1; + if (_lossList.getLength() > 0) { + return _lossList.getFirstSeqNum(); + } else { + return _largestReceivedSeqNum + 1; + } } void Connection::processReceivedSeqNum(SeqNum seq) { @@ -123,7 +126,19 @@ void Connection::processReceivedSeqNum(SeqNum seq) { _lossList.append(_largestReceivedSeqNum + 1, seq - 1); } - // TODO: Send loss report + // create the loss report packet, make it static so we can re-use it + static const int NAK_PACKET_PAYLOAD_BYTES = 2 * sizeof(SeqNum); + static auto lossReport = ControlPacket::create(ControlPacket::NAK, NAK_PACKET_PAYLOAD_BYTES); + lossReport->reset(); // We need to reset it every time. + + // pack in the loss report + lossReport->writePrimitive(_largestReceivedSeqNum + 1); + if (_largestReceivedSeqNum + 1 != seq - 1) { + lossReport->writePrimitive(seq - 1); + } + + // have the send queue send off our packet immediately + _sendQueue->sendPacket(*lossReport); } if (seq > _largestReceivedSeqNum) { diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index f67404a16b..5d8d309d94 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -42,7 +42,7 @@ public: void processControl(std::unique_ptr controlPacket); private: - LossList _lossList; + LossList _lossList; // List of all missing packets SeqNum _largestReceivedSeqNum; // The largest sequence number received from the peer SeqNum _lastSentACK; // The last sent ACK SeqNum _lastReceivedAcknowledgedACK; // The last sent ACK that has been acknowledged via an ACK2 from the peer diff --git a/libraries/networking/src/udt/LossList.cpp b/libraries/networking/src/udt/LossList.cpp index 0a3eb5b4fb..a8fe0912d5 100644 --- a/libraries/networking/src/udt/LossList.cpp +++ b/libraries/networking/src/udt/LossList.cpp @@ -93,7 +93,7 @@ void LossList::remove(SeqNum start, SeqNum end) { } } -SeqNum LossList::getFirstSeqNum() { +SeqNum LossList::getFirstSeqNum() const { assert(getLength() > 0); return _lossList.front().first; } \ No newline at end of file diff --git a/libraries/networking/src/udt/LossList.h b/libraries/networking/src/udt/LossList.h index ee20104b4d..dd7f1de2ff 100644 --- a/libraries/networking/src/udt/LossList.h +++ b/libraries/networking/src/udt/LossList.h @@ -30,7 +30,7 @@ public: void remove(SeqNum start, SeqNum end); int getLength() const { return _length; } - SeqNum getFirstSeqNum(); + SeqNum getFirstSeqNum() const; private: std::list> _lossList; From 89a53886cb828e766b50dc8eb8e59c2e33b053ff Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 27 Jul 2015 20:44:12 -0700 Subject: [PATCH 035/242] Read ACK, NAK control packet --- libraries/networking/src/udt/Connection.cpp | 37 +++++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index b66110979b..2f901d6540 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -36,6 +36,7 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { // setup the ACK packet, make it static so we can re-use it static auto ackPacket = ControlPacket::create(ControlPacket::ACK, ACK_PACKET_PAYLOAD_BYTES); + ackPacket->reset(); // We need to reset it every time. auto currentTime = high_resolution_clock::now(); @@ -63,9 +64,6 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { } } - // reset the ACK packet so we can fill it up and have it figure out what size it is - ackPacket->reset(); - // pack in the ACK sub-sequence number ackPacket->writePrimitive(_currentACKSubSequenceNumber++); @@ -152,13 +150,38 @@ void Connection::processReceivedSeqNum(SeqNum seq) { void Connection::processControl(unique_ptr controlPacket) { switch (controlPacket->getType()) { - case ControlPacket::ACK: + case ControlPacket::ACK: { + // read the ACK sub-sequence number + SeqNum currentACKSubSequenceNumber; + controlPacket->readPrimitive(¤tACKSubSequenceNumber); + + // read the ACK number + SeqNum nextACKNumber; + controlPacket->readPrimitive(&nextACKNumber); + + // read the RTT and variance + int32_t rtt{ 0 }, rttVariance{ 0 }; + controlPacket->readPrimitive(&rtt); + controlPacket->readPrimitive(&rttVariance); + break; - case ControlPacket::ACK2: + } + case ControlPacket::ACK2: { + break; - case ControlPacket::NAK: + } + case ControlPacket::NAK: { + // read the loss report + SeqNum start, end; + controlPacket->readPrimitive(&start); + if (controlPacket->bytesLeftToRead() >= (qint64)sizeof(SeqNum)) { + controlPacket->readPrimitive(&end); + } break; - case ControlPacket::PacketPair: + } + case ControlPacket::PacketPair: { + break; + } } } From 3efd1be3d656347a1ad3abe345604eb35a85f0d5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 09:54:46 -0700 Subject: [PATCH 036/242] resolve conflicts for removal of ACK2 process --- libraries/networking/src/udt/Connection.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 2a482ec4ec..18d3658270 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -167,10 +167,6 @@ void Connection::processControl(unique_ptr controlPacket) { break; } case ControlPacket::ACK2: { - // change the type of the packet to an ACK2 and send it back - controlPacket->setType(ControlPacket::ACK2); - _sendQueue->sendPacket(*controlPacket); - break; } case ControlPacket::NAK: { From c14fee2ee8d1fa316ea4a547e8b8695830e50247 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 09:56:31 -0700 Subject: [PATCH 037/242] remove initialization of rtt and rttVariance --- libraries/networking/src/udt/Connection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 18d3658270..abe0ef8986 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -160,7 +160,7 @@ void Connection::processControl(unique_ptr controlPacket) { controlPacket->readPrimitive(&nextACKNumber); // read the RTT and variance - int32_t rtt{ 0 }, rttVariance{ 0 }; + int32_t rtt, rttVariance; controlPacket->readPrimitive(&rtt); controlPacket->readPrimitive(&rttVariance); From 17d8085ab58fb42d02c54a377f19879836d9b218 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 10:01:38 -0700 Subject: [PATCH 038/242] add process methods for control packet types --- libraries/networking/src/udt/Connection.cpp | 54 +++++++++++++-------- libraries/networking/src/udt/Connection.h | 4 ++ 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index abe0ef8986..d12f2b9fdb 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -91,7 +91,6 @@ void Connection::sendLightACK() const { // create the light ACK packet, make it static so we can re-use it static const int LIGHT_ACK_PACKET_PAYLOAD_BYTES = sizeof(SeqNum); static auto lightACKPacket = ControlPacket::create(ControlPacket::ACK, LIGHT_ACK_PACKET_PAYLOAD_BYTES); - lightACKPacket->reset(); // We need to reset it every time. SeqNum nextACKNumber = nextACK(); @@ -100,6 +99,9 @@ void Connection::sendLightACK() const { return; } + // reset the lightACKPacket before we go to write the ACK to it + lightACKPacket->reset(); + // pack in the ACK lightACKPacket->writePrimitive(nextACKNumber); @@ -151,31 +153,15 @@ void Connection::processReceivedSeqNum(SeqNum seq) { void Connection::processControl(unique_ptr controlPacket) { switch (controlPacket->getType()) { case ControlPacket::ACK: { - // read the ACK sub-sequence number - SeqNum currentACKSubSequenceNumber; - controlPacket->readPrimitive(¤tACKSubSequenceNumber); - - // read the ACK number - SeqNum nextACKNumber; - controlPacket->readPrimitive(&nextACKNumber); - - // read the RTT and variance - int32_t rtt, rttVariance; - controlPacket->readPrimitive(&rtt); - controlPacket->readPrimitive(&rttVariance); - + processACK(move(controlPacket)); break; } case ControlPacket::ACK2: { + processACK2(move(controlPacket)); break; } case ControlPacket::NAK: { - // read the loss report - SeqNum start, end; - controlPacket->readPrimitive(&start); - if (controlPacket->bytesLeftToRead() >= (qint64)sizeof(SeqNum)) { - controlPacket->readPrimitive(&end); - } + processNAK(move(controlPacket)); break; } case ControlPacket::PacketPair: { @@ -184,3 +170,31 @@ void Connection::processControl(unique_ptr controlPacket) { } } } + +void Connection::processACK(std::unique_ptr controlPacket) { + // read the ACK sub-sequence number + SeqNum currentACKSubSequenceNumber; + controlPacket->readPrimitive(¤tACKSubSequenceNumber); + + // read the ACK number + SeqNum nextACKNumber; + controlPacket->readPrimitive(&nextACKNumber); + + // read the RTT and variance + int32_t rtt, rttVariance; + controlPacket->readPrimitive(&rtt); + controlPacket->readPrimitive(&rttVariance); +} + +void Connection::processACK2(std::unique_ptr controlPacket) { + +} + +void Connection::processNAK(std::unique_ptr controlPacket) { + // read the loss report + SeqNum start, end; + controlPacket->readPrimitive(&start); + if (controlPacket->bytesLeftToRead() >= (qint64)sizeof(SeqNum)) { + controlPacket->readPrimitive(&end); + } +} diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 5d8d309d94..f96a571de4 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -42,6 +42,10 @@ public: void processControl(std::unique_ptr controlPacket); private: + void processACK(std::unique_ptr controlPacket); + void processACK2(std::unique_ptr controlPacket); + void processNAK(std::unique_ptr controlPacket); + LossList _lossList; // List of all missing packets SeqNum _largestReceivedSeqNum; // The largest sequence number received from the peer SeqNum _lastSentACK; // The last sent ACK From 1c6b14d17c439de9e8dbb8944dc5ac8f6db195b8 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 10:24:56 -0700 Subject: [PATCH 039/242] add file path to clement's header --- libraries/networking/src/udt/SeqNum.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/SeqNum.cpp b/libraries/networking/src/udt/SeqNum.cpp index 506b8bf3c9..668c1f131a 100644 --- a/libraries/networking/src/udt/SeqNum.cpp +++ b/libraries/networking/src/udt/SeqNum.cpp @@ -1,6 +1,6 @@ // // SeqNum.cpp -// +// libraries/networking/src/udt // // Created by Clement on 7/23/15. // Copyright 2015 High Fidelity, Inc. From c32c95c707adf8ffbe4bb7ab5872f6d6a6509f17 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 10:33:07 -0700 Subject: [PATCH 040/242] complete implementation of processLightACK --- libraries/networking/src/udt/Connection.cpp | 22 ++++++++++++++++++++- libraries/networking/src/udt/Connection.h | 3 +++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index d12f2b9fdb..184597004e 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -153,7 +153,12 @@ void Connection::processReceivedSeqNum(SeqNum seq) { void Connection::processControl(unique_ptr controlPacket) { switch (controlPacket->getType()) { case ControlPacket::ACK: { - processACK(move(controlPacket)); + if (controlPacket->getPayloadSize() == sizeof(SeqNum)) { + processLightACK(move(controlPacket)); + } else { + processACK(move(controlPacket)); + } + break; } case ControlPacket::ACK2: { @@ -186,6 +191,21 @@ void Connection::processACK(std::unique_ptr controlPacket) { controlPacket->readPrimitive(&rttVariance); } +void Connection::processLightACK(std::unique_ptr controlPacket) { + // read the ACKed sequence number + SeqNum ack; + controlPacket->readPrimitive(&ack); + + // must be larger than the last received ACK to be processed + if (ack > _lastReceivedACK) { + // decrease the flow window size by the offset between the last received ACK and this ACK + _flowWindowSize -= seqoff(_lastReceivedACK, ack); + + // update the last received ACK to the this one + _lastReceivedACK = ack; + } +} + void Connection::processACK2(std::unique_ptr controlPacket) { } diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index f96a571de4..f1d2210b2f 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -43,17 +43,20 @@ public: private: void processACK(std::unique_ptr controlPacket); + void processLightACK(std::unique_ptr controlPacket); void processACK2(std::unique_ptr controlPacket); void processNAK(std::unique_ptr controlPacket); LossList _lossList; // List of all missing packets SeqNum _largestReceivedSeqNum; // The largest sequence number received from the peer SeqNum _lastSentACK; // The last sent ACK + SeqNum _lastReceivedACK; // The last ACK received SeqNum _lastReceivedAcknowledgedACK; // The last sent ACK that has been acknowledged via an ACK2 from the peer SeqNum _currentACKSubSequenceNumber; // The current ACK sub-sequence number (used for Acknowledgment of ACKs) int32_t _rtt; // RTT, in milliseconds int32_t _rttVariance; // RTT variance + int _flowWindowSize; // Flow control window size std::chrono::high_resolution_clock::time_point _lastACKTime; From 8b71462dc14c499f2a33c9d95b96f13d1c6d3476 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 28 Jul 2015 10:36:00 -0700 Subject: [PATCH 041/242] Remove unnecessary headers --- libraries/gpu/src/gpu/State.h | 1 - libraries/networking/src/udt/Connection.cpp | 9 +++------ libraries/octree/src/OctreeEditPacketSender.h | 10 +++++----- tests/render-utils/src/main.cpp | 1 - tests/ui/src/main.cpp | 1 - 5 files changed, 8 insertions(+), 14 deletions(-) diff --git a/libraries/gpu/src/gpu/State.h b/libraries/gpu/src/gpu/State.h index 39cad1445f..ac350b0f5f 100755 --- a/libraries/gpu/src/gpu/State.h +++ b/libraries/gpu/src/gpu/State.h @@ -15,7 +15,6 @@ #include #include -#include #include // Why a macro and not a fancy template you will ask me ? diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index d12f2b9fdb..386ad13bb6 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -152,18 +152,15 @@ void Connection::processReceivedSeqNum(SeqNum seq) { void Connection::processControl(unique_ptr controlPacket) { switch (controlPacket->getType()) { - case ControlPacket::ACK: { + case ControlPacket::ACK: processACK(move(controlPacket)); break; - } - case ControlPacket::ACK2: { + case ControlPacket::ACK2: processACK2(move(controlPacket)); break; - } - case ControlPacket::NAK: { + case ControlPacket::NAK: processNAK(move(controlPacket)); break; - } case ControlPacket::PacketPair: { break; diff --git a/libraries/octree/src/OctreeEditPacketSender.h b/libraries/octree/src/OctreeEditPacketSender.h index be44aec88d..d10870a221 100644 --- a/libraries/octree/src/OctreeEditPacketSender.h +++ b/libraries/octree/src/OctreeEditPacketSender.h @@ -12,19 +12,19 @@ #ifndef hifi_OctreeEditPacketSender_h #define hifi_OctreeEditPacketSender_h +#include + #include #include #include "JurisdictionMap.h" #include "SentPacketHistory.h" -namespace std { - template <> struct hash { +template <> struct std::hash { size_t operator()(const QUuid& uuid) const { - return qHash(uuid); + return qHash(uuid); } - }; -} +}; /// Utility for processing, packing, queueing and sending of outbound edit messages. class OctreeEditPacketSender : public PacketSender { diff --git a/tests/render-utils/src/main.cpp b/tests/render-utils/src/main.cpp index 3b7eb18368..0806e0b6df 100644 --- a/tests/render-utils/src/main.cpp +++ b/tests/render-utils/src/main.cpp @@ -26,7 +26,6 @@ #include #include -#include #include #include diff --git a/tests/ui/src/main.cpp b/tests/ui/src/main.cpp index f5647bd176..fa45e807ae 100644 --- a/tests/ui/src/main.cpp +++ b/tests/ui/src/main.cpp @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include From b25d860be4025dec88df8f1c1cf2032eb64ff9fe Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 28 Jul 2015 10:36:19 -0700 Subject: [PATCH 042/242] Remove seqcmp --- libraries/networking/src/udt/CongestionControl.cpp | 2 +- libraries/networking/src/udt/SeqNum.cpp | 5 ----- libraries/networking/src/udt/SeqNum.h | 2 -- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 4f32be0d06..a75f62bb5f 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -106,7 +106,7 @@ void UdtCC::onLoss(const std::vector& losslist) { _loss = true; - if (seqcmp(losslist[0], _lastDecSeq) > 0) { + if (losslist[0] > _lastDecSeq) { _lastDecPeriod = _packetSendPeriod; _packetSendPeriod = ceil(_packetSendPeriod * 1.125); diff --git a/libraries/networking/src/udt/SeqNum.cpp b/libraries/networking/src/udt/SeqNum.cpp index 506b8bf3c9..1e853bf4f3 100644 --- a/libraries/networking/src/udt/SeqNum.cpp +++ b/libraries/networking/src/udt/SeqNum.cpp @@ -11,11 +11,6 @@ #include "SeqNum.h" -int udt::seqcmp(const SeqNum& seq1, const SeqNum& seq2) { - return (glm::abs(seq1._value - seq2._value) < SeqNum::THRESHOLD) ? (seq1._value - seq2._value) - : (seq2._value - seq1._value); -} - int udt::seqlen(const SeqNum& seq1, const SeqNum& seq2) { return (seq1._value <= seq2._value) ? (seq2._value - seq1._value + 1) : (seq2._value - seq1._value + SeqNum::MAX + 2); diff --git a/libraries/networking/src/udt/SeqNum.h b/libraries/networking/src/udt/SeqNum.h index 5ae85f3dcf..90b327e4c9 100644 --- a/libraries/networking/src/udt/SeqNum.h +++ b/libraries/networking/src/udt/SeqNum.h @@ -84,7 +84,6 @@ public: friend SeqNum operator-(const SeqNum a, const Type& b); friend SeqNum operator-(const Type& a, const SeqNum b); - friend int seqcmp(const SeqNum& seq1, const SeqNum& seq2); friend int seqlen(const SeqNum& seq1, const SeqNum& seq2); friend int seqoff(const SeqNum& seq1, const SeqNum& seq2); @@ -132,7 +131,6 @@ inline SeqNum operator-(const SeqNum::Type& a, SeqNum b) { return b; } -int seqcmp(const SeqNum& seq1, const SeqNum& seq2); int seqlen(const SeqNum& seq1, const SeqNum& seq2); int seqoff(const SeqNum& seq1, const SeqNum& seq2); From c3995a6e235e247ed5cb7cc4ee7799c96d3856eb Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 28 Jul 2015 11:30:50 -0700 Subject: [PATCH 043/242] Fix LossList length not cumputed --- libraries/networking/src/udt/LossList.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/LossList.cpp b/libraries/networking/src/udt/LossList.cpp index a8fe0912d5..b1d3165f34 100644 --- a/libraries/networking/src/udt/LossList.cpp +++ b/libraries/networking/src/udt/LossList.cpp @@ -51,6 +51,7 @@ void LossList::remove(SeqNum seq) { it->second = seq - 1; _lossList.insert(it, make_pair(seq + 1, temp)); } + _length -= 1; } } @@ -68,11 +69,13 @@ void LossList::remove(SeqNum start, SeqNum end) { // or remove it altogether since it is fully contained it the range while (it != _lossList.end() && end >= it->second) { if (start <= it->first) { - // Segment is contained, erase it. + // Segment is contained, update new length and erase it. + _length -= seqlen(it->first, it->second); it = _lossList.erase(it); } else { // Beginning of segment not contained, modify end of segment. // Will only occur sometimes one the first loop + _length -= seqlen(start, it->second); it->second = start - 1; ++it; } @@ -82,9 +85,11 @@ void LossList::remove(SeqNum start, SeqNum end) { if (it != _lossList.end() && it->first <= end) { if (start <= it->first) { // Truncate beginning of segment + _length -= seqlen(it->first, end); it->first = end + 1; } else { // Cut it in half if the range we are removing is contained within one segment + _length -= seqlen(start, end); auto temp = it->second; it->second = start - 1; _lossList.insert(it, make_pair(end + 1, temp)); From c7ae4d5e59a2d5746647162f04ea8e96b0a6ab4e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 11:47:57 -0700 Subject: [PATCH 044/242] implement more of processACK in Connection --- libraries/networking/src/udt/Connection.cpp | 82 +++++++++++++++++++-- libraries/networking/src/udt/Connection.h | 13 +++- libraries/networking/src/udt/SendQueue.cpp | 1 + libraries/networking/src/udt/SendQueue.h | 4 + 4 files changed, 92 insertions(+), 8 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 184597004e..abdfcbcdb8 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -39,6 +39,7 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { ackPacket->reset(); // We need to reset it every time. auto currentTime = high_resolution_clock::now(); + static high_resolution_clock::time_point lastACKSendTime; SeqNum nextACKNumber = nextACK(); @@ -58,7 +59,8 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { } else if (nextACKNumber == _lastSentACK) { // We already sent this ACK, but check if we should re-send it. // We will re-send if it has been more than RTT + (4 * RTT variance) since the last ACK - milliseconds sinceLastACK = duration_cast(currentTime - _lastACKTime); + milliseconds sinceLastACK = duration_cast(currentTime - lastACKSendTime); + if (sinceLastACK.count() < (_rtt + (4 * _rttVariance))) { return; } @@ -80,7 +82,7 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { // pack in the receive speed and bandwidth // record this as the last ACK send time - _lastACKTime = high_resolution_clock::now(); + lastACKSendTime = high_resolution_clock::now(); } // have the send queue send off our packet @@ -181,14 +183,84 @@ void Connection::processACK(std::unique_ptr controlPacket) { SeqNum currentACKSubSequenceNumber; controlPacket->readPrimitive(¤tACKSubSequenceNumber); - // read the ACK number - SeqNum nextACKNumber; - controlPacket->readPrimitive(&nextACKNumber); + // check if we need send an ACK2 for this ACK + auto currentTime = high_resolution_clock::now(); + static high_resolution_clock::time_point lastACK2SendTime; + + milliseconds sinceLastACK2 = duration_cast(currentTime - lastACK2SendTime); + + if (sinceLastACK2.count() > _synInterval || currentACKSubSequenceNumber == _lastSentACK2) { + // setup a static ACK2 packet we will re-use + static const int ACK2_PAYLOAD_BYTES = sizeof(SeqNum); + static auto ack2Packet = ControlPacket::create(ControlPacket::ACK2, ACK2_PAYLOAD_BYTES); + + // reset the ACK2 Packet before writing the sub-sequence number to it + ack2Packet->reset(); + + // write the sub sequence number for this ACK2 + ack2Packet->writePrimitive(currentACKSubSequenceNumber); + + // update the last sent ACK2 and the last ACK2 send time + _lastSentACK2 = currentACKSubSequenceNumber; + lastACK2SendTime = high_resolution_clock::now(); + } + + // read the ACKed sequence number + SeqNum ack; + controlPacket->readPrimitive(&ack); + + // validate that this isn't a BS ACK + if (ack > (_sendQueue->getCurrentSeqNum() + 1)) { + // in UDT they specifically break the connection here - do we want to do anything? + return; + } // read the RTT and variance int32_t rtt, rttVariance; controlPacket->readPrimitive(&rtt); controlPacket->readPrimitive(&rttVariance); + + // read the desired flow window size + int flowWindowSize; + controlPacket->readPrimitive(&flowWindowSize); + + if (ack <= _lastReceivedACK) { + // this is a valid ACKed sequence number - update the flow window size and the last received ACK + _flowWindowSize = flowWindowSize; + _lastReceivedACK = ack; + } + + // make sure this isn't a repeated ACK + if (ack <= SeqNum(_atomicLastReceivedACK)) { + return; + } + + // ACK the send queue so it knows what was received + _sendQueue->ack(ack); + + // update the atomic for last received ACK, the send queue uses this to re-transmit + _atomicLastReceivedACK.store((uint32_t) _lastReceivedACK); + + // remove everything up to this ACK from the sender loss list + + // update the RTT + _rttVariance = (_rttVariance * 3 + abs(rtt - _rtt)) >> 2; + _rtt = (_rtt * 7 + rtt) >> 3; + + // set the RTT for congestion control + + if (controlPacket->getPayloadSize() > (qint64) (sizeof(SeqNum) + sizeof(SeqNum) + sizeof(rtt) + sizeof(rttVariance))) { + int32_t deliveryRate, bandwidth; + controlPacket->readPrimitive(&deliveryRate); + controlPacket->readPrimitive(&bandwidth); + + // set the delivery rate and bandwidth for congestion control + } + + // fire the onACK callback for congestion control + + // update the total count of received ACKs + ++_totalReceivedACKs; } void Connection::processLightACK(std::unique_ptr controlPacket) { diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index f1d2210b2f..de60b88b37 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -38,6 +38,8 @@ public: SeqNum nextACK() const; + SeqNum getLastReceivedACK() const { return SeqNum(_atomicLastReceivedACK); } + void processReceivedSeqNum(SeqNum seq); void processControl(std::unique_ptr controlPacket); @@ -47,19 +49,24 @@ private: void processACK2(std::unique_ptr controlPacket); void processNAK(std::unique_ptr controlPacket); + int _synInterval; // Periodical Rate Control Interval, defaults to 10ms + LossList _lossList; // List of all missing packets SeqNum _largestReceivedSeqNum; // The largest sequence number received from the peer - SeqNum _lastSentACK; // The last sent ACK SeqNum _lastReceivedACK; // The last ACK received + std::atomic _atomicLastReceivedACK; // Atomic for thread-safe get of last ACK received SeqNum _lastReceivedAcknowledgedACK; // The last sent ACK that has been acknowledged via an ACK2 from the peer SeqNum _currentACKSubSequenceNumber; // The current ACK sub-sequence number (used for Acknowledgment of ACKs) + SeqNum _lastSentACK; // The last sent ACK + SeqNum _lastSentACK2; // The last sent ACK sub-sequence number in an ACK2 + + int _totalReceivedACKs { 0 }; + int32_t _rtt; // RTT, in milliseconds int32_t _rttVariance; // RTT variance int _flowWindowSize; // Flow control window size - std::chrono::high_resolution_clock::time_point _lastACKTime; - std::unique_ptr _sendQueue; }; diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 65c1807d19..93b90379c7 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -114,6 +114,7 @@ void SendQueue::sendNextPacket() { if (_nextPacket) { _nextPacket->writeSequenceNumber(++_currentSeqNum); sendPacket(*_nextPacket); + _atomicCurrentSeqNum.store((uint32_t) _currentSeqNum); // Insert the packet we have just sent in the sent list QWriteLocker locker(&_sentLock); diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 9c14c19ede..729ed9e305 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -42,6 +42,8 @@ public: quint64 getLastSendTimestamp() const { return _lastSendTimestamp; } + SeqNum getCurrentSeqNum() const { return SeqNum(_atomicCurrentSeqNum); } + int getPacketSendPeriod() const { return _packetSendPeriod; } void setPacketSendPeriod(int newPeriod) { _packetSendPeriod = newPeriod; } @@ -73,6 +75,8 @@ private: SeqNum _currentSeqNum; // Last sequence number sent out SeqNum _lastAck; // ACKed sequence number + std::atomic _atomicCurrentSeqNum; // Atomic for last sequence number sent out + std::unique_ptr _sendTimer; // Send timer std::atomic _packetSendPeriod { 0 }; // Interval between two packet send envent in msec std::atomic _lastSendTimestamp { 0 }; // Record last time of packet departure From 155d339c2f47118a0b1214849af77650082fa17d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 11:53:40 -0700 Subject: [PATCH 045/242] remove RTT variance that isn't handled --- libraries/networking/src/udt/Connection.cpp | 21 ++++++++++++--------- libraries/networking/src/udt/Connection.h | 2 ++ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index abdfcbcdb8..dc8e3c9c11 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -32,7 +32,7 @@ void Connection::send(unique_ptr packet) { void Connection::sendACK(bool wasCausedBySyncTimeout) { static const int ACK_PACKET_PAYLOAD_BYTES = sizeof(_lastSentACK) + sizeof(_currentACKSubSequenceNumber) - + sizeof(_rtt) + sizeof(_rttVariance) + sizeof(int32_t) + sizeof(int32_t); + + sizeof(_rtt) + sizeof(int32_t) + sizeof(int32_t); // setup the ACK packet, make it static so we can re-use it static auto ackPacket = ControlPacket::create(ControlPacket::ACK, ACK_PACKET_PAYLOAD_BYTES); @@ -74,7 +74,6 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { // pack in the RTT and variance ackPacket->writePrimitive(_rtt); - ackPacket->writePrimitive(_rttVariance); // pack the available buffer size - must be a minimum of 2 @@ -215,10 +214,9 @@ void Connection::processACK(std::unique_ptr controlPacket) { return; } - // read the RTT and variance - int32_t rtt, rttVariance; + // read the RTT + int32_t rtt; controlPacket->readPrimitive(&rtt); - controlPacket->readPrimitive(&rttVariance); // read the desired flow window size int flowWindowSize; @@ -244,12 +242,11 @@ void Connection::processACK(std::unique_ptr controlPacket) { // remove everything up to this ACK from the sender loss list // update the RTT - _rttVariance = (_rttVariance * 3 + abs(rtt - _rtt)) >> 2; - _rtt = (_rtt * 7 + rtt) >> 3; + updateRTT(rtt); // set the RTT for congestion control - if (controlPacket->getPayloadSize() > (qint64) (sizeof(SeqNum) + sizeof(SeqNum) + sizeof(rtt) + sizeof(rttVariance))) { + if (controlPacket->getPayloadSize() > (qint64) (sizeof(SeqNum) + sizeof(SeqNum) + sizeof(rtt))) { int32_t deliveryRate, bandwidth; controlPacket->readPrimitive(&deliveryRate); controlPacket->readPrimitive(&bandwidth); @@ -260,7 +257,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { // fire the onACK callback for congestion control // update the total count of received ACKs - ++_totalReceivedACKs; + ++_totalReceivedACKs; } void Connection::processLightACK(std::unique_ptr controlPacket) { @@ -290,3 +287,9 @@ void Connection::processNAK(std::unique_ptr controlPacket) { controlPacket->readPrimitive(&end); } } + +void Connection::updateRTT(int32_t rtt) { + // this updates the RTT using exponential weighted moving average + _rttVariance = (_rttVariance * 3 + abs(rtt - _rtt)) >> 2; + _rtt = (_rtt * 7 + rtt) >> 3; +} diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index de60b88b37..d58cd21cff 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -49,6 +49,8 @@ private: void processACK2(std::unique_ptr controlPacket); void processNAK(std::unique_ptr controlPacket); + void updateRTT(int32_t rtt); + int _synInterval; // Periodical Rate Control Interval, defaults to 10ms LossList _lossList; // List of all missing packets From 0b8feed327ece8df87f2afc3d85f68a2a503659b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 13:01:35 -0700 Subject: [PATCH 046/242] add a map of sentACKs for ACK2 processing --- libraries/networking/src/udt/Connection.cpp | 27 ++++++++++++++++++++- libraries/networking/src/udt/Connection.h | 5 ++++ libraries/networking/src/udt/SendQueue.h | 2 +- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index dc8e3c9c11..0e6300b020 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -86,6 +86,9 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { // have the send queue send off our packet _sendQueue->sendPacket(*ackPacket); + + // write this ACK to the map of sent ACKs + _sentACKs[_currentACKSubSequenceNumber] = { nextACKNumber, high_resolution_clock::now() }; } void Connection::sendLightACK() const { @@ -276,7 +279,29 @@ void Connection::processLightACK(std::unique_ptr controlPacket) { } void Connection::processACK2(std::unique_ptr controlPacket) { - + // pull the sub sequence number from the packet + SeqNum subSequenceNumber; + controlPacket->readPrimitive(&subSequenceNumber); + + // check if we had that subsequence number in our map + auto it = _sentACKs.find(subSequenceNumber); + if (it != _sentACKs.end()) { + // update the RTT using the ACK window + SequenceNumberTimePair& pair = it->second; + + // calculate the RTT (time now - time ACK sent) + auto now = high_resolution_clock::now(); + int rtt = duration_cast(now - pair.second).count(); + + updateRTT(rtt); + + // set the RTT for congestion control + + // update the last ACKed ACK + if (pair.first > _lastReceivedAcknowledgedACK) { + _lastReceivedAcknowledgedACK = pair.first; + } + } } void Connection::processNAK(std::unique_ptr controlPacket) { diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index d58cd21cff..2643e06a43 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -29,6 +29,9 @@ class Socket; class Connection { public: + using SequenceNumberTimePair = std::pair>; + using SentACKMap = std::unordered_map; + Connection(Socket* parentSocket, HifiSockAddr destination); void send(std::unique_ptr packet); @@ -69,6 +72,8 @@ private: int32_t _rttVariance; // RTT variance int _flowWindowSize; // Flow control window size + SentACKMap _sentACKs; // Map of ACK sub-sequence numbers to ACKed sequence number and sent time + std::unique_ptr _sendQueue; }; diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 729ed9e305..b30a7a0cbb 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -73,7 +73,7 @@ private: Socket* _socket { nullptr }; // Socket to send packet on HifiSockAddr _destination; // Destination addr SeqNum _currentSeqNum; // Last sequence number sent out - SeqNum _lastAck; // ACKed sequence number + SeqNum _lastAck; // Last ACKed sequence number std::atomic _atomicCurrentSeqNum; // Atomic for last sequence number sent out From a8371cc3ed1b6ff7b2f87ca2457e74eea8a33989 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 13:28:01 -0700 Subject: [PATCH 047/242] send the actual sequence number being ACKed --- libraries/networking/src/udt/Connection.cpp | 6 +++--- libraries/networking/src/udt/Connection.h | 14 +++++++------- libraries/networking/src/udt/SendQueue.cpp | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 0e6300b020..dfe25176a2 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -1,6 +1,6 @@ // // Connection.cpp -// +// libraries/networking/src/udt // // Created by Clement on 7/27/15. // Copyright 2015 High Fidelity, Inc. @@ -117,7 +117,7 @@ SeqNum Connection::nextACK() const { if (_lossList.getLength() > 0) { return _lossList.getFirstSeqNum(); } else { - return _largestReceivedSeqNum + 1; + return _largestReceivedSeqNum; } } @@ -212,7 +212,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { controlPacket->readPrimitive(&ack); // validate that this isn't a BS ACK - if (ack > (_sendQueue->getCurrentSeqNum() + 1)) { + if (ack > _sendQueue->getCurrentSeqNum()) { // in UDT they specifically break the connection here - do we want to do anything? return; } diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 2643e06a43..ec2736ae8d 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -1,6 +1,6 @@ // // Connection.h -// +// libraries/networking/src/udt // // Created by Clement on 7/27/15. // Copyright 2015 High Fidelity, Inc. @@ -29,7 +29,7 @@ class Socket; class Connection { public: - using SequenceNumberTimePair = std::pair>; + using SequenceNumberTimePair = std::pair; using SentACKMap = std::unordered_map; Connection(Socket* parentSocket, HifiSockAddr destination); @@ -57,13 +57,13 @@ private: int _synInterval; // Periodical Rate Control Interval, defaults to 10ms LossList _lossList; // List of all missing packets - SeqNum _largestReceivedSeqNum; // The largest sequence number received from the peer - SeqNum _lastReceivedACK; // The last ACK received - std::atomic _atomicLastReceivedACK; // Atomic for thread-safe get of last ACK received - SeqNum _lastReceivedAcknowledgedACK; // The last sent ACK that has been acknowledged via an ACK2 from the peer + SeqNum _largestReceivedSeqNum { SeqNum::MAX }; // The largest sequence number received from the peer + SeqNum _lastReceivedACK { SeqNum::MAX }; // The last ACK received + std::atomic _atomicLastReceivedACK { (uint32_t) SeqNum::MAX }; // Atomic for thread-safe get of last ACK received + SeqNum _lastReceivedAcknowledgedACK { SeqNum::MAX }; // The last sent ACK that has been acknowledged via an ACK2 from the peer SeqNum _currentACKSubSequenceNumber; // The current ACK sub-sequence number (used for Acknowledgment of ACKs) - SeqNum _lastSentACK; // The last sent ACK + SeqNum _lastSentACK { SeqNum::MAX }; // The last sent ACK SeqNum _lastSentACK2; // The last sent ACK sub-sequence number in an ACK2 int _totalReceivedACKs { 0 }; diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 93b90379c7..bc88c646df 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -90,7 +90,7 @@ void SendQueue::ack(SeqNum ack) { } QWriteLocker locker(&_sentLock); - for (auto seq = _lastAck; seq != ack; ++seq) { + for (auto seq = _lastAck; seq <= ack; ++seq) { _sentPackets.erase(seq); } From 45f6a984af806d973ee8bedf575fe33658411684 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 13:28:58 -0700 Subject: [PATCH 048/242] add a comment to SendQueue ack --- libraries/networking/src/udt/SendQueue.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index bc88c646df..0648ef0292 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -89,6 +89,7 @@ void SendQueue::ack(SeqNum ack) { return; } + // remove any ACKed packets from the map of sent packets QWriteLocker locker(&_sentLock); for (auto seq = _lastAck; seq <= ack; ++seq) { _sentPackets.erase(seq); From 606356cdcfec6c62c96d3e0a756afa222d1cf412 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 13:39:19 -0700 Subject: [PATCH 049/242] rename SeqNum to SequenceNumber --- .../networking/src/udt/CongestionControl.cpp | 6 +- .../networking/src/udt/CongestionControl.h | 18 +-- libraries/networking/src/udt/Connection.cpp | 54 +++---- libraries/networking/src/udt/Connection.h | 24 +-- libraries/networking/src/udt/LossList.cpp | 14 +- libraries/networking/src/udt/LossList.h | 16 +- libraries/networking/src/udt/Packet.cpp | 8 +- libraries/networking/src/udt/Packet.h | 9 +- libraries/networking/src/udt/SendQueue.cpp | 10 +- libraries/networking/src/udt/SendQueue.h | 18 +-- libraries/networking/src/udt/SeqNum.cpp | 34 ---- libraries/networking/src/udt/SeqNum.h | 147 ------------------ .../networking/src/udt/SequenceNumber.cpp | 34 ++++ libraries/networking/src/udt/SequenceNumber.h | 147 ++++++++++++++++++ libraries/networking/src/udt/Socket.h | 4 +- 15 files changed, 271 insertions(+), 272 deletions(-) delete mode 100644 libraries/networking/src/udt/SeqNum.cpp delete mode 100644 libraries/networking/src/udt/SeqNum.h create mode 100644 libraries/networking/src/udt/SequenceNumber.cpp create mode 100644 libraries/networking/src/udt/SequenceNumber.h diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 4f32be0d06..070e946c92 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -21,13 +21,13 @@ void UdtCC::init() { setAckTimer(_rcInterval); _lastAck = _sendCurrSeqNum; - _lastDecSeq = SeqNum{ SeqNum::MAX }; + _lastDecSeq = SequenceNumber{ SequenceNumber::MAX }; _congestionWindowSize = 16.0; _packetSendPeriod = 1.0; } -void UdtCC::onACK(SeqNum ackNum) { +void UdtCC::onACK(SequenceNumber ackNum) { int64_t B = 0; double inc = 0; // Note: 1/24/2012 @@ -89,7 +89,7 @@ void UdtCC::onACK(SeqNum ackNum) { _packetSendPeriod = (_packetSendPeriod * _rcInterval) / (_packetSendPeriod * inc + _rcInterval); } -void UdtCC::onLoss(const std::vector& losslist) { +void UdtCC::onLoss(const std::vector& losslist) { //Slow Start stopped, if it hasn't yet if (_slowStart) { _slowStart = false; diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index fe7f4a15fc..9f61c7944f 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -14,7 +14,7 @@ #include -#include "SeqNum.h" +#include "SequenceNumber.h" namespace udt { @@ -29,8 +29,8 @@ public: virtual void init() {} virtual void close() {} - virtual void onAck(SeqNum ackNum) {} - virtual void onLoss(const std::vector& lossList) {} + virtual void onAck(SequenceNumber ackNum) {} + virtual void onLoss(const std::vector& lossList) {} virtual void onPacketSent(const Packet& packet) {} virtual void onPacketReceived(const Packet& packet) {} @@ -48,7 +48,7 @@ protected: double _maxCongestionWindowSize = 0.0; // maximum cwnd size, in packets int _mss = 0; // Maximum Packet Size, including all packet headers - SeqNum _sendCurrSeqNum; // current maximum seq num sent out + SequenceNumber _sendCurrSeqNum; // current maximum seq num sent out int _recvieveRate = 0; // packet arrive rate at receiver side, packets per second int _rtt = 0; // current estimated RTT, microsecond @@ -59,7 +59,7 @@ private: void setMss(int mss) { _mss = mss; } void setMaxCongestionWindowSize(int window) { _maxCongestionWindowSize = window; } void setBandwidth(int bandwidth) { _bandwidth = bandwidth; } - void setSndCurrSeqNum(SeqNum seqNum) { _sendCurrSeqNum = seqNum; } + void setSndCurrSeqNum(SequenceNumber seqNum) { _sendCurrSeqNum = seqNum; } void setRcvRate(int rate) { _recvieveRate = rate; } void setRtt(int rtt) { _rtt = rtt; } @@ -92,17 +92,17 @@ public: public: virtual void init(); - virtual void onACK(SeqNum ackNum); - virtual void onLoss(const std::vector& lossList); + virtual void onACK(SequenceNumber ackNum); + virtual void onLoss(const std::vector& lossList); virtual void onTimeout(); private: int _rcInterval = 0; // UDT Rate control interval uint64_t _lastRCTime = 0; // last rate increase time bool _slowStart = true; // if in slow start phase - SeqNum _lastAck; // last ACKed seq num + SequenceNumber _lastAck; // last ACKed seq num bool _loss = false; // if loss happened since last rate increase - SeqNum _lastDecSeq; // max pkt seq num sent out when last decrease happened + SequenceNumber _lastDecSeq; // max pkt seq num sent out when last decrease happened double _lastDecPeriod = 1; // value of pktsndperiod when last decrease happened int _nakCount = 0; // NAK counter int _decRandom = 1; // random threshold on decrease by number of loss events diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index dfe25176a2..9ecba087a5 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -41,7 +41,7 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { auto currentTime = high_resolution_clock::now(); static high_resolution_clock::time_point lastACKSendTime; - SeqNum nextACKNumber = nextACK(); + SequenceNumber nextACKNumber = nextACK(); if (nextACKNumber <= _lastReceivedAcknowledgedACK) { // we already got an ACK2 for this ACK we would be sending, don't bother @@ -93,10 +93,10 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { void Connection::sendLightACK() const { // create the light ACK packet, make it static so we can re-use it - static const int LIGHT_ACK_PACKET_PAYLOAD_BYTES = sizeof(SeqNum); + static const int LIGHT_ACK_PACKET_PAYLOAD_BYTES = sizeof(SequenceNumber); static auto lightACKPacket = ControlPacket::create(ControlPacket::ACK, LIGHT_ACK_PACKET_PAYLOAD_BYTES); - SeqNum nextACKNumber = nextACK(); + SequenceNumber nextACKNumber = nextACK(); if (nextACKNumber == _lastReceivedAcknowledgedACK) { // we already got an ACK2 for this ACK we would be sending, don't bother @@ -113,31 +113,31 @@ void Connection::sendLightACK() const { _sendQueue->sendPacket(*lightACKPacket); } -SeqNum Connection::nextACK() const { +SequenceNumber Connection::nextACK() const { if (_lossList.getLength() > 0) { - return _lossList.getFirstSeqNum(); + return _lossList.getFirstSequenceNumber(); } else { - return _largestReceivedSeqNum; + return _lastReceivedSequenceNumber; } } -void Connection::processReceivedSeqNum(SeqNum seq) { +void Connection::processReceivedSequenceNumber(SequenceNumber seq) { // If this is not the next sequence number, report loss - if (seq > _largestReceivedSeqNum + 1) { - if (_largestReceivedSeqNum + 1 == seq - 1) { - _lossList.append(_largestReceivedSeqNum + 1); + if (seq > _lastReceivedSequenceNumber + 1) { + if (_lastReceivedSequenceNumber + 1 == seq - 1) { + _lossList.append(_lastReceivedSequenceNumber + 1); } else { - _lossList.append(_largestReceivedSeqNum + 1, seq - 1); + _lossList.append(_lastReceivedSequenceNumber + 1, seq - 1); } // create the loss report packet, make it static so we can re-use it - static const int NAK_PACKET_PAYLOAD_BYTES = 2 * sizeof(SeqNum); + static const int NAK_PACKET_PAYLOAD_BYTES = 2 * sizeof(SequenceNumber); static auto lossReport = ControlPacket::create(ControlPacket::NAK, NAK_PACKET_PAYLOAD_BYTES); lossReport->reset(); // We need to reset it every time. // pack in the loss report - lossReport->writePrimitive(_largestReceivedSeqNum + 1); - if (_largestReceivedSeqNum + 1 != seq - 1) { + lossReport->writePrimitive(_lastReceivedSequenceNumber + 1); + if (_lastReceivedSequenceNumber + 1 != seq - 1) { lossReport->writePrimitive(seq - 1); } @@ -145,9 +145,9 @@ void Connection::processReceivedSeqNum(SeqNum seq) { _sendQueue->sendPacket(*lossReport); } - if (seq > _largestReceivedSeqNum) { + if (seq > _lastReceivedSequenceNumber) { // Update largest recieved sequence number - _largestReceivedSeqNum = seq; + _lastReceivedSequenceNumber = seq; } else { // Otherwise, it's a resend, remove it from the loss list _lossList.remove(seq); @@ -157,7 +157,7 @@ void Connection::processReceivedSeqNum(SeqNum seq) { void Connection::processControl(unique_ptr controlPacket) { switch (controlPacket->getType()) { case ControlPacket::ACK: { - if (controlPacket->getPayloadSize() == sizeof(SeqNum)) { + if (controlPacket->getPayloadSize() == sizeof(SequenceNumber)) { processLightACK(move(controlPacket)); } else { processACK(move(controlPacket)); @@ -182,7 +182,7 @@ void Connection::processControl(unique_ptr controlPacket) { void Connection::processACK(std::unique_ptr controlPacket) { // read the ACK sub-sequence number - SeqNum currentACKSubSequenceNumber; + SequenceNumber currentACKSubSequenceNumber; controlPacket->readPrimitive(¤tACKSubSequenceNumber); // check if we need send an ACK2 for this ACK @@ -193,7 +193,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { if (sinceLastACK2.count() > _synInterval || currentACKSubSequenceNumber == _lastSentACK2) { // setup a static ACK2 packet we will re-use - static const int ACK2_PAYLOAD_BYTES = sizeof(SeqNum); + static const int ACK2_PAYLOAD_BYTES = sizeof(SequenceNumber); static auto ack2Packet = ControlPacket::create(ControlPacket::ACK2, ACK2_PAYLOAD_BYTES); // reset the ACK2 Packet before writing the sub-sequence number to it @@ -208,11 +208,11 @@ void Connection::processACK(std::unique_ptr controlPacket) { } // read the ACKed sequence number - SeqNum ack; + SequenceNumber ack; controlPacket->readPrimitive(&ack); // validate that this isn't a BS ACK - if (ack > _sendQueue->getCurrentSeqNum()) { + if (ack > _sendQueue->getCurrentSequenceNumber()) { // in UDT they specifically break the connection here - do we want to do anything? return; } @@ -232,7 +232,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { } // make sure this isn't a repeated ACK - if (ack <= SeqNum(_atomicLastReceivedACK)) { + if (ack <= SequenceNumber(_atomicLastReceivedACK)) { return; } @@ -249,7 +249,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { // set the RTT for congestion control - if (controlPacket->getPayloadSize() > (qint64) (sizeof(SeqNum) + sizeof(SeqNum) + sizeof(rtt))) { + if (controlPacket->getPayloadSize() > (qint64) (sizeof(SequenceNumber) + sizeof(SequenceNumber) + sizeof(rtt))) { int32_t deliveryRate, bandwidth; controlPacket->readPrimitive(&deliveryRate); controlPacket->readPrimitive(&bandwidth); @@ -265,7 +265,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { void Connection::processLightACK(std::unique_ptr controlPacket) { // read the ACKed sequence number - SeqNum ack; + SequenceNumber ack; controlPacket->readPrimitive(&ack); // must be larger than the last received ACK to be processed @@ -280,7 +280,7 @@ void Connection::processLightACK(std::unique_ptr controlPacket) { void Connection::processACK2(std::unique_ptr controlPacket) { // pull the sub sequence number from the packet - SeqNum subSequenceNumber; + SequenceNumber subSequenceNumber; controlPacket->readPrimitive(&subSequenceNumber); // check if we had that subsequence number in our map @@ -306,9 +306,9 @@ void Connection::processACK2(std::unique_ptr controlPacket) { void Connection::processNAK(std::unique_ptr controlPacket) { // read the loss report - SeqNum start, end; + SequenceNumber start, end; controlPacket->readPrimitive(&start); - if (controlPacket->bytesLeftToRead() >= (qint64)sizeof(SeqNum)) { + if (controlPacket->bytesLeftToRead() >= (qint64)sizeof(SequenceNumber)) { controlPacket->readPrimitive(&end); } } diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index ec2736ae8d..5bf15f3031 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -29,8 +29,8 @@ class Socket; class Connection { public: - using SequenceNumberTimePair = std::pair; - using SentACKMap = std::unordered_map; + using SequenceNumberTimePair = std::pair; + using SentACKMap = std::unordered_map; Connection(Socket* parentSocket, HifiSockAddr destination); @@ -39,11 +39,11 @@ public: void sendACK(bool wasCausedBySyncTimeout = true); void sendLightACK() const; - SeqNum nextACK() const; + SequenceNumber nextACK() const; - SeqNum getLastReceivedACK() const { return SeqNum(_atomicLastReceivedACK); } + SequenceNumber getLastReceivedACK() const { return SequenceNumber(_atomicLastReceivedACK); } - void processReceivedSeqNum(SeqNum seq); + void processReceivedSequenceNumber(SequenceNumber seq); void processControl(std::unique_ptr controlPacket); private: @@ -57,14 +57,14 @@ private: int _synInterval; // Periodical Rate Control Interval, defaults to 10ms LossList _lossList; // List of all missing packets - SeqNum _largestReceivedSeqNum { SeqNum::MAX }; // The largest sequence number received from the peer - SeqNum _lastReceivedACK { SeqNum::MAX }; // The last ACK received - std::atomic _atomicLastReceivedACK { (uint32_t) SeqNum::MAX }; // Atomic for thread-safe get of last ACK received - SeqNum _lastReceivedAcknowledgedACK { SeqNum::MAX }; // The last sent ACK that has been acknowledged via an ACK2 from the peer - SeqNum _currentACKSubSequenceNumber; // The current ACK sub-sequence number (used for Acknowledgment of ACKs) + SequenceNumber _lastReceivedSequenceNumber { SequenceNumber::MAX }; // The largest sequence number received from the peer + SequenceNumber _lastReceivedACK { SequenceNumber::MAX }; // The last ACK received + std::atomic _atomicLastReceivedACK { (uint32_t) SequenceNumber::MAX }; // Atomic for thread-safe get of last ACK received + SequenceNumber _lastReceivedAcknowledgedACK { SequenceNumber::MAX }; // The last sent ACK that has been acknowledged via an ACK2 from the peer + SequenceNumber _currentACKSubSequenceNumber; // The current ACK sub-sequence number (used for Acknowledgment of ACKs) - SeqNum _lastSentACK { SeqNum::MAX }; // The last sent ACK - SeqNum _lastSentACK2; // The last sent ACK sub-sequence number in an ACK2 + SequenceNumber _lastSentACK { SequenceNumber::MAX }; // The last sent ACK + SequenceNumber _lastSentACK2; // The last sent ACK sub-sequence number in an ACK2 int _totalReceivedACKs { 0 }; diff --git a/libraries/networking/src/udt/LossList.cpp b/libraries/networking/src/udt/LossList.cpp index a8fe0912d5..2b4943b08b 100644 --- a/libraries/networking/src/udt/LossList.cpp +++ b/libraries/networking/src/udt/LossList.cpp @@ -14,7 +14,7 @@ using namespace udt; using namespace std; -void LossList::append(SeqNum seq) { +void LossList::append(SequenceNumber seq) { assert(_lossList.back().second < seq); if (getLength() > 0 && _lossList.back().second + 1 == seq) { @@ -25,7 +25,7 @@ void LossList::append(SeqNum seq) { _length += 1; } -void LossList::append(SeqNum start, SeqNum end) { +void LossList::append(SequenceNumber start, SequenceNumber end) { if (getLength() > 0 && _lossList.back().second + 1 == start) { _lossList.back().second = end; } else { @@ -34,8 +34,8 @@ void LossList::append(SeqNum start, SeqNum end) { _length += seqlen(start, end); } -void LossList::remove(SeqNum seq) { - auto it = find_if(_lossList.begin(), _lossList.end(), [&seq](pair pair) { +void LossList::remove(SequenceNumber seq) { + auto it = find_if(_lossList.begin(), _lossList.end(), [&seq](pair pair) { return pair.first <= seq && seq <= pair.second; }); @@ -54,9 +54,9 @@ void LossList::remove(SeqNum seq) { } } -void LossList::remove(SeqNum start, SeqNum end) { +void LossList::remove(SequenceNumber start, SequenceNumber end) { // Find the first segment sharing sequence numbers - auto it = find_if(_lossList.begin(), _lossList.end(), [&start, &end](pair pair) { + auto it = find_if(_lossList.begin(), _lossList.end(), [&start, &end](pair pair) { return (pair.first <= start && start <= pair.second) || (start <= pair.first && pair.first <= end); }); @@ -93,7 +93,7 @@ void LossList::remove(SeqNum start, SeqNum end) { } } -SeqNum LossList::getFirstSeqNum() const { +SequenceNumber LossList::getFirstSequenceNumber() const { assert(getLength() > 0); return _lossList.front().first; } \ No newline at end of file diff --git a/libraries/networking/src/udt/LossList.h b/libraries/networking/src/udt/LossList.h index dd7f1de2ff..ec26947dfc 100644 --- a/libraries/networking/src/udt/LossList.h +++ b/libraries/networking/src/udt/LossList.h @@ -14,7 +14,7 @@ #include -#include "SeqNum.h" +#include "SequenceNumber.h" namespace udt { @@ -23,20 +23,20 @@ public: LossList() {} // Should always add at the end - void append(SeqNum seq); - void append(SeqNum start, SeqNum end); + void append(SequenceNumber seq); + void append(SequenceNumber start, SequenceNumber end); - void remove(SeqNum seq); - void remove(SeqNum start, SeqNum end); + void remove(SequenceNumber seq); + void remove(SequenceNumber start, SequenceNumber end); int getLength() const { return _length; } - SeqNum getFirstSeqNum() const; + SequenceNumber getFirstSequenceNumber() const; private: - std::list> _lossList; + std::list> _lossList; int _length { 0 }; }; } -#endif // hifi_LossList_h \ No newline at end of file +#endif // hifi_LossList_h diff --git a/libraries/networking/src/udt/Packet.cpp b/libraries/networking/src/udt/Packet.cpp index a70c3bfbef..6bbb5a8972 100644 --- a/libraries/networking/src/udt/Packet.cpp +++ b/libraries/networking/src/udt/Packet.cpp @@ -65,7 +65,7 @@ Packet::Packet(qint64 size, bool isReliable, bool isPartOfMessage) : adjustPayloadStartAndCapacity(); // set the UDT header to default values - writeSequenceNumber(SeqNum()); + writeSequenceNumber(SequenceNumber()); } Packet::Packet(std::unique_ptr data, qint64 size, const HifiSockAddr& senderSockAddr) : @@ -131,13 +131,13 @@ void Packet::readIsPartOfMessage() { void Packet::readSequenceNumber() { SequenceNumberAndBitField seqNumBitField = *reinterpret_cast(_packet.get()); - _sequenceNumber = SeqNum{ seqNumBitField & ~BIT_FIELD_MASK }; // Remove the bit field + _sequenceNumber = SequenceNumber{ seqNumBitField & ~BIT_FIELD_MASK }; // Remove the bit field } -void Packet::writeSequenceNumber(SeqNum seqNum) { +void Packet::writeSequenceNumber(SequenceNumber seqNum) { // grab pointer to current SequenceNumberAndBitField SequenceNumberAndBitField* seqNumBitField = reinterpret_cast(_packet.get()); // write new value by ORing (old value & BIT_FIELD_MASK) with new seqNum - *seqNumBitField = (*seqNumBitField & BIT_FIELD_MASK) | (SeqNum::Type)seqNum; + *seqNumBitField = (*seqNumBitField & BIT_FIELD_MASK) | (SequenceNumber::Type) seqNum; } diff --git a/libraries/networking/src/udt/Packet.h b/libraries/networking/src/udt/Packet.h index 6189be0ef1..343b89107c 100644 --- a/libraries/networking/src/udt/Packet.h +++ b/libraries/networking/src/udt/Packet.h @@ -18,7 +18,7 @@ #include "BasePacket.h" #include "PacketHeaders.h" -#include "SeqNum.h" +#include "SequenceNumber.h" namespace udt { @@ -26,7 +26,6 @@ class Packet : public BasePacket { Q_OBJECT public: // NOTE: The SequenceNumber is only actually 29 bits to leave room for a bit field - using SequenceNumber = uint32_t; using SequenceNumberAndBitField = uint32_t; // NOTE: The MessageNumber is only actually 29 bits to leave room for a bit field @@ -49,8 +48,8 @@ public: virtual qint64 totalHeadersSize() const; // Cumulated size of all the headers - void writeSequenceNumber(SeqNum seqNum); - SeqNum getSequenceNumber() const { return _sequenceNumber; } + void writeSequenceNumber(SequenceNumber seqNum); + SequenceNumber getSequenceNumber() const { return _sequenceNumber; } protected: Packet(qint64 size, bool isReliable = false, bool isPartOfMessage = false); @@ -68,7 +67,7 @@ protected: bool _isReliable { false }; bool _isPartOfMessage { false }; - SeqNum _sequenceNumber; + SequenceNumber _sequenceNumber; }; } // namespace udt diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 0648ef0292..2fe1603200 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -84,7 +84,7 @@ void SendQueue::sendPacket(const BasePacket& packet) { _socket->writeUnreliablePacket(packet, _destination); } -void SendQueue::ack(SeqNum ack) { +void SendQueue::ack(SequenceNumber ack) { if (_lastAck == ack) { return; } @@ -98,7 +98,7 @@ void SendQueue::ack(SeqNum ack) { _lastAck = ack; } -void SendQueue::nak(std::list naks) { +void SendQueue::nak(std::list naks) { QWriteLocker locker(&_naksLock); _naks.splice(_naks.end(), naks); // Add naks at the end } @@ -113,9 +113,9 @@ void SendQueue::sendNextPacket() { _lastSendTimestamp = sendTime; if (_nextPacket) { - _nextPacket->writeSequenceNumber(++_currentSeqNum); + _nextPacket->writeSequenceNumber(++_currentSequenceNumber); sendPacket(*_nextPacket); - _atomicCurrentSeqNum.store((uint32_t) _currentSeqNum); + _atomicCurrentSequenceNumber.store((uint32_t) _currentSequenceNumber); // Insert the packet we have just sent in the sent list QWriteLocker locker(&_sentLock); @@ -125,7 +125,7 @@ void SendQueue::sendNextPacket() { } bool hasResend = false; - SeqNum seqNum; + SequenceNumber seqNum; { // Check nak list for packet to resend QWriteLocker locker(&_naksLock); diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index b30a7a0cbb..60ad1d0101 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -21,7 +21,7 @@ #include "../HifiSockAddr.h" -#include "SeqNum.h" +#include "SequenceNumber.h" namespace udt { @@ -42,7 +42,7 @@ public: quint64 getLastSendTimestamp() const { return _lastSendTimestamp; } - SeqNum getCurrentSeqNum() const { return SeqNum(_atomicCurrentSeqNum); } + SequenceNumber getCurrentSequenceNumber() const { return SequenceNumber(_atomicCurrentSequenceNumber); } int getPacketSendPeriod() const { return _packetSendPeriod; } void setPacketSendPeriod(int newPeriod) { _packetSendPeriod = newPeriod; } @@ -52,8 +52,8 @@ public slots: void stop(); void sendPacket(const BasePacket& packet); - void ack(SeqNum ack); - void nak(std::list naks); + void ack(SequenceNumber ack); + void nak(std::list naks); private slots: void sendNextPacket(); @@ -72,10 +72,10 @@ private: Socket* _socket { nullptr }; // Socket to send packet on HifiSockAddr _destination; // Destination addr - SeqNum _currentSeqNum; // Last sequence number sent out - SeqNum _lastAck; // Last ACKed sequence number + SequenceNumber _currentSequenceNumber; // Last sequence number sent out + SequenceNumber _lastAck; // Last ACKed sequence number - std::atomic _atomicCurrentSeqNum; // Atomic for last sequence number sent out + std::atomic _atomicCurrentSequenceNumber; // Atomic for last sequence number sent out std::unique_ptr _sendTimer; // Send timer std::atomic _packetSendPeriod { 0 }; // Interval between two packet send envent in msec @@ -83,10 +83,10 @@ private: std::atomic _running { false }; mutable QReadWriteLock _naksLock; // Protects the naks list. - std::list _naks; // Sequence numbers of packets to resend + std::list _naks; // Sequence numbers of packets to resend mutable QReadWriteLock _sentLock; // Protects the sent packet list - std::unordered_map> _sentPackets; // Packets waiting for ACK. + std::unordered_map> _sentPackets; // Packets waiting for ACK. }; } diff --git a/libraries/networking/src/udt/SeqNum.cpp b/libraries/networking/src/udt/SeqNum.cpp deleted file mode 100644 index 668c1f131a..0000000000 --- a/libraries/networking/src/udt/SeqNum.cpp +++ /dev/null @@ -1,34 +0,0 @@ -// -// SeqNum.cpp -// libraries/networking/src/udt -// -// Created by Clement on 7/23/15. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "SeqNum.h" - -int udt::seqcmp(const SeqNum& seq1, const SeqNum& seq2) { - return (glm::abs(seq1._value - seq2._value) < SeqNum::THRESHOLD) ? (seq1._value - seq2._value) - : (seq2._value - seq1._value); -} - -int udt::seqlen(const SeqNum& seq1, const SeqNum& seq2) { - return (seq1._value <= seq2._value) ? (seq2._value - seq1._value + 1) - : (seq2._value - seq1._value + SeqNum::MAX + 2); -} - -int udt::seqoff(const SeqNum& seq1, const SeqNum& seq2) { - if (glm::abs(seq1._value - seq2._value) < SeqNum::THRESHOLD) { - return seq2._value - seq1._value; - } - - if (seq1._value < seq2._value) { - return seq2._value - seq1._value - SeqNum::MAX - 1; - } - - return seq2._value - seq1._value + SeqNum::MAX + 1; -} diff --git a/libraries/networking/src/udt/SeqNum.h b/libraries/networking/src/udt/SeqNum.h deleted file mode 100644 index 5ae85f3dcf..0000000000 --- a/libraries/networking/src/udt/SeqNum.h +++ /dev/null @@ -1,147 +0,0 @@ -// -// SeqNum.h -// libraries/networking/src/udt -// -// Created by Clement on 7/23/15. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_SeqNum_h -#define hifi_SeqNum_h - -#include - -#include - -namespace udt { - -class SeqNum { -public: - // Base type of sequence numbers - using Type = uint32_t; - - // Values are for 29 bit SeqNum - static const Type THRESHOLD = 0x0FFFFFFF; // threshold for comparing sequence numbers - static const Type MAX = 0x1FFFFFFF; // maximum sequence number used in UDT - - SeqNum() = default; - SeqNum(const SeqNum& other) : _value(other._value) {} - - // Only explicit conversions - explicit SeqNum(char* value) { _value = (*reinterpret_cast(value)) & MAX; } - explicit SeqNum(Type value) { _value = (value <= MAX) ? value : MAX; } - explicit operator Type() { return _value; } - - inline SeqNum& operator++() { - _value = (_value == MAX) ? 0 : ++_value; - return *this; - } - inline SeqNum& operator--() { - _value = (_value == 0) ? MAX : --_value; - return *this; - } - inline SeqNum operator++(int) { - SeqNum before = *this; - (*this)++; - return before; - } - inline SeqNum operator--(int) { - SeqNum before = *this; - (*this)--; - return before; - } - - inline SeqNum& operator=(const SeqNum& other) { - _value = other._value; - return *this; - } - inline SeqNum& operator+=(Type inc) { - _value = (_value + inc > MAX) ? _value + inc - (MAX + 1) : _value + inc; - return *this; - } - inline SeqNum& operator-=(Type dec) { - _value = (_value < dec) ? MAX - (dec - _value + 1) : _value - dec; - return *this; - } - - inline bool operator==(const SeqNum& other) const { - return _value == other._value; - } - inline bool operator!=(const SeqNum& other) const { - return _value != other._value; - } - - friend bool operator<(const SeqNum& a, const SeqNum& b); - friend bool operator>(const SeqNum& a, const SeqNum& b); - friend bool operator<=(const SeqNum& a, const SeqNum& b); - friend bool operator>=(const SeqNum& a, const SeqNum& b); - - friend SeqNum operator+(const SeqNum a, const Type& b); - friend SeqNum operator+(const Type& a, const SeqNum b); - friend SeqNum operator-(const SeqNum a, const Type& b); - friend SeqNum operator-(const Type& a, const SeqNum b); - - friend int seqcmp(const SeqNum& seq1, const SeqNum& seq2); - friend int seqlen(const SeqNum& seq1, const SeqNum& seq2); - friend int seqoff(const SeqNum& seq1, const SeqNum& seq2); - -private: - Type _value { 0 }; - - friend struct std::hash; -}; - - -inline bool operator<(const SeqNum& a, const SeqNum& b) { - return (glm::abs(a._value - b._value) < SeqNum::THRESHOLD) ? a._value < b._value : b._value < a._value; -} - -inline bool operator>(const SeqNum& a, const SeqNum& b) { - return (glm::abs(a._value - b._value) < SeqNum::THRESHOLD) ? a._value > b._value : b._value > a._value; -} - -inline bool operator<=(const SeqNum& a, const SeqNum& b) { - return (glm::abs(a._value - b._value) < SeqNum::THRESHOLD) ? a._value <= b._value : b._value <= a._value; -} - -inline bool operator>=(const SeqNum& a, const SeqNum& b) { - return (glm::abs(a._value - b._value) < SeqNum::THRESHOLD) ? a._value >= b._value : b._value >= a._value; -} - - -inline SeqNum operator+(SeqNum a, const SeqNum::Type& b) { - a += b; - return a; -} - -inline SeqNum operator+(const SeqNum::Type& a, SeqNum b) { - b += a; - return b; -} - -inline SeqNum operator-(SeqNum a, const SeqNum::Type& b) { - a -= b; - return a; -} - -inline SeqNum operator-(const SeqNum::Type& a, SeqNum b) { - b -= a; - return b; -} - -int seqcmp(const SeqNum& seq1, const SeqNum& seq2); -int seqlen(const SeqNum& seq1, const SeqNum& seq2); -int seqoff(const SeqNum& seq1, const SeqNum& seq2); - -} - -template<> struct std::hash { - size_t operator()(const udt::SeqNum& seqNum) const { - return std::hash()(seqNum._value); - } -}; - -#endif // hifi_SeqNum_h diff --git a/libraries/networking/src/udt/SequenceNumber.cpp b/libraries/networking/src/udt/SequenceNumber.cpp new file mode 100644 index 0000000000..cd7a467d23 --- /dev/null +++ b/libraries/networking/src/udt/SequenceNumber.cpp @@ -0,0 +1,34 @@ +// +// SequenceNumber.cpp +// libraries/networking/src/udt +// +// Created by Clement on 7/23/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "SequenceNumber.h" + +int udt::seqcmp(const SequenceNumber& seq1, const SequenceNumber& seq2) { + return (glm::abs(seq1._value - seq2._value) < SequenceNumber::THRESHOLD) ? (seq1._value - seq2._value) + : (seq2._value - seq1._value); +} + +int udt::seqlen(const SequenceNumber& seq1, const SequenceNumber& seq2) { + return (seq1._value <= seq2._value) ? (seq2._value - seq1._value + 1) + : (seq2._value - seq1._value + SequenceNumber::MAX + 2); +} + +int udt::seqoff(const SequenceNumber& seq1, const SequenceNumber& seq2) { + if (glm::abs(seq1._value - seq2._value) < SequenceNumber::THRESHOLD) { + return seq2._value - seq1._value; + } + + if (seq1._value < seq2._value) { + return seq2._value - seq1._value - SequenceNumber::MAX - 1; + } + + return seq2._value - seq1._value + SequenceNumber::MAX + 1; +} diff --git a/libraries/networking/src/udt/SequenceNumber.h b/libraries/networking/src/udt/SequenceNumber.h new file mode 100644 index 0000000000..28d94d70fb --- /dev/null +++ b/libraries/networking/src/udt/SequenceNumber.h @@ -0,0 +1,147 @@ +// +// SequenceNumber.h +// libraries/networking/src/udt +// +// Created by Clement on 7/23/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_SequenceNumber_h +#define hifi_SequenceNumber_h + +#include + +#include + +namespace udt { + +class SequenceNumber { +public: + // Base type of sequence numbers + using Type = uint32_t; + + // Values are for 29 bit SequenceNumber + static const Type THRESHOLD = 0x0FFFFFFF; // threshold for comparing sequence numbers + static const Type MAX = 0x1FFFFFFF; // maximum sequence number used in UDT + + SequenceNumber() = default; + SequenceNumber(const SequenceNumber& other) : _value(other._value) {} + + // Only explicit conversions + explicit SequenceNumber(char* value) { _value = (*reinterpret_cast(value)) & MAX; } + explicit SequenceNumber(Type value) { _value = (value <= MAX) ? value : MAX; } + explicit operator Type() { return _value; } + + inline SequenceNumber& operator++() { + _value = (_value == MAX) ? 0 : ++_value; + return *this; + } + inline SequenceNumber& operator--() { + _value = (_value == 0) ? MAX : --_value; + return *this; + } + inline SequenceNumber operator++(int) { + SequenceNumber before = *this; + (*this)++; + return before; + } + inline SequenceNumber operator--(int) { + SequenceNumber before = *this; + (*this)--; + return before; + } + + inline SequenceNumber& operator=(const SequenceNumber& other) { + _value = other._value; + return *this; + } + inline SequenceNumber& operator+=(Type inc) { + _value = (_value + inc > MAX) ? _value + inc - (MAX + 1) : _value + inc; + return *this; + } + inline SequenceNumber& operator-=(Type dec) { + _value = (_value < dec) ? MAX - (dec - _value + 1) : _value - dec; + return *this; + } + + inline bool operator==(const SequenceNumber& other) const { + return _value == other._value; + } + inline bool operator!=(const SequenceNumber& other) const { + return _value != other._value; + } + + friend bool operator<(const SequenceNumber& a, const SequenceNumber& b); + friend bool operator>(const SequenceNumber& a, const SequenceNumber& b); + friend bool operator<=(const SequenceNumber& a, const SequenceNumber& b); + friend bool operator>=(const SequenceNumber& a, const SequenceNumber& b); + + friend SequenceNumber operator+(const SequenceNumber a, const Type& b); + friend SequenceNumber operator+(const Type& a, const SequenceNumber b); + friend SequenceNumber operator-(const SequenceNumber a, const Type& b); + friend SequenceNumber operator-(const Type& a, const SequenceNumber b); + + friend int seqcmp(const SequenceNumber& seq1, const SequenceNumber& seq2); + friend int seqlen(const SequenceNumber& seq1, const SequenceNumber& seq2); + friend int seqoff(const SequenceNumber& seq1, const SequenceNumber& seq2); + +private: + Type _value { 0 }; + + friend struct std::hash; +}; + + +inline bool operator<(const SequenceNumber& a, const SequenceNumber& b) { + return (glm::abs(a._value - b._value) < SequenceNumber::THRESHOLD) ? a._value < b._value : b._value < a._value; +} + +inline bool operator>(const SequenceNumber& a, const SequenceNumber& b) { + return (glm::abs(a._value - b._value) < SequenceNumber::THRESHOLD) ? a._value > b._value : b._value > a._value; +} + +inline bool operator<=(const SequenceNumber& a, const SequenceNumber& b) { + return (glm::abs(a._value - b._value) < SequenceNumber::THRESHOLD) ? a._value <= b._value : b._value <= a._value; +} + +inline bool operator>=(const SequenceNumber& a, const SequenceNumber& b) { + return (glm::abs(a._value - b._value) < SequenceNumber::THRESHOLD) ? a._value >= b._value : b._value >= a._value; +} + + +inline SequenceNumber operator+(SequenceNumber a, const SequenceNumber::Type& b) { + a += b; + return a; +} + +inline SequenceNumber operator+(const SequenceNumber::Type& a, SequenceNumber b) { + b += a; + return b; +} + +inline SequenceNumber operator-(SequenceNumber a, const SequenceNumber::Type& b) { + a -= b; + return a; +} + +inline SequenceNumber operator-(const SequenceNumber::Type& a, SequenceNumber b) { + b -= a; + return b; +} + +int seqcmp(const SequenceNumber& seq1, const SequenceNumber& seq2); +int seqlen(const SequenceNumber& seq1, const SequenceNumber& seq2); +int seqoff(const SequenceNumber& seq1, const SequenceNumber& seq2); + +} + +template<> struct std::hash { + size_t operator()(const udt::SequenceNumber& SequenceNumber) const { + return std::hash()(SequenceNumber._value); + } +}; + +#endif // hifi_SequenceNumber_h diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index f74ea49f0b..5b8ebb5d0f 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -28,7 +28,7 @@ namespace udt { class BasePacket; class ControlSender; class Packet; -class SeqNum; +class SequenceNumber; using PacketFilterOperator = std::function; @@ -70,7 +70,7 @@ private: std::unordered_map _unfilteredHandlers; - std::unordered_map _packetSequenceNumbers; + std::unordered_map _packetSequenceNumbers; int32_t _synInterval = 10; // 10ms QTimer _synTimer; From 00fdcf3133c87bb87ca92bd26f46837340b1e900 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 13:44:26 -0700 Subject: [PATCH 050/242] fix a comment that reference SequenceNumber --- libraries/networking/src/udt/SendQueue.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 2fe1603200..34761d57f6 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -136,7 +136,7 @@ void SendQueue::sendNextPacket() { } } - // Find packet in sent list using SeqNum + // Find packet in sent list using SequenceNumber if (hasResend) { QWriteLocker locker(&_sentLock); auto it = _sentPackets.find(seqNum); From f0bb71b5196ae7eb674c2f1e0338c66c52c2d500 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 28 Jul 2015 13:48:20 -0700 Subject: [PATCH 051/242] Change Packet read/write fcts --- libraries/networking/src/udt/Connection.cpp | 6 ++-- libraries/networking/src/udt/Packet.cpp | 38 ++++++++++++--------- libraries/networking/src/udt/Packet.h | 12 ++++--- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 6826b4b109..b1b298c0ff 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -156,7 +156,7 @@ void Connection::processReceivedSequenceNumber(SequenceNumber seq) { void Connection::processControl(unique_ptr controlPacket) { switch (controlPacket->getType()) { - case ControlPacket::ACK: { + case ControlPacket::ACK: if (controlPacket->getPayloadSize() == sizeof(SequenceNumber)) { processLightACK(move(controlPacket)); } else { @@ -169,10 +169,8 @@ void Connection::processControl(unique_ptr controlPacket) { case ControlPacket::NAK: processNAK(move(controlPacket)); break; - case ControlPacket::PacketPair: { - + case ControlPacket::PacketPair: break; - } } } diff --git a/libraries/networking/src/udt/Packet.cpp b/libraries/networking/src/udt/Packet.cpp index 6bbb5a8972..5f1aa88937 100644 --- a/libraries/networking/src/udt/Packet.cpp +++ b/libraries/networking/src/udt/Packet.cpp @@ -65,15 +65,13 @@ Packet::Packet(qint64 size, bool isReliable, bool isPartOfMessage) : adjustPayloadStartAndCapacity(); // set the UDT header to default values - writeSequenceNumber(SequenceNumber()); + writeHeader(); } Packet::Packet(std::unique_ptr data, qint64 size, const HifiSockAddr& senderSockAddr) : BasePacket(std::move(data), size, senderSockAddr) { - readIsReliable(); - readIsPartOfMessage(); - readSequenceNumber(); + readHeader(); adjustPayloadStartAndCapacity(_payloadSize > 0); } @@ -119,25 +117,33 @@ static const uint32_t RELIABILITY_BIT_MASK = 1 << (sizeof(Packet::SequenceNumber static const uint32_t MESSAGE_BIT_MASK = 1 << (sizeof(Packet::SequenceNumberAndBitField) - 3); static const uint32_t BIT_FIELD_MASK = CONTROL_BIT_MASK | RELIABILITY_BIT_MASK | MESSAGE_BIT_MASK; -void Packet::readIsReliable() { +void Packet::readHeader() { SequenceNumberAndBitField seqNumBitField = *reinterpret_cast(_packet.get()); + Q_ASSERT_X((bool) (seqNumBitField & CONTROL_BIT_MASK), + "Packet::readHeader()", "This should be a data packet"); _isReliable = (bool) (seqNumBitField & RELIABILITY_BIT_MASK); // Only keep reliability bit -} - -void Packet::readIsPartOfMessage() { - SequenceNumberAndBitField seqNumBitField = *reinterpret_cast(_packet.get()); - _isReliable = (bool) (seqNumBitField & MESSAGE_BIT_MASK); // Only keep message bit -} - -void Packet::readSequenceNumber() { - SequenceNumberAndBitField seqNumBitField = *reinterpret_cast(_packet.get()); + _isPartOfMessage = (bool) (seqNumBitField & MESSAGE_BIT_MASK); // Only keep message bit _sequenceNumber = SequenceNumber{ seqNumBitField & ~BIT_FIELD_MASK }; // Remove the bit field } -void Packet::writeSequenceNumber(SequenceNumber seqNum) { +void Packet::writeHeader() { // grab pointer to current SequenceNumberAndBitField SequenceNumberAndBitField* seqNumBitField = reinterpret_cast(_packet.get()); + *seqNumBitField &= ~CONTROL_BIT_MASK; + + if (_isPartOfMessage) { + *seqNumBitField |= MESSAGE_BIT_MASK; + } else { + *seqNumBitField &= ~MESSAGE_BIT_MASK; + } + + if (_isReliable) { + *seqNumBitField |= RELIABILITY_BIT_MASK; + } else { + *seqNumBitField &= ~RELIABILITY_BIT_MASK; + } + // write new value by ORing (old value & BIT_FIELD_MASK) with new seqNum - *seqNumBitField = (*seqNumBitField & BIT_FIELD_MASK) | (SequenceNumber::Type) seqNum; + *seqNumBitField = (*seqNumBitField & BIT_FIELD_MASK) | (SequenceNumber::Type)_sequenceNumber; } diff --git a/libraries/networking/src/udt/Packet.h b/libraries/networking/src/udt/Packet.h index 343b89107c..94bd1be4fa 100644 --- a/libraries/networking/src/udt/Packet.h +++ b/libraries/networking/src/udt/Packet.h @@ -47,9 +47,12 @@ public: virtual qint64 localHeaderSize() const; virtual qint64 totalHeadersSize() const; // Cumulated size of all the headers - - void writeSequenceNumber(SequenceNumber seqNum); + + bool isPartOfMessage() const { return _isPartOfMessage; } + bool isReliable() const { return _isReliable; } SequenceNumber getSequenceNumber() const { return _sequenceNumber; } + + void setSequenceNumber(); protected: Packet(qint64 size, bool isReliable = false, bool isPartOfMessage = false); @@ -61,9 +64,8 @@ protected: Packet& operator=(Packet&& other); // Header readers - these read data to member variables after pulling packet off wire - void readIsPartOfMessage(); - void readIsReliable(); - void readSequenceNumber(); + void readHeader(); + void writeHeader(); bool _isReliable { false }; bool _isPartOfMessage { false }; From f885fbad8cac8d0cf485aa3b83e4222c72d82ae8 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 13:49:11 -0700 Subject: [PATCH 052/242] remove sequence numbers from _naks in SendQueue --- libraries/networking/src/udt/SendQueue.cpp | 26 ++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 34761d57f6..0be80fe98d 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -89,10 +89,28 @@ void SendQueue::ack(SequenceNumber ack) { return; } - // remove any ACKed packets from the map of sent packets - QWriteLocker locker(&_sentLock); - for (auto seq = _lastAck; seq <= ack; ++seq) { - _sentPackets.erase(seq); + { + // remove any ACKed packets from the map of sent packets + QWriteLocker locker(&_sentLock); + for (auto seq = _lastAck; seq <= ack; ++seq) { + _sentPackets.erase(seq); + } + } + + { + // remove any sequence numbers equal to or lower than this ACK in the loss list + QWriteLocker nakLocker(&_naksLock); + + auto it = _naks.begin(); + + while (it != _naks.end()) { + if (*it <= ack) { + it = _naks.erase(it); + } else { + // the NAKs in the NAK list must be in order, so we can break if we hit one > ack + break; + } + } } _lastAck = ack; From c17f4b8991980206aa8fc5ac3a9d7f2115fd32da Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 13:58:21 -0700 Subject: [PATCH 053/242] add a missing bracket, comment for processACK --- libraries/networking/src/udt/Connection.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 6826b4b109..779c8179bf 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -163,6 +163,7 @@ void Connection::processControl(unique_ptr controlPacket) { processACK(move(controlPacket)); } break; + } case ControlPacket::ACK2: processACK2(move(controlPacket)); break; @@ -181,7 +182,9 @@ void Connection::processACK(std::unique_ptr controlPacket) { SequenceNumber currentACKSubSequenceNumber; controlPacket->readPrimitive(¤tACKSubSequenceNumber); - // check if we need send an ACK2 for this ACK + // Check if we need send an ACK2 for this ACK + // This will be the case if it has been longer than the sync interval OR + // it looks like they haven't received our ACK2 for this ACK auto currentTime = high_resolution_clock::now(); static high_resolution_clock::time_point lastACK2SendTime; From ca9969967424f5c01b682fc16430861730f8b64a Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 28 Jul 2015 14:00:59 -0700 Subject: [PATCH 054/242] Implement Packet::setSequenceNumber --- libraries/networking/src/udt/Packet.cpp | 6 ++++++ libraries/networking/src/udt/Packet.h | 2 +- libraries/networking/src/udt/SendQueue.cpp | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/Packet.cpp b/libraries/networking/src/udt/Packet.cpp index 5f1aa88937..e40c81accd 100644 --- a/libraries/networking/src/udt/Packet.cpp +++ b/libraries/networking/src/udt/Packet.cpp @@ -112,6 +112,11 @@ Packet& Packet::operator=(Packet&& other) { return *this; } +void Packet::setSequenceNumber(SequenceNumber sequenceNumber) { + _sequenceNumber = sequenceNumber; + writeHeader(); +} + static const uint32_t CONTROL_BIT_MASK = 1 << (sizeof(Packet::SequenceNumberAndBitField) - 1); static const uint32_t RELIABILITY_BIT_MASK = 1 << (sizeof(Packet::SequenceNumberAndBitField) - 2); static const uint32_t MESSAGE_BIT_MASK = 1 << (sizeof(Packet::SequenceNumberAndBitField) - 3); @@ -130,6 +135,7 @@ void Packet::writeHeader() { // grab pointer to current SequenceNumberAndBitField SequenceNumberAndBitField* seqNumBitField = reinterpret_cast(_packet.get()); + // 0 for data packets *seqNumBitField &= ~CONTROL_BIT_MASK; if (_isPartOfMessage) { diff --git a/libraries/networking/src/udt/Packet.h b/libraries/networking/src/udt/Packet.h index 94bd1be4fa..e7bd41b937 100644 --- a/libraries/networking/src/udt/Packet.h +++ b/libraries/networking/src/udt/Packet.h @@ -52,7 +52,7 @@ public: bool isReliable() const { return _isReliable; } SequenceNumber getSequenceNumber() const { return _sequenceNumber; } - void setSequenceNumber(); + void setSequenceNumber(SequenceNumber sequenceNumber); protected: Packet(qint64 size, bool isReliable = false, bool isPartOfMessage = false); diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 0be80fe98d..84d34f4012 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -131,7 +131,7 @@ void SendQueue::sendNextPacket() { _lastSendTimestamp = sendTime; if (_nextPacket) { - _nextPacket->writeSequenceNumber(++_currentSequenceNumber); + _nextPacket->setSequenceNumber(++_currentSequenceNumber); sendPacket(*_nextPacket); _atomicCurrentSequenceNumber.store((uint32_t) _currentSequenceNumber); From b6bdcadd3334d724a9b60fec5aa58f7fcde1ee34 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 14:02:13 -0700 Subject: [PATCH 055/242] make updateRTT take an int --- libraries/networking/src/udt/Connection.cpp | 2 +- libraries/networking/src/udt/Connection.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 779c8179bf..3807134776 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -312,7 +312,7 @@ void Connection::processNAK(std::unique_ptr controlPacket) { } } -void Connection::updateRTT(int32_t rtt) { +void Connection::updateRTT(int rtt) { // this updates the RTT using exponential weighted moving average _rttVariance = (_rttVariance * 3 + abs(rtt - _rtt)) >> 2; _rtt = (_rtt * 7 + rtt) >> 3; diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 5bf15f3031..e85c8a3c7e 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -52,7 +52,7 @@ private: void processACK2(std::unique_ptr controlPacket); void processNAK(std::unique_ptr controlPacket); - void updateRTT(int32_t rtt); + void updateRTT(int rtt); int _synInterval; // Periodical Rate Control Interval, defaults to 10ms From f53637f19e07aa2bd2856834c48d912c801f2955 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 28 Jul 2015 14:16:09 -0700 Subject: [PATCH 056/242] Added SendQueue::getNextSeqNum --- libraries/networking/src/udt/SendQueue.cpp | 15 +++++++++++---- libraries/networking/src/udt/SendQueue.h | 9 +++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 84d34f4012..b69b643ec8 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -80,8 +80,10 @@ void SendQueue::stop() { _running = false; } -void SendQueue::sendPacket(const BasePacket& packet) { - _socket->writeUnreliablePacket(packet, _destination); +void SendQueue::sendPacket(const Packet& packet) { + if (_socket) { + _socket->writePacket(packet, _destination); + } } void SendQueue::ack(SequenceNumber ack) { @@ -121,6 +123,11 @@ void SendQueue::nak(std::list naks) { _naks.splice(_naks.end(), naks); // Add naks at the end } +SequenceNumber SendQueue::getNextSequenceNumber() { + _atomicCurrentSequenceNumber = (SequenceNumber::Type)++_currentSequenceNumber; + return _currentSequenceNumber; +} + void SendQueue::sendNextPacket() { if (!_running) { return; @@ -131,9 +138,9 @@ void SendQueue::sendNextPacket() { _lastSendTimestamp = sendTime; if (_nextPacket) { - _nextPacket->setSequenceNumber(++_currentSequenceNumber); + // Write packet's sequence number and send it off + _nextPacket->setSequenceNumber(getNextSequenceNumber()); sendPacket(*_nextPacket); - _atomicCurrentSequenceNumber.store((uint32_t) _currentSequenceNumber); // Insert the packet we have just sent in the sent list QWriteLocker locker(&_sentLock); diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 60ad1d0101..9e9c5cc024 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -50,7 +50,6 @@ public: public slots: void start(); void stop(); - void sendPacket(const BasePacket& packet); void ack(SequenceNumber ack); void nak(std::list naks); @@ -66,15 +65,21 @@ private: SendQueue(SendQueue&& other) = delete; ~SendQueue(); + // Increments current sequence number and return it + SequenceNumber getNextSequenceNumber(); + + // Send a packet through the socket + void sendPacket(const Packet& packet); + mutable QReadWriteLock _packetsLock; // Protects the packets to be sent list. std::list> _packets; // List of packets to be sent std::unique_ptr _nextPacket; // Next packet to be sent Socket* _socket { nullptr }; // Socket to send packet on HifiSockAddr _destination; // Destination addr - SequenceNumber _currentSequenceNumber; // Last sequence number sent out SequenceNumber _lastAck; // Last ACKed sequence number + SequenceNumber _currentSequenceNumber; // Last sequence number sent out std::atomic _atomicCurrentSequenceNumber; // Atomic for last sequence number sent out std::unique_ptr _sendTimer; // Send timer From e3e8219c11b1c7c5b14fbe6cf763545b0f816da1 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 14:20:51 -0700 Subject: [PATCH 057/242] add some comments to help understand the RTT calculation --- libraries/networking/src/udt/Connection.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 3807134776..4bfc679c97 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -313,7 +313,16 @@ void Connection::processNAK(std::unique_ptr controlPacket) { } void Connection::updateRTT(int rtt) { - // this updates the RTT using exponential weighted moving average + // This updates the RTT using exponential weighted moving average + // This is the Jacobson's forumla for RTT estimation + // http://www.mathcs.emory.edu/~cheung/Courses/455/Syllabus/7-transport/Jacobson-88.pdf + + // Estimated RTT = (1 - x)(estimatedRTT) + (x)(sampleRTT) + // (where x = 0.125 via Jacobson) + + // Deviation = (1 - x)(deviation) + x |sampleRTT - estimatedRTT| + // (where x = 0.25 via Jacobson) + _rttVariance = (_rttVariance * 3 + abs(rtt - _rtt)) >> 2; _rtt = (_rtt * 7 + rtt) >> 3; } From 937e46abf22821f9313ba4069da36a561d41b990 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 14:28:19 -0700 Subject: [PATCH 058/242] add path to header for LossList --- libraries/networking/src/udt/LossList.cpp | 4 ++-- libraries/networking/src/udt/LossList.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/LossList.cpp b/libraries/networking/src/udt/LossList.cpp index abe26970e1..db312935b8 100644 --- a/libraries/networking/src/udt/LossList.cpp +++ b/libraries/networking/src/udt/LossList.cpp @@ -1,6 +1,6 @@ // // LossList.cpp -// +// libraries/networking/src/udt // // Created by Clement on 7/27/15. // Copyright 2015 High Fidelity, Inc. @@ -101,4 +101,4 @@ void LossList::remove(SequenceNumber start, SequenceNumber end) { SequenceNumber LossList::getFirstSequenceNumber() const { assert(getLength() > 0); return _lossList.front().first; -} \ No newline at end of file +} diff --git a/libraries/networking/src/udt/LossList.h b/libraries/networking/src/udt/LossList.h index ec26947dfc..387c69b70f 100644 --- a/libraries/networking/src/udt/LossList.h +++ b/libraries/networking/src/udt/LossList.h @@ -1,6 +1,6 @@ // // LossList.h -// +// libraries/networking/src/udt // // Created by Clement on 7/27/15. // Copyright 2015 High Fidelity, Inc. From cf30426636646633d83ba70b0317343a90e7f7fe Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 15:21:17 -0700 Subject: [PATCH 059/242] add a PacketTimeWindow to estimate bandwidth and report speed --- libraries/networking/src/udt/Connection.cpp | 18 +++- libraries/networking/src/udt/Connection.h | 3 + .../networking/src/udt/ControlPacket.cpp | 21 ---- libraries/networking/src/udt/ControlPacket.h | 6 +- .../networking/src/udt/PacketTimeWindow.cpp | 101 ++++++++++++++++++ .../networking/src/udt/PacketTimeWindow.h | 48 +++++++++ libraries/networking/src/udt/SendQueue.cpp | 2 + 7 files changed, 168 insertions(+), 31 deletions(-) create mode 100644 libraries/networking/src/udt/PacketTimeWindow.cpp create mode 100644 libraries/networking/src/udt/PacketTimeWindow.h diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 4bfc679c97..a9318fe0e3 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -78,7 +78,9 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { // pack the available buffer size - must be a minimum of 2 if (wasCausedBySyncTimeout) { - // pack in the receive speed and bandwidth + // pack in the receive speed and estimatedBandwidth + ackPacket->writePrimitive(_receiveWindow.getPacketReceiveSpeed()); + ackPacket->writePrimitive(_receiveWindow.getEstimatedBandwidth()); // record this as the last ACK send time lastACKSendTime = high_resolution_clock::now(); @@ -122,6 +124,16 @@ SequenceNumber Connection::nextACK() const { } void Connection::processReceivedSequenceNumber(SequenceNumber seq) { + + // check if this is a packet pair we should estimate bandwidth from, or just a regular packet + if (((uint32_t) seq & 0xF) == 0) { + _receiveWindow.onProbePair1Arrival(); + } else if (((uint32_t) seq & 0xF) == 1) { + _receiveWindow.onProbePair2Arrival(); + } else { + _receiveWindow.onPacketArrival(); + } + // If this is not the next sequence number, report loss if (seq > _lastReceivedSequenceNumber + 1) { if (_lastReceivedSequenceNumber + 1 == seq - 1) { @@ -170,10 +182,6 @@ void Connection::processControl(unique_ptr controlPacket) { case ControlPacket::NAK: processNAK(move(controlPacket)); break; - case ControlPacket::PacketPair: { - - break; - } } } diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index e85c8a3c7e..dc2c335455 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -16,6 +16,7 @@ #include #include "LossList.h" +#include "PacketTimeWindow.h" #include "SendQueue.h" class HifiSockAddr; @@ -74,6 +75,8 @@ private: SentACKMap _sentACKs; // Map of ACK sub-sequence numbers to ACKed sequence number and sent time + PacketTimeWindow _receiveWindow; // Window of received packets for bandwidth estimation and receive speed + std::unique_ptr _sendQueue; }; diff --git a/libraries/networking/src/udt/ControlPacket.cpp b/libraries/networking/src/udt/ControlPacket.cpp index 5f28b96750..057f3633f9 100644 --- a/libraries/networking/src/udt/ControlPacket.cpp +++ b/libraries/networking/src/udt/ControlPacket.cpp @@ -27,13 +27,6 @@ std::unique_ptr ControlPacket::create(Type type, qint64 size) { } } -ControlPacket::ControlPacketPair ControlPacket::createPacketPair(quint64 timestamp) { - // create each of the two packets in the packet pair - ControlPacketPair packetPair { std::unique_ptr(new ControlPacket(timestamp)), - std::unique_ptr(new ControlPacket(timestamp)) }; - return packetPair; -} - qint64 ControlPacket::localHeaderSize() { return sizeof(ControlBitAndType); } @@ -64,20 +57,6 @@ ControlPacket::ControlPacket(Type type, qint64 size) : writeControlBitAndType(); } -ControlPacket::ControlPacket(quint64 timestamp) : - BasePacket(localHeaderSize() + sizeof(timestamp)), - _type(Type::PacketPair) -{ - adjustPayloadStartAndCapacity(); - - open(QIODevice::ReadWrite); - - writeControlBitAndType(); - - // pack in the timestamp - writePrimitive(timestamp); -} - ControlPacket::ControlPacket(ControlPacket&& other) : BasePacket(std::move(other)) { diff --git a/libraries/networking/src/udt/ControlPacket.h b/libraries/networking/src/udt/ControlPacket.h index d17e12e182..4563b657a9 100644 --- a/libraries/networking/src/udt/ControlPacket.h +++ b/libraries/networking/src/udt/ControlPacket.h @@ -25,17 +25,14 @@ class ControlPacket : public BasePacket { Q_OBJECT public: using ControlBitAndType = uint32_t; - using ControlPacketPair = std::pair, std::unique_ptr>; enum Type : uint16_t { ACK, ACK2, - NAK, - PacketPair + NAK }; static std::unique_ptr create(Type type, qint64 size = -1); - static ControlPacketPair createPacketPair(quint64 timestamp); static qint64 localHeaderSize(); // Current level's header size virtual qint64 totalHeadersSize() const; // Cumulated size of all the headers @@ -46,7 +43,6 @@ public: private: ControlPacket(Type type); ControlPacket(Type type, qint64 size); - ControlPacket(quint64 timestamp); ControlPacket(ControlPacket&& other); ControlPacket(const ControlPacket& other) = delete; diff --git a/libraries/networking/src/udt/PacketTimeWindow.cpp b/libraries/networking/src/udt/PacketTimeWindow.cpp new file mode 100644 index 0000000000..c8c49e6603 --- /dev/null +++ b/libraries/networking/src/udt/PacketTimeWindow.cpp @@ -0,0 +1,101 @@ +// +// PacketTimeWindow.cpp +// libraries/networking/src/udt +// +// Created by Stephen Birarda on 2015-07-28. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "PacketTimeWindow.h" + +#include + +#include + +using namespace udt; +using namespace std::chrono; + +PacketTimeWindow::PacketTimeWindow(int numPacketIntervals, int numProbeIntervals) : + _numPacketIntervals(numPacketIntervals), + _numProbeIntervals(numProbeIntervals), + _packetIntervals({ _numPacketIntervals }), + _probeIntervals({ _numProbeIntervals }) +{ + +} + +int32_t meanOfMedianFilteredValues(std::vector intervals, int numValues, int valuesRequired = 0) { + // sort the intervals from smallest to largest + std::sort(intervals.begin(), intervals.end()); + + int median = 0; + if (numValues % 2 == 0) { + median = intervals[numValues / 2]; + } else { + median = (intervals[(numValues / 2) - 1] + intervals[numValues / 2]) / 2; + } + + int count = 0; + int sum = 0; + int upperBound = median * 8; + int lowerBound = median / 8; + + for (auto& interval : intervals) { + if ((interval < upperBound) && interval > lowerBound) { + ++count; + sum += interval; + } + } + + if (count >= valuesRequired) { + return (int32_t) ceil((double) USECS_PER_MSEC / ((double) sum) / ((double) count)); + } else { + return 0; + } +} + +int32_t PacketTimeWindow::getPacketReceiveSpeed() const { + // return the mean value of median filtered values (per second) - or zero if there are too few filtered values + return meanOfMedianFilteredValues(_packetIntervals, _numPacketIntervals, _numPacketIntervals / 2); +} + +int32_t PacketTimeWindow::getEstimatedBandwidth() const { + // return mean value of median filtered values (per second) + return meanOfMedianFilteredValues(_probeIntervals, _numProbeIntervals); +} + +void PacketTimeWindow::onPacketArrival() { + // take the current time + auto now = high_resolution_clock::now(); + + // record the interval between this packet and the last one + _packetIntervals[_currentPacketInterval++] = duration_cast(now - _lastPacketTime).count(); + + // reset the currentPacketInterval index when it wraps + if (_currentPacketInterval == _numPacketIntervals) { + _currentPacketInterval = 0; + } + + // remember this as the last packet arrival time + _lastPacketTime = now; +} + +void PacketTimeWindow::onProbePair1Arrival() { + // take the current time as the first probe time + _firstProbeTime = high_resolution_clock::now(); +} + +void PacketTimeWindow::onProbePair2Arrival() { + // store the interval between the two probes + auto now = high_resolution_clock::now(); + + _probeIntervals[_currentProbeInterval++] = duration_cast(now - _firstProbeTime).count(); + + // reset the currentProbeInterval index when it wraps + if (_currentProbeInterval == _numProbeIntervals) { + _currentProbeInterval = 0; + } +} diff --git a/libraries/networking/src/udt/PacketTimeWindow.h b/libraries/networking/src/udt/PacketTimeWindow.h new file mode 100644 index 0000000000..25e3df8a43 --- /dev/null +++ b/libraries/networking/src/udt/PacketTimeWindow.h @@ -0,0 +1,48 @@ +// +// PacketTimeWindow.h +// libraries/networking/src/udt +// +// Created by Stephen Birarda on 2015-07-28. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#pragma once + +#ifndef hifi_PacketTimeWindow_h +#define hifi_PacketTimeWindow_h + +#include +#include + +namespace udt { + +class PacketTimeWindow { +public: + PacketTimeWindow(int numPacketIntervals = 16, int numProbeIntervals = 16); + + void onPacketArrival(); + void onProbePair1Arrival(); + void onProbePair2Arrival(); + + int32_t getPacketReceiveSpeed() const; + int32_t getEstimatedBandwidth() const; +private: + int _numPacketIntervals { 0 }; // the number of packet intervals to store + int _numProbeIntervals { 0 }; // the number of probe intervals to store + + int _currentPacketInterval { 0 }; // index for the current packet interval + int _currentProbeInterval { 0 }; // index for the current probe interval + + std::vector _packetIntervals; // vector of microsecond intervals between packet arrivals + std::vector _probeIntervals; // vector of microsecond intervals between probe pair arrivals + + std::chrono::high_resolution_clock::time_point _lastPacketTime; // the time_point when last packet arrived + std::chrono::high_resolution_clock::time_point _firstProbeTime; // the time_point when first probe in pair arrived +}; + +} + +#endif // hifi_PacketTimeWindow_h diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 0be80fe98d..c324792475 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -174,6 +174,8 @@ void SendQueue::sendNextPacket() { _packets.pop_front(); } + // check if we need to fire off a packet pair - we do this + // How long before next packet send auto timeToSleep = (sendTime + _packetSendPeriod) - msecTimestampNow(); // msec _sendTimer->start(std::max((quint64)0, timeToSleep)); From b0147144cb48f275160535c086c3fc230160d824 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 15:25:08 -0700 Subject: [PATCH 060/242] fix for comment for PacketTimeWindow --- libraries/networking/src/udt/Connection.cpp | 2 +- libraries/networking/src/udt/Connection.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index a9318fe0e3..de6fbfbe1d 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -80,7 +80,7 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { if (wasCausedBySyncTimeout) { // pack in the receive speed and estimatedBandwidth ackPacket->writePrimitive(_receiveWindow.getPacketReceiveSpeed()); - ackPacket->writePrimitive(_receiveWindow.getEstimatedBandwidth()); + ackPacket->writePrimitive(_receiveWindow.getEstimatedBandwidth()); // record this as the last ACK send time lastACKSendTime = high_resolution_clock::now(); diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index dc2c335455..66885ef612 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -75,7 +75,7 @@ private: SentACKMap _sentACKs; // Map of ACK sub-sequence numbers to ACKed sequence number and sent time - PacketTimeWindow _receiveWindow; // Window of received packets for bandwidth estimation and receive speed + PacketTimeWindow _receiveWindow { 16, 64 }; // Window of interval between packets (16) and probes (64) for bandwidth and receive speed std::unique_ptr _sendQueue; }; From 24520c5856dd7b2d3e89b41b330a652d45fd42cc Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 28 Jul 2015 15:32:00 -0700 Subject: [PATCH 061/242] LNL send cleanup / Added connection hash --- ice-server/src/IceServer.cpp | 2 +- libraries/networking/src/LimitedNodeList.cpp | 74 +++++++++++--------- libraries/networking/src/LimitedNodeList.h | 10 +-- libraries/networking/src/udt/Connection.cpp | 18 +++-- libraries/networking/src/udt/Connection.h | 7 +- libraries/networking/src/udt/SendQueue.cpp | 4 +- libraries/networking/src/udt/SendQueue.h | 6 +- libraries/networking/src/udt/Socket.cpp | 28 ++++++-- libraries/networking/src/udt/Socket.h | 10 +-- 9 files changed, 97 insertions(+), 62 deletions(-) diff --git a/ice-server/src/IceServer.cpp b/ice-server/src/IceServer.cpp index 9ddb67677d..ec6f5ef825 100644 --- a/ice-server/src/IceServer.cpp +++ b/ice-server/src/IceServer.cpp @@ -145,7 +145,7 @@ void IceServer::sendPeerInformationPacket(const NetworkPeer& peer, const HifiSoc peerPacket->write(peer.toByteArray()); // write the current packet - _serverSocket.writeUnreliablePacket(*peerPacket, *destinationSockAddr); + _serverSocket.writePacket(*peerPacket, *destinationSockAddr); } void IceServer::clearInactivePeers() { diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index edfdd682e9..c705542bf5 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -244,18 +244,13 @@ bool LimitedNodeList::packetSourceAndHashMatch(const udt::Packet& packet) { return false; } -qint64 LimitedNodeList::writePacket(const NLPacket& packet, const Node& destinationNode) { - if (!destinationNode.getActiveSocket()) { - return 0; - } - - emit dataSent(destinationNode.getType(), packet.getDataSize()); - - return writePacket(packet, *destinationNode.getActiveSocket(), destinationNode.getConnectionSecret()); +void LimitedNodeList::collectPacketStats(const NLPacket& packet) { + // stat collection for packets + ++_numCollectedPackets; + _numCollectedBytes += packet.getDataSize(); } -qint64 LimitedNodeList::writePacket(const NLPacket& packet, const HifiSockAddr& destinationSockAddr, - const QUuid& connectionSecret) { +void LimitedNodeList::fillPacketHeader(const NLPacket& packet, const QUuid& connectionSecret) { if (!NON_SOURCED_PACKETS.contains(packet.getType())) { const_cast(packet).writeSourceID(getSessionUUID()); } @@ -265,55 +260,66 @@ qint64 LimitedNodeList::writePacket(const NLPacket& packet, const HifiSockAddr& && !NON_VERIFIED_PACKETS.contains(packet.getType())) { const_cast(packet).writeVerificationHashGivenSecret(connectionSecret); } - - emit dataSent(NodeType::Unassigned, packet.getDataSize()); - - return writePacketAndCollectStats(packet, destinationSockAddr); -} - -qint64 LimitedNodeList::writePacketAndCollectStats(const NLPacket& packet, const HifiSockAddr& destinationSockAddr) { - // XXX can BandwidthRecorder be used for this? - // stat collection for packets - ++_numCollectedPackets; - _numCollectedBytes += packet.getDataSize(); - - qint64 bytesWritten = _nodeSocket.writeUnreliablePacket(packet, destinationSockAddr); - - return bytesWritten; } qint64 LimitedNodeList::sendUnreliablePacket(const NLPacket& packet, const Node& destinationNode) { - return writePacket(packet, destinationNode); + if (!destinationNode.getActiveSocket()) { + return 0; + } + emit dataSent(destinationNode.getType(), packet.getDataSize()); + return sendUnreliablePacket(packet, *destinationNode.getActiveSocket(), destinationNode.getConnectionSecret()); } qint64 LimitedNodeList::sendUnreliablePacket(const NLPacket& packet, const HifiSockAddr& sockAddr, const QUuid& connectionSecret) { - return writePacket(packet, sockAddr, connectionSecret); + Q_ASSERT_X(!packet.isReliable(), "LimitedNodeList::sendUnreliablePacket", + "Trying to send a reliable packet unreliably."); + + collectPacketStats(packet); + fillPacketHeader(packet, connectionSecret); + + return _nodeSocket.writePacket(packet, sockAddr); } qint64 LimitedNodeList::sendPacket(std::unique_ptr packet, const Node& destinationNode) { - // Keep unique_ptr alive during write - auto result = writePacket(*packet, destinationNode); - return result; + if (!destinationNode.getActiveSocket()) { + return 0; + } + emit dataSent(destinationNode.getType(), packet->getDataSize()); + return sendPacket(std::move(packet), *destinationNode.getActiveSocket(), destinationNode.getConnectionSecret()); } qint64 LimitedNodeList::sendPacket(std::unique_ptr packet, const HifiSockAddr& sockAddr, const QUuid& connectionSecret) { - // Keep unique_ptr alive during write - auto result = writePacket(*packet, sockAddr, connectionSecret); - return result; + if (packet->isReliable()) { + collectPacketStats(*packet); + fillPacketHeader(*packet, connectionSecret); + + auto size = packet->getDataSize(); + _nodeSocket.writePacket(std::move(packet), sockAddr); + + return size; + } else { + return sendUnreliablePacket(*packet, sockAddr, connectionSecret); + } } qint64 LimitedNodeList::sendPacketList(NLPacketList& packetList, const Node& destinationNode) { + auto activeSocket = destinationNode.getActiveSocket(); + if (!activeSocket) { + return 0; + } qint64 bytesSent = 0; + auto connectionSecret = destinationNode.getConnectionSecret(); // close the last packet in the list packetList.closeCurrentPacket(); while (!packetList._packets.empty()) { - bytesSent += sendPacket(packetList.takeFront(), destinationNode); + bytesSent += sendPacket(packetList.takeFront(), *activeSocket, connectionSecret); } + emit dataSent(destinationNode.getType(), bytesSent); return bytesSent; } diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index e09617b88f..9c3ff058fc 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -247,11 +247,13 @@ protected: LimitedNodeList(LimitedNodeList const&); // Don't implement, needed to avoid copies of singleton void operator=(LimitedNodeList const&); // Don't implement, needed to avoid copies of singleton - qint64 writePacket(const NLPacket& packet, const Node& destinationNode); + qint64 sendPacket(std::unique_ptr packet, const Node& destinationNode, + const HifiSockAddr& overridenSockAddr); qint64 writePacket(const NLPacket& packet, const HifiSockAddr& destinationSockAddr, const QUuid& connectionSecret = QUuid()); - qint64 writePacketAndCollectStats(const NLPacket& packet, const HifiSockAddr& destinationSockAddr); - + void collectPacketStats(const NLPacket& packet); + void fillPacketHeader(const NLPacket& packet, const QUuid& connectionSecret); + bool isPacketVerified(const udt::Packet& packet); bool packetVersionMatch(const udt::Packet& packet); bool packetSourceAndHashMatch(const udt::Packet& packet); @@ -264,8 +266,6 @@ protected: void sendPacketToIceServer(PacketType packetType, const HifiSockAddr& iceServerSockAddr, const QUuid& clientID, const QUuid& peerRequestID = QUuid()); - qint64 sendPacket(std::unique_ptr packet, const Node& destinationNode, - const HifiSockAddr& overridenSockAddr); QUuid _sessionUUID; diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 8e72a53c69..a46af2bbde 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -20,14 +20,21 @@ using namespace udt; using namespace std; using namespace std::chrono; -Connection::Connection(Socket* parentSocket, HifiSockAddr destination) { - +Connection::Connection(Socket* parentSocket, HifiSockAddr destination) : + _parentSocket(parentSocket), + _destination(destination) +{ } -void Connection::send(unique_ptr packet) { - if (_sendQueue) { - _sendQueue->queuePacket(move(packet)); +void Connection::sendReliablePacket(unique_ptr packet) { + Q_ASSERT_X(packet->isReliable(), "Connection::send", "Trying to send an unreliable packet reliably."); + + if (!_sendQueue) { + // Lasily create send queue + _sendQueue = SendQueue::create(_parentSocket, _destination); } + + _sendQueue->queuePacket(move(packet)); } void Connection::sendACK(bool wasCausedBySyncTimeout) { @@ -163,7 +170,6 @@ void Connection::processControl(unique_ptr controlPacket) { processACK(move(controlPacket)); } break; - } case ControlPacket::ACK2: processACK2(move(controlPacket)); break; diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 5bf15f3031..bd3e6b82e4 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -17,8 +17,7 @@ #include "LossList.h" #include "SendQueue.h" - -class HifiSockAddr; +#include "../HifiSockAddr.h" namespace udt { @@ -34,7 +33,7 @@ public: Connection(Socket* parentSocket, HifiSockAddr destination); - void send(std::unique_ptr packet); + void sendReliablePacket(std::unique_ptr packet); void sendACK(bool wasCausedBySyncTimeout = true); void sendLightACK() const; @@ -74,6 +73,8 @@ private: SentACKMap _sentACKs; // Map of ACK sub-sequence numbers to ACKed sequence number and sent time + Socket* _parentSocket { nullptr }; + HifiSockAddr _destination; std::unique_ptr _sendQueue; }; diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index b69b643ec8..ef37bf7f9a 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -80,9 +80,9 @@ void SendQueue::stop() { _running = false; } -void SendQueue::sendPacket(const Packet& packet) { +void SendQueue::sendPacket(const BasePacket& packet) { if (_socket) { - _socket->writePacket(packet, _destination); + _socket->writeDatagram(packet.getData(), packet.getDataSize(), _destination); } } diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 9e9c5cc024..1c30d1580a 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -47,6 +47,9 @@ public: int getPacketSendPeriod() const { return _packetSendPeriod; } void setPacketSendPeriod(int newPeriod) { _packetSendPeriod = newPeriod; } + // Send a packet through the socket + void sendPacket(const BasePacket& packet); + public slots: void start(); void stop(); @@ -67,9 +70,6 @@ private: // Increments current sequence number and return it SequenceNumber getNextSequenceNumber(); - - // Send a packet through the socket - void sendPacket(const Packet& packet); mutable QReadWriteLock _packetsLock; // Protects the packets to be sent list. std::list> _packets; // List of packets to be sent diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 6923879201..a5719da9a4 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -14,6 +14,7 @@ #include #include "../NetworkLogging.h" +#include "ControlPacket.h" #include "Packet.h" using namespace udt; @@ -66,17 +67,36 @@ void Socket::setBufferSizes(int numBytes) { } } -qint64 Socket::writeUnreliablePacket(const BasePacket& packet, const HifiSockAddr& sockAddr) { +qint64 Socket::writePacket(const Packet& packet, const HifiSockAddr& sockAddr) { + Q_ASSERT_X(!packet.isReliable(), "Socket::writePacket", "Cannot send a reliable packet unreliably"); + + // TODO: write the correct sequence number to the Packet here + // const_cast(packet).writeSequenceNumber(sequenceNumber); + return writeDatagram(packet.getData(), packet.getDataSize(), sockAddr); } +qint64 Socket::writePacket(std::unique_ptr packet, const HifiSockAddr& sockAddr) { + if (packet->isReliable()) { + auto it = _connectionsHash.find(sockAddr); + if (it == _connectionsHash.end()) { + it = _connectionsHash.insert(it, std::make_pair(sockAddr, new Connection(this, sockAddr))); + } + it->second->sendReliablePacket(std::move(packet)); + return 0; + } + + return writePacket(*packet, sockAddr); +} + +qint64 Socket::writeDatagram(const char* data, qint64 size, const HifiSockAddr& sockAddr) { + return writeDatagram(QByteArray::fromRawData(data, size), sockAddr); +} + qint64 Socket::writeDatagram(const QByteArray& datagram, const HifiSockAddr& sockAddr) { qint64 bytesWritten = _udpSocket.writeDatagram(datagram, sockAddr.getAddress(), sockAddr.getPort()); - // TODO: write the correct sequence number to the Packet here - // const_cast(packet).writeSequenceNumber(sequenceNumber); - if (bytesWritten < 0) { qCDebug(networking) << "ERROR in writeDatagram:" << _udpSocket.error() << "-" << _udpSocket.errorString(); } diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 5b8ebb5d0f..24fa8c8404 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -22,6 +22,7 @@ #include #include "../HifiSockAddr.h" +#include "Connection.h" namespace udt { @@ -42,10 +43,10 @@ public: quint16 localPort() const { return _udpSocket.localPort(); } - qint64 writeUnreliablePacket(const BasePacket& packet, const HifiSockAddr& sockAddr); - - qint64 writeDatagram(const char* data, qint64 size, const HifiSockAddr& sockAddr) - { return writeDatagram(QByteArray::fromRawData(data, size), sockAddr); } + // Simple functions writing to the socket with no processing + qint64 writePacket(const Packet& packet, const HifiSockAddr& sockAddr); + qint64 writePacket(std::unique_ptr packet, const HifiSockAddr& sockAddr); + qint64 writeDatagram(const char* data, qint64 size, const HifiSockAddr& sockAddr); qint64 writeDatagram(const QByteArray& datagram, const HifiSockAddr& sockAddr); void bind(const QHostAddress& address, quint16 port = 0) { _udpSocket.bind(address, port); } @@ -71,6 +72,7 @@ private: std::unordered_map _unfilteredHandlers; std::unordered_map _packetSequenceNumbers; + std::unordered_map _connectionsHash; int32_t _synInterval = 10; // 10ms QTimer _synTimer; From ffdd54d41f74dddd21e9783bfca99ceacf8238db Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 28 Jul 2015 15:42:53 -0700 Subject: [PATCH 062/242] Make sendUnreliable take a non const --- libraries/networking/src/LimitedNodeList.cpp | 6 +++--- libraries/networking/src/LimitedNodeList.h | 4 ++-- libraries/networking/src/udt/Socket.cpp | 4 ++-- libraries/networking/src/udt/Socket.h | 4 +++- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index c705542bf5..8d0f1aa5d7 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -262,7 +262,7 @@ void LimitedNodeList::fillPacketHeader(const NLPacket& packet, const QUuid& conn } } -qint64 LimitedNodeList::sendUnreliablePacket(const NLPacket& packet, const Node& destinationNode) { +qint64 LimitedNodeList::sendUnreliablePacket(NLPacket& packet, const Node& destinationNode) { if (!destinationNode.getActiveSocket()) { return 0; } @@ -270,7 +270,7 @@ qint64 LimitedNodeList::sendUnreliablePacket(const NLPacket& packet, const Node& return sendUnreliablePacket(packet, *destinationNode.getActiveSocket(), destinationNode.getConnectionSecret()); } -qint64 LimitedNodeList::sendUnreliablePacket(const NLPacket& packet, const HifiSockAddr& sockAddr, +qint64 LimitedNodeList::sendUnreliablePacket(NLPacket& packet, const HifiSockAddr& sockAddr, const QUuid& connectionSecret) { Q_ASSERT_X(!packet.isReliable(), "LimitedNodeList::sendUnreliablePacket", "Trying to send a reliable packet unreliably."); @@ -516,7 +516,7 @@ unsigned int LimitedNodeList::broadcastToNodes(std::unique_ptr packet, eachNode([&](const SharedNodePointer& node){ if (node && destinationNodeTypes.contains(node->getType())) { - writePacket(*packet, *node); + sendUnreliablePacket(*packet, *node); ++n; } }); diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 9c3ff058fc..777d6f9679 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -116,8 +116,8 @@ public: PacketReceiver& getPacketReceiver() { return *_packetReceiver; } - qint64 sendUnreliablePacket(const NLPacket& packet, const Node& destinationNode); - qint64 sendUnreliablePacket(const NLPacket& packet, const HifiSockAddr& sockAddr, + qint64 sendUnreliablePacket(NLPacket& packet, const Node& destinationNode); + qint64 sendUnreliablePacket(NLPacket& packet, const HifiSockAddr& sockAddr, const QUuid& connectionSecret = QUuid()); qint64 sendPacket(std::unique_ptr packet, const Node& destinationNode); diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index a5719da9a4..360a59e53f 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -67,11 +67,11 @@ void Socket::setBufferSizes(int numBytes) { } } -qint64 Socket::writePacket(const Packet& packet, const HifiSockAddr& sockAddr) { +qint64 Socket::writePacket(Packet& packet, const HifiSockAddr& sockAddr) { Q_ASSERT_X(!packet.isReliable(), "Socket::writePacket", "Cannot send a reliable packet unreliably"); // TODO: write the correct sequence number to the Packet here - // const_cast(packet).writeSequenceNumber(sequenceNumber); + packet.setSequenceNumber(_currentUnreliableSequenceNumber); return writeDatagram(packet.getData(), packet.getDataSize(), sockAddr); } diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 24fa8c8404..67cc82d299 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -44,7 +44,7 @@ public: quint16 localPort() const { return _udpSocket.localPort(); } // Simple functions writing to the socket with no processing - qint64 writePacket(const Packet& packet, const HifiSockAddr& sockAddr); + qint64 writePacket(Packet& packet, const HifiSockAddr& sockAddr); qint64 writePacket(std::unique_ptr packet, const HifiSockAddr& sockAddr); qint64 writeDatagram(const char* data, qint64 size, const HifiSockAddr& sockAddr); qint64 writeDatagram(const QByteArray& datagram, const HifiSockAddr& sockAddr); @@ -69,6 +69,8 @@ private: PacketFilterOperator _packetFilterOperator; PacketHandler _packetHandler; + SequenceNumber _currentUnreliableSequenceNumber; + std::unordered_map _unfilteredHandlers; std::unordered_map _packetSequenceNumbers; From c2c188c5dd1633be100774acd71d6bd32e68e359 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 28 Jul 2015 15:46:26 -0700 Subject: [PATCH 063/242] Remove TODO --- libraries/networking/src/udt/Socket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 360a59e53f..f48df77db3 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -70,7 +70,7 @@ void Socket::setBufferSizes(int numBytes) { qint64 Socket::writePacket(Packet& packet, const HifiSockAddr& sockAddr) { Q_ASSERT_X(!packet.isReliable(), "Socket::writePacket", "Cannot send a reliable packet unreliably"); - // TODO: write the correct sequence number to the Packet here + // write the correct sequence number to the Packet here packet.setSequenceNumber(_currentUnreliableSequenceNumber); return writeDatagram(packet.getData(), packet.getDataSize(), sockAddr); From 65fa4c4e6d20b60d2a1917eb696fe93c5095e694 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 15:47:36 -0700 Subject: [PATCH 064/242] add the periodic rate control packet sending --- libraries/networking/src/udt/Socket.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 360a59e53f..8c2658cb12 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -144,7 +144,10 @@ void Socket::readPendingDatagrams() { void Socket::rateControlSync() { - // TODO: enumerate our list of connections and ask each of them to send off periodic ACK packet for rate control + // enumerate our list of connections and ask each of them to send off periodic ACK packet for rate control + for (auto& connection : _connectionsHash) { + connection.second->sendACK(); + } if (_synTimer.interval() != _synInterval) { // if the _synTimer interval doesn't match the current _synInterval (changes when the CC factory is changed) From 747f2b0c07babb1b517a9e363ef108c17ffb9bcb Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 28 Jul 2015 15:48:35 -0700 Subject: [PATCH 065/242] Remove seqcmp declaration --- libraries/networking/src/udt/SequenceNumber.h | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/networking/src/udt/SequenceNumber.h b/libraries/networking/src/udt/SequenceNumber.h index f04f532bad..6215d23494 100644 --- a/libraries/networking/src/udt/SequenceNumber.h +++ b/libraries/networking/src/udt/SequenceNumber.h @@ -131,7 +131,6 @@ inline SequenceNumber operator-(const SequenceNumber::Type& a, SequenceNumber b) return b; } -int seqcmp(const SequenceNumber& seq1, const SequenceNumber& seq2); int seqlen(const SequenceNumber& seq1, const SequenceNumber& seq2); int seqoff(const SequenceNumber& seq1, const SequenceNumber& seq2); From ba226d168dddb5a5e2c15a3f09de9f77af971ce7 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 15:48:37 -0700 Subject: [PATCH 066/242] remove SeqNum since it is replaced by SequenceNumber --- libraries/networking/src/udt/SeqNum.cpp | 29 ----- libraries/networking/src/udt/SeqNum.h | 145 ------------------------ 2 files changed, 174 deletions(-) delete mode 100644 libraries/networking/src/udt/SeqNum.cpp delete mode 100644 libraries/networking/src/udt/SeqNum.h diff --git a/libraries/networking/src/udt/SeqNum.cpp b/libraries/networking/src/udt/SeqNum.cpp deleted file mode 100644 index 53afbcada7..0000000000 --- a/libraries/networking/src/udt/SeqNum.cpp +++ /dev/null @@ -1,29 +0,0 @@ -// -// SeqNum.cpp -// libraries/networking/src/udt -// -// Created by Clement on 7/23/15. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "SeqNum.h" - -int udt::seqlen(const SeqNum& seq1, const SeqNum& seq2) { - return (seq1._value <= seq2._value) ? (seq2._value - seq1._value + 1) - : (seq2._value - seq1._value + SeqNum::MAX + 2); -} - -int udt::seqoff(const SeqNum& seq1, const SeqNum& seq2) { - if (glm::abs(seq1._value - seq2._value) < SeqNum::THRESHOLD) { - return seq2._value - seq1._value; - } - - if (seq1._value < seq2._value) { - return seq2._value - seq1._value - SeqNum::MAX - 1; - } - - return seq2._value - seq1._value + SeqNum::MAX + 1; -} diff --git a/libraries/networking/src/udt/SeqNum.h b/libraries/networking/src/udt/SeqNum.h deleted file mode 100644 index 90b327e4c9..0000000000 --- a/libraries/networking/src/udt/SeqNum.h +++ /dev/null @@ -1,145 +0,0 @@ -// -// SeqNum.h -// libraries/networking/src/udt -// -// Created by Clement on 7/23/15. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_SeqNum_h -#define hifi_SeqNum_h - -#include - -#include - -namespace udt { - -class SeqNum { -public: - // Base type of sequence numbers - using Type = uint32_t; - - // Values are for 29 bit SeqNum - static const Type THRESHOLD = 0x0FFFFFFF; // threshold for comparing sequence numbers - static const Type MAX = 0x1FFFFFFF; // maximum sequence number used in UDT - - SeqNum() = default; - SeqNum(const SeqNum& other) : _value(other._value) {} - - // Only explicit conversions - explicit SeqNum(char* value) { _value = (*reinterpret_cast(value)) & MAX; } - explicit SeqNum(Type value) { _value = (value <= MAX) ? value : MAX; } - explicit operator Type() { return _value; } - - inline SeqNum& operator++() { - _value = (_value == MAX) ? 0 : ++_value; - return *this; - } - inline SeqNum& operator--() { - _value = (_value == 0) ? MAX : --_value; - return *this; - } - inline SeqNum operator++(int) { - SeqNum before = *this; - (*this)++; - return before; - } - inline SeqNum operator--(int) { - SeqNum before = *this; - (*this)--; - return before; - } - - inline SeqNum& operator=(const SeqNum& other) { - _value = other._value; - return *this; - } - inline SeqNum& operator+=(Type inc) { - _value = (_value + inc > MAX) ? _value + inc - (MAX + 1) : _value + inc; - return *this; - } - inline SeqNum& operator-=(Type dec) { - _value = (_value < dec) ? MAX - (dec - _value + 1) : _value - dec; - return *this; - } - - inline bool operator==(const SeqNum& other) const { - return _value == other._value; - } - inline bool operator!=(const SeqNum& other) const { - return _value != other._value; - } - - friend bool operator<(const SeqNum& a, const SeqNum& b); - friend bool operator>(const SeqNum& a, const SeqNum& b); - friend bool operator<=(const SeqNum& a, const SeqNum& b); - friend bool operator>=(const SeqNum& a, const SeqNum& b); - - friend SeqNum operator+(const SeqNum a, const Type& b); - friend SeqNum operator+(const Type& a, const SeqNum b); - friend SeqNum operator-(const SeqNum a, const Type& b); - friend SeqNum operator-(const Type& a, const SeqNum b); - - friend int seqlen(const SeqNum& seq1, const SeqNum& seq2); - friend int seqoff(const SeqNum& seq1, const SeqNum& seq2); - -private: - Type _value { 0 }; - - friend struct std::hash; -}; - - -inline bool operator<(const SeqNum& a, const SeqNum& b) { - return (glm::abs(a._value - b._value) < SeqNum::THRESHOLD) ? a._value < b._value : b._value < a._value; -} - -inline bool operator>(const SeqNum& a, const SeqNum& b) { - return (glm::abs(a._value - b._value) < SeqNum::THRESHOLD) ? a._value > b._value : b._value > a._value; -} - -inline bool operator<=(const SeqNum& a, const SeqNum& b) { - return (glm::abs(a._value - b._value) < SeqNum::THRESHOLD) ? a._value <= b._value : b._value <= a._value; -} - -inline bool operator>=(const SeqNum& a, const SeqNum& b) { - return (glm::abs(a._value - b._value) < SeqNum::THRESHOLD) ? a._value >= b._value : b._value >= a._value; -} - - -inline SeqNum operator+(SeqNum a, const SeqNum::Type& b) { - a += b; - return a; -} - -inline SeqNum operator+(const SeqNum::Type& a, SeqNum b) { - b += a; - return b; -} - -inline SeqNum operator-(SeqNum a, const SeqNum::Type& b) { - a -= b; - return a; -} - -inline SeqNum operator-(const SeqNum::Type& a, SeqNum b) { - b -= a; - return b; -} - -int seqlen(const SeqNum& seq1, const SeqNum& seq2); -int seqoff(const SeqNum& seq1, const SeqNum& seq2); - -} - -template<> struct std::hash { - size_t operator()(const udt::SeqNum& seqNum) const { - return std::hash()(seqNum._value); - } -}; - -#endif // hifi_SeqNum_h From cfba30078652198de02baf4a1f49bb682bdb0fc0 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 15:51:50 -0700 Subject: [PATCH 067/242] send an unreliable sequence number from Socket --- libraries/networking/src/udt/Socket.cpp | 3 +-- libraries/networking/src/udt/Socket.h | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 8c2658cb12..6fc135118c 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -70,8 +70,7 @@ void Socket::setBufferSizes(int numBytes) { qint64 Socket::writePacket(Packet& packet, const HifiSockAddr& sockAddr) { Q_ASSERT_X(!packet.isReliable(), "Socket::writePacket", "Cannot send a reliable packet unreliably"); - // TODO: write the correct sequence number to the Packet here - packet.setSequenceNumber(_currentUnreliableSequenceNumber); + packet.setSequenceNumber(_unreliableSequenceNumbers[sockAddr]++); return writeDatagram(packet.getData(), packet.getDataSize(), sockAddr); } diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 67cc82d299..9ca4adcc74 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -73,7 +73,7 @@ private: std::unordered_map _unfilteredHandlers; - std::unordered_map _packetSequenceNumbers; + std::unordered_map _unreliableSequenceNumbers; std::unordered_map _connectionsHash; int32_t _synInterval = 10; // 10ms From a49ad92d87f4c425e526857a62ae00ec33053078 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 15:55:18 -0700 Subject: [PATCH 068/242] use a constant for the bounds for median filtering --- libraries/networking/src/udt/PacketTimeWindow.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/PacketTimeWindow.cpp b/libraries/networking/src/udt/PacketTimeWindow.cpp index c8c49e6603..ab540e9bc0 100644 --- a/libraries/networking/src/udt/PacketTimeWindow.cpp +++ b/libraries/networking/src/udt/PacketTimeWindow.cpp @@ -40,8 +40,9 @@ int32_t meanOfMedianFilteredValues(std::vector intervals, int numValues, in int count = 0; int sum = 0; - int upperBound = median * 8; - int lowerBound = median / 8; + static const int MEDIAN_FILTERING_BOUND_MULTIPLIER = 8; + int upperBound = median * MEDIAN_FILTERING_BOUND_MULTIPLIER; + int lowerBound = median / MEDIAN_FILTERING_BOUND_MULTIPLIER; for (auto& interval : intervals) { if ((interval < upperBound) && interval > lowerBound) { From 13568d3d694dae00ce2e85523f0a85d4e03a2552 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 15:59:13 -0700 Subject: [PATCH 069/242] add constants for RTT estimation calculations --- libraries/networking/src/udt/Connection.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index dbcee6e622..2ad4171f69 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -337,6 +337,11 @@ void Connection::updateRTT(int rtt) { // Deviation = (1 - x)(deviation) + x |sampleRTT - estimatedRTT| // (where x = 0.25 via Jacobson) - _rttVariance = (_rttVariance * 3 + abs(rtt - _rtt)) >> 2; - _rtt = (_rtt * 7 + rtt) >> 3; + static const int RTT_ESTIMATION_ALPHA_NUMERATOR = 8; + static const int RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR = 4; + + _rtt = (_rtt * (1 - RTT_ESTIMATION_ALPHA_NUMERATOR) + rtt) / RTT_ESTIMATION_ALPHA_NUMERATOR; + + _rttVariance = (_rttVariance * (1 - RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR) + + abs(rtt - _rtt)) / RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR; } From 64e1281d4751eac7dc7fff4da9464870a7cdde1b Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 28 Jul 2015 16:02:33 -0700 Subject: [PATCH 070/242] Restore const, make helper memebers mutable --- libraries/networking/src/LimitedNodeList.cpp | 4 ++-- libraries/networking/src/LimitedNodeList.h | 4 ++-- libraries/networking/src/udt/Packet.cpp | 6 +++--- libraries/networking/src/udt/Packet.h | 14 ++++++++------ libraries/networking/src/udt/SendQueue.cpp | 2 +- libraries/networking/src/udt/Socket.cpp | 4 ++-- libraries/networking/src/udt/Socket.h | 4 +--- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 8d0f1aa5d7..fc8e6ad8fe 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -262,7 +262,7 @@ void LimitedNodeList::fillPacketHeader(const NLPacket& packet, const QUuid& conn } } -qint64 LimitedNodeList::sendUnreliablePacket(NLPacket& packet, const Node& destinationNode) { +qint64 LimitedNodeList::sendUnreliablePacket(const NLPacket& packet, const Node& destinationNode) { if (!destinationNode.getActiveSocket()) { return 0; } @@ -270,7 +270,7 @@ qint64 LimitedNodeList::sendUnreliablePacket(NLPacket& packet, const Node& desti return sendUnreliablePacket(packet, *destinationNode.getActiveSocket(), destinationNode.getConnectionSecret()); } -qint64 LimitedNodeList::sendUnreliablePacket(NLPacket& packet, const HifiSockAddr& sockAddr, +qint64 LimitedNodeList::sendUnreliablePacket(const NLPacket& packet, const HifiSockAddr& sockAddr, const QUuid& connectionSecret) { Q_ASSERT_X(!packet.isReliable(), "LimitedNodeList::sendUnreliablePacket", "Trying to send a reliable packet unreliably."); diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 777d6f9679..9c3ff058fc 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -116,8 +116,8 @@ public: PacketReceiver& getPacketReceiver() { return *_packetReceiver; } - qint64 sendUnreliablePacket(NLPacket& packet, const Node& destinationNode); - qint64 sendUnreliablePacket(NLPacket& packet, const HifiSockAddr& sockAddr, + qint64 sendUnreliablePacket(const NLPacket& packet, const Node& destinationNode); + qint64 sendUnreliablePacket(const NLPacket& packet, const HifiSockAddr& sockAddr, const QUuid& connectionSecret = QUuid()); qint64 sendPacket(std::unique_ptr packet, const Node& destinationNode); diff --git a/libraries/networking/src/udt/Packet.cpp b/libraries/networking/src/udt/Packet.cpp index e40c81accd..4223986aee 100644 --- a/libraries/networking/src/udt/Packet.cpp +++ b/libraries/networking/src/udt/Packet.cpp @@ -112,7 +112,7 @@ Packet& Packet::operator=(Packet&& other) { return *this; } -void Packet::setSequenceNumber(SequenceNumber sequenceNumber) { +void Packet::writeSequenceNumber(SequenceNumber sequenceNumber) const { _sequenceNumber = sequenceNumber; writeHeader(); } @@ -122,7 +122,7 @@ static const uint32_t RELIABILITY_BIT_MASK = 1 << (sizeof(Packet::SequenceNumber static const uint32_t MESSAGE_BIT_MASK = 1 << (sizeof(Packet::SequenceNumberAndBitField) - 3); static const uint32_t BIT_FIELD_MASK = CONTROL_BIT_MASK | RELIABILITY_BIT_MASK | MESSAGE_BIT_MASK; -void Packet::readHeader() { +void Packet::readHeader() const { SequenceNumberAndBitField seqNumBitField = *reinterpret_cast(_packet.get()); Q_ASSERT_X((bool) (seqNumBitField & CONTROL_BIT_MASK), "Packet::readHeader()", "This should be a data packet"); @@ -131,7 +131,7 @@ void Packet::readHeader() { _sequenceNumber = SequenceNumber{ seqNumBitField & ~BIT_FIELD_MASK }; // Remove the bit field } -void Packet::writeHeader() { +void Packet::writeHeader() const { // grab pointer to current SequenceNumberAndBitField SequenceNumberAndBitField* seqNumBitField = reinterpret_cast(_packet.get()); diff --git a/libraries/networking/src/udt/Packet.h b/libraries/networking/src/udt/Packet.h index e7bd41b937..8a6832205f 100644 --- a/libraries/networking/src/udt/Packet.h +++ b/libraries/networking/src/udt/Packet.h @@ -52,7 +52,7 @@ public: bool isReliable() const { return _isReliable; } SequenceNumber getSequenceNumber() const { return _sequenceNumber; } - void setSequenceNumber(SequenceNumber sequenceNumber); + void writeSequenceNumber(SequenceNumber sequenceNumber) const; protected: Packet(qint64 size, bool isReliable = false, bool isPartOfMessage = false); @@ -63,13 +63,15 @@ protected: Packet& operator=(const Packet& other); Packet& operator=(Packet&& other); +private: // Header readers - these read data to member variables after pulling packet off wire - void readHeader(); - void writeHeader(); + void readHeader() const; + void writeHeader() const; - bool _isReliable { false }; - bool _isPartOfMessage { false }; - SequenceNumber _sequenceNumber; + // Simple holders to prevent multiple reading and bitwise ops + mutable bool _isReliable { false }; + mutable bool _isPartOfMessage { false }; + mutable SequenceNumber _sequenceNumber; }; } // namespace udt diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 7824e01d17..787bf60ca7 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -139,7 +139,7 @@ void SendQueue::sendNextPacket() { if (_nextPacket) { // Write packet's sequence number and send it off - _nextPacket->setSequenceNumber(getNextSequenceNumber()); + _nextPacket->writeSequenceNumber(getNextSequenceNumber()); sendPacket(*_nextPacket); // Insert the packet we have just sent in the sent list diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 2ccddde918..358fb95638 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -67,11 +67,11 @@ void Socket::setBufferSizes(int numBytes) { } } -qint64 Socket::writePacket(Packet& packet, const HifiSockAddr& sockAddr) { +qint64 Socket::writePacket(const Packet& packet, const HifiSockAddr& sockAddr) { Q_ASSERT_X(!packet.isReliable(), "Socket::writePacket", "Cannot send a reliable packet unreliably"); // write the correct sequence number to the Packet here - packet.setSequenceNumber(_currentUnreliableSequenceNumber); + packet.writeSequenceNumber(_currentUnreliableSequenceNumber); return writeDatagram(packet.getData(), packet.getDataSize(), sockAddr); } diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 67cc82d299..dee6af4117 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -44,7 +44,7 @@ public: quint16 localPort() const { return _udpSocket.localPort(); } // Simple functions writing to the socket with no processing - qint64 writePacket(Packet& packet, const HifiSockAddr& sockAddr); + qint64 writePacket(const Packet& packet, const HifiSockAddr& sockAddr); qint64 writePacket(std::unique_ptr packet, const HifiSockAddr& sockAddr); qint64 writeDatagram(const char* data, qint64 size, const HifiSockAddr& sockAddr); qint64 writeDatagram(const QByteArray& datagram, const HifiSockAddr& sockAddr); @@ -72,8 +72,6 @@ private: SequenceNumber _currentUnreliableSequenceNumber; std::unordered_map _unfilteredHandlers; - - std::unordered_map _packetSequenceNumbers; std::unordered_map _connectionsHash; int32_t _synInterval = 10; // 10ms From 09a497a59b7417dc87455d37dc62f85a543b0f96 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 16:23:10 -0700 Subject: [PATCH 071/242] add default interval values for PacketTimeWindow --- libraries/networking/src/udt/PacketTimeWindow.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/PacketTimeWindow.cpp b/libraries/networking/src/udt/PacketTimeWindow.cpp index ab540e9bc0..469915d311 100644 --- a/libraries/networking/src/udt/PacketTimeWindow.cpp +++ b/libraries/networking/src/udt/PacketTimeWindow.cpp @@ -18,11 +18,14 @@ using namespace udt; using namespace std::chrono; +static const int DEFAULT_PACKET_INTERVAL_MICROSECONDS = 1000000; +static const int DEFAULT_PROBE_INTERVAL_MICROSECONDS = 1000; + PacketTimeWindow::PacketTimeWindow(int numPacketIntervals, int numProbeIntervals) : _numPacketIntervals(numPacketIntervals), _numProbeIntervals(numProbeIntervals), - _packetIntervals({ _numPacketIntervals }), - _probeIntervals({ _numProbeIntervals }) + _packetIntervals({ _numPacketIntervals, DEFAULT_PACKET_INTERVAL_MICROSECONDS }), + _probeIntervals({ _numProbeIntervals, DEFAULT_PROBE_INTERVAL_MICROSECONDS }) { } From 74b0fa7e8734707991cfedc1a39d306e1506583f Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 16:43:21 -0700 Subject: [PATCH 072/242] use microseconds for intervals, setup timeout NAKs --- libraries/networking/src/udt/Connection.cpp | 51 +++++++++++++++++++-- libraries/networking/src/udt/Connection.h | 18 ++++++-- libraries/networking/src/udt/Socket.cpp | 2 +- 3 files changed, 60 insertions(+), 11 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 2ad4171f69..cc7d1f6c09 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -11,6 +11,8 @@ #include "Connection.h" +#include + #include "../HifiSockAddr.h" #include "ControlPacket.h" #include "Packet.h" @@ -37,6 +39,28 @@ void Connection::sendReliablePacket(unique_ptr packet) { _sendQueue->queuePacket(move(packet)); } +void Connection::sync() { + // we send out a periodic ACK every rate control interval + sendACK(); + + // check if we need to re-transmit a loss list + // we do this if it has been longer than the current nakInterval since we last sent + auto now = high_resolution_clock::now(); + + if (duration_cast(now - _lastNAKTime).count() >= _nakInterval) { + // construct a NAK packet that will hold all of the lost sequence numbers + auto lossListPacket = ControlPacket::create(ControlPacket::NAK, _lossList.getLength() * sizeof(SequenceNumber)); + + + + // have our SendQueue send off this control packet + _sendQueue->sendPacket(*lossListPacket); + + _lastNAKTime = high_resolution_clock::now(); + } + +} + void Connection::sendACK(bool wasCausedBySyncTimeout) { static const int ACK_PACKET_PAYLOAD_BYTES = sizeof(_lastSentACK) + sizeof(_currentACKSubSequenceNumber) + sizeof(_rtt) + sizeof(int32_t) + sizeof(int32_t); @@ -65,10 +89,10 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { } else if (nextACKNumber == _lastSentACK) { // We already sent this ACK, but check if we should re-send it. - // We will re-send if it has been more than RTT + (4 * RTT variance) since the last ACK - milliseconds sinceLastACK = duration_cast(currentTime - lastACKSendTime); + // We will re-send if it has been more than the estimated timeout since the last ACK + microseconds sinceLastACK = duration_cast(currentTime - lastACKSendTime); - if (sinceLastACK.count() < (_rtt + (4 * _rttVariance))) { + if (sinceLastACK.count() < estimatedTimeout()) { return; } } @@ -162,6 +186,23 @@ void Connection::processReceivedSequenceNumber(SequenceNumber seq) { // have the send queue send off our packet immediately _sendQueue->sendPacket(*lossReport); + + // record our last NAK time + _lastNAKTime = high_resolution_clock::now(); + + // figure out when we should send the next loss report, if we haven't heard anything back + _nakInterval = (_rtt + 4 * _rttVariance); + + int receivedPacketsPerSecond = _receiveWindow.getPacketReceiveSpeed(); + if (receivedPacketsPerSecond > 0) { + // the NAK interval is at least the _minNAKInterval + // but might be the time required for all lost packets to be retransmitted + _nakInterval = std::max((int) (_lossList.getLength() * (USECS_PER_SECOND / receivedPacketsPerSecond)), + _minNAKInterval); + } else { + // the NAK interval is at least the _minNAKInterval but might be the estimated timeout + _nakInterval = std::max(estimatedTimeout(), _minNAKInterval); + } } if (seq > _lastReceivedSequenceNumber) { @@ -202,7 +243,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { auto currentTime = high_resolution_clock::now(); static high_resolution_clock::time_point lastACK2SendTime; - milliseconds sinceLastACK2 = duration_cast(currentTime - lastACK2SendTime); + microseconds sinceLastACK2 = duration_cast(currentTime - lastACK2SendTime); if (sinceLastACK2.count() > _synInterval || currentACKSubSequenceNumber == _lastSentACK2) { // setup a static ACK2 packet we will re-use @@ -304,7 +345,7 @@ void Connection::processACK2(std::unique_ptr controlPacket) { // calculate the RTT (time now - time ACK sent) auto now = high_resolution_clock::now(); - int rtt = duration_cast(now - pair.second).count(); + int rtt = duration_cast(now - pair.second).count(); updateRTT(rtt); diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 11c5126023..27057da3a9 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -35,9 +35,8 @@ public: Connection(Socket* parentSocket, HifiSockAddr destination); void sendReliablePacket(std::unique_ptr packet); - - void sendACK(bool wasCausedBySyncTimeout = true); - void sendLightACK() const; + + void sync(); // rate control method, fired by Socket for all connections on SYN interval SequenceNumber nextACK() const; @@ -47,6 +46,9 @@ public: void processControl(std::unique_ptr controlPacket); private: + void sendACK(bool wasCausedBySyncTimeout = true); + void sendLightACK() const; + void processACK(std::unique_ptr controlPacket); void processLightACK(std::unique_ptr controlPacket); void processACK2(std::unique_ptr controlPacket); @@ -54,7 +56,13 @@ private: void updateRTT(int rtt); - int _synInterval; // Periodical Rate Control Interval, defaults to 10ms + int estimatedTimeout() const { return _rtt + _rttVariance * 4; } + + int _synInterval; // Periodical Rate Control Interval, in microseconds, defaults to 10ms + + int _nakInterval; // NAK timeout interval, in microseconds + int _minNAKInterval { 100000 }; // NAK timeout interval lower bound, default of 100ms + std::chrono::high_resolution_clock::time_point _lastNAKTime; LossList _lossList; // List of all missing packets SequenceNumber _lastReceivedSequenceNumber { SequenceNumber::MAX }; // The largest sequence number received from the peer @@ -68,7 +76,7 @@ private: int _totalReceivedACKs { 0 }; - int32_t _rtt; // RTT, in milliseconds + int32_t _rtt; // RTT, in microseconds int32_t _rttVariance; // RTT variance int _flowWindowSize; // Flow control window size diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 6fc135118c..15f0febc9b 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -145,7 +145,7 @@ void Socket::rateControlSync() { // enumerate our list of connections and ask each of them to send off periodic ACK packet for rate control for (auto& connection : _connectionsHash) { - connection.second->sendACK(); + connection.second->sync(); } if (_synTimer.interval() != _synInterval) { From 05d9845077d6991b6455f465af42a4c15ced2cf5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 17:01:13 -0700 Subject: [PATCH 073/242] add counting of total control packets sent/received --- libraries/networking/src/udt/Connection.cpp | 34 ++++++++++++++++++-- libraries/networking/src/udt/Connection.h | 22 +++++++++++-- libraries/networking/src/udt/ControlPacket.h | 3 +- 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index cc7d1f6c09..507e11f206 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -49,14 +49,17 @@ void Connection::sync() { if (duration_cast(now - _lastNAKTime).count() >= _nakInterval) { // construct a NAK packet that will hold all of the lost sequence numbers - auto lossListPacket = ControlPacket::create(ControlPacket::NAK, _lossList.getLength() * sizeof(SequenceNumber)); - + auto lossListPacket = ControlPacket::create(ControlPacket::TimeoutNAK, _lossList.getLength() * sizeof(SequenceNumber)); + // TODO: pack in the lost sequence numbers // have our SendQueue send off this control packet _sendQueue->sendPacket(*lossListPacket); + // record this as the last NAK time _lastNAKTime = high_resolution_clock::now(); + + ++_totalSentTimeoutNAKs; } } @@ -122,9 +125,11 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { // write this ACK to the map of sent ACKs _sentACKs[_currentACKSubSequenceNumber] = { nextACKNumber, high_resolution_clock::now() }; + + ++_totalSentACKs; } -void Connection::sendLightACK() const { +void Connection::sendLightACK() { // create the light ACK packet, make it static so we can re-use it static const int LIGHT_ACK_PACKET_PAYLOAD_BYTES = sizeof(SequenceNumber); static auto lightACKPacket = ControlPacket::create(ControlPacket::ACK, LIGHT_ACK_PACKET_PAYLOAD_BYTES); @@ -144,6 +149,8 @@ void Connection::sendLightACK() const { // have the send queue send off our packet immediately _sendQueue->sendPacket(*lightACKPacket); + + ++_totalSentLightACKs; } SequenceNumber Connection::nextACK() const { @@ -190,6 +197,8 @@ void Connection::processReceivedSequenceNumber(SequenceNumber seq) { // record our last NAK time _lastNAKTime = high_resolution_clock::now(); + ++_totalSentNAKs; + // figure out when we should send the next loss report, if we haven't heard anything back _nakInterval = (_rtt + 4 * _rttVariance); @@ -212,6 +221,8 @@ void Connection::processReceivedSequenceNumber(SequenceNumber seq) { // Otherwise, it's a resend, remove it from the loss list _lossList.remove(seq); } + + ++_totalReceivedDataPackets; } void Connection::processControl(unique_ptr controlPacket) { @@ -229,6 +240,9 @@ void Connection::processControl(unique_ptr controlPacket) { case ControlPacket::NAK: processNAK(move(controlPacket)); break; + case ControlPacket::TimeoutNAK: + processTimeoutNAK(move(controlPacket)); + break; } } @@ -259,6 +273,8 @@ void Connection::processACK(std::unique_ptr controlPacket) { // update the last sent ACK2 and the last ACK2 send time _lastSentACK2 = currentACKSubSequenceNumber; lastACK2SendTime = high_resolution_clock::now(); + + ++_totalSentACK2s; } // read the ACKed sequence number @@ -330,6 +346,8 @@ void Connection::processLightACK(std::unique_ptr controlPacket) { // update the last received ACK to the this one _lastReceivedACK = ack; } + + ++_totalReceivedLightACKs; } void Connection::processACK2(std::unique_ptr controlPacket) { @@ -356,6 +374,8 @@ void Connection::processACK2(std::unique_ptr controlPacket) { _lastReceivedAcknowledgedACK = pair.first; } } + + ++_totalReceivedACK2s; } void Connection::processNAK(std::unique_ptr controlPacket) { @@ -365,6 +385,14 @@ void Connection::processNAK(std::unique_ptr controlPacket) { if (controlPacket->bytesLeftToRead() >= (qint64)sizeof(SequenceNumber)) { controlPacket->readPrimitive(&end); } + + ++_totalReceivedNAKs; +} + +void Connection::processTimeoutNAK(std::unique_ptr controlPacket) { + // read the NAKed sequence numbers from the packet + + ++_totalReceivedTimeoutNAKs; } void Connection::updateRTT(int rtt) { diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 27057da3a9..76725cae42 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -47,12 +47,13 @@ public: private: void sendACK(bool wasCausedBySyncTimeout = true); - void sendLightACK() const; + void sendLightACK(); void processACK(std::unique_ptr controlPacket); void processLightACK(std::unique_ptr controlPacket); void processACK2(std::unique_ptr controlPacket); void processNAK(std::unique_ptr controlPacket); + void processTimeoutNAK(std::unique_ptr controlPacket); void updateRTT(int rtt); @@ -74,8 +75,6 @@ private: SequenceNumber _lastSentACK { SequenceNumber::MAX }; // The last sent ACK SequenceNumber _lastSentACK2; // The last sent ACK sub-sequence number in an ACK2 - int _totalReceivedACKs { 0 }; - int32_t _rtt; // RTT, in microseconds int32_t _rttVariance; // RTT variance int _flowWindowSize; // Flow control window size @@ -88,6 +87,23 @@ private: PacketTimeWindow _receiveWindow { 16, 64 }; // Window of interval between packets (16) and probes (64) for bandwidth and receive speed std::unique_ptr _sendQueue; + + // Control Packet stat collection + int _totalReceivedACKs { 0 }; + int _totalSentACKs { 0 }; + int _totalSentLightACKs { 0 }; + int _totalReceivedLightACKs { 0 }; + int _totalReceivedACK2s { 0 }; + int _totalSentACK2s { 0 }; + int _totalReceivedNAKs { 0 }; + int _totalSentNAKs { 0 }; + int _totalReceivedTimeoutNAKs { 0 }; + int _totalSentTimeoutNAKs { 0 }; + + // Data packet stat collection + int _totalReceivedDataPackets { 0 }; + + }; } diff --git a/libraries/networking/src/udt/ControlPacket.h b/libraries/networking/src/udt/ControlPacket.h index 4563b657a9..fdd0b65e36 100644 --- a/libraries/networking/src/udt/ControlPacket.h +++ b/libraries/networking/src/udt/ControlPacket.h @@ -29,7 +29,8 @@ public: enum Type : uint16_t { ACK, ACK2, - NAK + NAK, + TimeoutNAK }; static std::unique_ptr create(Type type, qint64 size = -1); From 988bd226ca82d705ef4b07ff9e359f586e5dd65a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 17:16:20 -0700 Subject: [PATCH 074/242] change SendQueue timing to use usleep and std::chrono --- libraries/networking/src/udt/Connection.cpp | 15 ++ libraries/networking/src/udt/Connection.h | 1 + libraries/networking/src/udt/SendQueue.cpp | 145 ++++++++++---------- libraries/networking/src/udt/SendQueue.h | 17 +-- 4 files changed, 95 insertions(+), 83 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 507e11f206..9b58dc41a5 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -11,6 +11,8 @@ #include "Connection.h" +#include + #include #include "../HifiSockAddr.h" @@ -28,6 +30,19 @@ Connection::Connection(Socket* parentSocket, HifiSockAddr destination) : { } +Connection::~Connection() { + if (_sendQueue) { + // tell our send queue to stop and wait until its send thread is done + QThread* sendQueueThread = _sendQueue->thread(); + + _sendQueue->stop(); + _sendQueue->deleteLater(); + + sendQueueThread->quit(); + sendQueueThread->wait(); + } +} + void Connection::sendReliablePacket(unique_ptr packet) { Q_ASSERT_X(packet->isReliable(), "Connection::send", "Trying to send an unreliable packet reliably."); diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 76725cae42..200f8acda8 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -33,6 +33,7 @@ public: using SentACKMap = std::unordered_map; Connection(Socket* parentSocket, HifiSockAddr destination); + ~Connection(); void sendReliablePacket(std::unique_ptr packet); diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 7824e01d17..c5542af978 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -13,6 +13,7 @@ #include +#include #include #include @@ -21,6 +22,7 @@ #include "Socket.h" using namespace udt; +using namespace std::chrono; std::unique_ptr SendQueue::create(Socket* socket, HifiSockAddr dest) { auto queue = std::unique_ptr(new SendQueue(socket, dest)); @@ -42,17 +44,7 @@ SendQueue::SendQueue(Socket* socket, HifiSockAddr dest) : _socket(socket), _destination(dest) { - _sendTimer.reset(new QTimer(this)); - _sendTimer->setSingleShot(true); - QObject::connect(_sendTimer.get(), &QTimer::timeout, this, &SendQueue::sendNextPacket); - _packetSendPeriod = DEFAULT_SEND_PERIOD; - _lastSendTimestamp = 0; -} - -SendQueue::~SendQueue() { - assert(thread() == QThread::currentThread()); - _sendTimer->stop(); } void SendQueue::queuePacket(std::unique_ptr packet) { @@ -60,24 +52,24 @@ void SendQueue::queuePacket(std::unique_ptr packet) { QWriteLocker locker(&_packetsLock); _packets.push_back(std::move(packet)); } - if (!_running) { - start(); + if (!_isRunning) { + run(); } } -void SendQueue::start() { +void SendQueue::run() { // We need to make sure this is called on the right thread if (thread() != QThread::currentThread()) { - QMetaObject::invokeMethod(this, "start", Qt::QueuedConnection); + QMetaObject::invokeMethod(this, "run", Qt::QueuedConnection); } - _running = true; + _isRunning = true; - // This will send a packet and fire the send timer - sendNextPacket(); + // This will loop and sleep to send packets + loop(); } void SendQueue::stop() { - _running = false; + _isRunning = false; } void SendQueue::sendPacket(const BasePacket& packet) { @@ -128,62 +120,69 @@ SequenceNumber SendQueue::getNextSequenceNumber() { return _currentSequenceNumber; } -void SendQueue::sendNextPacket() { - if (!_running) { - return; - } - - // Record timing - auto sendTime = msecTimestampNow(); // msec - _lastSendTimestamp = sendTime; - - if (_nextPacket) { - // Write packet's sequence number and send it off - _nextPacket->setSequenceNumber(getNextSequenceNumber()); - sendPacket(*_nextPacket); +void SendQueue::loop() { + while (_isRunning) { + // Record timing + _lastSendTimestamp = high_resolution_clock::now(); - // Insert the packet we have just sent in the sent list - QWriteLocker locker(&_sentLock); - _sentPackets[_nextPacket->getSequenceNumber()].swap(_nextPacket); - Q_ASSERT_X(!_nextPacket, - "SendQueue::sendNextPacket()", "Overriden packet in sent list"); - } - - bool hasResend = false; - SequenceNumber seqNum; - { - // Check nak list for packet to resend - QWriteLocker locker(&_naksLock); - if (!_naks.empty()) { - hasResend = true; - seqNum = _naks.front(); - _naks.pop_front(); + if (_nextPacket) { + // Write packet's sequence number and send it off + _nextPacket->setSequenceNumber(getNextSequenceNumber()); + sendPacket(*_nextPacket); + + // Insert the packet we have just sent in the sent list + QWriteLocker locker(&_sentLock); + _sentPackets[_nextPacket->getSequenceNumber()].swap(_nextPacket); + Q_ASSERT_X(!_nextPacket, + "SendQueue::sendNextPacket()", "Overriden packet in sent list"); + } + + bool hasResend = false; + SequenceNumber seqNum; + { + // Check nak list for packet to resend + QWriteLocker locker(&_naksLock); + if (!_naks.empty()) { + hasResend = true; + seqNum = _naks.front(); + _naks.pop_front(); + } + } + + // Find packet in sent list using SequenceNumber + if (hasResend) { + QWriteLocker locker(&_sentLock); + auto it = _sentPackets.find(seqNum); + Q_ASSERT_X(it != _sentPackets.end(), + "SendQueue::sendNextPacket()", "Couldn't find NAKed packet to resend"); + + if (it != _sentPackets.end()) { + it->second.swap(_nextPacket); + _sentPackets.erase(it); + } + } + + // If there is no packet to resend, grab the next one in the list + if (!_nextPacket) { + QWriteLocker locker(&_packetsLock); + _nextPacket.swap(_packets.front()); + _packets.pop_front(); + } + + // since we're a while loop, give the thread a chance to process events + QCoreApplication::processEvents(); + + // we just processed events so check now if we were just told to stop + if (!_isRunning) { + break; + } + + // sleep as long as we need until next packet send, if we can + auto now = high_resolution_clock::now(); + auto microsecondDuration = (_lastSendTimestamp + microseconds(_packetSendPeriod)) - now; + + if (microsecondDuration.count() > 0) { + usleep(microsecondDuration.count()); } } - - // Find packet in sent list using SequenceNumber - if (hasResend) { - QWriteLocker locker(&_sentLock); - auto it = _sentPackets.find(seqNum); - Q_ASSERT_X(it != _sentPackets.end(), - "SendQueue::sendNextPacket()", "Couldn't find NAKed packet to resend"); - - if (it != _sentPackets.end()) { - it->second.swap(_nextPacket); - _sentPackets.erase(it); - } - } - - // If there is no packet to resend, grab the next one in the list - if (!_nextPacket) { - QWriteLocker locker(&_packetsLock); - _nextPacket.swap(_packets.front()); - _packets.pop_front(); - } - - // check if we need to fire off a packet pair - we do this - - // How long before next packet send - auto timeToSleep = (sendTime + _packetSendPeriod) - msecTimestampNow(); // msec - _sendTimer->start(std::max((quint64)0, timeToSleep)); } diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 1c30d1580a..f532bf7fe2 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -12,6 +12,7 @@ #ifndef hifi_SendQueue_h #define hifi_SendQueue_h +#include #include #include @@ -33,15 +34,13 @@ class SendQueue : public QObject { Q_OBJECT public: - static const int DEFAULT_SEND_PERIOD = 16; // msec + static const int DEFAULT_SEND_PERIOD = 16 * 1000; // 16ms, in microseconds static std::unique_ptr create(Socket* socket, HifiSockAddr dest); void queuePacket(std::unique_ptr packet); int getQueueSize() const { QReadLocker locker(&_packetsLock); return _packets.size(); } - quint64 getLastSendTimestamp() const { return _lastSendTimestamp; } - SequenceNumber getCurrentSequenceNumber() const { return SequenceNumber(_atomicCurrentSequenceNumber); } int getPacketSendPeriod() const { return _packetSendPeriod; } @@ -51,14 +50,14 @@ public: void sendPacket(const BasePacket& packet); public slots: - void start(); + void run(); void stop(); void ack(SequenceNumber ack); void nak(std::list naks); private slots: - void sendNextPacket(); + void loop(); private: friend struct std::default_delete; @@ -66,7 +65,6 @@ private: SendQueue(Socket* socket, HifiSockAddr dest); SendQueue(SendQueue& other) = delete; SendQueue(SendQueue&& other) = delete; - ~SendQueue(); // Increments current sequence number and return it SequenceNumber getNextSequenceNumber(); @@ -82,10 +80,9 @@ private: SequenceNumber _currentSequenceNumber; // Last sequence number sent out std::atomic _atomicCurrentSequenceNumber; // Atomic for last sequence number sent out - std::unique_ptr _sendTimer; // Send timer - std::atomic _packetSendPeriod { 0 }; // Interval between two packet send envent in msec - std::atomic _lastSendTimestamp { 0 }; // Record last time of packet departure - std::atomic _running { false }; + std::atomic _packetSendPeriod { 0 }; // Interval between two packet send event in microseconds + std::chrono::high_resolution_clock::time_point _lastSendTimestamp; // Record last time of packet departure + std::atomic _isRunning { false }; mutable QReadWriteLock _naksLock; // Protects the naks list. std::list _naks; // Sequence numbers of packets to resend From 1df657c3594633a2d3a22fe5df2ed983f060e9c8 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 28 Jul 2015 17:39:05 -0700 Subject: [PATCH 075/242] Added insert function to LossList --- libraries/networking/src/udt/LossList.cpp | 47 ++++++++++++++++++++++- libraries/networking/src/udt/LossList.h | 3 ++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/LossList.cpp b/libraries/networking/src/udt/LossList.cpp index db312935b8..86b80fd3f2 100644 --- a/libraries/networking/src/udt/LossList.cpp +++ b/libraries/networking/src/udt/LossList.cpp @@ -34,6 +34,50 @@ void LossList::append(SequenceNumber start, SequenceNumber end) { _length += seqlen(start, end); } +void LossList::insert(SequenceNumber start, SequenceNumber end) { + auto it = find_if_not(_lossList.begin(), _lossList.end(), [&start](pair pair){ + return pair.second < start; + }); + + if (it == _lossList.end()) { + // No overlap, simply insert + _lossList.insert(it, make_pair(start, end)); + _length += seqlen(start, end); + } else { + // If it starts before segment, extend segment + if (start < it->first) { + _length += seqlen(start, it->first - 1); + it->first = start; + } + + if (end > it->second) { + // If it goes further, find the actual end + auto it2 = find_if(it, _lossList.end(), [&end](pair pair){ + return end <= pair.second; + }); + + // If it ends inside a segment, change end (segment will be deleted) + // Or backup iterator so segment doesn't get deleted + if (it2->first <= end) { + end = it2->second; + } else { + --it2; + } + + // Change the end of the original segment + _length += seqlen(it->second + 1, end); + it->second = end; + + // remove all underlapping segments + ++it; ++it2; + while (it != it2) { + _length -= seqlen(it->first, it->second); + it = _lossList.erase(it); + } + } + } +} + void LossList::remove(SequenceNumber seq) { auto it = find_if(_lossList.begin(), _lossList.end(), [&seq](pair pair) { return pair.first <= seq && seq <= pair.second; @@ -58,8 +102,7 @@ void LossList::remove(SequenceNumber seq) { void LossList::remove(SequenceNumber start, SequenceNumber end) { // Find the first segment sharing sequence numbers auto it = find_if(_lossList.begin(), _lossList.end(), [&start, &end](pair pair) { - return (pair.first <= start && start <= pair.second) || - (start <= pair.first && pair.first <= end); + return (pair.first <= start && start <= pair.second) || (start <= pair.first && pair.first <= end); }); // If we found one diff --git a/libraries/networking/src/udt/LossList.h b/libraries/networking/src/udt/LossList.h index 387c69b70f..53ccf3ba50 100644 --- a/libraries/networking/src/udt/LossList.h +++ b/libraries/networking/src/udt/LossList.h @@ -26,6 +26,9 @@ public: void append(SequenceNumber seq); void append(SequenceNumber start, SequenceNumber end); + // Inserts anywhere - MUCH slower + void insert(SequenceNumber start, SequenceNumber end); + void remove(SequenceNumber seq); void remove(SequenceNumber start, SequenceNumber end); From 5d1c0f69355822534ba0c410e4032b4154da9803 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 28 Jul 2015 17:56:11 -0700 Subject: [PATCH 076/242] Added pop to loss list --- libraries/networking/src/udt/LossList.cpp | 6 ++++++ libraries/networking/src/udt/LossList.h | 1 + 2 files changed, 7 insertions(+) diff --git a/libraries/networking/src/udt/LossList.cpp b/libraries/networking/src/udt/LossList.cpp index 86b80fd3f2..ab66634b3e 100644 --- a/libraries/networking/src/udt/LossList.cpp +++ b/libraries/networking/src/udt/LossList.cpp @@ -145,3 +145,9 @@ SequenceNumber LossList::getFirstSequenceNumber() const { assert(getLength() > 0); return _lossList.front().first; } + +SequenceNumber LossList::popFirstSequenceNumber() { + auto front = getFirstSequenceNumber(); + remove(front); + return front; +} diff --git a/libraries/networking/src/udt/LossList.h b/libraries/networking/src/udt/LossList.h index 53ccf3ba50..a669317e9d 100644 --- a/libraries/networking/src/udt/LossList.h +++ b/libraries/networking/src/udt/LossList.h @@ -34,6 +34,7 @@ public: int getLength() const { return _length; } SequenceNumber getFirstSequenceNumber() const; + SequenceNumber popFirstSequenceNumber(); private: std::list> _lossList; From 0e0968f7483c8bce5fd12b67704785fa09774877 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 17:58:49 -0700 Subject: [PATCH 077/242] initial support to get CongestionControl down to Connection --- .../networking/src/udt/CongestionControl.cpp | 23 ++++++++-------- .../networking/src/udt/CongestionControl.h | 26 +++++++++++-------- libraries/networking/src/udt/Connection.cpp | 8 ++++-- libraries/networking/src/udt/Connection.h | 5 +++- libraries/networking/src/udt/Socket.cpp | 10 ++++++- libraries/networking/src/udt/Socket.h | 5 ++++ 6 files changed, 50 insertions(+), 27 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 47a45485fa..5acba518e4 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -15,10 +15,9 @@ using namespace udt; -void UdtCC::init() { - _rcInterval = _synInterval; +void DefaultCC::init() { _lastRCTime = usecTimestampNow(); - setAckTimer(_rcInterval); + setAckTimer(synInterval()); _lastAck = _sendCurrSeqNum; _lastDecSeq = SequenceNumber{ SequenceNumber::MAX }; @@ -27,7 +26,7 @@ void UdtCC::init() { _packetSendPeriod = 1.0; } -void UdtCC::onACK(SequenceNumber ackNum) { +void DefaultCC::onACK(SequenceNumber ackNum) { int64_t B = 0; double inc = 0; // Note: 1/24/2012 @@ -37,7 +36,7 @@ void UdtCC::onACK(SequenceNumber ackNum) { const double min_inc = 0.01; uint64_t currtime = usecTimestampNow(); - if (currtime - _lastRCTime < (uint64_t)_rcInterval) { + if (currtime - _lastRCTime < (uint64_t)synInterval()) { return; } @@ -52,11 +51,11 @@ void UdtCC::onACK(SequenceNumber ackNum) { if (_recvieveRate > 0) { _packetSendPeriod = 1000000.0 / _recvieveRate; } else { - _packetSendPeriod = (_rtt + _rcInterval) / _congestionWindowSize; + _packetSendPeriod = (_rtt + synInterval()) / _congestionWindowSize; } } } else { - _congestionWindowSize = _recvieveRate / 1000000.0 * (_rtt + _rcInterval) + 16; + _congestionWindowSize = _recvieveRate / 1000000.0 * (_rtt + synInterval()) + 16; } // During Slow Start, no rate increase @@ -86,10 +85,10 @@ void UdtCC::onACK(SequenceNumber ackNum) { } } - _packetSendPeriod = (_packetSendPeriod * _rcInterval) / (_packetSendPeriod * inc + _rcInterval); + _packetSendPeriod = (_packetSendPeriod * synInterval()) / (_packetSendPeriod * inc + synInterval()); } -void UdtCC::onLoss(const std::vector& losslist) { +void DefaultCC::onLoss(const std::vector& losslist) { //Slow Start stopped, if it hasn't yet if (_slowStart) { _slowStart = false; @@ -101,7 +100,7 @@ void UdtCC::onLoss(const std::vector& losslist) { // If no receiving rate is observed, we have to compute the sending // rate according to the current window size, and decrease it // using the method below. - _packetSendPeriod = _congestionWindowSize / (_rtt + _rcInterval); + _packetSendPeriod = _congestionWindowSize / (_rtt + synInterval()); } _loss = true; @@ -128,13 +127,13 @@ void UdtCC::onLoss(const std::vector& losslist) { } } -void UdtCC::onTimeout() { +void DefaultCC::onTimeout() { if (_slowStart) { _slowStart = false; if (_recvieveRate > 0) { _packetSendPeriod = 1000000.0 / _recvieveRate; } else { - _packetSendPeriod = _congestionWindowSize / (_rtt + _rcInterval); + _packetSendPeriod = _congestionWindowSize / (_rtt + synInterval()); } } else { /* diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 9f61c7944f..c54405e73d 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -17,15 +17,19 @@ #include "SequenceNumber.h" namespace udt { + +static const int32_t DEFAULT_SYN_INTERVAL = 10000; // 10 ms class Packet; class CongestionControl { public: - static const int32_t DEFAULT_SYN_INTERVAL = 10000; // 10 ms - CongestionControl() {} + CongestionControl() {}; + CongestionControl(int synInterval) : _synInterval(synInterval) {} virtual ~CongestionControl() {} + + int synInterval() const { return _synInterval; } virtual void init() {} virtual void close() {} @@ -35,12 +39,10 @@ public: virtual void onPacketReceived(const Packet& packet) {} protected: - void setAckTimer(int syn) { _ackPeriod = (syn > _synInterval) ? _synInterval : syn; } - void setAckInterval(int interval) { _ackInterval = interval; } + void setAckTimer(int period) { _ackPeriod = (period > _synInterval) ? _synInterval : period; } + void setAckInterval(int ackInterval) { _ackInterval = ackInterval; } void setRto(int rto) { _userDefinedRto = true; _rto = rto; } - int32_t _synInterval = DEFAULT_SYN_INTERVAL; // UDT constant parameter, SYN - double _packetSendPeriod = 1.0; // Packet sending period, in microseconds double _congestionWindowSize = 16.0; // Congestion window size, in packets @@ -66,6 +68,8 @@ private: int _ackPeriod = 0; // Periodical timer to send an ACK, in milliseconds int _ackInterval = 0; // How many packets to send one ACK, in packets + int _synInterval { DEFAULT_SYN_INTERVAL }; + bool _userDefinedRto = false; // if the RTO value is defined by users int _rto = -1; // RTO value, microseconds }; @@ -75,6 +79,8 @@ class CongestionControlVirtualFactory { public: virtual ~CongestionControlVirtualFactory() {} + static int synInterval() { return DEFAULT_SYN_INTERVAL; } + virtual std::unique_ptr create() = 0; }; @@ -82,13 +88,12 @@ template class CongestionControlFactory: public CongestionControlVirtu { public: virtual ~CongestionControlFactory() {} - virtual std::unique_ptr create() { return std::unique_ptr(new T()); } }; -class UdtCC: public CongestionControl { +class DefaultCC: public CongestionControl { public: - UdtCC() {} + DefaultCC() {} public: virtual void init(); @@ -97,7 +102,6 @@ public: virtual void onTimeout(); private: - int _rcInterval = 0; // UDT Rate control interval uint64_t _lastRCTime = 0; // last rate increase time bool _slowStart = true; // if in slow start phase SequenceNumber _lastAck; // last ACKed seq num @@ -112,4 +116,4 @@ private: } -#endif // hifi_CongestionControl_h \ No newline at end of file +#endif // hifi_CongestionControl_h diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 9b58dc41a5..0f09e67a72 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -16,6 +16,7 @@ #include #include "../HifiSockAddr.h" +#include "CongestionControl.h" #include "ControlPacket.h" #include "Packet.h" #include "Socket.h" @@ -24,10 +25,12 @@ using namespace udt; using namespace std; using namespace std::chrono; -Connection::Connection(Socket* parentSocket, HifiSockAddr destination) : +Connection::Connection(Socket* parentSocket, HifiSockAddr destination, unique_ptr congestionControl) : _parentSocket(parentSocket), - _destination(destination) + _destination(destination), + _congestionControl(move(congestionControl)) { + } Connection::~Connection() { @@ -343,6 +346,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { } // fire the onACK callback for congestion control + _congestionControl->onAck(ack); // update the total count of received ACKs ++_totalReceivedACKs; diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 200f8acda8..7cc2563fc5 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -22,6 +22,7 @@ namespace udt { +class CongestionControl; class ControlPacket; class Packet; class Socket; @@ -32,7 +33,7 @@ public: using SequenceNumberTimePair = std::pair; using SentACKMap = std::unordered_map; - Connection(Socket* parentSocket, HifiSockAddr destination); + Connection(Socket* parentSocket, HifiSockAddr destination, std::unique_ptr congestionControl); ~Connection(); void sendReliablePacket(std::unique_ptr packet); @@ -89,6 +90,8 @@ private: std::unique_ptr _sendQueue; + std::unique_ptr _congestionControl; + // Control Packet stat collection int _totalReceivedACKs { 0 }; int _totalSentACKs { 0 }; diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 15f0febc9b..c2ad40b9f5 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -79,7 +79,7 @@ qint64 Socket::writePacket(std::unique_ptr packet, const HifiSockAddr& s if (packet->isReliable()) { auto it = _connectionsHash.find(sockAddr); if (it == _connectionsHash.end()) { - it = _connectionsHash.insert(it, std::make_pair(sockAddr, new Connection(this, sockAddr))); + it = _connectionsHash.insert(it, std::make_pair(sockAddr, new Connection(this, sockAddr, _ccFactory->create()))); } it->second->sendReliablePacket(std::move(packet)); return 0; @@ -154,3 +154,11 @@ void Socket::rateControlSync() { _synTimer.start(_synInterval); } } + +void Socket::setCongestionControlFactory(std::unique_ptr ccFactory) { + // swap the current unique_ptr for the new factory + _ccFactory.swap(ccFactory); + + // update the _synInterval to the value from the factory + _synInterval = _ccFactory->synInterval(); +} diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 9ca4adcc74..497c98b296 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -22,6 +22,7 @@ #include #include "../HifiSockAddr.h" +#include "CongestionControl.h" #include "Connection.h" namespace udt { @@ -59,6 +60,8 @@ public: void addUnfilteredHandler(const HifiSockAddr& senderSockAddr, BasePacketHandler handler) { _unfilteredHandlers[senderSockAddr] = handler; } + + void setCongestionControlFactory(std::unique_ptr ccFactory); private slots: void readPendingDatagrams(); @@ -78,6 +81,8 @@ private: int32_t _synInterval = 10; // 10ms QTimer _synTimer; + + std::unique_ptr _ccFactory { new CongestionControlFactory() }; }; } // namespace udt From cb134f4685dc04721510f3446865d08707a4a33a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 18:04:18 -0700 Subject: [PATCH 078/242] remove onPacketSent and onPacketReceived from CC, use setRTT --- libraries/networking/src/udt/CongestionControl.h | 4 ++-- libraries/networking/src/udt/Connection.cpp | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index c54405e73d..27c1c8e8fd 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -20,9 +20,11 @@ namespace udt { static const int32_t DEFAULT_SYN_INTERVAL = 10000; // 10 ms +class Connection; class Packet; class CongestionControl { + friend class Connection; public: CongestionControl() {}; @@ -35,8 +37,6 @@ public: virtual void close() {} virtual void onAck(SequenceNumber ackNum) {} virtual void onLoss(const std::vector& lossList) {} - virtual void onPacketSent(const Packet& packet) {} - virtual void onPacketReceived(const Packet& packet) {} protected: void setAckTimer(int period) { _ackPeriod = (period > _synInterval) ? _synInterval : period; } diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 0f09e67a72..d4dd0be7bf 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -336,6 +336,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { updateRTT(rtt); // set the RTT for congestion control + _congestionControl->setRtt(_rtt); if (controlPacket->getPayloadSize() > (qint64) (sizeof(SequenceNumber) + sizeof(SequenceNumber) + sizeof(rtt))) { int32_t deliveryRate, bandwidth; @@ -343,6 +344,8 @@ void Connection::processACK(std::unique_ptr controlPacket) { controlPacket->readPrimitive(&bandwidth); // set the delivery rate and bandwidth for congestion control + _congestionControl->setRcvRate(deliveryRate); + _congestionControl->setBandwidth(bandwidth); } // fire the onACK callback for congestion control @@ -387,6 +390,7 @@ void Connection::processACK2(std::unique_ptr controlPacket) { updateRTT(rtt); // set the RTT for congestion control + _congestionControl->setRtt(_rtt); // update the last ACKed ACK if (pair.first > _lastReceivedAcknowledgedACK) { @@ -401,10 +405,13 @@ void Connection::processNAK(std::unique_ptr controlPacket) { // read the loss report SequenceNumber start, end; controlPacket->readPrimitive(&start); + if (controlPacket->bytesLeftToRead() >= (qint64)sizeof(SequenceNumber)) { controlPacket->readPrimitive(&end); } + + ++_totalReceivedNAKs; } From b9b4245dfee687927600dd8641ce13e5068c54fc Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 18:05:42 -0700 Subject: [PATCH 079/242] NAK interval is in microseconds --- libraries/networking/src/udt/Connection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index d4dd0be7bf..d43901f705 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -65,7 +65,7 @@ void Connection::sync() { // we do this if it has been longer than the current nakInterval since we last sent auto now = high_resolution_clock::now(); - if (duration_cast(now - _lastNAKTime).count() >= _nakInterval) { + if (duration_cast(now - _lastNAKTime).count() >= _nakInterval) { // construct a NAK packet that will hold all of the lost sequence numbers auto lossListPacket = ControlPacket::create(ControlPacket::TimeoutNAK, _lossList.getLength() * sizeof(SequenceNumber)); From b5b7fa850489a063e9bfb7ebcfdc5ad5af3caf18 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 28 Jul 2015 18:08:02 -0700 Subject: [PATCH 080/242] SendQueue uses LossList --- libraries/networking/src/udt/SendQueue.cpp | 27 ++++++---------------- libraries/networking/src/udt/SendQueue.h | 5 ++-- 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 787bf60ca7..bc619d95bc 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -91,36 +91,24 @@ void SendQueue::ack(SequenceNumber ack) { return; } - { - // remove any ACKed packets from the map of sent packets + { // remove any ACKed packets from the map of sent packets QWriteLocker locker(&_sentLock); for (auto seq = _lastAck; seq <= ack; ++seq) { _sentPackets.erase(seq); } } - { - // remove any sequence numbers equal to or lower than this ACK in the loss list + { // remove any sequence numbers equal to or lower than this ACK in the loss list QWriteLocker nakLocker(&_naksLock); - - auto it = _naks.begin(); - - while (it != _naks.end()) { - if (*it <= ack) { - it = _naks.erase(it); - } else { - // the NAKs in the NAK list must be in order, so we can break if we hit one > ack - break; - } - } + _naks.remove(_naks.getFirstSequenceNumber(), ack); } _lastAck = ack; } -void SendQueue::nak(std::list naks) { +void SendQueue::nak(SequenceNumber start, SequenceNumber end) { QWriteLocker locker(&_naksLock); - _naks.splice(_naks.end(), naks); // Add naks at the end + _naks.insert(start, end); } SequenceNumber SendQueue::getNextSequenceNumber() { @@ -154,10 +142,9 @@ void SendQueue::sendNextPacket() { { // Check nak list for packet to resend QWriteLocker locker(&_naksLock); - if (!_naks.empty()) { + if (_naks.getLength() > 0) { hasResend = true; - seqNum = _naks.front(); - _naks.pop_front(); + seqNum = _naks.popFirstSequenceNumber(); } } diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 1c30d1580a..82c569f6c5 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -22,6 +22,7 @@ #include "../HifiSockAddr.h" #include "SequenceNumber.h" +#include "LossList.h" namespace udt { @@ -55,7 +56,7 @@ public slots: void stop(); void ack(SequenceNumber ack); - void nak(std::list naks); + void nak(SequenceNumber start, SequenceNumber end); private slots: void sendNextPacket(); @@ -88,7 +89,7 @@ private: std::atomic _running { false }; mutable QReadWriteLock _naksLock; // Protects the naks list. - std::list _naks; // Sequence numbers of packets to resend + LossList _naks; // Sequence numbers of packets to resend mutable QReadWriteLock _sentLock; // Protects the sent packet list std::unordered_map> _sentPackets; // Packets waiting for ACK. From dc65a2fe1ac7b7360978b15f97a0d27fd542c448 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 18:11:21 -0700 Subject: [PATCH 081/242] use user defined RTO for timeout (if defined) --- libraries/networking/src/udt/Connection.cpp | 4 ++++ libraries/networking/src/udt/Connection.h | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index d43901f705..5ba749ad30 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -440,3 +440,7 @@ void Connection::updateRTT(int rtt) { _rttVariance = (_rttVariance * (1 - RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR) + abs(rtt - _rtt)) / RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR; } + +int Connection::estimatedTimeout() const { + return _congestionControl->_userDefinedRto ? _rtt + _rttVariance * 4 : _congestionControl->_rto; +} diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 7cc2563fc5..c0a8cf1aa1 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -59,7 +59,7 @@ private: void updateRTT(int rtt); - int estimatedTimeout() const { return _rtt + _rttVariance * 4; } + int estimatedTimeout() const; int _synInterval; // Periodical Rate Control Interval, in microseconds, defaults to 10ms From e69cc2aae8b69d97ee760e51be6c1967730ae4be Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 18:20:37 -0700 Subject: [PATCH 082/242] pass const ref LossList to CongestionControl --- libraries/networking/src/udt/CongestionControl.cpp | 4 ++-- libraries/networking/src/udt/CongestionControl.h | 5 +++-- libraries/networking/src/udt/Connection.cpp | 10 ++++++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 5acba518e4..1f241a1e09 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -88,7 +88,7 @@ void DefaultCC::onACK(SequenceNumber ackNum) { _packetSendPeriod = (_packetSendPeriod * synInterval()) / (_packetSendPeriod * inc + synInterval()); } -void DefaultCC::onLoss(const std::vector& losslist) { +void DefaultCC::onLoss(const LossList& losslist) { //Slow Start stopped, if it hasn't yet if (_slowStart) { _slowStart = false; @@ -105,7 +105,7 @@ void DefaultCC::onLoss(const std::vector& losslist) { _loss = true; - if (losslist[0] > _lastDecSeq) { + if (losslist.getFirstSequenceNumber() > _lastDecSeq) { _lastDecPeriod = _packetSendPeriod; _packetSendPeriod = ceil(_packetSendPeriod * 1.125); diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 27c1c8e8fd..c328afb9dd 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -14,6 +14,7 @@ #include +#include "LossList.h" #include "SequenceNumber.h" namespace udt { @@ -36,7 +37,7 @@ public: virtual void init() {} virtual void close() {} virtual void onAck(SequenceNumber ackNum) {} - virtual void onLoss(const std::vector& lossList) {} + virtual void onLoss(const LossList& lossList) {} protected: void setAckTimer(int period) { _ackPeriod = (period > _synInterval) ? _synInterval : period; } @@ -98,7 +99,7 @@ public: public: virtual void init(); virtual void onACK(SequenceNumber ackNum); - virtual void onLoss(const std::vector& lossList); + virtual void onLoss(const LossList& lossList); virtual void onTimeout(); private: diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 5ba749ad30..b5ca118730 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -217,6 +217,9 @@ void Connection::processReceivedSequenceNumber(SequenceNumber seq) { ++_totalSentNAKs; + // tell the CC that there was loss + _congestionControl->onLoss(_lossList); + // figure out when we should send the next loss report, if we haven't heard anything back _nakInterval = (_rtt + 4 * _rttVariance); @@ -406,12 +409,15 @@ void Connection::processNAK(std::unique_ptr controlPacket) { SequenceNumber start, end; controlPacket->readPrimitive(&start); + end = start; + if (controlPacket->bytesLeftToRead() >= (qint64)sizeof(SequenceNumber)) { controlPacket->readPrimitive(&end); } - - + // send that off to the send queue so it knows there was loss + _sendQueue->nak(start, end); + ++_totalReceivedNAKs; } From 3b7d0ab62aecadf1a6371bb86900daa29cb1f04d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 18:21:00 -0700 Subject: [PATCH 083/242] make TODO a TODO --- libraries/networking/src/udt/Connection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index b5ca118730..0418298d9d 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -333,7 +333,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { // update the atomic for last received ACK, the send queue uses this to re-transmit _atomicLastReceivedACK.store((uint32_t) _lastReceivedACK); - // remove everything up to this ACK from the sender loss list + // TODO: remove everything up to this ACK from the sender loss list // update the RTT updateRTT(rtt); From 43e81420bdda60c0dfc7c22c6b4b652b7b47831a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 18:21:32 -0700 Subject: [PATCH 084/242] add a TODO for processTimeoutNAK --- libraries/networking/src/udt/Connection.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 0418298d9d..e237535f2f 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -424,6 +424,8 @@ void Connection::processNAK(std::unique_ptr controlPacket) { void Connection::processTimeoutNAK(std::unique_ptr controlPacket) { // read the NAKed sequence numbers from the packet + // TODO: enumerate the received NAKs and create ranges if possible, then call _sendQueue->nak + ++_totalReceivedTimeoutNAKs; } From b7d80ca435a70bb56dfe9a7e6d2e4aa01760f9fd Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 18:22:33 -0700 Subject: [PATCH 085/242] remove a TODO that is actually done --- libraries/networking/src/udt/Connection.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index e237535f2f..0e3724b030 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -333,8 +333,6 @@ void Connection::processACK(std::unique_ptr controlPacket) { // update the atomic for last received ACK, the send queue uses this to re-transmit _atomicLastReceivedACK.store((uint32_t) _lastReceivedACK); - // TODO: remove everything up to this ACK from the sender loss list - // update the RTT updateRTT(rtt); From 065353eb6bcc89d04a958438d3f36d90dbe66df8 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 28 Jul 2015 18:27:02 -0700 Subject: [PATCH 086/242] Fix LossList::insert --- libraries/networking/src/udt/LossList.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/LossList.cpp b/libraries/networking/src/udt/LossList.cpp index ab66634b3e..5dbdac6de0 100644 --- a/libraries/networking/src/udt/LossList.cpp +++ b/libraries/networking/src/udt/LossList.cpp @@ -52,9 +52,10 @@ void LossList::insert(SequenceNumber start, SequenceNumber end) { if (end > it->second) { // If it goes further, find the actual end - auto it2 = find_if(it, _lossList.end(), [&end](pair pair){ + auto it2 = find_if_not(it, _lossList.end(), [&end](pair pair){ return end <= pair.second; }); + --it2; // If it ends inside a segment, change end (segment will be deleted) // Or backup iterator so segment doesn't get deleted From 99f9a4a712d434b241126bd252c50cbfeecabc66 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Jul 2015 19:23:38 -0700 Subject: [PATCH 087/242] add TODOs for onLoss, handle ACK from packet count --- libraries/networking/src/udt/Connection.cpp | 22 ++++++++++++++++----- libraries/networking/src/udt/Connection.h | 1 + 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 0e3724b030..29706148f0 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -79,7 +79,6 @@ void Connection::sync() { ++_totalSentTimeoutNAKs; } - } void Connection::sendACK(bool wasCausedBySyncTimeout) { @@ -144,6 +143,9 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { // write this ACK to the map of sent ACKs _sentACKs[_currentACKSubSequenceNumber] = { nextACKNumber, high_resolution_clock::now() }; + // reset the number of data packets received since last ACK + _packetsSinceACK = 0; + ++_totalSentACKs; } @@ -217,9 +219,6 @@ void Connection::processReceivedSequenceNumber(SequenceNumber seq) { ++_totalSentNAKs; - // tell the CC that there was loss - _congestionControl->onLoss(_lossList); - // figure out when we should send the next loss report, if we haven't heard anything back _nakInterval = (_rtt + 4 * _rttVariance); @@ -243,7 +242,14 @@ void Connection::processReceivedSequenceNumber(SequenceNumber seq) { _lossList.remove(seq); } + // increment the counters for data packets received + ++_packetsSinceACK; ++_totalReceivedDataPackets; + + // check if we need to send an ACK, according to CC params + if (_congestionControl->_ackInterval > 0 && _packetsSinceACK >= _congestionControl->_ackInterval) { + sendACK(false); + } } void Connection::processControl(unique_ptr controlPacket) { @@ -413,9 +419,12 @@ void Connection::processNAK(std::unique_ptr controlPacket) { controlPacket->readPrimitive(&end); } + // TODO: tell the congestion control object that there was loss + // _congestionControl->onLoss(); + // send that off to the send queue so it knows there was loss _sendQueue->nak(start, end); - + ++_totalReceivedNAKs; } @@ -424,6 +433,9 @@ void Connection::processTimeoutNAK(std::unique_ptr controlPacket) // TODO: enumerate the received NAKs and create ranges if possible, then call _sendQueue->nak + // TODO: tell the congestion control object that there was loss + // _congestionControl->onLoss(); + ++_totalReceivedTimeoutNAKs; } diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index c0a8cf1aa1..a98f4c394c 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -106,6 +106,7 @@ private: // Data packet stat collection int _totalReceivedDataPackets { 0 }; + int _packetsSinceACK { 0 }; // The number of packets that have been received during the current ACK interval }; From 7b3761f07a8805e236d3eb93429936ad971e2457 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 28 Jul 2015 19:27:29 -0700 Subject: [PATCH 088/242] Implement NAK timeout packets read/write --- libraries/networking/src/udt/Connection.cpp | 12 ++++------ libraries/networking/src/udt/LossList.cpp | 26 +++++++++++++++++++++ libraries/networking/src/udt/LossList.h | 5 ++++ libraries/networking/src/udt/SendQueue.cpp | 5 ++++ libraries/networking/src/udt/SendQueue.h | 1 + 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 0e3724b030..8fdcbd14f7 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -30,7 +30,6 @@ Connection::Connection(Socket* parentSocket, HifiSockAddr destination, unique_pt _destination(destination), _congestionControl(move(congestionControl)) { - } Connection::~Connection() { @@ -69,7 +68,8 @@ void Connection::sync() { // construct a NAK packet that will hold all of the lost sequence numbers auto lossListPacket = ControlPacket::create(ControlPacket::TimeoutNAK, _lossList.getLength() * sizeof(SequenceNumber)); - // TODO: pack in the lost sequence numbers + // Pack in the lost sequence numbers + _lossList.write(*lossListPacket); // have our SendQueue send off this control packet _sendQueue->sendPacket(*lossListPacket); @@ -79,7 +79,6 @@ void Connection::sync() { ++_totalSentTimeoutNAKs; } - } void Connection::sendACK(bool wasCausedBySyncTimeout) { @@ -331,7 +330,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { _sendQueue->ack(ack); // update the atomic for last received ACK, the send queue uses this to re-transmit - _atomicLastReceivedACK.store((uint32_t) _lastReceivedACK); + _atomicLastReceivedACK = (SequenceNumber::Type)_lastReceivedACK; // update the RTT updateRTT(rtt); @@ -420,9 +419,8 @@ void Connection::processNAK(std::unique_ptr controlPacket) { } void Connection::processTimeoutNAK(std::unique_ptr controlPacket) { - // read the NAKed sequence numbers from the packet - - // TODO: enumerate the received NAKs and create ranges if possible, then call _sendQueue->nak + // Override SendQueue's LossList with the timeout NAK list + _sendQueue->overrideNAKListFromPacket(*controlPacket); ++_totalReceivedTimeoutNAKs; } diff --git a/libraries/networking/src/udt/LossList.cpp b/libraries/networking/src/udt/LossList.cpp index 5dbdac6de0..2e1346daae 100644 --- a/libraries/networking/src/udt/LossList.cpp +++ b/libraries/networking/src/udt/LossList.cpp @@ -11,6 +11,8 @@ #include "LossList.h" +#include "ControlPacket.h" + using namespace udt; using namespace std; @@ -152,3 +154,27 @@ SequenceNumber LossList::popFirstSequenceNumber() { remove(front); return front; } + +void LossList::write(ControlPacket& packet) { + for(const auto& pair : _lossList) { + packet.writePrimitive(pair.first); + packet.writePrimitive(pair.second); + } +} + +void LossList::read(ControlPacket& packet) { + _lossList.clear(); + + SequenceNumber first, second; + while (packet.bytesLeftToRead() > (qint64)(2 * sizeof(SequenceNumber))) { + packet.readPrimitive(&first); + packet.readPrimitive(&second); + + if (first == second) { + append(first); + } else { + append(first, second); + } + + } +} diff --git a/libraries/networking/src/udt/LossList.h b/libraries/networking/src/udt/LossList.h index a669317e9d..c88bef8baa 100644 --- a/libraries/networking/src/udt/LossList.h +++ b/libraries/networking/src/udt/LossList.h @@ -18,6 +18,8 @@ namespace udt { +class ControlPacket; + class LossList { public: LossList() {} @@ -36,6 +38,9 @@ public: SequenceNumber getFirstSequenceNumber() const; SequenceNumber popFirstSequenceNumber(); + void write(ControlPacket& packet); + void read(ControlPacket& packet); + private: std::list> _lossList; int _length { 0 }; diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 0931159483..e9f835b918 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -103,6 +103,11 @@ void SendQueue::nak(SequenceNumber start, SequenceNumber end) { _naks.insert(start, end); } +void SendQueue::overrideNAKListFromPacket(ControlPacket& packet) { + QWriteLocker locker(&_naksLock); + _naks.read(packet); +} + SequenceNumber SendQueue::getNextSequenceNumber() { _atomicCurrentSequenceNumber = (SequenceNumber::Type)++_currentSequenceNumber; return _currentSequenceNumber; diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 10b35079b0..2bb61a5002 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -56,6 +56,7 @@ public slots: void ack(SequenceNumber ack); void nak(SequenceNumber start, SequenceNumber end); + void overrideNAKListFromPacket(ControlPacket& packet); private slots: void loop(); From 7084c12434332e2977721a1597b7009bea8a38fb Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 10:28:59 -0700 Subject: [PATCH 089/242] make the CongestionControl object take a range for loss --- libraries/networking/src/udt/CongestionControl.cpp | 4 ++-- libraries/networking/src/udt/CongestionControl.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 1f241a1e09..a7cd45cfe7 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -88,7 +88,7 @@ void DefaultCC::onACK(SequenceNumber ackNum) { _packetSendPeriod = (_packetSendPeriod * synInterval()) / (_packetSendPeriod * inc + synInterval()); } -void DefaultCC::onLoss(const LossList& losslist) { +void DefaultCC::onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) { //Slow Start stopped, if it hasn't yet if (_slowStart) { _slowStart = false; @@ -105,7 +105,7 @@ void DefaultCC::onLoss(const LossList& losslist) { _loss = true; - if (losslist.getFirstSequenceNumber() > _lastDecSeq) { + if (rangeStart > _lastDecSeq) { _lastDecPeriod = _packetSendPeriod; _packetSendPeriod = ceil(_packetSendPeriod * 1.125); diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index c328afb9dd..93e842a21a 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -37,7 +37,7 @@ public: virtual void init() {} virtual void close() {} virtual void onAck(SequenceNumber ackNum) {} - virtual void onLoss(const LossList& lossList) {} + virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) {} protected: void setAckTimer(int period) { _ackPeriod = (period > _synInterval) ? _synInterval : period; } @@ -99,7 +99,7 @@ public: public: virtual void init(); virtual void onACK(SequenceNumber ackNum); - virtual void onLoss(const LossList& lossList); + virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd); virtual void onTimeout(); private: From 94db77155ea5547069e5e7072522f3a4d3b78bf9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 10:55:22 -0700 Subject: [PATCH 090/242] fix for RTT numerator calculations --- libraries/networking/src/udt/Connection.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index cc421e5298..4de12d28b0 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -452,9 +452,9 @@ void Connection::updateRTT(int rtt) { static const int RTT_ESTIMATION_ALPHA_NUMERATOR = 8; static const int RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR = 4; - _rtt = (_rtt * (1 - RTT_ESTIMATION_ALPHA_NUMERATOR) + rtt) / RTT_ESTIMATION_ALPHA_NUMERATOR; + _rtt = (_rtt * (RTT_ESTIMATION_ALPHA_NUMERATOR - 1) + rtt) / RTT_ESTIMATION_ALPHA_NUMERATOR; - _rttVariance = (_rttVariance * (1 - RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR) + _rttVariance = (_rttVariance * (RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR - 1) + abs(rtt - _rtt)) / RTT_ESTIMATION_VARIANCE_ALPHA_NUMERATOR; } From d344ba57434ff85ec918c6cec76f780fd0a94c57 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 10:56:38 -0700 Subject: [PATCH 091/242] add a comment for Connection processTimeoutNAK --- libraries/networking/src/udt/Connection.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 4de12d28b0..af8a0e1f75 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -432,8 +432,8 @@ void Connection::processTimeoutNAK(std::unique_ptr controlPacket) // Override SendQueue's LossList with the timeout NAK list _sendQueue->overrideNAKListFromPacket(*controlPacket); - // TODO: tell the congestion control object that there was loss - // _congestionControl->onLoss(); + // we don't tell the congestion control object there was loss here - this matches UDTs implementation + // a possible improvement would be to tell it which new loss this timeout packet told us about ++_totalReceivedTimeoutNAKs; } From 2cdaea4e730a8a82c8f5d6c209ef019decbabaf0 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 29 Jul 2015 11:09:39 -0700 Subject: [PATCH 092/242] Remove LossList::read method and add clear --- libraries/networking/src/udt/LossList.cpp | 17 ----------------- libraries/networking/src/udt/LossList.h | 3 ++- libraries/networking/src/udt/SendQueue.cpp | 15 ++++++++++++++- libraries/networking/src/udt/SendQueue.h | 3 ++- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/libraries/networking/src/udt/LossList.cpp b/libraries/networking/src/udt/LossList.cpp index 2e1346daae..0bf08fbc1d 100644 --- a/libraries/networking/src/udt/LossList.cpp +++ b/libraries/networking/src/udt/LossList.cpp @@ -161,20 +161,3 @@ void LossList::write(ControlPacket& packet) { packet.writePrimitive(pair.second); } } - -void LossList::read(ControlPacket& packet) { - _lossList.clear(); - - SequenceNumber first, second; - while (packet.bytesLeftToRead() > (qint64)(2 * sizeof(SequenceNumber))) { - packet.readPrimitive(&first); - packet.readPrimitive(&second); - - if (first == second) { - append(first); - } else { - append(first, second); - } - - } -} diff --git a/libraries/networking/src/udt/LossList.h b/libraries/networking/src/udt/LossList.h index c88bef8baa..5b5fb9f0c8 100644 --- a/libraries/networking/src/udt/LossList.h +++ b/libraries/networking/src/udt/LossList.h @@ -24,6 +24,8 @@ class LossList { public: LossList() {} + void clear() { _length = 0; _lossList.clear(); } + // Should always add at the end void append(SequenceNumber seq); void append(SequenceNumber start, SequenceNumber end); @@ -39,7 +41,6 @@ public: SequenceNumber popFirstSequenceNumber(); void write(ControlPacket& packet); - void read(ControlPacket& packet); private: std::list> _lossList; diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index e9f835b918..ce1ce41d13 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -18,6 +18,7 @@ #include +#include "ControlPacket.h" #include "Packet.h" #include "Socket.h" @@ -105,7 +106,19 @@ void SendQueue::nak(SequenceNumber start, SequenceNumber end) { void SendQueue::overrideNAKListFromPacket(ControlPacket& packet) { QWriteLocker locker(&_naksLock); - _naks.read(packet); + _naks.clear(); + + SequenceNumber first, second; + while (packet.bytesLeftToRead() > (qint64)(2 * sizeof(SequenceNumber))) { + packet.readPrimitive(&first); + packet.readPrimitive(&second); + + if (first == second) { + _naks.append(first); + } else { + _naks.append(first, second); + } + } } SequenceNumber SendQueue::getNextSequenceNumber() { diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 2bb61a5002..4437565ded 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -27,9 +27,10 @@ namespace udt { -class Socket; class BasePacket; +class ControlPacket; class Packet; +class Socket; class SendQueue : public QObject { Q_OBJECT From 89e32e4a6beddd0ffee571d8fd465e42f56646f9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 11:11:06 -0700 Subject: [PATCH 093/242] some initial cleanup in CongestionControl --- .../networking/src/udt/CongestionControl.cpp | 41 ++++++++++--------- .../networking/src/udt/CongestionControl.h | 15 +++---- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index a7cd45cfe7..969909e956 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -11,16 +11,17 @@ #include "CongestionControl.h" -#include +#include using namespace udt; +using namespace std::chrono; void DefaultCC::init() { - _lastRCTime = usecTimestampNow(); + _lastRCTime = high_resolution_clock::now(); setAckTimer(synInterval()); _lastAck = _sendCurrSeqNum; - _lastDecSeq = SequenceNumber{ SequenceNumber::MAX }; + _lastDecreaseMaxSeq = SequenceNumber { SequenceNumber::MAX }; _congestionWindowSize = 16.0; _packetSendPeriod = 1.0; @@ -35,12 +36,12 @@ void DefaultCC::onACK(SequenceNumber ackNum) { // for long time. const double min_inc = 0.01; - uint64_t currtime = usecTimestampNow(); - if (currtime - _lastRCTime < (uint64_t)synInterval()) { + auto now = high_resolution_clock::now(); + if (duration_cast(now - _lastRCTime).count() < synInterval()) { return; } - _lastRCTime = currtime; + _lastRCTime = now; if (_slowStart) { _congestionWindowSize += seqlen(_lastAck, ackNum); @@ -48,14 +49,14 @@ void DefaultCC::onACK(SequenceNumber ackNum) { if (_congestionWindowSize > _maxCongestionWindowSize) { _slowStart = false; - if (_recvieveRate > 0) { - _packetSendPeriod = 1000000.0 / _recvieveRate; + if (_receiveRate > 0) { + _packetSendPeriod = USECS_PER_SECOND / _receiveRate; } else { _packetSendPeriod = (_rtt + synInterval()) / _congestionWindowSize; } } } else { - _congestionWindowSize = _recvieveRate / 1000000.0 * (_rtt + synInterval()) + 16; + _congestionWindowSize = _receiveRate / USECS_PER_SECOND * (_rtt + synInterval()) + 16; } // During Slow Start, no rate increase @@ -69,7 +70,7 @@ void DefaultCC::onACK(SequenceNumber ackNum) { } B = (int64_t)(_bandwidth - 1000000.0 / _packetSendPeriod); - if ((_packetSendPeriod > _lastDecPeriod) && ((_bandwidth / 9) < B)) { + if ((_packetSendPeriod > _lastDecreasePeriod) && ((_bandwidth / 9) < B)) { B = _bandwidth / 9; } if (B <= 0) { @@ -92,9 +93,9 @@ void DefaultCC::onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) { //Slow Start stopped, if it hasn't yet if (_slowStart) { _slowStart = false; - if (_recvieveRate > 0) { + if (_receiveRate > 0) { // Set the sending rate to the receiving rate. - _packetSendPeriod = 1000000.0 / _recvieveRate; + _packetSendPeriod = USECS_PER_SECOND / _receiveRate; return; } // If no receiving rate is observed, we have to compute the sending @@ -105,33 +106,33 @@ void DefaultCC::onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) { _loss = true; - if (rangeStart > _lastDecSeq) { - _lastDecPeriod = _packetSendPeriod; + if (rangeStart > _lastDecreaseMaxSeq) { + _lastDecreasePeriod = _packetSendPeriod; _packetSendPeriod = ceil(_packetSendPeriod * 1.125); _avgNAKNum = (int)ceil(_avgNAKNum * 0.875 + _nakCount * 0.125); _nakCount = 1; _decCount = 1; - _lastDecSeq = _sendCurrSeqNum; + _lastDecreaseMaxSeq = _sendCurrSeqNum; // remove global synchronization using randomization - srand((uint32_t)_lastDecSeq); + srand((uint32_t)_lastDecreaseMaxSeq); _decRandom = (int)ceil(_avgNAKNum * (double(rand()) / RAND_MAX)); if (_decRandom < 1) _decRandom = 1; - } else if ((_decCount ++ < 5) && (0 == (++ _nakCount % _decRandom))) { + } else if ((_decCount++ < 5) && (0 == (++_nakCount % _decRandom))) { // 0.875^5 = 0.51, rate should not be decreased by more than half within a congestion period _packetSendPeriod = ceil(_packetSendPeriod * 1.125); - _lastDecSeq = _sendCurrSeqNum; + _lastDecreaseMaxSeq = _sendCurrSeqNum; } } void DefaultCC::onTimeout() { if (_slowStart) { _slowStart = false; - if (_recvieveRate > 0) { - _packetSendPeriod = 1000000.0 / _recvieveRate; + if (_receiveRate > 0) { + _packetSendPeriod = USECS_PER_SECOND / _receiveRate; } else { _packetSendPeriod = _congestionWindowSize / (_rtt + synInterval()); } diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 93e842a21a..83959f034f 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -12,6 +12,7 @@ #ifndef hifi_CongestionControl_h #define hifi_CongestionControl_h +#include #include #include "LossList.h" @@ -52,7 +53,7 @@ protected: int _mss = 0; // Maximum Packet Size, including all packet headers SequenceNumber _sendCurrSeqNum; // current maximum seq num sent out - int _recvieveRate = 0; // packet arrive rate at receiver side, packets per second + int _receiveRate = 0; // packet arrive rate at receiver side, packets per second int _rtt = 0; // current estimated RTT, microsecond private: @@ -62,9 +63,9 @@ private: void setMss(int mss) { _mss = mss; } void setMaxCongestionWindowSize(int window) { _maxCongestionWindowSize = window; } void setBandwidth(int bandwidth) { _bandwidth = bandwidth; } - void setSndCurrSeqNum(SequenceNumber seqNum) { _sendCurrSeqNum = seqNum; } - void setRcvRate(int rate) { _recvieveRate = rate; } - void setRtt(int rtt) { _rtt = rtt; } + void setSendCurrentSequenceNumber(SequenceNumber seqNum) { _sendCurrSeqNum = seqNum; } + void setReceiveRate(int rate) { _receiveRate = rate; } + void setRTT(int rtt) { _rtt = rtt; } int _ackPeriod = 0; // Periodical timer to send an ACK, in milliseconds int _ackInterval = 0; // How many packets to send one ACK, in packets @@ -103,12 +104,12 @@ public: virtual void onTimeout(); private: - uint64_t _lastRCTime = 0; // last rate increase time + std::chrono::high_resolution_clock::time_point _lastRCTime; // last rate increase time bool _slowStart = true; // if in slow start phase SequenceNumber _lastAck; // last ACKed seq num bool _loss = false; // if loss happened since last rate increase - SequenceNumber _lastDecSeq; // max pkt seq num sent out when last decrease happened - double _lastDecPeriod = 1; // value of pktsndperiod when last decrease happened + SequenceNumber _lastDecreaseMaxSeq; // max pkt seq num sent out when last decrease happened + double _lastDecreasePeriod = 1; // value of _packetSendPeriod when last decrease happened int _nakCount = 0; // NAK counter int _decRandom = 1; // random threshold on decrease by number of loss events int _avgNAKNum = 0; // average number of NAKs per congestion From f65472cbc760195552586c71f6eed3f6431227d4 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 11:14:03 -0700 Subject: [PATCH 094/242] use a double USECS_PER_SECOND in CongestionControl --- libraries/networking/src/udt/CongestionControl.cpp | 8 ++++---- libraries/networking/src/udt/Connection.cpp | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 969909e956..37b198ffb9 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -11,11 +11,11 @@ #include "CongestionControl.h" -#include - using namespace udt; using namespace std::chrono; +static const double USECS_PER_SECOND = 1000000.0; + void DefaultCC::init() { _lastRCTime = high_resolution_clock::now(); setAckTimer(synInterval()); @@ -69,7 +69,7 @@ void DefaultCC::onACK(SequenceNumber ackNum) { return; } - B = (int64_t)(_bandwidth - 1000000.0 / _packetSendPeriod); + B = (int64_t)(_bandwidth - USECS_PER_SECOND/ _packetSendPeriod); if ((_packetSendPeriod > _lastDecreasePeriod) && ((_bandwidth / 9) < B)) { B = _bandwidth / 9; } @@ -121,7 +121,7 @@ void DefaultCC::onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) { _decRandom = (int)ceil(_avgNAKNum * (double(rand()) / RAND_MAX)); if (_decRandom < 1) _decRandom = 1; - } else if ((_decCount++ < 5) && (0 == (++_nakCount % _decRandom))) { + } else if ((_decCount ++ < 5) && (0 == (++ _nakCount % _decRandom))) { // 0.875^5 = 0.51, rate should not be decreased by more than half within a congestion period _packetSendPeriod = ceil(_packetSendPeriod * 1.125); _lastDecreaseMaxSeq = _sendCurrSeqNum; diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index af8a0e1f75..a3fb200302 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -343,7 +343,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { updateRTT(rtt); // set the RTT for congestion control - _congestionControl->setRtt(_rtt); + _congestionControl->setRTT(_rtt); if (controlPacket->getPayloadSize() > (qint64) (sizeof(SequenceNumber) + sizeof(SequenceNumber) + sizeof(rtt))) { int32_t deliveryRate, bandwidth; @@ -351,7 +351,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { controlPacket->readPrimitive(&bandwidth); // set the delivery rate and bandwidth for congestion control - _congestionControl->setRcvRate(deliveryRate); + _congestionControl->setReceiveRate(deliveryRate); _congestionControl->setBandwidth(bandwidth); } @@ -397,7 +397,7 @@ void Connection::processACK2(std::unique_ptr controlPacket) { updateRTT(rtt); // set the RTT for congestion control - _congestionControl->setRtt(_rtt); + _congestionControl->setRTT(_rtt); // update the last ACKed ACK if (pair.first > _lastReceivedAcknowledgedACK) { From 169cb97db501e1ae068d9f84c810c6a26d1575ab Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 11:35:06 -0700 Subject: [PATCH 095/242] fix bandwidth and receive rate EWMA in Connection --- libraries/networking/src/udt/Connection.cpp | 14 ++++++++++---- libraries/networking/src/udt/Connection.h | 3 +++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index a3fb200302..d952b245f2 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -346,13 +346,19 @@ void Connection::processACK(std::unique_ptr controlPacket) { _congestionControl->setRTT(_rtt); if (controlPacket->getPayloadSize() > (qint64) (sizeof(SequenceNumber) + sizeof(SequenceNumber) + sizeof(rtt))) { - int32_t deliveryRate, bandwidth; - controlPacket->readPrimitive(&deliveryRate); + int32_t receiveRate, bandwidth; + controlPacket->readPrimitive(&receiveRate); controlPacket->readPrimitive(&bandwidth); // set the delivery rate and bandwidth for congestion control - _congestionControl->setReceiveRate(deliveryRate); - _congestionControl->setBandwidth(bandwidth); + // these are calculated using an EWMA + static const int EMWA_ALPHA_NUMERATOR = 8; + + _deliveryRate = (_deliveryRate * (EMWA_ALPHA_NUMERATOR - 1) + _deliveryRate) / EMWA_ALPHA_NUMERATOR; + _bandwidth = (_bandwidth * (EMWA_ALPHA_NUMERATOR - 1) + _bandwidth) / EMWA_ALPHA_NUMERATOR; + + _congestionControl->setReceiveRate(_deliveryRate); + _congestionControl->setBandwidth(_bandwidth); } // fire the onACK callback for congestion control diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index a98f4c394c..8a385ca02d 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -81,6 +81,9 @@ private: int32_t _rttVariance; // RTT variance int _flowWindowSize; // Flow control window size + int _bandwidth { 1 }; // Exponential moving average for estimated bandwidth, in packets per second + int _deliveryRate { 16 }; // Exponential moving average for receiver's receive rate, in packets per second + SentACKMap _sentACKs; // Map of ACK sub-sequence numbers to ACKed sequence number and sent time Socket* _parentSocket { nullptr }; From abfe434b3034651f5861cc2bc2e7b8811137f1d8 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 29 Jul 2015 11:39:33 -0700 Subject: [PATCH 096/242] Synchronise SendQueue and CongestionControl --- libraries/networking/src/udt/Connection.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index d952b245f2..9e68d05699 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -362,7 +362,9 @@ void Connection::processACK(std::unique_ptr controlPacket) { } // fire the onACK callback for congestion control + _congestionControl->setSendCurrentSequenceNumber(_sendQueue->getCurrentSequenceNumber()); _congestionControl->onAck(ack); + _sendQueue->setPacketSendPeriod(_congestionControl->_packetSendPeriod); // update the total count of received ACKs ++_totalReceivedACKs; @@ -425,12 +427,14 @@ void Connection::processNAK(std::unique_ptr controlPacket) { controlPacket->readPrimitive(&end); } - // TODO: tell the congestion control object that there was loss - // _congestionControl->onLoss(); - // send that off to the send queue so it knows there was loss _sendQueue->nak(start, end); + // Tell the congestion control object that there was loss + _congestionControl->setSendCurrentSequenceNumber(_sendQueue->getCurrentSequenceNumber()); + _congestionControl->onLoss(start, end); + _sendQueue->setPacketSendPeriod(_congestionControl->_packetSendPeriod); + ++_totalReceivedNAKs; } From c35fa6eaae3effa8bf01c828e3a36f678149c81c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 11:48:00 -0700 Subject: [PATCH 097/242] some more cleanup in congestion control --- .../networking/src/udt/CongestionControl.cpp | 37 ++++++++++++------- .../networking/src/udt/CongestionControl.h | 2 +- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 37b198ffb9..aef86c8923 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -28,13 +28,13 @@ void DefaultCC::init() { } void DefaultCC::onACK(SequenceNumber ackNum) { - int64_t B = 0; - double inc = 0; + double increase = 0; + // Note: 1/24/2012 // The minimum increase parameter is increased from "1.0 / _mss" to 0.01 // because the original was too small and caused sending rate to stay at low level // for long time. - const double min_inc = 0.01; + const double minimumIncrease = 0.01; auto now = high_resolution_clock::now(); if (duration_cast(now - _lastRCTime).count() < synInterval()) { @@ -69,24 +69,30 @@ void DefaultCC::onACK(SequenceNumber ackNum) { return; } - B = (int64_t)(_bandwidth - USECS_PER_SECOND/ _packetSendPeriod); - if ((_packetSendPeriod > _lastDecreasePeriod) && ((_bandwidth / 9) < B)) { - B = _bandwidth / 9; + int capacitySpeedDelta = (int) (_bandwidth - USECS_PER_SECOND / _packetSendPeriod); + + if ((_packetSendPeriod > _lastDecreasePeriod) && ((_bandwidth / 9) < capacitySpeedDelta)) { + capacitySpeedDelta = _bandwidth / 9; } - if (B <= 0) { - inc = min_inc; + + if (capacitySpeedDelta <= 0) { + increase = minimumIncrease; } else { - // inc = max(10 ^ ceil(log10( B * MSS * 8 ) * Beta / MSS, 1/MSS) + // inc = max(10 ^ ceil(log10(B * MSS * 8 ) * Beta / MSS, minimumIncrease) + // B = estimated link capacity // Beta = 1.5 * 10^(-6) - inc = pow(10.0, ceil(log10(B * _mss * 8.0))) * 0.0000015 / _mss; + static const double BETA = 0.0000015; + static const double BITS_PER_BYTE = 8.0; - if (inc < min_inc) { - inc = min_inc; + increase = pow(10.0, ceil(log10(capacitySpeedDelta * _mss * BITS_PER_BYTE))) * BETA / _mss; + + if (increase < minimumIncrease) { + increase = minimumIncrease; } } - _packetSendPeriod = (_packetSendPeriod * synInterval()) / (_packetSendPeriod * inc + synInterval()); + _packetSendPeriod = (_packetSendPeriod * synInterval()) / (_packetSendPeriod * increase + synInterval()); } void DefaultCC::onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) { @@ -119,8 +125,11 @@ void DefaultCC::onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) { // remove global synchronization using randomization srand((uint32_t)_lastDecreaseMaxSeq); _decRandom = (int)ceil(_avgNAKNum * (double(rand()) / RAND_MAX)); - if (_decRandom < 1) + + if (_decRandom < 1) { _decRandom = 1; + } + } else if ((_decCount ++ < 5) && (0 == (++ _nakCount % _decRandom))) { // 0.875^5 = 0.51, rate should not be decreased by more than half within a congestion period _packetSendPeriod = ceil(_packetSendPeriod * 1.125); diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 83959f034f..c65901ce15 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -60,7 +60,7 @@ private: CongestionControl(const CongestionControl& other) = delete; CongestionControl& operator=(const CongestionControl& other) = delete; - void setMss(int mss) { _mss = mss; } + void setMSS(int mss) { _mss = mss; } void setMaxCongestionWindowSize(int window) { _maxCongestionWindowSize = window; } void setBandwidth(int bandwidth) { _bandwidth = bandwidth; } void setSendCurrentSequenceNumber(SequenceNumber seqNum) { _sendCurrSeqNum = seqNum; } From 236c09785337555d0740b433e83518f8bf44d6ef Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 29 Jul 2015 12:30:55 -0700 Subject: [PATCH 098/242] Bit of code cleanup --- libraries/networking/src/NetworkPeer.cpp | 4 +- libraries/networking/src/NetworkPeer.h | 4 +- .../networking/src/udt/CongestionControl.h | 53 ++++++++++--------- libraries/networking/src/udt/Connection.cpp | 1 + libraries/networking/src/udt/Connection.h | 2 - 5 files changed, 32 insertions(+), 32 deletions(-) diff --git a/libraries/networking/src/NetworkPeer.cpp b/libraries/networking/src/NetworkPeer.cpp index 28f19d4f60..52e53e9665 100644 --- a/libraries/networking/src/NetworkPeer.cpp +++ b/libraries/networking/src/NetworkPeer.cpp @@ -32,7 +32,7 @@ NetworkPeer::NetworkPeer(QObject* parent) : _wakeTimestamp(QDateTime::currentMSecsSinceEpoch()), _connectionAttempts(0) { - _lastHeardMicrostamp.store(usecTimestampNow()); + _lastHeardMicrostamp = usecTimestampNow(); } NetworkPeer::NetworkPeer(const QUuid& uuid, const HifiSockAddr& publicSocket, const HifiSockAddr& localSocket, QObject* parent) : @@ -45,7 +45,7 @@ NetworkPeer::NetworkPeer(const QUuid& uuid, const HifiSockAddr& publicSocket, co _wakeTimestamp(QDateTime::currentMSecsSinceEpoch()), _connectionAttempts(0) { - _lastHeardMicrostamp.store(usecTimestampNow()); + _lastHeardMicrostamp = usecTimestampNow(); } void NetworkPeer::setPublicSocket(const HifiSockAddr& publicSocket) { diff --git a/libraries/networking/src/NetworkPeer.h b/libraries/networking/src/NetworkPeer.h index 1e03677587..8446586121 100644 --- a/libraries/networking/src/NetworkPeer.h +++ b/libraries/networking/src/NetworkPeer.h @@ -61,8 +61,8 @@ public: quint64 getWakeTimestamp() const { return _wakeTimestamp; } void setWakeTimestamp(quint64 wakeTimestamp) { _wakeTimestamp = wakeTimestamp; } - quint64 getLastHeardMicrostamp() const { return _lastHeardMicrostamp.load(); } - void setLastHeardMicrostamp(quint64 lastHeardMicrostamp) { _lastHeardMicrostamp.store(lastHeardMicrostamp); } + quint64 getLastHeardMicrostamp() const { return _lastHeardMicrostamp; } + void setLastHeardMicrostamp(quint64 lastHeardMicrostamp) { _lastHeardMicrostamp = lastHeardMicrostamp; } QByteArray toByteArray() const; diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 83959f034f..43a6413361 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -45,21 +45,6 @@ protected: void setAckInterval(int ackInterval) { _ackInterval = ackInterval; } void setRto(int rto) { _userDefinedRto = true; _rto = rto; } - double _packetSendPeriod = 1.0; // Packet sending period, in microseconds - double _congestionWindowSize = 16.0; // Congestion window size, in packets - - int _bandwidth = 0; // estimated bandwidth, packets per second - double _maxCongestionWindowSize = 0.0; // maximum cwnd size, in packets - - int _mss = 0; // Maximum Packet Size, including all packet headers - SequenceNumber _sendCurrSeqNum; // current maximum seq num sent out - int _receiveRate = 0; // packet arrive rate at receiver side, packets per second - int _rtt = 0; // current estimated RTT, microsecond - -private: - CongestionControl(const CongestionControl& other) = delete; - CongestionControl& operator=(const CongestionControl& other) = delete; - void setMss(int mss) { _mss = mss; } void setMaxCongestionWindowSize(int window) { _maxCongestionWindowSize = window; } void setBandwidth(int bandwidth) { _bandwidth = bandwidth; } @@ -67,13 +52,29 @@ private: void setReceiveRate(int rate) { _receiveRate = rate; } void setRTT(int rtt) { _rtt = rtt; } - int _ackPeriod = 0; // Periodical timer to send an ACK, in milliseconds - int _ackInterval = 0; // How many packets to send one ACK, in packets + + double _packetSendPeriod { 1.0 }; // Packet sending period, in microseconds + double _congestionWindowSize { 16.0 }; // Congestion window size, in packets + + int _bandwidth { 0 }; // estimated bandwidth, packets per second + double _maxCongestionWindowSize { 0.0 }; // maximum cwnd size, in packets + + int _mss { 0 }; // Maximum Packet Size, including all packet headers + SequenceNumber _sendCurrSeqNum; // current maximum seq num sent out + int _receiveRate { 0 }; // packet arrive rate at receiver side, packets per second + int _rtt { 0 }; // current estimated RTT, microsecond + +private: + CongestionControl(const CongestionControl& other) = delete; + CongestionControl& operator=(const CongestionControl& other) = delete; + + int _ackPeriod { 0 }; // Periodical timer to send an ACK, in milliseconds + int _ackInterval { 0 }; // How many packets to send one ACK, in packets int _synInterval { DEFAULT_SYN_INTERVAL }; - bool _userDefinedRto = false; // if the RTO value is defined by users - int _rto = -1; // RTO value, microseconds + bool _userDefinedRto { false }; // if the RTO value is defined by users + int _rto { -1 }; // RTO value, microseconds }; @@ -105,15 +106,15 @@ public: private: std::chrono::high_resolution_clock::time_point _lastRCTime; // last rate increase time - bool _slowStart = true; // if in slow start phase + bool _slowStart { true }; // if in slow start phase SequenceNumber _lastAck; // last ACKed seq num - bool _loss = false; // if loss happened since last rate increase + bool _loss { false }; // if loss happened since last rate increase SequenceNumber _lastDecreaseMaxSeq; // max pkt seq num sent out when last decrease happened - double _lastDecreasePeriod = 1; // value of _packetSendPeriod when last decrease happened - int _nakCount = 0; // NAK counter - int _decRandom = 1; // random threshold on decrease by number of loss events - int _avgNAKNum = 0; // average number of NAKs per congestion - int _decCount = 0; // number of decreases in a congestion epoch + double _lastDecreasePeriod { 1 }; // value of _packetSendPeriod when last decrease happened + int _nakCount { 0 }; // NAK counter + int _decRandom { 1 }; // random threshold on decrease by number of loss events + int _avgNAKNum { 0 }; // average number of NAKs per congestion + int _decCount { 0 }; // number of decreases in a congestion epoch }; } diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 9e68d05699..146714df10 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -253,6 +253,7 @@ void Connection::processReceivedSequenceNumber(SequenceNumber seq) { } void Connection::processControl(unique_ptr controlPacket) { + // Simple dispatch to control packets processing methods based on their type switch (controlPacket->getType()) { case ControlPacket::ACK: if (controlPacket->getPayloadSize() == sizeof(SequenceNumber)) { diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 8a385ca02d..b42ef0d8ea 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -110,8 +110,6 @@ private: // Data packet stat collection int _totalReceivedDataPackets { 0 }; int _packetsSinceACK { 0 }; // The number of packets that have been received during the current ACK interval - - }; } From 94fd7b428d28a8a8248de564e3d2be062b202253 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 12:37:36 -0700 Subject: [PATCH 099/242] add some comments to CongestionControl --- .../networking/src/udt/CongestionControl.cpp | 43 +++++++++++++++---- .../networking/src/udt/CongestionControl.h | 2 +- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index aef86c8923..a5e5b23304 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -20,7 +20,7 @@ void DefaultCC::init() { _lastRCTime = high_resolution_clock::now(); setAckTimer(synInterval()); - _lastAck = _sendCurrSeqNum; + _slowStartLastAck = _sendCurrSeqNum; _lastDecreaseMaxSeq = SequenceNumber { SequenceNumber::MAX }; _congestionWindowSize = 16.0; @@ -36,34 +36,45 @@ void DefaultCC::onACK(SequenceNumber ackNum) { // for long time. const double minimumIncrease = 0.01; + // we will only adjust once per sync interval so check that it has been at least that long now auto now = high_resolution_clock::now(); if (duration_cast(now - _lastRCTime).count() < synInterval()) { return; } + // our last rate increase time is now _lastRCTime = now; - + if (_slowStart) { - _congestionWindowSize += seqlen(_lastAck, ackNum); - _lastAck = ackNum; + // we are in slow start phase - increase the congestion window size by the number of packets just ACKed + _congestionWindowSize += seqlen(_slowStartLastAck, ackNum); + // update the last ACK + _slowStartLastAck = ackNum; + + // check if we can get out of slow start (is our new congestion window size bigger than the max) if (_congestionWindowSize > _maxCongestionWindowSize) { _slowStart = false; + if (_receiveRate > 0) { + // if we have a valid receive rate we set the send period to whatever the receive rate dictates _packetSendPeriod = USECS_PER_SECOND / _receiveRate; } else { + // no valid receive rate, packet send period is dictated by estimated RTT and current congestion window size _packetSendPeriod = (_rtt + synInterval()) / _congestionWindowSize; } } } else { + // not in slow start - window size should be arrival rate * (RTT + SYN) + 16 _congestionWindowSize = _receiveRate / USECS_PER_SECOND * (_rtt + synInterval()) + 16; } - // During Slow Start, no rate increase + // during slow start we perform no rate increases if (_slowStart) { return; } + // if loss has happened since the last rate increase we do not perform another increase if (_loss) { _loss = false; return; @@ -71,13 +82,19 @@ void DefaultCC::onACK(SequenceNumber ackNum) { int capacitySpeedDelta = (int) (_bandwidth - USECS_PER_SECOND / _packetSendPeriod); - if ((_packetSendPeriod > _lastDecreasePeriod) && ((_bandwidth / 9) < capacitySpeedDelta)) { - capacitySpeedDelta = _bandwidth / 9; + // UDT uses what they call DAIMD - additive increase multiplicative decrease with decreasing increases + // This factor is a protocol parameter that is part of the DAIMD algorithim + static const int AIMD_DECREASING_INCREASE_FACTOR = 9; + + if ((_packetSendPeriod > _lastDecreasePeriod) && ((_bandwidth / AIMD_DECREASING_INCREASE_FACTOR) < capacitySpeedDelta)) { + capacitySpeedDelta = _bandwidth / AIMD_DECREASING_INCREASE_FACTOR; } if (capacitySpeedDelta <= 0) { increase = minimumIncrease; } else { + // use UDTs DAIMD algorithm to figure out what the send period increase factor should be + // inc = max(10 ^ ceil(log10(B * MSS * 8 ) * Beta / MSS, minimumIncrease) // B = estimated link capacity // Beta = 1.5 * 10^(-6) @@ -112,11 +129,19 @@ void DefaultCC::onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) { _loss = true; + // check if this NAK starts a new congestion period - this will be the case if the + // NAK received occured for a packet sent after the last decrease if (rangeStart > _lastDecreaseMaxSeq) { _lastDecreasePeriod = _packetSendPeriod; - _packetSendPeriod = ceil(_packetSendPeriod * 1.125); - _avgNAKNum = (int)ceil(_avgNAKNum * 0.875 + _nakCount * 0.125); + static const double INTER_PACKET_ARRIVAL_INCREASE = 1.125; + _packetSendPeriod = ceil(_packetSendPeriod * INTER_PACKET_ARRIVAL_INCREASE); + + // use EWMA to update the average number of NAKs per congestion + static const double NAK_EWMA_ALPHA = 0.125; + _avgNAKNum = (int)ceil(_avgNAKNum * (1 - NAK_EWMA_ALPHA) + _nakCount * NAK_EWMA_ALPHA); + + _nakCount = 1; _decCount = 1; diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index c65901ce15..bf7161fffc 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -106,7 +106,7 @@ public: private: std::chrono::high_resolution_clock::time_point _lastRCTime; // last rate increase time bool _slowStart = true; // if in slow start phase - SequenceNumber _lastAck; // last ACKed seq num + SequenceNumber _slowStartLastAck; // last ACKed seq num, used only during slow start bool _loss = false; // if loss happened since last rate increase SequenceNumber _lastDecreaseMaxSeq; // max pkt seq num sent out when last decrease happened double _lastDecreasePeriod = 1; // value of _packetSendPeriod when last decrease happened From 7602c352d6d632defd4582d5af0812eccb650b95 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 13:57:05 -0700 Subject: [PATCH 100/242] finish cleanup in CongestionControl --- .../networking/src/udt/CongestionControl.cpp | 70 ++++++++++--------- .../networking/src/udt/CongestionControl.h | 8 ++- 2 files changed, 42 insertions(+), 36 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 0f158f1b15..14d88a4761 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -112,68 +112,72 @@ void DefaultCC::onACK(SequenceNumber ackNum) { } void DefaultCC::onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) { - //Slow Start stopped, if it hasn't yet + // stop the slow start if we haven't yet if (_slowStart) { - _slowStart = false; - if (_receiveRate > 0) { - // Set the sending rate to the receiving rate. - _packetSendPeriod = USECS_PER_SECOND / _receiveRate; - return; - } - // If no receiving rate is observed, we have to compute the sending - // rate according to the current window size, and decrease it - // using the method below. - _packetSendPeriod = _congestionWindowSize / (_rtt + synInterval()); + stopSlowStart(); } _loss = true; + static const double INTER_PACKET_ARRIVAL_INCREASE = 1.125; + static const int MAX_DECREASES_PER_CONGESTION_EPOCH = 5; + // check if this NAK starts a new congestion period - this will be the case if the // NAK received occured for a packet sent after the last decrease if (rangeStart > _lastDecreaseMaxSeq) { _lastDecreasePeriod = _packetSendPeriod; - static const double INTER_PACKET_ARRIVAL_INCREASE = 1.125; _packetSendPeriod = ceil(_packetSendPeriod * INTER_PACKET_ARRIVAL_INCREASE); // use EWMA to update the average number of NAKs per congestion static const double NAK_EWMA_ALPHA = 0.125; _avgNAKNum = (int)ceil(_avgNAKNum * (1 - NAK_EWMA_ALPHA) + _nakCount * NAK_EWMA_ALPHA); - + // update the count of NAKs and count of decreases in this interval _nakCount = 1; - _decCount = 1; + _decreaseCount = 1; _lastDecreaseMaxSeq = _sendCurrSeqNum; - // remove global synchronization using randomization - srand((uint32_t)_lastDecreaseMaxSeq); - _decRandom = (int)ceil(_avgNAKNum * (double(rand()) / RAND_MAX)); + // avoid synchronous rate decrease across connections using randomization + srand((unsigned) _lastDecreaseMaxSeq); + _randomDecreaseThreshold = (int) ceil(_avgNAKNum * (double(rand()) / RAND_MAX)); - if (_decRandom < 1) { - _decRandom = 1; + if (_randomDecreaseThreshold < 1) { + _randomDecreaseThreshold = 1; } - } else if ((_decCount ++ < 5) && (0 == (++ _nakCount % _decRandom))) { - // 0.875^5 = 0.51, rate should not be decreased by more than half within a congestion period - _packetSendPeriod = ceil(_packetSendPeriod * 1.125); + } else if ((_decreaseCount++ < MAX_DECREASES_PER_CONGESTION_EPOCH) && ((++_nakCount % _randomDecreaseThreshold) == 0)) { + // there have been fewer than MAX_DECREASES_PER_CONGESTION_EPOCH AND this NAK matches the random count at which we + // decided we would decrease the packet send period + + _packetSendPeriod = ceil(_packetSendPeriod * INTER_PACKET_ARRIVAL_INCREASE); _lastDecreaseMaxSeq = _sendCurrSeqNum; } } void DefaultCC::onTimeout() { if (_slowStart) { - _slowStart = false; - if (_receiveRate > 0) { - _packetSendPeriod = USECS_PER_SECOND / _receiveRate; - } else { - _packetSendPeriod = _congestionWindowSize / (_rtt + synInterval()); - } + stopSlowStart(); } else { - /* - _lastDecPeriod = _packetSendPeriod; - _packetSendPeriod = ceil(_packetSendPeriod * 2); - _lastDecSeq = _lastAck; - */ + // UDT used to do the following on timeout if not in slow start - we should check if it could be helpful + // _lastDecreasePeriod = _packetSendPeriod; + // _packetSendPeriod = ceil(_packetSendPeriod * 2); + + // this seems odd - the last ack they were setting _lastDecreaseMaxSeq to only applies to slow start + // _lastDecreaseMaxSeq = _slowStartLastAck; } } + +void DefaultCC::stopSlowStart() { + _slowStart = false; + if (_receiveRate > 0) { + // Set the sending rate to the receiving rate. + _packetSendPeriod = USECS_PER_SECOND / _receiveRate; + return; + } + // If no receiving rate is observed, we have to compute the sending + // rate according to the current window size, and decrease it + // using the method below. + _packetSendPeriod = _congestionWindowSize / (_rtt + synInterval()); +} diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index d37c5c883e..75590d88b5 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -103,16 +103,18 @@ public: virtual void onTimeout(); private: + void stopSlowStart(); // stops the slow start on loss or timeout, if it's still on + std::chrono::high_resolution_clock::time_point _lastRCTime; // last rate increase time bool _slowStart { true }; // if in slow start phase SequenceNumber _slowStartLastAck; // last ACKed seq num bool _loss { false }; // if loss happened since last rate increase SequenceNumber _lastDecreaseMaxSeq; // max pkt seq num sent out when last decrease happened double _lastDecreasePeriod { 1 }; // value of _packetSendPeriod when last decrease happened - int _nakCount { 0 }; // NAK counter - int _decRandom { 1 }; // random threshold on decrease by number of loss events + int _nakCount { 0 }; // number of NAKs in congestion epoch + int _randomDecreaseThreshold { 1 }; // random threshold on decrease by number of loss events int _avgNAKNum { 0 }; // average number of NAKs per congestion - int _decCount { 0 }; // number of decreases in a congestion epoch + int _decreaseCount { 0 }; // number of decreases in a congestion epoch }; } From bb62231ebcf362c25be1488816811eb2cd13b17a Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 29 Jul 2015 14:14:28 -0700 Subject: [PATCH 101/242] Code cleanup --- libraries/networking/src/udt/SendQueue.h | 2 -- libraries/networking/src/udt/Socket.cpp | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 4437565ded..a75b784d9b 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -63,8 +63,6 @@ private slots: void loop(); private: - friend struct std::default_delete; - SendQueue(Socket* socket, HifiSockAddr dest); SendQueue(SendQueue& other) = delete; SendQueue(SendQueue&& other) = delete; diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index d82915c68f..c7c1d90666 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -136,7 +136,7 @@ void Socket::readPendingDatagrams() { if (!_packetFilterOperator || _packetFilterOperator(*packet)) { if (_packetHandler) { // call the verified packet callback to let it handle this packet - return _packetHandler(std::move(packet)); + _packetHandler(std::move(packet)); } } } From 85156b9d0fd8d18acfeaa12c6620e9ad56a2e880 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 15:11:12 -0700 Subject: [PATCH 102/242] add a constants file for UDT --- libraries/networking/src/LimitedNodeList.cpp | 5 ---- libraries/networking/src/udt/BasePacket.h | 3 +-- .../networking/src/udt/CongestionControl.cpp | 3 +++ libraries/networking/src/udt/Constants.h | 26 +++++++++++++++++++ libraries/networking/src/udt/Socket.cpp | 8 +++++- libraries/networking/src/udt/Socket.h | 4 +-- libraries/networking/src/udt/udt.cpp | 12 --------- libraries/networking/src/udt/udt.h | 15 ----------- 8 files changed, 39 insertions(+), 37 deletions(-) create mode 100644 libraries/networking/src/udt/Constants.h delete mode 100644 libraries/networking/src/udt/udt.cpp delete mode 100644 libraries/networking/src/udt/udt.h diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index fc8e6ad8fe..2ca4e89c12 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -33,8 +33,6 @@ #include "UUID.h" #include "NetworkLogging.h" -#include "udt/udt.h" - const char SOLO_NODE_TYPES[2] = { NodeType::AvatarMixer, NodeType::AudioMixer @@ -81,9 +79,6 @@ LimitedNodeList::LimitedNodeList(unsigned short socketListenPort, unsigned short qCDebug(networking) << "NodeList DTLS socket is listening on" << _dtlsSocket->localPort(); } - const int LARGER_BUFFER_SIZE = 1048576; - _nodeSocket.setBufferSizes(LARGER_BUFFER_SIZE); - // check for local socket updates every so often const int LOCAL_SOCKET_UPDATE_INTERVAL_MSECS = 5 * 1000; QTimer* localSocketUpdate = new QTimer(this); diff --git a/libraries/networking/src/udt/BasePacket.h b/libraries/networking/src/udt/BasePacket.h index d943e05204..8dcc3ab9be 100644 --- a/libraries/networking/src/udt/BasePacket.h +++ b/libraries/networking/src/udt/BasePacket.h @@ -17,11 +17,10 @@ #include #include "../HifiSockAddr.h" +#include "Constants.h" namespace udt { -static const int MAX_PACKET_SIZE = 1450; - class BasePacket : public QIODevice { Q_OBJECT public: diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 14d88a4761..06b63ec60e 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -10,6 +10,7 @@ // #include "CongestionControl.h" +#include "Packet.h" using namespace udt; using namespace std::chrono; @@ -19,6 +20,8 @@ static const double USECS_PER_SECOND = 1000000.0; void DefaultCC::init() { _lastRCTime = high_resolution_clock::now(); + _mss = udt::MAX_PACKET_SIZE; + _slowStartLastAck = _sendCurrSeqNum; _lastDecreaseMaxSeq = SequenceNumber { SequenceNumber::MAX }; diff --git a/libraries/networking/src/udt/Constants.h b/libraries/networking/src/udt/Constants.h new file mode 100644 index 0000000000..5c5ba392a1 --- /dev/null +++ b/libraries/networking/src/udt/Constants.h @@ -0,0 +1,26 @@ +// +// Constants.h +// libraries/networking/src/udt +// +// Created by Clement on 7/13/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#pragma once + +#ifndef hifi_udt_Constants_h +#define hifi_udt_Constants_h + +namespace udt { + static const int MAX_PACKET_SIZE = 1450; + static const int MAX_PACKETS_IN_FLIGHT = 25600; + static const int CONNECTION_RECEIVE_BUFFER_SIZE_PACKETS = 8192; + static const int CONNECTION_SEND_BUFFER_SIZE_PACKETS = 8192; + static const int UDP_SEND_BUFFER_SIZE_BYTES = 1048576; + static const int UDP_RECEIVE_BUFFER_SIZE_BYTES = 1048576; +} + +#endif // hifi_udt_Constants_h diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index d82915c68f..1307aef931 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -22,6 +22,8 @@ using namespace udt; Socket::Socket(QObject* parent) : QObject(parent) { + setSystemBufferSizes(); + connect(&_udpSocket, &QUdpSocket::readyRead, this, &Socket::readPendingDatagrams); // make sure our synchronization method is called every SYN interval @@ -38,17 +40,21 @@ void Socket::rebind() { _udpSocket.bind(QHostAddress::AnyIPv4, oldPort); } -void Socket::setBufferSizes(int numBytes) { +void Socket::setSystemBufferSizes() { for (int i = 0; i < 2; i++) { QAbstractSocket::SocketOption bufferOpt; QString bufferTypeString; + int numBytes = 0; + if (i == 0) { bufferOpt = QAbstractSocket::SendBufferSizeSocketOption; + numBytes = udt::UDP_SEND_BUFFER_SIZE_BYTES; bufferTypeString = "send"; } else { bufferOpt = QAbstractSocket::ReceiveBufferSizeSocketOption; + numBytes = udt::UDP_RECEIVE_BUFFER_SIZE_BYTES; bufferTypeString = "receive"; } diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 7cf4283cf3..0bc23cdf0a 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -56,8 +56,6 @@ public: void setPacketFilterOperator(PacketFilterOperator filterOperator) { _packetFilterOperator = filterOperator; } void setPacketHandler(PacketHandler handler) { _packetHandler = handler; } - void setBufferSizes(int numBytes); - void addUnfilteredHandler(const HifiSockAddr& senderSockAddr, BasePacketHandler handler) { _unfilteredHandlers[senderSockAddr] = handler; } @@ -68,6 +66,8 @@ private slots: void rateControlSync(); private: + void setSystemBufferSizes(); + QUdpSocket _udpSocket { this }; PacketFilterOperator _packetFilterOperator; PacketHandler _packetHandler; diff --git a/libraries/networking/src/udt/udt.cpp b/libraries/networking/src/udt/udt.cpp deleted file mode 100644 index d580a9f7f8..0000000000 --- a/libraries/networking/src/udt/udt.cpp +++ /dev/null @@ -1,12 +0,0 @@ -// -// udt.cpp -// -// -// Created by Clement on 7/13/15. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "udt.h" diff --git a/libraries/networking/src/udt/udt.h b/libraries/networking/src/udt/udt.h deleted file mode 100644 index 74803151d1..0000000000 --- a/libraries/networking/src/udt/udt.h +++ /dev/null @@ -1,15 +0,0 @@ -// -// udt.h -// -// -// Created by Clement on 7/13/15. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_udt_h -#define hifi_udt_h - -#endif // hifi_udt_h \ No newline at end of file From 05421aaf8ab002589f852c6ecaf825ad50a0358b Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 29 Jul 2015 15:17:34 -0700 Subject: [PATCH 103/242] Added ConnectionStats object for monitoring --- .../networking/src/udt/ConnectionStats.cpp | 92 +++++++++++++++++++ .../networking/src/udt/ConnectionStats.h | 67 ++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 libraries/networking/src/udt/ConnectionStats.cpp create mode 100644 libraries/networking/src/udt/ConnectionStats.h diff --git a/libraries/networking/src/udt/ConnectionStats.cpp b/libraries/networking/src/udt/ConnectionStats.cpp new file mode 100644 index 0000000000..394448afec --- /dev/null +++ b/libraries/networking/src/udt/ConnectionStats.cpp @@ -0,0 +1,92 @@ +// +// ConnectionStats.cpp +// libraries/networking/src/udt +// +// Created by Clement on 7/29/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "ConnectionStats.h" + +using namespace udt; +using namespace std::chrono; + +ConnectionStats::ConnectionStats() { + auto now = duration_cast(high_resolution_clock::now().time_since_epoch()); + _currentSample.startTime = now; + _total.startTime = now; +} + +ConnectionStats::Stats ConnectionStats::sample() { + Stats sample; + std::swap(sample, _currentSample); + + auto now = duration_cast(high_resolution_clock::now().time_since_epoch()); + sample.endTime = now; + _currentSample.startTime = now; + + return sample; +} + +void ConnectionStats::recordSentACK() { + ++_currentSample.sentACKs; + ++_total.sentACKs; +} + +void ConnectionStats::recordReceivedACK() { + ++_currentSample.receivedACKs; + ++_total.receivedACKs; +} + +void ConnectionStats::recordSentLightACK() { + ++_currentSample.sentLightACKs; + ++_total.sentLightACKs; +} + +void ConnectionStats::recordReceivedLightACK() { + ++_currentSample.receivedLightACKs; + ++_total.receivedLightACKs; +} + +void ConnectionStats::recordSentACK2() { + ++_currentSample.sentACK2s; + ++_total.sentACK2s; +} + +void ConnectionStats::recordReceivedACK2() { + ++_currentSample.receivedACK2s; + ++_total.receivedACK2s; +} + +void ConnectionStats::recordSentNAK() { + ++_currentSample.sentNAKs; + ++_total.sentNAKs; +} + +void ConnectionStats::recordReceivedNAK() { + ++_currentSample.receivedNAKs; + ++_total.receivedNAKs; +} + +void ConnectionStats::recordSentTimeoutNAK() { + ++_currentSample.sentTimeoutNAKs; + ++_total.sentTimeoutNAKs; +} + +void ConnectionStats::recordReceivedTimeoutNAK() { + ++_currentSample.receivedTimeoutNAKs; + ++_total.receivedTimeoutNAKs; +} + +void ConnectionStats::recordSentPackets() { + ++_currentSample.sentPackets; + ++_total.sentPackets; +} + +void ConnectionStats::recordReceivedPackets() { + ++_currentSample.recievedPackets; + ++_total.recievedPackets; +} \ No newline at end of file diff --git a/libraries/networking/src/udt/ConnectionStats.h b/libraries/networking/src/udt/ConnectionStats.h new file mode 100644 index 0000000000..df7dfd1545 --- /dev/null +++ b/libraries/networking/src/udt/ConnectionStats.h @@ -0,0 +1,67 @@ +// +// ConnectionStats.h +// libraries/networking/src/udt +// +// Created by Clement on 7/29/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_ConnectionStats_h +#define hifi_ConnectionStats_h + +#include + +namespace udt { + +class ConnectionStats { +public: + struct Stats { + std::chrono::microseconds startTime; + std::chrono::microseconds endTime; + + // Control Packet stat collection + int sentACKs { 0 }; + int receivedACKs { 0 }; + int sentLightACKs { 0 }; + int receivedLightACKs { 0 }; + int sentACK2s { 0 }; + int receivedACK2s { 0 }; + int sentNAKs { 0 }; + int receivedNAKs { 0 }; + int sentTimeoutNAKs { 0 }; + int receivedTimeoutNAKs { 0 }; + + int sentPackets { 0 }; + int recievedPackets { 0 }; + }; + + ConnectionStats(); + + Stats sample(); + Stats getTotalStats(); + + void recordSentACK(); + void recordReceivedACK(); + void recordSentLightACK(); + void recordReceivedLightACK(); + void recordSentACK2(); + void recordReceivedACK2(); + void recordSentNAK(); + void recordReceivedNAK(); + void recordSentTimeoutNAK(); + void recordReceivedTimeoutNAK(); + + void recordSentPackets(); + void recordReceivedPackets(); + +private: + Stats _currentSample; + Stats _total; +}; + +} + +#endif // hifi_ConnectionStats_h \ No newline at end of file From 5236abe561992296aa0b1c47d86e0645d5edc562 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 29 Jul 2015 15:18:42 -0700 Subject: [PATCH 104/242] Connetion record stats/factored send for control packets --- libraries/networking/src/udt/Connection.cpp | 140 +++++++++++--------- libraries/networking/src/udt/Connection.h | 18 +-- 2 files changed, 84 insertions(+), 74 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 146714df10..46915ce229 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -65,30 +65,12 @@ void Connection::sync() { auto now = high_resolution_clock::now(); if (duration_cast(now - _lastNAKTime).count() >= _nakInterval) { - // construct a NAK packet that will hold all of the lost sequence numbers - auto lossListPacket = ControlPacket::create(ControlPacket::TimeoutNAK, _lossList.getLength() * sizeof(SequenceNumber)); - - // Pack in the lost sequence numbers - _lossList.write(*lossListPacket); - - // have our SendQueue send off this control packet - _sendQueue->sendPacket(*lossListPacket); - - // record this as the last NAK time - _lastNAKTime = high_resolution_clock::now(); - - ++_totalSentTimeoutNAKs; + // Send a timeout NAK packet + sendTimeoutNAK(); } } void Connection::sendACK(bool wasCausedBySyncTimeout) { - static const int ACK_PACKET_PAYLOAD_BYTES = sizeof(_lastSentACK) + sizeof(_currentACKSubSequenceNumber) - + sizeof(_rtt) + sizeof(int32_t) + sizeof(int32_t); - - // setup the ACK packet, make it static so we can re-use it - static auto ackPacket = ControlPacket::create(ControlPacket::ACK, ACK_PACKET_PAYLOAD_BYTES); - ackPacket->reset(); // We need to reset it every time. - auto currentTime = high_resolution_clock::now(); static high_resolution_clock::time_point lastACKSendTime; @@ -117,6 +99,12 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { } } + // setup the ACK packet, make it static so we can re-use it + static const int ACK_PACKET_PAYLOAD_BYTES = sizeof(_lastSentACK) + sizeof(_currentACKSubSequenceNumber) + + sizeof(_rtt) + sizeof(int32_t) + sizeof(int32_t); + static auto ackPacket = ControlPacket::create(ControlPacket::ACK, ACK_PACKET_PAYLOAD_BYTES); + ackPacket->reset(); // We need to reset it every time. + // pack in the ACK sub-sequence number ackPacket->writePrimitive(_currentACKSubSequenceNumber++); @@ -146,14 +134,10 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { // reset the number of data packets received since last ACK _packetsSinceACK = 0; - ++_totalSentACKs; + _stats.recordSentACK(); } void Connection::sendLightACK() { - // create the light ACK packet, make it static so we can re-use it - static const int LIGHT_ACK_PACKET_PAYLOAD_BYTES = sizeof(SequenceNumber); - static auto lightACKPacket = ControlPacket::create(ControlPacket::ACK, LIGHT_ACK_PACKET_PAYLOAD_BYTES); - SequenceNumber nextACKNumber = nextACK(); if (nextACKNumber == _lastReceivedAcknowledgedACK) { @@ -161,6 +145,10 @@ void Connection::sendLightACK() { return; } + // create the light ACK packet, make it static so we can re-use it + static const int LIGHT_ACK_PACKET_PAYLOAD_BYTES = sizeof(SequenceNumber); + static auto lightACKPacket = ControlPacket::create(ControlPacket::ACK, LIGHT_ACK_PACKET_PAYLOAD_BYTES); + // reset the lightACKPacket before we go to write the ACK to it lightACKPacket->reset(); @@ -170,7 +158,62 @@ void Connection::sendLightACK() { // have the send queue send off our packet immediately _sendQueue->sendPacket(*lightACKPacket); - ++_totalSentLightACKs; + _stats.recordSentLightACK(); +} + +void Connection::sendACK2(SequenceNumber currentACKSubSequenceNumber) { + // setup a static ACK2 packet we will re-use + static const int ACK2_PAYLOAD_BYTES = sizeof(SequenceNumber); + static auto ack2Packet = ControlPacket::create(ControlPacket::ACK2, ACK2_PAYLOAD_BYTES); + + // reset the ACK2 Packet before writing the sub-sequence number to it + ack2Packet->reset(); + + // write the sub sequence number for this ACK2 + ack2Packet->writePrimitive(currentACKSubSequenceNumber); + + // update the last sent ACK2 and the last ACK2 send time + _lastSentACK2 = currentACKSubSequenceNumber; + + _stats.recordSentACK2(); +} + +void Connection::sendNAK(SequenceNumber sequenceNumberRecieved) { + // create the loss report packet, make it static so we can re-use it + static const int NAK_PACKET_PAYLOAD_BYTES = 2 * sizeof(SequenceNumber); + static auto lossReport = ControlPacket::create(ControlPacket::NAK, NAK_PACKET_PAYLOAD_BYTES); + lossReport->reset(); // We need to reset it every time. + + // pack in the loss report + lossReport->writePrimitive(_lastReceivedSequenceNumber + 1); + if (_lastReceivedSequenceNumber + 1 != sequenceNumberRecieved - 1) { + lossReport->writePrimitive(sequenceNumberRecieved - 1); + } + + // have the send queue send off our packet immediately + _sendQueue->sendPacket(*lossReport); + + // record our last NAK time + _lastNAKTime = high_resolution_clock::now(); + + _stats.recordSentNAK(); +} + +void Connection::sendTimeoutNAK() { + // construct a NAK packet that will hold all of the lost sequence numbers + // TODO size is wrong, fix it. + auto lossListPacket = ControlPacket::create(ControlPacket::TimeoutNAK, _lossList.getLength() * sizeof(SequenceNumber)); + + // Pack in the lost sequence numbers + _lossList.write(*lossListPacket); + + // have our SendQueue send off this control packet + _sendQueue->sendPacket(*lossListPacket); + + // record this as the last NAK time + _lastNAKTime = high_resolution_clock::now(); + + _stats.recordSentTimeoutNAK(); } SequenceNumber Connection::nextACK() const { @@ -200,24 +243,8 @@ void Connection::processReceivedSequenceNumber(SequenceNumber seq) { _lossList.append(_lastReceivedSequenceNumber + 1, seq - 1); } - // create the loss report packet, make it static so we can re-use it - static const int NAK_PACKET_PAYLOAD_BYTES = 2 * sizeof(SequenceNumber); - static auto lossReport = ControlPacket::create(ControlPacket::NAK, NAK_PACKET_PAYLOAD_BYTES); - lossReport->reset(); // We need to reset it every time. - - // pack in the loss report - lossReport->writePrimitive(_lastReceivedSequenceNumber + 1); - if (_lastReceivedSequenceNumber + 1 != seq - 1) { - lossReport->writePrimitive(seq - 1); - } - - // have the send queue send off our packet immediately - _sendQueue->sendPacket(*lossReport); - - // record our last NAK time - _lastNAKTime = high_resolution_clock::now(); - - ++_totalSentNAKs; + // Send a NAK packet + sendNAK(seq); // figure out when we should send the next loss report, if we haven't heard anything back _nakInterval = (_rtt + 4 * _rttVariance); @@ -288,21 +315,10 @@ void Connection::processACK(std::unique_ptr controlPacket) { microseconds sinceLastACK2 = duration_cast(currentTime - lastACK2SendTime); if (sinceLastACK2.count() > _synInterval || currentACKSubSequenceNumber == _lastSentACK2) { - // setup a static ACK2 packet we will re-use - static const int ACK2_PAYLOAD_BYTES = sizeof(SequenceNumber); - static auto ack2Packet = ControlPacket::create(ControlPacket::ACK2, ACK2_PAYLOAD_BYTES); + // Send ACK2 packet + sendACK2(currentACKSubSequenceNumber); - // reset the ACK2 Packet before writing the sub-sequence number to it - ack2Packet->reset(); - - // write the sub sequence number for this ACK2 - ack2Packet->writePrimitive(currentACKSubSequenceNumber); - - // update the last sent ACK2 and the last ACK2 send time - _lastSentACK2 = currentACKSubSequenceNumber; lastACK2SendTime = high_resolution_clock::now(); - - ++_totalSentACK2s; } // read the ACKed sequence number @@ -368,7 +384,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { _sendQueue->setPacketSendPeriod(_congestionControl->_packetSendPeriod); // update the total count of received ACKs - ++_totalReceivedACKs; + _stats.recordReceivedACK(); } void Connection::processLightACK(std::unique_ptr controlPacket) { @@ -385,7 +401,7 @@ void Connection::processLightACK(std::unique_ptr controlPacket) { _lastReceivedACK = ack; } - ++_totalReceivedLightACKs; + _stats.recordReceivedLightACK(); } void Connection::processACK2(std::unique_ptr controlPacket) { @@ -414,7 +430,7 @@ void Connection::processACK2(std::unique_ptr controlPacket) { } } - ++_totalReceivedACK2s; + _stats.recordReceivedACK2(); } void Connection::processNAK(std::unique_ptr controlPacket) { @@ -436,7 +452,7 @@ void Connection::processNAK(std::unique_ptr controlPacket) { _congestionControl->onLoss(start, end); _sendQueue->setPacketSendPeriod(_congestionControl->_packetSendPeriod); - ++_totalReceivedNAKs; + _stats.recordReceivedNAK(); } void Connection::processTimeoutNAK(std::unique_ptr controlPacket) { @@ -446,7 +462,7 @@ void Connection::processTimeoutNAK(std::unique_ptr controlPacket) // we don't tell the congestion control object there was loss here - this matches UDTs implementation // a possible improvement would be to tell it which new loss this timeout packet told us about - ++_totalReceivedTimeoutNAKs; + _stats.recordReceivedTimeoutNAK(); } void Connection::updateRTT(int rtt) { diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index b42ef0d8ea..c5ff009fc9 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -15,6 +15,7 @@ #include #include +#include "ConnectionStats.h" #include "LossList.h" #include "PacketTimeWindow.h" #include "SendQueue.h" @@ -50,6 +51,9 @@ public: private: void sendACK(bool wasCausedBySyncTimeout = true); void sendLightACK(); + void sendACK2(SequenceNumber currentACKSubSequenceNumber); + void sendNAK(SequenceNumber sequenceNumberRecieved); + void sendTimeoutNAK(); void processACK(std::unique_ptr controlPacket); void processLightACK(std::unique_ptr controlPacket); @@ -95,21 +99,11 @@ private: std::unique_ptr _congestionControl; - // Control Packet stat collection - int _totalReceivedACKs { 0 }; - int _totalSentACKs { 0 }; - int _totalSentLightACKs { 0 }; - int _totalReceivedLightACKs { 0 }; - int _totalReceivedACK2s { 0 }; - int _totalSentACK2s { 0 }; - int _totalReceivedNAKs { 0 }; - int _totalSentNAKs { 0 }; - int _totalReceivedTimeoutNAKs { 0 }; - int _totalSentTimeoutNAKs { 0 }; - // Data packet stat collection int _totalReceivedDataPackets { 0 }; int _packetsSinceACK { 0 }; // The number of packets that have been received during the current ACK interval + + ConnectionStats _stats; }; } From 55555cf13e10579b800a7e26f1849ae9659b843c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 15:28:20 -0700 Subject: [PATCH 105/242] setup default for RTT, RTT variance, _synInterval --- libraries/networking/src/udt/CongestionControl.cpp | 2 +- libraries/networking/src/udt/Connection.cpp | 8 +++++++- libraries/networking/src/udt/Connection.h | 5 +++-- libraries/networking/src/udt/Constants.h | 4 +++- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 06b63ec60e..1450e60930 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -20,7 +20,7 @@ static const double USECS_PER_SECOND = 1000000.0; void DefaultCC::init() { _lastRCTime = high_resolution_clock::now(); - _mss = udt::MAX_PACKET_SIZE; + _mss = udt::MAX_PACKET_SIZE_WITH_UDP_HEADER; _slowStartLastAck = _sendCurrSeqNum; _lastDecreaseMaxSeq = SequenceNumber { SequenceNumber::MAX }; diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 146714df10..39e4254895 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -30,6 +30,10 @@ Connection::Connection(Socket* parentSocket, HifiSockAddr destination, unique_pt _destination(destination), _congestionControl(move(congestionControl)) { + // setup default SYN, RTT and RTT Variance based on the SYN interval in CongestionControl object + _synInterval = _congestionControl->synInterval(); + _rtt = _synInterval * 10; + _rttVariance = _rtt / 2; } Connection::~Connection() { @@ -126,7 +130,9 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { // pack in the RTT and variance ackPacket->writePrimitive(_rtt); - // pack the available buffer size - must be a minimum of 2 + // pack the available buffer size, in packets + // in our implementation we have no hard limit on receive buffer size, send the default value + ackPacket->writePrimitive((int32_t) udt::CONNECTION_RECEIVE_BUFFER_SIZE_PACKETS); if (wasCausedBySyncTimeout) { // pack in the receive speed and estimatedBandwidth diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index b42ef0d8ea..d957339b9f 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -15,6 +15,7 @@ #include #include +#include "Constants.h" #include "LossList.h" #include "PacketTimeWindow.h" #include "SendQueue.h" @@ -61,7 +62,7 @@ private: int estimatedTimeout() const; - int _synInterval; // Periodical Rate Control Interval, in microseconds, defaults to 10ms + int _synInterval; // Periodical Rate Control Interval, in microseconds int _nakInterval; // NAK timeout interval, in microseconds int _minNAKInterval { 100000 }; // NAK timeout interval lower bound, default of 100ms @@ -79,7 +80,7 @@ private: int32_t _rtt; // RTT, in microseconds int32_t _rttVariance; // RTT variance - int _flowWindowSize; // Flow control window size + int _flowWindowSize { udt::MAX_PACKETS_IN_FLIGHT }; // Flow control window size int _bandwidth { 1 }; // Exponential moving average for estimated bandwidth, in packets per second int _deliveryRate { 16 }; // Exponential moving average for receiver's receive rate, in packets per second diff --git a/libraries/networking/src/udt/Constants.h b/libraries/networking/src/udt/Constants.h index 5c5ba392a1..e8a7658bf9 100644 --- a/libraries/networking/src/udt/Constants.h +++ b/libraries/networking/src/udt/Constants.h @@ -15,12 +15,14 @@ #define hifi_udt_Constants_h namespace udt { - static const int MAX_PACKET_SIZE = 1450; + static const int MAX_PACKET_SIZE_WITH_UDP_HEADER = 1500; + static const int MAX_PACKET_SIZE = MAX_PACKET_SIZE_WITH_UDP_HEADER - 28; static const int MAX_PACKETS_IN_FLIGHT = 25600; static const int CONNECTION_RECEIVE_BUFFER_SIZE_PACKETS = 8192; static const int CONNECTION_SEND_BUFFER_SIZE_PACKETS = 8192; static const int UDP_SEND_BUFFER_SIZE_BYTES = 1048576; static const int UDP_RECEIVE_BUFFER_SIZE_BYTES = 1048576; + static const int DEFAULT_SYN_INTERVAL_USECS = 10 * 1000; } #endif // hifi_udt_Constants_h From 785d14f8352fb625a235959271cc72c19decd433 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 29 Jul 2015 15:43:44 -0700 Subject: [PATCH 106/242] Cleanup sendACK --- libraries/networking/src/udt/Connection.cpp | 36 ++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 46915ce229..bce650b85a 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -63,7 +63,7 @@ void Connection::sync() { // check if we need to re-transmit a loss list // we do this if it has been longer than the current nakInterval since we last sent auto now = high_resolution_clock::now(); - + if (duration_cast(now - _lastNAKTime).count() >= _nakInterval) { // Send a timeout NAK packet sendTimeoutNAK(); @@ -71,26 +71,19 @@ void Connection::sync() { } void Connection::sendACK(bool wasCausedBySyncTimeout) { - auto currentTime = high_resolution_clock::now(); static high_resolution_clock::time_point lastACKSendTime; + auto currentTime = high_resolution_clock::now(); SequenceNumber nextACKNumber = nextACK(); + Q_ASSERT_X(nextACKNumber < _lastSentACK, "Connection::sendACK", "Sending lower ACK, something is wrong"); - if (nextACKNumber <= _lastReceivedAcknowledgedACK) { - // we already got an ACK2 for this ACK we would be sending, don't bother - return; - } - - if (nextACKNumber >= _lastSentACK) { - // we have received new packets since the last sent ACK - - // update the last sent ACK - _lastSentACK = nextACKNumber; - - // remove the ACKed packets from the receive queue - - } else if (nextACKNumber == _lastSentACK) { + if (nextACKNumber == _lastSentACK) { // We already sent this ACK, but check if we should re-send it. + if (nextACKNumber <= _lastReceivedAcknowledgedACK) { + // we already got an ACK2 for this ACK we would be sending, don't bother + return; + } + // We will re-send if it has been more than the estimated timeout since the last ACK microseconds sinceLastACK = duration_cast(currentTime - lastACKSendTime); @@ -98,6 +91,13 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { return; } } + // we have received new packets since the last sent ACK + + // update the last sent ACK + _lastSentACK = nextACKNumber; + + // remove the ACKed packets from the receive queue + // TODO? // setup the ACK packet, make it static so we can re-use it static const int ACK_PACKET_PAYLOAD_BYTES = sizeof(_lastSentACK) + sizeof(_currentACKSubSequenceNumber) @@ -129,7 +129,7 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { _sendQueue->sendPacket(*ackPacket); // write this ACK to the map of sent ACKs - _sentACKs[_currentACKSubSequenceNumber] = { nextACKNumber, high_resolution_clock::now() }; + _sentACKs[_currentACKSubSequenceNumber] = { nextACKNumber, currentTime }; // reset the number of data packets received since last ACK _packetsSinceACK = 0; @@ -218,7 +218,7 @@ void Connection::sendTimeoutNAK() { SequenceNumber Connection::nextACK() const { if (_lossList.getLength() > 0) { - return _lossList.getFirstSequenceNumber(); + return _lossList.getFirstSequenceNumber() - 1; } else { return _lastReceivedSequenceNumber; } From 4907418587cedb26ce30942f316d4bc231642328 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 15:53:44 -0700 Subject: [PATCH 107/242] update the flow window size for the send queue --- libraries/networking/src/udt/Connection.cpp | 14 +++- libraries/networking/src/udt/SendQueue.cpp | 84 +++++++++++---------- libraries/networking/src/udt/SendQueue.h | 9 ++- 3 files changed, 63 insertions(+), 44 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 39e4254895..6e4080f4f2 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -368,10 +368,15 @@ void Connection::processACK(std::unique_ptr controlPacket) { _congestionControl->setBandwidth(_bandwidth); } - // fire the onACK callback for congestion control + // update the last sent sequence number in congestion control _congestionControl->setSendCurrentSequenceNumber(_sendQueue->getCurrentSequenceNumber()); + + // fire the onACK callback for congestion control _congestionControl->onAck(ack); + + // now that we've updated the congestion control, update the packet send period and flow window size _sendQueue->setPacketSendPeriod(_congestionControl->_packetSendPeriod); + _sendQueue->setFlowWindowSize(std::min(_flowWindowSize, (int) _congestionControl->_congestionWindowSize)); // update the total count of received ACKs ++_totalReceivedACKs; @@ -437,10 +442,15 @@ void Connection::processNAK(std::unique_ptr controlPacket) { // send that off to the send queue so it knows there was loss _sendQueue->nak(start, end); - // Tell the congestion control object that there was loss + // update the last sent sequence number in congestion control _congestionControl->setSendCurrentSequenceNumber(_sendQueue->getCurrentSequenceNumber()); + + // give the loss to the congestion control object _congestionControl->onLoss(start, end); + + // now that we've updated the congestion control, update the packet send period and flow window size _sendQueue->setPacketSendPeriod(_congestionControl->_packetSendPeriod); + _sendQueue->setFlowWindowSize(std::min(_flowWindowSize, (int) _congestionControl->_congestionWindowSize)); ++_totalReceivedNAKs; } diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index ce1ce41d13..df8260b4f5 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -80,13 +80,13 @@ void SendQueue::sendPacket(const BasePacket& packet) { } void SendQueue::ack(SequenceNumber ack) { - if (_lastAck == ack) { + if (_lastACKSequenceNumber == (uint32_t) ack) { return; } { // remove any ACKed packets from the map of sent packets QWriteLocker locker(&_sentLock); - for (auto seq = _lastAck; seq <= ack; ++seq) { + for (auto seq = SequenceNumber { (uint32_t) _lastACKSequenceNumber }; seq <= ack; ++seq) { _sentPackets.erase(seq); } } @@ -96,7 +96,7 @@ void SendQueue::ack(SequenceNumber ack) { _naks.remove(_naks.getFirstSequenceNumber(), ack); } - _lastAck = ack; + _lastACKSequenceNumber = (uint32_t) ack; } void SendQueue::nak(SequenceNumber start, SequenceNumber end) { @@ -131,48 +131,52 @@ void SendQueue::loop() { // Record timing _lastSendTimestamp = high_resolution_clock::now(); - if (_nextPacket) { + // we're only allowed to send if the flow window size + // is greater than or equal to the gap between the last ACKed sent and the one we are about to send + if (seqlen(SequenceNumber { (uint32_t) _lastACKSequenceNumber }, _currentSequenceNumber + 1) <= _flowWindowSize) { + bool hasResend = false; + SequenceNumber seqNum; + { + // Check nak list for packet to resend + QWriteLocker locker(&_naksLock); + if (_naks.getLength() > 0) { + hasResend = true; + seqNum = _naks.popFirstSequenceNumber(); + } + } + + std::unique_ptr nextPacket; + + // Find packet in sent list using SequenceNumber + if (hasResend) { + QWriteLocker locker(&_sentLock); + auto it = _sentPackets.find(seqNum); + Q_ASSERT_X(it != _sentPackets.end(), + "SendQueue::sendNextPacket()", "Couldn't find NAKed packet to resend"); + + if (it != _sentPackets.end()) { + it->second.swap(nextPacket); + _sentPackets.erase(it); + } + } + + // If there is no packet to resend, grab the next one in the list + if (!nextPacket) { + QWriteLocker locker(&_packetsLock); + nextPacket.swap(_packets.front()); + _packets.pop_front(); + } + // Write packet's sequence number and send it off - _nextPacket->writeSequenceNumber(getNextSequenceNumber()); - sendPacket(*_nextPacket); + nextPacket->writeSequenceNumber(getNextSequenceNumber()); + sendPacket(*nextPacket); // Insert the packet we have just sent in the sent list QWriteLocker locker(&_sentLock); - _sentPackets[_nextPacket->getSequenceNumber()].swap(_nextPacket); - Q_ASSERT_X(!_nextPacket, + _sentPackets[nextPacket->getSequenceNumber()].swap(nextPacket); + Q_ASSERT_X(!nextPacket, "SendQueue::sendNextPacket()", "Overriden packet in sent list"); - } - - bool hasResend = false; - SequenceNumber seqNum; - { - // Check nak list for packet to resend - QWriteLocker locker(&_naksLock); - if (_naks.getLength() > 0) { - hasResend = true; - seqNum = _naks.popFirstSequenceNumber(); - } - } - - // Find packet in sent list using SequenceNumber - if (hasResend) { - QWriteLocker locker(&_sentLock); - auto it = _sentPackets.find(seqNum); - Q_ASSERT_X(it != _sentPackets.end(), - "SendQueue::sendNextPacket()", "Couldn't find NAKed packet to resend"); - - if (it != _sentPackets.end()) { - it->second.swap(_nextPacket); - _sentPackets.erase(it); - } - } - - // If there is no packet to resend, grab the next one in the list - if (!_nextPacket) { - QWriteLocker locker(&_packetsLock); - _nextPacket.swap(_packets.front()); - _packets.pop_front(); - } + } // since we're a while loop, give the thread a chance to process events QCoreApplication::processEvents(); diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 4437565ded..0b3919a25b 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -22,6 +22,7 @@ #include "../HifiSockAddr.h" +#include "Constants.h" #include "SequenceNumber.h" #include "LossList.h" @@ -45,6 +46,8 @@ public: SequenceNumber getCurrentSequenceNumber() const { return SequenceNumber(_atomicCurrentSequenceNumber); } + void setFlowWindowSize(int flowWindowSize) { _flowWindowSize = flowWindowSize; } + int getPacketSendPeriod() const { return _packetSendPeriod; } void setPacketSendPeriod(int newPeriod) { _packetSendPeriod = newPeriod; } @@ -74,11 +77,11 @@ private: mutable QReadWriteLock _packetsLock; // Protects the packets to be sent list. std::list> _packets; // List of packets to be sent - std::unique_ptr _nextPacket; // Next packet to be sent Socket* _socket { nullptr }; // Socket to send packet on HifiSockAddr _destination; // Destination addr - SequenceNumber _lastAck; // Last ACKed sequence number + + std::atomic _lastACKSequenceNumber; // Last ACKed sequence number SequenceNumber _currentSequenceNumber; // Last sequence number sent out std::atomic _atomicCurrentSequenceNumber; // Atomic for last sequence number sent out @@ -87,6 +90,8 @@ private: std::chrono::high_resolution_clock::time_point _lastSendTimestamp; // Record last time of packet departure std::atomic _isRunning { false }; + std::atomic _flowWindowSize { udt::MAX_PACKETS_IN_FLIGHT }; // Flow control window size (number of packets that can be on wire) + mutable QReadWriteLock _naksLock; // Protects the naks list. LossList _naks; // Sequence numbers of packets to resend From f7f7ff385fbc551a931a1a713779be389f704b4e Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 29 Jul 2015 16:04:13 -0700 Subject: [PATCH 108/242] Code cleanup --- libraries/networking/src/udt/Connection.cpp | 27 ++++++++++----------- libraries/networking/src/udt/Connection.h | 3 --- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index de675bfca8..07c3f1a143 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -334,6 +334,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { // validate that this isn't a BS ACK if (ack > _sendQueue->getCurrentSequenceNumber()) { // in UDT they specifically break the connection here - do we want to do anything? + Q_ASSERT_X(true, "Connection::processACK", "ACK recieved higher than largest sent sequence number"); return; } @@ -341,26 +342,24 @@ void Connection::processACK(std::unique_ptr controlPacket) { int32_t rtt; controlPacket->readPrimitive(&rtt); - // read the desired flow window size - int flowWindowSize; - controlPacket->readPrimitive(&flowWindowSize); - - if (ack <= _lastReceivedACK) { - // this is a valid ACKed sequence number - update the flow window size and the last received ACK - _flowWindowSize = flowWindowSize; - _lastReceivedACK = ack; - } - - // make sure this isn't a repeated ACK - if (ack <= SequenceNumber(_atomicLastReceivedACK)) { + if (ack < _lastReceivedACK) { + // Bail return; } + // this is a valid ACKed sequence number - update the flow window size and the last received ACK + controlPacket->readPrimitive(&_flowWindowSize); + + if (ack == _lastReceivedACK) { + // Bail + return; + } + + _lastReceivedACK = ack; + // ACK the send queue so it knows what was received _sendQueue->ack(ack); - // update the atomic for last received ACK, the send queue uses this to re-transmit - _atomicLastReceivedACK = (SequenceNumber::Type)_lastReceivedACK; // update the RTT updateRTT(rtt); diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 2af89000e6..1ce81734b2 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -44,8 +44,6 @@ public: SequenceNumber nextACK() const; - SequenceNumber getLastReceivedACK() const { return SequenceNumber(_atomicLastReceivedACK); } - void processReceivedSequenceNumber(SequenceNumber seq); void processControl(std::unique_ptr controlPacket); @@ -75,7 +73,6 @@ private: LossList _lossList; // List of all missing packets SequenceNumber _lastReceivedSequenceNumber { SequenceNumber::MAX }; // The largest sequence number received from the peer SequenceNumber _lastReceivedACK { SequenceNumber::MAX }; // The last ACK received - std::atomic _atomicLastReceivedACK { (uint32_t) SequenceNumber::MAX }; // Atomic for thread-safe get of last ACK received SequenceNumber _lastReceivedAcknowledgedACK { SequenceNumber::MAX }; // The last sent ACK that has been acknowledged via an ACK2 from the peer SequenceNumber _currentACKSubSequenceNumber; // The current ACK sub-sequence number (used for Acknowledgment of ACKs) From 5fc3e7dbf1b19a3ef518d564b29a30b411807d0c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 16:12:00 -0700 Subject: [PATCH 109/242] add an updater that takes a lambda for CC update --- libraries/networking/src/udt/Connection.cpp | 38 +++++++++++---------- libraries/networking/src/udt/Connection.h | 2 ++ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 935622f4d8..626833c5c5 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -383,15 +383,10 @@ void Connection::processACK(std::unique_ptr controlPacket) { _congestionControl->setBandwidth(_bandwidth); } - // update the last sent sequence number in congestion control - _congestionControl->setSendCurrentSequenceNumber(_sendQueue->getCurrentSequenceNumber()); - - // fire the onACK callback for congestion control - _congestionControl->onAck(ack); - - // now that we've updated the congestion control, update the packet send period and flow window size - _sendQueue->setPacketSendPeriod(_congestionControl->_packetSendPeriod); - _sendQueue->setFlowWindowSize(std::min(_flowWindowSize, (int) _congestionControl->_congestionWindowSize)); + // give this ACK to the congestion control and update the send queue parameters + updateCongestionControlAndSentQueue([this, ack](){ + _congestionControl->onAck(ack); + }); // update the total count of received ACKs _stats.recordReceivedACK(); @@ -457,15 +452,10 @@ void Connection::processNAK(std::unique_ptr controlPacket) { // send that off to the send queue so it knows there was loss _sendQueue->nak(start, end); - // update the last sent sequence number in congestion control - _congestionControl->setSendCurrentSequenceNumber(_sendQueue->getCurrentSequenceNumber()); - - // give the loss to the congestion control object - _congestionControl->onLoss(start, end); - - // now that we've updated the congestion control, update the packet send period and flow window size - _sendQueue->setPacketSendPeriod(_congestionControl->_packetSendPeriod); - _sendQueue->setFlowWindowSize(std::min(_flowWindowSize, (int) _congestionControl->_congestionWindowSize)); + // give the loss to the congestion control object and update the send queue parameters + updateCongestionControlAndSentQueue([this, start, end](){ + _congestionControl->onLoss(start, end); + }); _stats.recordReceivedNAK(); } @@ -503,3 +493,15 @@ void Connection::updateRTT(int rtt) { int Connection::estimatedTimeout() const { return _congestionControl->_userDefinedRto ? _rtt + _rttVariance * 4 : _congestionControl->_rto; } + +void Connection::updateCongestionControlAndSentQueue(std::function congestionCallback) { + // update the last sent sequence number in congestion control + _congestionControl->setSendCurrentSequenceNumber(_sendQueue->getCurrentSequenceNumber()); + + // fire congestion control callback + congestionCallback(); + + // now that we've update the congestion control, update the packet send period and flow window size + _sendQueue->setPacketSendPeriod(_congestionControl->_packetSendPeriod); + _sendQueue->setFlowWindowSize(std::min(_flowWindowSize, (int) _congestionControl->_congestionWindowSize)); +} diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 1ce81734b2..b46d2c37cf 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -64,6 +64,8 @@ private: int estimatedTimeout() const; + void updateCongestionControlAndSentQueue(std::function congestionCallback); + int _synInterval; // Periodical Rate Control Interval, in microseconds int _nakInterval; // NAK timeout interval, in microseconds From 35f00f9ba1388f362d2c21e4bfe2e7407cfdc23a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 16:12:50 -0700 Subject: [PATCH 110/242] rename the CC and send queue updater --- libraries/networking/src/udt/Connection.cpp | 6 +++--- libraries/networking/src/udt/Connection.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 626833c5c5..ec175d8f39 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -384,7 +384,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { } // give this ACK to the congestion control and update the send queue parameters - updateCongestionControlAndSentQueue([this, ack](){ + updateCongestionControlAndSendQueue([this, ack](){ _congestionControl->onAck(ack); }); @@ -453,7 +453,7 @@ void Connection::processNAK(std::unique_ptr controlPacket) { _sendQueue->nak(start, end); // give the loss to the congestion control object and update the send queue parameters - updateCongestionControlAndSentQueue([this, start, end](){ + updateCongestionControlAndSendQueue([this, start, end](){ _congestionControl->onLoss(start, end); }); @@ -494,7 +494,7 @@ int Connection::estimatedTimeout() const { return _congestionControl->_userDefinedRto ? _rtt + _rttVariance * 4 : _congestionControl->_rto; } -void Connection::updateCongestionControlAndSentQueue(std::function congestionCallback) { +void Connection::updateCongestionControlAndSendQueue(std::function congestionCallback) { // update the last sent sequence number in congestion control _congestionControl->setSendCurrentSequenceNumber(_sendQueue->getCurrentSequenceNumber()); diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index b46d2c37cf..eaa55797d9 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -64,7 +64,7 @@ private: int estimatedTimeout() const; - void updateCongestionControlAndSentQueue(std::function congestionCallback); + void updateCongestionControlAndSendQueue(std::function congestionCallback); int _synInterval; // Periodical Rate Control Interval, in microseconds From ce1fc4baa9f7863020d6c5f5c3103d78bd9c41f5 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 29 Jul 2015 16:24:58 -0700 Subject: [PATCH 111/242] Send light acks based on CC member --- libraries/networking/src/udt/CongestionControl.h | 1 + libraries/networking/src/udt/Connection.cpp | 2 ++ 2 files changed, 3 insertions(+) diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 75590d88b5..f8f4860a89 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -68,6 +68,7 @@ private: CongestionControl& operator=(const CongestionControl& other) = delete; int _ackInterval { 0 }; // How many packets to send one ACK, in packets + int _lightACKInterval { 64 }; // How many packets to send one light ACK, in packets int _synInterval { DEFAULT_SYN_INTERVAL }; diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 935622f4d8..2051ca9627 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -282,6 +282,8 @@ void Connection::processReceivedSequenceNumber(SequenceNumber seq) { // check if we need to send an ACK, according to CC params if (_congestionControl->_ackInterval > 0 && _packetsSinceACK >= _congestionControl->_ackInterval) { sendACK(false); + } else if (_congestionControl->_lightACKInterval > 0 && _packetsSinceACK >= _congestionControl->_lightACKInterval) { + sendLightACK(); } } From 116ff17fcfcfc8482e80388c306dd1dc9e018248 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 29 Jul 2015 16:38:37 -0700 Subject: [PATCH 112/242] SendQueue auto thread cleanup --- libraries/networking/src/udt/Connection.cpp | 7 +------ libraries/networking/src/udt/Connection.h | 3 +-- libraries/networking/src/udt/SendQueue.cpp | 2 +- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 9170017297..17c11100eb 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -38,14 +38,9 @@ Connection::Connection(Socket* parentSocket, HifiSockAddr destination, unique_pt Connection::~Connection() { if (_sendQueue) { - // tell our send queue to stop and wait until its send thread is done - QThread* sendQueueThread = _sendQueue->thread(); - _sendQueue->stop(); _sendQueue->deleteLater(); - - sendQueueThread->quit(); - sendQueueThread->wait(); + _sendQueue.release(); } } diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index eaa55797d9..e68efd2887 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -42,8 +42,6 @@ public: void sync(); // rate control method, fired by Socket for all connections on SYN interval - SequenceNumber nextACK() const; - void processReceivedSequenceNumber(SequenceNumber seq); void processControl(std::unique_ptr controlPacket); @@ -60,6 +58,7 @@ private: void processNAK(std::unique_ptr controlPacket); void processTimeoutNAK(std::unique_ptr controlPacket); + SequenceNumber nextACK() const; void updateRTT(int rtt); int estimatedTimeout() const; diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index df8260b4f5..05d69dde6c 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -29,7 +29,7 @@ std::unique_ptr SendQueue::create(Socket* socket, HifiSockAddr dest) auto queue = std::unique_ptr(new SendQueue(socket, dest)); // Setup queue private thread - QThread* thread = new QThread(queue.get()); + QThread* thread = new QThread(); thread->setObjectName("Networking: SendQueue"); // Name thread for easier debug connect(queue.get(), &QObject::destroyed, thread, &QThread::quit); // Thread auto cleanup connect(thread, &QThread::finished, thread, &QThread::deleteLater); // Thread auto cleanup From 66c308b43682a1b8df364148c12bcc5c9a758a9c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 16:48:14 -0700 Subject: [PATCH 113/242] actually process control and sequence numbers from Socket --- libraries/networking/src/udt/BasePacket.h | 3 +- libraries/networking/src/udt/Connection.cpp | 26 +++++++----- libraries/networking/src/udt/Connection.h | 2 +- libraries/networking/src/udt/Constants.h | 3 ++ .../networking/src/udt/ControlPacket.cpp | 39 +++++++++++++++++- libraries/networking/src/udt/ControlPacket.h | 6 +++ libraries/networking/src/udt/LossList.cpp | 8 +++- libraries/networking/src/udt/LossList.h | 2 +- libraries/networking/src/udt/Packet.cpp | 1 - libraries/networking/src/udt/Socket.cpp | 41 +++++++++++++++---- 10 files changed, 106 insertions(+), 25 deletions(-) diff --git a/libraries/networking/src/udt/BasePacket.h b/libraries/networking/src/udt/BasePacket.h index 8dcc3ab9be..7ebf2d1e65 100644 --- a/libraries/networking/src/udt/BasePacket.h +++ b/libraries/networking/src/udt/BasePacket.h @@ -27,7 +27,8 @@ public: static const qint64 PACKET_WRITE_ERROR; static std::unique_ptr create(qint64 size = -1); - static std::unique_ptr fromReceivedPacket(std::unique_ptr data, qint64 size, const HifiSockAddr& senderSockAddr); + static std::unique_ptr fromReceivedPacket(std::unique_ptr data, qint64 size, + const HifiSockAddr& senderSockAddr); static qint64 maxPayloadSize() { return MAX_PACKET_SIZE; } // The maximum payload size this packet can use to fit in MTU static qint64 localHeaderSize() { return 0; } // Current level's header size diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index ec175d8f39..9867584e96 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -230,27 +230,27 @@ SequenceNumber Connection::nextACK() const { } } -void Connection::processReceivedSequenceNumber(SequenceNumber seq) { +bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber) { // check if this is a packet pair we should estimate bandwidth from, or just a regular packet - if (((uint32_t) seq & 0xF) == 0) { + if (((uint32_t) sequenceNumber & 0xF) == 0) { _receiveWindow.onProbePair1Arrival(); - } else if (((uint32_t) seq & 0xF) == 1) { + } else if (((uint32_t) sequenceNumber & 0xF) == 1) { _receiveWindow.onProbePair2Arrival(); } else { _receiveWindow.onPacketArrival(); } // If this is not the next sequence number, report loss - if (seq > _lastReceivedSequenceNumber + 1) { - if (_lastReceivedSequenceNumber + 1 == seq - 1) { + if (sequenceNumber > _lastReceivedSequenceNumber + 1) { + if (_lastReceivedSequenceNumber + 1 == sequenceNumber - 1) { _lossList.append(_lastReceivedSequenceNumber + 1); } else { - _lossList.append(_lastReceivedSequenceNumber + 1, seq - 1); + _lossList.append(_lastReceivedSequenceNumber + 1, sequenceNumber - 1); } // Send a NAK packet - sendNAK(seq); + sendNAK(sequenceNumber); // figure out when we should send the next loss report, if we haven't heard anything back _nakInterval = (_rtt + 4 * _rttVariance); @@ -267,12 +267,14 @@ void Connection::processReceivedSequenceNumber(SequenceNumber seq) { } } - if (seq > _lastReceivedSequenceNumber) { + bool wasDuplicate = false; + + if (sequenceNumber > _lastReceivedSequenceNumber) { // Update largest recieved sequence number - _lastReceivedSequenceNumber = seq; + _lastReceivedSequenceNumber = sequenceNumber; } else { - // Otherwise, it's a resend, remove it from the loss list - _lossList.remove(seq); + // Otherwise, it could be a resend, try and remove it from the loss list + wasDuplicate = !_lossList.remove(sequenceNumber); } // increment the counters for data packets received @@ -283,6 +285,8 @@ void Connection::processReceivedSequenceNumber(SequenceNumber seq) { if (_congestionControl->_ackInterval > 0 && _packetsSinceACK >= _congestionControl->_ackInterval) { sendACK(false); } + + return wasDuplicate; } void Connection::processControl(unique_ptr controlPacket) { diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index eaa55797d9..2e6d665e4f 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -44,7 +44,7 @@ public: SequenceNumber nextACK() const; - void processReceivedSequenceNumber(SequenceNumber seq); + bool processReceivedSequenceNumber(SequenceNumber sequenceNumber); // returns indicates if this packet was a duplicate void processControl(std::unique_ptr controlPacket); private: diff --git a/libraries/networking/src/udt/Constants.h b/libraries/networking/src/udt/Constants.h index e8a7658bf9..5e0f130f13 100644 --- a/libraries/networking/src/udt/Constants.h +++ b/libraries/networking/src/udt/Constants.h @@ -14,6 +14,8 @@ #ifndef hifi_udt_Constants_h #define hifi_udt_Constants_h +#include "SequenceNumber.h" + namespace udt { static const int MAX_PACKET_SIZE_WITH_UDP_HEADER = 1500; static const int MAX_PACKET_SIZE = MAX_PACKET_SIZE_WITH_UDP_HEADER - 28; @@ -23,6 +25,7 @@ namespace udt { static const int UDP_SEND_BUFFER_SIZE_BYTES = 1048576; static const int UDP_RECEIVE_BUFFER_SIZE_BYTES = 1048576; static const int DEFAULT_SYN_INTERVAL_USECS = 10 * 1000; + static const uint32_t CONTROL_BIT_MASK = 1 << (sizeof(SequenceNumber) - 1); } #endif // hifi_udt_Constants_h diff --git a/libraries/networking/src/udt/ControlPacket.cpp b/libraries/networking/src/udt/ControlPacket.cpp index 057f3633f9..2bc081e197 100644 --- a/libraries/networking/src/udt/ControlPacket.cpp +++ b/libraries/networking/src/udt/ControlPacket.cpp @@ -11,8 +11,26 @@ #include "ControlPacket.h" +#include "Constants.h" + using namespace udt; +std::unique_ptr ControlPacket::fromReceivedPacket(std::unique_ptr data, qint64 size, + const HifiSockAddr &senderSockAddr) { + // Fail with null data + Q_ASSERT(data); + + // Fail with invalid size + Q_ASSERT(size >= 0); + + // allocate memory + auto packet = std::unique_ptr(new ControlPacket(std::move(data), size, senderSockAddr)); + + packet->open(QIODevice::ReadOnly); + + return packet; +} + std::unique_ptr ControlPacket::create(Type type, qint64 size) { std::unique_ptr controlPacket; @@ -57,6 +75,17 @@ ControlPacket::ControlPacket(Type type, qint64 size) : writeControlBitAndType(); } +ControlPacket::ControlPacket(std::unique_ptr data, qint64 size, const HifiSockAddr& senderSockAddr) : + BasePacket(std::move(data), size, senderSockAddr) +{ + // sanity check before we decrease the payloadSize with the payloadCapacity + Q_ASSERT(_payloadSize == _payloadCapacity); + + adjustPayloadStartAndCapacity(_payloadSize > 0); + + readType(); +} + ControlPacket::ControlPacket(ControlPacket&& other) : BasePacket(std::move(other)) { @@ -71,8 +100,6 @@ ControlPacket& ControlPacket::operator=(ControlPacket&& other) { return *this; } -static const uint32_t CONTROL_BIT_MASK = 1 << (sizeof(ControlPacket::ControlBitAndType) - 1); - void ControlPacket::setType(udt::ControlPacket::Type type) { _type = type; @@ -94,3 +121,11 @@ void ControlPacket::writeType() { // write the type by OR'ing the new type with the current value & CONTROL_BIT_MASK *bitAndType = (*bitAndType & CONTROL_BIT_MASK) | (_type << sizeof((ControlPacket::Type) - 1)); } + +void ControlPacket::readType() { + ControlBitAndType bitAndType = *reinterpret_cast(_packet.get()); + + // read the type + uint32_t oversizeType = (uint32_t) (bitAndType & ~CONTROL_BIT_MASK); + _type = (Type) oversizeType; +} diff --git a/libraries/networking/src/udt/ControlPacket.h b/libraries/networking/src/udt/ControlPacket.h index fdd0b65e36..6d13c06d33 100644 --- a/libraries/networking/src/udt/ControlPacket.h +++ b/libraries/networking/src/udt/ControlPacket.h @@ -33,6 +33,8 @@ public: TimeoutNAK }; + static std::unique_ptr fromReceivedPacket(std::unique_ptr data, qint64 size, + const HifiSockAddr& senderSockAddr); static std::unique_ptr create(Type type, qint64 size = -1); static qint64 localHeaderSize(); // Current level's header size @@ -44,6 +46,7 @@ public: private: ControlPacket(Type type); ControlPacket(Type type, qint64 size); + ControlPacket(std::unique_ptr data, qint64 size, const HifiSockAddr& senderSockAddr); ControlPacket(ControlPacket&& other); ControlPacket(const ControlPacket& other) = delete; @@ -54,6 +57,9 @@ private: void writeControlBitAndType(); void writeType(); + // Header readers + void readType(); + Type _type; }; diff --git a/libraries/networking/src/udt/LossList.cpp b/libraries/networking/src/udt/LossList.cpp index 0bf08fbc1d..32812be10b 100644 --- a/libraries/networking/src/udt/LossList.cpp +++ b/libraries/networking/src/udt/LossList.cpp @@ -81,7 +81,7 @@ void LossList::insert(SequenceNumber start, SequenceNumber end) { } } -void LossList::remove(SequenceNumber seq) { +bool LossList::remove(SequenceNumber seq) { auto it = find_if(_lossList.begin(), _lossList.end(), [&seq](pair pair) { return pair.first <= seq && seq <= pair.second; }); @@ -99,6 +99,12 @@ void LossList::remove(SequenceNumber seq) { _lossList.insert(it, make_pair(seq + 1, temp)); } _length -= 1; + + // this sequence number was found in the loss list, return true + return true; + } else { + // this sequence number was not found in the loss list, return false + return false; } } diff --git a/libraries/networking/src/udt/LossList.h b/libraries/networking/src/udt/LossList.h index 5b5fb9f0c8..fc97206282 100644 --- a/libraries/networking/src/udt/LossList.h +++ b/libraries/networking/src/udt/LossList.h @@ -33,7 +33,7 @@ public: // Inserts anywhere - MUCH slower void insert(SequenceNumber start, SequenceNumber end); - void remove(SequenceNumber seq); + bool remove(SequenceNumber seq); void remove(SequenceNumber start, SequenceNumber end); int getLength() const { return _length; } diff --git a/libraries/networking/src/udt/Packet.cpp b/libraries/networking/src/udt/Packet.cpp index 4223986aee..96cf5210b4 100644 --- a/libraries/networking/src/udt/Packet.cpp +++ b/libraries/networking/src/udt/Packet.cpp @@ -117,7 +117,6 @@ void Packet::writeSequenceNumber(SequenceNumber sequenceNumber) const { writeHeader(); } -static const uint32_t CONTROL_BIT_MASK = 1 << (sizeof(Packet::SequenceNumberAndBitField) - 1); static const uint32_t RELIABILITY_BIT_MASK = 1 << (sizeof(Packet::SequenceNumberAndBitField) - 2); static const uint32_t MESSAGE_BIT_MASK = 1 << (sizeof(Packet::SequenceNumberAndBitField) - 3); static const uint32_t BIT_FIELD_MASK = CONTROL_BIT_MASK | RELIABILITY_BIT_MASK | MESSAGE_BIT_MASK; diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 650bdd7906..5ded97135b 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -135,14 +135,41 @@ void Socket::readPendingDatagrams() { return; } - // setup a Packet from the data we just read - auto packet = Packet::fromReceivedPacket(std::move(buffer), packetSizeWithHeader, senderSockAddr); + // check if this was a control packet or a data packet + bool isControlPacket = *buffer & CONTROL_BIT_MASK; - // call our verification operator to see if this packet is verified - if (!_packetFilterOperator || _packetFilterOperator(*packet)) { - if (_packetHandler) { - // call the verified packet callback to let it handle this packet - _packetHandler(std::move(packet)); + if (isControlPacket) { + // setup a control packet from the data we just read + auto controlPacket = ControlPacket::fromReceivedPacket(std::move(buffer), packetSizeWithHeader, senderSockAddr); + + // move this control packet to the matching connection + auto it = _connectionsHash.find(senderSockAddr); + + if (it != _connectionsHash.end()) { + it->second->processControl(move(controlPacket)); + } + + } else { + // setup a Packet from the data we just read + auto packet = Packet::fromReceivedPacket(std::move(buffer), packetSizeWithHeader, senderSockAddr); + + // call our verification operator to see if this packet is verified + if (!_packetFilterOperator || _packetFilterOperator(*packet)) { + + if (packet->isReliable()) { + // if this was a reliable packet then signal the matching connection with the sequence number + // assuming it exists + auto it = _connectionsHash.find(senderSockAddr); + + if (it != _connectionsHash.end()) { + it->second->processReceivedSequenceNumber(packet->getSequenceNumber()); + } + } + + if (_packetHandler) { + // call the verified packet callback to let it handle this packet + _packetHandler(std::move(packet)); + } } } } From b8085086c65238a72f392366b1d2ffeeef04c828 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 29 Jul 2015 17:28:37 -0700 Subject: [PATCH 114/242] HifiSockAddr have object names --- assignment-client/src/AssignmentClient.cpp | 2 ++ libraries/networking/src/DomainHandler.cpp | 3 +++ libraries/networking/src/HifiSockAddr.cpp | 15 ++++++++++----- libraries/networking/src/NetworkPeer.cpp | 6 ++++++ libraries/networking/src/Node.cpp | 12 +++++++++++- libraries/networking/src/Node.h | 2 +- libraries/networking/src/udt/SendQueue.cpp | 2 +- libraries/networking/src/udt/Socket.cpp | 1 + libraries/networking/src/udt/Socket.h | 2 +- 9 files changed, 36 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/AssignmentClient.cpp b/assignment-client/src/AssignmentClient.cpp index 40aef1c707..6b7365fc01 100644 --- a/assignment-client/src/AssignmentClient.cpp +++ b/assignment-client/src/AssignmentClient.cpp @@ -95,6 +95,7 @@ AssignmentClient::AssignmentClient(Assignment::Type requestAssignmentType, QStri } _assignmentServerSocket = HifiSockAddr(_assignmentServerHostname, assignmentServerPort, true); + _assignmentServerSocket.setObjectName("AssigmentServer"); nodeList->setAssignmentServerSocket(_assignmentServerSocket); qDebug() << "Assignment server socket is" << _assignmentServerSocket; @@ -119,6 +120,7 @@ AssignmentClient::AssignmentClient(Assignment::Type requestAssignmentType, QStri // did we get an assignment-client monitor port? if (assignmentMonitorPort > 0) { _assignmentClientMonitorSocket = HifiSockAddr(DEFAULT_ASSIGNMENT_CLIENT_MONITOR_HOSTNAME, assignmentMonitorPort); + _assignmentClientMonitorSocket.setObjectName("AssignmentClientMonitor"); qDebug() << "Assignment-client monitor socket is" << _assignmentClientMonitorSocket; diff --git a/libraries/networking/src/DomainHandler.cpp b/libraries/networking/src/DomainHandler.cpp index afb362053e..d38ae35826 100644 --- a/libraries/networking/src/DomainHandler.cpp +++ b/libraries/networking/src/DomainHandler.cpp @@ -37,6 +37,8 @@ DomainHandler::DomainHandler(QObject* parent) : _settingsObject(), _failedSettingsRequests(0) { + _sockAddr.setObjectName("DomainServer"); + // if we get a socket that make sure our NetworkPeer ping timer stops connect(this, &DomainHandler::completedSocketDiscovery, &_icePeer, &NetworkPeer::stopPingTimer); } @@ -145,6 +147,7 @@ void DomainHandler::setIceServerHostnameAndID(const QString& iceServerHostname, HifiSockAddr* replaceableSockAddr = &_iceServerSockAddr; replaceableSockAddr->~HifiSockAddr(); replaceableSockAddr = new (replaceableSockAddr) HifiSockAddr(iceServerHostname, ICE_SERVER_DEFAULT_PORT); + _iceServerSockAddr.setObjectName("IceServer"); auto nodeList = DependencyManager::get(); diff --git a/libraries/networking/src/HifiSockAddr.cpp b/libraries/networking/src/HifiSockAddr.cpp index 39210db81e..813d19d22c 100644 --- a/libraries/networking/src/HifiSockAddr.cpp +++ b/libraries/networking/src/HifiSockAddr.cpp @@ -33,16 +33,16 @@ HifiSockAddr::HifiSockAddr(const QHostAddress& address, quint16 port) : } HifiSockAddr::HifiSockAddr(const HifiSockAddr& otherSockAddr) : - QObject(), _address(otherSockAddr._address), _port(otherSockAddr._port) { - + setObjectName(otherSockAddr.objectName()); } HifiSockAddr& HifiSockAddr::operator=(const HifiSockAddr& rhsSockAddr) { - HifiSockAddr temp(rhsSockAddr); - swap(temp); + setObjectName(rhsSockAddr.objectName()); + _address = rhsSockAddr._address; + _port = rhsSockAddr._port; return *this; } @@ -76,9 +76,14 @@ HifiSockAddr::HifiSockAddr(const sockaddr* sockaddr) { void HifiSockAddr::swap(HifiSockAddr& otherSockAddr) { using std::swap; - + swap(_address, otherSockAddr._address); swap(_port, otherSockAddr._port); + + // Swap objects name + auto temp = otherSockAddr.objectName(); + otherSockAddr.setObjectName(objectName()); + setObjectName(temp); } bool HifiSockAddr::operator==(const HifiSockAddr& rhsSockAddr) const { diff --git a/libraries/networking/src/NetworkPeer.cpp b/libraries/networking/src/NetworkPeer.cpp index 52e53e9665..9e51bc5dac 100644 --- a/libraries/networking/src/NetworkPeer.cpp +++ b/libraries/networking/src/NetworkPeer.cpp @@ -57,7 +57,9 @@ void NetworkPeer::setPublicSocket(const HifiSockAddr& publicSocket) { bool wasOldSocketNull = _publicSocket.isNull(); + auto temp = _publicSocket.objectName(); _publicSocket = publicSocket; + _publicSocket.setObjectName(temp); if (!wasOldSocketNull) { qCDebug(networking) << "Public socket change for node" << *this; @@ -74,7 +76,9 @@ void NetworkPeer::setLocalSocket(const HifiSockAddr& localSocket) { bool wasOldSocketNull = _localSocket.isNull(); + auto temp = _localSocket.objectName(); _localSocket = localSocket; + _localSocket.setObjectName(temp); if (!wasOldSocketNull) { qCDebug(networking) << "Local socket change for node" << *this; @@ -91,7 +95,9 @@ void NetworkPeer::setSymmetricSocket(const HifiSockAddr& symmetricSocket) { bool wasOldSocketNull = _symmetricSocket.isNull(); + auto temp = _symmetricSocket.objectName(); _symmetricSocket = symmetricSocket; + _symmetricSocket.setObjectName(temp); if (!wasOldSocketNull) { qCDebug(networking) << "Symmetric socket change for node" << *this; diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index 355aa6994e..6f10e6dbdb 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -55,13 +55,23 @@ Node::Node(const QUuid& uuid, NodeType_t type, const HifiSockAddr& publicSocket, _canAdjustLocks(canAdjustLocks), _canRez(canRez) { - + // Update socket's object name + setType(_type); } Node::~Node() { delete _linkedData; } +void Node::setType(char type) { + _type = type; + + auto typeString = NodeType::getNodeTypeName(type); + _publicSocket.setObjectName(typeString); + _localSocket.setObjectName(typeString); + _symmetricSocket.setObjectName(typeString); +} + void Node::updateClockSkewUsec(int clockSkewSample) { _clockSkewMovingPercentile.updatePercentile((float)clockSkewSample); _clockSkewUsec = (int)_clockSkewMovingPercentile.getValueAtPercentile(); diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index 53a2aced8c..573569f92b 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -40,7 +40,7 @@ public: bool operator!=(const Node& otherNode) const { return !(*this == otherNode); } char getType() const { return _type; } - void setType(char type) { _type = type; } + void setType(char type); const QUuid& getConnectionSecret() const { return _connectionSecret; } void setConnectionSecret(const QUuid& connectionSecret) { _connectionSecret = connectionSecret; } diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 05d69dde6c..9a2e371616 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -30,7 +30,7 @@ std::unique_ptr SendQueue::create(Socket* socket, HifiSockAddr dest) // Setup queue private thread QThread* thread = new QThread(); - thread->setObjectName("Networking: SendQueue"); // Name thread for easier debug + thread->setObjectName("Networking: SendQueue " + dest.objectName()); // Name thread for easier debug connect(queue.get(), &QObject::destroyed, thread, &QThread::quit); // Thread auto cleanup connect(thread, &QThread::finished, thread, &QThread::deleteLater); // Thread auto cleanup diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 650bdd7906..4c8b5a4329 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -14,6 +14,7 @@ #include #include "../NetworkLogging.h" +#include "Connection.h" #include "ControlPacket.h" #include "Packet.h" diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 0bc23cdf0a..1827dc8234 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -23,11 +23,11 @@ #include "../HifiSockAddr.h" #include "CongestionControl.h" -#include "Connection.h" namespace udt { class BasePacket; +class Connection; class ControlSender; class Packet; class SequenceNumber; From 7c87ee3a725c93dfc6484583854fc9610c8d0aa7 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 17:37:33 -0700 Subject: [PATCH 115/242] type and version fixes after refactor --- libraries/networking/src/NLPacket.cpp | 12 ++++++------ libraries/networking/src/NodeList.cpp | 1 + libraries/networking/src/udt/BasePacket.cpp | 3 +-- libraries/networking/src/udt/BasePacket.h | 2 +- libraries/networking/src/udt/Constants.h | 3 ++- libraries/networking/src/udt/ControlPacket.cpp | 8 +++++--- libraries/networking/src/udt/Packet.cpp | 13 +++++++------ libraries/networking/src/udt/Socket.cpp | 3 +++ 8 files changed, 26 insertions(+), 19 deletions(-) diff --git a/libraries/networking/src/NLPacket.cpp b/libraries/networking/src/NLPacket.cpp index 5359d6cf2c..4568561fa2 100644 --- a/libraries/networking/src/NLPacket.cpp +++ b/libraries/networking/src/NLPacket.cpp @@ -26,7 +26,7 @@ qint64 NLPacket::maxPayloadSize() const { } qint64 NLPacket::totalHeadersSize() const { - return localHeaderSize() + Packet::localHeaderSize(); + return Packet::totalHeadersSize() + localHeaderSize(); } qint64 NLPacket::localHeaderSize() const { @@ -84,7 +84,7 @@ NLPacket::NLPacket(PacketType type, bool isReliable, bool isPartOfMessage) : _type(type), _version(versionForPacketType(type)) { - adjustPayloadStartAndCapacity(); + adjustPayloadStartAndCapacity(localHeaderSize()); writeTypeAndVersion(); } @@ -96,7 +96,7 @@ NLPacket::NLPacket(PacketType type, qint64 size, bool isReliable, bool isPartOfM { Q_ASSERT(size >= 0); - adjustPayloadStartAndCapacity(); + adjustPayloadStartAndCapacity(localHeaderSize()); writeTypeAndVersion(); } @@ -108,7 +108,7 @@ NLPacket::NLPacket(std::unique_ptr packet) : readVersion(); readSourceID(); - adjustPayloadStartAndCapacity(_payloadSize > 0); + adjustPayloadStartAndCapacity(localHeaderSize(), _payloadSize > 0); } NLPacket::NLPacket(const NLPacket& other) : Packet(other) { @@ -123,7 +123,7 @@ NLPacket::NLPacket(std::unique_ptr data, qint64 size, const HifiSockAddr& // sanity check before we decrease the payloadSize with the payloadCapacity Q_ASSERT(_payloadSize == _payloadCapacity); - adjustPayloadStartAndCapacity(_payloadSize > 0); + adjustPayloadStartAndCapacity(localHeaderSize(), _payloadSize > 0); readType(); readVersion(); @@ -194,7 +194,7 @@ QByteArray NLPacket::hashForPacketAndSecret(const udt::Packet& packet, const QUu } void NLPacket::writeTypeAndVersion() { - auto headerOffset = Packet::totalHeadersSize(); + auto headerOffset = Packet::localHeaderSize(); // Pack the packet type memcpy(_packet.get() + headerOffset, &_type, sizeof(PacketType)); diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 0864cb9fcd..d55489a84d 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -252,6 +252,7 @@ void NodeList::sendDomainServerCheckIn() { } auto domainPacket = NLPacket::create(domainPacketType); + QDataStream packetStream(domainPacket.get()); if (domainPacketType == PacketType::DomainConnectRequest) { diff --git a/libraries/networking/src/udt/BasePacket.cpp b/libraries/networking/src/udt/BasePacket.cpp index 083a62b155..38fb9e849e 100644 --- a/libraries/networking/src/udt/BasePacket.cpp +++ b/libraries/networking/src/udt/BasePacket.cpp @@ -177,8 +177,7 @@ qint64 BasePacket::readData(char* dest, qint64 maxSize) { return numBytesToRead; } -void BasePacket::adjustPayloadStartAndCapacity(bool shouldDecreasePayloadSize) { - qint64 headerSize = localHeaderSize(); +void BasePacket::adjustPayloadStartAndCapacity(qint64 headerSize, bool shouldDecreasePayloadSize) { _payloadStart += headerSize; _payloadCapacity -= headerSize; diff --git a/libraries/networking/src/udt/BasePacket.h b/libraries/networking/src/udt/BasePacket.h index 7ebf2d1e65..d8c1aaeddf 100644 --- a/libraries/networking/src/udt/BasePacket.h +++ b/libraries/networking/src/udt/BasePacket.h @@ -86,7 +86,7 @@ protected: virtual qint64 writeData(const char* data, qint64 maxSize); virtual qint64 readData(char* data, qint64 maxSize); - virtual void adjustPayloadStartAndCapacity(bool shouldDecreasePayloadSize = false); + virtual void adjustPayloadStartAndCapacity(qint64 headerSize, bool shouldDecreasePayloadSize = false); qint64 _packetSize = 0; // Total size of the allocated memory std::unique_ptr _packet; // Allocated memory diff --git a/libraries/networking/src/udt/Constants.h b/libraries/networking/src/udt/Constants.h index 5e0f130f13..eaad77d03e 100644 --- a/libraries/networking/src/udt/Constants.h +++ b/libraries/networking/src/udt/Constants.h @@ -25,7 +25,8 @@ namespace udt { static const int UDP_SEND_BUFFER_SIZE_BYTES = 1048576; static const int UDP_RECEIVE_BUFFER_SIZE_BYTES = 1048576; static const int DEFAULT_SYN_INTERVAL_USECS = 10 * 1000; - static const uint32_t CONTROL_BIT_MASK = 1 << (sizeof(SequenceNumber) - 1); + static const int SEQUENCE_NUMBER_BITS = sizeof(SequenceNumber) * 8; + static const uint32_t CONTROL_BIT_MASK = uint32_t(1) << (SEQUENCE_NUMBER_BITS - 1); } #endif // hifi_udt_Constants_h diff --git a/libraries/networking/src/udt/ControlPacket.cpp b/libraries/networking/src/udt/ControlPacket.cpp index 2bc081e197..01d92bfa07 100644 --- a/libraries/networking/src/udt/ControlPacket.cpp +++ b/libraries/networking/src/udt/ControlPacket.cpp @@ -57,7 +57,7 @@ ControlPacket::ControlPacket(Type type) : BasePacket(-1), _type(type) { - adjustPayloadStartAndCapacity(); + adjustPayloadStartAndCapacity(localHeaderSize()); open(QIODevice::ReadWrite); @@ -68,7 +68,7 @@ ControlPacket::ControlPacket(Type type, qint64 size) : BasePacket(localHeaderSize() + size), _type(type) { - adjustPayloadStartAndCapacity(); + adjustPayloadStartAndCapacity(localHeaderSize()); open(QIODevice::ReadWrite); @@ -119,12 +119,14 @@ void ControlPacket::writeType() { ControlBitAndType* bitAndType = reinterpret_cast(_packet.get()); // write the type by OR'ing the new type with the current value & CONTROL_BIT_MASK - *bitAndType = (*bitAndType & CONTROL_BIT_MASK) | (_type << sizeof((ControlPacket::Type) - 1)); + *bitAndType = (*bitAndType & CONTROL_BIT_MASK) | (_type << (sizeof(ControlPacket::Type) * 8 - 1)); } void ControlPacket::readType() { ControlBitAndType bitAndType = *reinterpret_cast(_packet.get()); + Q_ASSERT_X(bitAndType & CONTROL_BIT_MASK, "ControlPacket::readHeader()", "This should be a control packet"); + // read the type uint32_t oversizeType = (uint32_t) (bitAndType & ~CONTROL_BIT_MASK); _type = (Type) oversizeType; diff --git a/libraries/networking/src/udt/Packet.cpp b/libraries/networking/src/udt/Packet.cpp index 96cf5210b4..6ec8ac398e 100644 --- a/libraries/networking/src/udt/Packet.cpp +++ b/libraries/networking/src/udt/Packet.cpp @@ -50,7 +50,7 @@ qint64 Packet::maxPayloadSize() const { } qint64 Packet::totalHeadersSize() const { - return BasePacket::localHeaderSize() + localHeaderSize(); + return BasePacket::totalHeadersSize() + Packet::localHeaderSize(); } qint64 Packet::localHeaderSize() const { @@ -62,7 +62,7 @@ Packet::Packet(qint64 size, bool isReliable, bool isPartOfMessage) : _isReliable(isReliable), _isPartOfMessage(isPartOfMessage) { - adjustPayloadStartAndCapacity(); + adjustPayloadStartAndCapacity(localHeaderSize()); // set the UDT header to default values writeHeader(); @@ -117,14 +117,15 @@ void Packet::writeSequenceNumber(SequenceNumber sequenceNumber) const { writeHeader(); } -static const uint32_t RELIABILITY_BIT_MASK = 1 << (sizeof(Packet::SequenceNumberAndBitField) - 2); -static const uint32_t MESSAGE_BIT_MASK = 1 << (sizeof(Packet::SequenceNumberAndBitField) - 3); +static const uint32_t RELIABILITY_BIT_MASK = uint32_t(1) << (SEQUENCE_NUMBER_BITS - 2); +static const uint32_t MESSAGE_BIT_MASK = uint32_t(1) << (SEQUENCE_NUMBER_BITS - 3); static const uint32_t BIT_FIELD_MASK = CONTROL_BIT_MASK | RELIABILITY_BIT_MASK | MESSAGE_BIT_MASK; void Packet::readHeader() const { SequenceNumberAndBitField seqNumBitField = *reinterpret_cast(_packet.get()); - Q_ASSERT_X((bool) (seqNumBitField & CONTROL_BIT_MASK), - "Packet::readHeader()", "This should be a data packet"); + + Q_ASSERT_X(!(seqNumBitField & CONTROL_BIT_MASK), "Packet::readHeader()", "This should be a data packet"); + _isReliable = (bool) (seqNumBitField & RELIABILITY_BIT_MASK); // Only keep reliability bit _isPartOfMessage = (bool) (seqNumBitField & MESSAGE_BIT_MASK); // Only keep message bit _sequenceNumber = SequenceNumber{ seqNumBitField & ~BIT_FIELD_MASK }; // Remove the bit field diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 5ded97135b..8b18bdddcd 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -16,6 +16,7 @@ #include "../NetworkLogging.h" #include "ControlPacket.h" #include "Packet.h" +#include "../NLPacket.h" using namespace udt; @@ -138,6 +139,8 @@ void Socket::readPendingDatagrams() { // check if this was a control packet or a data packet bool isControlPacket = *buffer & CONTROL_BIT_MASK; + qDebug() << "IS CONTROL" << isControlPacket; + if (isControlPacket) { // setup a control packet from the data we just read auto controlPacket = ControlPacket::fromReceivedPacket(std::move(buffer), packetSizeWithHeader, senderSockAddr); From 50b80c3c20515cd59843544e2945e68652d0f24e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 17:39:16 -0700 Subject: [PATCH 116/242] remove debug for isControl --- libraries/networking/src/udt/Socket.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 8b18bdddcd..8e1b1685bd 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -139,8 +139,6 @@ void Socket::readPendingDatagrams() { // check if this was a control packet or a data packet bool isControlPacket = *buffer & CONTROL_BIT_MASK; - qDebug() << "IS CONTROL" << isControlPacket; - if (isControlPacket) { // setup a control packet from the data we just read auto controlPacket = ControlPacket::fromReceivedPacket(std::move(buffer), packetSizeWithHeader, senderSockAddr); From cb703a9d7800ee5f118c65807c22e3258ec5284f Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 17:43:40 -0700 Subject: [PATCH 117/242] repair two calls to adjust payload start and capacity --- libraries/networking/src/udt/ControlPacket.cpp | 2 +- libraries/networking/src/udt/Packet.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/ControlPacket.cpp b/libraries/networking/src/udt/ControlPacket.cpp index 01d92bfa07..bd9e851a79 100644 --- a/libraries/networking/src/udt/ControlPacket.cpp +++ b/libraries/networking/src/udt/ControlPacket.cpp @@ -81,7 +81,7 @@ ControlPacket::ControlPacket(std::unique_ptr data, qint64 size, const Hifi // sanity check before we decrease the payloadSize with the payloadCapacity Q_ASSERT(_payloadSize == _payloadCapacity); - adjustPayloadStartAndCapacity(_payloadSize > 0); + adjustPayloadStartAndCapacity(localHeaderSize(), _payloadSize > 0); readType(); } diff --git a/libraries/networking/src/udt/Packet.cpp b/libraries/networking/src/udt/Packet.cpp index 6ec8ac398e..cbe9c986bd 100644 --- a/libraries/networking/src/udt/Packet.cpp +++ b/libraries/networking/src/udt/Packet.cpp @@ -73,7 +73,7 @@ Packet::Packet(std::unique_ptr data, qint64 size, const HifiSockAddr& send { readHeader(); - adjustPayloadStartAndCapacity(_payloadSize > 0); + adjustPayloadStartAndCapacity(localHeaderSize(), _payloadSize > 0); } Packet::Packet(const Packet& other) : From 38037664ba3a435d1df8828f48a143f1be14eb2b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 29 Jul 2015 17:51:50 -0700 Subject: [PATCH 118/242] don't attempt to send stats while not connected --- .../networking/src/ThreadedAssignment.cpp | 24 +++++++++++++++---- libraries/networking/src/ThreadedAssignment.h | 2 ++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/ThreadedAssignment.cpp b/libraries/networking/src/ThreadedAssignment.cpp index 1c425806c9..a5df256e29 100644 --- a/libraries/networking/src/ThreadedAssignment.cpp +++ b/libraries/networking/src/ThreadedAssignment.cpp @@ -70,10 +70,11 @@ void ThreadedAssignment::commonInit(const QString& targetName, NodeType_t nodeTy _domainServerTimer->start(DOMAIN_SERVER_CHECK_IN_MSECS); if (shouldSendStats) { - // send a stats packet every 1 second - _statsTimer = new QTimer(); - connect(_statsTimer, &QTimer::timeout, this, &ThreadedAssignment::sendStatsPacket); - _statsTimer->start(1000); + // start sending stats packet once we connect to the domain + connect(&nodeList->getDomainHandler(), &DomainHandler::connectedToDomain, this, &ThreadedAssignment::startSendingStats); + + // stop sending stats if we disconnect + connect(&nodeList->getDomainHandler(), &DomainHandler::disconnectedFromDomain, this, &ThreadedAssignment::stopSendingStats); } } @@ -96,6 +97,21 @@ void ThreadedAssignment::sendStatsPacket() { addPacketStatsAndSendStatsPacket(statsObject); } +void ThreadedAssignment::startSendingStats() { + // send the stats packet every 1s + if (!_statsTimer) { + _statsTimer = new QTimer(); + connect(_statsTimer, &QTimer::timeout, this, &ThreadedAssignment::sendStatsPacket); + } + + _statsTimer->start(1000); +} + +void ThreadedAssignment::stopSendingStats() { + // stop sending stats, we just disconnected from domain + _statsTimer->stop(); +} + void ThreadedAssignment::checkInWithDomainServerOrExit() { if (DependencyManager::get()->getNumNoReplyDomainCheckIns() == MAX_SILENT_DOMAIN_SERVER_CHECK_INS) { setFinished(true); diff --git a/libraries/networking/src/ThreadedAssignment.h b/libraries/networking/src/ThreadedAssignment.h index 51917cd74d..9ff3b34add 100644 --- a/libraries/networking/src/ThreadedAssignment.h +++ b/libraries/networking/src/ThreadedAssignment.h @@ -42,6 +42,8 @@ protected: QTimer* _statsTimer = nullptr; private slots: + void startSendingStats(); + void stopSendingStats(); void checkInWithDomainServerOrExit(); }; From db5e5a4f7503076d15cefbf138794722a227653e Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 29 Jul 2015 20:04:29 -0700 Subject: [PATCH 119/242] Remove Environment Server --- interface/src/ui/BandwidthDialog.cpp | 2 +- libraries/networking/src/NodeType.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/interface/src/ui/BandwidthDialog.cpp b/interface/src/ui/BandwidthDialog.cpp index b3bc934006..6272ddf76a 100644 --- a/interface/src/ui/BandwidthDialog.cpp +++ b/interface/src/ui/BandwidthDialog.cpp @@ -90,7 +90,7 @@ BandwidthDialog::BandwidthDialog(QWidget* parent) : new BandwidthChannelDisplay({NodeType::Unassigned}, form, "Other", "Kbps", 1.0, COLOR2); _allChannelDisplays[5] = _totalChannelDisplay = new BandwidthChannelDisplay({NodeType::DomainServer, NodeType::EntityServer, - NodeType::EnvironmentServer, NodeType::AudioMixer, NodeType::Agent, + NodeType::AudioMixer, NodeType::Agent, NodeType::AvatarMixer, NodeType::Unassigned}, form, "Total", "Kbps", 1.0, COLOR2); diff --git a/libraries/networking/src/NodeType.h b/libraries/networking/src/NodeType.h index 4427b87158..38590e1b03 100644 --- a/libraries/networking/src/NodeType.h +++ b/libraries/networking/src/NodeType.h @@ -19,7 +19,6 @@ typedef quint8 NodeType_t; namespace NodeType { const NodeType_t DomainServer = 'D'; const NodeType_t EntityServer = 'o'; // was ModelServer - const NodeType_t EnvironmentServer = 'E'; const NodeType_t Agent = 'I'; const NodeType_t AudioMixer = 'M'; const NodeType_t AvatarMixer = 'W'; From da480bcaf07104dd7b802e5f231296bbae223493 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 29 Jul 2015 20:05:15 -0700 Subject: [PATCH 120/242] Fix ControlPackets read/write header --- .../networking/src/udt/ControlPacket.cpp | 20 +++++-------------- libraries/networking/src/udt/Packet.cpp | 19 +++++++----------- libraries/networking/src/udt/SequenceNumber.h | 1 + 3 files changed, 13 insertions(+), 27 deletions(-) diff --git a/libraries/networking/src/udt/ControlPacket.cpp b/libraries/networking/src/udt/ControlPacket.cpp index bd9e851a79..cee2436165 100644 --- a/libraries/networking/src/udt/ControlPacket.cpp +++ b/libraries/networking/src/udt/ControlPacket.cpp @@ -61,7 +61,7 @@ ControlPacket::ControlPacket(Type type) : open(QIODevice::ReadWrite); - writeControlBitAndType(); + writeType(); } ControlPacket::ControlPacket(Type type, qint64 size) : @@ -72,7 +72,7 @@ ControlPacket::ControlPacket(Type type, qint64 size) : open(QIODevice::ReadWrite); - writeControlBitAndType(); + writeType(); } ControlPacket::ControlPacket(std::unique_ptr data, qint64 size, const HifiSockAddr& senderSockAddr) : @@ -106,20 +106,11 @@ void ControlPacket::setType(udt::ControlPacket::Type type) { writeType(); } -void ControlPacket::writeControlBitAndType() { - ControlBitAndType* bitAndType = reinterpret_cast(_packet.get()); - - // write the control bit by OR'ing the current value with the CONTROL_BIT_MASK - *bitAndType = (*bitAndType | CONTROL_BIT_MASK); - - writeType(); -} - void ControlPacket::writeType() { ControlBitAndType* bitAndType = reinterpret_cast(_packet.get()); - // write the type by OR'ing the new type with the current value & CONTROL_BIT_MASK - *bitAndType = (*bitAndType & CONTROL_BIT_MASK) | (_type << (sizeof(ControlPacket::Type) * 8 - 1)); + // We override the control bit here by writing the type but it's okay, it'll always be 1 + *bitAndType = CONTROL_BIT_MASK | (ControlBitAndType(_type) << (8 * sizeof(Type))); } void ControlPacket::readType() { @@ -128,6 +119,5 @@ void ControlPacket::readType() { Q_ASSERT_X(bitAndType & CONTROL_BIT_MASK, "ControlPacket::readHeader()", "This should be a control packet"); // read the type - uint32_t oversizeType = (uint32_t) (bitAndType & ~CONTROL_BIT_MASK); - _type = (Type) oversizeType; + _type = (Type) ((bitAndType & ~CONTROL_BIT_MASK) >> (8 * sizeof(Type))); } diff --git a/libraries/networking/src/udt/Packet.cpp b/libraries/networking/src/udt/Packet.cpp index cbe9c986bd..673013a4a7 100644 --- a/libraries/networking/src/udt/Packet.cpp +++ b/libraries/networking/src/udt/Packet.cpp @@ -135,21 +135,16 @@ void Packet::writeHeader() const { // grab pointer to current SequenceNumberAndBitField SequenceNumberAndBitField* seqNumBitField = reinterpret_cast(_packet.get()); - // 0 for data packets - *seqNumBitField &= ~CONTROL_BIT_MASK; - - if (_isPartOfMessage) { - *seqNumBitField |= MESSAGE_BIT_MASK; - } else { - *seqNumBitField &= ~MESSAGE_BIT_MASK; - } + // Write sequence number and reset bit field + Q_ASSERT_X(!((SequenceNumber::Type)_sequenceNumber & BIT_FIELD_MASK), + "Packet::writeHeader()", "Sequence number is overflowing into bit field"); + *seqNumBitField = ((SequenceNumber::Type)_sequenceNumber); if (_isReliable) { *seqNumBitField |= RELIABILITY_BIT_MASK; - } else { - *seqNumBitField &= ~RELIABILITY_BIT_MASK; } - // write new value by ORing (old value & BIT_FIELD_MASK) with new seqNum - *seqNumBitField = (*seqNumBitField & BIT_FIELD_MASK) | (SequenceNumber::Type)_sequenceNumber; + if (_isPartOfMessage) { + *seqNumBitField |= MESSAGE_BIT_MASK; + } } diff --git a/libraries/networking/src/udt/SequenceNumber.h b/libraries/networking/src/udt/SequenceNumber.h index 6215d23494..d5b61c0a95 100644 --- a/libraries/networking/src/udt/SequenceNumber.h +++ b/libraries/networking/src/udt/SequenceNumber.h @@ -92,6 +92,7 @@ private: friend struct std::hash; }; +static_assert(sizeof(SequenceNumber) == sizeof(uint32_t), "SequenceNumber invalid size"); inline bool operator<(const SequenceNumber& a, const SequenceNumber& b) { From 3d523552f8242ec8164721935cbd724bda380326 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 30 Jul 2015 01:02:53 -0700 Subject: [PATCH 121/242] Cast buffer ptr before bitwise op --- libraries/networking/src/udt/Socket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index dc808ff286..a93a4d0e9e 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -138,7 +138,7 @@ void Socket::readPendingDatagrams() { } // check if this was a control packet or a data packet - bool isControlPacket = *buffer & CONTROL_BIT_MASK; + bool isControlPacket = *reinterpret_cast(buffer.get()) & CONTROL_BIT_MASK; if (isControlPacket) { // setup a control packet from the data we just read From 57025cff74352634fe760197c48ab3cb65a86faf Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 30 Jul 2015 10:20:09 -0700 Subject: [PATCH 122/242] fix packet pair sending from SendQueue --- libraries/networking/src/udt/Connection.cpp | 4 +- libraries/networking/src/udt/SendQueue.cpp | 43 +++++++++++++++++++-- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index f7f32d7480..690f47adab 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -228,9 +228,9 @@ SequenceNumber Connection::nextACK() const { bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber) { // check if this is a packet pair we should estimate bandwidth from, or just a regular packet - if (((uint32_t) sequenceNumber & 0xF) == 0) { + if (((uint32_t) sequenceNumber & 0xFF) == 0) { _receiveWindow.onProbePair1Arrival(); - } else if (((uint32_t) sequenceNumber & 0xF) == 1) { + } else if (((uint32_t) sequenceNumber & 0xFF) == 1) { _receiveWindow.onProbePair2Arrival(); } else { _receiveWindow.onPacketArrival(); diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 9a2e371616..628ac020f5 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -135,13 +135,13 @@ void SendQueue::loop() { // is greater than or equal to the gap between the last ACKed sent and the one we are about to send if (seqlen(SequenceNumber { (uint32_t) _lastACKSequenceNumber }, _currentSequenceNumber + 1) <= _flowWindowSize) { bool hasResend = false; - SequenceNumber seqNum; + SequenceNumber sequenceNumber; { // Check nak list for packet to resend QWriteLocker locker(&_naksLock); if (_naks.getLength() > 0) { hasResend = true; - seqNum = _naks.popFirstSequenceNumber(); + sequenceNumber = _naks.popFirstSequenceNumber(); } } @@ -150,7 +150,7 @@ void SendQueue::loop() { // Find packet in sent list using SequenceNumber if (hasResend) { QWriteLocker locker(&_sentLock); - auto it = _sentPackets.find(seqNum); + auto it = _sentPackets.find(sequenceNumber); Q_ASSERT_X(it != _sentPackets.end(), "SendQueue::sendNextPacket()", "Couldn't find NAKed packet to resend"); @@ -167,8 +167,20 @@ void SendQueue::loop() { _packets.pop_front(); } + bool shouldSendSecondOfPair = false; + + if (!hasResend) { + // if we're not re-sending a packet then need to check if this should be a packet pair + sequenceNumber = getNextSequenceNumber(); + + // the first packet in the pair is every 16 (rightmost 16 bits = 0) packets + if (((uint32_t) sequenceNumber & 0xFF) == 0) { + shouldSendSecondOfPair = true; + } + } + // Write packet's sequence number and send it off - nextPacket->writeSequenceNumber(getNextSequenceNumber()); + nextPacket->writeSequenceNumber(sequenceNumber); sendPacket(*nextPacket); // Insert the packet we have just sent in the sent list @@ -176,6 +188,29 @@ void SendQueue::loop() { _sentPackets[nextPacket->getSequenceNumber()].swap(nextPacket); Q_ASSERT_X(!nextPacket, "SendQueue::sendNextPacket()", "Overriden packet in sent list"); + + if (shouldSendSecondOfPair) { + std::unique_ptr pairedPacket; + + // we've detected we should send the second packet in a pair, do that now before sleeping + { + QWriteLocker locker(&_packetsLock); + pairedPacket.swap(_packets.front()); + _packets.pop_front(); + } + + if (pairedPacket) { + // write this packet's sequence number and send it off + pairedPacket->writeSequenceNumber(getNextSequenceNumber()); + sendPacket(*pairedPacket); + + // add the paired packet to the sent list + QWriteLocker locker(&_sentLock); + _sentPackets[pairedPacket->getSequenceNumber()].swap(pairedPacket); + Q_ASSERT_X(!pairedPacket, + "SendQueue::sendNextPacket()", "Overriden packet in sent list"); + } + } } // since we're a while loop, give the thread a chance to process events From e7f5bec3d0a45012e0c8e3f27c8029a8e040676b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 30 Jul 2015 10:30:14 -0700 Subject: [PATCH 123/242] take out the incorrect attempt to cast for bitwise --- libraries/networking/src/udt/Connection.cpp | 4 ++-- libraries/networking/src/udt/SendQueue.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 690f47adab..f7f32d7480 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -228,9 +228,9 @@ SequenceNumber Connection::nextACK() const { bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber) { // check if this is a packet pair we should estimate bandwidth from, or just a regular packet - if (((uint32_t) sequenceNumber & 0xFF) == 0) { + if (((uint32_t) sequenceNumber & 0xF) == 0) { _receiveWindow.onProbePair1Arrival(); - } else if (((uint32_t) sequenceNumber & 0xFF) == 1) { + } else if (((uint32_t) sequenceNumber & 0xF) == 1) { _receiveWindow.onProbePair2Arrival(); } else { _receiveWindow.onPacketArrival(); diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 628ac020f5..e6bc59c6cb 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -174,7 +174,7 @@ void SendQueue::loop() { sequenceNumber = getNextSequenceNumber(); // the first packet in the pair is every 16 (rightmost 16 bits = 0) packets - if (((uint32_t) sequenceNumber & 0xFF) == 0) { + if (((uint32_t) sequenceNumber & 0xF) == 0) { shouldSendSecondOfPair = true; } } From 3f0eecc599f044dc5a4991fc621ef42a83a6c45c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 30 Jul 2015 15:04:48 -0700 Subject: [PATCH 124/242] actually change the system socket buffer size --- libraries/networking/src/udt/Socket.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index a93a4d0e9e..3dd00d0834 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -39,7 +39,7 @@ void Socket::rebind() { quint16 oldPort = _udpSocket.localPort(); _udpSocket.close(); - _udpSocket.bind(QHostAddress::AnyIPv4, oldPort); + bind(QHostAddress::AnyIPv4, oldPort); } void Socket::setSystemBufferSizes() { @@ -63,6 +63,7 @@ void Socket::setSystemBufferSizes() { int oldBufferSize = _udpSocket.socketOption(bufferOpt).toInt(); if (oldBufferSize < numBytes) { + _udpSocket.setSocketOption(bufferOpt, QVariant(numBytes)); int newBufferSize = _udpSocket.socketOption(bufferOpt).toInt(); qCDebug(networking) << "Changed socket" << bufferTypeString << "buffer size from" << oldBufferSize << "to" From 9e7644ee785af686baa14e932d23fe1717c34018 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 30 Jul 2015 15:05:51 -0700 Subject: [PATCH 125/242] Change headers/payload size computation --- .../octree/OctreeInboundPacketProcessor.cpp | 2 +- libraries/networking/src/NLPacket.cpp | 62 ++++++++----------- libraries/networking/src/NLPacket.h | 14 ++--- libraries/networking/src/udt/BasePacket.cpp | 16 ++++- libraries/networking/src/udt/BasePacket.h | 14 +++-- .../networking/src/udt/ControlPacket.cpp | 26 ++++---- libraries/networking/src/udt/ControlPacket.h | 20 +++--- libraries/networking/src/udt/Packet.cpp | 38 +++++------- libraries/networking/src/udt/Packet.h | 13 ++-- 9 files changed, 102 insertions(+), 103 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 8d76be2c78..1d9adab28b 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -128,7 +128,7 @@ void OctreeInboundPacketProcessor::processPacket(QSharedPointer packet } if (debugProcessPacket) { - qDebug() << " numBytesPacketHeader=" << packet->totalHeadersSize(); + qDebug() << " numBytesPacketHeader=" << NLPacket::totalHeaderSize(packetType); qDebug() << " sizeof(sequence)=" << sizeof(sequence); qDebug() << " sizeof(sentAt)=" << sizeof(sentAt); } diff --git a/libraries/networking/src/NLPacket.cpp b/libraries/networking/src/NLPacket.cpp index 4568561fa2..a5e7feb6f5 100644 --- a/libraries/networking/src/NLPacket.cpp +++ b/libraries/networking/src/NLPacket.cpp @@ -11,26 +11,17 @@ #include "NLPacket.h" -qint64 NLPacket::maxPayloadSize(PacketType type) { - return Packet::maxPayloadSize(false) - localHeaderSize(type); -} - -qint64 NLPacket::localHeaderSize(PacketType type) { - qint64 optionalSize = ((NON_SOURCED_PACKETS.contains(type)) ? 0 : NUM_BYTES_RFC4122_UUID) + - ((NON_SOURCED_PACKETS.contains(type) || NON_VERIFIED_PACKETS.contains(type)) ? 0 : NUM_BYTES_MD5_HASH); +int NLPacket::localHeaderSize(PacketType type) { + bool sourced = NON_SOURCED_PACKETS.contains(type); + bool verified = NON_VERIFIED_PACKETS.contains(type); + qint64 optionalSize = (sourced ? 0 : NUM_BYTES_RFC4122_UUID) + ((sourced || verified) ? 0 : NUM_BYTES_MD5_HASH); return sizeof(PacketType) + sizeof(PacketVersion) + optionalSize; } - -qint64 NLPacket::maxPayloadSize() const { - return Packet::maxPayloadSize() - localHeaderSize(); +int NLPacket::totalHeaderSize(PacketType type, bool isPartOfMessage) { + return Packet::totalHeaderSize(isPartOfMessage) + NLPacket::localHeaderSize(type); } - -qint64 NLPacket::totalHeadersSize() const { - return Packet::totalHeadersSize() + localHeaderSize(); -} - -qint64 NLPacket::localHeaderSize() const { - return localHeaderSize(_type); +int NLPacket::maxPayloadSize(PacketType type, bool isPartOfMessage) { + return Packet::maxPayloadSize(isPartOfMessage) - NLPacket::localHeaderSize(type); } std::unique_ptr NLPacket::create(PacketType type, qint64 size, bool isReliable, bool isPartOfMessage) { @@ -84,19 +75,19 @@ NLPacket::NLPacket(PacketType type, bool isReliable, bool isPartOfMessage) : _type(type), _version(versionForPacketType(type)) { - adjustPayloadStartAndCapacity(localHeaderSize()); + adjustPayloadStartAndCapacity(NLPacket::localHeaderSize(_type)); writeTypeAndVersion(); } NLPacket::NLPacket(PacketType type, qint64 size, bool isReliable, bool isPartOfMessage) : - Packet(localHeaderSize(type) + size, isReliable, isPartOfMessage), + Packet(NLPacket::localHeaderSize(type) + size, isReliable, isPartOfMessage), _type(type), _version(versionForPacketType(type)) { Q_ASSERT(size >= 0); - adjustPayloadStartAndCapacity(localHeaderSize()); + adjustPayloadStartAndCapacity(NLPacket::localHeaderSize(_type)); writeTypeAndVersion(); } @@ -108,7 +99,7 @@ NLPacket::NLPacket(std::unique_ptr packet) : readVersion(); readSourceID(); - adjustPayloadStartAndCapacity(localHeaderSize(), _payloadSize > 0); + adjustPayloadStartAndCapacity(NLPacket::localHeaderSize(_type), _payloadSize > 0); } NLPacket::NLPacket(const NLPacket& other) : Packet(other) { @@ -123,11 +114,11 @@ NLPacket::NLPacket(std::unique_ptr data, qint64 size, const HifiSockAddr& // sanity check before we decrease the payloadSize with the payloadCapacity Q_ASSERT(_payloadSize == _payloadCapacity); - adjustPayloadStartAndCapacity(localHeaderSize(), _payloadSize > 0); - readType(); readVersion(); readSourceID(); + + adjustPayloadStartAndCapacity(NLPacket::localHeaderSize(_type), _payloadSize > 0); } NLPacket::NLPacket(NLPacket&& other) : @@ -160,29 +151,29 @@ NLPacket& NLPacket::operator=(NLPacket&& other) { } PacketType NLPacket::typeInHeader(const udt::Packet& packet) { - auto headerOffset = packet.Packet::totalHeadersSize(); + auto headerOffset = Packet::totalHeaderSize(packet.isPartOfMessage()); return *reinterpret_cast(packet.getData() + headerOffset); } PacketVersion NLPacket::versionInHeader(const udt::Packet& packet) { - auto headerOffset = packet.Packet::totalHeadersSize(); + auto headerOffset = Packet::totalHeaderSize(packet.isPartOfMessage()); return *reinterpret_cast(packet.getData() + headerOffset + sizeof(PacketType)); } QUuid NLPacket::sourceIDInHeader(const udt::Packet& packet) { - int offset = packet.Packet::totalHeadersSize() + sizeof(PacketType) + sizeof(PacketVersion); + int offset = Packet::totalHeaderSize(packet.isPartOfMessage()) + sizeof(PacketType) + sizeof(PacketVersion); return QUuid::fromRfc4122(QByteArray::fromRawData(packet.getData() + offset, NUM_BYTES_RFC4122_UUID)); } QByteArray NLPacket::verificationHashInHeader(const udt::Packet& packet) { - int offset = packet.Packet::totalHeadersSize() + sizeof(PacketType) + sizeof(PacketVersion) + NUM_BYTES_RFC4122_UUID; + int offset = Packet::totalHeaderSize(packet.isPartOfMessage()) + sizeof(PacketType) + sizeof(PacketVersion) + NUM_BYTES_RFC4122_UUID; return QByteArray(packet.getData() + offset, NUM_BYTES_MD5_HASH); } QByteArray NLPacket::hashForPacketAndSecret(const udt::Packet& packet, const QUuid& connectionSecret) { QCryptographicHash hash(QCryptographicHash::Md5); - int offset = packet.Packet::totalHeadersSize() + sizeof(PacketType) + sizeof(PacketVersion) + int offset = Packet::totalHeaderSize(packet.isPartOfMessage()) + sizeof(PacketType) + sizeof(PacketVersion) + NUM_BYTES_RFC4122_UUID + NUM_BYTES_MD5_HASH; // add the packet payload and the connection UUID @@ -194,7 +185,7 @@ QByteArray NLPacket::hashForPacketAndSecret(const udt::Packet& packet, const QUu } void NLPacket::writeTypeAndVersion() { - auto headerOffset = Packet::localHeaderSize(); + auto headerOffset = Packet::localHeaderSize(isPartOfMessage()); // Pack the packet type memcpy(_packet.get() + headerOffset, &_type, sizeof(PacketType)); @@ -204,16 +195,14 @@ void NLPacket::writeTypeAndVersion() { } void NLPacket::setType(PacketType type) { - auto currentHeaderSize = totalHeadersSize(); + // Setting new packet type with a different header size not currently supported + Q_ASSERT(NLPacket::totalHeaderSize(_type, isPartOfMessage()) == + NLPacket::totalHeaderSize(type, isPartOfMessage())); _type = type; _version = versionForPacketType(_type); writeTypeAndVersion(); - - // Setting new packet type with a different header size not currently supported - Q_ASSERT(currentHeaderSize == totalHeadersSize()); - Q_UNUSED(currentHeaderSize); } void NLPacket::readType() { @@ -233,7 +222,7 @@ void NLPacket::readSourceID() { void NLPacket::writeSourceID(const QUuid& sourceID) { Q_ASSERT(!NON_SOURCED_PACKETS.contains(_type)); - auto offset = Packet::totalHeadersSize() + sizeof(PacketType) + sizeof(PacketVersion); + auto offset = Packet::totalHeaderSize(isPartOfMessage()) + sizeof(PacketType) + sizeof(PacketVersion); memcpy(_packet.get() + offset, sourceID.toRfc4122().constData(), NUM_BYTES_RFC4122_UUID); _sourceID = sourceID; @@ -242,7 +231,8 @@ void NLPacket::writeSourceID(const QUuid& sourceID) { void NLPacket::writeVerificationHashGivenSecret(const QUuid& connectionSecret) { Q_ASSERT(!NON_SOURCED_PACKETS.contains(_type) && !NON_VERIFIED_PACKETS.contains(_type)); - auto offset = Packet::totalHeadersSize() + sizeof(PacketType) + sizeof(PacketVersion) + NUM_BYTES_RFC4122_UUID; + auto offset = Packet::totalHeaderSize(isPartOfMessage()) + sizeof(PacketType) + sizeof(PacketVersion) + + NUM_BYTES_RFC4122_UUID; QByteArray verificationHash = hashForPacketAndSecret(*this, connectionSecret); memcpy(_packet.get() + offset, verificationHash.data(), verificationHash.size()); diff --git a/libraries/networking/src/NLPacket.h b/libraries/networking/src/NLPacket.h index 896fbaa66d..25efd673aa 100644 --- a/libraries/networking/src/NLPacket.h +++ b/libraries/networking/src/NLPacket.h @@ -35,6 +35,13 @@ public: // Provided for convenience, try to limit use static std::unique_ptr createCopy(const NLPacket& other); + // Current level's header size + static int localHeaderSize(PacketType type); + // Cumulated size of all the headers + static int totalHeaderSize(PacketType type, bool isPartOfMessage = false); + // The maximum payload size this packet can use to fit in MTU + static int maxPayloadSize(PacketType type, bool isPartOfMessage = false); + static PacketType typeInHeader(const udt::Packet& packet); static PacketVersion versionInHeader(const udt::Packet& packet); @@ -42,13 +49,6 @@ public: static QByteArray verificationHashInHeader(const udt::Packet& packet); static QByteArray hashForPacketAndSecret(const udt::Packet& packet, const QUuid& connectionSecret); - static qint64 maxPayloadSize(PacketType type); - static qint64 localHeaderSize(PacketType type); - - virtual qint64 maxPayloadSize() const; // The maximum payload size this packet can use to fit in MTU - virtual qint64 totalHeadersSize() const; // Cumulated size of all the headers - virtual qint64 localHeaderSize() const; // Current level's header size - PacketType getType() const { return _type; } void setType(PacketType type); diff --git a/libraries/networking/src/udt/BasePacket.cpp b/libraries/networking/src/udt/BasePacket.cpp index 38fb9e849e..e8b660dd75 100644 --- a/libraries/networking/src/udt/BasePacket.cpp +++ b/libraries/networking/src/udt/BasePacket.cpp @@ -15,6 +15,16 @@ using namespace udt; const qint64 BasePacket::PACKET_WRITE_ERROR = -1; +int BasePacket::localHeaderSize() { + return 0; +} +int BasePacket::totalHeaderSize() { + return 0; +} +int BasePacket::maxPayloadSize() { + return MAX_PACKET_SIZE; +} + std::unique_ptr BasePacket::create(qint64 size) { auto packet = std::unique_ptr(new BasePacket(size)); @@ -37,7 +47,7 @@ std::unique_ptr BasePacket::fromReceivedPacket(std::unique_ptr } BasePacket::BasePacket(qint64 size) { - auto maxPayload = maxPayloadSize(); + auto maxPayload = BasePacket::maxPayloadSize(); if (size == -1) { // default size of -1, means biggest packet possible @@ -116,6 +126,10 @@ BasePacket& BasePacket::operator=(BasePacket&& other) { return *this; } +qint64 BasePacket::getDataSize() const { + return (_payloadStart - _packet.get()) + _payloadSize; +} + void BasePacket::setPayloadSize(qint64 payloadSize) { if (isWritable()) { Q_ASSERT(payloadSize <= _payloadCapacity); diff --git a/libraries/networking/src/udt/BasePacket.h b/libraries/networking/src/udt/BasePacket.h index d8c1aaeddf..959ef8900c 100644 --- a/libraries/networking/src/udt/BasePacket.h +++ b/libraries/networking/src/udt/BasePacket.h @@ -30,10 +30,12 @@ public: static std::unique_ptr fromReceivedPacket(std::unique_ptr data, qint64 size, const HifiSockAddr& senderSockAddr); - static qint64 maxPayloadSize() { return MAX_PACKET_SIZE; } // The maximum payload size this packet can use to fit in MTU - static qint64 localHeaderSize() { return 0; } // Current level's header size - - virtual qint64 totalHeadersSize() const { return 0; } // Cumulated size of all the headers + // Current level's header size + static int localHeaderSize(); + // Cumulated size of all the headers + static int totalHeaderSize(); + // The maximum payload size this packet can use to fit in MTU + static int maxPayloadSize(); // Payload direct access to the payload, use responsibly! char* getPayload() { return _payloadStart; } @@ -44,7 +46,7 @@ public: const char* getData() const { return _packet.get(); } // Returns the size of the packet, including the header - qint64 getDataSize() const { return totalHeadersSize() + _payloadSize; } + qint64 getDataSize() const; // Returns the size of the payload only qint64 getPayloadSize() const { return _payloadSize; } @@ -86,7 +88,7 @@ protected: virtual qint64 writeData(const char* data, qint64 maxSize); virtual qint64 readData(char* data, qint64 maxSize); - virtual void adjustPayloadStartAndCapacity(qint64 headerSize, bool shouldDecreasePayloadSize = false); + void adjustPayloadStartAndCapacity(qint64 headerSize, bool shouldDecreasePayloadSize = false); qint64 _packetSize = 0; // Total size of the allocated memory std::unique_ptr _packet; // Allocated memory diff --git a/libraries/networking/src/udt/ControlPacket.cpp b/libraries/networking/src/udt/ControlPacket.cpp index cee2436165..7378b2f314 100644 --- a/libraries/networking/src/udt/ControlPacket.cpp +++ b/libraries/networking/src/udt/ControlPacket.cpp @@ -15,6 +15,16 @@ using namespace udt; +int ControlPacket::localHeaderSize() { + return sizeof(ControlPacket::ControlBitAndType); +} +int ControlPacket::totalHeaderSize() { + return BasePacket::totalHeaderSize() + ControlPacket::localHeaderSize(); +} +int ControlPacket::maxPayloadSize() { + return BasePacket::maxPayloadSize() - ControlPacket::localHeaderSize(); +} + std::unique_ptr ControlPacket::fromReceivedPacket(std::unique_ptr data, qint64 size, const HifiSockAddr &senderSockAddr) { // Fail with null data @@ -45,19 +55,11 @@ std::unique_ptr ControlPacket::create(Type type, qint64 size) { } } -qint64 ControlPacket::localHeaderSize() { - return sizeof(ControlBitAndType); -} - -qint64 ControlPacket::totalHeadersSize() const { - return BasePacket::totalHeadersSize() + localHeaderSize(); -} - ControlPacket::ControlPacket(Type type) : BasePacket(-1), _type(type) { - adjustPayloadStartAndCapacity(localHeaderSize()); + adjustPayloadStartAndCapacity(ControlPacket::localHeaderSize()); open(QIODevice::ReadWrite); @@ -65,10 +67,10 @@ ControlPacket::ControlPacket(Type type) : } ControlPacket::ControlPacket(Type type, qint64 size) : - BasePacket(localHeaderSize() + size), + BasePacket(ControlPacket::localHeaderSize() + size), _type(type) { - adjustPayloadStartAndCapacity(localHeaderSize()); + adjustPayloadStartAndCapacity(ControlPacket::localHeaderSize()); open(QIODevice::ReadWrite); @@ -81,7 +83,7 @@ ControlPacket::ControlPacket(std::unique_ptr data, qint64 size, const Hifi // sanity check before we decrease the payloadSize with the payloadCapacity Q_ASSERT(_payloadSize == _payloadCapacity); - adjustPayloadStartAndCapacity(localHeaderSize(), _payloadSize > 0); + adjustPayloadStartAndCapacity(ControlPacket::localHeaderSize(), _payloadSize > 0); readType(); } diff --git a/libraries/networking/src/udt/ControlPacket.h b/libraries/networking/src/udt/ControlPacket.h index 6d13c06d33..c4ad7065a7 100644 --- a/libraries/networking/src/udt/ControlPacket.h +++ b/libraries/networking/src/udt/ControlPacket.h @@ -33,12 +33,15 @@ public: TimeoutNAK }; + static std::unique_ptr create(Type type, qint64 size = -1); static std::unique_ptr fromReceivedPacket(std::unique_ptr data, qint64 size, const HifiSockAddr& senderSockAddr); - static std::unique_ptr create(Type type, qint64 size = -1); - - static qint64 localHeaderSize(); // Current level's header size - virtual qint64 totalHeadersSize() const; // Cumulated size of all the headers + // Current level's header size + static int localHeaderSize(); + // Cumulated size of all the headers + static int totalHeaderSize(); + // The maximum payload size this packet can use to fit in MTU + static int maxPayloadSize(); Type getType() const { return _type; } void setType(Type type); @@ -53,16 +56,13 @@ private: ControlPacket& operator=(ControlPacket&& other); ControlPacket& operator=(const ControlPacket& other) = delete; - // Header writers - void writeControlBitAndType(); - void writeType(); - - // Header readers + // Header read/write void readType(); + void writeType(); Type _type; }; - + } // namespace udt diff --git a/libraries/networking/src/udt/Packet.cpp b/libraries/networking/src/udt/Packet.cpp index 673013a4a7..754d1534b1 100644 --- a/libraries/networking/src/udt/Packet.cpp +++ b/libraries/networking/src/udt/Packet.cpp @@ -13,6 +13,20 @@ using namespace udt; +int Packet::localHeaderSize(bool isPartOfMessage) { + return sizeof(Packet::SequenceNumberAndBitField) + + (isPartOfMessage ? sizeof(Packet::MessageNumberAndBitField) : 0); +} + +int Packet::totalHeaderSize(bool isPartOfMessage) { + return BasePacket::totalHeaderSize() + Packet::localHeaderSize(isPartOfMessage); +} + +int Packet::maxPayloadSize(bool isPartOfMessage) { + return BasePacket::maxPayloadSize() - Packet::localHeaderSize(isPartOfMessage); +} + + std::unique_ptr Packet::create(qint64 size, bool isReliable, bool isPartOfMessage) { auto packet = std::unique_ptr(new Packet(size, isReliable, isPartOfMessage)); @@ -37,32 +51,12 @@ std::unique_ptr Packet::createCopy(const Packet& other) { return std::unique_ptr(new Packet(other)); } -qint64 Packet::maxPayloadSize(bool isPartOfMessage) { - return MAX_PACKET_SIZE - localHeaderSize(isPartOfMessage); -} - -qint64 Packet::localHeaderSize(bool isPartOfMessage) { - return sizeof(SequenceNumberAndBitField) + (isPartOfMessage ? sizeof(MessageNumberAndBitField) : 0); -} - -qint64 Packet::maxPayloadSize() const { - return MAX_PACKET_SIZE - localHeaderSize(); -} - -qint64 Packet::totalHeadersSize() const { - return BasePacket::totalHeadersSize() + Packet::localHeaderSize(); -} - -qint64 Packet::localHeaderSize() const { - return localHeaderSize(_isPartOfMessage); -} - Packet::Packet(qint64 size, bool isReliable, bool isPartOfMessage) : BasePacket(size), _isReliable(isReliable), _isPartOfMessage(isPartOfMessage) { - adjustPayloadStartAndCapacity(localHeaderSize()); + adjustPayloadStartAndCapacity(Packet::localHeaderSize(_isPartOfMessage)); // set the UDT header to default values writeHeader(); @@ -73,7 +67,7 @@ Packet::Packet(std::unique_ptr data, qint64 size, const HifiSockAddr& send { readHeader(); - adjustPayloadStartAndCapacity(localHeaderSize(), _payloadSize > 0); + adjustPayloadStartAndCapacity(Packet::localHeaderSize(_isPartOfMessage), _payloadSize > 0); } Packet::Packet(const Packet& other) : diff --git a/libraries/networking/src/udt/Packet.h b/libraries/networking/src/udt/Packet.h index 8a6832205f..22eec97d97 100644 --- a/libraries/networking/src/udt/Packet.h +++ b/libraries/networking/src/udt/Packet.h @@ -38,15 +38,12 @@ public: // Provided for convenience, try to limit use static std::unique_ptr createCopy(const Packet& other); - // The maximum payload size this packet can use to fit in MTU - static qint64 maxPayloadSize(bool isPartOfMessage); - virtual qint64 maxPayloadSize() const; - // Current level's header size - static qint64 localHeaderSize(bool isPartOfMessage); - virtual qint64 localHeaderSize() const; - - virtual qint64 totalHeadersSize() const; // Cumulated size of all the headers + static int localHeaderSize(bool isPartOfMessage = false); + // Cumulated size of all the headers + static int totalHeaderSize(bool isPartOfMessage = false); + // The maximum payload size this packet can use to fit in MTU + static int maxPayloadSize(bool isPartOfMessage = false); bool isPartOfMessage() const { return _isPartOfMessage; } bool isReliable() const { return _isReliable; } From 8e55655ec3aed5698a9e724e1213ab9f3720e060 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 30 Jul 2015 15:07:49 -0700 Subject: [PATCH 126/242] call setSystemBufferSizes once bound --- libraries/networking/src/udt/Socket.cpp | 2 -- libraries/networking/src/udt/Socket.h | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 3dd00d0834..f8515affe6 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -24,8 +24,6 @@ using namespace udt; Socket::Socket(QObject* parent) : QObject(parent) { - setSystemBufferSizes(); - connect(&_udpSocket, &QUdpSocket::readyRead, this, &Socket::readPendingDatagrams); // make sure our synchronization method is called every SYN interval diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 1827dc8234..e923f4b307 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -50,7 +50,7 @@ public: qint64 writeDatagram(const char* data, qint64 size, const HifiSockAddr& sockAddr); qint64 writeDatagram(const QByteArray& datagram, const HifiSockAddr& sockAddr); - void bind(const QHostAddress& address, quint16 port = 0) { _udpSocket.bind(address, port); } + void bind(const QHostAddress& address, quint16 port = 0) { _udpSocket.bind(address, port); setSystemBufferSizes(); } void rebind(); void setPacketFilterOperator(PacketFilterOperator filterOperator) { _packetFilterOperator = filterOperator; } From 784cc880a7d4186b67d199e4e21915e1f04b4c44 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 30 Jul 2015 15:15:51 -0700 Subject: [PATCH 127/242] Fix Packet ctor --- libraries/networking/src/udt/Packet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Packet.cpp b/libraries/networking/src/udt/Packet.cpp index 754d1534b1..67fb3b8378 100644 --- a/libraries/networking/src/udt/Packet.cpp +++ b/libraries/networking/src/udt/Packet.cpp @@ -52,7 +52,7 @@ std::unique_ptr Packet::createCopy(const Packet& other) { } Packet::Packet(qint64 size, bool isReliable, bool isPartOfMessage) : - BasePacket(size), + BasePacket(Packet::localHeaderSize() + size), _isReliable(isReliable), _isPartOfMessage(isPartOfMessage) { From 532ac690912b65f8657511b37c50d11e5df9ee72 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 30 Jul 2015 15:36:54 -0700 Subject: [PATCH 128/242] initial version of UDTTest tool --- tools/CMakeLists.txt | 4 ++- tools/udt-test/CMakeLists.txt | 6 +++++ tools/udt-test/src/UDTTest.cpp | 46 ++++++++++++++++++++++++++++++++++ tools/udt-test/src/UDTTest.h | 32 +++++++++++++++++++++++ tools/udt-test/src/main.cpp | 19 ++++++++++++++ 5 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 tools/udt-test/CMakeLists.txt create mode 100644 tools/udt-test/src/UDTTest.cpp create mode 100644 tools/udt-test/src/UDTTest.h create mode 100644 tools/udt-test/src/main.cpp diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 55994f3d89..2056044a4b 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -5,6 +5,8 @@ set_target_properties(mtc PROPERTIES FOLDER "Tools") add_subdirectory(scribe) set_target_properties(scribe PROPERTIES FOLDER "Tools") +add_subdirectory(udt-test) +set_target_properties(udt-test PROPERTIES FOLDER "Tools") + add_subdirectory(vhacd-util) set_target_properties(vhacd-util PROPERTIES FOLDER "Tools") - diff --git a/tools/udt-test/CMakeLists.txt b/tools/udt-test/CMakeLists.txt new file mode 100644 index 0000000000..7f47677269 --- /dev/null +++ b/tools/udt-test/CMakeLists.txt @@ -0,0 +1,6 @@ +set(TARGET_NAME udt-test) +setup_hifi_project() + +link_hifi_libraries(networking) + +copy_dlls_beside_windows_executable() \ No newline at end of file diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp new file mode 100644 index 0000000000..d2f8bf6e40 --- /dev/null +++ b/tools/udt-test/src/UDTTest.cpp @@ -0,0 +1,46 @@ +// +// UDTTest.cpp +// tools/udt-test/src +// +// Created by Stephen Birarda on 2015-07-30. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "UDTTest.h" + +#include + +const QCommandLineOption PORT_OPTION { "p", "listening port for socket (defaults to random)", "port", 0 }; + +UDTTest::UDTTest(int& argc, char** argv) : + QCoreApplication(argc, argv) +{ + parseArguments(); + + _socket.bind(QHostAddress::LocalHost, _argumentParser.value(PORT_OPTION).toUInt()); + qDebug() << "Test socket is listening on" << _socket.localPort(); +} + +void UDTTest::parseArguments() { + // use a QCommandLineParser to setup command line arguments and give helpful output + _argumentParser.setApplicationDescription("High Fidelity Assignment Client"); + _argumentParser.addHelpOption(); + + const QCommandLineOption helpOption = _argumentParser.addHelpOption(); + + _argumentParser.addOption(PORT_OPTION); + + if (!_argumentParser.parse(QCoreApplication::arguments())) { + qCritical() << _argumentParser.errorText(); + _argumentParser.showHelp(); + Q_UNREACHABLE(); + } + + if (_argumentParser.isSet(helpOption)) { + _argumentParser.showHelp(); + Q_UNREACHABLE(); + } +} diff --git a/tools/udt-test/src/UDTTest.h b/tools/udt-test/src/UDTTest.h new file mode 100644 index 0000000000..86d15974d0 --- /dev/null +++ b/tools/udt-test/src/UDTTest.h @@ -0,0 +1,32 @@ +// +// UDTTest.h +// tools/udt-test/src +// +// Created by Stephen Birarda on 2015-07-30. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#pragma once + +#ifndef hifi_UDTTest_h +#define hifi_UDTTest_h + +#include +#include + +#include + +class UDTTest : public QCoreApplication { +public: + UDTTest(int& argc, char** argv); +private: + void parseArguments(); + + QCommandLineParser _argumentParser; + udt::Socket _socket; +}; + +#endif // hifi_UDTTest_h diff --git a/tools/udt-test/src/main.cpp b/tools/udt-test/src/main.cpp new file mode 100644 index 0000000000..ccb7d0af0f --- /dev/null +++ b/tools/udt-test/src/main.cpp @@ -0,0 +1,19 @@ +// +// main.cpp +// tools/udt-test/src +// +// Created by Stephen Birarda on 7/30/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html + +#include + +#include "UDTTest.h" + +int main(int argc, char* argv[]) { + UDTTest app(argc, argv); + return app.exec(); +} + From 3a9aedea68daf7ace5d86c9b848916db0a1e7f27 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 30 Jul 2015 15:56:42 -0700 Subject: [PATCH 129/242] Fix packet ctors size param --- libraries/networking/src/NLPacket.cpp | 25 ++----------------------- libraries/networking/src/NLPacket.h | 3 +-- libraries/networking/src/udt/Packet.cpp | 2 +- 3 files changed, 4 insertions(+), 26 deletions(-) diff --git a/libraries/networking/src/NLPacket.cpp b/libraries/networking/src/NLPacket.cpp index a5e7feb6f5..146054ccc7 100644 --- a/libraries/networking/src/NLPacket.cpp +++ b/libraries/networking/src/NLPacket.cpp @@ -25,16 +25,7 @@ int NLPacket::maxPayloadSize(PacketType type, bool isPartOfMessage) { } std::unique_ptr NLPacket::create(PacketType type, qint64 size, bool isReliable, bool isPartOfMessage) { - std::unique_ptr packet; - - if (size == -1) { - packet = std::unique_ptr(new NLPacket(type, isReliable, isPartOfMessage)); - } else { - // Fail with invalid size - Q_ASSERT(size >= 0); - - packet = std::unique_ptr(new NLPacket(type, size, isReliable, isPartOfMessage)); - } + auto packet = std::unique_ptr(new NLPacket(type, size, isReliable, isPartOfMessage)); packet->open(QIODevice::ReadWrite); @@ -70,23 +61,11 @@ std::unique_ptr NLPacket::createCopy(const NLPacket& other) { return std::unique_ptr(new NLPacket(other)); } -NLPacket::NLPacket(PacketType type, bool isReliable, bool isPartOfMessage) : - Packet(-1, isReliable, isPartOfMessage), - _type(type), - _version(versionForPacketType(type)) -{ - adjustPayloadStartAndCapacity(NLPacket::localHeaderSize(_type)); - - writeTypeAndVersion(); -} - NLPacket::NLPacket(PacketType type, qint64 size, bool isReliable, bool isPartOfMessage) : - Packet(NLPacket::localHeaderSize(type) + size, isReliable, isPartOfMessage), + Packet((size == -1) ? -1 : NLPacket::localHeaderSize(type) + size, isReliable, isPartOfMessage), _type(type), _version(versionForPacketType(type)) { - Q_ASSERT(size >= 0); - adjustPayloadStartAndCapacity(NLPacket::localHeaderSize(_type)); writeTypeAndVersion(); diff --git a/libraries/networking/src/NLPacket.h b/libraries/networking/src/NLPacket.h index 25efd673aa..2303150e66 100644 --- a/libraries/networking/src/NLPacket.h +++ b/libraries/networking/src/NLPacket.h @@ -61,8 +61,7 @@ public: protected: - NLPacket(PacketType type, bool forceReliable = false, bool isPartOfMessage = false); - NLPacket(PacketType type, qint64 size, bool forceReliable = false, bool isPartOfMessage = false); + NLPacket(PacketType type, qint64 size = -1, bool forceReliable = false, bool isPartOfMessage = false); NLPacket(std::unique_ptr data, qint64 size, const HifiSockAddr& senderSockAddr); NLPacket(std::unique_ptr packet); NLPacket(const NLPacket& other); diff --git a/libraries/networking/src/udt/Packet.cpp b/libraries/networking/src/udt/Packet.cpp index 67fb3b8378..4e77079a22 100644 --- a/libraries/networking/src/udt/Packet.cpp +++ b/libraries/networking/src/udt/Packet.cpp @@ -52,7 +52,7 @@ std::unique_ptr Packet::createCopy(const Packet& other) { } Packet::Packet(qint64 size, bool isReliable, bool isPartOfMessage) : - BasePacket(Packet::localHeaderSize() + size), + BasePacket((size == -1) ? -1 : (Packet::localHeaderSize() + size)), _isReliable(isReliable), _isPartOfMessage(isPartOfMessage) { From e8ffe962325d261de9430e83355c2fc8649be906 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 30 Jul 2015 15:58:05 -0700 Subject: [PATCH 130/242] Remove const_cast --- libraries/networking/src/Assignment.cpp | 4 +--- libraries/networking/src/LimitedNodeList.cpp | 4 ++-- libraries/networking/src/NLPacket.cpp | 4 ++-- libraries/networking/src/NLPacket.h | 6 +++--- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/libraries/networking/src/Assignment.cpp b/libraries/networking/src/Assignment.cpp index 293d86475f..e8ba67c4a6 100644 --- a/libraries/networking/src/Assignment.cpp +++ b/libraries/networking/src/Assignment.cpp @@ -150,11 +150,9 @@ QDataStream& operator<<(QDataStream &out, const Assignment& assignment) { QDataStream& operator>>(QDataStream &in, Assignment& assignment) { quint8 packedType; - in >> packedType; + in >> packedType >> assignment._uuid >> assignment._pool >> assignment._payload; assignment._type = (Assignment::Type) packedType; - in >> assignment._uuid >> assignment._pool >> assignment._payload; - if (assignment._command == Assignment::RequestCommand) { in >> assignment._walletUUID; } diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 2ca4e89c12..5f547ecaf3 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -247,13 +247,13 @@ void LimitedNodeList::collectPacketStats(const NLPacket& packet) { void LimitedNodeList::fillPacketHeader(const NLPacket& packet, const QUuid& connectionSecret) { if (!NON_SOURCED_PACKETS.contains(packet.getType())) { - const_cast(packet).writeSourceID(getSessionUUID()); + packet.writeSourceID(getSessionUUID()); } if (!connectionSecret.isNull() && !NON_SOURCED_PACKETS.contains(packet.getType()) && !NON_VERIFIED_PACKETS.contains(packet.getType())) { - const_cast(packet).writeVerificationHashGivenSecret(connectionSecret); + packet.writeVerificationHashGivenSecret(connectionSecret); } } diff --git a/libraries/networking/src/NLPacket.cpp b/libraries/networking/src/NLPacket.cpp index 146054ccc7..84330b0d77 100644 --- a/libraries/networking/src/NLPacket.cpp +++ b/libraries/networking/src/NLPacket.cpp @@ -198,7 +198,7 @@ void NLPacket::readSourceID() { } } -void NLPacket::writeSourceID(const QUuid& sourceID) { +void NLPacket::writeSourceID(const QUuid& sourceID) const { Q_ASSERT(!NON_SOURCED_PACKETS.contains(_type)); auto offset = Packet::totalHeaderSize(isPartOfMessage()) + sizeof(PacketType) + sizeof(PacketVersion); @@ -207,7 +207,7 @@ void NLPacket::writeSourceID(const QUuid& sourceID) { _sourceID = sourceID; } -void NLPacket::writeVerificationHashGivenSecret(const QUuid& connectionSecret) { +void NLPacket::writeVerificationHashGivenSecret(const QUuid& connectionSecret) const { Q_ASSERT(!NON_SOURCED_PACKETS.contains(_type) && !NON_VERIFIED_PACKETS.contains(_type)); auto offset = Packet::totalHeaderSize(isPartOfMessage()) + sizeof(PacketType) + sizeof(PacketVersion) diff --git a/libraries/networking/src/NLPacket.h b/libraries/networking/src/NLPacket.h index 2303150e66..f5c08b308a 100644 --- a/libraries/networking/src/NLPacket.h +++ b/libraries/networking/src/NLPacket.h @@ -56,8 +56,8 @@ public: const QUuid& getSourceID() const { return _sourceID; } - void writeSourceID(const QUuid& sourceID); - void writeVerificationHashGivenSecret(const QUuid& connectionSecret); + void writeSourceID(const QUuid& sourceID) const; + void writeVerificationHashGivenSecret(const QUuid& connectionSecret) const; protected: @@ -80,7 +80,7 @@ protected: PacketType _type; PacketVersion _version; - QUuid _sourceID; + mutable QUuid _sourceID; }; #endif // hifi_NLPacket_h From 80ef80ec2e4f6015bf435d2168addf85d3a4cf42 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 30 Jul 2015 16:07:14 -0700 Subject: [PATCH 131/242] add all initial command line options for UDTTest --- tools/udt-test/src/UDTTest.cpp | 95 ++++++++++++++++++++++++++++++++-- tools/udt-test/src/UDTTest.h | 12 ++++- 2 files changed, 103 insertions(+), 4 deletions(-) diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index d2f8bf6e40..6bfac4ad26 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -13,7 +13,32 @@ #include +#include + const QCommandLineOption PORT_OPTION { "p", "listening port for socket (defaults to random)", "port", 0 }; +const QCommandLineOption TARGET_OPTION { + "target", "target for sent packets (default is listen only)", + "IP:PORT or HOSTNAME:PORT" +}; +const QCommandLineOption PACKET_SIZE { + "packet-size", "size for sent packets in bytes (defaults to 1500)", "bytes", + QString(udt::MAX_PACKET_SIZE_WITH_UDP_HEADER) +}; +const QCommandLineOption MIN_PACKET_SIZE { + "min-packet-size", "min size for sent packets in bytes", "min-bytes" +}; +const QCommandLineOption MAX_PACKET_SIZE { + "max-packet-size", "max size for sent packets in bytes", "max-bytes" +}; +const QCommandLineOption MAX_SEND_BYTES { + "max-send-bytes", "number of bytes to send before stopping (default is infinite)", "max-bytes" +}; +const QCommandLineOption MAX_SEND_PACKETS { + "max-send-packets", "number of packets to send before stopping (default is infinite)", "max-packets" +}; +const QCommandLineOption UNRELIABLE_PACKETS { + "unreliable", "send unreliable packets (default is reliable)" +}; UDTTest::UDTTest(int& argc, char** argv) : QCoreApplication(argc, argv) @@ -22,18 +47,82 @@ UDTTest::UDTTest(int& argc, char** argv) : _socket.bind(QHostAddress::LocalHost, _argumentParser.value(PORT_OPTION).toUInt()); qDebug() << "Test socket is listening on" << _socket.localPort(); + + if (_argumentParser.isSet(TARGET_OPTION)) { + // parse the IP and port combination for this target + QString hostnamePortString = _argumentParser.value(TARGET_OPTION); + + QHostAddress address { hostnamePortString.left(hostnamePortString.indexOf(':')) }; + quint16 port { (quint16) hostnamePortString.right(hostnamePortString.indexOf(':') + 1).toUInt() }; + + if (address.isNull() || port == 0) { + qCritical() << "Could not parse an IP address and port combination from" << hostnamePortString << "-" << + "The parsed IP was" << address.toString() << "and the parsed port was" << port; + + QMetaObject::invokeMethod(this, "quit", Qt::QueuedConnection); + } else { + _target = HifiSockAddr(address, port); + } + } + + if (_argumentParser.isSet(PACKET_SIZE)) { + // parse the desired packet size + _minPacketSize = _maxPacketSize = _argumentParser.value(PACKET_SIZE).toInt(); + + if (_argumentParser.isSet(MIN_PACKET_SIZE) || _argumentParser.isSet(MAX_PACKET_SIZE)) { + qCritical() << "Cannot set a min packet size or max packet size AND a specific packet size."; + QMetaObject::invokeMethod(this, "quit", Qt::QueuedConnection); + } + } else { + + bool customMinSize = false; + + if (_argumentParser.isSet(MIN_PACKET_SIZE)) { + _minPacketSize = _argumentParser.value(MIN_PACKET_SIZE).toInt(); + customMinSize = true; + } + + if (_argumentParser.isSet(MAX_PACKET_SIZE)) { + _maxPacketSize = _argumentParser.value(MAX_PACKET_SIZE).toInt(); + + // if we don't have a min packet size we should make it zero, because we have a max + if (customMinSize) { + _minPacketSize = 0; + } + } + + if (_maxPacketSize < _minPacketSize) { + qCritical() << "Cannot set a max packet size that is smaller than the min packet size."; + QMetaObject::invokeMethod(this, "quit", Qt::QueuedConnection); + } + } + + if (_argumentParser.isSet(MAX_SEND_BYTES)) { + _maxSendBytes = _argumentParser.value(MAX_SEND_BYTES).toInt(); + } + + if (_argumentParser.isSet(MAX_SEND_PACKETS)) { + _maxSendPackets = _argumentParser.value(MAX_SEND_PACKETS).toInt(); + } + + if (_argumentParser.isSet(UNRELIABLE_PACKETS)) { + _sendReliable = false; + } } void UDTTest::parseArguments() { // use a QCommandLineParser to setup command line arguments and give helpful output - _argumentParser.setApplicationDescription("High Fidelity Assignment Client"); + _argumentParser.setApplicationDescription("High Fidelity UDT Protocol Test Client"); _argumentParser.addHelpOption(); const QCommandLineOption helpOption = _argumentParser.addHelpOption(); - _argumentParser.addOption(PORT_OPTION); + _argumentParser.addOptions({ + PORT_OPTION, TARGET_OPTION, PACKET_SIZE, MIN_PACKET_SIZE, MAX_PACKET_SIZE, + MAX_SEND_BYTES, MAX_SEND_PACKETS, UNRELIABLE_PACKETS + }); - if (!_argumentParser.parse(QCoreApplication::arguments())) { + if (!_argumentParser.parse(arguments())) { qCritical() << _argumentParser.errorText(); _argumentParser.showHelp(); Q_UNREACHABLE(); diff --git a/tools/udt-test/src/UDTTest.h b/tools/udt-test/src/UDTTest.h index 86d15974d0..678a067114 100644 --- a/tools/udt-test/src/UDTTest.h +++ b/tools/udt-test/src/UDTTest.h @@ -17,7 +17,8 @@ #include #include -#include +#include +#include class UDTTest : public QCoreApplication { public: @@ -27,6 +28,15 @@ private: QCommandLineParser _argumentParser; udt::Socket _socket; + + HifiSockAddr _target; // the target for sent packets + + int _minPacketSize = udt::MAX_PACKET_SIZE_WITH_UDP_HEADER; + int _maxPacketSize = udt::MAX_PACKET_SIZE_WITH_UDP_HEADER; + int _maxSendBytes = -1; // the number of bytes to send to the target before stopping + int _maxSendPackets = -1; // the number of packets to send to the target before stopping + + bool _sendReliable = true; // wether packets are sent reliably or unreliably }; #endif // hifi_UDTTest_h From d340515ab3f513eef0095f4903cfe1f293d842fc Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 30 Jul 2015 17:09:25 -0700 Subject: [PATCH 132/242] complete the initial version of UDTTest --- libraries/entities/src/EntityItemID.cpp | 2 + libraries/networking/src/AddressManager.cpp | 4 +- libraries/networking/src/NLPacket.h | 1 + libraries/networking/src/NetworkPeer.cpp | 5 +- libraries/networking/src/Node.cpp | 3 +- libraries/{shared => networking}/src/UUID.cpp | 0 libraries/{shared => networking}/src/UUID.h | 0 .../networking/src/WalletTransaction.cpp | 8 +-- libraries/networking/src/udt/Connection.cpp | 2 + libraries/networking/src/udt/Connection.h | 9 ++- libraries/networking/src/udt/PacketHeaders.h | 2 - libraries/networking/src/udt/SendQueue.cpp | 4 ++ libraries/networking/src/udt/SendQueue.h | 3 + libraries/networking/src/udt/Socket.cpp | 30 +++++--- libraries/networking/src/udt/Socket.h | 3 + tools/udt-test/src/UDTTest.cpp | 69 ++++++++++++++++++- tools/udt-test/src/UDTTest.h | 21 ++++-- 17 files changed, 133 insertions(+), 33 deletions(-) rename libraries/{shared => networking}/src/UUID.cpp (100%) rename libraries/{shared => networking}/src/UUID.h (100%) diff --git a/libraries/entities/src/EntityItemID.cpp b/libraries/entities/src/EntityItemID.cpp index ab2c39324f..36664f5457 100644 --- a/libraries/entities/src/EntityItemID.cpp +++ b/libraries/entities/src/EntityItemID.cpp @@ -11,8 +11,10 @@ #include #include + #include #include +#include #include "RegisteredMetaTypes.h" #include "EntityItemID.h" diff --git a/libraries/networking/src/AddressManager.cpp b/libraries/networking/src/AddressManager.cpp index 3f3234a307..ed4728ad78 100644 --- a/libraries/networking/src/AddressManager.cpp +++ b/libraries/networking/src/AddressManager.cpp @@ -18,11 +18,11 @@ #include #include -#include +#include "AddressManager.h" #include "NodeList.h" #include "NetworkLogging.h" -#include "AddressManager.h" +#include "UUID.h" const QString ADDRESS_MANAGER_SETTINGS_GROUP = "AddressManager"; const QString SETTINGS_CURRENT_ADDRESS_KEY = "address"; diff --git a/libraries/networking/src/NLPacket.h b/libraries/networking/src/NLPacket.h index 25efd673aa..11cebd2468 100644 --- a/libraries/networking/src/NLPacket.h +++ b/libraries/networking/src/NLPacket.h @@ -14,6 +14,7 @@ #include +#include "UUID.h" #include "udt/Packet.h" class NLPacket : public udt::Packet { diff --git a/libraries/networking/src/NetworkPeer.cpp b/libraries/networking/src/NetworkPeer.cpp index 9e51bc5dac..0ffb87fe6c 100644 --- a/libraries/networking/src/NetworkPeer.cpp +++ b/libraries/networking/src/NetworkPeer.cpp @@ -16,11 +16,10 @@ #include #include -#include - -#include "NetworkLogging.h" #include "BandwidthRecorder.h" +#include "NetworkLogging.h" +#include "UUID.h" NetworkPeer::NetworkPeer(QObject* parent) : QObject(parent), diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index 6f10e6dbdb..9f580c6d20 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -12,10 +12,9 @@ #include #include -#include - #include "Node.h" #include "SharedUtil.h" +#include "UUID.h" #include #include diff --git a/libraries/shared/src/UUID.cpp b/libraries/networking/src/UUID.cpp similarity index 100% rename from libraries/shared/src/UUID.cpp rename to libraries/networking/src/UUID.cpp diff --git a/libraries/shared/src/UUID.h b/libraries/networking/src/UUID.h similarity index 100% rename from libraries/shared/src/UUID.h rename to libraries/networking/src/UUID.h diff --git a/libraries/networking/src/WalletTransaction.cpp b/libraries/networking/src/WalletTransaction.cpp index 9c8226a908..67fc73de07 100644 --- a/libraries/networking/src/WalletTransaction.cpp +++ b/libraries/networking/src/WalletTransaction.cpp @@ -1,6 +1,6 @@ // // WalletTransaction.cpp -// domain-server/src +// libraries/networking/src // // Created by Stephen Birarda on 2014-05-20. // Copyright 2014 High Fidelity, Inc. @@ -11,10 +11,10 @@ #include -#include - +#include "UUID.h" #include "WalletTransaction.h" + WalletTransaction::WalletTransaction() : _uuid(), _destinationUUID(), @@ -64,4 +64,4 @@ void WalletTransaction::loadFromJson(const QJsonObject& jsonObject) { _uuid = QUuid(transactionObject.value(TRANSACTION_ID_KEY).toString()); _destinationUUID = QUuid(transactionObject.value(TRANSACTION_DESTINATION_WALLET_ID_KEY).toString()); _amount = transactionObject.value(TRANSACTION_AMOUNT_KEY).toInt(); -} \ No newline at end of file +} diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index f7f32d7480..8f6d9a2812 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -50,6 +50,8 @@ void Connection::sendReliablePacket(unique_ptr packet) { if (!_sendQueue) { // Lasily create send queue _sendQueue = SendQueue::create(_parentSocket, _destination); + + QObject::connect(_sendQueue.get(), &SendQueue::packetSent, this, &Connection::packetSent); } _sendQueue->queuePacket(move(packet)); diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 0c9fa831df..a86c8f817f 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -15,6 +15,8 @@ #include #include +#include + #include "ConnectionStats.h" #include "Constants.h" #include "LossList.h" @@ -29,8 +31,8 @@ class ControlPacket; class Packet; class Socket; -class Connection { - +class Connection : public QObject { + Q_OBJECT public: using SequenceNumberTimePair = std::pair; using SentACKMap = std::unordered_map; @@ -44,6 +46,9 @@ public: bool processReceivedSequenceNumber(SequenceNumber sequenceNumber); // returns indicates if this packet was a duplicate void processControl(std::unique_ptr controlPacket); + +signals: + void packetSent(); private: void sendACK(bool wasCausedBySyncTimeout = true); diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index d8bc1d52e6..8283a95e4f 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -21,8 +21,6 @@ #include #include -#include "UUID.h" - // If adding a new packet packetType, you can replace one marked usable or add at the end. // If you want the name of the packet packetType to be available for debugging or logging, update nameForPacketType() as well // This enum must hold 256 or fewer packet types (so the value is <= 255) since it is statically typed as a uint8_t diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index e6bc59c6cb..7d0edf9d7c 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -189,6 +189,8 @@ void SendQueue::loop() { Q_ASSERT_X(!nextPacket, "SendQueue::sendNextPacket()", "Overriden packet in sent list"); + emit packetSent(); + if (shouldSendSecondOfPair) { std::unique_ptr pairedPacket; @@ -209,6 +211,8 @@ void SendQueue::loop() { _sentPackets[pairedPacket->getSequenceNumber()].swap(pairedPacket); Q_ASSERT_X(!pairedPacket, "SendQueue::sendNextPacket()", "Overriden packet in sent list"); + + emit packetSent(); } } } diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 4b8bf6a11f..1b6b6181b8 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -61,6 +61,9 @@ public slots: void ack(SequenceNumber ack); void nak(SequenceNumber start, SequenceNumber end); void overrideNAKListFromPacket(ControlPacket& packet); + +signals: + void packetSent(); private slots: void loop(); diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index f8515affe6..99817f9790 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -85,11 +85,7 @@ qint64 Socket::writePacket(const Packet& packet, const HifiSockAddr& sockAddr) { qint64 Socket::writePacket(std::unique_ptr packet, const HifiSockAddr& sockAddr) { if (packet->isReliable()) { - auto it = _connectionsHash.find(sockAddr); - if (it == _connectionsHash.end()) { - it = _connectionsHash.insert(it, std::make_pair(sockAddr, new Connection(this, sockAddr, _ccFactory->create()))); - } - it->second->sendReliablePacket(std::move(packet)); + auto connection = findOrCreateConnection(sockAddr); return 0; } @@ -111,6 +107,16 @@ qint64 Socket::writeDatagram(const QByteArray& datagram, const HifiSockAddr& soc return bytesWritten; } +Connection* Socket::findOrCreateConnection(const HifiSockAddr& sockAddr) { + auto it = _connectionsHash.find(sockAddr); + + if (it == _connectionsHash.end()) { + it = _connectionsHash.insert(it, std::make_pair(sockAddr, new Connection(this, sockAddr, _ccFactory->create()))); + } + + return it->second; +} + void Socket::readPendingDatagrams() { while (_udpSocket.hasPendingDatagrams()) { // setup a HifiSockAddr to read into @@ -144,11 +150,8 @@ void Socket::readPendingDatagrams() { auto controlPacket = ControlPacket::fromReceivedPacket(std::move(buffer), packetSizeWithHeader, senderSockAddr); // move this control packet to the matching connection - auto it = _connectionsHash.find(senderSockAddr); - - if (it != _connectionsHash.end()) { - it->second->processControl(move(controlPacket)); - } + auto connection = findOrCreateConnection(senderSockAddr); + connection->processControl(move(controlPacket)); } else { // setup a Packet from the data we just read @@ -176,6 +179,13 @@ void Socket::readPendingDatagrams() { } } +void Socket::connectToSendSignal(const HifiSockAddr& destinationAddr, QObject* receiver, const char* slot) { + auto it = _connectionsHash.find(destinationAddr); + if (it != _connectionsHash.end()) { + connect(it->second, SIGNAL(packetSent()), receiver, slot); + } +} + void Socket::rateControlSync() { // enumerate our list of connections and ask each of them to send off periodic ACK packet for rate control diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index e923f4b307..8c8ecb32b2 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -60,6 +60,8 @@ public: { _unfilteredHandlers[senderSockAddr] = handler; } void setCongestionControlFactory(std::unique_ptr ccFactory); + + void connectToSendSignal(const HifiSockAddr& destinationAddr, QObject* receiver, const char* slot); private slots: void readPendingDatagrams(); @@ -67,6 +69,7 @@ private slots: private: void setSystemBufferSizes(); + Connection* findOrCreateConnection(const HifiSockAddr& sockAddr); QUdpSocket _udpSocket { this }; PacketFilterOperator _packetFilterOperator; diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 6bfac4ad26..810f68ccb6 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -14,6 +14,7 @@ #include #include +#include const QCommandLineOption PORT_OPTION { "p", "listening port for socket (defaults to random)", "port", 0 }; const QCommandLineOption TARGET_OPTION { @@ -45,6 +46,9 @@ UDTTest::UDTTest(int& argc, char** argv) : { parseArguments(); + // randomize the seed for packet size randomization + srand(time(NULL)); + _socket.bind(QHostAddress::LocalHost, _argumentParser.value(PORT_OPTION).toUInt()); qDebug() << "Test socket is listening on" << _socket.localPort(); @@ -53,7 +57,7 @@ UDTTest::UDTTest(int& argc, char** argv) : QString hostnamePortString = _argumentParser.value(TARGET_OPTION); QHostAddress address { hostnamePortString.left(hostnamePortString.indexOf(':')) }; - quint16 port { (quint16) hostnamePortString.right(hostnamePortString.indexOf(':') + 1).toUInt() }; + quint16 port { (quint16) hostnamePortString.mid(hostnamePortString.indexOf(':') + 1).toUInt() }; if (address.isNull() || port == 0) { qCritical() << "Could not parse an IP address and port combination from" << hostnamePortString << "-" << @@ -85,9 +89,9 @@ UDTTest::UDTTest(int& argc, char** argv) : if (_argumentParser.isSet(MAX_PACKET_SIZE)) { _maxPacketSize = _argumentParser.value(MAX_PACKET_SIZE).toInt(); - // if we don't have a min packet size we should make it zero, because we have a max + // if we don't have a min packet size we should make it 1, because we have a max if (customMinSize) { - _minPacketSize = 0; + _minPacketSize = 1; } } @@ -108,6 +112,10 @@ UDTTest::UDTTest(int& argc, char** argv) : if (_argumentParser.isSet(UNRELIABLE_PACKETS)) { _sendReliable = false; } + + if (!_target.isNull()) { + sendInitialPackets(); + } } void UDTTest::parseArguments() { @@ -133,3 +141,58 @@ void UDTTest::parseArguments() { Q_UNREACHABLE(); } } + +void UDTTest::sendInitialPackets() { + static const int NUM_INITIAL_PACKETS = 500; + + int numPackets = std::max(NUM_INITIAL_PACKETS, _maxSendPackets); + + for (int i = 0; i < numPackets; ++i) { + sendPacket(); + } + + if (numPackets == NUM_INITIAL_PACKETS) { + // we've put 500 initial packets in the queue, everytime we hear one has gone out we should add a new one + _socket.connectToSendSignal(_target, this, SLOT(refillPacket())); + } +} + +void UDTTest::sendPacket() { + + qDebug() << "Sending packet" << _totalQueuedPackets + 1; + + if (_maxSendPackets != -1 && _totalQueuedPackets > _maxSendPackets) { + // don't send more packets, we've hit max + return; + } + + if (_maxSendBytes != -1 && _totalQueuedBytes > _maxSendBytes) { + // don't send more packets, we've hit max + return; + } + + // we're good to send a new packet, construct it now + + // figure out what size the packet will be + int packetPayloadSize = 0; + + if (_minPacketSize == _maxPacketSize) { + // we know what size we want - figure out the payload size + packetPayloadSize = _maxPacketSize - udt::Packet::localHeaderSize(false); + } else { + // pick a random size in our range + int randomPacketSize = rand() % _maxPacketSize + _minPacketSize; + packetPayloadSize = randomPacketSize - udt::Packet::localHeaderSize(false); + } + + auto newPacket = udt::Packet::create(packetPayloadSize, _sendReliable); + + // queue or send this packet by calling write packet on the socket for our target + if (_sendReliable) { + _socket.writePacket(std::move(newPacket), _target); + } else { + _socket.writePacket(*newPacket, _target); + } + + ++_totalQueuedPackets; +} diff --git a/tools/udt-test/src/UDTTest.h b/tools/udt-test/src/UDTTest.h index 678a067114..050c5dc75a 100644 --- a/tools/udt-test/src/UDTTest.h +++ b/tools/udt-test/src/UDTTest.h @@ -21,22 +21,33 @@ #include class UDTTest : public QCoreApplication { + Q_OBJECT public: UDTTest(int& argc, char** argv); + +public slots: + void refillPacket() { sendPacket(); } // adds a new packet to the queue when we are told one is sent + private: void parseArguments(); + void sendInitialPackets(); // fills the queue with packets to start + void sendPacket(); // constructs and sends a packet according to the test parameters + QCommandLineParser _argumentParser; udt::Socket _socket; HifiSockAddr _target; // the target for sent packets - int _minPacketSize = udt::MAX_PACKET_SIZE_WITH_UDP_HEADER; - int _maxPacketSize = udt::MAX_PACKET_SIZE_WITH_UDP_HEADER; - int _maxSendBytes = -1; // the number of bytes to send to the target before stopping - int _maxSendPackets = -1; // the number of packets to send to the target before stopping + int _minPacketSize { udt::MAX_PACKET_SIZE }; + int _maxPacketSize { udt::MAX_PACKET_SIZE }; + int _maxSendBytes { -1 }; // the number of bytes to send to the target before stopping + int _maxSendPackets { -1 }; // the number of packets to send to the target before stopping - bool _sendReliable = true; // wether packets are sent reliably or unreliably + bool _sendReliable { true }; // wether packets are sent reliably or unreliably + + int _totalQueuedPackets { 0 }; // keeps track of the number of packets we have already queued + int _totalQueuedBytes { 0 }; // keeps track of the number of bytes we have already queued }; #endif // hifi_UDTTest_h From 0f1d1a16a71f9a6a97f036221c6cf5b1d73d469d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 30 Jul 2015 17:12:42 -0700 Subject: [PATCH 133/242] actually call sendReliablePacket for write of reliable --- libraries/networking/src/udt/Socket.cpp | 2 +- tools/udt-test/src/UDTTest.cpp | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 99817f9790..810eaa73e2 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -85,7 +85,7 @@ qint64 Socket::writePacket(const Packet& packet, const HifiSockAddr& sockAddr) { qint64 Socket::writePacket(std::unique_ptr packet, const HifiSockAddr& sockAddr) { if (packet->isReliable()) { - auto connection = findOrCreateConnection(sockAddr); + findOrCreateConnection(sockAddr)->sendReliablePacket(move(packet)); return 0; } diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 810f68ccb6..f6aad9ee8a 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -143,7 +143,7 @@ void UDTTest::parseArguments() { } void UDTTest::sendInitialPackets() { - static const int NUM_INITIAL_PACKETS = 500; + static const int NUM_INITIAL_PACKETS = 10; int numPackets = std::max(NUM_INITIAL_PACKETS, _maxSendPackets); @@ -187,6 +187,8 @@ void UDTTest::sendPacket() { auto newPacket = udt::Packet::create(packetPayloadSize, _sendReliable); + _totalQueuedBytes += newPacket->getDataSize(); + // queue or send this packet by calling write packet on the socket for our target if (_sendReliable) { _socket.writePacket(std::move(newPacket), _target); From 1724e3c6320fa621dfe68b8bb1b0204d39b18168 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 30 Jul 2015 17:17:03 -0700 Subject: [PATCH 134/242] fix a couple of bugs in SendQueue --- libraries/networking/src/udt/SendQueue.cpp | 94 ++++++++++++---------- 1 file changed, 50 insertions(+), 44 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 7d0edf9d7c..d0ec6ee38e 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -62,7 +62,9 @@ void SendQueue::run() { // We need to make sure this is called on the right thread if (thread() != QThread::currentThread()) { QMetaObject::invokeMethod(this, "run", Qt::QueuedConnection); + return; } + _isRunning = true; // This will loop and sleep to send packets @@ -161,60 +163,64 @@ void SendQueue::loop() { } // If there is no packet to resend, grab the next one in the list - if (!nextPacket) { + if (!nextPacket && _packets.size() > 0) { QWriteLocker locker(&_packetsLock); nextPacket.swap(_packets.front()); _packets.pop_front(); } - bool shouldSendSecondOfPair = false; - - if (!hasResend) { - // if we're not re-sending a packet then need to check if this should be a packet pair - sequenceNumber = getNextSequenceNumber(); + if (nextPacket) { + bool shouldSendSecondOfPair = false; - // the first packet in the pair is every 16 (rightmost 16 bits = 0) packets - if (((uint32_t) sequenceNumber & 0xF) == 0) { - shouldSendSecondOfPair = true; + if (!hasResend) { + // if we're not re-sending a packet then need to check if this should be a packet pair + sequenceNumber = getNextSequenceNumber(); + + // the first packet in the pair is every 16 (rightmost 16 bits = 0) packets + if (((uint32_t) sequenceNumber & 0xF) == 0) { + shouldSendSecondOfPair = true; + } + } + + // Write packet's sequence number and send it off + nextPacket->writeSequenceNumber(sequenceNumber); + sendPacket(*nextPacket); + + // Insert the packet we have just sent in the sent list + QWriteLocker locker(&_sentLock); + _sentPackets[nextPacket->getSequenceNumber()].swap(nextPacket); + Q_ASSERT_X(!nextPacket, + "SendQueue::sendNextPacket()", "Overriden packet in sent list"); + + emit packetSent(); + + if (shouldSendSecondOfPair) { + std::unique_ptr pairedPacket; + + // we've detected we should send the second packet in a pair, do that now before sleeping + { + QWriteLocker locker(&_packetsLock); + pairedPacket.swap(_packets.front()); + _packets.pop_front(); + } + + if (pairedPacket) { + // write this packet's sequence number and send it off + pairedPacket->writeSequenceNumber(getNextSequenceNumber()); + sendPacket(*pairedPacket); + + // add the paired packet to the sent list + QWriteLocker locker(&_sentLock); + _sentPackets[pairedPacket->getSequenceNumber()].swap(pairedPacket); + Q_ASSERT_X(!pairedPacket, + "SendQueue::sendNextPacket()", "Overriden packet in sent list"); + + emit packetSent(); + } } } - // Write packet's sequence number and send it off - nextPacket->writeSequenceNumber(sequenceNumber); - sendPacket(*nextPacket); - // Insert the packet we have just sent in the sent list - QWriteLocker locker(&_sentLock); - _sentPackets[nextPacket->getSequenceNumber()].swap(nextPacket); - Q_ASSERT_X(!nextPacket, - "SendQueue::sendNextPacket()", "Overriden packet in sent list"); - - emit packetSent(); - - if (shouldSendSecondOfPair) { - std::unique_ptr pairedPacket; - - // we've detected we should send the second packet in a pair, do that now before sleeping - { - QWriteLocker locker(&_packetsLock); - pairedPacket.swap(_packets.front()); - _packets.pop_front(); - } - - if (pairedPacket) { - // write this packet's sequence number and send it off - pairedPacket->writeSequenceNumber(getNextSequenceNumber()); - sendPacket(*pairedPacket); - - // add the paired packet to the sent list - QWriteLocker locker(&_sentLock); - _sentPackets[pairedPacket->getSequenceNumber()].swap(pairedPacket); - Q_ASSERT_X(!pairedPacket, - "SendQueue::sendNextPacket()", "Overriden packet in sent list"); - - emit packetSent(); - } - } } // since we're a while loop, give the thread a chance to process events From 367fa7b07b11fa828c18a38b3a214acab7648f4b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 30 Jul 2015 17:20:22 -0700 Subject: [PATCH 135/242] fixes for assert in Connection and ControlPacket create --- libraries/networking/src/udt/Connection.cpp | 2 +- libraries/networking/src/udt/ControlPacket.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 8f6d9a2812..e26f524c31 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -76,7 +76,7 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { auto currentTime = high_resolution_clock::now(); SequenceNumber nextACKNumber = nextACK(); - Q_ASSERT_X(nextACKNumber < _lastSentACK, "Connection::sendACK", "Sending lower ACK, something is wrong"); + Q_ASSERT_X(nextACKNumber >= _lastSentACK, "Connection::sendACK", "Sending lower ACK, something is wrong"); if (nextACKNumber == _lastSentACK) { // We already sent this ACK, but check if we should re-send it. diff --git a/libraries/networking/src/udt/ControlPacket.cpp b/libraries/networking/src/udt/ControlPacket.cpp index 7378b2f314..626e126c03 100644 --- a/libraries/networking/src/udt/ControlPacket.cpp +++ b/libraries/networking/src/udt/ControlPacket.cpp @@ -46,12 +46,12 @@ std::unique_ptr ControlPacket::create(Type type, qint64 size) { std::unique_ptr controlPacket; if (size == -1) { - return ControlPacket::create(type); + return std::unique_ptr(new ControlPacket(type)); } else { // Fail with invalid size Q_ASSERT(size >= 0); - return ControlPacket::create(type, size); + return std::unique_ptr(new ControlPacket(type, size)); } } From 805c5a6affce36fceb345b55ebdc6dd1256383b3 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 30 Jul 2015 17:22:25 -0700 Subject: [PATCH 136/242] reset initial to 500, remove debug --- tools/udt-test/src/UDTTest.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index f6aad9ee8a..8d9655437d 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -143,7 +143,7 @@ void UDTTest::parseArguments() { } void UDTTest::sendInitialPackets() { - static const int NUM_INITIAL_PACKETS = 10; + static const int NUM_INITIAL_PACKETS = 500; int numPackets = std::max(NUM_INITIAL_PACKETS, _maxSendPackets); @@ -159,8 +159,6 @@ void UDTTest::sendInitialPackets() { void UDTTest::sendPacket() { - qDebug() << "Sending packet" << _totalQueuedPackets + 1; - if (_maxSendPackets != -1 && _totalQueuedPackets > _maxSendPackets) { // don't send more packets, we've hit max return; From 8e786cb95313dde784b70fa5e6786ef6d156ca44 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 30 Jul 2015 17:40:51 -0700 Subject: [PATCH 137/242] repairs while testing with UDTTest --- libraries/networking/src/udt/Connection.cpp | 30 +++++++++++---------- libraries/networking/src/udt/SendQueue.cpp | 25 ++++++++--------- libraries/networking/src/udt/SendQueue.h | 3 +-- tools/udt-test/src/UDTTest.cpp | 2 ++ 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index e26f524c31..0d0140dbca 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -203,20 +203,22 @@ void Connection::sendNAK(SequenceNumber sequenceNumberRecieved) { } void Connection::sendTimeoutNAK() { - // construct a NAK packet that will hold all of the lost sequence numbers - // TODO size is wrong, fix it. - auto lossListPacket = ControlPacket::create(ControlPacket::TimeoutNAK, _lossList.getLength() * sizeof(SequenceNumber)); - - // Pack in the lost sequence numbers - _lossList.write(*lossListPacket); - - // have our SendQueue send off this control packet - _sendQueue->sendPacket(*lossListPacket); - - // record this as the last NAK time - _lastNAKTime = high_resolution_clock::now(); - - _stats.recordSentTimeoutNAK(); + if (_lossList.getLength() > 0) { + // construct a NAK packet that will hold all of the lost sequence numbers + // TODO size is wrong, fix it. + auto lossListPacket = ControlPacket::create(ControlPacket::TimeoutNAK, _lossList.getLength() * sizeof(SequenceNumber)); + + // Pack in the lost sequence numbers + _lossList.write(*lossListPacket); + + // have our SendQueue send off this control packet + _sendQueue->sendPacket(*lossListPacket); + + // record this as the last NAK time + _lastNAKTime = high_resolution_clock::now(); + + _stats.recordSentTimeoutNAK(); + } } SequenceNumber Connection::nextACK() const { diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index d0ec6ee38e..66051ad1c4 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -58,19 +58,6 @@ void SendQueue::queuePacket(std::unique_ptr packet) { } } -void SendQueue::run() { - // We need to make sure this is called on the right thread - if (thread() != QThread::currentThread()) { - QMetaObject::invokeMethod(this, "run", Qt::QueuedConnection); - return; - } - - _isRunning = true; - - // This will loop and sleep to send packets - loop(); -} - void SendQueue::stop() { _isRunning = false; } @@ -128,7 +115,16 @@ SequenceNumber SendQueue::getNextSequenceNumber() { return _currentSequenceNumber; } -void SendQueue::loop() { +void SendQueue::run() { + + // We need to make sure this is called on the right thread + if (thread() != QThread::currentThread()) { + QMetaObject::invokeMethod(this, "run", Qt::QueuedConnection); + return; + } + + _isRunning = true; + while (_isRunning) { // Record timing _lastSendTimestamp = high_resolution_clock::now(); @@ -170,6 +166,7 @@ void SendQueue::loop() { } if (nextPacket) { + qDebug() << "the next packet is" << nextPacket->getDataSize() << "bytes"; bool shouldSendSecondOfPair = false; if (!hasResend) { diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 1b6b6181b8..3915414e1f 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -55,7 +55,6 @@ public: void sendPacket(const BasePacket& packet); public slots: - void run(); void stop(); void ack(SequenceNumber ack); @@ -66,7 +65,7 @@ signals: void packetSent(); private slots: - void loop(); + void run(); private: SendQueue(Socket* socket, HifiSockAddr dest); diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 8d9655437d..2bfcd48a76 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -66,6 +66,7 @@ UDTTest::UDTTest(int& argc, char** argv) : QMetaObject::invokeMethod(this, "quit", Qt::QueuedConnection); } else { _target = HifiSockAddr(address, port); + qDebug() << "Packets will be sent to" << _target; } } @@ -184,6 +185,7 @@ void UDTTest::sendPacket() { } auto newPacket = udt::Packet::create(packetPayloadSize, _sendReliable); + newPacket->setPayloadSize(packetPayloadSize); _totalQueuedBytes += newPacket->getDataSize(); From a38e7b0431f55958818d3855ff0a9b61838002a9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 30 Jul 2015 17:59:27 -0700 Subject: [PATCH 138/242] cleanup SendQueue start and UDTTest bind --- libraries/networking/src/udt/SendQueue.cpp | 43 ++++++++++------------ libraries/networking/src/udt/SendQueue.h | 4 +- tools/udt-test/src/UDTTest.cpp | 2 +- 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 66051ad1c4..5e3d1cc6e4 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -31,6 +31,9 @@ std::unique_ptr SendQueue::create(Socket* socket, HifiSockAddr dest) // Setup queue private thread QThread* thread = new QThread(); thread->setObjectName("Networking: SendQueue " + dest.objectName()); // Name thread for easier debug + + connect(thread, &QThread::started, queue.get(), &SendQueue::run); + connect(queue.get(), &QObject::destroyed, thread, &QThread::quit); // Thread auto cleanup connect(thread, &QThread::finished, thread, &QThread::deleteLater); // Thread auto cleanup @@ -45,7 +48,7 @@ SendQueue::SendQueue(Socket* socket, HifiSockAddr dest) : _socket(socket), _destination(dest) { - _packetSendPeriod = DEFAULT_SEND_PERIOD; + } void SendQueue::queuePacket(std::unique_ptr packet) { @@ -53,8 +56,8 @@ void SendQueue::queuePacket(std::unique_ptr packet) { QWriteLocker locker(&_packetsLock); _packets.push_back(std::move(packet)); } - if (!_isRunning) { - run(); + if (!this->thread()->isRunning()) { + this->thread()->start(); } } @@ -116,13 +119,6 @@ SequenceNumber SendQueue::getNextSequenceNumber() { } void SendQueue::run() { - - // We need to make sure this is called on the right thread - if (thread() != QThread::currentThread()) { - QMetaObject::invokeMethod(this, "run", Qt::QueuedConnection); - return; - } - _isRunning = true; while (_isRunning) { @@ -166,7 +162,6 @@ void SendQueue::run() { } if (nextPacket) { - qDebug() << "the next packet is" << nextPacket->getDataSize() << "bytes"; bool shouldSendSecondOfPair = false; if (!hasResend) { @@ -183,11 +178,13 @@ void SendQueue::run() { nextPacket->writeSequenceNumber(sequenceNumber); sendPacket(*nextPacket); - // Insert the packet we have just sent in the sent list - QWriteLocker locker(&_sentLock); - _sentPackets[nextPacket->getSequenceNumber()].swap(nextPacket); - Q_ASSERT_X(!nextPacket, - "SendQueue::sendNextPacket()", "Overriden packet in sent list"); + { + // Insert the packet we have just sent in the sent list + QWriteLocker locker(&_sentLock); + _sentPackets[nextPacket->getSequenceNumber()].swap(nextPacket); + Q_ASSERT_X(!nextPacket, + "SendQueue::sendNextPacket()", "Overriden packet in sent list"); + } emit packetSent(); @@ -206,18 +203,18 @@ void SendQueue::run() { pairedPacket->writeSequenceNumber(getNextSequenceNumber()); sendPacket(*pairedPacket); - // add the paired packet to the sent list - QWriteLocker locker(&_sentLock); - _sentPackets[pairedPacket->getSequenceNumber()].swap(pairedPacket); - Q_ASSERT_X(!pairedPacket, - "SendQueue::sendNextPacket()", "Overriden packet in sent list"); + { + // add the paired packet to the sent list + QWriteLocker locker(&_sentLock); + _sentPackets[pairedPacket->getSequenceNumber()].swap(pairedPacket); + Q_ASSERT_X(!pairedPacket, + "SendQueue::sendNextPacket()", "Overriden packet in sent list"); + } emit packetSent(); } } } - - } // since we're a while loop, give the thread a chance to process events diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 3915414e1f..f8ffa91c96 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -37,7 +37,7 @@ class SendQueue : public QObject { Q_OBJECT public: - static const int DEFAULT_SEND_PERIOD = 16 * 1000; // 16ms, in microseconds + static const int DEFAULT_SEND_PERIOD = 1; // in microseconds static std::unique_ptr create(Socket* socket, HifiSockAddr dest); @@ -86,7 +86,7 @@ private: SequenceNumber _currentSequenceNumber; // Last sequence number sent out std::atomic _atomicCurrentSequenceNumber; // Atomic for last sequence number sent out - std::atomic _packetSendPeriod { 0 }; // Interval between two packet send event in microseconds + std::atomic _packetSendPeriod { DEFAULT_SEND_PERIOD }; // Interval between two packet send event in microseconds std::chrono::high_resolution_clock::time_point _lastSendTimestamp; // Record last time of packet departure std::atomic _isRunning { false }; diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 2bfcd48a76..9f149dce52 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -49,7 +49,7 @@ UDTTest::UDTTest(int& argc, char** argv) : // randomize the seed for packet size randomization srand(time(NULL)); - _socket.bind(QHostAddress::LocalHost, _argumentParser.value(PORT_OPTION).toUInt()); + _socket.bind(QHostAddress::AnyIPv4, _argumentParser.value(PORT_OPTION).toUInt()); qDebug() << "Test socket is listening on" << _socket.localPort(); if (_argumentParser.isSet(TARGET_OPTION)) { From a714f8081d963c13028b29e236712080725dd4f9 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 30 Jul 2015 18:10:59 -0700 Subject: [PATCH 139/242] Fix a couple networking bugs --- libraries/networking/src/udt/LossList.cpp | 2 +- libraries/networking/src/udt/SequenceNumber.h | 4 ++-- libraries/networking/src/udt/Socket.cpp | 8 ++------ 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/libraries/networking/src/udt/LossList.cpp b/libraries/networking/src/udt/LossList.cpp index 32812be10b..90399e319d 100644 --- a/libraries/networking/src/udt/LossList.cpp +++ b/libraries/networking/src/udt/LossList.cpp @@ -17,7 +17,7 @@ using namespace udt; using namespace std; void LossList::append(SequenceNumber seq) { - assert(_lossList.back().second < seq); + assert(_lossList.empty() || _lossList.back().second < seq); if (getLength() > 0 && _lossList.back().second + 1 == seq) { ++_lossList.back().second; diff --git a/libraries/networking/src/udt/SequenceNumber.h b/libraries/networking/src/udt/SequenceNumber.h index d5b61c0a95..c77a14edf9 100644 --- a/libraries/networking/src/udt/SequenceNumber.h +++ b/libraries/networking/src/udt/SequenceNumber.h @@ -45,12 +45,12 @@ public: } inline SequenceNumber operator++(int) { SequenceNumber before = *this; - (*this)++; + ++(*this); return before; } inline SequenceNumber operator--(int) { SequenceNumber before = *this; - (*this)--; + --(*this); return before; } diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 810eaa73e2..ba0e196669 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -162,12 +162,8 @@ void Socket::readPendingDatagrams() { if (packet->isReliable()) { // if this was a reliable packet then signal the matching connection with the sequence number - // assuming it exists - auto it = _connectionsHash.find(senderSockAddr); - - if (it != _connectionsHash.end()) { - it->second->processReceivedSequenceNumber(packet->getSequenceNumber()); - } + auto connection = findOrCreateConnection(senderSockAddr); + connection->processReceivedSequenceNumber(packet->getSequenceNumber()); } if (_packetHandler) { From 98385ce8cd0a1d9e6a1661a358b704224697b464 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 30 Jul 2015 18:14:54 -0700 Subject: [PATCH 140/242] default sequence numbers to 0 --- libraries/networking/src/udt/Connection.h | 8 ++++---- libraries/networking/src/udt/SendQueue.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index a86c8f817f..606359e6f5 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -77,12 +77,12 @@ private: std::chrono::high_resolution_clock::time_point _lastNAKTime; LossList _lossList; // List of all missing packets - SequenceNumber _lastReceivedSequenceNumber { SequenceNumber::MAX }; // The largest sequence number received from the peer - SequenceNumber _lastReceivedACK { SequenceNumber::MAX }; // The last ACK received - SequenceNumber _lastReceivedAcknowledgedACK { SequenceNumber::MAX }; // The last sent ACK that has been acknowledged via an ACK2 from the peer + SequenceNumber _lastReceivedSequenceNumber; // The largest sequence number received from the peer + SequenceNumber _lastReceivedACK; // The last ACK received + SequenceNumber _lastReceivedAcknowledgedACK; // The last sent ACK that has been acknowledged via an ACK2 from the peer SequenceNumber _currentACKSubSequenceNumber; // The current ACK sub-sequence number (used for Acknowledgment of ACKs) - SequenceNumber _lastSentACK { SequenceNumber::MAX }; // The last sent ACK + SequenceNumber _lastSentACK; // The last sent ACK SequenceNumber _lastSentACK2; // The last sent ACK sub-sequence number in an ACK2 int32_t _rtt; // RTT, in microseconds diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index f8ffa91c96..24bdea27df 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -84,7 +84,7 @@ private: std::atomic _lastACKSequenceNumber; // Last ACKed sequence number SequenceNumber _currentSequenceNumber; // Last sequence number sent out - std::atomic _atomicCurrentSequenceNumber; // Atomic for last sequence number sent out + std::atomic _atomicCurrentSequenceNumber;// Atomic for last sequence number sent out std::atomic _packetSendPeriod { DEFAULT_SEND_PERIOD }; // Interval between two packet send event in microseconds std::chrono::high_resolution_clock::time_point _lastSendTimestamp; // Record last time of packet departure From e81e1c42b0eb9b48db64f03e09b08b873edf131c Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 30 Jul 2015 18:22:27 -0700 Subject: [PATCH 141/242] Add send queue getter --- libraries/networking/src/udt/Connection.cpp | 33 +++++++++++---------- libraries/networking/src/udt/Connection.h | 2 ++ 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 0d0140dbca..474088671f 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -44,9 +44,7 @@ Connection::~Connection() { } } -void Connection::sendReliablePacket(unique_ptr packet) { - Q_ASSERT_X(packet->isReliable(), "Connection::send", "Trying to send an unreliable packet reliably."); - +SendQueue& Connection::getSendQueue() { if (!_sendQueue) { // Lasily create send queue _sendQueue = SendQueue::create(_parentSocket, _destination); @@ -54,7 +52,12 @@ void Connection::sendReliablePacket(unique_ptr packet) { QObject::connect(_sendQueue.get(), &SendQueue::packetSent, this, &Connection::packetSent); } - _sendQueue->queuePacket(move(packet)); + return *_sendQueue; +} + +void Connection::sendReliablePacket(unique_ptr packet) { + Q_ASSERT_X(packet->isReliable(), "Connection::send", "Trying to send an unreliable packet reliably."); + getSendQueue().queuePacket(move(packet)); } void Connection::sync() { @@ -129,7 +132,7 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { } // have the send queue send off our packet - _sendQueue->sendPacket(*ackPacket); + getSendQueue().sendPacket(*ackPacket); // write this ACK to the map of sent ACKs _sentACKs[_currentACKSubSequenceNumber] = { nextACKNumber, currentTime }; @@ -159,7 +162,7 @@ void Connection::sendLightACK() { lightACKPacket->writePrimitive(nextACKNumber); // have the send queue send off our packet immediately - _sendQueue->sendPacket(*lightACKPacket); + getSendQueue().sendPacket(*lightACKPacket); _stats.recordSentLightACK(); } @@ -194,7 +197,7 @@ void Connection::sendNAK(SequenceNumber sequenceNumberRecieved) { } // have the send queue send off our packet immediately - _sendQueue->sendPacket(*lossReport); + getSendQueue().sendPacket(*lossReport); // record our last NAK time _lastNAKTime = high_resolution_clock::now(); @@ -212,7 +215,7 @@ void Connection::sendTimeoutNAK() { _lossList.write(*lossListPacket); // have our SendQueue send off this control packet - _sendQueue->sendPacket(*lossListPacket); + getSendQueue().sendPacket(*lossListPacket); // record this as the last NAK time _lastNAKTime = high_resolution_clock::now(); @@ -337,7 +340,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { controlPacket->readPrimitive(&ack); // validate that this isn't a BS ACK - if (ack > _sendQueue->getCurrentSequenceNumber()) { + if (ack > getSendQueue().getCurrentSequenceNumber()) { // in UDT they specifically break the connection here - do we want to do anything? Q_ASSERT_X(true, "Connection::processACK", "ACK recieved higher than largest sent sequence number"); return; @@ -363,7 +366,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { _lastReceivedACK = ack; // ACK the send queue so it knows what was received - _sendQueue->ack(ack); + getSendQueue().ack(ack); // update the RTT @@ -455,7 +458,7 @@ void Connection::processNAK(std::unique_ptr controlPacket) { } // send that off to the send queue so it knows there was loss - _sendQueue->nak(start, end); + getSendQueue().nak(start, end); // give the loss to the congestion control object and update the send queue parameters updateCongestionControlAndSendQueue([this, start, end](){ @@ -467,7 +470,7 @@ void Connection::processNAK(std::unique_ptr controlPacket) { void Connection::processTimeoutNAK(std::unique_ptr controlPacket) { // Override SendQueue's LossList with the timeout NAK list - _sendQueue->overrideNAKListFromPacket(*controlPacket); + getSendQueue().overrideNAKListFromPacket(*controlPacket); // we don't tell the congestion control object there was loss here - this matches UDTs implementation // a possible improvement would be to tell it which new loss this timeout packet told us about @@ -501,12 +504,12 @@ int Connection::estimatedTimeout() const { void Connection::updateCongestionControlAndSendQueue(std::function congestionCallback) { // update the last sent sequence number in congestion control - _congestionControl->setSendCurrentSequenceNumber(_sendQueue->getCurrentSequenceNumber()); + _congestionControl->setSendCurrentSequenceNumber(getSendQueue().getCurrentSequenceNumber()); // fire congestion control callback congestionCallback(); // now that we've update the congestion control, update the packet send period and flow window size - _sendQueue->setPacketSendPeriod(_congestionControl->_packetSendPeriod); - _sendQueue->setFlowWindowSize(std::min(_flowWindowSize, (int) _congestionControl->_congestionWindowSize)); + getSendQueue().setPacketSendPeriod(_congestionControl->_packetSendPeriod); + getSendQueue().setFlowWindowSize(std::min(_flowWindowSize, (int) _congestionControl->_congestionWindowSize)); } diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 606359e6f5..b006003526 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -63,6 +63,8 @@ private: void processNAK(std::unique_ptr controlPacket); void processTimeoutNAK(std::unique_ptr controlPacket); + + SendQueue& getSendQueue(); SequenceNumber nextACK() const; void updateRTT(int rtt); From 6c4a36fb7f877373ba7fbc111125b85fcf93a565 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 30 Jul 2015 18:22:43 -0700 Subject: [PATCH 142/242] lock then check size --- libraries/networking/src/udt/SendQueue.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 5e3d1cc6e4..249cb6e069 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -155,10 +155,13 @@ void SendQueue::run() { } // If there is no packet to resend, grab the next one in the list - if (!nextPacket && _packets.size() > 0) { + if (!nextPacket) { QWriteLocker locker(&_packetsLock); - nextPacket.swap(_packets.front()); - _packets.pop_front(); + + if (_packets.size() > 0) { + nextPacket.swap(_packets.front()); + _packets.pop_front(); + } } if (nextPacket) { @@ -194,8 +197,11 @@ void SendQueue::run() { // we've detected we should send the second packet in a pair, do that now before sleeping { QWriteLocker locker(&_packetsLock); - pairedPacket.swap(_packets.front()); - _packets.pop_front(); + + if (_packets.size() > 0) { + pairedPacket.swap(_packets.front()); + _packets.pop_front(); + } } if (pairedPacket) { From 0ea1bb578f28591141a68f4c81f9a2cadc131c10 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 08:57:25 -0700 Subject: [PATCH 143/242] support max bandwidth in congestion control --- libraries/networking/src/udt/CongestionControl.cpp | 8 ++++++++ libraries/networking/src/udt/CongestionControl.h | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 1450e60930..65f358e0bf 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -112,6 +112,14 @@ void DefaultCC::onACK(SequenceNumber ackNum) { } _packetSendPeriod = (_packetSendPeriod * synInterval()) / (_packetSendPeriod * increase + synInterval()); + + if (_maxBandwidth > 0) { + // anytime the packet send period is about to be increased, make sure it stays below the minimum period, + // calculated based on the maximum desired bandwidth + int minPacketSendPeriod = USECS_PER_SECOND / (double(_maxBandwidth) / _mss); + _packetSendPeriod = std::max(_packetSendPeriod, (double) minPacketSendPeriod); + } + } void DefaultCC::onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) { diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index f8f4860a89..e5af8acac8 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -47,15 +47,16 @@ protected: void setMSS(int mss) { _mss = mss; } void setMaxCongestionWindowSize(int window) { _maxCongestionWindowSize = window; } void setBandwidth(int bandwidth) { _bandwidth = bandwidth; } + void setMaxBandwidth(int maxBandwidth) { _maxBandwidth = maxBandwidth; } void setSendCurrentSequenceNumber(SequenceNumber seqNum) { _sendCurrSeqNum = seqNum; } void setReceiveRate(int rate) { _receiveRate = rate; } void setRTT(int rtt) { _rtt = rtt; } - double _packetSendPeriod { 1.0 }; // Packet sending period, in microseconds double _congestionWindowSize { 16.0 }; // Congestion window size, in packets int _bandwidth { 0 }; // estimated bandwidth, packets per second + int _maxBandwidth { -1 }; // Maximum desired bandwidth, packets per second double _maxCongestionWindowSize { 0.0 }; // maximum cwnd size, in packets int _mss { 0 }; // Maximum Packet Size, including all packet headers From c2d39f0f6afe5e1c40009ffccc194ead8c016896 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 09:02:43 -0700 Subject: [PATCH 144/242] make packet send period update respect max bandwidth --- .../networking/src/udt/CongestionControl.cpp | 21 +++++++++++-------- .../networking/src/udt/CongestionControl.h | 1 + 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 65f358e0bf..984d520d07 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -17,6 +17,17 @@ using namespace std::chrono; static const double USECS_PER_SECOND = 1000000.0; +void CongestionControl::setPacketSendPeriod(double newSendPeriod) { + if (_maxBandwidth > 0) { + // anytime the packet send period is about to be increased, make sure it stays below the minimum period, + // calculated based on the maximum desired bandwidth + int minPacketSendPeriod = USECS_PER_SECOND / (double(_maxBandwidth) / _mss); + _packetSendPeriod = std::max(newSendPeriod, (double) minPacketSendPeriod); + } else { + _packetSendPeriod = newSendPeriod; + } +} + void DefaultCC::init() { _lastRCTime = high_resolution_clock::now(); @@ -111,15 +122,7 @@ void DefaultCC::onACK(SequenceNumber ackNum) { } } - _packetSendPeriod = (_packetSendPeriod * synInterval()) / (_packetSendPeriod * increase + synInterval()); - - if (_maxBandwidth > 0) { - // anytime the packet send period is about to be increased, make sure it stays below the minimum period, - // calculated based on the maximum desired bandwidth - int minPacketSendPeriod = USECS_PER_SECOND / (double(_maxBandwidth) / _mss); - _packetSendPeriod = std::max(_packetSendPeriod, (double) minPacketSendPeriod); - } - + setPacketSendPeriod((_packetSendPeriod * synInterval()) / (_packetSendPeriod * increase + synInterval())); } void DefaultCC::onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) { diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index e5af8acac8..75970ed2fd 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -51,6 +51,7 @@ protected: void setSendCurrentSequenceNumber(SequenceNumber seqNum) { _sendCurrSeqNum = seqNum; } void setReceiveRate(int rate) { _receiveRate = rate; } void setRTT(int rtt) { _rtt = rtt; } + void setPacketSendPeriod(double newSendPeriod); // call this internally to ensure send period doesn't go past max bandwidth double _packetSendPeriod { 1.0 }; // Packet sending period, in microseconds double _congestionWindowSize { 16.0 }; // Congestion window size, in packets From f2ab2fb08a0112a9fe053262fd8f696a0f1318ca Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 09:36:55 -0700 Subject: [PATCH 145/242] start with a congestion window of 16, not 25 grand --- libraries/networking/src/udt/Connection.cpp | 4 ++++ libraries/networking/src/udt/SendQueue.h | 6 ++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 0d0140dbca..b315611cf3 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -52,6 +52,10 @@ void Connection::sendReliablePacket(unique_ptr packet) { _sendQueue = SendQueue::create(_parentSocket, _destination); QObject::connect(_sendQueue.get(), &SendQueue::packetSent, this, &Connection::packetSent); + + // set defaults on the send queue from our congestion control object + _sendQueue->setPacketSendPeriod(_congestionControl->_packetSendPeriod); + _sendQueue->setFlowWindowSize(std::min(_flowWindowSize, (int) _congestionControl->_congestionWindowSize)); } _sendQueue->queuePacket(move(packet)); diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 24bdea27df..9c8d6878c0 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -37,8 +37,6 @@ class SendQueue : public QObject { Q_OBJECT public: - static const int DEFAULT_SEND_PERIOD = 1; // in microseconds - static std::unique_ptr create(Socket* socket, HifiSockAddr dest); void queuePacket(std::unique_ptr packet); @@ -86,11 +84,11 @@ private: SequenceNumber _currentSequenceNumber; // Last sequence number sent out std::atomic _atomicCurrentSequenceNumber;// Atomic for last sequence number sent out - std::atomic _packetSendPeriod { DEFAULT_SEND_PERIOD }; // Interval between two packet send event in microseconds + std::atomic _packetSendPeriod; // Interval between two packet send event in microseconds, set from CC std::chrono::high_resolution_clock::time_point _lastSendTimestamp; // Record last time of packet departure std::atomic _isRunning { false }; - std::atomic _flowWindowSize { udt::MAX_PACKETS_IN_FLIGHT }; // Flow control window size (number of packets that can be on wire) + std::atomic _flowWindowSize; // Flow control window size (number of packets that can be on wire) - set from CC mutable QReadWriteLock _naksLock; // Protects the naks list. LossList _naks; // Sequence numbers of packets to resend From 98a53cbd723b484a2d223f2cbf184fd486ddf2dd Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 09:56:50 -0700 Subject: [PATCH 146/242] don't require a sendQueue for control packet sending --- libraries/networking/src/udt/Connection.cpp | 18 ++++++++++-------- libraries/networking/src/udt/Connection.h | 6 +++--- libraries/networking/src/udt/SendQueue.cpp | 8 ++++---- libraries/networking/src/udt/SendQueue.h | 5 ++--- libraries/networking/src/udt/Socket.cpp | 10 ++++++++++ libraries/networking/src/udt/Socket.h | 1 + 6 files changed, 30 insertions(+), 18 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index b315611cf3..b6b3f9f12a 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -30,6 +30,8 @@ Connection::Connection(Socket* parentSocket, HifiSockAddr destination, unique_pt _destination(destination), _congestionControl(move(congestionControl)) { + Q_ASSERT_X(socket, "Connection::Connection", "Must be called with a valid Socket*"); + // setup default SYN, RTT and RTT Variance based on the SYN interval in CongestionControl object _synInterval = _congestionControl->synInterval(); _rtt = _synInterval * 10; @@ -132,8 +134,8 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { lastACKSendTime = high_resolution_clock::now(); } - // have the send queue send off our packet - _sendQueue->sendPacket(*ackPacket); + // have the socket send off our packet + _parentSocket->writeBasePacket(*ackPacket, _destination); // write this ACK to the map of sent ACKs _sentACKs[_currentACKSubSequenceNumber] = { nextACKNumber, currentTime }; @@ -162,8 +164,8 @@ void Connection::sendLightACK() { // pack in the ACK lightACKPacket->writePrimitive(nextACKNumber); - // have the send queue send off our packet immediately - _sendQueue->sendPacket(*lightACKPacket); + // have the socket send off our packet immediately + _parentSocket->writeBasePacket(*lightACKPacket, _destination); _stats.recordSentLightACK(); } @@ -197,8 +199,8 @@ void Connection::sendNAK(SequenceNumber sequenceNumberRecieved) { lossReport->writePrimitive(sequenceNumberRecieved - 1); } - // have the send queue send off our packet immediately - _sendQueue->sendPacket(*lossReport); + // have the parent socket send off our packet immediately + _parentSocket->writeBasePacket(*lossReport, _destination); // record our last NAK time _lastNAKTime = high_resolution_clock::now(); @@ -215,8 +217,8 @@ void Connection::sendTimeoutNAK() { // Pack in the lost sequence numbers _lossList.write(*lossListPacket); - // have our SendQueue send off this control packet - _sendQueue->sendPacket(*lossListPacket); + // have our parent socket send off this control packet + _parentSocket->writeBasePacket(*lossListPacket, _destination); // record this as the last NAK time _lastNAKTime = high_resolution_clock::now(); diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 606359e6f5..71cae880d0 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -98,11 +98,11 @@ private: HifiSockAddr _destination; PacketTimeWindow _receiveWindow { 16, 64 }; // Window of interval between packets (16) and probes (64) for bandwidth and receive speed - - std::unique_ptr _sendQueue; - + std::unique_ptr _congestionControl; + std::unique_ptr _sendQueue; + // Data packet stat collection int _totalReceivedDataPackets { 0 }; int _packetsSinceACK { 0 }; // The number of packets that have been received during the current ACK interval diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 249cb6e069..c092d089c4 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -28,6 +28,8 @@ using namespace std::chrono; std::unique_ptr SendQueue::create(Socket* socket, HifiSockAddr dest) { auto queue = std::unique_ptr(new SendQueue(socket, dest)); + Q_ASSERT_X(socket, "SendQueue::create", "Must be called with a valid Socket*"); + // Setup queue private thread QThread* thread = new QThread(); thread->setObjectName("Networking: SendQueue " + dest.objectName()); // Name thread for easier debug @@ -65,10 +67,8 @@ void SendQueue::stop() { _isRunning = false; } -void SendQueue::sendPacket(const BasePacket& packet) { - if (_socket) { - _socket->writeDatagram(packet.getData(), packet.getDataSize(), _destination); - } +void SendQueue::sendPacket(const Packet& packet) { + _socket->writeDatagram(packet.getData(), packet.getDataSize(), _destination); } void SendQueue::ack(SequenceNumber ack) { diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 9c8d6878c0..4a4cf1ffea 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -49,9 +49,6 @@ public: int getPacketSendPeriod() const { return _packetSendPeriod; } void setPacketSendPeriod(int newPeriod) { _packetSendPeriod = newPeriod; } - // Send a packet through the socket - void sendPacket(const BasePacket& packet); - public slots: void stop(); @@ -70,6 +67,8 @@ private: SendQueue(SendQueue& other) = delete; SendQueue(SendQueue&& other) = delete; + void sendPacket(const Packet& packet); + // Increments current sequence number and return it SequenceNumber getNextSequenceNumber(); diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index ba0e196669..99cdb06b29 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -74,6 +74,16 @@ void Socket::setSystemBufferSizes() { } } +qint64 Socket::writeBasePacket(const udt::BasePacket& packet, const HifiSockAddr &sockAddr) { + // Since this is a base packet we have no way to know if this is reliable or not - we just fire it off + + // this should not be called with an instance of Packet + Q_ASSERT_X(!dynamic_cast(&packet), + "Socket::writeBasePacket", "Cannot send a Packet/NLPacket via writeBasePacket"); + + return writeDatagram(packet.getData(), packet.getDataSize(), sockAddr); +} + qint64 Socket::writePacket(const Packet& packet, const HifiSockAddr& sockAddr) { Q_ASSERT_X(!packet.isReliable(), "Socket::writePacket", "Cannot send a reliable packet unreliably"); diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 8c8ecb32b2..4f947e4043 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -45,6 +45,7 @@ public: quint16 localPort() const { return _udpSocket.localPort(); } // Simple functions writing to the socket with no processing + qint64 writeBasePacket(const BasePacket& packet, const HifiSockAddr& sockAddr); qint64 writePacket(const Packet& packet, const HifiSockAddr& sockAddr); qint64 writePacket(std::unique_ptr packet, const HifiSockAddr& sockAddr); qint64 writeDatagram(const char* data, qint64 size, const HifiSockAddr& sockAddr); From da7a8f6e2faecdc827171d062d56cb91b2d186cd Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 31 Jul 2015 10:41:16 -0700 Subject: [PATCH 147/242] Added new connection stats --- .../networking/src/udt/ConnectionStats.cpp | 55 +++++++++++++++++-- .../networking/src/udt/ConnectionStats.h | 24 +++++++- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/udt/ConnectionStats.cpp b/libraries/networking/src/udt/ConnectionStats.cpp index 394448afec..0b53e35618 100644 --- a/libraries/networking/src/udt/ConnectionStats.cpp +++ b/libraries/networking/src/udt/ConnectionStats.cpp @@ -21,8 +21,8 @@ ConnectionStats::ConnectionStats() { } ConnectionStats::Stats ConnectionStats::sample() { - Stats sample; - std::swap(sample, _currentSample); + Stats sample = _currentSample; + _currentSample = Stats(); auto now = duration_cast(high_resolution_clock::now().time_since_epoch()); sample.endTime = now; @@ -81,12 +81,57 @@ void ConnectionStats::recordReceivedTimeoutNAK() { ++_total.receivedTimeoutNAKs; } -void ConnectionStats::recordSentPackets() { + +void ConnectionStats::recordSentPackets(int payload, int total) { ++_currentSample.sentPackets; ++_total.sentPackets; + + _currentSample.sentUtilBytes += payload; + _total.sentUtilBytes += payload; + + _currentSample.sentBytes += total; + _total.sentBytes += total; } -void ConnectionStats::recordReceivedPackets() { +void ConnectionStats::recordReceivedPackets(int payload, int total) { ++_currentSample.recievedPackets; ++_total.recievedPackets; -} \ No newline at end of file + + _currentSample.recievedUtilBytes += payload; + _total.recievedUtilBytes += payload; + + _currentSample.sentBytes += total; + _total.recievedBytes += total; +} + +void ConnectionStats::recordUnreliableSentPackets(int payload, int total) { + ++_currentSample.sentUnreliablePackets; + ++_total.sentUnreliablePackets; + + _currentSample.sentUnreliableUtilBytes += payload; + _total.sentUnreliableUtilBytes += payload; + + _currentSample.sentUnreliableBytes += total; + _total.sentUnreliableBytes += total; +} + +void ConnectionStats::recordUnreliableReceivedPackets(int payload, int total) { + ++_currentSample.recievedUnreliablePackets; + ++_total.recievedUnreliablePackets; + + _currentSample.recievedUnreliableUtilBytes += payload; + _total.recievedUnreliableUtilBytes += payload; + + _currentSample.sentUnreliableBytes += total; + _total.recievedUnreliableBytes += total; +} + +void ConnectionStats::recordRetransmition() { + ++_currentSample.retransmissions; + ++_total.retransmissions; +} + +void ConnectionStats::recordDrop() { + ++_currentSample.drops; + ++_total.drops; +} diff --git a/libraries/networking/src/udt/ConnectionStats.h b/libraries/networking/src/udt/ConnectionStats.h index df7dfd1545..aa3687e481 100644 --- a/libraries/networking/src/udt/ConnectionStats.h +++ b/libraries/networking/src/udt/ConnectionStats.h @@ -36,6 +36,20 @@ public: int sentPackets { 0 }; int recievedPackets { 0 }; + int sentUtilBytes { 0 }; + int recievedUtilBytes { 0 }; + int sentBytes { 0 }; + int recievedBytes { 0 }; + + int sentUnreliablePackets { 0 }; + int recievedUnreliablePackets { 0 }; + int sentUnreliableUtilBytes { 0 }; + int recievedUnreliableUtilBytes { 0 }; + int sentUnreliableBytes { 0 }; + int recievedUnreliableBytes { 0 }; + + int retransmissions { 0 }; + int drops { 0 }; }; ConnectionStats(); @@ -54,8 +68,14 @@ public: void recordSentTimeoutNAK(); void recordReceivedTimeoutNAK(); - void recordSentPackets(); - void recordReceivedPackets(); + void recordSentPackets(int payload, int total); + void recordReceivedPackets(int payload, int total); + + void recordUnreliableSentPackets(int payload, int total); + void recordUnreliableReceivedPackets(int payload, int total); + + void recordRetransmition(); + void recordDrop(); private: Stats _currentSample; From b437f3bca9eaa7e1e51adb4e716a6075919f8108 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 31 Jul 2015 10:42:43 -0700 Subject: [PATCH 148/242] Made connection hash hold unique ptr --- libraries/networking/src/udt/Connection.cpp | 1 - libraries/networking/src/udt/Connection.h | 6 ++---- libraries/networking/src/udt/Socket.cpp | 19 ++++++++++--------- libraries/networking/src/udt/Socket.h | 6 +++--- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index ff10994e27..50bccc1ab3 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -287,7 +287,6 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber) { // increment the counters for data packets received ++_packetsSinceACK; - ++_totalReceivedDataPackets; // check if we need to send an ACK, according to CC params if (_congestionControl->_ackInterval > 0 && _packetsSinceACK >= _congestionControl->_ackInterval) { diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 2cacdf9fe9..c8313ed2fd 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -39,7 +39,7 @@ public: Connection(Socket* parentSocket, HifiSockAddr destination, std::unique_ptr congestionControl); ~Connection(); - + void sendReliablePacket(std::unique_ptr packet); void sync(); // rate control method, fired by Socket for all connections on SYN interval @@ -103,10 +103,8 @@ private: std::unique_ptr _congestionControl; - std::unique_ptr _sendQueue; + std::unique_ptr _sendQueue; - // Data packet stat collection - int _totalReceivedDataPackets { 0 }; int _packetsSinceACK { 0 }; // The number of packets that have been received during the current ACK interval ConnectionStats _stats; diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 99cdb06b29..04fa835f2c 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -95,7 +95,7 @@ qint64 Socket::writePacket(const Packet& packet, const HifiSockAddr& sockAddr) { qint64 Socket::writePacket(std::unique_ptr packet, const HifiSockAddr& sockAddr) { if (packet->isReliable()) { - findOrCreateConnection(sockAddr)->sendReliablePacket(move(packet)); + findOrCreateConnection(sockAddr).sendReliablePacket(move(packet)); return 0; } @@ -117,14 +117,15 @@ qint64 Socket::writeDatagram(const QByteArray& datagram, const HifiSockAddr& soc return bytesWritten; } -Connection* Socket::findOrCreateConnection(const HifiSockAddr& sockAddr) { +Connection& Socket::findOrCreateConnection(const HifiSockAddr& sockAddr) { auto it = _connectionsHash.find(sockAddr); if (it == _connectionsHash.end()) { - it = _connectionsHash.insert(it, std::make_pair(sockAddr, new Connection(this, sockAddr, _ccFactory->create()))); + it = _connectionsHash.insert(it, std::make_pair(sockAddr, + std::unique_ptr(new Connection(this, sockAddr, _ccFactory->create())))); } - return it->second; + return *it->second; } void Socket::readPendingDatagrams() { @@ -160,8 +161,8 @@ void Socket::readPendingDatagrams() { auto controlPacket = ControlPacket::fromReceivedPacket(std::move(buffer), packetSizeWithHeader, senderSockAddr); // move this control packet to the matching connection - auto connection = findOrCreateConnection(senderSockAddr); - connection->processControl(move(controlPacket)); + auto& connection = findOrCreateConnection(senderSockAddr); + connection.processControl(move(controlPacket)); } else { // setup a Packet from the data we just read @@ -172,8 +173,8 @@ void Socket::readPendingDatagrams() { if (packet->isReliable()) { // if this was a reliable packet then signal the matching connection with the sequence number - auto connection = findOrCreateConnection(senderSockAddr); - connection->processReceivedSequenceNumber(packet->getSequenceNumber()); + auto& connection = findOrCreateConnection(senderSockAddr); + connection.processReceivedSequenceNumber(packet->getSequenceNumber()); } if (_packetHandler) { @@ -188,7 +189,7 @@ void Socket::readPendingDatagrams() { void Socket::connectToSendSignal(const HifiSockAddr& destinationAddr, QObject* receiver, const char* slot) { auto it = _connectionsHash.find(destinationAddr); if (it != _connectionsHash.end()) { - connect(it->second, SIGNAL(packetSent()), receiver, slot); + connect(it->second.get(), SIGNAL(packetSent()), receiver, slot); } } diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 4f947e4043..a8ff255b8d 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -23,11 +23,11 @@ #include "../HifiSockAddr.h" #include "CongestionControl.h" +#include "Connection.h" namespace udt { class BasePacket; -class Connection; class ControlSender; class Packet; class SequenceNumber; @@ -70,7 +70,7 @@ private slots: private: void setSystemBufferSizes(); - Connection* findOrCreateConnection(const HifiSockAddr& sockAddr); + Connection& findOrCreateConnection(const HifiSockAddr& sockAddr); QUdpSocket _udpSocket { this }; PacketFilterOperator _packetFilterOperator; @@ -78,7 +78,7 @@ private: std::unordered_map _unfilteredHandlers; std::unordered_map _unreliableSequenceNumbers; - std::unordered_map _connectionsHash; + std::unordered_map> _connectionsHash; int32_t _synInterval = 10; // 10ms QTimer _synTimer; From 50c0b59ba2e2a113fe75705f2386ba4c938c0c32 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 10:46:52 -0700 Subject: [PATCH 149/242] handle priority of re-transmission of loss --- libraries/networking/src/udt/SendQueue.cpp | 194 +++++++++++---------- libraries/networking/src/udt/SendQueue.h | 1 + 2 files changed, 106 insertions(+), 89 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index c092d089c4..d65cdf951c 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -85,7 +85,10 @@ void SendQueue::ack(SequenceNumber ack) { { // remove any sequence numbers equal to or lower than this ACK in the loss list QWriteLocker nakLocker(&_naksLock); - _naks.remove(_naks.getFirstSequenceNumber(), ack); + + if (_naks.getLength() > 0) { + _naks.remove(_naks.getFirstSequenceNumber(), ack); + } } _lastACKSequenceNumber = (uint32_t) ack; @@ -118,6 +121,21 @@ SequenceNumber SendQueue::getNextSequenceNumber() { return _currentSequenceNumber; } +void SendQueue::sendNewPacketAndAddToSentList(std::unique_ptr newPacket, SequenceNumber sequenceNumber) { + // write the sequence number and send the packet + newPacket->writeSequenceNumber(sequenceNumber); + sendPacket(*newPacket); + + { + // Insert the packet we have just sent in the sent list + QWriteLocker locker(&_sentLock); + _sentPackets[newPacket->getSequenceNumber()].swap(newPacket); + Q_ASSERT_X(!newPacket, "SendQueue::sendNewPacketAndAddToSentList()", "Overriden packet in sent list"); + } + + emit packetSent(); +} + void SendQueue::run() { _isRunning = true; @@ -125,104 +143,102 @@ void SendQueue::run() { // Record timing _lastSendTimestamp = high_resolution_clock::now(); - // we're only allowed to send if the flow window size - // is greater than or equal to the gap between the last ACKed sent and the one we are about to send - if (seqlen(SequenceNumber { (uint32_t) _lastACKSequenceNumber }, _currentSequenceNumber + 1) <= _flowWindowSize) { - bool hasResend = false; - SequenceNumber sequenceNumber; - { - // Check nak list for packet to resend - QWriteLocker locker(&_naksLock); - if (_naks.getLength() > 0) { - hasResend = true; - sequenceNumber = _naks.popFirstSequenceNumber(); - } - } + qDebug() << _lastACKSequenceNumber + << (uint32_t) (_currentSequenceNumber + 1) + << seqlen(SequenceNumber { (uint32_t) _lastACKSequenceNumber }, _currentSequenceNumber + 1); + + bool resentPacket = false; + + while (!resentPacket) { + // prioritize a loss retransmission + _naksLock.lockForWrite(); - std::unique_ptr nextPacket; - - // Find packet in sent list using SequenceNumber - if (hasResend) { - QWriteLocker locker(&_sentLock); - auto it = _sentPackets.find(sequenceNumber); - Q_ASSERT_X(it != _sentPackets.end(), - "SendQueue::sendNextPacket()", "Couldn't find NAKed packet to resend"); + if (_naks.getLength() > 0) { + + // pull the sequence number we need to re-send + SequenceNumber resendNumber = _naks.popFirstSequenceNumber(); + _naksLock.unlock(); + + // pull the packet to re-send from the sent packets list + _sentLock.lockForRead(); + + // see if we can find the packet to re-send + auto it = _sentPackets.find(resendNumber); if (it != _sentPackets.end()) { - it->second.swap(nextPacket); - _sentPackets.erase(it); - } - } - - // If there is no packet to resend, grab the next one in the list - if (!nextPacket) { - QWriteLocker locker(&_packetsLock); - - if (_packets.size() > 0) { - nextPacket.swap(_packets.front()); - _packets.pop_front(); - } - } - - if (nextPacket) { - bool shouldSendSecondOfPair = false; - - if (!hasResend) { - // if we're not re-sending a packet then need to check if this should be a packet pair - sequenceNumber = getNextSequenceNumber(); + // we found the packet - grab it + auto& resendPacket = *(it->second); - // the first packet in the pair is every 16 (rightmost 16 bits = 0) packets - if (((uint32_t) sequenceNumber & 0xF) == 0) { - shouldSendSecondOfPair = true; - } - } - - // Write packet's sequence number and send it off - nextPacket->writeSequenceNumber(sequenceNumber); - sendPacket(*nextPacket); - - { - // Insert the packet we have just sent in the sent list - QWriteLocker locker(&_sentLock); - _sentPackets[nextPacket->getSequenceNumber()].swap(nextPacket); - Q_ASSERT_X(!nextPacket, - "SendQueue::sendNextPacket()", "Overriden packet in sent list"); - } - - emit packetSent(); - - if (shouldSendSecondOfPair) { - std::unique_ptr pairedPacket; + // unlock the sent packets + _sentLock.unlock(); - // we've detected we should send the second packet in a pair, do that now before sleeping - { - QWriteLocker locker(&_packetsLock); - - if (_packets.size() > 0) { - pairedPacket.swap(_packets.front()); - _packets.pop_front(); - } - } + // send it off + sendPacket(resendPacket); - if (pairedPacket) { - // write this packet's sequence number and send it off - pairedPacket->writeSequenceNumber(getNextSequenceNumber()); - sendPacket(*pairedPacket); - - { - // add the paired packet to the sent list - QWriteLocker locker(&_sentLock); - _sentPackets[pairedPacket->getSequenceNumber()].swap(pairedPacket); - Q_ASSERT_X(!pairedPacket, - "SendQueue::sendNextPacket()", "Overriden packet in sent list"); - } - - emit packetSent(); - } + // mark that we did resend a packet + resentPacket = true; + + // break out of our while now that we have re-sent a packet + break; + } else { + // we didn't find this packet in the sentPackets queue - assume this means it was ACKed + // we'll fire the loop again to see if there is another to re-send + + // unlock the sent packets + _sentLock.unlock(); } + + } else { + // unlock the loss list, it's empty + _naksLock.unlock(); + + // break from the while, we didn't resend a packet + break; } } + if (!resentPacket + && seqlen(SequenceNumber { (uint32_t) _lastACKSequenceNumber }, _currentSequenceNumber + 1) <= _flowWindowSize) { + + // we didn't re-send a packet, so time to send a new one + _packetsLock.lockForWrite(); + + if (_packets.size() > 0) { + SequenceNumber nextNumber = getNextSequenceNumber(); + + // grab the first packet we will send + std::unique_ptr firstPacket; + firstPacket.swap(_packets.front()); + _packets.pop_front(); + + std::unique_ptr secondPacket; + + if (((uint32_t) nextNumber & 0xF) == 0) { + // the first packet is the first in a probe pair - every 16 (rightmost 16 bits = 0) packets + // pull off a second packet if we can before we unlock + if (_packets.size() > 0) { + secondPacket.swap(_packets.front()); + _packets.pop_front(); + } + } + + // unlock the packets, we're done pulling + _packetsLock.unlock(); + + // definitely send the first packet + sendNewPacketAndAddToSentList(move(firstPacket), nextNumber); + + // do we have a second in a pair to send as well? + if (secondPacket) { + nextNumber = getNextSequenceNumber(); + sendNewPacketAndAddToSentList(move(secondPacket), nextNumber); + } + + } else { + _packetsLock.unlock(); + } + } + // since we're a while loop, give the thread a chance to process events QCoreApplication::processEvents(); diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 4a4cf1ffea..a438c5635f 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -68,6 +68,7 @@ private: SendQueue(SendQueue&& other) = delete; void sendPacket(const Packet& packet); + void sendNewPacketAndAddToSentList(std::unique_ptr newPacket, SequenceNumber sequenceNumber); // Increments current sequence number and return it SequenceNumber getNextSequenceNumber(); From ee9ae9ceec1881c6ca1bc3cace1be124ab588e4b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 10:51:17 -0700 Subject: [PATCH 150/242] fix for vector initialization in PacketTimeWindow --- libraries/networking/src/udt/PacketTimeWindow.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/PacketTimeWindow.cpp b/libraries/networking/src/udt/PacketTimeWindow.cpp index 469915d311..9db837bc12 100644 --- a/libraries/networking/src/udt/PacketTimeWindow.cpp +++ b/libraries/networking/src/udt/PacketTimeWindow.cpp @@ -24,8 +24,8 @@ static const int DEFAULT_PROBE_INTERVAL_MICROSECONDS = 1000; PacketTimeWindow::PacketTimeWindow(int numPacketIntervals, int numProbeIntervals) : _numPacketIntervals(numPacketIntervals), _numProbeIntervals(numProbeIntervals), - _packetIntervals({ _numPacketIntervals, DEFAULT_PACKET_INTERVAL_MICROSECONDS }), - _probeIntervals({ _numProbeIntervals, DEFAULT_PROBE_INTERVAL_MICROSECONDS }) + _packetIntervals(_numPacketIntervals, DEFAULT_PACKET_INTERVAL_MICROSECONDS), + _probeIntervals(_numProbeIntervals, DEFAULT_PROBE_INTERVAL_MICROSECONDS) { } From e346dbcfee2b288d713a464f9c5357b47f0a3e61 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 10:59:06 -0700 Subject: [PATCH 151/242] seqlen fix in SendQueue --- libraries/networking/src/udt/SendQueue.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index d65cdf951c..2c3c540fb2 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -143,10 +143,6 @@ void SendQueue::run() { // Record timing _lastSendTimestamp = high_resolution_clock::now(); - qDebug() << _lastACKSequenceNumber - << (uint32_t) (_currentSequenceNumber + 1) - << seqlen(SequenceNumber { (uint32_t) _lastACKSequenceNumber }, _currentSequenceNumber + 1); - bool resentPacket = false; while (!resentPacket) { @@ -195,10 +191,10 @@ void SendQueue::run() { // break from the while, we didn't resend a packet break; } - } + } if (!resentPacket - && seqlen(SequenceNumber { (uint32_t) _lastACKSequenceNumber }, _currentSequenceNumber + 1) <= _flowWindowSize) { + && seqlen(SequenceNumber { (uint32_t) _lastACKSequenceNumber }, _currentSequenceNumber) <= _flowWindowSize) { // we didn't re-send a packet, so time to send a new one _packetsLock.lockForWrite(); From ce212041512aae9e2290b74da5746c8b2295166c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 15:09:30 -0700 Subject: [PATCH 152/242] add output of connection stats in UDTTest --- libraries/networking/src/udt/Connection.cpp | 25 ++++++++++++++-- libraries/networking/src/udt/Connection.h | 2 ++ .../networking/src/udt/ConnectionStats.cpp | 30 ++++++++++++++++++- .../networking/src/udt/ConnectionStats.h | 15 +++++++++- libraries/networking/src/udt/Socket.cpp | 17 +++++++++++ libraries/networking/src/udt/Socket.h | 2 ++ tools/udt-test/src/UDTTest.cpp | 11 +++++++ tools/udt-test/src/UDTTest.h | 1 + 8 files changed, 98 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index b6b3f9f12a..71141656ae 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -126,9 +126,17 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { ackPacket->writePrimitive((int32_t) udt::CONNECTION_RECEIVE_BUFFER_SIZE_PACKETS); if (wasCausedBySyncTimeout) { + // grab the up to date packet receive speed and estimated bandwidth + int32_t packetReceiveSpeed = _receiveWindow.getPacketReceiveSpeed(); + int32_t estimatedBandwidth = _receiveWindow.getEstimatedBandwidth(); + + // update those values in our connection stats + _stats.recordReceiveRate(packetReceiveSpeed); + _stats.recordEstimatedBandwidth(estimatedBandwidth); + // pack in the receive speed and estimatedBandwidth - ackPacket->writePrimitive(_receiveWindow.getPacketReceiveSpeed()); - ackPacket->writePrimitive(_receiveWindow.getEstimatedBandwidth()); + ackPacket->writePrimitive(packetReceiveSpeed); + ackPacket->writePrimitive(estimatedBandwidth); // record this as the last ACK send time lastACKSendTime = high_resolution_clock::now(); @@ -261,6 +269,7 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber) { _nakInterval = (_rtt + 4 * _rttVariance); int receivedPacketsPerSecond = _receiveWindow.getPacketReceiveSpeed(); + if (receivedPacketsPerSecond > 0) { // the NAK interval is at least the _minNAKInterval // but might be the time required for all lost packets to be retransmitted @@ -371,10 +380,12 @@ void Connection::processACK(std::unique_ptr controlPacket) { // ACK the send queue so it knows what was received _sendQueue->ack(ack); - // update the RTT updateRTT(rtt); + // write this RTT to stats + _stats.recordRTT(rtt); + // set the RTT for congestion control _congestionControl->setRTT(_rtt); @@ -387,6 +398,10 @@ void Connection::processACK(std::unique_ptr controlPacket) { // these are calculated using an EWMA static const int EMWA_ALPHA_NUMERATOR = 8; + // record these samples in connection stats + _stats.recordReceiveRate(receiveRate); + _stats.recordEstimatedBandwidth(bandwidth); + _deliveryRate = (_deliveryRate * (EMWA_ALPHA_NUMERATOR - 1) + _deliveryRate) / EMWA_ALPHA_NUMERATOR; _bandwidth = (_bandwidth * (EMWA_ALPHA_NUMERATOR - 1) + _bandwidth) / EMWA_ALPHA_NUMERATOR; @@ -515,4 +530,8 @@ void Connection::updateCongestionControlAndSendQueue(std::function cong // now that we've update the congestion control, update the packet send period and flow window size _sendQueue->setPacketSendPeriod(_congestionControl->_packetSendPeriod); _sendQueue->setFlowWindowSize(std::min(_flowWindowSize, (int) _congestionControl->_congestionWindowSize)); + + // record connection stats + _stats.recordPacketSendPeriod(_congestionControl->_packetSendPeriod); + _stats.recordCongestionWindowSize(_congestionControl->_congestionWindowSize); } diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 71cae880d0..83afb88db8 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -46,6 +46,8 @@ public: bool processReceivedSequenceNumber(SequenceNumber sequenceNumber); // returns indicates if this packet was a duplicate void processControl(std::unique_ptr controlPacket); + + ConnectionStats::Stats sampleStats() { return _stats.sample(); } signals: void packetSent(); diff --git a/libraries/networking/src/udt/ConnectionStats.cpp b/libraries/networking/src/udt/ConnectionStats.cpp index 394448afec..b46eb3ee34 100644 --- a/libraries/networking/src/udt/ConnectionStats.cpp +++ b/libraries/networking/src/udt/ConnectionStats.cpp @@ -89,4 +89,32 @@ void ConnectionStats::recordSentPackets() { void ConnectionStats::recordReceivedPackets() { ++_currentSample.recievedPackets; ++_total.recievedPackets; -} \ No newline at end of file +} + +static const double EWMA_CURRENT_SAMPLE_WEIGHT = 0.125; +static const double EWMA_PREVIOUS_SAMPLES_WEIGHT = 1 - 0.125; + +void ConnectionStats::recordReceiveRate(int sample) { + _currentSample.receiveRate = sample; + _total.receiveRate = (_total.receiveRate * EWMA_PREVIOUS_SAMPLES_WEIGHT) + (sample * EWMA_CURRENT_SAMPLE_WEIGHT); +} + +void ConnectionStats::recordEstimatedBandwidth(int sample) { + _currentSample.estimatedBandwith = sample; + _total.estimatedBandwith = (_total.estimatedBandwith * EWMA_PREVIOUS_SAMPLES_WEIGHT) + (sample * EWMA_CURRENT_SAMPLE_WEIGHT); +} + +void ConnectionStats::recordRTT(int sample) { + _currentSample.rtt = sample; + _total.rtt = (_total.rtt * EWMA_PREVIOUS_SAMPLES_WEIGHT) + (sample * EWMA_CURRENT_SAMPLE_WEIGHT); +} + +void ConnectionStats::recordCongestionWindowSize(int sample) { + _currentSample.congestionWindowSize = sample; + _total.congestionWindowSize = (_total.congestionWindowSize * EWMA_PREVIOUS_SAMPLES_WEIGHT) + (sample * EWMA_CURRENT_SAMPLE_WEIGHT); +} + +void ConnectionStats::recordPacketSendPeriod(int sample) { + _currentSample.packetSendPeriod = sample; + _total.packetSendPeriod = (_total.packetSendPeriod * EWMA_PREVIOUS_SAMPLES_WEIGHT) + (sample * EWMA_CURRENT_SAMPLE_WEIGHT); +} diff --git a/libraries/networking/src/udt/ConnectionStats.h b/libraries/networking/src/udt/ConnectionStats.h index df7dfd1545..fed1158bf7 100644 --- a/libraries/networking/src/udt/ConnectionStats.h +++ b/libraries/networking/src/udt/ConnectionStats.h @@ -36,6 +36,13 @@ public: int sentPackets { 0 }; int recievedPackets { 0 }; + + // the following stats are trailing averages in the result, not totals + int receiveRate { 0 }; + int estimatedBandwith { 0 }; + int rtt { 0 }; + int congestionWindowSize { 0 }; + int packetSendPeriod { 0 }; }; ConnectionStats(); @@ -57,6 +64,12 @@ public: void recordSentPackets(); void recordReceivedPackets(); + void recordReceiveRate(int sample); + void recordEstimatedBandwidth(int sample); + void recordRTT(int sample); + void recordCongestionWindowSize(int sample); + void recordPacketSendPeriod(int sample); + private: Stats _currentSample; Stats _total; @@ -64,4 +77,4 @@ private: } -#endif // hifi_ConnectionStats_h \ No newline at end of file +#endif // hifi_ConnectionStats_h diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 99cdb06b29..d1e953d650 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -213,3 +213,20 @@ void Socket::setCongestionControlFactory(std::unique_ptrsynInterval(); } + +void Socket::sampleAndPrintConnectionStats() { + if (thread() != QThread::currentThread()) { + QMetaObject::invokeMethod(this, "sampleAndPrintConnectionStats"); + return; + } + + for(auto& connection : _connectionsHash) { + ConnectionStats::Stats sampleStats = connection.second->sampleStats(); + + qDebug() << connection.first + << sampleStats.receiveRate << sampleStats.rtt + << sampleStats.congestionWindowSize << sampleStats.packetSendPeriod + << sampleStats.sentPackets + << sampleStats.receivedACKs << sampleStats.receivedLightACKs << sampleStats.receivedNAKs; + } +} diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 4f947e4043..02aa733e93 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -63,6 +63,8 @@ public: void setCongestionControlFactory(std::unique_ptr ccFactory); void connectToSendSignal(const HifiSockAddr& destinationAddr, QObject* receiver, const char* slot); + + void sampleAndPrintConnectionStats(); private slots: void readPendingDatagrams(); diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 9f149dce52..55a8a78f6f 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -116,6 +116,13 @@ UDTTest::UDTTest(int& argc, char** argv) : if (!_target.isNull()) { sendInitialPackets(); + + // the sender reports stats every 1 second + static const int STATS_SAMPLE_INTERVAL = 1000; + + QTimer* statsTimer = new QTimer(this); + connect(statsTimer, &QTimer::timeout, this, &UDTTest::sampleStats); + statsTimer->start(STATS_SAMPLE_INTERVAL); } } @@ -198,3 +205,7 @@ void UDTTest::sendPacket() { ++_totalQueuedPackets; } + +void UDTTest::sampleStats() { + _socket.sampleAndPrintConnectionStats(); +} diff --git a/tools/udt-test/src/UDTTest.h b/tools/udt-test/src/UDTTest.h index 050c5dc75a..1fd1836cf9 100644 --- a/tools/udt-test/src/UDTTest.h +++ b/tools/udt-test/src/UDTTest.h @@ -27,6 +27,7 @@ public: public slots: void refillPacket() { sendPacket(); } // adds a new packet to the queue when we are told one is sent + void sampleStats(); private: void parseArguments(); From fcec53bdce9aa7f25cbddcdb5529dc100b5cd5b1 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 31 Jul 2015 15:09:58 -0700 Subject: [PATCH 153/242] Fix sequence number crash --- libraries/networking/src/udt/SendQueue.cpp | 2 +- libraries/networking/src/udt/SequenceNumber.h | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 2c3c540fb2..a4f9d16d08 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -86,7 +86,7 @@ void SendQueue::ack(SequenceNumber ack) { { // remove any sequence numbers equal to or lower than this ACK in the loss list QWriteLocker nakLocker(&_naksLock); - if (_naks.getLength() > 0) { + if (_naks.getLength() > 0 && _naks.getFirstSequenceNumber() <= ack) { _naks.remove(_naks.getFirstSequenceNumber(), ack); } } diff --git a/libraries/networking/src/udt/SequenceNumber.h b/libraries/networking/src/udt/SequenceNumber.h index c77a14edf9..d0bea86777 100644 --- a/libraries/networking/src/udt/SequenceNumber.h +++ b/libraries/networking/src/udt/SequenceNumber.h @@ -21,7 +21,8 @@ namespace udt { class SequenceNumber { public: // Base type of sequence numbers - using Type = uint32_t; + using Type = int32_t; + using UType = uint32_t; // Values are for 29 bit SequenceNumber static const Type THRESHOLD = 0x0FFFFFFF; // threshold for comparing sequence numbers @@ -32,8 +33,10 @@ public: // Only explicit conversions explicit SequenceNumber(char* value) { _value = (*reinterpret_cast(value)) & MAX; } - explicit SequenceNumber(Type value) { _value = (value <= MAX) ? value : MAX; } + explicit SequenceNumber(Type value) { _value = (value <= MAX) ? ((value >= 0) ? value : 0) : MAX; } + explicit SequenceNumber(UType value) { _value = (value <= MAX) ? value : MAX; } explicit operator Type() { return _value; } + explicit operator UType() { return static_cast(_value); } inline SequenceNumber& operator++() { _value = (_value == MAX) ? 0 : ++_value; From c5b92d319fcf254227473a82b2b1896369396593 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 31 Jul 2015 16:14:19 -0700 Subject: [PATCH 154/242] Update stats gathering --- libraries/networking/src/udt/Connection.cpp | 19 +++++++++++++++++-- libraries/networking/src/udt/Connection.h | 10 +++++++--- .../networking/src/udt/ConnectionStats.cpp | 8 ++++---- .../networking/src/udt/ConnectionStats.h | 6 +++--- libraries/networking/src/udt/SendQueue.cpp | 7 ++++++- libraries/networking/src/udt/SendQueue.h | 3 ++- libraries/networking/src/udt/Socket.cpp | 4 +++- 7 files changed, 42 insertions(+), 15 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index e141e680c2..b399942a19 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -52,6 +52,8 @@ SendQueue& Connection::getSendQueue() { _sendQueue = SendQueue::create(_parentSocket, _destination); QObject::connect(_sendQueue.get(), &SendQueue::packetSent, this, &Connection::packetSent); + QObject::connect(_sendQueue.get(), &SendQueue::packetSent, this, &Connection::recordSentPackets); + QObject::connect(_sendQueue.get(), &SendQueue::packetRetransmitted, this, &Connection::recordRetransmission); // set defaults on the send queue from our congestion control object _sendQueue->setPacketSendPeriod(_congestionControl->_packetSendPeriod); @@ -80,6 +82,14 @@ void Connection::sync() { } } +void Connection::recordSentPackets(int dataSize, int payloadSize) { + _stats.recordSentPackets(payloadSize, dataSize); +} + +void Connection::recordRetransmission() { + _stats.recordRetransmission(); +} + void Connection::sendACK(bool wasCausedBySyncTimeout) { static high_resolution_clock::time_point lastACKSendTime; auto currentTime = high_resolution_clock::now(); @@ -246,8 +256,7 @@ SequenceNumber Connection::nextACK() const { } } -bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber) { - +bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, int packetSize, int payloadSize) { // check if this is a packet pair we should estimate bandwidth from, or just a regular packet if (((uint32_t) sequenceNumber & 0xF) == 0) { _receiveWindow.onProbePair1Arrival(); @@ -304,6 +313,12 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber) { sendLightACK(); } + if (wasDuplicate) { + _stats.recordDuplicates(); + } else { + _stats.recordReceivedPackets(payloadSize, packetSize); + } + return wasDuplicate; } diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index d0aa359d08..23a77bf4db 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -43,8 +43,9 @@ public: void sendReliablePacket(std::unique_ptr packet); void sync(); // rate control method, fired by Socket for all connections on SYN interval - - bool processReceivedSequenceNumber(SequenceNumber sequenceNumber); // returns indicates if this packet was a duplicate + + // returns indicates if this packet was a duplicate + bool processReceivedSequenceNumber(SequenceNumber sequenceNumber, int packetSize, int payloadSize); void processControl(std::unique_ptr controlPacket); ConnectionStats::Stats sampleStats() { return _stats.sample(); } @@ -52,6 +53,10 @@ public: signals: void packetSent(); +private slots: + void recordSentPackets(int payload, int total); + void recordRetransmission(); + private: void sendACK(bool wasCausedBySyncTimeout = true); void sendLightACK(); @@ -65,7 +70,6 @@ private: void processNAK(std::unique_ptr controlPacket); void processTimeoutNAK(std::unique_ptr controlPacket); - SendQueue& getSendQueue(); SequenceNumber nextACK() const; void updateRTT(int rtt); diff --git a/libraries/networking/src/udt/ConnectionStats.cpp b/libraries/networking/src/udt/ConnectionStats.cpp index 73dadfbe7e..62497ab496 100644 --- a/libraries/networking/src/udt/ConnectionStats.cpp +++ b/libraries/networking/src/udt/ConnectionStats.cpp @@ -126,14 +126,14 @@ void ConnectionStats::recordUnreliableReceivedPackets(int payload, int total) { _total.recievedUnreliableBytes += total; } -void ConnectionStats::recordRetransmition() { +void ConnectionStats::recordRetransmission() { ++_currentSample.retransmissions; ++_total.retransmissions; } -void ConnectionStats::recordDrop() { - ++_currentSample.drops; - ++_total.drops; +void ConnectionStats::recordDuplicates() { + ++_currentSample.duplicates; + ++_total.duplicates; } static const double EWMA_CURRENT_SAMPLE_WEIGHT = 0.125; diff --git a/libraries/networking/src/udt/ConnectionStats.h b/libraries/networking/src/udt/ConnectionStats.h index cf7c908a83..f4b912ace5 100644 --- a/libraries/networking/src/udt/ConnectionStats.h +++ b/libraries/networking/src/udt/ConnectionStats.h @@ -49,7 +49,7 @@ public: int recievedUnreliableBytes { 0 }; int retransmissions { 0 }; - int drops { 0 }; + int duplicates { 0 }; // the following stats are trailing averages in the result, not totals int receiveRate { 0 }; @@ -81,8 +81,8 @@ public: void recordUnreliableSentPackets(int payload, int total); void recordUnreliableReceivedPackets(int payload, int total); - void recordRetransmition(); - void recordDrop(); + void recordRetransmission(); + void recordDuplicates(); void recordReceiveRate(int sample); void recordEstimatedBandwidth(int sample); diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index a4f9d16d08..f271a1c685 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -126,6 +126,10 @@ void SendQueue::sendNewPacketAndAddToSentList(std::unique_ptr newPacket, newPacket->writeSequenceNumber(sequenceNumber); sendPacket(*newPacket); + // Save packet/payload size before we move it + auto packetSize = newPacket->getDataSize(); + auto payloadSize = newPacket->getPayloadSize(); + { // Insert the packet we have just sent in the sent list QWriteLocker locker(&_sentLock); @@ -133,7 +137,7 @@ void SendQueue::sendNewPacketAndAddToSentList(std::unique_ptr newPacket, Q_ASSERT_X(!newPacket, "SendQueue::sendNewPacketAndAddToSentList()", "Overriden packet in sent list"); } - emit packetSent(); + emit packetSent(packetSize, payloadSize); } void SendQueue::run() { @@ -170,6 +174,7 @@ void SendQueue::run() { // send it off sendPacket(resendPacket); + emit packetRetransmitted(); // mark that we did resend a packet resentPacket = true; diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index a438c5635f..004938051e 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -57,7 +57,8 @@ public slots: void overrideNAKListFromPacket(ControlPacket& packet); signals: - void packetSent(); + void packetSent(int dataSize, int payloadSize); + void packetRetransmitted(); private slots: void run(); diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 6bb7bf08d0..14fad37005 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -174,7 +174,9 @@ void Socket::readPendingDatagrams() { if (packet->isReliable()) { // if this was a reliable packet then signal the matching connection with the sequence number auto& connection = findOrCreateConnection(senderSockAddr); - connection.processReceivedSequenceNumber(packet->getSequenceNumber()); + connection.processReceivedSequenceNumber(packet->getSequenceNumber(), + packet->getDataSize(), + packet->getPayloadSize()); } if (_packetHandler) { From d420bca88eece3f7f55640629b6ecbb577006b2e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 16:14:33 -0700 Subject: [PATCH 155/242] fix for ACK2 sending, output prettier stats --- libraries/networking/src/udt/Connection.cpp | 3 +++ libraries/networking/src/udt/Socket.cpp | 22 ++++++++---------- libraries/networking/src/udt/Socket.h | 2 +- tools/udt-test/src/UDTTest.cpp | 25 ++++++++++++++++++++- 4 files changed, 37 insertions(+), 15 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index e141e680c2..5afd3772a0 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -192,6 +192,9 @@ void Connection::sendACK2(SequenceNumber currentACKSubSequenceNumber) { // write the sub sequence number for this ACK2 ack2Packet->writePrimitive(currentACKSubSequenceNumber); + // send the ACK2 packet + _parentSocket->writeBasePacket(*ack2Packet, _destination); + // update the last sent ACK2 and the last ACK2 send time _lastSentACK2 = currentACKSubSequenceNumber; diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 6bb7bf08d0..25335c99df 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -215,19 +215,15 @@ void Socket::setCongestionControlFactory(std::unique_ptrsynInterval(); } -void Socket::sampleAndPrintConnectionStats() { - if (thread() != QThread::currentThread()) { - QMetaObject::invokeMethod(this, "sampleAndPrintConnectionStats"); - return; - } +ConnectionStats::Stats Socket::sampleStatsForConnection(const HifiSockAddr& destination) { + Q_ASSERT_X(thread() == QThread::currentThread(), + "Socket::sampleStatsForConnection", + "Stats sampling for connection must be on socket thread"); - for(auto& connection : _connectionsHash) { - ConnectionStats::Stats sampleStats = connection.second->sampleStats(); - - qDebug() << connection.first - << sampleStats.receiveRate << sampleStats.rtt - << sampleStats.congestionWindowSize << sampleStats.packetSendPeriod - << sampleStats.sentPackets - << sampleStats.receivedACKs << sampleStats.receivedLightACKs << sampleStats.receivedNAKs; + auto it = _connectionsHash.find(destination); + if (it != _connectionsHash.end()) { + return it->second->sampleStats(); + } else { + return ConnectionStats::Stats(); } } diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 6b798d64d2..0fee52e755 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -64,7 +64,7 @@ public: void connectToSendSignal(const HifiSockAddr& destinationAddr, QObject* receiver, const char* slot); - void sampleAndPrintConnectionStats(); + ConnectionStats::Stats sampleStatsForConnection(const HifiSockAddr& destination); private slots: void readPendingDatagrams(); diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 55a8a78f6f..067a7ec746 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -41,6 +41,10 @@ const QCommandLineOption UNRELIABLE_PACKETS { "unreliable", "send unreliable packets (default is reliable)" }; +const QStringList STATS_TABLE_HEADERS { + "Send Rate (P/s)", "RTT(ms)", "CW (P)", "Send Period (us)", "Received ACK", "Received NAK", "Sent ACK2" +}; + UDTTest::UDTTest(int& argc, char** argv) : QCoreApplication(argc, argv) { @@ -123,6 +127,9 @@ UDTTest::UDTTest(int& argc, char** argv) : QTimer* statsTimer = new QTimer(this); connect(statsTimer, &QTimer::timeout, this, &UDTTest::sampleStats); statsTimer->start(STATS_SAMPLE_INTERVAL); + + // output the headers for stats for our table + qDebug() << qPrintable(STATS_TABLE_HEADERS.join(" | ")); } } @@ -207,5 +214,21 @@ void UDTTest::sendPacket() { } void UDTTest::sampleStats() { - _socket.sampleAndPrintConnectionStats(); + udt::ConnectionStats::Stats stats = _socket.sampleStatsForConnection(_target); + + int headerIndex = -1; + + // setup a list of left justified values + QStringList values { + QString::number(stats.receiveRate).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.rtt).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.congestionWindowSize).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.packetSendPeriod).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.receivedACKs).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.receivedNAKs).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.sentACK2s).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()) + }; + + // output this line of values + qDebug() << qPrintable(values.join(" | ")); } From 7083d843728d7eb5343821fb7831ed89eacc7cdd Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 31 Jul 2015 16:15:21 -0700 Subject: [PATCH 156/242] Fix ack timestamps --- libraries/networking/src/udt/Connection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index b399942a19..df03b7ea8f 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -159,7 +159,7 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { _parentSocket->writeBasePacket(*ackPacket, _destination); // write this ACK to the map of sent ACKs - _sentACKs[_currentACKSubSequenceNumber] = { nextACKNumber, currentTime }; + _sentACKs[_currentACKSubSequenceNumber] = { nextACKNumber, high_resolution_clock::now() }; // reset the number of data packets received since last ACK _packetsSinceACK = 0; From e39611dc1cba4fb3b0f7bea47c25ac053defde3e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 16:31:08 -0700 Subject: [PATCH 157/242] fix for PacketTimeWindow calculations --- libraries/networking/src/udt/PacketTimeWindow.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/PacketTimeWindow.cpp b/libraries/networking/src/udt/PacketTimeWindow.cpp index 9db837bc12..a16c736622 100644 --- a/libraries/networking/src/udt/PacketTimeWindow.cpp +++ b/libraries/networking/src/udt/PacketTimeWindow.cpp @@ -48,14 +48,15 @@ int32_t meanOfMedianFilteredValues(std::vector intervals, int numValues, in int lowerBound = median / MEDIAN_FILTERING_BOUND_MULTIPLIER; for (auto& interval : intervals) { - if ((interval < upperBound) && interval > lowerBound) { + if ((interval < upperBound) && (interval > lowerBound)) { ++count; sum += interval; } } if (count >= valuesRequired) { - return (int32_t) ceil((double) USECS_PER_MSEC / ((double) sum) / ((double) count)); + static const double USECS_PER_SEC = 1000000; + return (int32_t) ceil(USECS_PER_SEC / ((double) sum) / ((double) count)); } else { return 0; } From 681c9afe9082fd19d0336e1e370081cc904c05f0 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 16:31:42 -0700 Subject: [PATCH 158/242] add parens to PacketTimeWindow division --- libraries/networking/src/udt/PacketTimeWindow.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/PacketTimeWindow.cpp b/libraries/networking/src/udt/PacketTimeWindow.cpp index a16c736622..3a942108a8 100644 --- a/libraries/networking/src/udt/PacketTimeWindow.cpp +++ b/libraries/networking/src/udt/PacketTimeWindow.cpp @@ -56,7 +56,7 @@ int32_t meanOfMedianFilteredValues(std::vector intervals, int numValues, in if (count >= valuesRequired) { static const double USECS_PER_SEC = 1000000; - return (int32_t) ceil(USECS_PER_SEC / ((double) sum) / ((double) count)); + return (int32_t) ceil(USECS_PER_SEC / (((double) sum) / ((double) count))); } else { return 0; } From 2db540ffde4a5fb588e0a28e00340a322d7b21c6 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 16:35:42 -0700 Subject: [PATCH 159/242] fix for double in PacketTimeWindow --- libraries/networking/src/udt/PacketTimeWindow.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/PacketTimeWindow.cpp b/libraries/networking/src/udt/PacketTimeWindow.cpp index 3a942108a8..e28b276ed9 100644 --- a/libraries/networking/src/udt/PacketTimeWindow.cpp +++ b/libraries/networking/src/udt/PacketTimeWindow.cpp @@ -55,7 +55,7 @@ int32_t meanOfMedianFilteredValues(std::vector intervals, int numValues, in } if (count >= valuesRequired) { - static const double USECS_PER_SEC = 1000000; + static const double USECS_PER_SEC = 1000000.0; return (int32_t) ceil(USECS_PER_SEC / (((double) sum) / ((double) count))); } else { return 0; From 9b6c8bcf77cbfdcd02e24877e1a695bf8b24b34e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 16:37:38 -0700 Subject: [PATCH 160/242] update EWMA for delivery rate and bandwidth --- libraries/networking/src/udt/Connection.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index d7fcdd20a1..0a4d974d08 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -422,8 +422,8 @@ void Connection::processACK(std::unique_ptr controlPacket) { _stats.recordReceiveRate(receiveRate); _stats.recordEstimatedBandwidth(bandwidth); - _deliveryRate = (_deliveryRate * (EMWA_ALPHA_NUMERATOR - 1) + _deliveryRate) / EMWA_ALPHA_NUMERATOR; - _bandwidth = (_bandwidth * (EMWA_ALPHA_NUMERATOR - 1) + _bandwidth) / EMWA_ALPHA_NUMERATOR; + _deliveryRate = (_deliveryRate * (EMWA_ALPHA_NUMERATOR - 1) + receiveRate) / EMWA_ALPHA_NUMERATOR; + _bandwidth = (_bandwidth * (EMWA_ALPHA_NUMERATOR - 1) + bandwidth) / EMWA_ALPHA_NUMERATOR; _congestionControl->setReceiveRate(_deliveryRate); _congestionControl->setBandwidth(_bandwidth); From 84b8fc9f063f9e9b54470a2b0cbdd3b0930424fa Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 16:41:22 -0700 Subject: [PATCH 161/242] fix casing of onACK method in CC --- libraries/networking/src/udt/CongestionControl.h | 2 +- libraries/networking/src/udt/Connection.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 75970ed2fd..81ade13b41 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -37,7 +37,7 @@ public: virtual void init() {} virtual void close() {} - virtual void onAck(SequenceNumber ackNum) {} + virtual void onACK(SequenceNumber ackNum) {} virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) {} protected: diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 0a4d974d08..d668ed09fb 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -431,7 +431,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { // give this ACK to the congestion control and update the send queue parameters updateCongestionControlAndSendQueue([this, ack](){ - _congestionControl->onAck(ack); + _congestionControl->onACK(ack); }); // update the total count of received ACKs From fa6628e7eb8ffda609567ad5a86b6b897903090d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 16:52:09 -0700 Subject: [PATCH 162/242] make sure we init CongestionControl --- libraries/networking/src/udt/CongestionControl.cpp | 6 ++++-- libraries/networking/src/udt/Connection.cpp | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 984d520d07..8d8efec51e 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -18,11 +18,13 @@ using namespace std::chrono; static const double USECS_PER_SECOND = 1000000.0; void CongestionControl::setPacketSendPeriod(double newSendPeriod) { + Q_ASSERT_X(newSendPeriod >= 0, "CongestionControl::setPacketPeriod", "Can not set a negative packet send period"); + if (_maxBandwidth > 0) { // anytime the packet send period is about to be increased, make sure it stays below the minimum period, // calculated based on the maximum desired bandwidth - int minPacketSendPeriod = USECS_PER_SECOND / (double(_maxBandwidth) / _mss); - _packetSendPeriod = std::max(newSendPeriod, (double) minPacketSendPeriod); + double minPacketSendPeriod = USECS_PER_SECOND / (((double) _maxBandwidth) / _mss); + _packetSendPeriod = std::max(newSendPeriod, minPacketSendPeriod); } else { _packetSendPeriod = newSendPeriod; } diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index d668ed09fb..8b16e8987f 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -32,6 +32,9 @@ Connection::Connection(Socket* parentSocket, HifiSockAddr destination, unique_pt { Q_ASSERT_X(socket, "Connection::Connection", "Must be called with a valid Socket*"); + Q_ASSERT_X(congestionControl, "Connection::Connection", "Must be called with a valid CongestionControl object"); + congestionControl->init(); + // setup default SYN, RTT and RTT Variance based on the SYN interval in CongestionControl object _synInterval = _congestionControl->synInterval(); _rtt = _synInterval * 10; From 15ce9aabc54080869e1eb0b1f7ebd91b84645b1d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 16:52:59 -0700 Subject: [PATCH 163/242] don't assert on moved CC --- libraries/networking/src/udt/Connection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 8b16e8987f..a2a8942508 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -32,7 +32,7 @@ Connection::Connection(Socket* parentSocket, HifiSockAddr destination, unique_pt { Q_ASSERT_X(socket, "Connection::Connection", "Must be called with a valid Socket*"); - Q_ASSERT_X(congestionControl, "Connection::Connection", "Must be called with a valid CongestionControl object"); + Q_ASSERT_X(_congestionControl, "Connection::Connection", "Must be called with a valid CongestionControl object"); congestionControl->init(); // setup default SYN, RTT and RTT Variance based on the SYN interval in CongestionControl object From cba51ac63df7ce91a5533648ecc57289ced49a0e Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 31 Jul 2015 16:53:43 -0700 Subject: [PATCH 164/242] Fix congestion control --- libraries/networking/src/udt/Connection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index a2a8942508..93f61c11aa 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -33,7 +33,7 @@ Connection::Connection(Socket* parentSocket, HifiSockAddr destination, unique_pt Q_ASSERT_X(socket, "Connection::Connection", "Must be called with a valid Socket*"); Q_ASSERT_X(_congestionControl, "Connection::Connection", "Must be called with a valid CongestionControl object"); - congestionControl->init(); + _congestionControl->init(); // setup default SYN, RTT and RTT Variance based on the SYN interval in CongestionControl object _synInterval = _congestionControl->synInterval(); From 0e3403833d5dc16f2783840ef69935abcd6794ee Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 17:00:53 -0700 Subject: [PATCH 165/242] make capacitySpeedDelta a double --- libraries/networking/src/udt/CongestionControl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 8d8efec51e..16ee9a0c75 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -95,7 +95,7 @@ void DefaultCC::onACK(SequenceNumber ackNum) { return; } - int capacitySpeedDelta = (int) (_bandwidth - USECS_PER_SECOND / _packetSendPeriod); + double capacitySpeedDelta = (_bandwidth - USECS_PER_SECOND / _packetSendPeriod); // UDT uses what they call DAIMD - additive increase multiplicative decrease with decreasing increases // This factor is a protocol parameter that is part of the DAIMD algorithim From 2946cb7ed5a86b1d6639b5f196e8a127366763eb Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 31 Jul 2015 17:15:10 -0700 Subject: [PATCH 166/242] Collapse ControlPacket ctors --- .../networking/src/udt/ControlPacket.cpp | 25 ++----------------- libraries/networking/src/udt/ControlPacket.h | 3 +-- 2 files changed, 3 insertions(+), 25 deletions(-) diff --git a/libraries/networking/src/udt/ControlPacket.cpp b/libraries/networking/src/udt/ControlPacket.cpp index 626e126c03..b64e471428 100644 --- a/libraries/networking/src/udt/ControlPacket.cpp +++ b/libraries/networking/src/udt/ControlPacket.cpp @@ -42,32 +42,11 @@ std::unique_ptr ControlPacket::fromReceivedPacket(std::unique_ptr } std::unique_ptr ControlPacket::create(Type type, qint64 size) { - - std::unique_ptr controlPacket; - - if (size == -1) { - return std::unique_ptr(new ControlPacket(type)); - } else { - // Fail with invalid size - Q_ASSERT(size >= 0); - - return std::unique_ptr(new ControlPacket(type, size)); - } -} - -ControlPacket::ControlPacket(Type type) : - BasePacket(-1), - _type(type) -{ - adjustPayloadStartAndCapacity(ControlPacket::localHeaderSize()); - - open(QIODevice::ReadWrite); - - writeType(); + return std::unique_ptr(new ControlPacket(type, size)); } ControlPacket::ControlPacket(Type type, qint64 size) : - BasePacket(ControlPacket::localHeaderSize() + size), + BasePacket((size == -1) ? -1 : ControlPacket::localHeaderSize() + size), _type(type) { adjustPayloadStartAndCapacity(ControlPacket::localHeaderSize()); diff --git a/libraries/networking/src/udt/ControlPacket.h b/libraries/networking/src/udt/ControlPacket.h index c4ad7065a7..e0e3e38796 100644 --- a/libraries/networking/src/udt/ControlPacket.h +++ b/libraries/networking/src/udt/ControlPacket.h @@ -47,8 +47,7 @@ public: void setType(Type type); private: - ControlPacket(Type type); - ControlPacket(Type type, qint64 size); + ControlPacket(Type type, qint64 size = -1); ControlPacket(std::unique_ptr data, qint64 size, const HifiSockAddr& senderSockAddr); ControlPacket(ControlPacket&& other); ControlPacket(const ControlPacket& other) = delete; From df1e97c591cd88d2df7fd8a441acf3fdc40d5d33 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 17:19:14 -0700 Subject: [PATCH 167/242] cleanup RTT stats, add assert for CT --- libraries/networking/src/udt/ControlPacket.cpp | 5 ++++- tools/udt-test/src/UDTTest.cpp | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/ControlPacket.cpp b/libraries/networking/src/udt/ControlPacket.cpp index 626e126c03..f1e3b71b67 100644 --- a/libraries/networking/src/udt/ControlPacket.cpp +++ b/libraries/networking/src/udt/ControlPacket.cpp @@ -120,6 +120,9 @@ void ControlPacket::readType() { Q_ASSERT_X(bitAndType & CONTROL_BIT_MASK, "ControlPacket::readHeader()", "This should be a control packet"); + uint16_t packetType = (bitAndType & ~CONTROL_BIT_MASK) >> (8 * sizeof(Type)); + Q_ASSERT_X(packetType < ControlPacket::Type::TimeoutNAK, "ControlPacket::readType()", "Received a control packet with wrong type"); + // read the type - _type = (Type) ((bitAndType & ~CONTROL_BIT_MASK) >> (8 * sizeof(Type))); + _type = (Type) packetType; } diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 067a7ec746..d0c7b159df 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -218,10 +218,12 @@ void UDTTest::sampleStats() { int headerIndex = -1; + static const double USECS_PER_MSEC = 1000.0; + // setup a list of left justified values QStringList values { QString::number(stats.receiveRate).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.rtt).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.rtt / USECS_PER_MSEC).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.congestionWindowSize).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.packetSendPeriod).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.receivedACKs).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), From 22ead79988a4681b947af1c6b19443771d51ba55 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 17:30:22 -0700 Subject: [PATCH 168/242] fix another char allocation in PacketTests --- tests/networking/src/PacketTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/networking/src/PacketTests.cpp b/tests/networking/src/PacketTests.cpp index 8ac4a6d94e..cbb949aa84 100644 --- a/tests/networking/src/PacketTests.cpp +++ b/tests/networking/src/PacketTests.cpp @@ -18,7 +18,7 @@ QTEST_MAIN(PacketTests) std::unique_ptr copyToReadPacket(std::unique_ptr& packet) { auto size = packet->getDataSize(); - auto data = std::unique_ptr(new char[size]); + auto data = std::unique_ptr(new char[size]); memcpy(data.get(), packet->getData(), size); return Packet::fromReceivedPacket(std::move(data), size, HifiSockAddr()); } From 9537aea5c0aa1db1c4325487d2a006093657081e Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 31 Jul 2015 17:31:30 -0700 Subject: [PATCH 169/242] Fix nak timeout packet size --- libraries/networking/src/udt/Connection.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 93f61c11aa..56fc1292b6 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -238,8 +238,7 @@ void Connection::sendNAK(SequenceNumber sequenceNumberRecieved) { void Connection::sendTimeoutNAK() { if (_lossList.getLength() > 0) { // construct a NAK packet that will hold all of the lost sequence numbers - // TODO size is wrong, fix it. - auto lossListPacket = ControlPacket::create(ControlPacket::TimeoutNAK, _lossList.getLength() * sizeof(SequenceNumber)); + auto lossListPacket = ControlPacket::create(ControlPacket::TimeoutNAK, 2 * _lossList.getLength() * sizeof(SequenceNumber)); // Pack in the lost sequence numbers _lossList.write(*lossListPacket); From e74f47b64c4467b9af16b25b09428e44e782662a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 18:13:11 -0700 Subject: [PATCH 170/242] use a list of pairs of pairs for ACK2 --- libraries/networking/src/udt/Connection.cpp | 48 +++++++++++++-------- libraries/networking/src/udt/Connection.h | 6 ++- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 56fc1292b6..62336a31ae 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -161,8 +161,11 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { // have the socket send off our packet _parentSocket->writeBasePacket(*ackPacket, _destination); + Q_ASSERT_X(_sentACKs.empty() || _sentACKs.back().first + 1 == _currentACKSubSequenceNumber, + "Connection::sendACK", "Adding an invalid ACK to _sentACKs"); + // write this ACK to the map of sent ACKs - _sentACKs[_currentACKSubSequenceNumber] = { nextACKNumber, high_resolution_clock::now() }; + _sentACKs.push_back({ _currentACKSubSequenceNumber, { nextACKNumber, high_resolution_clock::now() }}); // reset the number of data packets received since last ACK _packetsSinceACK = 0; @@ -372,7 +375,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { // read the ACKed sequence number SequenceNumber ack; controlPacket->readPrimitive(&ack); - + // validate that this isn't a BS ACK if (ack > getSendQueue().getCurrentSequenceNumber()) { // in UDT they specifically break the connection here - do we want to do anything? @@ -463,23 +466,32 @@ void Connection::processACK2(std::unique_ptr controlPacket) { controlPacket->readPrimitive(&subSequenceNumber); // check if we had that subsequence number in our map - auto it = _sentACKs.find(subSequenceNumber); + auto it = std::find_if_not(_sentACKs.begin(), _sentACKs.end(), [subSequenceNumber](const ACKListPair& pair){ + return subSequenceNumber < pair.first; + }); + if (it != _sentACKs.end()) { - // update the RTT using the ACK window - SequenceNumberTimePair& pair = it->second; - - // calculate the RTT (time now - time ACK sent) - auto now = high_resolution_clock::now(); - int rtt = duration_cast(now - pair.second).count(); - - updateRTT(rtt); - - // set the RTT for congestion control - _congestionControl->setRTT(_rtt); - - // update the last ACKed ACK - if (pair.first > _lastReceivedAcknowledgedACK) { - _lastReceivedAcknowledgedACK = pair.first; + if (it->first == subSequenceNumber){ + // update the RTT using the ACK window + + // calculate the RTT (time now - time ACK sent) + auto now = high_resolution_clock::now(); + int rtt = duration_cast(now - it->second.second).count(); + + updateRTT(rtt); + + // set the RTT for congestion control + _congestionControl->setRTT(_rtt); + + // update the last ACKed ACK + if (it->second.first > _lastReceivedAcknowledgedACK) { + _lastReceivedAcknowledgedACK = it->second.first; + } + + // erase this sub-sequence number and anything below it now that we've gotten our timing information + _sentACKs.erase(_sentACKs.begin(), it); + } else { + Q_UNREACHABLE(); } } diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 23a77bf4db..dd86f9bc5b 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -13,6 +13,7 @@ #define hifi_Connection_h #include +#include #include #include @@ -35,7 +36,8 @@ class Connection : public QObject { Q_OBJECT public: using SequenceNumberTimePair = std::pair; - using SentACKMap = std::unordered_map; + using ACKListPair = std::pair; + using SentACKList = std::list; Connection(Socket* parentSocket, HifiSockAddr destination, std::unique_ptr congestionControl); ~Connection(); @@ -100,7 +102,7 @@ private: int _bandwidth { 1 }; // Exponential moving average for estimated bandwidth, in packets per second int _deliveryRate { 16 }; // Exponential moving average for receiver's receive rate, in packets per second - SentACKMap _sentACKs; // Map of ACK sub-sequence numbers to ACKed sequence number and sent time + SentACKList _sentACKs; // Map of ACK sub-sequence numbers to ACKed sequence number and sent time Socket* _parentSocket { nullptr }; HifiSockAddr _destination; From 8f54cd6f7ebc32b33b6537fa6bd73e366e74bfa9 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 31 Jul 2015 18:19:15 -0700 Subject: [PATCH 171/242] Fix find_if_not --- libraries/networking/src/udt/Connection.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 62336a31ae..dd34a2bad2 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -466,8 +466,8 @@ void Connection::processACK2(std::unique_ptr controlPacket) { controlPacket->readPrimitive(&subSequenceNumber); // check if we had that subsequence number in our map - auto it = std::find_if_not(_sentACKs.begin(), _sentACKs.end(), [subSequenceNumber](const ACKListPair& pair){ - return subSequenceNumber < pair.first; + auto it = std::find_if(_sentACKs.begin(), _sentACKs.end(), [subSequenceNumber](const ACKListPair& pair){ + return subSequenceNumber >= pair.first; }); if (it != _sentACKs.end()) { From 8bc94454b447c4918f1dd5ad506e8df2af96421d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 18:39:09 -0700 Subject: [PATCH 172/242] pre-increment the _currentACKSubSequenceNumber --- libraries/networking/src/udt/Connection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index dd34a2bad2..5c3e1846ce 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -129,7 +129,7 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { ackPacket->reset(); // We need to reset it every time. // pack in the ACK sub-sequence number - ackPacket->writePrimitive(_currentACKSubSequenceNumber++); + ackPacket->writePrimitive(++_currentACKSubSequenceNumber); // pack in the ACK number ackPacket->writePrimitive(nextACKNumber); From ca342fb3b4bbf1579a0e815b2789c94c5f08b701 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 31 Jul 2015 18:39:24 -0700 Subject: [PATCH 173/242] Fix std::find_if --- libraries/networking/src/udt/Connection.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index dd34a2bad2..18e3fb9668 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -466,8 +466,8 @@ void Connection::processACK2(std::unique_ptr controlPacket) { controlPacket->readPrimitive(&subSequenceNumber); // check if we had that subsequence number in our map - auto it = std::find_if(_sentACKs.begin(), _sentACKs.end(), [subSequenceNumber](const ACKListPair& pair){ - return subSequenceNumber >= pair.first; + auto it = std::find_if_not(_sentACKs.begin(), _sentACKs.end(), [&subSequenceNumber](const ACKListPair& pair){ + return pair.first < subSequenceNumber; }); if (it != _sentACKs.end()) { @@ -487,14 +487,14 @@ void Connection::processACK2(std::unique_ptr controlPacket) { if (it->second.first > _lastReceivedAcknowledgedACK) { _lastReceivedAcknowledgedACK = it->second.first; } - - // erase this sub-sequence number and anything below it now that we've gotten our timing information - _sentACKs.erase(_sentACKs.begin(), it); - } else { + } else if (it->first < subSequenceNumber) { Q_UNREACHABLE(); } } + // erase this sub-sequence number and anything below it now that we've gotten our timing information + _sentACKs.erase(_sentACKs.begin(), it); + _stats.recordReceivedACK2(); } From 3833623cfe87b9cfeca20eb7bd69ec6345a7b120 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 18:41:46 -0700 Subject: [PATCH 174/242] fix assert for control packet type --- libraries/networking/src/udt/ControlPacket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/ControlPacket.cpp b/libraries/networking/src/udt/ControlPacket.cpp index 30bf8d452e..b8407d42a4 100644 --- a/libraries/networking/src/udt/ControlPacket.cpp +++ b/libraries/networking/src/udt/ControlPacket.cpp @@ -100,7 +100,7 @@ void ControlPacket::readType() { Q_ASSERT_X(bitAndType & CONTROL_BIT_MASK, "ControlPacket::readHeader()", "This should be a control packet"); uint16_t packetType = (bitAndType & ~CONTROL_BIT_MASK) >> (8 * sizeof(Type)); - Q_ASSERT_X(packetType < ControlPacket::Type::TimeoutNAK, "ControlPacket::readType()", "Received a control packet with wrong type"); + Q_ASSERT_X(packetType <= ControlPacket::Type::TimeoutNAK, "ControlPacket::readType()", "Received a control packet with wrong type"); // read the type _type = (Type) packetType; From 74a2d985ebb1fd6a985d1d516e79b67102557a43 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 19:17:02 -0700 Subject: [PATCH 175/242] tell the send queue about ACKs on light ACK --- libraries/networking/src/udt/Connection.cpp | 13 ++++++++----- tools/udt-test/src/UDTTest.cpp | 5 +++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 24911d04cb..8ad7259347 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -102,7 +102,7 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { if (nextACKNumber == _lastSentACK) { // We already sent this ACK, but check if we should re-send it. - if (nextACKNumber <= _lastReceivedAcknowledgedACK) { + if (nextACKNumber < _lastReceivedAcknowledgedACK) { // we already got an ACK2 for this ACK we would be sending, don't bother return; } @@ -375,11 +375,11 @@ void Connection::processACK(std::unique_ptr controlPacket) { // read the ACKed sequence number SequenceNumber ack; controlPacket->readPrimitive(&ack); - + // validate that this isn't a BS ACK if (ack > getSendQueue().getCurrentSequenceNumber()) { // in UDT they specifically break the connection here - do we want to do anything? - Q_ASSERT_X(true, "Connection::processACK", "ACK recieved higher than largest sent sequence number"); + Q_ASSERT_X(false, "Connection::processACK", "ACK recieved higher than largest sent sequence number"); return; } @@ -388,7 +388,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { controlPacket->readPrimitive(&rtt); if (ack < _lastReceivedACK) { - // Bail + // this is an out of order ACK, bail return; } @@ -396,7 +396,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { controlPacket->readPrimitive(&_flowWindowSize); if (ack == _lastReceivedACK) { - // Bail + // processing an already received ACK, bail return; } @@ -455,6 +455,9 @@ void Connection::processLightACK(std::unique_ptr controlPacket) { // update the last received ACK to the this one _lastReceivedACK = ack; + + // send light ACK to the send queue + getSendQueue().ack(ack); } _stats.recordReceivedLightACK(); diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index d0c7b159df..97de033dfe 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -42,7 +42,7 @@ const QCommandLineOption UNRELIABLE_PACKETS { }; const QStringList STATS_TABLE_HEADERS { - "Send Rate (P/s)", "RTT(ms)", "CW (P)", "Send Period (us)", "Received ACK", "Received NAK", "Sent ACK2" + "Send Rate (P/s)", "RTT(ms)", "CW (P)", "Send Period (us)", "Received ACK", "Received NAK", "Sent ACK2", "Re-sent Packets" }; UDTTest::UDTTest(int& argc, char** argv) : @@ -228,7 +228,8 @@ void UDTTest::sampleStats() { QString::number(stats.packetSendPeriod).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.receivedACKs).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.receivedNAKs).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.sentACK2s).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()) + QString::number(stats.sentACK2s).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.retransmissions).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()) }; // output this line of values From 7f8c993bd7726090db7fea914615d7f325ea169e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 19:42:24 -0700 Subject: [PATCH 176/242] make send rate actually send rate, don't sync as sender --- libraries/networking/src/udt/Connection.cpp | 27 +++++++++++-------- libraries/networking/src/udt/Connection.h | 2 ++ .../networking/src/udt/ConnectionStats.cpp | 5 ++++ .../networking/src/udt/ConnectionStats.h | 2 ++ libraries/networking/src/udt/SendQueue.cpp | 2 ++ tools/udt-test/src/UDTTest.cpp | 2 +- 6 files changed, 28 insertions(+), 12 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 8ad7259347..c27d5d5bed 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -72,16 +72,18 @@ void Connection::sendReliablePacket(unique_ptr packet) { } void Connection::sync() { - // we send out a periodic ACK every rate control interval - sendACK(); - - // check if we need to re-transmit a loss list - // we do this if it has been longer than the current nakInterval since we last sent - auto now = high_resolution_clock::now(); - - if (duration_cast(now - _lastNAKTime).count() >= _nakInterval) { - // Send a timeout NAK packet - sendTimeoutNAK(); + if (_hasReceivedFirstPacket) { + // we send out a periodic ACK every rate control interval + sendACK(); + + // check if we need to re-transmit a loss list + // we do this if it has been longer than the current nakInterval since we last sent + auto now = high_resolution_clock::now(); + + if (duration_cast(now - _lastNAKTime).count() >= _nakInterval) { + // Send a timeout NAK packet + sendTimeoutNAK(); + } } } @@ -265,6 +267,9 @@ SequenceNumber Connection::nextACK() const { } bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, int packetSize, int payloadSize) { + + _hasReceivedFirstPacket = true; + // check if this is a packet pair we should estimate bandwidth from, or just a regular packet if (((uint32_t) sequenceNumber & 0xF) == 0) { _receiveWindow.onProbePair1Arrival(); @@ -424,7 +429,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { static const int EMWA_ALPHA_NUMERATOR = 8; // record these samples in connection stats - _stats.recordReceiveRate(receiveRate); + _stats.recordSendRate(receiveRate); _stats.recordEstimatedBandwidth(bandwidth); _deliveryRate = (_deliveryRate * (EMWA_ALPHA_NUMERATOR - 1) + receiveRate) / EMWA_ALPHA_NUMERATOR; diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index dd86f9bc5b..0876a814e3 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -86,6 +86,8 @@ private: int _minNAKInterval { 100000 }; // NAK timeout interval lower bound, default of 100ms std::chrono::high_resolution_clock::time_point _lastNAKTime; + bool _hasReceivedFirstPacket { false }; + LossList _lossList; // List of all missing packets SequenceNumber _lastReceivedSequenceNumber; // The largest sequence number received from the peer SequenceNumber _lastReceivedACK; // The last ACK received diff --git a/libraries/networking/src/udt/ConnectionStats.cpp b/libraries/networking/src/udt/ConnectionStats.cpp index 62497ab496..1f8308bf5d 100644 --- a/libraries/networking/src/udt/ConnectionStats.cpp +++ b/libraries/networking/src/udt/ConnectionStats.cpp @@ -139,6 +139,11 @@ void ConnectionStats::recordDuplicates() { static const double EWMA_CURRENT_SAMPLE_WEIGHT = 0.125; static const double EWMA_PREVIOUS_SAMPLES_WEIGHT = 1 - 0.125; +void ConnectionStats::recordSendRate(int sample) { + _currentSample.sendRate = sample; + _total.sendRate = (_total.sendRate * EWMA_PREVIOUS_SAMPLES_WEIGHT) + (sample * EWMA_CURRENT_SAMPLE_WEIGHT); +} + void ConnectionStats::recordReceiveRate(int sample) { _currentSample.receiveRate = sample; _total.receiveRate = (_total.receiveRate * EWMA_PREVIOUS_SAMPLES_WEIGHT) + (sample * EWMA_CURRENT_SAMPLE_WEIGHT); diff --git a/libraries/networking/src/udt/ConnectionStats.h b/libraries/networking/src/udt/ConnectionStats.h index f4b912ace5..27d8bbd7a1 100644 --- a/libraries/networking/src/udt/ConnectionStats.h +++ b/libraries/networking/src/udt/ConnectionStats.h @@ -52,6 +52,7 @@ public: int duplicates { 0 }; // the following stats are trailing averages in the result, not totals + int sendRate { 0 }; int receiveRate { 0 }; int estimatedBandwith { 0 }; int rtt { 0 }; @@ -84,6 +85,7 @@ public: void recordRetransmission(); void recordDuplicates(); + void recordSendRate(int sample); void recordReceiveRate(int sample); void recordEstimatedBandwidth(int sample); void recordRTT(int sample); diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index f271a1c685..508e674278 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -108,6 +108,8 @@ void SendQueue::overrideNAKListFromPacket(ControlPacket& packet) { packet.readPrimitive(&first); packet.readPrimitive(&second); + qDebug() << "NAK" << (uint32_t) first << (uint32_t) second; + if (first == second) { _naks.append(first); } else { diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 97de033dfe..780e035a89 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -222,7 +222,7 @@ void UDTTest::sampleStats() { // setup a list of left justified values QStringList values { - QString::number(stats.receiveRate).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.sendRate).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.rtt / USECS_PER_MSEC).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.congestionWindowSize).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.packetSendPeriod).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), From 0c44fc53ded8d4560820c2fe1cef1e50d7b6a8de Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 19:45:43 -0700 Subject: [PATCH 177/242] fix for process of timeout NAKs --- libraries/networking/src/udt/SendQueue.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 508e674278..7186316bc0 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -104,12 +104,10 @@ void SendQueue::overrideNAKListFromPacket(ControlPacket& packet) { _naks.clear(); SequenceNumber first, second; - while (packet.bytesLeftToRead() > (qint64)(2 * sizeof(SequenceNumber))) { + while (packet.bytesLeftToRead() >= (qint64)(2 * sizeof(SequenceNumber))) { packet.readPrimitive(&first); packet.readPrimitive(&second); - qDebug() << "NAK" << (uint32_t) first << (uint32_t) second; - if (first == second) { _naks.append(first); } else { From 826c8d5150a1457be57f04380ea21f814edbe86e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 19:52:20 -0700 Subject: [PATCH 178/242] more stats in UDTTest for sender --- tools/udt-test/src/UDTTest.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 780e035a89..eaa7530989 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -42,7 +42,9 @@ const QCommandLineOption UNRELIABLE_PACKETS { }; const QStringList STATS_TABLE_HEADERS { - "Send Rate (P/s)", "RTT(ms)", "CW (P)", "Send Period (us)", "Received ACK", "Received NAK", "Sent ACK2", "Re-sent Packets" + "Send Rate (P/s)", "RTT(ms)", "CW (P)", "Send Period (us)", + "Received ACK", "Received L-ACK", "Received NAK", "Received TNAK", + "Sent ACK2", "Re-sent Packets" }; UDTTest::UDTTest(int& argc, char** argv) : @@ -227,7 +229,9 @@ void UDTTest::sampleStats() { QString::number(stats.congestionWindowSize).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.packetSendPeriod).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.receivedACKs).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.receivedLightACKs).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.receivedNAKs).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.receivedTimeoutNAKs).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.sentACK2s).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.retransmissions).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()) }; From bc5ddd38374776e72909711ede5c3f3bf41867af Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 31 Jul 2015 20:12:48 -0700 Subject: [PATCH 179/242] send a light ACK every 64, not each after 64 --- libraries/networking/src/udt/Connection.cpp | 7 ++++++- libraries/networking/src/udt/Connection.h | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index c27d5d5bed..41bdb36c98 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -73,6 +73,9 @@ void Connection::sendReliablePacket(unique_ptr packet) { void Connection::sync() { if (_hasReceivedFirstPacket) { + // reset the number of light ACKS during this sync interval + _lightACKsDuringSYN = 1; + // we send out a periodic ACK every rate control interval sendACK(); @@ -322,8 +325,10 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in // check if we need to send an ACK, according to CC params if (_congestionControl->_ackInterval > 0 && _packetsSinceACK >= _congestionControl->_ackInterval) { sendACK(false); - } else if (_congestionControl->_lightACKInterval > 0 && _packetsSinceACK >= _congestionControl->_lightACKInterval) { + } else if (_congestionControl->_lightACKInterval > 0 + && _packetsSinceACK >= _congestionControl->_lightACKInterval * _lightACKsDuringSYN) { sendLightACK(); + ++_lightACKsDuringSYN; } if (wasDuplicate) { diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 0876a814e3..99328d601f 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -97,6 +97,8 @@ private: SequenceNumber _lastSentACK; // The last sent ACK SequenceNumber _lastSentACK2; // The last sent ACK sub-sequence number in an ACK2 + int _lightACKsDuringSYN { 1 }; // The number of lite ACKs sent during SYN interval + int32_t _rtt; // RTT, in microseconds int32_t _rttVariance; // RTT variance int _flowWindowSize { udt::MAX_PACKETS_IN_FLIGHT }; // Flow control window size From ffb5fcee7b9c273a9098a016b6e9f6d2ad83db0d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 3 Aug 2015 10:25:40 -0700 Subject: [PATCH 180/242] use nth element for median in PacketTimeWindow --- .../networking/src/udt/PacketTimeWindow.cpp | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/libraries/networking/src/udt/PacketTimeWindow.cpp b/libraries/networking/src/udt/PacketTimeWindow.cpp index e28b276ed9..269d158b7d 100644 --- a/libraries/networking/src/udt/PacketTimeWindow.cpp +++ b/libraries/networking/src/udt/PacketTimeWindow.cpp @@ -11,6 +11,7 @@ #include "PacketTimeWindow.h" +#include #include #include @@ -30,22 +31,32 @@ PacketTimeWindow::PacketTimeWindow(int numPacketIntervals, int numProbeIntervals } -int32_t meanOfMedianFilteredValues(std::vector intervals, int numValues, int valuesRequired = 0) { - // sort the intervals from smallest to largest - std::sort(intervals.begin(), intervals.end()); +template +int median(Iterator begin, Iterator end) { + // use std::nth_element to grab the middle - for an even number of elements this is the upper middle + Iterator middle = begin + (end - begin) / 2; + std::nth_element(begin, middle, end); - int median = 0; - if (numValues % 2 == 0) { - median = intervals[numValues / 2]; + if ((end - begin) % 2 != 0) { + // odd number of elements, just return the middle + return *middle; } else { - median = (intervals[(numValues / 2) - 1] + intervals[numValues / 2]) / 2; + // even number of elements, return the mean of the upper middle and the lower middle + Iterator lowerMiddle = std::max_element(begin, middle); + return (*middle + *lowerMiddle) / 2; } +} + +int32_t meanOfMedianFilteredValues(std::vector intervals, int numValues, int valuesRequired = 0) { + // grab the median value of the intervals vector + int intervalsMedian = median(intervals.begin(), intervals.end()); - int count = 0; - int sum = 0; static const int MEDIAN_FILTERING_BOUND_MULTIPLIER = 8; - int upperBound = median * MEDIAN_FILTERING_BOUND_MULTIPLIER; - int lowerBound = median / MEDIAN_FILTERING_BOUND_MULTIPLIER; + int upperBound = intervalsMedian * MEDIAN_FILTERING_BOUND_MULTIPLIER; + int lowerBound = intervalsMedian / MEDIAN_FILTERING_BOUND_MULTIPLIER; + + int sum = 0; + int count = 0; for (auto& interval : intervals) { if ((interval < upperBound) && (interval > lowerBound)) { From 4230d021848a40f1e1218c709e07f12db9357bc5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 3 Aug 2015 11:06:48 -0700 Subject: [PATCH 181/242] rename col for light ACKs --- tools/udt-test/src/UDTTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index eaa7530989..babf4b8c25 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -43,7 +43,7 @@ const QCommandLineOption UNRELIABLE_PACKETS { const QStringList STATS_TABLE_HEADERS { "Send Rate (P/s)", "RTT(ms)", "CW (P)", "Send Period (us)", - "Received ACK", "Received L-ACK", "Received NAK", "Received TNAK", + "Received ACK", "Received LACK", "Received NAK", "Received TNAK", "Sent ACK2", "Re-sent Packets" }; From f513a28953da79aa1f770038b9aecbeceb63027f Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 3 Aug 2015 11:06:57 -0700 Subject: [PATCH 182/242] Added better assert in LossList --- libraries/networking/src/udt/LossList.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/LossList.cpp b/libraries/networking/src/udt/LossList.cpp index 90399e319d..bf9c081943 100644 --- a/libraries/networking/src/udt/LossList.cpp +++ b/libraries/networking/src/udt/LossList.cpp @@ -17,7 +17,8 @@ using namespace udt; using namespace std; void LossList::append(SequenceNumber seq) { - assert(_lossList.empty() || _lossList.back().second < seq); + Q_ASSERT_X(_lossList.empty() || (_lossList.back().second < seq), "LossList::append(SequenceNumber)", + "SequenceNumber appended is not greater than the last SequenceNumber in the list"); if (getLength() > 0 && _lossList.back().second + 1 == seq) { ++_lossList.back().second; @@ -28,6 +29,12 @@ void LossList::append(SequenceNumber seq) { } void LossList::append(SequenceNumber start, SequenceNumber end) { + Q_ASSERT_X(_lossList.empty() || (_lossList.back().second < start), + "LossList::append(SequenceNumber, SequenceNumber)", + "SequenceNumber range appended is not greater than the last SequenceNumber in the list"); + Q_ASSERT_X(start <= end, + "LossList::append(SequenceNumber, SequenceNumber)", "Range start greater than range end"); + if (getLength() > 0 && _lossList.back().second + 1 == start) { _lossList.back().second = end; } else { @@ -37,6 +44,9 @@ void LossList::append(SequenceNumber start, SequenceNumber end) { } void LossList::insert(SequenceNumber start, SequenceNumber end) { + Q_ASSERT_X(start <= end, + "LossList::insert(SequenceNumber, SequenceNumber)", "Range start greater than range end"); + auto it = find_if_not(_lossList.begin(), _lossList.end(), [&start](pair pair){ return pair.second < start; }); @@ -109,6 +119,8 @@ bool LossList::remove(SequenceNumber seq) { } void LossList::remove(SequenceNumber start, SequenceNumber end) { + Q_ASSERT_X(start <= end, + "LossList::remove(SequenceNumber, SequenceNumber)", "Range start greater than range end"); // Find the first segment sharing sequence numbers auto it = find_if(_lossList.begin(), _lossList.end(), [&start, &end](pair pair) { return (pair.first <= start && start <= pair.second) || (start <= pair.first && pair.first <= end); @@ -151,7 +163,7 @@ void LossList::remove(SequenceNumber start, SequenceNumber end) { } SequenceNumber LossList::getFirstSequenceNumber() const { - assert(getLength() > 0); + Q_ASSERT_X(getLength() > 0, "LossList::getFirstSequenceNumber()", "Trying to get first element of an empty list"); return _lossList.front().first; } From de8f730f784fce7e91e412bc0d59173579b1118d Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 3 Aug 2015 11:07:58 -0700 Subject: [PATCH 183/242] Fix connection stats error --- libraries/networking/src/udt/ConnectionStats.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/ConnectionStats.cpp b/libraries/networking/src/udt/ConnectionStats.cpp index 1f8308bf5d..26112eb24f 100644 --- a/libraries/networking/src/udt/ConnectionStats.cpp +++ b/libraries/networking/src/udt/ConnectionStats.cpp @@ -100,7 +100,7 @@ void ConnectionStats::recordReceivedPackets(int payload, int total) { _currentSample.recievedUtilBytes += payload; _total.recievedUtilBytes += payload; - _currentSample.sentBytes += total; + _currentSample.recievedBytes += total; _total.recievedBytes += total; } From 62d4467f416026d94ff054843eb26df673f2d946 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 3 Aug 2015 12:09:25 -0700 Subject: [PATCH 184/242] use a vector for events in ConnectionStats --- libraries/networking/src/udt/Connection.cpp | 28 +++++---- .../networking/src/udt/ConnectionStats.cpp | 62 +------------------ .../networking/src/udt/ConnectionStats.h | 48 +++++++------- tools/udt-test/src/UDTTest.cpp | 15 ++--- 4 files changed, 47 insertions(+), 106 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 41bdb36c98..1786f7558e 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -95,7 +95,7 @@ void Connection::recordSentPackets(int dataSize, int payloadSize) { } void Connection::recordRetransmission() { - _stats.recordRetransmission(); + _stats.record(ConnectionStats::Stats::Retransmission); } void Connection::sendACK(bool wasCausedBySyncTimeout) { @@ -175,7 +175,7 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { // reset the number of data packets received since last ACK _packetsSinceACK = 0; - _stats.recordSentACK(); + _stats.record(ConnectionStats::Stats::SentACK); } void Connection::sendLightACK() { @@ -199,7 +199,7 @@ void Connection::sendLightACK() { // have the socket send off our packet immediately _parentSocket->writeBasePacket(*lightACKPacket, _destination); - _stats.recordSentLightACK(); + _stats.record(ConnectionStats::Stats::SentLightACK); } void Connection::sendACK2(SequenceNumber currentACKSubSequenceNumber) { @@ -219,7 +219,7 @@ void Connection::sendACK2(SequenceNumber currentACKSubSequenceNumber) { // update the last sent ACK2 and the last ACK2 send time _lastSentACK2 = currentACKSubSequenceNumber; - _stats.recordSentACK2(); + _stats.record(ConnectionStats::Stats::SentACK2); } void Connection::sendNAK(SequenceNumber sequenceNumberRecieved) { @@ -240,7 +240,7 @@ void Connection::sendNAK(SequenceNumber sequenceNumberRecieved) { // record our last NAK time _lastNAKTime = high_resolution_clock::now(); - _stats.recordSentNAK(); + _stats.record(ConnectionStats::Stats::SentNAK); } void Connection::sendTimeoutNAK() { @@ -257,7 +257,7 @@ void Connection::sendTimeoutNAK() { // record this as the last NAK time _lastNAKTime = high_resolution_clock::now(); - _stats.recordSentTimeoutNAK(); + _stats.record(ConnectionStats::Stats::SentTimeoutNAK); } } @@ -332,7 +332,7 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in } if (wasDuplicate) { - _stats.recordDuplicates(); + _stats.record(ConnectionStats::Stats::Duplicate); } else { _stats.recordReceivedPackets(payloadSize, packetSize); } @@ -386,6 +386,9 @@ void Connection::processACK(std::unique_ptr controlPacket) { SequenceNumber ack; controlPacket->readPrimitive(&ack); + // update the total count of received ACKs + _stats.record(ConnectionStats::Stats::ReceivedACK); + // validate that this isn't a BS ACK if (ack > getSendQueue().getCurrentSequenceNumber()) { // in UDT they specifically break the connection here - do we want to do anything? @@ -449,8 +452,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { _congestionControl->onACK(ack); }); - // update the total count of received ACKs - _stats.recordReceivedACK(); + _stats.record(ConnectionStats::Stats::ProcessedACK); } void Connection::processLightACK(std::unique_ptr controlPacket) { @@ -470,7 +472,7 @@ void Connection::processLightACK(std::unique_ptr controlPacket) { getSendQueue().ack(ack); } - _stats.recordReceivedLightACK(); + _stats.record(ConnectionStats::Stats::ReceivedLightACK); } void Connection::processACK2(std::unique_ptr controlPacket) { @@ -508,7 +510,7 @@ void Connection::processACK2(std::unique_ptr controlPacket) { // erase this sub-sequence number and anything below it now that we've gotten our timing information _sentACKs.erase(_sentACKs.begin(), it); - _stats.recordReceivedACK2(); + _stats.record(ConnectionStats::Stats::ReceivedACK2); } void Connection::processNAK(std::unique_ptr controlPacket) { @@ -530,7 +532,7 @@ void Connection::processNAK(std::unique_ptr controlPacket) { _congestionControl->onLoss(start, end); }); - _stats.recordReceivedNAK(); + _stats.record(ConnectionStats::Stats::ReceivedNAK); } void Connection::processTimeoutNAK(std::unique_ptr controlPacket) { @@ -540,7 +542,7 @@ void Connection::processTimeoutNAK(std::unique_ptr controlPacket) // we don't tell the congestion control object there was loss here - this matches UDTs implementation // a possible improvement would be to tell it which new loss this timeout packet told us about - _stats.recordReceivedTimeoutNAK(); + _stats.record(ConnectionStats::Stats::ReceivedTimeoutNAK); } void Connection::updateRTT(int rtt) { diff --git a/libraries/networking/src/udt/ConnectionStats.cpp b/libraries/networking/src/udt/ConnectionStats.cpp index 26112eb24f..91eeb6d49c 100644 --- a/libraries/networking/src/udt/ConnectionStats.cpp +++ b/libraries/networking/src/udt/ConnectionStats.cpp @@ -31,57 +31,11 @@ ConnectionStats::Stats ConnectionStats::sample() { return sample; } -void ConnectionStats::recordSentACK() { - ++_currentSample.sentACKs; - ++_total.sentACKs; +void ConnectionStats::record(Stats::Event event) { + ++_currentSample.events[(int) event]; + ++_total.events[(int) event]; } -void ConnectionStats::recordReceivedACK() { - ++_currentSample.receivedACKs; - ++_total.receivedACKs; -} - -void ConnectionStats::recordSentLightACK() { - ++_currentSample.sentLightACKs; - ++_total.sentLightACKs; -} - -void ConnectionStats::recordReceivedLightACK() { - ++_currentSample.receivedLightACKs; - ++_total.receivedLightACKs; -} - -void ConnectionStats::recordSentACK2() { - ++_currentSample.sentACK2s; - ++_total.sentACK2s; -} - -void ConnectionStats::recordReceivedACK2() { - ++_currentSample.receivedACK2s; - ++_total.receivedACK2s; -} - -void ConnectionStats::recordSentNAK() { - ++_currentSample.sentNAKs; - ++_total.sentNAKs; -} - -void ConnectionStats::recordReceivedNAK() { - ++_currentSample.receivedNAKs; - ++_total.receivedNAKs; -} - -void ConnectionStats::recordSentTimeoutNAK() { - ++_currentSample.sentTimeoutNAKs; - ++_total.sentTimeoutNAKs; -} - -void ConnectionStats::recordReceivedTimeoutNAK() { - ++_currentSample.receivedTimeoutNAKs; - ++_total.receivedTimeoutNAKs; -} - - void ConnectionStats::recordSentPackets(int payload, int total) { ++_currentSample.sentPackets; ++_total.sentPackets; @@ -126,16 +80,6 @@ void ConnectionStats::recordUnreliableReceivedPackets(int payload, int total) { _total.recievedUnreliableBytes += total; } -void ConnectionStats::recordRetransmission() { - ++_currentSample.retransmissions; - ++_total.retransmissions; -} - -void ConnectionStats::recordDuplicates() { - ++_currentSample.duplicates; - ++_total.duplicates; -} - static const double EWMA_CURRENT_SAMPLE_WEIGHT = 0.125; static const double EWMA_PREVIOUS_SAMPLES_WEIGHT = 1 - 0.125; diff --git a/libraries/networking/src/udt/ConnectionStats.h b/libraries/networking/src/udt/ConnectionStats.h index 27d8bbd7a1..2800db7720 100644 --- a/libraries/networking/src/udt/ConnectionStats.h +++ b/libraries/networking/src/udt/ConnectionStats.h @@ -13,6 +13,7 @@ #define hifi_ConnectionStats_h #include +#include namespace udt { @@ -22,18 +23,26 @@ public: std::chrono::microseconds startTime; std::chrono::microseconds endTime; - // Control Packet stat collection - int sentACKs { 0 }; - int receivedACKs { 0 }; - int sentLightACKs { 0 }; - int receivedLightACKs { 0 }; - int sentACK2s { 0 }; - int receivedACK2s { 0 }; - int sentNAKs { 0 }; - int receivedNAKs { 0 }; - int sentTimeoutNAKs { 0 }; - int receivedTimeoutNAKs { 0 }; + enum Event { + SentACK, + ReceivedACK, + ProcessedACK, + SentLightACK, + ReceivedLightACK, + SentACK2, + ReceivedACK2, + SentNAK, + ReceivedNAK, + SentTimeoutNAK, + ReceivedTimeoutNAK, + Retransmission, + Duplicate + }; + // construct a vector for the events of the size of our Enum - default value is zero + std::vector events = std::vector((int) Event::Duplicate + 1, 0); + + // packet counts and sizes int sentPackets { 0 }; int recievedPackets { 0 }; int sentUtilBytes { 0 }; @@ -47,9 +56,6 @@ public: int recievedUnreliableUtilBytes { 0 }; int sentUnreliableBytes { 0 }; int recievedUnreliableBytes { 0 }; - - int retransmissions { 0 }; - int duplicates { 0 }; // the following stats are trailing averages in the result, not totals int sendRate { 0 }; @@ -65,16 +71,7 @@ public: Stats sample(); Stats getTotalStats(); - void recordSentACK(); - void recordReceivedACK(); - void recordSentLightACK(); - void recordReceivedLightACK(); - void recordSentACK2(); - void recordReceivedACK2(); - void recordSentNAK(); - void recordReceivedNAK(); - void recordSentTimeoutNAK(); - void recordReceivedTimeoutNAK(); + void record(Stats::Event event); void recordSentPackets(int payload, int total); void recordReceivedPackets(int payload, int total); @@ -82,9 +79,6 @@ public: void recordUnreliableSentPackets(int payload, int total); void recordUnreliableReceivedPackets(int payload, int total); - void recordRetransmission(); - void recordDuplicates(); - void recordSendRate(int sample); void recordReceiveRate(int sample); void recordEstimatedBandwidth(int sample); diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index babf4b8c25..340c6a16a4 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -43,7 +43,7 @@ const QCommandLineOption UNRELIABLE_PACKETS { const QStringList STATS_TABLE_HEADERS { "Send Rate (P/s)", "RTT(ms)", "CW (P)", "Send Period (us)", - "Received ACK", "Received LACK", "Received NAK", "Received TNAK", + "Received ACK", "Processed ACK", "Received LACK", "Received NAK", "Received TNAK", "Sent ACK2", "Re-sent Packets" }; @@ -228,12 +228,13 @@ void UDTTest::sampleStats() { QString::number(stats.rtt / USECS_PER_MSEC).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.congestionWindowSize).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.packetSendPeriod).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.receivedACKs).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.receivedLightACKs).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.receivedNAKs).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.receivedTimeoutNAKs).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.sentACK2s).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.retransmissions).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()) + QString::number(stats.events[udt::ConnectionStats::Stats::ReceivedACK]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::ProcessedACK]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::ReceivedLightACK]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::ReceivedNAK]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::ReceivedTimeoutNAK]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::SentACK2]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::Retransmission]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()) }; // output this line of values From 00ab4a58f308ad5e60e5b40d8733b68b37cc4b8e Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 3 Aug 2015 12:12:21 -0700 Subject: [PATCH 185/242] Fix LossList::insert --- libraries/networking/src/udt/LossList.cpp | 40 ++++++++++------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/libraries/networking/src/udt/LossList.cpp b/libraries/networking/src/udt/LossList.cpp index bf9c081943..488ac6610f 100644 --- a/libraries/networking/src/udt/LossList.cpp +++ b/libraries/networking/src/udt/LossList.cpp @@ -51,10 +51,10 @@ void LossList::insert(SequenceNumber start, SequenceNumber end) { return pair.second < start; }); - if (it == _lossList.end()) { + if (it == _lossList.end() || end < it->first) { // No overlap, simply insert - _lossList.insert(it, make_pair(start, end)); _length += seqlen(start, end); + _lossList.insert(it, make_pair(start, end)); } else { // If it starts before segment, extend segment if (start < it->first) { @@ -62,31 +62,25 @@ void LossList::insert(SequenceNumber start, SequenceNumber end) { it->first = start; } + // If it ends after segment, extend segment if (end > it->second) { - // If it goes further, find the actual end - auto it2 = find_if_not(it, _lossList.end(), [&end](pair pair){ - return end <= pair.second; - }); - --it2; - - // If it ends inside a segment, change end (segment will be deleted) - // Or backup iterator so segment doesn't get deleted - if (it2->first <= end) { - end = it2->second; - } else { - --it2; - } - - // Change the end of the original segment _length += seqlen(it->second + 1, end); it->second = end; - - // remove all underlapping segments - ++it; ++it2; - while (it != it2) { - _length -= seqlen(it->first, it->second); - it = _lossList.erase(it); + } + + auto it2 = it; + ++it2; + // For all ranges touching the current range + while (it2 != _lossList.end() && it->second >= it2->first - 1) { + // extend current range if necessary + if (it->second < it2->second) { + _length += seqlen(it->second + 1, it2->second); + it->second = it2->second; } + + // Remove overlapping range + _length -= seqlen(it2->first, it2->second); + it2 = _lossList.erase(it2); } } } From 51b47a12120ffd65b16079585f5101a5e62fec06 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 3 Aug 2015 13:27:00 -0700 Subject: [PATCH 186/242] change stats sampling to 100ms --- tools/udt-test/src/UDTTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 340c6a16a4..f6ce1cc5ac 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -123,8 +123,8 @@ UDTTest::UDTTest(int& argc, char** argv) : if (!_target.isNull()) { sendInitialPackets(); - // the sender reports stats every 1 second - static const int STATS_SAMPLE_INTERVAL = 1000; + // the sender reports stats every 100ms + static const int STATS_SAMPLE_INTERVAL = 100; QTimer* statsTimer = new QTimer(this); connect(statsTimer, &QTimer::timeout, this, &UDTTest::sampleStats); From f2b6db584a13d03fabee8ae6ac0e73e81e52e24c Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 3 Aug 2015 13:34:40 -0700 Subject: [PATCH 187/242] Fix list insert --- libraries/networking/src/udt/LossList.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/LossList.cpp b/libraries/networking/src/udt/LossList.cpp index 488ac6610f..4cb182c2f7 100644 --- a/libraries/networking/src/udt/LossList.cpp +++ b/libraries/networking/src/udt/LossList.cpp @@ -100,7 +100,7 @@ bool LossList::remove(SequenceNumber seq) { } else { auto temp = it->second; it->second = seq - 1; - _lossList.insert(it, make_pair(seq + 1, temp)); + it = _lossList.insert(++it, make_pair(seq + 1, temp)); } _length -= 1; @@ -150,7 +150,7 @@ void LossList::remove(SequenceNumber start, SequenceNumber end) { _length -= seqlen(start, end); auto temp = it->second; it->second = start - 1; - _lossList.insert(it, make_pair(end + 1, temp)); + _lossList.insert(++it, make_pair(end + 1, temp)); } } } From 20f33b3b64f40349e7dbca4ce4e66aae55aa5011 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 3 Aug 2015 14:34:12 -0700 Subject: [PATCH 188/242] fix sendqueue sleep timing to use microseconds --- libraries/networking/src/udt/SendQueue.cpp | 2 +- tools/udt-test/src/UDTTest.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 7186316bc0..3aeeec4d4a 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -250,7 +250,7 @@ void SendQueue::run() { // sleep as long as we need until next packet send, if we can auto now = high_resolution_clock::now(); - auto microsecondDuration = (_lastSendTimestamp + microseconds(_packetSendPeriod)) - now; + auto microsecondDuration = duration_cast((_lastSendTimestamp + microseconds(_packetSendPeriod)) - now); if (microsecondDuration.count() > 0) { usleep(microsecondDuration.count()); diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index f6ce1cc5ac..5bc020c370 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -44,7 +44,7 @@ const QCommandLineOption UNRELIABLE_PACKETS { const QStringList STATS_TABLE_HEADERS { "Send Rate (P/s)", "RTT(ms)", "CW (P)", "Send Period (us)", "Received ACK", "Processed ACK", "Received LACK", "Received NAK", "Received TNAK", - "Sent ACK2", "Re-sent Packets" + "Sent ACK2", "Sent Packets", "Re-sent Packets" }; UDTTest::UDTTest(int& argc, char** argv) : @@ -234,6 +234,7 @@ void UDTTest::sampleStats() { QString::number(stats.events[udt::ConnectionStats::Stats::ReceivedNAK]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.events[udt::ConnectionStats::Stats::ReceivedTimeoutNAK]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.events[udt::ConnectionStats::Stats::SentACK2]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.sentPackets).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.events[udt::ConnectionStats::Stats::Retransmission]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()) }; From a1bd558d71545115a9ca823302a61af1b4a15950 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 3 Aug 2015 14:39:51 -0700 Subject: [PATCH 189/242] Safe locks --- libraries/networking/src/udt/SendQueue.cpp | 23 ++++++++-------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 3aeeec4d4a..111b676499 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -151,16 +151,15 @@ void SendQueue::run() { while (!resentPacket) { // prioritize a loss retransmission - _naksLock.lockForWrite(); + QWriteLocker naksLocker(&_naksLock); if (_naks.getLength() > 0) { - // pull the sequence number we need to re-send SequenceNumber resendNumber = _naks.popFirstSequenceNumber(); - _naksLock.unlock(); + naksLocker.unlock(); // pull the packet to re-send from the sent packets list - _sentLock.lockForRead(); + QReadLocker sentLocker(&_sentLock); // see if we can find the packet to re-send auto it = _sentPackets.find(resendNumber); @@ -170,7 +169,7 @@ void SendQueue::run() { auto& resendPacket = *(it->second); // unlock the sent packets - _sentLock.unlock(); + sentLocker.unlock(); // send it off sendPacket(resendPacket); @@ -184,18 +183,12 @@ void SendQueue::run() { } else { // we didn't find this packet in the sentPackets queue - assume this means it was ACKed // we'll fire the loop again to see if there is another to re-send - - // unlock the sent packets - _sentLock.unlock(); + continue; } - - } else { - // unlock the loss list, it's empty - _naksLock.unlock(); - - // break from the while, we didn't resend a packet - break; } + + // break from the while, we didn't resend a packet + break; } if (!resentPacket From 8704f058127546bed0ad89c297168683be446f1c Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 3 Aug 2015 14:40:18 -0700 Subject: [PATCH 190/242] Added UDTTest server stats --- libraries/networking/src/udt/Socket.cpp | 9 ++ libraries/networking/src/udt/Socket.h | 1 + tools/udt-test/src/UDTTest.cpp | 111 ++++++++++++++++-------- 3 files changed, 87 insertions(+), 34 deletions(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 2056957ef1..6a3a3d619f 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -229,3 +229,12 @@ ConnectionStats::Stats Socket::sampleStatsForConnection(const HifiSockAddr& dest return ConnectionStats::Stats(); } } + +std::vector Socket::getSockAddr() { + std::vector addr; + addr.reserve(_connectionsHash.size()); + for (const auto& connectionPair : _connectionsHash) { + addr.push_back(connectionPair.first); + } + return std::move(addr); +} diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 0fee52e755..602cfaf8ec 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -65,6 +65,7 @@ public: void connectToSendSignal(const HifiSockAddr& destinationAddr, QObject* receiver, const char* slot); ConnectionStats::Stats sampleStatsForConnection(const HifiSockAddr& destination); + std::vector getSockAddr(); private slots: void readPendingDatagrams(); diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 5bc020c370..312beffa05 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -41,12 +41,18 @@ const QCommandLineOption UNRELIABLE_PACKETS { "unreliable", "send unreliable packets (default is reliable)" }; -const QStringList STATS_TABLE_HEADERS { +const QStringList CLIENT_STATS_TABLE_HEADERS { "Send Rate (P/s)", "RTT(ms)", "CW (P)", "Send Period (us)", "Received ACK", "Processed ACK", "Received LACK", "Received NAK", "Received TNAK", "Sent ACK2", "Sent Packets", "Re-sent Packets" }; +const QStringList SERVER_STATS_TABLE_HEADERS { + "Recieve Rate (P/s)", "Total Bytes", "Util Bytes", "Ratio (%)", + "Sent ACK", "Sent LACK", "Sent NAK", "Sent TNAK", + "Recieved ACK2", "Duplicate Packets" +}; + UDTTest::UDTTest(int& argc, char** argv) : QCoreApplication(argc, argv) { @@ -122,17 +128,14 @@ UDTTest::UDTTest(int& argc, char** argv) : if (!_target.isNull()) { sendInitialPackets(); - - // the sender reports stats every 100ms - static const int STATS_SAMPLE_INTERVAL = 100; - - QTimer* statsTimer = new QTimer(this); - connect(statsTimer, &QTimer::timeout, this, &UDTTest::sampleStats); - statsTimer->start(STATS_SAMPLE_INTERVAL); - - // output the headers for stats for our table - qDebug() << qPrintable(STATS_TABLE_HEADERS.join(" | ")); } + + // the sender reports stats every 100 milliseconds + static const int STATS_SAMPLE_INTERVAL = 100; + + QTimer* statsTimer = new QTimer(this); + connect(statsTimer, &QTimer::timeout, this, &UDTTest::sampleStats); + statsTimer->start(STATS_SAMPLE_INTERVAL); } void UDTTest::parseArguments() { @@ -216,28 +219,68 @@ void UDTTest::sendPacket() { } void UDTTest::sampleStats() { - udt::ConnectionStats::Stats stats = _socket.sampleStatsForConnection(_target); + static bool first = true; - int headerIndex = -1; - - static const double USECS_PER_MSEC = 1000.0; - - // setup a list of left justified values - QStringList values { - QString::number(stats.sendRate).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.rtt / USECS_PER_MSEC).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.congestionWindowSize).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.packetSendPeriod).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.events[udt::ConnectionStats::Stats::ReceivedACK]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.events[udt::ConnectionStats::Stats::ProcessedACK]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.events[udt::ConnectionStats::Stats::ReceivedLightACK]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.events[udt::ConnectionStats::Stats::ReceivedNAK]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.events[udt::ConnectionStats::Stats::ReceivedTimeoutNAK]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.events[udt::ConnectionStats::Stats::SentACK2]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.sentPackets).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.events[udt::ConnectionStats::Stats::Retransmission]).leftJustified(STATS_TABLE_HEADERS[++headerIndex].size()) - }; - - // output this line of values - qDebug() << qPrintable(values.join(" | ")); + if (!_target.isNull()) { + if (first) { + // output the headers for stats for our table + qDebug() << qPrintable(CLIENT_STATS_TABLE_HEADERS.join(" | ")); + first = false; + } + + udt::ConnectionStats::Stats stats = _socket.sampleStatsForConnection(_target); + + int headerIndex = -1; + + static const double USECS_PER_MSEC = 1000.0; + + // setup a list of left justified values + QStringList values { + QString::number(stats.sendRate).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.rtt / USECS_PER_MSEC).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.congestionWindowSize).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.packetSendPeriod).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::ReceivedACK]).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::ProcessedACK]).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::ReceivedLightACK]).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::ReceivedNAK]).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::ReceivedTimeoutNAK]).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::SentACK2]).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.sentPackets).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::Retransmission]).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()) + }; + + // output this line of values + qDebug() << qPrintable(values.join(" | ")); + } else { + if (first) { + // output the headers for stats for our table + qDebug() << qPrintable(SERVER_STATS_TABLE_HEADERS.join(" | ")); + first = false; + } + + auto sockets = _socket.getSockAddr(); + if (sockets.size() > 0) { + udt::ConnectionStats::Stats stats = _socket.sampleStatsForConnection(sockets.front()); + + int headerIndex = -1; + + // setup a list of left justified values + QStringList values { + QString::number(stats.receiveRate).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.recievedBytes).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.recievedUtilBytes).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(100 * (double)stats.recievedUtilBytes / (double)stats.recievedBytes).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::SentACK]).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::SentLightACK]).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::SentNAK]).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::SentTimeoutNAK]).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::ReceivedACK2]).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.events[udt::ConnectionStats::Stats::Duplicate]).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()) + }; + + // output this line of values + qDebug() << qPrintable(values.join(" | ")); + } + } } From d937cf1cc7b78338830893a3f0b4dd5b3a3542d4 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 3 Aug 2015 15:05:41 -0700 Subject: [PATCH 191/242] set RTT on congestion control in ctor --- libraries/networking/src/udt/Connection.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 1786f7558e..1fb4a7db9c 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -37,8 +37,12 @@ Connection::Connection(Socket* parentSocket, HifiSockAddr destination, unique_pt // setup default SYN, RTT and RTT Variance based on the SYN interval in CongestionControl object _synInterval = _congestionControl->synInterval(); + _rtt = _synInterval * 10; _rttVariance = _rtt / 2; + + // set the initial RTT on congestion control object + _congestionControl->setRTT(_rtt); } Connection::~Connection() { @@ -375,7 +379,7 @@ void Connection::processACK(std::unique_ptr controlPacket) { microseconds sinceLastACK2 = duration_cast(currentTime - lastACK2SendTime); - if (sinceLastACK2.count() > _synInterval || currentACKSubSequenceNumber == _lastSentACK2) { + if (sinceLastACK2.count() >= _synInterval || currentACKSubSequenceNumber == _lastSentACK2) { // Send ACK2 packet sendACK2(currentACKSubSequenceNumber); From b5ec02bd014d15cf08a5a46ba403dccfb92d9f06 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 3 Aug 2015 15:36:08 -0700 Subject: [PATCH 192/242] go back to previous PacketTimeWindow code --- libraries/networking/src/udt/Connection.cpp | 6 ++++-- libraries/networking/src/udt/Connection.h | 3 ++- libraries/networking/src/udt/PacketTimeWindow.cpp | 12 ++++++++---- tools/udt-test/src/UDTTest.cpp | 3 ++- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 1fb4a7db9c..4f9b7a38f7 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -77,8 +77,9 @@ void Connection::sendReliablePacket(unique_ptr packet) { void Connection::sync() { if (_hasReceivedFirstPacket) { - // reset the number of light ACKS during this sync interval + // reset the number of light ACKs or non SYN ACKs during this sync interval _lightACKsDuringSYN = 1; + _acksDuringSYN = 1; // we send out a periodic ACK every rate control interval sendACK(); @@ -327,7 +328,8 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in ++_packetsSinceACK; // check if we need to send an ACK, according to CC params - if (_congestionControl->_ackInterval > 0 && _packetsSinceACK >= _congestionControl->_ackInterval) { + if (_congestionControl->_ackInterval > 0 && _packetsSinceACK >= _congestionControl->_ackInterval * _acksDuringSYN) { + _acksDuringSYN++; sendACK(false); } else if (_congestionControl->_lightACKInterval > 0 && _packetsSinceACK >= _congestionControl->_lightACKInterval * _lightACKsDuringSYN) { diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 99328d601f..9bdc6baf24 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -96,7 +96,8 @@ private: SequenceNumber _lastSentACK; // The last sent ACK SequenceNumber _lastSentACK2; // The last sent ACK sub-sequence number in an ACK2 - + + int _acksDuringSYN { 1 }; // The number of non-SYN ACKs sent during SYN int _lightACKsDuringSYN { 1 }; // The number of lite ACKs sent during SYN interval int32_t _rtt; // RTT, in microseconds diff --git a/libraries/networking/src/udt/PacketTimeWindow.cpp b/libraries/networking/src/udt/PacketTimeWindow.cpp index 269d158b7d..67aa61bc78 100644 --- a/libraries/networking/src/udt/PacketTimeWindow.cpp +++ b/libraries/networking/src/udt/PacketTimeWindow.cpp @@ -11,7 +11,6 @@ #include "PacketTimeWindow.h" -#include #include #include @@ -49,11 +48,16 @@ int median(Iterator begin, Iterator end) { int32_t meanOfMedianFilteredValues(std::vector intervals, int numValues, int valuesRequired = 0) { // grab the median value of the intervals vector - int intervalsMedian = median(intervals.begin(), intervals.end()); + int median = 0; + if (numValues % 2 == 0) { + median = intervals[numValues / 2]; + } else { + median = (intervals[(numValues / 2) - 1] + intervals[numValues / 2]) / 2; + } static const int MEDIAN_FILTERING_BOUND_MULTIPLIER = 8; - int upperBound = intervalsMedian * MEDIAN_FILTERING_BOUND_MULTIPLIER; - int lowerBound = intervalsMedian / MEDIAN_FILTERING_BOUND_MULTIPLIER; + int upperBound = median * MEDIAN_FILTERING_BOUND_MULTIPLIER; + int lowerBound = median / MEDIAN_FILTERING_BOUND_MULTIPLIER; int sum = 0; int count = 0; diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 312beffa05..6344860442 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -42,7 +42,7 @@ const QCommandLineOption UNRELIABLE_PACKETS { }; const QStringList CLIENT_STATS_TABLE_HEADERS { - "Send Rate (P/s)", "RTT(ms)", "CW (P)", "Send Period (us)", + "Send Rate (P/s)", "Bandwidth (P/s)", "RTT(ms)", "CW (P)", "Send Period (us)", "Received ACK", "Processed ACK", "Received LACK", "Received NAK", "Received TNAK", "Sent ACK2", "Sent Packets", "Re-sent Packets" }; @@ -237,6 +237,7 @@ void UDTTest::sampleStats() { // setup a list of left justified values QStringList values { QString::number(stats.sendRate).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.estimatedBandwith).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.rtt / USECS_PER_MSEC).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.congestionWindowSize).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.packetSendPeriod).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), From acd7a7a732d390a1d91b9a9d6bba9de25f331106 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 3 Aug 2015 16:04:08 -0700 Subject: [PATCH 193/242] fix send/process for ACK packets --- libraries/networking/src/udt/Connection.cpp | 13 ++++++++++--- libraries/networking/src/udt/PacketTimeWindow.cpp | 12 ++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 4f9b7a38f7..dab6c5b934 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -134,7 +134,7 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { // setup the ACK packet, make it static so we can re-use it static const int ACK_PACKET_PAYLOAD_BYTES = sizeof(_lastSentACK) + sizeof(_currentACKSubSequenceNumber) - + sizeof(_rtt) + sizeof(int32_t) + sizeof(int32_t); + + sizeof(_rtt) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t); static auto ackPacket = ControlPacket::create(ControlPacket::ACK, ACK_PACKET_PAYLOAD_BYTES); ackPacket->reset(); // We need to reset it every time. @@ -412,7 +412,10 @@ void Connection::processACK(std::unique_ptr controlPacket) { } // this is a valid ACKed sequence number - update the flow window size and the last received ACK - controlPacket->readPrimitive(&_flowWindowSize); + int32_t packedFlowWindow; + controlPacket->readPrimitive(&packedFlowWindow); + + _flowWindowSize = packedFlowWindow; if (ack == _lastReceivedACK) { // processing an already received ACK, bail @@ -433,8 +436,12 @@ void Connection::processACK(std::unique_ptr controlPacket) { // set the RTT for congestion control _congestionControl->setRTT(_rtt); - if (controlPacket->getPayloadSize() > (qint64) (sizeof(SequenceNumber) + sizeof(SequenceNumber) + sizeof(rtt))) { + if (controlPacket->bytesLeftToRead() > 0) { int32_t receiveRate, bandwidth; + + Q_ASSERT_X(controlPacket->bytesLeftToRead() == sizeof(receiveRate) + sizeof(bandwidth), + "Connection::processACK", "sync interval ACK packet does not contain expected data"); + controlPacket->readPrimitive(&receiveRate); controlPacket->readPrimitive(&bandwidth); diff --git a/libraries/networking/src/udt/PacketTimeWindow.cpp b/libraries/networking/src/udt/PacketTimeWindow.cpp index 67aa61bc78..269d158b7d 100644 --- a/libraries/networking/src/udt/PacketTimeWindow.cpp +++ b/libraries/networking/src/udt/PacketTimeWindow.cpp @@ -11,6 +11,7 @@ #include "PacketTimeWindow.h" +#include #include #include @@ -48,16 +49,11 @@ int median(Iterator begin, Iterator end) { int32_t meanOfMedianFilteredValues(std::vector intervals, int numValues, int valuesRequired = 0) { // grab the median value of the intervals vector - int median = 0; - if (numValues % 2 == 0) { - median = intervals[numValues / 2]; - } else { - median = (intervals[(numValues / 2) - 1] + intervals[numValues / 2]) / 2; - } + int intervalsMedian = median(intervals.begin(), intervals.end()); static const int MEDIAN_FILTERING_BOUND_MULTIPLIER = 8; - int upperBound = median * MEDIAN_FILTERING_BOUND_MULTIPLIER; - int lowerBound = median / MEDIAN_FILTERING_BOUND_MULTIPLIER; + int upperBound = intervalsMedian * MEDIAN_FILTERING_BOUND_MULTIPLIER; + int lowerBound = intervalsMedian / MEDIAN_FILTERING_BOUND_MULTIPLIER; int sum = 0; int count = 0; From 36a0ef50d4b681875775614c58457b6a9180ccf9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 3 Aug 2015 16:21:17 -0700 Subject: [PATCH 194/242] add assert for BasePacket write, fix time in Connection --- libraries/networking/src/udt/BasePacket.cpp | 6 +++--- libraries/networking/src/udt/Connection.cpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/udt/BasePacket.cpp b/libraries/networking/src/udt/BasePacket.cpp index 80564143d4..2bfa2b6805 100644 --- a/libraries/networking/src/udt/BasePacket.cpp +++ b/libraries/networking/src/udt/BasePacket.cpp @@ -156,13 +156,13 @@ bool BasePacket::reset() { } qint64 BasePacket::writeData(const char* data, qint64 maxSize) { + + Q_ASSERT_X(maxSize <= bytesAvailableForWrite(), "BasePacket::writeData", "not enough space for write"); // make sure we have the space required to write this block if (maxSize <= bytesAvailableForWrite()) { qint64 currentPos = pos(); - - Q_ASSERT(currentPos < _payloadCapacity); - + // good to go - write the data memcpy(_payloadStart + currentPos, data, maxSize); diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index dab6c5b934..082de32204 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -283,10 +283,10 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in _receiveWindow.onProbePair1Arrival(); } else if (((uint32_t) sequenceNumber & 0xF) == 1) { _receiveWindow.onProbePair2Arrival(); - } else { - _receiveWindow.onPacketArrival(); } + _receiveWindow.onPacketArrival(); + // If this is not the next sequence number, report loss if (sequenceNumber > _lastReceivedSequenceNumber + 1) { if (_lastReceivedSequenceNumber + 1 == sequenceNumber - 1) { From c0852f0eb25dc9b8aa39fd3e1ecdf8e9dd0f2d1a Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 3 Aug 2015 17:14:42 -0700 Subject: [PATCH 195/242] Remove call to has pending datagrams --- libraries/networking/src/udt/Connection.cpp | 3 +-- libraries/networking/src/udt/Socket.cpp | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index dab6c5b934..8bf3501590 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -283,9 +283,8 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in _receiveWindow.onProbePair1Arrival(); } else if (((uint32_t) sequenceNumber & 0xF) == 1) { _receiveWindow.onProbePair2Arrival(); - } else { - _receiveWindow.onPacketArrival(); } + _receiveWindow.onPacketArrival(); // If this is not the next sequence number, report loss if (sequenceNumber > _lastReceivedSequenceNumber + 1) { diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 6a3a3d619f..dafe50d5c8 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -129,12 +129,12 @@ Connection& Socket::findOrCreateConnection(const HifiSockAddr& sockAddr) { } void Socket::readPendingDatagrams() { - while (_udpSocket.hasPendingDatagrams()) { + int packetSizeWithHeader = -1; + while ((packetSizeWithHeader = _udpSocket.pendingDatagramSize()) != -1) { // setup a HifiSockAddr to read into HifiSockAddr senderSockAddr; // setup a buffer to read the packet into - int packetSizeWithHeader = _udpSocket.pendingDatagramSize(); auto buffer = std::unique_ptr(new char[packetSizeWithHeader]); // pull the datagram From 40520c204ff099eb2e67c24f6846042bb89d8320 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 3 Aug 2015 17:39:09 -0700 Subject: [PATCH 196/242] Stats improvement --- libraries/networking/src/udt/Connection.cpp | 2 ++ libraries/networking/src/udt/PacketTimeWindow.cpp | 4 ++-- tools/udt-test/src/UDTTest.cpp | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 85f103f438..2219de6fd6 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -508,6 +508,8 @@ void Connection::processACK2(std::unique_ptr controlPacket) { int rtt = duration_cast(now - it->second.second).count(); updateRTT(rtt); + // write this RTT to stats + _stats.recordRTT(rtt); // set the RTT for congestion control _congestionControl->setRTT(_rtt); diff --git a/libraries/networking/src/udt/PacketTimeWindow.cpp b/libraries/networking/src/udt/PacketTimeWindow.cpp index 269d158b7d..9193e8e0af 100644 --- a/libraries/networking/src/udt/PacketTimeWindow.cpp +++ b/libraries/networking/src/udt/PacketTimeWindow.cpp @@ -19,8 +19,8 @@ using namespace udt; using namespace std::chrono; -static const int DEFAULT_PACKET_INTERVAL_MICROSECONDS = 1000000; -static const int DEFAULT_PROBE_INTERVAL_MICROSECONDS = 1000; +static const int DEFAULT_PACKET_INTERVAL_MICROSECONDS = 1000000; // 1s +static const int DEFAULT_PROBE_INTERVAL_MICROSECONDS = 1000; // 1ms PacketTimeWindow::PacketTimeWindow(int numPacketIntervals, int numProbeIntervals) : _numPacketIntervals(numPacketIntervals), diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 6344860442..8cc141c83f 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -220,6 +220,7 @@ void UDTTest::sendPacket() { void UDTTest::sampleStats() { static bool first = true; + static const double USECS_PER_MSEC = 1000.0; if (!_target.isNull()) { if (first) { @@ -232,8 +233,6 @@ void UDTTest::sampleStats() { int headerIndex = -1; - static const double USECS_PER_MSEC = 1000.0; - // setup a list of left justified values QStringList values { QString::number(stats.sendRate).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), @@ -269,6 +268,9 @@ void UDTTest::sampleStats() { // setup a list of left justified values QStringList values { QString::number(stats.receiveRate).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.estimatedBandwith).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.rtt / USECS_PER_MSEC).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), + QString::number(stats.congestionWindowSize).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.recievedBytes).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.recievedUtilBytes).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(100 * (double)stats.recievedUtilBytes / (double)stats.recievedBytes).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), From a10746a448307a3cb1dc77df4f4256c8b97919f1 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 3 Aug 2015 19:12:37 -0700 Subject: [PATCH 197/242] set the max window size, correct a typo --- libraries/networking/src/udt/Connection.cpp | 3 ++- tools/udt-test/src/UDTTest.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 2219de6fd6..1fce27ee5a 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -41,8 +41,9 @@ Connection::Connection(Socket* parentSocket, HifiSockAddr destination, unique_pt _rtt = _synInterval * 10; _rttVariance = _rtt / 2; - // set the initial RTT on congestion control object + // set the initial RTT and flow window size on congestion control object _congestionControl->setRTT(_rtt); + _congestionControl->setMaxCongestionWindowSize(_flowWindowSize); } Connection::~Connection() { diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 8cc141c83f..d13c82e8f1 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -48,7 +48,7 @@ const QStringList CLIENT_STATS_TABLE_HEADERS { }; const QStringList SERVER_STATS_TABLE_HEADERS { - "Recieve Rate (P/s)", "Total Bytes", "Util Bytes", "Ratio (%)", + "Receive Rate (P/s)", "Total Bytes", "Util Bytes", "Ratio (%)", "Sent ACK", "Sent LACK", "Sent NAK", "Sent TNAK", "Recieved ACK2", "Duplicate Packets" }; From 924a4d3c21a65190c85ee8b60c9cf51e77101946 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 3 Aug 2015 19:12:58 -0700 Subject: [PATCH 198/242] Change connection stats --- tools/udt-test/src/UDTTest.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 8cc141c83f..db3b483889 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -48,7 +48,8 @@ const QStringList CLIENT_STATS_TABLE_HEADERS { }; const QStringList SERVER_STATS_TABLE_HEADERS { - "Recieve Rate (P/s)", "Total Bytes", "Util Bytes", "Ratio (%)", + "Recieve Rate (P/s)", "Bandwidth (P/s)", "RTT(ms)", "CW (P)", + //"Total Bytes", "Util Bytes", "Ratio (%)", "Sent ACK", "Sent LACK", "Sent NAK", "Sent TNAK", "Recieved ACK2", "Duplicate Packets" }; @@ -271,9 +272,9 @@ void UDTTest::sampleStats() { QString::number(stats.estimatedBandwith).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.rtt / USECS_PER_MSEC).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.congestionWindowSize).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.recievedBytes).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(stats.recievedUtilBytes).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), - QString::number(100 * (double)stats.recievedUtilBytes / (double)stats.recievedBytes).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), +// QString::number(stats.recievedBytes).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), +// QString::number(stats.recievedUtilBytes).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), +// QString::number(100 * (double)stats.recievedUtilBytes / (double)stats.recievedBytes).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.events[udt::ConnectionStats::Stats::SentACK]).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.events[udt::ConnectionStats::Stats::SentLightACK]).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.events[udt::ConnectionStats::Stats::SentNAK]).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), From 3d0c71d95c644f5353c8d7aad0e1297021f65b16 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 4 Aug 2015 12:29:22 -0700 Subject: [PATCH 199/242] remove a double on arrival time record --- libraries/networking/src/udt/Connection.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 1fce27ee5a..0817c23e8d 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -285,7 +285,6 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in } else if (((uint32_t) sequenceNumber & 0xF) == 1) { _receiveWindow.onProbePair2Arrival(); } - _receiveWindow.onPacketArrival(); _receiveWindow.onPacketArrival(); From 925cb4bd566002ab2d52ec555d3b7856cfba2f68 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 4 Aug 2015 12:48:58 -0700 Subject: [PATCH 200/242] fix for slow start algo on loss --- .../networking/src/udt/CongestionControl.cpp | 16 +++++++++++----- libraries/networking/src/udt/CongestionControl.h | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 16ee9a0c75..d338e648fe 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -131,6 +131,11 @@ void DefaultCC::onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) { // stop the slow start if we haven't yet if (_slowStart) { stopSlowStart(); + + // if the change to send rate was driven by a known receive rate, then we don't continue with the decrease + if (_receiveRate > 0) { + return; + } } _loss = true; @@ -187,13 +192,14 @@ void DefaultCC::onTimeout() { void DefaultCC::stopSlowStart() { _slowStart = false; + if (_receiveRate > 0) { // Set the sending rate to the receiving rate. _packetSendPeriod = USECS_PER_SECOND / _receiveRate; - return; + } else { + // If no receiving rate is observed, we have to compute the sending + // rate according to the current window size, and decrease it + // using the method below. + _packetSendPeriod = _congestionWindowSize / (_rtt + synInterval()); } - // If no receiving rate is observed, we have to compute the sending - // rate according to the current window size, and decrease it - // using the method below. - _packetSendPeriod = _congestionWindowSize / (_rtt + synInterval()); } diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 81ade13b41..3aac2dd54e 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -106,7 +106,7 @@ public: virtual void onTimeout(); private: - void stopSlowStart(); // stops the slow start on loss or timeout, if it's still on + void stopSlowStart(); // stops the slow start on loss or timeout std::chrono::high_resolution_clock::time_point _lastRCTime; // last rate increase time bool _slowStart { true }; // if in slow start phase From 4b21581b3686c3d6d30f12c1ecc3885ca4202efb Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 4 Aug 2015 14:32:48 -0700 Subject: [PATCH 201/242] fix for min NAK interval in Connection --- libraries/networking/src/udt/Connection.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 0817c23e8d..9e5bcc3dde 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -300,8 +300,6 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in sendNAK(sequenceNumber); // figure out when we should send the next loss report, if we haven't heard anything back - _nakInterval = (_rtt + 4 * _rttVariance); - int receivedPacketsPerSecond = _receiveWindow.getPacketReceiveSpeed(); if (receivedPacketsPerSecond > 0) { @@ -582,7 +580,7 @@ void Connection::updateRTT(int rtt) { } int Connection::estimatedTimeout() const { - return _congestionControl->_userDefinedRto ? _rtt + _rttVariance * 4 : _congestionControl->_rto; + return _congestionControl->_userDefinedRto ? _congestionControl->_rto : _rtt + _rttVariance * 4; } void Connection::updateCongestionControlAndSendQueue(std::function congestionCallback) { From 01749ad0de09df8e541d12d1901ea7ee69f3c327 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 4 Aug 2015 15:07:04 -0700 Subject: [PATCH 202/242] TimeoutNAK fixes --- libraries/networking/src/udt/Connection.cpp | 16 +++++++--------- libraries/networking/src/udt/ConnectionStats.cpp | 2 +- libraries/networking/src/udt/LossList.cpp | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 0817c23e8d..56f76d2f5b 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -285,7 +285,6 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in } else if (((uint32_t) sequenceNumber & 0xF) == 1) { _receiveWindow.onProbePair2Arrival(); } - _receiveWindow.onPacketArrival(); // If this is not the next sequence number, report loss @@ -300,19 +299,18 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in sendNAK(sequenceNumber); // figure out when we should send the next loss report, if we haven't heard anything back - _nakInterval = (_rtt + 4 * _rttVariance); + _nakInterval = estimatedTimeout(); int receivedPacketsPerSecond = _receiveWindow.getPacketReceiveSpeed(); - if (receivedPacketsPerSecond > 0) { // the NAK interval is at least the _minNAKInterval // but might be the time required for all lost packets to be retransmitted - _nakInterval = std::max((int) (_lossList.getLength() * (USECS_PER_SECOND / receivedPacketsPerSecond)), - _minNAKInterval); - } else { - // the NAK interval is at least the _minNAKInterval but might be the estimated timeout - _nakInterval = std::max(estimatedTimeout(), _minNAKInterval); + _nakInterval += (int) (_lossList.getLength() * (USECS_PER_SECOND / receivedPacketsPerSecond)); } + + // the NAK interval is at least the _minNAKInterval but might be the estimated timeout + _nakInterval = std::max(_nakInterval, _minNAKInterval); + } bool wasDuplicate = false; @@ -582,7 +580,7 @@ void Connection::updateRTT(int rtt) { } int Connection::estimatedTimeout() const { - return _congestionControl->_userDefinedRto ? _rtt + _rttVariance * 4 : _congestionControl->_rto; + return _congestionControl->_userDefinedRto ? _congestionControl->_rto : _rtt + _rttVariance * 4; } void Connection::updateCongestionControlAndSendQueue(std::function congestionCallback) { diff --git a/libraries/networking/src/udt/ConnectionStats.cpp b/libraries/networking/src/udt/ConnectionStats.cpp index 91eeb6d49c..8c36750782 100644 --- a/libraries/networking/src/udt/ConnectionStats.cpp +++ b/libraries/networking/src/udt/ConnectionStats.cpp @@ -81,7 +81,7 @@ void ConnectionStats::recordUnreliableReceivedPackets(int payload, int total) { } static const double EWMA_CURRENT_SAMPLE_WEIGHT = 0.125; -static const double EWMA_PREVIOUS_SAMPLES_WEIGHT = 1 - 0.125; +static const double EWMA_PREVIOUS_SAMPLES_WEIGHT = 1.0 - EWMA_CURRENT_SAMPLE_WEIGHT; void ConnectionStats::recordSendRate(int sample) { _currentSample.sendRate = sample; diff --git a/libraries/networking/src/udt/LossList.cpp b/libraries/networking/src/udt/LossList.cpp index 4cb182c2f7..ab860a768d 100644 --- a/libraries/networking/src/udt/LossList.cpp +++ b/libraries/networking/src/udt/LossList.cpp @@ -100,7 +100,7 @@ bool LossList::remove(SequenceNumber seq) { } else { auto temp = it->second; it->second = seq - 1; - it = _lossList.insert(++it, make_pair(seq + 1, temp)); + _lossList.insert(++it, make_pair(seq + 1, temp)); } _length -= 1; From 23e1ee71ccc3d67f3ab9c19eb2181be8f01fa3ad Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 4 Aug 2015 16:18:12 -0700 Subject: [PATCH 203/242] suppress writeDatagram errors for saturated links --- libraries/networking/src/udt/Socket.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index dafe50d5c8..9bef5ca6e5 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -13,6 +13,8 @@ #include +#include + #include "../NetworkLogging.h" #include "Connection.h" #include "ControlPacket.h" @@ -111,7 +113,12 @@ qint64 Socket::writeDatagram(const QByteArray& datagram, const HifiSockAddr& soc qint64 bytesWritten = _udpSocket.writeDatagram(datagram, sockAddr.getAddress(), sockAddr.getPort()); if (bytesWritten < 0) { - qCDebug(networking) << "ERROR in writeDatagram:" << _udpSocket.error() << "-" << _udpSocket.errorString(); + // when saturating a link this isn't an uncommon message - suppress it so it doesn't bomb the debug + static const QString WRITE_ERROR_REGEX = "writeDatagram error: QAbstractSocket::NetworkError - Unable to send a message"; + static QString repeatedMessage + = LogHandler::getInstance().addRepeatedMessageRegex(WRITE_ERROR_REGEX); + + qCDebug(networking) << "writeDatagram error:" << _udpSocket.error() << "-" << qPrintable(_udpSocket.errorString()); } return bytesWritten; From 7b8ca2c0f2626c9f50839a8f25c731bfee3fe69e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 4 Aug 2015 16:29:53 -0700 Subject: [PATCH 204/242] fix suppression of link saturation messages --- libraries/networking/src/udt/Socket.cpp | 4 ++-- tools/udt-test/CMakeLists.txt | 2 +- tools/udt-test/src/UDTTest.cpp | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 9bef5ca6e5..a7dd6377ba 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -114,11 +114,11 @@ qint64 Socket::writeDatagram(const QByteArray& datagram, const HifiSockAddr& soc if (bytesWritten < 0) { // when saturating a link this isn't an uncommon message - suppress it so it doesn't bomb the debug - static const QString WRITE_ERROR_REGEX = "writeDatagram error: QAbstractSocket::NetworkError - Unable to send a message"; + static const QString WRITE_ERROR_REGEX = "Socket::writeDatagram QAbstractSocket::NetworkError - Unable to send a message"; static QString repeatedMessage = LogHandler::getInstance().addRepeatedMessageRegex(WRITE_ERROR_REGEX); - qCDebug(networking) << "writeDatagram error:" << _udpSocket.error() << "-" << qPrintable(_udpSocket.errorString()); + qCDebug(networking) << "Socket::writeDatagram" << _udpSocket.error() << "-" << qPrintable(_udpSocket.errorString()); } return bytesWritten; diff --git a/tools/udt-test/CMakeLists.txt b/tools/udt-test/CMakeLists.txt index 7f47677269..648ef6f00c 100644 --- a/tools/udt-test/CMakeLists.txt +++ b/tools/udt-test/CMakeLists.txt @@ -1,6 +1,6 @@ set(TARGET_NAME udt-test) setup_hifi_project() -link_hifi_libraries(networking) +link_hifi_libraries(networking shared) copy_dlls_beside_windows_executable() \ No newline at end of file diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 83f5ba1751..7508263429 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -16,6 +16,8 @@ #include #include +#include + const QCommandLineOption PORT_OPTION { "p", "listening port for socket (defaults to random)", "port", 0 }; const QCommandLineOption TARGET_OPTION { "target", "target for sent packets (default is listen only)", From 5124fc1ea0ae86443560c20813352f13b3840ceb Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 4 Aug 2015 16:31:40 -0700 Subject: [PATCH 205/242] add LogHandler to UDTTest --- tools/udt-test/src/UDTTest.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 7508263429..6c163a726e 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -59,6 +59,8 @@ const QStringList SERVER_STATS_TABLE_HEADERS { UDTTest::UDTTest(int& argc, char** argv) : QCoreApplication(argc, argv) { + qInstallMessageHandler(LogHandler::verboseMessageHandler); + parseArguments(); // randomize the seed for packet size randomization From 5d7dac9b8ecca1b67268b4074d6d1245138adc8c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 7 Aug 2015 17:33:33 -0700 Subject: [PATCH 206/242] do DefaultCC setup in constructor --- libraries/networking/src/udt/CongestionControl.cpp | 11 +++++------ libraries/networking/src/udt/CongestionControl.h | 3 +-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index d338e648fe..307ef528a9 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -30,14 +30,13 @@ void CongestionControl::setPacketSendPeriod(double newSendPeriod) { } } -void DefaultCC::init() { - _lastRCTime = high_resolution_clock::now(); - +DefaultCC::DefaultCC() : + _lastRCTime(high_resolution_clock::now()), + _slowStartLastAck(_sendCurrSeqNum), + _lastDecreaseMaxSeq(SequenceNumber {SequenceNumber::MAX }) +{ _mss = udt::MAX_PACKET_SIZE_WITH_UDP_HEADER; - _slowStartLastAck = _sendCurrSeqNum; - _lastDecreaseMaxSeq = SequenceNumber { SequenceNumber::MAX }; - _congestionWindowSize = 16.0; _packetSendPeriod = 1.0; } diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index 3aac2dd54e..f131324acb 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -97,10 +97,9 @@ public: class DefaultCC: public CongestionControl { public: - DefaultCC() {} + DefaultCC(); public: - virtual void init(); virtual void onACK(SequenceNumber ackNum); virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd); virtual void onTimeout(); From 9ac760aed63108cf16796d8b44f903641644ce57 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 10 Aug 2015 09:59:34 -0700 Subject: [PATCH 207/242] use c++11 number randomization for NAK --- libraries/networking/src/udt/CongestionControl.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 307ef528a9..ccb904f67e 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -10,6 +10,9 @@ // #include "CongestionControl.h" + +#include + #include "Packet.h" using namespace udt; @@ -160,12 +163,11 @@ void DefaultCC::onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) { _lastDecreaseMaxSeq = _sendCurrSeqNum; // avoid synchronous rate decrease across connections using randomization - srand((unsigned) _lastDecreaseMaxSeq); - _randomDecreaseThreshold = (int) ceil(_avgNAKNum * (double(rand()) / RAND_MAX)); + std::random_device rd; + std::mt19937 generator(rd()); + std::uniform_int_distribution<> distribution(1, _avgNAKNum); - if (_randomDecreaseThreshold < 1) { - _randomDecreaseThreshold = 1; - } + _randomDecreaseThreshold = distribution(generator); } else if ((_decreaseCount++ < MAX_DECREASES_PER_CONGESTION_EPOCH) && ((++_nakCount % _randomDecreaseThreshold) == 0)) { // there have been fewer than MAX_DECREASES_PER_CONGESTION_EPOCH AND this NAK matches the random count at which we From 5e3d3fc06ba418a4cdd54a6a36f7fab714cf1af2 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 10 Aug 2015 10:03:53 -0700 Subject: [PATCH 208/242] fix for indentation in BandwidthDialog --- interface/src/ui/BandwidthDialog.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/interface/src/ui/BandwidthDialog.cpp b/interface/src/ui/BandwidthDialog.cpp index 6272ddf76a..e086005783 100644 --- a/interface/src/ui/BandwidthDialog.cpp +++ b/interface/src/ui/BandwidthDialog.cpp @@ -89,10 +89,11 @@ BandwidthDialog::BandwidthDialog(QWidget* parent) : _allChannelDisplays[4] = _otherChannelDisplay = new BandwidthChannelDisplay({NodeType::Unassigned}, form, "Other", "Kbps", 1.0, COLOR2); _allChannelDisplays[5] = _totalChannelDisplay = - new BandwidthChannelDisplay({NodeType::DomainServer, NodeType::EntityServer, - NodeType::AudioMixer, NodeType::Agent, - NodeType::AvatarMixer, NodeType::Unassigned}, - form, "Total", "Kbps", 1.0, COLOR2); + new BandwidthChannelDisplay({ + NodeType::DomainServer, NodeType::EntityServer, + NodeType::AudioMixer, NodeType::Agent, + NodeType::AvatarMixer, NodeType::Unassigned + }, form, "Total", "Kbps", 1.0, COLOR2); connect(averageUpdateTimer, SIGNAL(timeout()), this, SLOT(updateTimerTimeout())); averageUpdateTimer->start(1000); From 04c1d154ab0e680e9b660da0f68fb80719a44e9a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 10 Aug 2015 10:10:43 -0700 Subject: [PATCH 209/242] fix header size for NLPacket writeTypeAndVersion --- libraries/networking/src/NLPacket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/NLPacket.cpp b/libraries/networking/src/NLPacket.cpp index cba73fa6cf..2ca794f7b9 100644 --- a/libraries/networking/src/NLPacket.cpp +++ b/libraries/networking/src/NLPacket.cpp @@ -164,7 +164,7 @@ QByteArray NLPacket::hashForPacketAndSecret(const udt::Packet& packet, const QUu } void NLPacket::writeTypeAndVersion() { - auto headerOffset = Packet::localHeaderSize(isPartOfMessage()); + auto headerOffset = Packet::totalHeaderSize(isPartOfMessage()); // Pack the packet type memcpy(_packet.get() + headerOffset, &_type, sizeof(PacketType)); From 6d9504935de5e3569fb2ef102a74f92e0b65acd2 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 10 Aug 2015 10:12:24 -0700 Subject: [PATCH 210/242] add a comment for onTimeout in CongestionControl --- libraries/networking/src/udt/CongestionControl.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index ccb904f67e..555f48cfd6 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -178,6 +178,7 @@ void DefaultCC::onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) { } } +// Note: This isn't currently being called by anything since we, unlike UDT, don't have TTL on our packets void DefaultCC::onTimeout() { if (_slowStart) { stopSlowStart(); From eb17ddc04fd45dfdbfe67c5fc32eb75f6438411d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 10 Aug 2015 10:13:50 -0700 Subject: [PATCH 211/242] fix capitalization of RTO for CongestionControl --- libraries/networking/src/udt/CongestionControl.h | 4 ++-- libraries/networking/src/udt/Connection.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index f131324acb..a8a4ba3342 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -42,7 +42,7 @@ public: protected: void setAckInterval(int ackInterval) { _ackInterval = ackInterval; } - void setRto(int rto) { _userDefinedRto = true; _rto = rto; } + void setRTO(int rto) { _userDefinedRTO = true; _rto = rto; } void setMSS(int mss) { _mss = mss; } void setMaxCongestionWindowSize(int window) { _maxCongestionWindowSize = window; } @@ -74,7 +74,7 @@ private: int _synInterval { DEFAULT_SYN_INTERVAL }; - bool _userDefinedRto { false }; // if the RTO value is defined by users + bool _userDefinedRTO { false }; // if the RTO value is defined by users int _rto { -1 }; // RTO value, microseconds }; diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 56f76d2f5b..ba3ae29e98 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -580,7 +580,7 @@ void Connection::updateRTT(int rtt) { } int Connection::estimatedTimeout() const { - return _congestionControl->_userDefinedRto ? _congestionControl->_rto : _rtt + _rttVariance * 4; + return _congestionControl->_userDefinedRTO ? _congestionControl->_rto : _rtt + _rttVariance * 4; } void Connection::updateCongestionControlAndSendQueue(std::function congestionCallback) { From d9254aa2e98f7cff8ccdb03383096ec072793d76 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 10 Aug 2015 10:14:50 -0700 Subject: [PATCH 212/242] remove comment that won't come into play until ordering --- libraries/networking/src/udt/Connection.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index ba3ae29e98..149b8a478e 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -130,9 +130,6 @@ void Connection::sendACK(bool wasCausedBySyncTimeout) { // update the last sent ACK _lastSentACK = nextACKNumber; - // remove the ACKed packets from the receive queue - // TODO? - // setup the ACK packet, make it static so we can re-use it static const int ACK_PACKET_PAYLOAD_BYTES = sizeof(_lastSentACK) + sizeof(_currentACKSubSequenceNumber) + sizeof(_rtt) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t); From 42b5b37b96fba13aea0957e039a3bd9ba296b267 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Aug 2015 15:47:43 -0700 Subject: [PATCH 213/242] force timeout NAK to fit MTU --- libraries/networking/src/udt/Connection.cpp | 8 ++++++-- libraries/networking/src/udt/LossList.cpp | 10 +++++++++- libraries/networking/src/udt/LossList.h | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 149b8a478e..273bb36378 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -248,11 +248,15 @@ void Connection::sendNAK(SequenceNumber sequenceNumberRecieved) { void Connection::sendTimeoutNAK() { if (_lossList.getLength() > 0) { + + int timeoutPayloadSize = std::min((int) (_lossList.getLength() * 2 * sizeof(SequenceNumber)), + ControlPacket::maxPayloadSize()); + // construct a NAK packet that will hold all of the lost sequence numbers - auto lossListPacket = ControlPacket::create(ControlPacket::TimeoutNAK, 2 * _lossList.getLength() * sizeof(SequenceNumber)); + auto lossListPacket = ControlPacket::create(ControlPacket::TimeoutNAK, timeoutPayloadSize); // Pack in the lost sequence numbers - _lossList.write(*lossListPacket); + _lossList.write(*lossListPacket, timeoutPayloadSize / 2); // have our parent socket send off this control packet _parentSocket->writeBasePacket(*lossListPacket, _destination); diff --git a/libraries/networking/src/udt/LossList.cpp b/libraries/networking/src/udt/LossList.cpp index ab860a768d..c02d12c34a 100644 --- a/libraries/networking/src/udt/LossList.cpp +++ b/libraries/networking/src/udt/LossList.cpp @@ -167,9 +167,17 @@ SequenceNumber LossList::popFirstSequenceNumber() { return front; } -void LossList::write(ControlPacket& packet) { +void LossList::write(ControlPacket& packet, int maxPairs) { + int writtenPairs = 0; for(const auto& pair : _lossList) { packet.writePrimitive(pair.first); packet.writePrimitive(pair.second); + + ++writtenPairs; + + // check if we've written the maximum number we were told to write + if (maxPairs != -1 && writtenPairs >= maxPairs) { + break; + } } } diff --git a/libraries/networking/src/udt/LossList.h b/libraries/networking/src/udt/LossList.h index fc97206282..d3cbef7ebe 100644 --- a/libraries/networking/src/udt/LossList.h +++ b/libraries/networking/src/udt/LossList.h @@ -40,7 +40,7 @@ public: SequenceNumber getFirstSequenceNumber() const; SequenceNumber popFirstSequenceNumber(); - void write(ControlPacket& packet); + void write(ControlPacket& packet, int maxPairs = -1); private: std::list> _lossList; From ec82e65aec02ae914bf486576c4eebc083fcd97a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Aug 2015 16:07:07 -0700 Subject: [PATCH 214/242] comment flow window size changes for light ACK --- libraries/networking/src/udt/Connection.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 273bb36378..ae4d04e930 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -475,8 +475,12 @@ void Connection::processLightACK(std::unique_ptr controlPacket) { // must be larger than the last received ACK to be processed if (ack > _lastReceivedACK) { + // NOTE: the following makes sense in UDT where there is a dynamic receive buffer. + // Since we have a receive buffer that is always of a default size, we don't use this light ACK to + // drop the flow window size. + // decrease the flow window size by the offset between the last received ACK and this ACK - _flowWindowSize -= seqoff(_lastReceivedACK, ack); + // _flowWindowSize -= seqoff(_lastReceivedACK, ack); // update the last received ACK to the this one _lastReceivedACK = ack; From 8c22627f3248727175f843ec4a4cbd35fe9a5e32 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Aug 2015 16:07:29 -0700 Subject: [PATCH 215/242] fix a typo in comment --- libraries/networking/src/udt/Connection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index ae4d04e930..c26e758050 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -595,7 +595,7 @@ void Connection::updateCongestionControlAndSendQueue(std::function cong // fire congestion control callback congestionCallback(); - // now that we've update the congestion control, update the packet send period and flow window size + // now that we've updated the congestion control, update the packet send period and flow window size getSendQueue().setPacketSendPeriod(_congestionControl->_packetSendPeriod); getSendQueue().setFlowWindowSize(std::min(_flowWindowSize, (int) _congestionControl->_congestionWindowSize)); From bc6846e08cbb73a40aa0349a81a85565b3505381 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Aug 2015 16:07:55 -0700 Subject: [PATCH 216/242] fix a typo in Connection header --- libraries/networking/src/udt/Connection.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 9bdc6baf24..65e7fe74f0 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -46,7 +46,7 @@ public: void sync(); // rate control method, fired by Socket for all connections on SYN interval - // returns indicates if this packet was a duplicate + // return indicates if this packet was a duplicate bool processReceivedSequenceNumber(SequenceNumber sequenceNumber, int packetSize, int payloadSize); void processControl(std::unique_ptr controlPacket); From 71b2d90c64ca3dba30cf7f482ef73959dc2984de Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Aug 2015 16:08:34 -0700 Subject: [PATCH 217/242] clarify comment in LossList header --- libraries/networking/src/udt/LossList.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/LossList.h b/libraries/networking/src/udt/LossList.h index d3cbef7ebe..f0f2b92988 100644 --- a/libraries/networking/src/udt/LossList.h +++ b/libraries/networking/src/udt/LossList.h @@ -26,11 +26,11 @@ public: void clear() { _length = 0; _lossList.clear(); } - // Should always add at the end + // must always add at the end - faster than insert void append(SequenceNumber seq); void append(SequenceNumber start, SequenceNumber end); - // Inserts anywhere - MUCH slower + // inserts anywhere - MUCH slower void insert(SequenceNumber start, SequenceNumber end); bool remove(SequenceNumber seq); From 1c176e55ed6570e4c668dac96024837fd7fe278c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Aug 2015 16:13:14 -0700 Subject: [PATCH 218/242] add some comments to PacketTimeWindow --- libraries/networking/src/udt/PacketTimeWindow.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libraries/networking/src/udt/PacketTimeWindow.cpp b/libraries/networking/src/udt/PacketTimeWindow.cpp index 9193e8e0af..5c389f6d26 100644 --- a/libraries/networking/src/udt/PacketTimeWindow.cpp +++ b/libraries/networking/src/udt/PacketTimeWindow.cpp @@ -51,6 +51,7 @@ int32_t meanOfMedianFilteredValues(std::vector intervals, int numValues, in // grab the median value of the intervals vector int intervalsMedian = median(intervals.begin(), intervals.end()); + // figure out our bounds for median filtering static const int MEDIAN_FILTERING_BOUND_MULTIPLIER = 8; int upperBound = intervalsMedian * MEDIAN_FILTERING_BOUND_MULTIPLIER; int lowerBound = intervalsMedian / MEDIAN_FILTERING_BOUND_MULTIPLIER; @@ -58,6 +59,7 @@ int32_t meanOfMedianFilteredValues(std::vector intervals, int numValues, in int sum = 0; int count = 0; + // sum the values that are inside the median filtered bounds for (auto& interval : intervals) { if ((interval < upperBound) && (interval > lowerBound)) { ++count; @@ -65,7 +67,9 @@ int32_t meanOfMedianFilteredValues(std::vector intervals, int numValues, in } } + // make sure we hit our threshold of values required if (count >= valuesRequired) { + // return the frequency (per second) for the mean interval static const double USECS_PER_SEC = 1000000.0; return (int32_t) ceil(USECS_PER_SEC / (((double) sum) / ((double) count))); } else { From feddb613e2736f3b4805dc2423d232c85e36ef85 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Aug 2015 16:28:33 -0700 Subject: [PATCH 219/242] add comments, use scoped locker in SendQueue --- libraries/networking/src/udt/SendQueue.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 111b676499..d004deea2f 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -149,8 +149,8 @@ void SendQueue::run() { bool resentPacket = false; + // the following while makes sure that we find a packet to re-send, if there is one while (!resentPacket) { - // prioritize a loss retransmission QWriteLocker naksLocker(&_naksLock); if (_naks.getLength() > 0) { @@ -191,11 +191,13 @@ void SendQueue::run() { break; } + // if we didn't find a packet to re-send AND we think we can fit a new packet on the wire + // (this is according to the current flow window size) then we send out a new packet if (!resentPacket && seqlen(SequenceNumber { (uint32_t) _lastACKSequenceNumber }, _currentSequenceNumber) <= _flowWindowSize) { // we didn't re-send a packet, so time to send a new one - _packetsLock.lockForWrite(); + QWriteLocker locker(&_packetsLock); if (_packets.size() > 0) { SequenceNumber nextNumber = getNextSequenceNumber(); @@ -217,7 +219,7 @@ void SendQueue::run() { } // unlock the packets, we're done pulling - _packetsLock.unlock(); + locker.unlock(); // definitely send the first packet sendNewPacketAndAddToSentList(move(firstPacket), nextNumber); @@ -229,7 +231,7 @@ void SendQueue::run() { } } else { - _packetsLock.unlock(); + locker.unlock(); } } From fe100695f0f0be0632c531dd3deef5bcffd119f5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Aug 2015 16:29:21 -0700 Subject: [PATCH 220/242] rename dest to destination for SendQueue --- libraries/networking/src/udt/SendQueue.cpp | 6 +++--- libraries/networking/src/udt/SendQueue.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index d004deea2f..d3095bff15 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -25,14 +25,14 @@ using namespace udt; using namespace std::chrono; -std::unique_ptr SendQueue::create(Socket* socket, HifiSockAddr dest) { - auto queue = std::unique_ptr(new SendQueue(socket, dest)); +std::unique_ptr SendQueue::create(Socket* socket, HifiSockAddr destination) { + auto queue = std::unique_ptr(new SendQueue(socket, destination)); Q_ASSERT_X(socket, "SendQueue::create", "Must be called with a valid Socket*"); // Setup queue private thread QThread* thread = new QThread(); - thread->setObjectName("Networking: SendQueue " + dest.objectName()); // Name thread for easier debug + thread->setObjectName("Networking: SendQueue " + destination.objectName()); // Name thread for easier debug connect(thread, &QThread::started, queue.get(), &SendQueue::run); diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 004938051e..bbb1dc3798 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -37,7 +37,7 @@ class SendQueue : public QObject { Q_OBJECT public: - static std::unique_ptr create(Socket* socket, HifiSockAddr dest); + static std::unique_ptr create(Socket* socket, HifiSockAddr destination); void queuePacket(std::unique_ptr packet); int getQueueSize() const { QReadLocker locker(&_packetsLock); return _packets.size(); } From d89c54112dadc7440fbcb7cca7b8f6467b6678bb Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Aug 2015 16:35:23 -0700 Subject: [PATCH 221/242] cleanup indentation in Socket --- libraries/networking/src/udt/Socket.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index a7dd6377ba..a280a68716 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -128,8 +128,8 @@ Connection& Socket::findOrCreateConnection(const HifiSockAddr& sockAddr) { auto it = _connectionsHash.find(sockAddr); if (it == _connectionsHash.end()) { - it = _connectionsHash.insert(it, std::make_pair(sockAddr, - std::unique_ptr(new Connection(this, sockAddr, _ccFactory->create())))); + auto connection = std::unique_ptr(new Connection(this, sockAddr, _ccFactory->create())); + it = _connectionsHash.insert(it, std::make_pair(sockAddr, std::move(connection))); } return *it->second; From 2ec4d8ff3cd3634e15f719c358f6716291a865db Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Aug 2015 16:38:59 -0700 Subject: [PATCH 222/242] rename return of HifiSockAddr objects for connections --- libraries/networking/src/udt/Socket.cpp | 2 +- libraries/networking/src/udt/Socket.h | 4 ++-- tools/udt-test/src/UDTTest.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index a280a68716..a55a129c0a 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -237,7 +237,7 @@ ConnectionStats::Stats Socket::sampleStatsForConnection(const HifiSockAddr& dest } } -std::vector Socket::getSockAddr() { +std::vector Socket::getConnectionSockAddrs() { std::vector addr; addr.reserve(_connectionsHash.size()); for (const auto& connectionPair : _connectionsHash) { diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 602cfaf8ec..e3cf347905 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -65,7 +65,7 @@ public: void connectToSendSignal(const HifiSockAddr& destinationAddr, QObject* receiver, const char* slot); ConnectionStats::Stats sampleStatsForConnection(const HifiSockAddr& destination); - std::vector getSockAddr(); + std::vector getConnectionSockAddrs(); private slots: void readPendingDatagrams(); @@ -83,7 +83,7 @@ private: std::unordered_map _unreliableSequenceNumbers; std::unordered_map> _connectionsHash; - int32_t _synInterval = 10; // 10ms + int _synInterval = 10; // 10ms QTimer _synTimer; std::unique_ptr _ccFactory { new CongestionControlFactory() }; diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 6c163a726e..d24a9282e9 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -264,7 +264,7 @@ void UDTTest::sampleStats() { first = false; } - auto sockets = _socket.getSockAddr(); + auto sockets = _socket.getConnectionSockAddrs(); if (sockets.size() > 0) { udt::ConnectionStats::Stats stats = _socket.sampleStatsForConnection(sockets.front()); From 47d17102082ac707cf82c9837154f85156662d59 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Aug 2015 16:39:29 -0700 Subject: [PATCH 223/242] remove msecTimestampNow that is not used --- libraries/shared/src/SharedUtil.cpp | 4 ---- libraries/shared/src/SharedUtil.h | 1 - 2 files changed, 5 deletions(-) diff --git a/libraries/shared/src/SharedUtil.cpp b/libraries/shared/src/SharedUtil.cpp index d73e8f446e..f78c8c47e0 100644 --- a/libraries/shared/src/SharedUtil.cpp +++ b/libraries/shared/src/SharedUtil.cpp @@ -39,10 +39,6 @@ void usecTimestampNowForceClockSkew(int clockSkew) { ::usecTimestampNowAdjust = clockSkew; } -quint64 msecTimestampNow(bool wantDebug) { - return usecTimestampNow() / 1000; -} - quint64 usecTimestampNow(bool wantDebug) { static bool usecTimestampNowIsInitialized = false; static qint64 TIME_REFERENCE = 0; // in usec diff --git a/libraries/shared/src/SharedUtil.h b/libraries/shared/src/SharedUtil.h index a60c0defd0..17a32e7910 100644 --- a/libraries/shared/src/SharedUtil.h +++ b/libraries/shared/src/SharedUtil.h @@ -64,7 +64,6 @@ inline bool operator!=(const xColor& lhs, const xColor& rhs) // Use a custom User-Agent to avoid ModSecurity filtering, e.g. by hosting providers. const QByteArray HIGH_FIDELITY_USER_AGENT = "Mozilla/5.0 (HighFidelityInterface)"; -quint64 msecTimestampNow(bool wantDebug = false); quint64 usecTimestampNow(bool wantDebug = false); void usecTimestampNowForceClockSkew(int clockSkew); From abf07dc06df1c0c339c3edd9dcb65fef4eecfa06 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Aug 2015 16:39:52 -0700 Subject: [PATCH 224/242] move hasher for UUID to UUID.h --- libraries/networking/src/UUID.h | 6 ++++++ libraries/octree/src/OctreeEditPacketSender.h | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libraries/networking/src/UUID.h b/libraries/networking/src/UUID.h index 7e7048486f..b8cb1b56ca 100644 --- a/libraries/networking/src/UUID.h +++ b/libraries/networking/src/UUID.h @@ -18,4 +18,10 @@ const int NUM_BYTES_RFC4122_UUID = 16; QString uuidStringWithoutCurlyBraces(const QUuid& uuid); +template <> struct std::hash { + size_t operator()(const QUuid& uuid) const { + return qHash(uuid); + } +}; + #endif // hifi_UUID_h diff --git a/libraries/octree/src/OctreeEditPacketSender.h b/libraries/octree/src/OctreeEditPacketSender.h index d10870a221..d8761e6161 100644 --- a/libraries/octree/src/OctreeEditPacketSender.h +++ b/libraries/octree/src/OctreeEditPacketSender.h @@ -20,12 +20,6 @@ #include "JurisdictionMap.h" #include "SentPacketHistory.h" -template <> struct std::hash { - size_t operator()(const QUuid& uuid) const { - return qHash(uuid); - } -}; - /// Utility for processing, packing, queueing and sending of outbound edit messages. class OctreeEditPacketSender : public PacketSender { Q_OBJECT From d9f445d5f69001a306f1862dad2d00a721d397c0 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Aug 2015 16:45:15 -0700 Subject: [PATCH 225/242] put custom hashing for UUID in UUIDHasher --- assignment-client/src/avatars/AvatarMixerClientData.h | 2 +- libraries/networking/src/UUID.h | 6 ------ libraries/networking/src/UUIDHasher.h | 10 +++++++--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index fd0971294c..3dbe917ee2 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -81,7 +81,7 @@ private: AvatarData _avatar; uint16_t _lastReceivedSequenceNumber { 0 }; - std::unordered_map _lastBroadcastSequenceNumbers; + std::unordered_map _lastBroadcastSequenceNumbers; bool _hasReceivedFirstPackets = false; quint64 _billboardChangeTimestamp = 0; diff --git a/libraries/networking/src/UUID.h b/libraries/networking/src/UUID.h index b8cb1b56ca..7e7048486f 100644 --- a/libraries/networking/src/UUID.h +++ b/libraries/networking/src/UUID.h @@ -18,10 +18,4 @@ const int NUM_BYTES_RFC4122_UUID = 16; QString uuidStringWithoutCurlyBraces(const QUuid& uuid); -template <> struct std::hash { - size_t operator()(const QUuid& uuid) const { - return qHash(uuid); - } -}; - #endif // hifi_UUID_h diff --git a/libraries/networking/src/UUIDHasher.h b/libraries/networking/src/UUIDHasher.h index 3ece236812..b05e517841 100644 --- a/libraries/networking/src/UUIDHasher.h +++ b/libraries/networking/src/UUIDHasher.h @@ -20,9 +20,13 @@ class UUIDHasher { public: size_t operator()(const QUuid& uuid) const { - return uuid.data1 ^ uuid.data2 ^ (uuid.data3 << 16) - ^ ((uuid.data4[0] << 24) | (uuid.data4[1] << 16) | (uuid.data4[2] << 8) | uuid.data4[3]) - ^ ((uuid.data4[4] << 24) | (uuid.data4[5] << 16) | (uuid.data4[6] << 8) | uuid.data4[7]); + return qHash(uuid); + } +}; + +template <> struct std::hash { + size_t operator()(const QUuid& uuid) const { + return qHash(uuid); } }; From 75e7de3019831a0dc45b3c6194074399ac82df81 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Aug 2015 16:49:08 -0700 Subject: [PATCH 226/242] remove getConnectionSockAddrs vector move --- libraries/networking/src/udt/Socket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index a55a129c0a..530d268425 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -243,5 +243,5 @@ std::vector Socket::getConnectionSockAddrs() { for (const auto& connectionPair : _connectionsHash) { addr.push_back(connectionPair.first); } - return std::move(addr); + return addr; } From f039851f13546d877d4179e386282419d6a3b452 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Aug 2015 16:57:42 -0700 Subject: [PATCH 227/242] remove std namespacing from Connection --- libraries/networking/src/udt/Connection.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index c26e758050..e07d987c4a 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -22,10 +22,9 @@ #include "Socket.h" using namespace udt; -using namespace std; using namespace std::chrono; -Connection::Connection(Socket* parentSocket, HifiSockAddr destination, unique_ptr congestionControl) : +Connection::Connection(Socket* parentSocket, HifiSockAddr destination, std::unique_ptr congestionControl) : _parentSocket(parentSocket), _destination(destination), _congestionControl(move(congestionControl)) @@ -71,7 +70,7 @@ SendQueue& Connection::getSendQueue() { return *_sendQueue; } -void Connection::sendReliablePacket(unique_ptr packet) { +void Connection::sendReliablePacket(std::unique_ptr packet) { Q_ASSERT_X(packet->isReliable(), "Connection::send", "Trying to send an unreliable packet reliably."); getSendQueue().queuePacket(move(packet)); } @@ -346,7 +345,7 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in return wasDuplicate; } -void Connection::processControl(unique_ptr controlPacket) { +void Connection::processControl(std::unique_ptr controlPacket) { // Simple dispatch to control packets processing methods based on their type switch (controlPacket->getType()) { case ControlPacket::ACK: From 4e540828bd3a7e17fd93e053f6da0a7d83072a40 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Aug 2015 09:33:22 -0700 Subject: [PATCH 228/242] use mod in place of branching for SequenceNumber --- libraries/networking/src/udt/SequenceNumber.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/SequenceNumber.h b/libraries/networking/src/udt/SequenceNumber.h index d0bea86777..9827fc79b2 100644 --- a/libraries/networking/src/udt/SequenceNumber.h +++ b/libraries/networking/src/udt/SequenceNumber.h @@ -39,11 +39,11 @@ public: explicit operator UType() { return static_cast(_value); } inline SequenceNumber& operator++() { - _value = (_value == MAX) ? 0 : ++_value; + _value = (_value + 1) % (MAX + 1); return *this; } inline SequenceNumber& operator--() { - _value = (_value == 0) ? MAX : --_value; + _value = (_value - 1) % (MAX + 1); return *this; } inline SequenceNumber operator++(int) { From 426a8909e19b45d217e5ba35e62fd5a434f20c63 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Aug 2015 09:40:13 -0700 Subject: [PATCH 229/242] wait on the SendQueue thread in Connection --- libraries/networking/src/udt/Connection.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index e07d987c4a..d8aee60432 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -47,9 +47,16 @@ Connection::Connection(Socket* parentSocket, HifiSockAddr destination, std::uniq Connection::~Connection() { if (_sendQueue) { + // grab the send queue thread so we can wait on it + QThread* sendQueueThread = _sendQueue->thread(); + + // tell the send queue to stop and be deleted _sendQueue->stop(); _sendQueue->deleteLater(); _sendQueue.release(); + + // wait on the send queue thread so we know the send queue is gone + sendQueueThread->wait(); } } From ffa2070a46365d7f26721adaace830e608684161 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Aug 2015 09:44:56 -0700 Subject: [PATCH 230/242] remove commented stats in UDTTest --- tools/udt-test/src/UDTTest.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index d24a9282e9..1780231b99 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -276,9 +276,6 @@ void UDTTest::sampleStats() { QString::number(stats.estimatedBandwith).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.rtt / USECS_PER_MSEC).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.congestionWindowSize).leftJustified(CLIENT_STATS_TABLE_HEADERS[++headerIndex].size()), -// QString::number(stats.recievedBytes).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), -// QString::number(stats.recievedUtilBytes).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), -// QString::number(100 * (double)stats.recievedUtilBytes / (double)stats.recievedBytes).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.events[udt::ConnectionStats::Stats::SentACK]).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.events[udt::ConnectionStats::Stats::SentLightACK]).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), QString::number(stats.events[udt::ConnectionStats::Stats::SentNAK]).leftJustified(SERVER_STATS_TABLE_HEADERS[++headerIndex].size()), From 5c2348cf0d8bc44e59878be48608ff85c048a747 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Aug 2015 09:49:39 -0700 Subject: [PATCH 231/242] remove header for commented stats --- tools/udt-test/src/UDTTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 1780231b99..1f55fd1665 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -51,7 +51,6 @@ const QStringList CLIENT_STATS_TABLE_HEADERS { const QStringList SERVER_STATS_TABLE_HEADERS { "Receive Rate (P/s)", "Bandwidth (P/s)", "RTT(ms)", "CW (P)", - //"Total Bytes", "Util Bytes", "Ratio (%)", "Sent ACK", "Sent LACK", "Sent NAK", "Sent TNAK", "Recieved ACK2", "Duplicate Packets" }; From b7d779bb25b35b31f8e571cf40cc0f198c9c2cac Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Aug 2015 09:56:00 -0700 Subject: [PATCH 232/242] put back branching for signed Type --- libraries/networking/src/udt/SequenceNumber.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/SequenceNumber.h b/libraries/networking/src/udt/SequenceNumber.h index 9827fc79b2..fae5a7bb01 100644 --- a/libraries/networking/src/udt/SequenceNumber.h +++ b/libraries/networking/src/udt/SequenceNumber.h @@ -43,7 +43,7 @@ public: return *this; } inline SequenceNumber& operator--() { - _value = (_value - 1) % (MAX + 1); + _value = (_value == 0) ? MAX : --_value; return *this; } inline SequenceNumber operator++(int) { From bf919f105a5c96bbf6c6df1fbd607b1260cacca2 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Aug 2015 11:29:07 -0700 Subject: [PATCH 233/242] add a simple container-test, speed HifiSockAddr hashing --- libraries/networking/src/HifiSockAddr.h | 18 +++++- tools/CMakeLists.txt | 3 + tools/container-test/CMakeLists.txt | 4 ++ tools/container-test/src/main.cpp | 84 +++++++++++++++++++++++++ 4 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 tools/container-test/CMakeLists.txt create mode 100644 tools/container-test/src/main.cpp diff --git a/libraries/networking/src/HifiSockAddr.h b/libraries/networking/src/HifiSockAddr.h index 25f7ad2194..e6f018413f 100644 --- a/libraries/networking/src/HifiSockAddr.h +++ b/libraries/networking/src/HifiSockAddr.h @@ -70,10 +70,24 @@ uint qHash(const HifiSockAddr& key, uint seed); template <> struct std::hash { + // NOTE: this hashing specifically ignores IPv6 addresses - if we begin to support those we will need + // to conditionally hash the bytes that represent an IPv6 address std::size_t operator()(const HifiSockAddr& sockAddr) const { // use XOR of implemented std::hash templates for new hash - return std::hash()(sockAddr.getAddress().toString().toStdString()) - ^ std::hash()((uint16_t) sockAddr.getPort()); + // depending on the type of address we're looking at + + if (sockAddr.getAddress().protocol() == QAbstractSocket::IPv4Protocol) { + return std::hash()((uint32_t) sockAddr.getAddress().toIPv4Address()) + ^ std::hash()((uint16_t) sockAddr.getPort()); + } else if (sockAddr.getAddress().protocol() == QAbstractSocket::IPv6Protocol) { + // use XOR of implemented std::hash templates for new hash + return std::hash()(reinterpret_cast(sockAddr.getAddress().toIPv6Address().c)) + ^ std::hash()((uint16_t) sockAddr.getPort()); + } else { + return std::hash()(sockAddr.getAddress().toString().toStdString()) + ^ std::hash()((uint16_t) sockAddr.getPort()); + + } } }; diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 2056044a4b..7ab4525707 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -8,5 +8,8 @@ set_target_properties(scribe PROPERTIES FOLDER "Tools") add_subdirectory(udt-test) set_target_properties(udt-test PROPERTIES FOLDER "Tools") +add_subdirectory(container-test) +set_target_properties(container-test PROPERTIES FOLDER "Tools") + add_subdirectory(vhacd-util) set_target_properties(vhacd-util PROPERTIES FOLDER "Tools") diff --git a/tools/container-test/CMakeLists.txt b/tools/container-test/CMakeLists.txt new file mode 100644 index 0000000000..0eea7ddd84 --- /dev/null +++ b/tools/container-test/CMakeLists.txt @@ -0,0 +1,4 @@ +set(TARGET_NAME container-test) +setup_hifi_project() + +link_hifi_libraries(networking) \ No newline at end of file diff --git a/tools/container-test/src/main.cpp b/tools/container-test/src/main.cpp new file mode 100644 index 0000000000..7b570b4172 --- /dev/null +++ b/tools/container-test/src/main.cpp @@ -0,0 +1,84 @@ +// +// main.cpp +// tools/container-test/src +// +// Created by Stephen Birarda on 8/18/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html + +#include +#include +#include +#include +#include + +#include + +using namespace std::chrono; + +int main(int argc, char* argv[]) { + static const int NUM_ELEMENTS = 50; + + std::random_device rd; + std::mt19937 generator(rd()); + std::uniform_int_distribution<> ipDistribution(1, UINT32_MAX); + std::uniform_int_distribution<> portDistribution(1, UINT16_MAX); + + // from 1 to NUM_ELEMENTS, create a vector and unordered map with that many elements and compare how long it takes to pull the last value + for (int i = 1; i <= NUM_ELEMENTS; ++i) { + // create our vector with i elements + std::vector> vectorElements(i); + + // create our unordered map with i elements + std::unordered_map hashElements; + + HifiSockAddr lastAddress; + + // fill the structures with HifiSockAddr + for (int j = 0; j < i; ++j) { + // create a random IP address + quint32 randomNumber = ipDistribution(generator); + quint32 randomIP = (randomNumber >> 24 & 0xFF) << 24 | + (randomNumber >> 16 & 0xFF) << 16 | + (randomNumber >> 8 & 0xFF) << 8 | + (randomNumber & 0xFF); + + HifiSockAddr randomAddress(QHostAddress(randomIP), portDistribution(generator)); + + vectorElements.push_back({ randomAddress, 12 }); + hashElements.insert({ randomAddress, 12 }); + + if (j == i - 1) { + // this is the last element - store it for lookup purposes + lastAddress = randomAddress; + } + } + + // time how long it takes to get the value for the key lastAddress, for each container + auto before = high_resolution_clock::now(); + int vectorResult; + + for (auto& vi : vectorElements) { + if (vi.first == lastAddress) { + vectorResult = vi.second;; + break; + } + } + + auto vectorDuration = duration_cast(high_resolution_clock::now() - before); + + Q_UNUSED(vectorResult); + + before = high_resolution_clock::now(); + auto mi = hashElements.find(lastAddress); + int mapResult = mi->second; + auto mapDuration = duration_cast(high_resolution_clock::now() - before); + + Q_UNUSED(mapResult); + + qDebug() << i << QString("v: %1ns").arg(vectorDuration.count()) << QString("m: %2ns").arg(mapDuration.count()); + } +} + From c6467f7fbc4a7be03d22d96f78a0b50b02077f87 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Aug 2015 11:31:11 -0700 Subject: [PATCH 234/242] rename the container-test to container-profile --- tools/CMakeLists.txt | 4 ++-- tools/{container-test => container-profile}/CMakeLists.txt | 2 +- tools/{container-test => container-profile}/src/main.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename tools/{container-test => container-profile}/CMakeLists.txt (60%) rename tools/{container-test => container-profile}/src/main.cpp (98%) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 7ab4525707..3dcd7aab57 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -8,8 +8,8 @@ set_target_properties(scribe PROPERTIES FOLDER "Tools") add_subdirectory(udt-test) set_target_properties(udt-test PROPERTIES FOLDER "Tools") -add_subdirectory(container-test) -set_target_properties(container-test PROPERTIES FOLDER "Tools") +add_subdirectory(container-profile) +set_target_properties(container-profile PROPERTIES FOLDER "Tools") add_subdirectory(vhacd-util) set_target_properties(vhacd-util PROPERTIES FOLDER "Tools") diff --git a/tools/container-test/CMakeLists.txt b/tools/container-profile/CMakeLists.txt similarity index 60% rename from tools/container-test/CMakeLists.txt rename to tools/container-profile/CMakeLists.txt index 0eea7ddd84..9c0adb99f2 100644 --- a/tools/container-test/CMakeLists.txt +++ b/tools/container-profile/CMakeLists.txt @@ -1,4 +1,4 @@ -set(TARGET_NAME container-test) +set(TARGET_NAME container-profile) setup_hifi_project() link_hifi_libraries(networking) \ No newline at end of file diff --git a/tools/container-test/src/main.cpp b/tools/container-profile/src/main.cpp similarity index 98% rename from tools/container-test/src/main.cpp rename to tools/container-profile/src/main.cpp index 7b570b4172..3cde3b2a23 100644 --- a/tools/container-test/src/main.cpp +++ b/tools/container-profile/src/main.cpp @@ -1,6 +1,6 @@ // // main.cpp -// tools/container-test/src +// tools/container-profile/src // // Created by Stephen Birarda on 8/18/15. // Copyright 2015 High Fidelity, Inc. From 972cf8cb02594375c37e65d01c65ff617f6a35e1 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Aug 2015 11:32:00 -0700 Subject: [PATCH 235/242] remove the container-profile tool --- tools/CMakeLists.txt | 3 - tools/container-profile/CMakeLists.txt | 4 -- tools/container-profile/src/main.cpp | 84 -------------------------- 3 files changed, 91 deletions(-) delete mode 100644 tools/container-profile/CMakeLists.txt delete mode 100644 tools/container-profile/src/main.cpp diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 3dcd7aab57..2056044a4b 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -8,8 +8,5 @@ set_target_properties(scribe PROPERTIES FOLDER "Tools") add_subdirectory(udt-test) set_target_properties(udt-test PROPERTIES FOLDER "Tools") -add_subdirectory(container-profile) -set_target_properties(container-profile PROPERTIES FOLDER "Tools") - add_subdirectory(vhacd-util) set_target_properties(vhacd-util PROPERTIES FOLDER "Tools") diff --git a/tools/container-profile/CMakeLists.txt b/tools/container-profile/CMakeLists.txt deleted file mode 100644 index 9c0adb99f2..0000000000 --- a/tools/container-profile/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(TARGET_NAME container-profile) -setup_hifi_project() - -link_hifi_libraries(networking) \ No newline at end of file diff --git a/tools/container-profile/src/main.cpp b/tools/container-profile/src/main.cpp deleted file mode 100644 index 3cde3b2a23..0000000000 --- a/tools/container-profile/src/main.cpp +++ /dev/null @@ -1,84 +0,0 @@ -// -// main.cpp -// tools/container-profile/src -// -// Created by Stephen Birarda on 8/18/15. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html - -#include -#include -#include -#include -#include - -#include - -using namespace std::chrono; - -int main(int argc, char* argv[]) { - static const int NUM_ELEMENTS = 50; - - std::random_device rd; - std::mt19937 generator(rd()); - std::uniform_int_distribution<> ipDistribution(1, UINT32_MAX); - std::uniform_int_distribution<> portDistribution(1, UINT16_MAX); - - // from 1 to NUM_ELEMENTS, create a vector and unordered map with that many elements and compare how long it takes to pull the last value - for (int i = 1; i <= NUM_ELEMENTS; ++i) { - // create our vector with i elements - std::vector> vectorElements(i); - - // create our unordered map with i elements - std::unordered_map hashElements; - - HifiSockAddr lastAddress; - - // fill the structures with HifiSockAddr - for (int j = 0; j < i; ++j) { - // create a random IP address - quint32 randomNumber = ipDistribution(generator); - quint32 randomIP = (randomNumber >> 24 & 0xFF) << 24 | - (randomNumber >> 16 & 0xFF) << 16 | - (randomNumber >> 8 & 0xFF) << 8 | - (randomNumber & 0xFF); - - HifiSockAddr randomAddress(QHostAddress(randomIP), portDistribution(generator)); - - vectorElements.push_back({ randomAddress, 12 }); - hashElements.insert({ randomAddress, 12 }); - - if (j == i - 1) { - // this is the last element - store it for lookup purposes - lastAddress = randomAddress; - } - } - - // time how long it takes to get the value for the key lastAddress, for each container - auto before = high_resolution_clock::now(); - int vectorResult; - - for (auto& vi : vectorElements) { - if (vi.first == lastAddress) { - vectorResult = vi.second;; - break; - } - } - - auto vectorDuration = duration_cast(high_resolution_clock::now() - before); - - Q_UNUSED(vectorResult); - - before = high_resolution_clock::now(); - auto mi = hashElements.find(lastAddress); - int mapResult = mi->second; - auto mapDuration = duration_cast(high_resolution_clock::now() - before); - - Q_UNUSED(mapResult); - - qDebug() << i << QString("v: %1ns").arg(vectorDuration.count()) << QString("m: %2ns").arg(mapDuration.count()); - } -} - From 1c3543febc2ff3a4c39740e995369bb1e7028fb4 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Aug 2015 11:39:04 -0700 Subject: [PATCH 236/242] use string style hashing for IPv6 addresses --- libraries/networking/src/HifiSockAddr.h | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/HifiSockAddr.h b/libraries/networking/src/HifiSockAddr.h index e6f018413f..c66579eca1 100644 --- a/libraries/networking/src/HifiSockAddr.h +++ b/libraries/networking/src/HifiSockAddr.h @@ -79,14 +79,11 @@ struct std::hash { if (sockAddr.getAddress().protocol() == QAbstractSocket::IPv4Protocol) { return std::hash()((uint32_t) sockAddr.getAddress().toIPv4Address()) ^ std::hash()((uint16_t) sockAddr.getPort()); - } else if (sockAddr.getAddress().protocol() == QAbstractSocket::IPv6Protocol) { - // use XOR of implemented std::hash templates for new hash - return std::hash()(reinterpret_cast(sockAddr.getAddress().toIPv6Address().c)) - ^ std::hash()((uint16_t) sockAddr.getPort()); } else { + // NOTE: if we start to use IPv6 addresses, it's possible their hashing + // can be faster by XORing the hash for each 64 bits in the address return std::hash()(sockAddr.getAddress().toString().toStdString()) ^ std::hash()((uint16_t) sockAddr.getPort()); - } } }; From 3d09f405ed41adb0b31cf1a2b44446989b2938ae Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Aug 2015 11:50:23 -0700 Subject: [PATCH 237/242] fix bad merge for State.h --- libraries/gpu/src/gpu/State.h | 714 +++++++++++++++++----------------- 1 file changed, 357 insertions(+), 357 deletions(-) diff --git a/libraries/gpu/src/gpu/State.h b/libraries/gpu/src/gpu/State.h index 8f09669f64..5f2a78690b 100755 --- a/libraries/gpu/src/gpu/State.h +++ b/libraries/gpu/src/gpu/State.h @@ -22,387 +22,387 @@ // Because some of the fields are bool packed tightly in the State::Cache class // and it s just not good anymore for template T& variable manipulation... #define SET_FIELD(field, defaultValue, value, dest) {\ - dest = value;\ - if (value == defaultValue) {\ - _signature.reset(field);\ - } else {\ - _signature.set(field);\ - }\ - _stamp++;\ +dest = value;\ +if (value == defaultValue) {\ +_signature.reset(field);\ +} else {\ +_signature.set(field);\ +}\ +_stamp++;\ }\ namespace gpu { - -class GPUObject; - -class State { -public: - State(); - virtual ~State(); - - Stamp getStamp() const { return _stamp; } - - typedef ::gpu::ComparisonFunction ComparisonFunction; - - enum FillMode { - FILL_POINT = 0, - FILL_LINE, - FILL_FACE, - - NUM_FILL_MODES, - }; - - enum CullMode { - CULL_NONE = 0, - CULL_FRONT, - CULL_BACK, - - NUM_CULL_MODES, - }; - - enum StencilOp { - STENCIL_OP_KEEP = 0, - STENCIL_OP_ZERO, - STENCIL_OP_REPLACE, - STENCIL_OP_INCR_SAT, - STENCIL_OP_DECR_SAT, - STENCIL_OP_INVERT, - STENCIL_OP_INCR, - STENCIL_OP_DECR, - - NUM_STENCIL_OPS, - }; - - enum BlendArg { - ZERO = 0, - ONE, - SRC_COLOR, - INV_SRC_COLOR, - SRC_ALPHA, - INV_SRC_ALPHA, - DEST_ALPHA, - INV_DEST_ALPHA, - DEST_COLOR, - INV_DEST_COLOR, - SRC_ALPHA_SAT, - FACTOR_COLOR, - INV_FACTOR_COLOR, - FACTOR_ALPHA, - INV_FACTOR_ALPHA, - - NUM_BLEND_ARGS, - }; - - enum BlendOp { - BLEND_OP_ADD = 0, - BLEND_OP_SUBTRACT, - BLEND_OP_REV_SUBTRACT, - BLEND_OP_MIN, - BLEND_OP_MAX, - - NUM_BLEND_OPS, - }; - - enum ColorMask - { - WRITE_NONE = 0, - WRITE_RED = 1, - WRITE_GREEN = 2, - WRITE_BLUE = 4, - WRITE_ALPHA = 8, - WRITE_ALL = (WRITE_RED | WRITE_GREEN | WRITE_BLUE | WRITE_ALPHA ), - }; - - class DepthTest { - uint8 _function = LESS; - uint8 _writeMask = true; - uint8 _enabled = false; - uint8 _spare = 0; + + class GPUObject; + + class State { public: - DepthTest(bool enabled = false, bool writeMask = true, ComparisonFunction func = LESS) : + State(); + virtual ~State(); + + Stamp getStamp() const { return _stamp; } + + typedef ::gpu::ComparisonFunction ComparisonFunction; + + enum FillMode { + FILL_POINT = 0, + FILL_LINE, + FILL_FACE, + + NUM_FILL_MODES, + }; + + enum CullMode { + CULL_NONE = 0, + CULL_FRONT, + CULL_BACK, + + NUM_CULL_MODES, + }; + + enum StencilOp { + STENCIL_OP_KEEP = 0, + STENCIL_OP_ZERO, + STENCIL_OP_REPLACE, + STENCIL_OP_INCR_SAT, + STENCIL_OP_DECR_SAT, + STENCIL_OP_INVERT, + STENCIL_OP_INCR, + STENCIL_OP_DECR, + + NUM_STENCIL_OPS, + }; + + enum BlendArg { + ZERO = 0, + ONE, + SRC_COLOR, + INV_SRC_COLOR, + SRC_ALPHA, + INV_SRC_ALPHA, + DEST_ALPHA, + INV_DEST_ALPHA, + DEST_COLOR, + INV_DEST_COLOR, + SRC_ALPHA_SAT, + FACTOR_COLOR, + INV_FACTOR_COLOR, + FACTOR_ALPHA, + INV_FACTOR_ALPHA, + + NUM_BLEND_ARGS, + }; + + enum BlendOp { + BLEND_OP_ADD = 0, + BLEND_OP_SUBTRACT, + BLEND_OP_REV_SUBTRACT, + BLEND_OP_MIN, + BLEND_OP_MAX, + + NUM_BLEND_OPS, + }; + + enum ColorMask + { + WRITE_NONE = 0, + WRITE_RED = 1, + WRITE_GREEN = 2, + WRITE_BLUE = 4, + WRITE_ALPHA = 8, + WRITE_ALL = (WRITE_RED | WRITE_GREEN | WRITE_BLUE | WRITE_ALPHA ), + }; + + class DepthTest { + uint8 _function = LESS; + uint8 _writeMask = true; + uint8 _enabled = false; + uint8 _spare = 0; + public: + DepthTest(bool enabled = false, bool writeMask = true, ComparisonFunction func = LESS) : _function(func), _writeMask(writeMask), _enabled(enabled) {} - - bool isEnabled() const { return _enabled != 0; } - ComparisonFunction getFunction() const { return ComparisonFunction(_function); } - uint8 getWriteMask() const { return _writeMask; } - - int32 getRaw() const { return *(reinterpret_cast(this)); } - DepthTest(int32 raw) { *(reinterpret_cast(this)) = raw; } - bool operator== (const DepthTest& right) const { return getRaw() == right.getRaw(); } - bool operator!= (const DepthTest& right) const { return getRaw() != right.getRaw(); } - }; - - class StencilTest { - static const int FUNC_MASK = 0x000f; - static const int FAIL_OP_MASK = 0x00f0; - static const int DEPTH_FAIL_OP_MASK = 0x0f00; - static const int PASS_OP_MASK = 0xf000; - static const int FAIL_OP_OFFSET = 4; - static const int DEPTH_FAIL_OP_OFFSET = 8; - static const int PASS_OP_OFFSET = 12; - - uint16 _functionAndOperations; - uint8 _reference = 0; - uint8 _readMask = 0xff; - public: - - StencilTest(uint8 reference = 0, uint8 readMask =0xFF, ComparisonFunction func = ALWAYS, StencilOp failOp = STENCIL_OP_KEEP, StencilOp depthFailOp = STENCIL_OP_KEEP, StencilOp passOp = STENCIL_OP_KEEP) : - _functionAndOperations(func | (failOp << FAIL_OP_OFFSET) | (depthFailOp << DEPTH_FAIL_OP_OFFSET) | (passOp << PASS_OP_OFFSET)), + + bool isEnabled() const { return _enabled != 0; } + ComparisonFunction getFunction() const { return ComparisonFunction(_function); } + uint8 getWriteMask() const { return _writeMask; } + + int32 getRaw() const { return *(reinterpret_cast(this)); } + DepthTest(int32 raw) { *(reinterpret_cast(this)) = raw; } + bool operator== (const DepthTest& right) const { return getRaw() == right.getRaw(); } + bool operator!= (const DepthTest& right) const { return getRaw() != right.getRaw(); } + }; + + class StencilTest { + static const int FUNC_MASK = 0x000f; + static const int FAIL_OP_MASK = 0x00f0; + static const int DEPTH_FAIL_OP_MASK = 0x0f00; + static const int PASS_OP_MASK = 0xf000; + static const int FAIL_OP_OFFSET = 4; + static const int DEPTH_FAIL_OP_OFFSET = 8; + static const int PASS_OP_OFFSET = 12; + + uint16 _functionAndOperations; + uint8 _reference = 0; + uint8 _readMask = 0xff; + public: + + StencilTest(uint8 reference = 0, uint8 readMask =0xFF, ComparisonFunction func = ALWAYS, StencilOp failOp = STENCIL_OP_KEEP, StencilOp depthFailOp = STENCIL_OP_KEEP, StencilOp passOp = STENCIL_OP_KEEP) : + _functionAndOperations(func | (failOp << FAIL_OP_OFFSET) | (depthFailOp << DEPTH_FAIL_OP_OFFSET) | (passOp << PASS_OP_OFFSET)), _reference(reference), _readMask(readMask) {} - - ComparisonFunction getFunction() const { return ComparisonFunction(_functionAndOperations & FUNC_MASK); } - StencilOp getFailOp() const { return StencilOp((_functionAndOperations & FAIL_OP_MASK) >> FAIL_OP_OFFSET); } - StencilOp getDepthFailOp() const { return StencilOp((_functionAndOperations & DEPTH_FAIL_OP_MASK) >> DEPTH_FAIL_OP_OFFSET); } - StencilOp getPassOp() const { return StencilOp((_functionAndOperations & PASS_OP_MASK) >> PASS_OP_OFFSET); } - - uint8 getReference() const { return _reference; } - uint8 getReadMask() const { return _readMask; } - - int32 getRaw() const { return *(reinterpret_cast(this)); } - StencilTest(int32 raw) { *(reinterpret_cast(this)) = raw; } - bool operator== (const StencilTest& right) const { return getRaw() == right.getRaw(); } - bool operator!= (const StencilTest& right) const { return getRaw() != right.getRaw(); } - }; - - class StencilActivation { - uint8 _frontWriteMask = 0xFF; - uint8 _backWriteMask = 0xFF; - uint16 _enabled = 0; - public: - - StencilActivation(bool enabled, uint8 frontWriteMask = 0xFF, uint8 backWriteMask = 0xFF) : + + ComparisonFunction getFunction() const { return ComparisonFunction(_functionAndOperations & FUNC_MASK); } + StencilOp getFailOp() const { return StencilOp((_functionAndOperations & FAIL_OP_MASK) >> FAIL_OP_OFFSET); } + StencilOp getDepthFailOp() const { return StencilOp((_functionAndOperations & DEPTH_FAIL_OP_MASK) >> DEPTH_FAIL_OP_OFFSET); } + StencilOp getPassOp() const { return StencilOp((_functionAndOperations & PASS_OP_MASK) >> PASS_OP_OFFSET); } + + uint8 getReference() const { return _reference; } + uint8 getReadMask() const { return _readMask; } + + int32 getRaw() const { return *(reinterpret_cast(this)); } + StencilTest(int32 raw) { *(reinterpret_cast(this)) = raw; } + bool operator== (const StencilTest& right) const { return getRaw() == right.getRaw(); } + bool operator!= (const StencilTest& right) const { return getRaw() != right.getRaw(); } + }; + + class StencilActivation { + uint8 _frontWriteMask = 0xFF; + uint8 _backWriteMask = 0xFF; + uint16 _enabled = 0; + public: + + StencilActivation(bool enabled, uint8 frontWriteMask = 0xFF, uint8 backWriteMask = 0xFF) : _frontWriteMask(frontWriteMask), _backWriteMask(backWriteMask), _enabled(enabled) {} - - bool isEnabled() const { return (_enabled != 0); } - uint8 getWriteMaskFront() const { return _frontWriteMask; } - uint8 getWriteMaskBack() const { return _backWriteMask; } - - int32 getRaw() const { return *(reinterpret_cast(this)); } - StencilActivation(int32 raw) { *(reinterpret_cast(this)) = raw; } - bool operator== (const StencilActivation& right) const { return getRaw() == right.getRaw(); } - bool operator!= (const StencilActivation& right) const { return getRaw() != right.getRaw(); } - }; - - class BlendFunction { - static const int COLOR_MASK = 0x0f; - static const int ALPHA_MASK = 0xf0; - static const int ALPHA_OFFSET = 4; - - uint8 _enabled; - uint8 _source; - uint8 _destination; - uint8 _operation; - public: - - BlendFunction(bool enabled, - BlendArg sourceColor, BlendOp operationColor, BlendArg destinationColor, - BlendArg sourceAlpha, BlendOp operationAlpha, BlendArg destinationAlpha) : + + bool isEnabled() const { return (_enabled != 0); } + uint8 getWriteMaskFront() const { return _frontWriteMask; } + uint8 getWriteMaskBack() const { return _backWriteMask; } + + int32 getRaw() const { return *(reinterpret_cast(this)); } + StencilActivation(int32 raw) { *(reinterpret_cast(this)) = raw; } + bool operator== (const StencilActivation& right) const { return getRaw() == right.getRaw(); } + bool operator!= (const StencilActivation& right) const { return getRaw() != right.getRaw(); } + }; + + class BlendFunction { + static const int COLOR_MASK = 0x0f; + static const int ALPHA_MASK = 0xf0; + static const int ALPHA_OFFSET = 4; + + uint8 _enabled; + uint8 _source; + uint8 _destination; + uint8 _operation; + public: + + BlendFunction(bool enabled, + BlendArg sourceColor, BlendOp operationColor, BlendArg destinationColor, + BlendArg sourceAlpha, BlendOp operationAlpha, BlendArg destinationAlpha) : _enabled(enabled), _source(sourceColor | (sourceAlpha << ALPHA_OFFSET)), _destination(destinationColor | (destinationAlpha << ALPHA_OFFSET)), _operation(operationColor | (operationAlpha << ALPHA_OFFSET)) {} - - BlendFunction(bool enabled, BlendArg source = ONE, BlendOp operation = BLEND_OP_ADD, BlendArg destination = ZERO) : + + BlendFunction(bool enabled, BlendArg source = ONE, BlendOp operation = BLEND_OP_ADD, BlendArg destination = ZERO) : _enabled(enabled), _source(source | (source << ALPHA_OFFSET)), _destination(destination | (destination << ALPHA_OFFSET)), _operation(operation | (operation << ALPHA_OFFSET)) {} - - bool isEnabled() const { return (_enabled != 0); } - - BlendArg getSourceColor() const { return BlendArg(_source & COLOR_MASK); } - BlendArg getDestinationColor() const { return BlendArg(_destination & COLOR_MASK); } - BlendOp getOperationColor() const { return BlendOp(_operation & COLOR_MASK); } - - BlendArg getSourceAlpha() const { return BlendArg((_source & ALPHA_MASK) >> ALPHA_OFFSET); } - BlendArg getDestinationAlpha() const { return BlendArg((_destination & ALPHA_MASK) >> ALPHA_OFFSET); } - BlendOp getOperationAlpha() const { return BlendOp((_operation & ALPHA_MASK) >> ALPHA_OFFSET); } - - int32 getRaw() const { return *(reinterpret_cast(this)); } - BlendFunction(int32 raw) { *(reinterpret_cast(this)) = raw; } - bool operator== (const BlendFunction& right) const { return getRaw() == right.getRaw(); } - bool operator!= (const BlendFunction& right) const { return getRaw() != right.getRaw(); } - }; - - // The Data class is the full explicit description of the State class fields value. - // Useful for having one const static called Default for reference or for the gpu::Backend to keep track of the current value - class Data { - public: - float depthBias = 0.0f; - float depthBiasSlopeScale = 0.0f; - - DepthTest depthTest = DepthTest(false, true, LESS); - - StencilActivation stencilActivation = StencilActivation(false); - StencilTest stencilTestFront = StencilTest(0, 0xff, ALWAYS, STENCIL_OP_KEEP, STENCIL_OP_KEEP, STENCIL_OP_KEEP); - StencilTest stencilTestBack = StencilTest(0, 0xff, ALWAYS, STENCIL_OP_KEEP, STENCIL_OP_KEEP, STENCIL_OP_KEEP); - - uint32 sampleMask = 0xFFFFFFFF; - - BlendFunction blendFunction = BlendFunction(false); - - uint8 fillMode = FILL_FACE; - uint8 cullMode = CULL_NONE; - - uint8 colorWriteMask = WRITE_ALL; - - bool frontFaceClockwise : 1; - bool depthClampEnable : 1; - bool scissorEnable : 1; - bool multisampleEnable : 1; - bool antialisedLineEnable : 1; - bool alphaToCoverageEnable : 1; - - Data() : + + bool isEnabled() const { return (_enabled != 0); } + + BlendArg getSourceColor() const { return BlendArg(_source & COLOR_MASK); } + BlendArg getDestinationColor() const { return BlendArg(_destination & COLOR_MASK); } + BlendOp getOperationColor() const { return BlendOp(_operation & COLOR_MASK); } + + BlendArg getSourceAlpha() const { return BlendArg((_source & ALPHA_MASK) >> ALPHA_OFFSET); } + BlendArg getDestinationAlpha() const { return BlendArg((_destination & ALPHA_MASK) >> ALPHA_OFFSET); } + BlendOp getOperationAlpha() const { return BlendOp((_operation & ALPHA_MASK) >> ALPHA_OFFSET); } + + int32 getRaw() const { return *(reinterpret_cast(this)); } + BlendFunction(int32 raw) { *(reinterpret_cast(this)) = raw; } + bool operator== (const BlendFunction& right) const { return getRaw() == right.getRaw(); } + bool operator!= (const BlendFunction& right) const { return getRaw() != right.getRaw(); } + }; + + // The Data class is the full explicit description of the State class fields value. + // Useful for having one const static called Default for reference or for the gpu::Backend to keep track of the current value + class Data { + public: + float depthBias = 0.0f; + float depthBiasSlopeScale = 0.0f; + + DepthTest depthTest = DepthTest(false, true, LESS); + + StencilActivation stencilActivation = StencilActivation(false); + StencilTest stencilTestFront = StencilTest(0, 0xff, ALWAYS, STENCIL_OP_KEEP, STENCIL_OP_KEEP, STENCIL_OP_KEEP); + StencilTest stencilTestBack = StencilTest(0, 0xff, ALWAYS, STENCIL_OP_KEEP, STENCIL_OP_KEEP, STENCIL_OP_KEEP); + + uint32 sampleMask = 0xFFFFFFFF; + + BlendFunction blendFunction = BlendFunction(false); + + uint8 fillMode = FILL_FACE; + uint8 cullMode = CULL_NONE; + + uint8 colorWriteMask = WRITE_ALL; + + bool frontFaceClockwise : 1; + bool depthClampEnable : 1; + bool scissorEnable : 1; + bool multisampleEnable : 1; + bool antialisedLineEnable : 1; + bool alphaToCoverageEnable : 1; + + Data() : frontFaceClockwise(false), depthClampEnable(false), scissorEnable(false), multisampleEnable(false), antialisedLineEnable(true), alphaToCoverageEnable(false) - {} + {} + }; + + // The unique default values for all the fields + static const Data DEFAULT; + void setFillMode(FillMode fill) { SET_FIELD(FILL_MODE, DEFAULT.fillMode, fill, _values.fillMode); } + FillMode getFillMode() const { return FillMode(_values.fillMode); } + + void setCullMode(CullMode cull) { SET_FIELD(CULL_MODE, DEFAULT.cullMode, cull, _values.cullMode); } + CullMode getCullMode() const { return CullMode(_values.cullMode); } + + void setFrontFaceClockwise(bool isClockwise) { SET_FIELD(FRONT_FACE_CLOCKWISE, DEFAULT.frontFaceClockwise, isClockwise, _values.frontFaceClockwise); } + bool isFrontFaceClockwise() const { return _values.frontFaceClockwise; } + + void setDepthClampEnable(bool enable) { SET_FIELD(DEPTH_CLAMP_ENABLE, DEFAULT.depthClampEnable, enable, _values.depthClampEnable); } + bool isDepthClampEnable() const { return _values.depthClampEnable; } + + void setScissorEnable(bool enable) { SET_FIELD(SCISSOR_ENABLE, DEFAULT.scissorEnable, enable, _values.scissorEnable); } + bool isScissorEnable() const { return _values.scissorEnable; } + + void setMultisampleEnable(bool enable) { SET_FIELD(MULTISAMPLE_ENABLE, DEFAULT.multisampleEnable, enable, _values.multisampleEnable); } + bool isMultisampleEnable() const { return _values.multisampleEnable; } + + void setAntialiasedLineEnable(bool enable) { SET_FIELD(ANTIALISED_LINE_ENABLE, DEFAULT.antialisedLineEnable, enable, _values.antialisedLineEnable); } + bool isAntialiasedLineEnable() const { return _values.antialisedLineEnable; } + + // Depth Bias + void setDepthBias(float bias) { SET_FIELD(DEPTH_BIAS, DEFAULT.depthBias, bias, _values.depthBias); } + float getDepthBias() const { return _values.depthBias; } + + void setDepthBiasSlopeScale(float scale) { SET_FIELD(DEPTH_BIAS_SLOPE_SCALE, DEFAULT.depthBiasSlopeScale, scale, _values.depthBiasSlopeScale); } + float getDepthBiasSlopeScale() const { return _values.depthBiasSlopeScale; } + + // Depth Test + void setDepthTest(DepthTest depthTest) { SET_FIELD(DEPTH_TEST, DEFAULT.depthTest, depthTest, _values.depthTest); } + void setDepthTest(bool enable, bool writeMask, ComparisonFunction func) { setDepthTest(DepthTest(enable, writeMask, func)); } + DepthTest getDepthTest() const { return _values.depthTest; } + + bool isDepthTestEnabled() const { return getDepthTest().isEnabled(); } + uint8 getDepthTestWriteMask() const { return getDepthTest().getWriteMask(); } + ComparisonFunction getDepthTestFunc() const { return getDepthTest().getFunction(); } + + // Stencil test + void setStencilTest(bool enabled, uint8 frontWriteMask, StencilTest frontTest, uint8 backWriteMask, StencilTest backTest) { + SET_FIELD(STENCIL_ACTIVATION, DEFAULT.stencilActivation, StencilActivation(enabled, frontWriteMask, backWriteMask), _values.stencilActivation); + SET_FIELD(STENCIL_TEST_FRONT, DEFAULT.stencilTestFront, frontTest, _values.stencilTestFront); + SET_FIELD(STENCIL_TEST_BACK, DEFAULT.stencilTestBack, backTest, _values.stencilTestBack); } + void setStencilTest(bool enabled, uint8 frontWriteMask, StencilTest frontTest) { + setStencilTest(enabled, frontWriteMask, frontTest, frontWriteMask, frontTest); } + + StencilActivation getStencilActivation() const { return _values.stencilActivation; } + StencilTest getStencilTestFront() const { return _values.stencilTestFront; } + StencilTest getStencilTestBack() const { return _values.stencilTestBack; } + + bool isStencilEnabled() const { return getStencilActivation().isEnabled(); } + uint8 getStencilWriteMaskFront() const { return getStencilActivation().getWriteMaskFront(); } + uint8 getStencilWriteMaskBack() const { return getStencilActivation().getWriteMaskBack(); } + + // Alpha to coverage + void setAlphaToCoverageEnable(bool enable) { SET_FIELD(ALPHA_TO_COVERAGE_ENABLE, DEFAULT.alphaToCoverageEnable, enable, _values.alphaToCoverageEnable); } + bool isAlphaToCoverageEnabled() const { return _values.alphaToCoverageEnable; } + + // Sample mask + void setSampleMask(uint32 mask) { SET_FIELD(SAMPLE_MASK, DEFAULT.sampleMask, mask, _values.sampleMask); } + uint32 getSampleMask() const { return _values.sampleMask; } + + // Blend Function + void setBlendFunction(BlendFunction function) { SET_FIELD(BLEND_FUNCTION, DEFAULT.blendFunction, function, _values.blendFunction); } + BlendFunction getBlendFunction() const { return _values.blendFunction; } + + void setBlendFunction(bool enabled, BlendArg sourceColor, BlendOp operationColor, BlendArg destinationColor, BlendArg sourceAlpha, BlendOp operationAlpha, BlendArg destinationAlpha) { + setBlendFunction(BlendFunction(enabled, sourceColor, operationColor, destinationColor, sourceAlpha, operationAlpha, destinationAlpha)); } + void setBlendFunction(bool enabled, BlendArg source, BlendOp operation, BlendArg destination) { + setBlendFunction(BlendFunction(enabled, source, operation, destination)); } + + bool isBlendEnabled() const { return getBlendFunction().isEnabled(); } + + // Color write mask + void setColorWriteMask(uint8 mask) { SET_FIELD(COLOR_WRITE_MASK, DEFAULT.colorWriteMask, mask, _values.colorWriteMask); } + void setColorWriteMask(bool red, bool green, bool blue, bool alpha) { uint32 value = ((WRITE_RED * red) | (WRITE_GREEN * green) | (WRITE_BLUE * blue) | (WRITE_ALPHA * alpha)); SET_FIELD(COLOR_WRITE_MASK, DEFAULT.colorWriteMask, value, _values.colorWriteMask); } + uint8 getColorWriteMask() const { return _values.colorWriteMask; } + + // All the possible fields + enum Field { + FILL_MODE, + CULL_MODE, + FRONT_FACE_CLOCKWISE, + DEPTH_CLAMP_ENABLE, + SCISSOR_ENABLE, + MULTISAMPLE_ENABLE, + ANTIALISED_LINE_ENABLE, + + DEPTH_BIAS, + DEPTH_BIAS_SLOPE_SCALE, + + DEPTH_TEST, + + STENCIL_ACTIVATION, + STENCIL_TEST_FRONT, + STENCIL_TEST_BACK, + + SAMPLE_MASK, + ALPHA_TO_COVERAGE_ENABLE, + + BLEND_FUNCTION, + + COLOR_WRITE_MASK, + + NUM_FIELDS, // not a valid field, just the count + }; + + // The signature of the state tells which fields of the state are not default + // this way during rendering the Backend can compare it's current state and try to minimize the job to do + typedef std::bitset Signature; + + Signature getSignature() const { return _signature; } + + static Signature evalSignature(const Data& state); + + // For convenience, create a State from the values directly + State(const Data& values); + const Data& getValues() const { return _values; } + + protected: + State(const State& state); + State& operator=(const State& state); + + Data _values; + Signature _signature{0}; + Stamp _stamp{0}; + + // This shouldn't be used by anything else than the Backend class with the proper casting. + mutable GPUObject* _gpuObject = nullptr; + void setGPUObject(GPUObject* gpuObject) const { _gpuObject = gpuObject; } + GPUObject* getGPUObject() const { return _gpuObject; } + friend class Backend; }; - - // The unique default values for all the fields - static const Data DEFAULT; - void setFillMode(FillMode fill) { SET_FIELD(FILL_MODE, DEFAULT.fillMode, fill, _values.fillMode); } - FillMode getFillMode() const { return FillMode(_values.fillMode); } - - void setCullMode(CullMode cull) { SET_FIELD(CULL_MODE, DEFAULT.cullMode, cull, _values.cullMode); } - CullMode getCullMode() const { return CullMode(_values.cullMode); } - - void setFrontFaceClockwise(bool isClockwise) { SET_FIELD(FRONT_FACE_CLOCKWISE, DEFAULT.frontFaceClockwise, isClockwise, _values.frontFaceClockwise); } - bool isFrontFaceClockwise() const { return _values.frontFaceClockwise; } - - void setDepthClampEnable(bool enable) { SET_FIELD(DEPTH_CLAMP_ENABLE, DEFAULT.depthClampEnable, enable, _values.depthClampEnable); } - bool isDepthClampEnable() const { return _values.depthClampEnable; } - - void setScissorEnable(bool enable) { SET_FIELD(SCISSOR_ENABLE, DEFAULT.scissorEnable, enable, _values.scissorEnable); } - bool isScissorEnable() const { return _values.scissorEnable; } - - void setMultisampleEnable(bool enable) { SET_FIELD(MULTISAMPLE_ENABLE, DEFAULT.multisampleEnable, enable, _values.multisampleEnable); } - bool isMultisampleEnable() const { return _values.multisampleEnable; } - - void setAntialiasedLineEnable(bool enable) { SET_FIELD(ANTIALISED_LINE_ENABLE, DEFAULT.antialisedLineEnable, enable, _values.antialisedLineEnable); } - bool isAntialiasedLineEnable() const { return _values.antialisedLineEnable; } - - // Depth Bias - void setDepthBias(float bias) { SET_FIELD(DEPTH_BIAS, DEFAULT.depthBias, bias, _values.depthBias); } - float getDepthBias() const { return _values.depthBias; } - - void setDepthBiasSlopeScale(float scale) { SET_FIELD(DEPTH_BIAS_SLOPE_SCALE, DEFAULT.depthBiasSlopeScale, scale, _values.depthBiasSlopeScale); } - float getDepthBiasSlopeScale() const { return _values.depthBiasSlopeScale; } - - // Depth Test - void setDepthTest(DepthTest depthTest) { SET_FIELD(DEPTH_TEST, DEFAULT.depthTest, depthTest, _values.depthTest); } - void setDepthTest(bool enable, bool writeMask, ComparisonFunction func) { setDepthTest(DepthTest(enable, writeMask, func)); } - DepthTest getDepthTest() const { return _values.depthTest; } - - bool isDepthTestEnabled() const { return getDepthTest().isEnabled(); } - uint8 getDepthTestWriteMask() const { return getDepthTest().getWriteMask(); } - ComparisonFunction getDepthTestFunc() const { return getDepthTest().getFunction(); } - - // Stencil test - void setStencilTest(bool enabled, uint8 frontWriteMask, StencilTest frontTest, uint8 backWriteMask, StencilTest backTest) { - SET_FIELD(STENCIL_ACTIVATION, DEFAULT.stencilActivation, StencilActivation(enabled, frontWriteMask, backWriteMask), _values.stencilActivation); - SET_FIELD(STENCIL_TEST_FRONT, DEFAULT.stencilTestFront, frontTest, _values.stencilTestFront); - SET_FIELD(STENCIL_TEST_BACK, DEFAULT.stencilTestBack, backTest, _values.stencilTestBack); } - void setStencilTest(bool enabled, uint8 frontWriteMask, StencilTest frontTest) { - setStencilTest(enabled, frontWriteMask, frontTest, frontWriteMask, frontTest); } - - StencilActivation getStencilActivation() const { return _values.stencilActivation; } - StencilTest getStencilTestFront() const { return _values.stencilTestFront; } - StencilTest getStencilTestBack() const { return _values.stencilTestBack; } - - bool isStencilEnabled() const { return getStencilActivation().isEnabled(); } - uint8 getStencilWriteMaskFront() const { return getStencilActivation().getWriteMaskFront(); } - uint8 getStencilWriteMaskBack() const { return getStencilActivation().getWriteMaskBack(); } - - // Alpha to coverage - void setAlphaToCoverageEnable(bool enable) { SET_FIELD(ALPHA_TO_COVERAGE_ENABLE, DEFAULT.alphaToCoverageEnable, enable, _values.alphaToCoverageEnable); } - bool isAlphaToCoverageEnabled() const { return _values.alphaToCoverageEnable; } - - // Sample mask - void setSampleMask(uint32 mask) { SET_FIELD(SAMPLE_MASK, DEFAULT.sampleMask, mask, _values.sampleMask); } - uint32 getSampleMask() const { return _values.sampleMask; } - - // Blend Function - void setBlendFunction(BlendFunction function) { SET_FIELD(BLEND_FUNCTION, DEFAULT.blendFunction, function, _values.blendFunction); } - BlendFunction getBlendFunction() const { return _values.blendFunction; } - - void setBlendFunction(bool enabled, BlendArg sourceColor, BlendOp operationColor, BlendArg destinationColor, BlendArg sourceAlpha, BlendOp operationAlpha, BlendArg destinationAlpha) { - setBlendFunction(BlendFunction(enabled, sourceColor, operationColor, destinationColor, sourceAlpha, operationAlpha, destinationAlpha)); } - void setBlendFunction(bool enabled, BlendArg source, BlendOp operation, BlendArg destination) { - setBlendFunction(BlendFunction(enabled, source, operation, destination)); } - - bool isBlendEnabled() const { return getBlendFunction().isEnabled(); } - - // Color write mask - void setColorWriteMask(uint8 mask) { SET_FIELD(COLOR_WRITE_MASK, DEFAULT.colorWriteMask, mask, _values.colorWriteMask); } - void setColorWriteMask(bool red, bool green, bool blue, bool alpha) { uint32 value = ((WRITE_RED * red) | (WRITE_GREEN * green) | (WRITE_BLUE * blue) | (WRITE_ALPHA * alpha)); SET_FIELD(COLOR_WRITE_MASK, DEFAULT.colorWriteMask, value, _values.colorWriteMask); } - uint8 getColorWriteMask() const { return _values.colorWriteMask; } - - // All the possible fields - enum Field { - FILL_MODE, - CULL_MODE, - FRONT_FACE_CLOCKWISE, - DEPTH_CLAMP_ENABLE, - SCISSOR_ENABLE, - MULTISAMPLE_ENABLE, - ANTIALISED_LINE_ENABLE, - - DEPTH_BIAS, - DEPTH_BIAS_SLOPE_SCALE, - - DEPTH_TEST, - - STENCIL_ACTIVATION, - STENCIL_TEST_FRONT, - STENCIL_TEST_BACK, - - SAMPLE_MASK, - ALPHA_TO_COVERAGE_ENABLE, - - BLEND_FUNCTION, - - COLOR_WRITE_MASK, - - NUM_FIELDS, // not a valid field, just the count - }; - - // The signature of the state tells which fields of the state are not default - // this way during rendering the Backend can compare it's current state and try to minimize the job to do - typedef std::bitset Signature; - - Signature getSignature() const { return _signature; } - - static Signature evalSignature(const Data& state); - - // For convenience, create a State from the values directly - State(const Data& values); - const Data& getValues() const { return _values; } - -protected: - State(const State& state); - State& operator=(const State& state); - - Data _values; - Signature _signature{0}; - Stamp _stamp{0}; - - // This shouldn't be used by anything else than the Backend class with the proper casting. - mutable GPUObject* _gpuObject = nullptr; - void setGPUObject(GPUObject* gpuObject) const { _gpuObject = gpuObject; } - GPUObject* getGPUObject() const { return _gpuObject; } - friend class Backend; -}; - -typedef std::shared_ptr< State > StatePointer; -typedef std::vector< StatePointer > States; - + + typedef std::shared_ptr< State > StatePointer; + typedef std::vector< StatePointer > States; + }; #endif From 23c6e8bd451bb29e902f56765e9c7cea2a86e452 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Aug 2015 12:03:49 -0700 Subject: [PATCH 238/242] fixes for packet version bumps --- libraries/networking/src/udt/PacketHeaders.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index 8c0e27db5e..1c2a38a2a6 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -32,16 +32,14 @@ const QSet NON_SOURCED_PACKETS = QSet() const QSet RELIABLE_PACKETS = QSet(); -PacketVersion versionForPacketType(PacketType::Value packetType) { +PacketVersion versionForPacketType(PacketType packetType) { switch (packetType) { case PacketType::EntityAdd: case PacketType::EntityEdit: case PacketType::EntityData: return VERSION_ENTITIES_PROTOCOL_HEADER_SWAP; - case AvatarData: - return 13; default: - return 12; + return 14; } } From bc7d8d7029cf0b021bb01af4c3bffd41f8d2f0be Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Aug 2015 12:06:45 -0700 Subject: [PATCH 239/242] change UDT note comment --- libraries/networking/src/udt/CongestionControl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/CongestionControl.cpp b/libraries/networking/src/udt/CongestionControl.cpp index 555f48cfd6..9dc1880df6 100644 --- a/libraries/networking/src/udt/CongestionControl.cpp +++ b/libraries/networking/src/udt/CongestionControl.cpp @@ -47,7 +47,7 @@ DefaultCC::DefaultCC() : void DefaultCC::onACK(SequenceNumber ackNum) { double increase = 0; - // Note: 1/24/2012 + // Note from UDT original code: // The minimum increase parameter is increased from "1.0 / _mss" to 0.01 // because the original was too small and caused sending rate to stay at low level // for long time. From a58c6b9320ff6c722e11aeb625ccd6bfeef364ea Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Aug 2015 12:09:28 -0700 Subject: [PATCH 240/242] remove CC close given that it is not called --- libraries/networking/src/udt/CongestionControl.h | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index a8a4ba3342..f1ebe9ada7 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -36,7 +36,6 @@ public: int synInterval() const { return _synInterval; } virtual void init() {} - virtual void close() {} virtual void onACK(SequenceNumber ackNum) {} virtual void onLoss(SequenceNumber rangeStart, SequenceNumber rangeEnd) {} From bd56a5074d6103bc21721b9d3839548036fd6b64 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Aug 2015 12:10:05 -0700 Subject: [PATCH 241/242] inline a curly bracket after template --- libraries/networking/src/udt/CongestionControl.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/CongestionControl.h b/libraries/networking/src/udt/CongestionControl.h index f1ebe9ada7..5208ff9d3b 100644 --- a/libraries/networking/src/udt/CongestionControl.h +++ b/libraries/networking/src/udt/CongestionControl.h @@ -87,8 +87,7 @@ public: virtual std::unique_ptr create() = 0; }; -template class CongestionControlFactory: public CongestionControlVirtualFactory -{ +template class CongestionControlFactory: public CongestionControlVirtualFactory { public: virtual ~CongestionControlFactory() {} virtual std::unique_ptr create() { return std::unique_ptr(new T()); } From e4c1de78455404d82057d8e35372946a357ef496 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Aug 2015 17:15:09 -0700 Subject: [PATCH 242/242] fix variable names for sourced/verified --- libraries/networking/src/NLPacket.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/NLPacket.cpp b/libraries/networking/src/NLPacket.cpp index 2ca794f7b9..39f4f9c541 100644 --- a/libraries/networking/src/NLPacket.cpp +++ b/libraries/networking/src/NLPacket.cpp @@ -12,9 +12,9 @@ #include "NLPacket.h" int NLPacket::localHeaderSize(PacketType type) { - bool sourced = NON_SOURCED_PACKETS.contains(type); - bool verified = NON_VERIFIED_PACKETS.contains(type); - qint64 optionalSize = (sourced ? 0 : NUM_BYTES_RFC4122_UUID) + ((sourced || verified) ? 0 : NUM_BYTES_MD5_HASH); + bool nonSourced = NON_SOURCED_PACKETS.contains(type); + bool nonVerified = NON_VERIFIED_PACKETS.contains(type); + qint64 optionalSize = (nonSourced ? 0 : NUM_BYTES_RFC4122_UUID) + ((nonSourced || nonVerified) ? 0 : NUM_BYTES_MD5_HASH); return sizeof(PacketType) + sizeof(PacketVersion) + optionalSize; } int NLPacket::totalHeaderSize(PacketType type, bool isPartOfMessage) {