Mirrors/overte-thingvellir
3
0
Fork 0
mirror of https://thingvellir.net/git/overte synced 2025-03-27 23:52:03 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #3246 from wangyix/inbound_audio_stream

Browse source 
Improved SequenceNumberStats handling of out-of-sync seq nums
This commit is contained in:
Philip Rosedale 2014-08-04 14:30:45 -07:00
commit d552b7ca97
6 changed files with 94 additions and 168 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
examples/bot_procedural.js. → examples/bot_procedural.js
View file

4
interface/src/ui/OctreeStatsDialog.cpp
View file

@ -372,7 +372,6 @@ void OctreeStatsDialog::showOctreeServersOfType(int& serverCount, NodeType_t ser
QString incomingEarlyString = locale.toString((uint)seqStats.getEarly());
QString incomingLikelyLostString = locale.toString((uint)seqStats.getLost());
QString incomingRecovered = locale.toString((uint)seqStats.getRecovered());
QString incomingDuplicateString = locale.toString((uint)seqStats.getDuplicate());
int clockSkewInMS = node->getClockSkewUsec() / (int)USECS_PER_MSEC;
QString incomingFlightTimeString = locale.toString((int)stats.getIncomingFlightTimeAverage());
@ -386,8 +385,7 @@ void OctreeStatsDialog::showOctreeServersOfType(int& serverCount, NodeType_t ser
serverDetails << "<br/>" << " Out of Order: " << qPrintable(incomingOutOfOrderString) <<
"/ Early: " << qPrintable(incomingEarlyString) <<
"/ Late: " << qPrintable(incomingLateString) <<
"/ Unreasonable: " << qPrintable(incomingUnreasonableString) <<
"/ Duplicate: " << qPrintable(incomingDuplicateString);
"/ Unreasonable: " << qPrintable(incomingUnreasonableString);
serverDetails << "<br/>" <<
" Average Flight Time: " << qPrintable(incomingFlightTimeString) << " msecs";

189
libraries/networking/src/SequenceNumberStats.cpp
View file

@ -14,70 +14,23 @@
#include <limits>
SequenceNumberStats::SequenceNumberStats(int statsHistoryLength, bool canDetectOutOfSync)
: _received(0),
_lastReceivedSequence(0),
: _lastReceivedSequence(0),
_missingSet(),
_stats(),
_lastSenderUUID(),
_statsHistory(statsHistoryLength),
_canHaveChild(canDetectOutOfSync),
_childInstance(NULL),
_consecutiveReasonable(0)
_lastUnreasonableSequence(0),
_consecutiveUnreasonableOnTime(0)
{
}
SequenceNumberStats::SequenceNumberStats(const SequenceNumberStats& other)
: _received(other._received),
_lastReceivedSequence(other._lastReceivedSequence),
_missingSet(other._missingSet),
_stats(other._stats),
_lastSenderUUID(other._lastSenderUUID),
_statsHistory(other._statsHistory),
_childInstance(NULL),
_canHaveChild(other._canHaveChild),
_consecutiveReasonable(other._consecutiveReasonable)
{
if (other._childInstance) {
_childInstance = new SequenceNumberStats(*other._childInstance);
}
}
SequenceNumberStats& SequenceNumberStats::operator=(const SequenceNumberStats& rhs) {
_received = rhs._received;
_lastReceivedSequence = rhs._lastReceivedSequence;
_missingSet = rhs._missingSet;
_stats = rhs._stats;
_lastSenderUUID = rhs._lastSenderUUID;
_statsHistory = rhs._statsHistory;
_canHaveChild = rhs._canHaveChild;
_consecutiveReasonable = rhs._consecutiveReasonable;
if (rhs._childInstance) {
_childInstance = new SequenceNumberStats(*rhs._childInstance);
} else {
_childInstance = NULL;
}
return *this;
}
SequenceNumberStats::~SequenceNumberStats() {
if (_childInstance) {
delete _childInstance;
}
}
void SequenceNumberStats::reset() {
_received = 0;
_missingSet.clear();
_stats = PacketStreamStats();
_lastSenderUUID = QUuid();
_statsHistory.clear();
if (_childInstance) {
delete _childInstance;
_childInstance = NULL;
}
_consecutiveReasonable = 0;
_lastUnreasonableSequence = 0;
_consecutiveUnreasonableOnTime = 0;
}
static const int UINT16_RANGE = std::numeric_limits<uint16_t>::max() + 1;
@ -88,7 +41,7 @@ SequenceNumberStats::ArrivalInfo SequenceNumberStats::sequenceNumberReceived(qui
// if the sender node has changed, reset all stats
if (senderUUID != _lastSenderUUID) {
if (_received > 0) {
if (_stats._received > 0) {
qDebug() << "sequence number stats was reset due to new sender node";
qDebug() << "previous:" << _lastSenderUUID << "current:" << senderUUID;
reset();
@ -97,9 +50,9 @@ SequenceNumberStats::ArrivalInfo SequenceNumberStats::sequenceNumberReceived(qui
}
// determine our expected sequence number... handle rollover appropriately
quint16 expected = _received > 0 ? _lastReceivedSequence + (quint16)1 : incoming;
quint16 expected = _stats._received > 0 ? _lastReceivedSequence + (quint16)1 : incoming;
_received++;
_stats._received++;
if (incoming == expected) { // on time
arrivalInfo._status = OnTime;
@ -130,72 +83,11 @@ SequenceNumberStats::ArrivalInfo SequenceNumberStats::sequenceNumberReceived(qui
qDebug() << "unreasonable sequence number:" << incoming << "previous:" << _lastReceivedSequence;
_stats._unreasonable++;
_consecutiveReasonable = 0;
// if _canHaveChild, create a child instance of SequenceNumberStats to track this unreasonable seq num and ones in the future.
// if the child instance detects a valid stream of seq nums up to some length, the seq nums sender probably
// fell out of sync with us.
if (_canHaveChild) {
if (!_childInstance) {
_childInstance = new SequenceNumberStats(0, false);
}
ArrivalInfo unreasonableTrackerArrivalInfo = _childInstance->sequenceNumberReceived(incoming);
// the child instance will be used to detect some threshold number seq nums in a row that are perfectly
// in order.
const int UNREASONABLE_TRACKER_RECEIVED_THRESHOLD = 8;
if (unreasonableTrackerArrivalInfo._status != OnTime) {
_childInstance->reset();
} else if (_childInstance->getReceived() >= UNREASONABLE_TRACKER_RECEIVED_THRESHOLD) {
// the child instance has detected a threshold number of consecutive seq nums.
// copy its state to this instance.
_received = _childInstance->_received;
_lastReceivedSequence = _childInstance->_lastReceivedSequence;
_missingSet = _childInstance->_missingSet;
_stats = _childInstance->_stats;
// don't copy _lastSenderUUID; _unreasonableTracker always has null UUID for that member.
// ours should be up-to-date.
// don't copy _statsHistory; _unreasonableTracker keeps a history of length 0.
// simply clear ours.
_statsHistory.clear();
arrivalInfo = unreasonableTrackerArrivalInfo;
// delete child instance;
delete _childInstance;
_childInstance = NULL;
}
}
receivedUnreasonable(incoming);
return arrivalInfo;
}
_consecutiveReasonable++;
// if we got a reasonable seq num but have a child instance tracking unreasonable seq nums,
// reset it. if many consecutive reasonable seq nums have occurred (implying the unreasonable seq num
// that caused the creation of the child instance was probably a fluke), delete our child instance.
if (_childInstance) {
const int CONSECUTIVE_REASONABLE_CHILD_DELETE_THRESHOLD = 8;
if (_consecutiveReasonable >= CONSECUTIVE_REASONABLE_CHILD_DELETE_THRESHOLD) {
_childInstance->reset();
} else {
delete _childInstance;
_childInstance = NULL;
}
}
// now that rollover has been corrected for (if it occurred), incomingInt and expectedInt can be
// compared to each other directly, though one of them might be negative
@ -219,7 +111,7 @@ SequenceNumberStats::ArrivalInfo SequenceNumberStats::sequenceNumberReceived(qui
for (int missingInt = expectedInt; missingInt < incomingInt; missingInt++) {
_missingSet.insert((quint16)(missingInt < 0 ? missingInt + UINT16_RANGE : missingInt));
}
// prune missing sequence list if it gets too big; sequence numbers that are older than MAX_REASONABLE_SEQUENCE_GAP
// will be removed.
if (_missingSet.size() > MAX_REASONABLE_SEQUENCE_GAP) {
@ -236,7 +128,7 @@ SequenceNumberStats::ArrivalInfo SequenceNumberStats::sequenceNumberReceived(qui
// remove this from missing sequence number if it's in there
if (_missingSet.remove(incoming)) {
arrivalInfo._status = Late;
arrivalInfo._status = Recovered;
if (wantExtraDebugging) {
qDebug() << "found it in _missingSet";
@ -244,19 +136,62 @@ SequenceNumberStats::ArrivalInfo SequenceNumberStats::sequenceNumberReceived(qui
_stats._lost--;
_stats._recovered++;
} else {
arrivalInfo._status = Duplicate;
// this late seq num is not in our missing set. it is possibly a duplicate, or possibly a late
// packet that should have arrived before our first received packet. we'll count these
// as unreasonable.
if (wantExtraDebugging) {
qDebug() << "sequence:" << incoming << "was NOT found in _missingSet and is probably a duplicate";
}
_stats._duplicate++;
arrivalInfo._status = Unreasonable;
qDebug() << "unreasonable sequence number:" << incoming << "(possible duplicate)";
_stats._unreasonable++;
receivedUnreasonable(incoming);
return arrivalInfo;
}
}
}
// if we've made it here, we received a reasonable seq number.
_consecutiveUnreasonableOnTime = 0;
return arrivalInfo;
}
void SequenceNumberStats::receivedUnreasonable(quint16 incoming) {
const int CONSECUTIVE_UNREASONABLE_ON_TIME_THRESHOLD = 8;
quint16 expected = _consecutiveUnreasonableOnTime > 0 ? _lastUnreasonableSequence + (quint16)1 : incoming;
if (incoming == expected) {
_consecutiveUnreasonableOnTime++;
_lastUnreasonableSequence = incoming;
if (_consecutiveUnreasonableOnTime >= CONSECUTIVE_UNREASONABLE_ON_TIME_THRESHOLD) {
// we've received many unreasonable seq numbers in a row, all in order. we're probably out of sync with
// the seq num sender. update our state to get back in sync with the sender.
_lastReceivedSequence = incoming;
_missingSet.clear();
_stats._received = CONSECUTIVE_UNREASONABLE_ON_TIME_THRESHOLD;
_stats._unreasonable = 0;
_stats._early = 0;
_stats._late = 0;
_stats._lost = 0;
_stats._recovered = 0;
_stats._expectedReceived = CONSECUTIVE_UNREASONABLE_ON_TIME_THRESHOLD;
_statsHistory.clear();
_consecutiveUnreasonableOnTime = 0;
qDebug() << "re-synced with sequence number sender";
}
} else {
_consecutiveUnreasonableOnTime = 0;
}
}
void SequenceNumberStats::pruneMissingSet(const bool wantExtraDebugging) {
if (wantExtraDebugging) {
qDebug() << "pruning _missingSet! size:" << _missingSet.size();
@ -311,7 +246,7 @@ PacketStreamStats SequenceNumberStats::getStatsForHistoryWindow() const {
const PacketStreamStats* newestStats = _statsHistory.getNewestEntry();
const PacketStreamStats* oldestStats = _statsHistory.get(_statsHistory.getNumEntries() - 1);
// this catches cases where history is length 1 or 0 (both are NULL in case of 0)
if (newestStats == oldestStats) {
return PacketStreamStats();
@ -319,13 +254,13 @@ PacketStreamStats SequenceNumberStats::getStatsForHistoryWindow() const {
// calculate difference between newest stats and oldest stats to get window stats
PacketStreamStats windowStats;
windowStats._expectedReceived = newestStats->_expectedReceived - oldestStats->_expectedReceived;
windowStats._received = newestStats->_received - oldestStats->_received;
windowStats._unreasonable = newestStats->_unreasonable - oldestStats->_unreasonable;
windowStats._early = newestStats->_early - oldestStats->_early;
windowStats._late = newestStats->_late - oldestStats->_late;
windowStats._lost = newestStats->_lost - oldestStats->_lost;
windowStats._recovered = newestStats->_recovered - oldestStats->_recovered;
windowStats._duplicate = newestStats->_duplicate - oldestStats->_duplicate;
windowStats._expectedReceived = newestStats->_expectedReceived - oldestStats->_expectedReceived;
return windowStats;
}

34
libraries/networking/src/SequenceNumberStats.h
View file

@ -22,27 +22,24 @@ const int MAX_REASONABLE_SEQUENCE_GAP = 1000;
class PacketStreamStats {
public:
PacketStreamStats()
: _expectedReceived(0),
: _received(0),
_unreasonable(0),
_early(0),
_late(0),
_lost(0),
_recovered(0),
_duplicate(0)
_expectedReceived(0)
{}
float getOutOfOrderRate() const { return (float)(_early + _late) / _expectedReceived; }
float getEaryRate() const { return (float)_early / _expectedReceived; }
float getLateRate() const { return (float)_late / _expectedReceived; }
float getLostRate() const { return (float)_lost / _expectedReceived; }
quint32 _expectedReceived;
quint32 _received;
quint32 _unreasonable;
quint32 _early;
quint32 _late;
quint32 _lost;
quint32 _recovered;
quint32 _duplicate;
quint32 _expectedReceived;
};
class SequenceNumberStats {
@ -51,8 +48,7 @@ public:
OnTime,
Unreasonable,
Early,
Late, // recovered
Duplicate
Recovered,
};
class ArrivalInfo {
@ -63,18 +59,13 @@ public:
SequenceNumberStats(int statsHistoryLength = 0, bool canDetectOutOfSync = true);
SequenceNumberStats(const SequenceNumberStats& other);
SequenceNumberStats& operator=(const SequenceNumberStats& rhs);
~SequenceNumberStats();
void reset();
ArrivalInfo sequenceNumberReceived(quint16 incoming, QUuid senderUUID = QUuid(), const bool wantExtraDebugging = false);
void pruneMissingSet(const bool wantExtraDebugging = false);
void pushStatsToHistory() { _statsHistory.insert(_stats); }
quint32 getReceived() const { return _received; }
float getUnreasonableRate() const { return _stats._unreasonable / _received; }
quint32 getReceived() const { return _stats._received; }
quint32 getExpectedReceived() const { return _stats._expectedReceived; }
quint32 getUnreasonable() const { return _stats._unreasonable; }
quint32 getOutOfOrder() const { return _stats._early + _stats._late; }
@ -82,15 +73,15 @@ public:
quint32 getLate() const { return _stats._late; }
quint32 getLost() const { return _stats._lost; }
quint32 getRecovered() const { return _stats._recovered; }
quint32 getDuplicate() const { return _stats._duplicate; }
const PacketStreamStats& getStats() const { return _stats; }
PacketStreamStats getStatsForHistoryWindow() const;
const QSet<quint16>& getMissingSet() const { return _missingSet; }
private:
int _received;
void receivedUnreasonable(quint16 incoming);
private:
quint16 _lastReceivedSequence;
QSet<quint16> _missingSet;
@ -100,13 +91,8 @@ private:
RingBufferHistory<PacketStreamStats> _statsHistory;
// to deal with the incoming seq nums going out of sync with this tracker, we'll create another instance
// of this class when we encounter an unreasonable
bool _canHaveChild;
SequenceNumberStats* _childInstance;
int _consecutiveReasonable;
quint16 _lastUnreasonableSequence;
int _consecutiveUnreasonableOnTime;
};
#endif // hifi_SequenceNumberStats_h

33
tests/networking/src/SequenceNumberStatsTests.cpp
View file

@ -20,7 +20,7 @@ void SequenceNumberStatsTests::runAllTests() {
earlyLateTest();
duplicateTest();
pruneTest();
recursiveTest();
resyncTest();
}
const quint32 UINT16_RANGE = std::numeric_limits<quint16>::max() + 1;
@ -38,7 +38,6 @@ void SequenceNumberStatsTests::rolloverTest() {
stats.sequenceNumberReceived(seq);
seq = seq + (quint16)1;
assert(stats.getDuplicate() == 0);
assert(stats.getEarly() == 0);
assert(stats.getLate() == 0);
assert(stats.getLost() == 0);
@ -69,7 +68,6 @@ void SequenceNumberStatsTests::earlyLateTest() {
seq = seq + (quint16)1;
numSent++;
assert(stats.getDuplicate() == 0);
assert(stats.getEarly() == numEarly);
assert(stats.getLate() == numLate);
assert(stats.getLost() == numLost);
@ -89,7 +87,6 @@ void SequenceNumberStatsTests::earlyLateTest() {
seq = seq + (quint16)1;
numSent++;
assert(stats.getDuplicate() == 0);
assert(stats.getEarly() == numEarly);
assert(stats.getLate() == numLate);
assert(stats.getLost() == numLost);
@ -106,7 +103,6 @@ void SequenceNumberStatsTests::earlyLateTest() {
numLost--;
numRecovered++;
assert(stats.getDuplicate() == 0);
assert(stats.getEarly() == numEarly);
assert(stats.getLate() == numLate);
assert(stats.getLost() == numLost);
@ -135,7 +131,7 @@ void SequenceNumberStatsTests::duplicateTest() {
quint32 numLost = 0;
for (int R = 0; R < 2; R++) {
for (int T = 0; T < 10000; T++) {
for (int T = 0; T < 5; T++) {
quint16 duplicate = seq;
@ -145,7 +141,7 @@ void SequenceNumberStatsTests::duplicateTest() {
seq = seq + (quint16)1;
numSent++;
assert(stats.getDuplicate() == numDuplicate);
assert(stats.getUnreasonable() == numDuplicate);
assert(stats.getEarly() == numEarly);
assert(stats.getLate() == numLate);
assert(stats.getLost() == numLost);
@ -167,7 +163,7 @@ void SequenceNumberStatsTests::duplicateTest() {
seq = seq + (quint16)1;
numSent++;
assert(stats.getDuplicate() == numDuplicate);
assert(stats.getUnreasonable() == numDuplicate);
assert(stats.getEarly() == numEarly);
assert(stats.getLate() == numLate);
assert(stats.getLost() == numLost);
@ -183,7 +179,7 @@ void SequenceNumberStatsTests::duplicateTest() {
numDuplicate++;
numLate++;
assert(stats.getDuplicate() == numDuplicate);
assert(stats.getUnreasonable() == numDuplicate);
assert(stats.getEarly() == numEarly);
assert(stats.getLate() == numLate);
assert(stats.getLost() == numLost);
@ -199,7 +195,7 @@ void SequenceNumberStatsTests::duplicateTest() {
numDuplicate++;
numLate++;
assert(stats.getDuplicate() == numDuplicate);
assert(stats.getUnreasonable() == numDuplicate);
assert(stats.getEarly() == numEarly);
assert(stats.getLate() == numLate);
assert(stats.getLost() == numLost);
@ -279,7 +275,7 @@ void SequenceNumberStatsTests::pruneTest() {
}
}
void SequenceNumberStatsTests::recursiveTest() {
void SequenceNumberStatsTests::resyncTest() {
SequenceNumberStats stats(0);
@ -302,9 +298,20 @@ void SequenceNumberStatsTests::recursiveTest() {
sequence = 0;
for (int R = 0; R < 7; R++) {
stats.sequenceNumberReceived(sequence);
sequence += (quint16)2000;
sequence += (quint16)1;
}
assert(stats.getUnreasonable() == 7);
sequence = 6000;
for (int R = 0; R < 7; R++) {
stats.sequenceNumberReceived(sequence);
sequence += (quint16)1;
}
assert(stats.getUnreasonable() == 14);
sequence = 9000;
for (int i = 0; i < 10; i++) {
stats.sequenceNumberReceived(sequence);
sequence += (quint16)1;

2
tests/networking/src/SequenceNumberStatsTests.h
View file

@ -23,7 +23,7 @@ namespace SequenceNumberStatsTests {
void earlyLateTest();
void duplicateTest();
void pruneTest();
void recursiveTest();
void resyncTest();
};
#endif // hifi_SequenceNumberStatsTests_h