added SequenceNumberStats and unit test

libraries/networking/src/SequenceNumbersStats.cpp

@@ -0,0 +1,156 @@
+//
+//  SequenceNumberStats.cpp
+//  libraries/networking/src
+//
+//  Created by Yixin Wang on 6/25/2014
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "SequenceNumbersStats.h"
+
+#include <limits>
+
+SequenceNumberStats::SequenceNumberStats()
+    : _lastReceived(std::numeric_limits<quint16>::max()),
+    _missingSet(),
+    _numReceived(0),
+    _numEarly(0),
+    _numLate(0),
+    _numLost(0),
+    _numRecovered(0),
+    _numDuplicate(0)
+{
+}
+
+void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, const bool wantExtraDebugging) {
+
+    static const int UINT16_RANGE = std::numeric_limits<uint16_t>::max() + 1;
+    static const int MAX_REASONABLE_SEQUENCE_GAP = 1000;  // this must be less than UINT16_RANGE / 2 for rollover handling to work
+
+    // determine our expected sequence number... handle rollover appropriately
+    quint16 expected = _numReceived > 0 ? _lastReceived + (quint16)1 : incoming;
+
+    if (incoming == expected) { // on time
+        _lastReceived = incoming;
+    } else { // out of order
+
+        if (wantExtraDebugging) {
+            qDebug() << "out of order... got:" << incoming << "expected:" << expected;
+        }
+
+        int incomingInt = (int)incoming;
+        int expectedInt = (int)expected;
+
+        // check if the gap between incoming and expected is reasonable, taking possible rollover into consideration
+        int absGap = std::abs(incomingInt - expectedInt);
+        if (absGap >= UINT16_RANGE - MAX_REASONABLE_SEQUENCE_GAP) {
+            // rollover likely occurred between incoming and expected.
+            // correct the larger of the two so that it's within [-UINT16_RANGE, -1] while the other remains within [0, UINT16_RANGE-1]
+            if (incomingInt > expectedInt) {
+                incomingInt -= UINT16_RANGE;
+            } else {
+                expectedInt -= UINT16_RANGE;
+            }
+        } else if (absGap > MAX_REASONABLE_SEQUENCE_GAP) {
+            // ignore packet if gap is unreasonable
+            qDebug() << "ignoring unreasonable packet... sequence:" << incoming
+                << "previous:" << _lastReceived;
+            return;
+        }
+
+        // now that rollover has been corrected for (if it occurred), incoming and expected can be
+        // compared to each other directly, though one of them might be negative
+        if (incomingInt > expectedInt) { // early
+            if (wantExtraDebugging) {
+                qDebug() << "this packet is earlier than expected...";
+                qDebug() << ">>>>>>>> missing gap=" << (incomingInt - expectedInt);
+            }
+
+            _numEarly++;
+            _numLost += (incomingInt - expectedInt);
+
+            // add all sequence numbers that were skipped to the missing sequence numbers list
+            for (int missingInt = expectedInt; missingInt < incomingInt; missingInt++) {
+                _missingSet.insert((quint16)(missingInt < 0 ? missingInt + UINT16_RANGE : missingInt));
+            }
+            _lastReceived = incoming;
+        } else { // late
+            if (wantExtraDebugging) {
+                qDebug() << "this packet is later than expected...";
+            }
+            _numLate++;
+
+            // remove this from missing sequence number if it's in there
+            if (_missingSet.remove(incoming)) {
+                if (wantExtraDebugging) {
+                    qDebug() << "found it in _missingSet";
+                }
+                _numLost--;
+                _numRecovered++;
+            } else {
+                if (wantExtraDebugging) {
+                    qDebug() << "sequence:" << incoming << "was NOT found in _missingSet and is probably a duplicate";
+                }
+                _numDuplicate++;
+            }
+
+            // do not update _incomingLastSequence; it shouldn't become smaller
+        }
+    }
+    _numReceived++;
+
+    // prune missing sequence list if it gets too big; sequence numbers that are older than MAX_REASONABLE_SEQUENCE_GAP
+    // will be removed.
+    if (_missingSet.size() > MAX_REASONABLE_SEQUENCE_GAP) {
+        if (wantExtraDebugging) {
+            qDebug() << "_missingSet is too large! size:" << _missingSet.size();
+        }
+
+        // some older sequence numbers may be from before a rollover point; this must be handled.
+        // some sequence numbers in this list may be larger than _incomingLastSequence, indicating that they were received
+        // before the most recent rollover.
+        int cutoff = (int)_lastReceived - MAX_REASONABLE_SEQUENCE_GAP;
+        if (cutoff >= 0) {
+            quint16 nonRolloverCutoff = (quint16)cutoff;
+            QSet<quint16>::iterator i = _missingSet.begin();
+            while (i != _missingSet.end()) {
+                quint16 missing = *i;
+                if (wantExtraDebugging) {
+                    qDebug() << "checking item:" << missing << "is it in need of pruning?";
+                    qDebug() << "old age cutoff:" << nonRolloverCutoff;
+                }
+
+                if (missing > _lastReceived || missing < nonRolloverCutoff) {
+                    i = _missingSet.erase(i);
+                    if (wantExtraDebugging) {
+                        qDebug() << "pruning really old missing sequence:" << missing;
+                    }
+                } else {
+                    i++;
+                }
+            }
+        } else {
+            quint16 rolloverCutoff = (quint16)(cutoff + UINT16_RANGE);
+            QSet<quint16>::iterator i = _missingSet.begin();
+            while (i != _missingSet.end()) {
+                quint16 missing = *i;
+                if (wantExtraDebugging) {
+                    qDebug() << "checking item:" << missing << "is it in need of pruning?";
+                    qDebug() << "old age cutoff:" << rolloverCutoff;
+                }
+
+                if (missing > _lastReceived && missing < rolloverCutoff) {
+                    i = _missingSet.erase(i);
+                    if (wantExtraDebugging) {
+                        qDebug() << "pruning really old missing sequence:" << missing;
+                    }
+                } else {
+                    i++;
+                }
+            }
+        }
+    }
+}

libraries/networking/src/SequenceNumbersStats.h

@@ -0,0 +1,43 @@
+//
+//  SequenceNumberStats.h
+//  libraries/networking/src
+//
+//  Created by Yixin Wang on 6/25/2014
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_SequenceNumberStats_h
+#define hifi_SequenceNumberStats_h
+
+#include "SharedUtil.h"
+
+class SequenceNumberStats {
+public:
+    SequenceNumberStats();
+
+    void sequenceNumberReceived(quint16 incoming, const bool wantExtraDebugging = false);
+
+    quint32 getNumReceived() const { return _numReceived; }
+    quint32 getNumEarly() const { return _numEarly; }
+    quint32 getNumLate() const { return _numLate; }
+    quint32 getNumLost() const { return _numLost; }
+    quint32 getNumRecovered() const { return _numRecovered; }
+    quint32 getNumDuplicate() const { return _numDuplicate; }
+    const QSet<quint16>& getMissingSet() const { return _missingSet; }
+
+private:
+    quint16 _lastReceived;
+    QSet<quint16> _missingSet;
+
+    quint32 _numReceived;
+    quint32 _numEarly;
+    quint32 _numLate;
+    quint32 _numLost;
+    quint32 _numRecovered;
+    quint32 _numDuplicate;
+};
+
+#endif // hifi_SequenceNumberStats_h

tests/networking/CMakeLists.txt

@@ -0,0 +1,39 @@
+cmake_minimum_required(VERSION 2.8)
+
+if (WIN32)
+  cmake_policy (SET CMP0020 NEW)
+endif (WIN32)
+
+set(TARGET_NAME networking-tests)
+
+set(ROOT_DIR ../..)
+set(MACRO_DIR ${ROOT_DIR}/cmake/macros)
+
+# setup for find modules
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/modules/")
+
+#find_package(Qt5Network REQUIRED)
+#find_package(Qt5Script REQUIRED)
+#find_package(Qt5Widgets REQUIRED)
+
+include(${MACRO_DIR}/SetupHifiProject.cmake)
+setup_hifi_project(${TARGET_NAME} TRUE)
+
+include(${MACRO_DIR}/AutoMTC.cmake)
+auto_mtc(${TARGET_NAME} ${ROOT_DIR})
+
+#qt5_use_modules(${TARGET_NAME} Network Script Widgets)
+
+#include glm
+include(${MACRO_DIR}/IncludeGLM.cmake)
+include_glm(${TARGET_NAME} ${ROOT_DIR})
+
+# link in the shared libraries
+include(${MACRO_DIR}/LinkHifiLibrary.cmake)
+link_hifi_library(shared ${TARGET_NAME} ${ROOT_DIR})
+link_hifi_library(networking ${TARGET_NAME} ${ROOT_DIR})
+
+IF (WIN32)
+    target_link_libraries(${TARGET_NAME} Winmm Ws2_32)
+ENDIF(WIN32)
+

tests/networking/src/SequenceNumberStatsTests.cpp

@@ -0,0 +1,258 @@
+//
+//  AudioRingBufferTests.cpp
+//  tests/networking/src
+//
+//  Created by Yixin Wang on 6/24/2014
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "SequenceNumberStatsTests.h"
+
+#include "SharedUtil.h"
+#include <limits>
+
+
+void SequenceNumberStatsTests::runAllTests() {
+
+    rolloverTest();
+    earlyLateTest();
+    duplicateTest();
+    pruneTest();
+}
+
+const int UINT16_RANGE = std::numeric_limits<quint16>::max() + 1;
+
+
+void SequenceNumberStatsTests::rolloverTest() {
+
+    SequenceNumberStats stats;
+
+    // insert enough samples to cause 3 rollovers
+    quint16 seq = 79;   // start on some random number
+    for (int i = 0; i < 3 * UINT16_RANGE; i++) {
+        stats.sequenceNumberReceived(seq);
+        seq = seq + (quint16)1;
+
+        assert(stats.getNumDuplicate() == 0);
+        assert(stats.getNumEarly() == 0);
+        assert(stats.getNumLate() == 0);
+        assert(stats.getNumLost() == 0);
+        assert(stats.getNumReceived() == i+1);
+        assert(stats.getNumRecovered() == 0);
+    }
+}
+
+void SequenceNumberStatsTests::earlyLateTest() {
+
+    SequenceNumberStats stats;
+    quint16 seq = 65530;
+    int numSent = 0;
+
+    int numEarly = 0;
+    int numLate = 0;
+    int numLost = 0;
+    int numRecovered = 0;
+
+
+    for (int T = 0; T < 10000; T++) {
+
+        // insert 7 consecutive
+        for (int i = 0; i < 7; i++) {
+            stats.sequenceNumberReceived(seq);
+            seq = seq + (quint16)1;
+            numSent++;
+
+            assert(stats.getNumDuplicate() == 0);
+            assert(stats.getNumEarly() == numEarly);
+            assert(stats.getNumLate() == numLate);
+            assert(stats.getNumLost() == numLost);
+            assert(stats.getNumReceived() == numSent);
+            assert(stats.getNumRecovered() == numRecovered);
+        }
+
+        // skip 10
+        quint16 skipped = seq;
+        seq = seq + (quint16)10;
+
+        // insert 36 consecutive
+        numEarly++;
+        numLost += 10;
+        for (int i = 0; i < 36; i++) {
+            stats.sequenceNumberReceived(seq);
+            seq = seq + (quint16)1;
+            numSent++;
+
+            assert(stats.getNumDuplicate() == 0);
+            assert(stats.getNumEarly() == numEarly);
+            assert(stats.getNumLate() == numLate);
+            assert(stats.getNumLost() == numLost);
+            assert(stats.getNumReceived() == numSent);
+            assert(stats.getNumRecovered() == numRecovered);
+        }
+
+        // send ones we skipped
+        for (int i = 0; i < 10; i++) {
+            stats.sequenceNumberReceived(skipped);
+            skipped = skipped + (quint16)1;
+            numSent++;
+            numLate++;
+            numLost--;
+            numRecovered++;
+
+            assert(stats.getNumDuplicate() == 0);
+            assert(stats.getNumEarly() == numEarly);
+            assert(stats.getNumLate() == numLate);
+            assert(stats.getNumLost() == numLost);
+            assert(stats.getNumReceived() == numSent);
+            assert(stats.getNumRecovered() == numRecovered);
+        }
+    }
+}
+
+void SequenceNumberStatsTests::duplicateTest() {
+
+    SequenceNumberStats stats;
+    quint16 seq = 12345;
+    int numSent = 0;
+
+    int numDuplicate = 0;
+    int numEarly = 0;
+    int numLate = 0;
+    int numLost = 0;
+
+
+    for (int T = 0; T < 10000; T++) {
+
+        quint16 duplicate = seq;
+
+        // insert 7 consecutive
+        for (int i = 0; i < 7; i++) {
+            stats.sequenceNumberReceived(seq);
+            seq = seq + (quint16)1;
+            numSent++;
+
+            assert(stats.getNumDuplicate() == numDuplicate);
+            assert(stats.getNumEarly() == numEarly);
+            assert(stats.getNumLate() == numLate);
+            assert(stats.getNumLost() == numLost);
+            assert(stats.getNumReceived() == numSent);
+            assert(stats.getNumRecovered() == 0);
+        }
+
+        // skip 10
+        quint16 skipped = seq;
+        seq = seq + (quint16)10;
+
+
+        quint16 duplicate2 = seq;
+
+        numEarly++;
+        numLost += 10;
+        // insert 36 consecutive
+        for (int i = 0; i < 36; i++) {
+            stats.sequenceNumberReceived(seq);
+            seq = seq + (quint16)1;
+            numSent++;
+
+            assert(stats.getNumDuplicate() == numDuplicate);
+            assert(stats.getNumEarly() == numEarly);
+            assert(stats.getNumLate() == numLate);
+            assert(stats.getNumLost() == numLost);
+            assert(stats.getNumReceived() == numSent);
+            assert(stats.getNumRecovered() == 0);
+        }
+
+        // send 5 duplicates from before skip
+        for (int i = 0; i < 5; i++) {
+            stats.sequenceNumberReceived(duplicate);
+            duplicate = duplicate + (quint16)1;
+            numSent++;
+            numDuplicate++;
+            numLate++;
+
+            assert(stats.getNumDuplicate() == numDuplicate);
+            assert(stats.getNumEarly() == numEarly);
+            assert(stats.getNumLate() == numLate);
+            assert(stats.getNumLost() == numLost);
+            assert(stats.getNumReceived() == numSent);
+            assert(stats.getNumRecovered() == 0);
+        }
+
+        // send 5 duplicates from after skip
+        for (int i = 0; i < 5; i++) {
+            stats.sequenceNumberReceived(duplicate2);
+            duplicate2 = duplicate2 + (quint16)1;
+            numSent++;
+            numDuplicate++;
+            numLate++;
+
+            assert(stats.getNumDuplicate() == numDuplicate);
+            assert(stats.getNumEarly() == numEarly);
+            assert(stats.getNumLate() == numLate);
+            assert(stats.getNumLost() == numLost);
+            assert(stats.getNumReceived() == numSent);
+            assert(stats.getNumRecovered() == 0);
+        }
+    }
+}
+
+void SequenceNumberStatsTests::pruneTest() {
+    
+    SequenceNumberStats stats;
+    quint16 seq = 54321;
+    int numSent = 0;
+
+    int numEarly = 0;
+    int numLate = 0;
+    int numLost = 0;
+
+    for (int T = 0; T < 1000; T++) {
+        // insert 1 seq
+        stats.sequenceNumberReceived(seq);
+        seq = seq + (quint16)1;
+        numSent++;
+
+        // skip 1000 seq
+        seq = seq + (quint16)1000;
+        quint16 highestSkipped = seq - (quint16)1;
+
+        // insert 1 seq
+        stats.sequenceNumberReceived(seq);
+        seq = seq + (quint16)1;
+        numSent++;
+        numEarly++;
+        numLost += 1000;
+
+        // skip 10 seq
+        seq = seq + (quint16)10;
+        quint16 highestSkipped2 = seq - (quint16)1;
+
+        // insert 1 seq
+        // insert 1 seq
+        stats.sequenceNumberReceived(seq);
+        seq = seq + (quint16)1;
+        numSent++;
+        numEarly++;
+        numLost += 10;
+
+        const QSet<quint16>& missingSet = stats.getMissingSet();
+        assert(missingSet.size() <= 1000);
+
+        for (int i = 0; i < 10; i++) {
+            assert(missingSet.contains(highestSkipped2));
+            highestSkipped2 = highestSkipped2 - (quint16)1;
+        }
+
+        for (int i = 0; i < 989; i++) {
+            assert(missingSet.contains(highestSkipped));
+            highestSkipped = highestSkipped - (quint16)1;
+        }
+        for (int i = 0; i < 11; i++) {
+            assert(!missingSet.contains(highestSkipped));
+            highestSkipped = highestSkipped - (quint16)1;
+        }
+    }
+}

tests/networking/src/SequenceNumberStatsTests.h

@@ -0,0 +1,28 @@
+//
+//  AudioRingBufferTests.h
+//  tests/networking/src
+//
+//  Created by Yixin Wang on 6/24/2014
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_SequenceNumberStatsTests_h
+#define hifi_SequenceNumberStatsTests_h
+
+#include "SequenceNumberStatsTests.h"
+#include "SequenceNumbersStats.h"
+
+namespace SequenceNumberStatsTests {
+
+    void runAllTests();
+
+    void rolloverTest();
+    void earlyLateTest();
+    void duplicateTest();
+    void pruneTest();
+};
+
+#endif // hifi_SequenceNumberStatsTests_h

tests/networking/src/main.cpp

@@ -0,0 +1,19 @@
+//
+//  main.cpp
+//  tests/networking/src
+//
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "SequenceNumberStatsTests.h"
+#include <stdio.h>
+
+int main(int argc, char** argv) {
+    SequenceNumberStatsTests::runAllTests();
+    printf("tests passed! press enter to exit");
+    getchar();
+    return 0;
+}