Tests, fixes for SpanList.

libraries/metavoxels/src/DatagramSequencer.cpp

@@ -520,7 +520,9 @@ int SpanList::set(int offset, int length) {
 
     // look for an intersection within the list
     int position = 0;
-    for (QList<Span>::iterator it = _spans.begin(); it != _spans.end(); it++) {
+    for (int i = 0; i < _spans.size(); i++) {
+        QList<Span>::iterator it = _spans.begin() + i;
+    
         // if we intersect the unset portion, contract it
         position += it->unset;
         if (offset <= position) {
@@ -530,16 +532,20 @@ int SpanList::set(int offset, int length) {
             // if we continue into the set portion, expand it and consume following spans
             int extra = offset + length - position;
             if (extra >= 0) {
-                int amount = setSpans(it + 1, extra);
-                it->set += amount;
-                _totalSet += amount;
-            
+                extra -= it->set;
+                it->set += remove;
+                _totalSet += remove;
+                if (extra > 0) {
+                    int amount = setSpans(it + 1, extra);
+                    _spans[i].set += amount;
+                    _totalSet += amount;
+                }
             // otherwise, insert a new span
             } else {        
-                Span span = { it->unset, length + extra };
-                _spans.insert(it, span);
+                Span span = { it->unset, length };
                 it->unset = -extra;
-                _totalSet += span.set;
+                _spans.insert(it, span);
+                _totalSet += length;
             }
             return 0;
         }
@@ -548,9 +554,11 @@ int SpanList::set(int offset, int length) {
         position += it->set;
         if (offset <= position) {
             int extra = offset + length - position;
-            int amount = setSpans(it + 1, extra);
-            it->set += amount;
-            _totalSet += amount;
+            if (extra > 0) {
+                int amount = setSpans(it + 1, extra);
+                _spans[i].set += amount;
+                _totalSet += amount;
+            }
             return 0;
         }
     }
@@ -629,67 +637,71 @@ ReliableChannel::ReliableChannel(DatagramSequencer* sequencer, int index, bool o
 }
 
 void ReliableChannel::writeData(QDataStream& out, int bytes, QVector<DatagramSequencer::ChannelSpan>& spans) {
-    // find out how many spans we want to write
-    int spanCount = 0;
-    int remainingBytes = bytes;
-    bool first = true;
-    while (remainingBytes > 0) {
+    if (bytes > 0) {
+        _writePosition %= _buffer.pos();
+        
         int position = 0;
-        foreach (const SpanList::Span& span, _acknowledged.getSpans()) {
-            if (remainingBytes <= 0) {
-                break;
+        for (int i = 0; i < _acknowledged.getSpans().size(); i++) {
+            const SpanList::Span& span = _acknowledged.getSpans().at(i);
+            position += span.unset;
+            if (_writePosition < position) {
+                int start = qMax(position - span.unset, _writePosition);
+                int length = qMin(bytes, position - start);
+                writeSpan(out, start, length, spans);
+                writeFullSpans(out, bytes - length, i + 1, position + span.set, spans);
+                out << (quint32)0;
+                return;
             }
-            spanCount++;
-            remainingBytes -= getBytesToWrite(first, qMin(remainingBytes, span.unset));
-            position += (span.unset + span.set);
+            position += span.set;
         }
         int leftover = _buffer.pos() - position;
-        if (remainingBytes > 0 && leftover > 0) {
-            spanCount++;
-            remainingBytes -= getBytesToWrite(first, qMin(remainingBytes, leftover));
+        position = _buffer.pos();
+        
+        if (_writePosition < position && leftover > 0) {
+            int start = qMax(position - leftover, _writePosition);
+            int length = qMin(bytes, position - start);
+            writeSpan(out, start, length, spans);
+            writeFullSpans(out, bytes - length, 0, 0, spans);
         }
     }
-    
-    // write the count and the spans
-    out << (quint32)spanCount;
-    remainingBytes = bytes;
-    first = true;
-    while (remainingBytes > 0) {
-        int position = 0;
-        foreach (const SpanList::Span& span, _acknowledged.getSpans()) {
-            if (remainingBytes <= 0) {
-                break;
-            }
-            remainingBytes -= writeSpan(out, first, position, qMin(remainingBytes, span.unset), spans);
-            position += (span.unset + span.set);
+    out << (quint32)0;
+}
+
+void ReliableChannel::writeFullSpans(QDataStream& out, int bytes, int startingIndex, int position,
+        QVector<DatagramSequencer::ChannelSpan>& spans) {
+    int expandedSize = _acknowledged.getSpans().size() + 1;
+    for (int i = 0; i < expandedSize; i++) {
+        if (bytes == 0) {
+            return;
         }
-        int leftover = _buffer.pos() - position;
-        if (remainingBytes > 0 && leftover > 0) {
-            remainingBytes -= writeSpan(out, first, position, qMin(remainingBytes, leftover), spans);
+        int index = (startingIndex + i) % expandedSize;
+        if (index == _acknowledged.getSpans().size()) {
+            int leftover = _buffer.pos() - position;
+            if (leftover > 0) {
+                int length = qMin(leftover, bytes);
+                writeSpan(out, position, length, spans);
+                bytes -= length;
+            }
+            position = 0;
+            
+        } else {
+            const SpanList::Span& span = _acknowledged.getSpans().at(index);
+            int length = qMin(span.unset, bytes);
+            writeSpan(out, position, length, spans);
+            bytes -= length;
+            position += (span.unset + span.set);
         }
     }
 }
 
-int ReliableChannel::getBytesToWrite(bool& first, int length) const {
-    if (first) {
-        first = false;
-        return length - (_writePosition % length);
-    }
-    return length;
-}
-
-int ReliableChannel::writeSpan(QDataStream& out, bool& first, int position, int length, QVector<DatagramSequencer::ChannelSpan>& spans) {
-    if (first) {
-        first = false;
-        position = _writePosition % length;
-        length -= position;
-        _writePosition += length;
-    }
+int ReliableChannel::writeSpan(QDataStream& out, int position, int length, QVector<DatagramSequencer::ChannelSpan>& spans) {
     DatagramSequencer::ChannelSpan span = { _index, _offset + position, length };
     spans.append(span);
-    out << (quint32)span.offset;
     out << (quint32)length;
+    out << (quint32)span.offset;
     _buffer.writeToStream(position, length, out);
+    _writePosition = position + length;
+    
     return length;
 }
 
@@ -700,17 +712,20 @@ void ReliableChannel::spanAcknowledged(const DatagramSequencer::ChannelSpan& spa
         _buffer.seek(_buffer.size());
         
         _offset += advancement;
-        _writePosition = qMax(_writePosition - advancement, 0);
-    } 
+        _writePosition = qMax(_writePosition - advancement, 0);   
+    }
 }
 
 void ReliableChannel::readData(QDataStream& in) {
-    quint32 segments;
-    in >> segments;
     bool readSome = false;
-    for (quint32 i = 0; i < segments; i++) {
-        quint32 offset, size;
-        in >> offset >> size;
+    forever {
+        quint32 size;
+        in >> size;
+        if (size == 0) {
+            break;
+        }
+        quint32 offset;
+        in >> offset;
         
         int position = offset - _offset;
         int end = position + size;

libraries/metavoxels/src/DatagramSequencer.h

@@ -343,8 +343,9 @@ private:
     ReliableChannel(DatagramSequencer* sequencer, int index, bool output);
     
     void writeData(QDataStream& out, int bytes, QVector<DatagramSequencer::ChannelSpan>& spans);
-    int getBytesToWrite(bool& first, int length) const;
-    int writeSpan(QDataStream& out, bool& first, int position, int length, QVector<DatagramSequencer::ChannelSpan>& spans);
+    void writeFullSpans(QDataStream& out, int bytes, int startingIndex, int position,
+        QVector<DatagramSequencer::ChannelSpan>& spans);
+    int writeSpan(QDataStream& out, int position, int length, QVector<DatagramSequencer::ChannelSpan>& spans);
     
     void spanAcknowledged(const DatagramSequencer::ChannelSpan& span);
     

tests/metavoxels/src/MetavoxelTests.cpp

@@ -28,6 +28,119 @@ MetavoxelTests::MetavoxelTests(int& argc, char** argv) :
     QCoreApplication(argc, argv) {
 }
 
+static bool testSpanList() {
+    SpanList list;
+
+    if (list.getTotalSet() != 0 || !list.getSpans().isEmpty()) {
+        qDebug() << "Failed empty state test.";
+        return true;
+    }
+    
+    if (list.set(-5, 15) != 10 || list.getTotalSet() != 0 || !list.getSpans().isEmpty()) {
+        qDebug() << "Failed initial front set.";
+        return true;
+    } 
+    
+    if (list.set(5, 15) != 0 || list.getTotalSet() != 15 || list.getSpans().size() != 1 ||
+            list.getSpans().at(0).unset != 5 || list.getSpans().at(0).set != 15) {
+        qDebug() << "Failed initial middle set.";
+        return true;
+    }
+    
+    if (list.set(25, 5) != 0 || list.getTotalSet() != 20 || list.getSpans().size() != 2 ||
+            list.getSpans().at(0).unset != 5 || list.getSpans().at(0).set != 15 ||
+            list.getSpans().at(1).unset != 5 || list.getSpans().at(1).set != 5) {
+        qDebug() << "Failed initial end set.";
+        return true;
+    }
+    
+    if (list.set(1, 3) != 0 || list.getTotalSet() != 23 || list.getSpans().size() != 3 ||
+            list.getSpans().at(0).unset != 1 || list.getSpans().at(0).set != 3 ||
+            list.getSpans().at(1).unset != 1 || list.getSpans().at(1).set != 15 ||
+            list.getSpans().at(2).unset != 5 || list.getSpans().at(2).set != 5) {
+        qDebug() << "Failed second front set.";
+        return true;
+    }
+    SpanList threeSet = list;
+    
+    if (list.set(20, 5) != 0 || list.getTotalSet() != 28 || list.getSpans().size() != 2 ||
+            list.getSpans().at(0).unset != 1 || list.getSpans().at(0).set != 3 ||
+            list.getSpans().at(1).unset != 1 || list.getSpans().at(1).set != 25) {
+        qDebug() << "Failed minimal join last two.";
+        return true;
+    }
+    
+    list = threeSet;
+    if (list.set(5, 25) != 0 || list.getTotalSet() != 28 || list.getSpans().size() != 2 ||
+            list.getSpans().at(0).unset != 1 || list.getSpans().at(0).set != 3 ||
+            list.getSpans().at(1).unset != 1 || list.getSpans().at(1).set != 25) {
+        qDebug() << "Failed maximal join last two.";
+        return true;
+    }
+    
+    list = threeSet;
+    if (list.set(10, 18) != 0 || list.getTotalSet() != 28 || list.getSpans().size() != 2 ||
+            list.getSpans().at(0).unset != 1 || list.getSpans().at(0).set != 3 ||
+            list.getSpans().at(1).unset != 1 || list.getSpans().at(1).set != 25) {
+        qDebug() << "Failed middle join last two.";
+        return true;
+    }
+    
+    list = threeSet;
+    if (list.set(10, 18) != 0 || list.getTotalSet() != 28 || list.getSpans().size() != 2 ||
+            list.getSpans().at(0).unset != 1 || list.getSpans().at(0).set != 3 ||
+            list.getSpans().at(1).unset != 1 || list.getSpans().at(1).set != 25) {
+        qDebug() << "Failed middle join last two.";
+        return true;
+    }
+    
+    list = threeSet;
+    if (list.set(2, 26) != 0 || list.getTotalSet() != 29 || list.getSpans().size() != 1 ||
+            list.getSpans().at(0).unset != 1 || list.getSpans().at(0).set != 29) {
+        qDebug() << "Failed middle join three.";
+        return true;
+    }
+    
+    list = threeSet;
+    if (list.set(0, 2) != 4 || list.getTotalSet() != 20 || list.getSpans().size() != 2 ||
+            list.getSpans().at(0).unset != 1 || list.getSpans().at(0).set != 15 ||
+            list.getSpans().at(1).unset != 5 || list.getSpans().at(1).set != 5) {
+        qDebug() << "Failed front advance.";
+        return true;
+    }
+    
+    list = threeSet;
+    if (list.set(-10, 15) != 20 || list.getTotalSet() != 5 || list.getSpans().size() != 1 ||
+            list.getSpans().at(0).unset != 5 || list.getSpans().at(0).set != 5) {
+        qDebug() << "Failed middle advance.";
+        return true;
+    }
+    
+    list = threeSet;
+    if (list.set(-10, 38) != 30 || list.getTotalSet() != 0 || list.getSpans().size() != 0) {
+        qDebug() << "Failed end advance.";
+        return true;
+    }
+    
+    list = threeSet;
+    if (list.set(-10, 100) != 90 || list.getTotalSet() != 0 || list.getSpans().size() != 0) {
+        qDebug() << "Failed clobber advance.";
+        return true;
+    }
+    
+    list = threeSet;
+    if (list.set(21, 3) != 0 || list.getTotalSet() != 26 || list.getSpans().size() != 4 ||
+            list.getSpans().at(0).unset != 1 || list.getSpans().at(0).set != 3 ||
+            list.getSpans().at(1).unset != 1 || list.getSpans().at(1).set != 15 ||
+            list.getSpans().at(2).unset != 1 || list.getSpans().at(2).set != 3 ||
+            list.getSpans().at(3).unset != 1 || list.getSpans().at(3).set != 5) {
+        qDebug() << "Failed adding fourth.";
+        return true;
+    }
+    
+    return false;
+}
+
 static int datagramsSent = 0;
 static int datagramsReceived = 0;
 static int bytesSent = 0;
@@ -332,9 +445,18 @@ bool MetavoxelTests::run() {
     QStringList arguments = this->arguments();
     int test = (arguments.size() > 1) ? arguments.at(1).toInt() : 0;
 
+    if (test == 0 || test == 1) {
+        qDebug() << "Running SpanList test...";
+        qDebug();
+        
+        if (testSpanList()) {
+            return true;
+        }
+    }
+
     QByteArray datagramHeader("testheader");
     const int SIMULATION_ITERATIONS = 10000;
-    if (test == 0 || test == 1) {
+    if (test == 0 || test == 2) {
         qDebug() << "Running transmission test...";
         qDebug();
     
@@ -364,7 +486,7 @@ bool MetavoxelTests::run() {
         qDebug();
     }
     
-    if (test == 0 || test == 2) {
+    if (test == 0 || test == 3) {
         qDebug() << "Running congestion control test...";
         qDebug();
         
@@ -394,7 +516,7 @@ bool MetavoxelTests::run() {
         qDebug() << "Efficiency:" << ((float)streamedBytesReceived / bytesReceived);
     }
     
-    if (test == 0 || test == 3) {
+    if (test == 0 || test == 4) {
         qDebug() << "Running serialization test...";
         qDebug();
         
@@ -403,7 +525,7 @@ bool MetavoxelTests::run() {
         }
     }
     
-    if (test == 0 || test == 4) {
+    if (test == 0 || test == 5) {
         qDebug() << "Running metavoxel data test...";
         qDebug();
     
@@ -532,6 +654,13 @@ Endpoint::Endpoint(const QByteArray& datagramHeader, Mode mode) :
     if (mode == CONGESTION_MODE) {
         const int HUGE_STREAM_BYTES = 50 * 1024 * 1024;
         bytes = createRandomBytes(HUGE_STREAM_BYTES, HUGE_STREAM_BYTES);
+        
+        // initialize the pipeline
+        for (int i = 0; i < 10; i++) {
+            _pipeline.append(ByteArrayVector());
+        }
+        _remainingPipelineCapacity = 100 * 1024;
+        
     } else {
         const int MIN_STREAM_BYTES = 100000;
         const int MAX_STREAM_BYTES = 200000;
@@ -669,10 +798,9 @@ int MutateVisitor::visit(MetavoxelInfo& info) {
 
 bool Endpoint::simulate(int iterationNumber) {
     // update/send our delayed datagrams
-    for (QList<QPair<QByteArray, int> >::iterator it = _delayedDatagrams.begin(); it != _delayedDatagrams.end(); ) {
+    for (QList<ByteArrayIntPair>::iterator it = _delayedDatagrams.begin(); it != _delayedDatagrams.end(); ) {
         if (it->second-- == 1) {
-            _other->_sequencer->receivedDatagram(it->first);
-            datagramsReceived++;    
+            _other->receiveDatagram(it->first);
             it = _delayedDatagrams.erase(it);
         
         } else {
@@ -683,6 +811,16 @@ bool Endpoint::simulate(int iterationNumber) {
     int oldDatagramsSent = datagramsSent;
     int oldBytesSent = bytesSent;
     if (_mode == CONGESTION_MODE) {
+        // cycle our pipeline
+        ByteArrayVector datagrams = _pipeline.takeLast();
+        _pipeline.prepend(ByteArrayVector());
+        foreach (const QByteArray& datagram, datagrams) {
+            _sequencer->receivedDatagram(datagram);
+            datagramsReceived++;
+            bytesReceived += datagram.size();
+            _remainingPipelineCapacity += datagram.size();
+        }
+    
         Bitstream& out = _sequencer->startPacket();
         out << QVariant();
         _sequencer->endPacket();    
@@ -804,7 +942,7 @@ void Endpoint::sendDatagram(const QByteArray& datagram) {
         const int MIN_DELAY = 1;
         const int MAX_DELAY = 5;
         // have to copy the datagram; the one we're passed is a reference to a shared buffer
-        _delayedDatagrams.append(QPair<QByteArray, int>(QByteArray(datagram.constData(), datagram.size()),
+        _delayedDatagrams.append(ByteArrayIntPair(QByteArray(datagram.constData(), datagram.size()),
             randIntInRange(MIN_DELAY, MAX_DELAY)));
         
         // and some are duplicated
@@ -814,9 +952,7 @@ void Endpoint::sendDatagram(const QByteArray& datagram) {
         }
     }
     
-    _other->_sequencer->receivedDatagram(datagram);
-    datagramsReceived++;
-    bytesReceived += datagram.size();
+    _other->receiveDatagram(datagram);
 }
 
 void Endpoint::handleHighPriorityMessage(const QVariant& message) {
@@ -942,6 +1078,20 @@ void Endpoint::clearReceiveRecordsBefore(int index) {
     _receiveRecords.erase(_receiveRecords.begin(), _receiveRecords.begin() + index + 1);
 }
 
+void Endpoint::receiveDatagram(const QByteArray& datagram) {
+    if (_mode == CONGESTION_MODE) {
+        if (datagram.size() <= _remainingPipelineCapacity) {
+            // have to copy the datagram; the one we're passed is a reference to a shared buffer
+            _pipeline[0].append(QByteArray(datagram.constData(), datagram.size()));
+            _remainingPipelineCapacity -= datagram.size();   
+        }
+    } else {
+        _sequencer->receivedDatagram(datagram);
+        datagramsReceived++;
+        bytesReceived += datagram.size();
+    }
+}
+
 void Endpoint::handleMessage(const QVariant& message, Bitstream& in) {
     int userType = message.userType();
     if (userType == ClientStateMessage::Type) {

tests/metavoxels/src/MetavoxelTests.h

@@ -63,6 +63,8 @@ private slots:
 
 private:
     
+    void receiveDatagram(const QByteArray& datagram);
+    
     void handleMessage(const QVariant& message, Bitstream& in);
     
     class SendRecord {
@@ -96,7 +98,14 @@ private:
     SharedObjectPointer _sphere;
     
     Endpoint* _other;
-    QList<QPair<QByteArray, int> > _delayedDatagrams;
+    
+    typedef QPair<QByteArray, int> ByteArrayIntPair;
+    QList<ByteArrayIntPair> _delayedDatagrams;
+
+    typedef QVector<QByteArray> ByteArrayVector;
+    QList<ByteArrayVector> _pipeline;
+    int _remainingPipelineCapacity;
+
     float _highPriorityMessagesToSend;
     QVariantList _highPriorityMessagesSent;
     QList<SequencedTestMessage> _unreliableMessagesSent;