added seq number to all Audio types, untested

assignment-client/src/Agent.cpp

@@ -147,6 +147,8 @@ void Agent::readPendingDatagrams() {
                 }
 
             } else if (datagramPacketType == PacketTypeMixedAudio) {
+                // TODO: track sequence numbers for mixed audio???
+
                 // parse the data and grab the average loudness
                 _receivedAudioBuffer.parseData(receivedPacket);
                 

assignment-client/src/audio/AudioMixer.cpp

@@ -519,7 +519,7 @@ void AudioMixer::run() {
     QElapsedTimer timer;
     timer.start();
     
-    char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO
+    char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO + sizeof(quint16)
                                      + numBytesForPacketHeaderGivenPacketType(PacketTypeMixedAudio)];
     
     int usecToSleep = BUFFER_SEND_INTERVAL_USECS;
@@ -602,18 +602,32 @@ void AudioMixer::run() {
         foreach (const SharedNodePointer& node, nodeList->getNodeHash()) {
             if (node->getType() == NodeType::Agent && node->getActiveSocket() && node->getLinkedData()
                 && ((AudioMixerClientData*) node->getLinkedData())->getAvatarAudioRingBuffer()) {
+
+                AudioMixerClientData* nodeData = (AudioMixerClientData*)node->getLinkedData();
+
                 prepareMixForListeningNode(node.data());
                 
+                // pack header
                 int numBytesPacketHeader = populatePacketHeader(clientMixBuffer, PacketTypeMixedAudio);
+                char* dataAt = clientMixBuffer + numBytesPacketHeader;
 
-                memcpy(clientMixBuffer + numBytesPacketHeader, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO);
+                // pack sequence number
-                nodeList->writeDatagram(clientMixBuffer, NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader, node);
+                quint16 sequence = nodeData->getOutgoingSequenceNumber();
+                memcpy(dataAt, &sequence, sizeof(quint16));
+                dataAt += sizeof(quint16);
+
+                // pack mixed audio samples
+                memcpy(dataAt, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO);
+                dataAt += NETWORK_BUFFER_LENGTH_BYTES_STEREO;
+
+                // send mixed audio packet
+                nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node);
+                nodeData->incrementOutgoingSequenceNumber();
                 
                 // send an audio stream stats packet if it's time
                 if (sendAudioStreamStats) {
-                    int numBytesWritten = ((AudioMixerClientData*)node->getLinkedData())
+                    int numBytesAudioStreamStatsPacket = nodeData->encodeAudioStreamStatsPacket(audioStreamStatsPacket);
-                        ->encodeAudioStreamStatsPacket(audioStreamStatsPacket);
+                    nodeList->writeDatagram(audioStreamStatsPacket, numBytesAudioStreamStatsPacket, node);
-                    nodeList->writeDatagram(audioStreamStatsPacket, numBytesWritten, node);
                 }
 
                 ++_sumListeners;

assignment-client/src/audio/AudioMixerClientData.cpp

@@ -20,7 +20,9 @@
 #include "AudioMixerClientData.h"
 
 AudioMixerClientData::AudioMixerClientData() :
-    _ringBuffers()
+    _ringBuffers(),
+    _outgoingSequenceNumber(0),
+    _incomingAvatarSequenceNumberStats()
 {
     
 }
@@ -49,13 +51,14 @@ int AudioMixerClientData::parseData(const QByteArray& packet) {
     int numBytesPacketHeader = numBytesForPacketHeader(packet);
     const char* sequenceAt = packet.constData() + numBytesPacketHeader;
     quint16 sequence = *(reinterpret_cast<const quint16*>(sequenceAt));
-    _sequenceNumberStats.sequenceNumberReceived(sequence);
 
     PacketType packetType = packetTypeForPacket(packet);
     if (packetType == PacketTypeMicrophoneAudioWithEcho
         || packetType == PacketTypeMicrophoneAudioNoEcho
         || packetType == PacketTypeSilentAudioFrame) {
 
+        _incomingAvatarSequenceNumberStats.sequenceNumberReceived(sequence);
+
         // grab the AvatarAudioRingBuffer from the vector (or create it if it doesn't exist)
         AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer();
         
@@ -85,6 +88,8 @@ int AudioMixerClientData::parseData(const QByteArray& packet) {
         // grab the stream identifier for this injected audio
         QUuid streamIdentifier = QUuid::fromRfc4122(packet.mid(numBytesForPacketHeader(packet), NUM_BYTES_RFC4122_UUID));
 
+        _incomingInjectedSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence);
+
         InjectedAudioRingBuffer* matchingInjectedRingBuffer = NULL;
 
         for (int i = 0; i < _ringBuffers.size(); i++) {
@@ -140,6 +145,9 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() {
         } else if (audioBuffer->getType() == PositionalAudioRingBuffer::Injector
                    && audioBuffer->hasStarted() && audioBuffer->isStarved()) {
             // this is an empty audio buffer that has starved, safe to delete
+            // also delete its sequence number stats
+            QUuid streamIdentifier = ((InjectedAudioRingBuffer*)audioBuffer)->getStreamIdentifier();
+            _incomingInjectedSequenceNumberStatsMap.remove(streamIdentifier);
             delete audioBuffer;
             i = _ringBuffers.erase(i);
             continue;

assignment-client/src/audio/AudioMixerClientData.h

@@ -38,11 +38,17 @@ public:
     
     QString getJitterBufferStatsString() const;
 
-    const SequenceNumberStats& getSequenceNumberStats() const { return _sequenceNumberStats; }
+    void incrementOutgoingSequenceNumber() { _outgoingSequenceNumber++; }
+
+    quint16 getOutgoingSequenceNumber() const { return _outgoingSequenceNumber; }
+    //const SequenceNumberStats& getIncomingSequenceNumberStats() const { return _incomingSequenceNumberStats; }
 
 private:
     QList<PositionalAudioRingBuffer*> _ringBuffers;
-    SequenceNumberStats _sequenceNumberStats;
+
+    quint16 _outgoingSequenceNumber;
+    SequenceNumberStats _incomingAvatarSequenceNumberStats;
+    QHash<QUuid, SequenceNumberStats> _incomingInjectedSequenceNumberStatsMap;
 };
 
 #endif // hifi_AudioMixerClientData_h

interface/src/Application.cpp

@@ -3608,6 +3608,9 @@ ScriptEngine* Application::loadScript(const QString& scriptName, bool loadScript
     // when the application is about to quit, stop our script engine so it unwinds properly
     connect(this, SIGNAL(aboutToQuit()), scriptEngine, SLOT(stop()));
 
+    NodeList* nodeList = NodeList::getInstance();
+    connect(nodeList, &NodeList::nodeKilled, scriptEngine, &ScriptEngine::nodeKilled);
+
     scriptEngine->moveToThread(workerThread);
 
     // Starts an event loop, and emits workerThread->started()

interface/src/Audio.cpp

@@ -104,7 +104,7 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) :
     _scopeOutputLeft(0),
     _scopeOutputRight(0),
     _audioMixerJitterBufferStats(),
-    _sequenceNumber(0)
+    _outgoingSequenceNumber(0)
 {
     // clear the array of locally injected samples
     memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL);
@@ -120,7 +120,7 @@ void Audio::init(QGLWidget *parent) {
 
 void Audio::reset() {
     _ringBuffer.reset();
-    _sequenceNumber = 0;
+    _outgoingSequenceNumber = 0;
 }
 
 QAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName) {
@@ -656,8 +656,8 @@ void Audio::handleAudioInput() {
 
             char* currentPacketPtr = audioDataPacket + populatePacketHeader(audioDataPacket, packetType);
             
-            // pack seq number
+            // pack sequence number
-            memcpy(currentPacketPtr, &_sequenceNumber, sizeof(quint16));
+            memcpy(currentPacketPtr, &_outgoingSequenceNumber, sizeof(quint16));
             currentPacketPtr += sizeof(quint16);
 
             // set the mono/stereo byte
@@ -672,13 +672,13 @@ void Audio::handleAudioInput() {
             currentPacketPtr += sizeof(headOrientation);
             
             nodeList->writeDatagram(audioDataPacket, numAudioBytes + leadingBytes, audioMixer);
-            _sequenceNumber++;
+            _outgoingSequenceNumber++;
 
             Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO)
                 .updateValue(numAudioBytes + leadingBytes);
         } else {
             // reset seq numbers if there's no connection with an audiomixer
-            _sequenceNumber = 0;
+            _outgoingSequenceNumber = 0;
         }
         delete[] inputAudioSamples;
     }
@@ -827,6 +827,14 @@ void Audio::toggleStereoInput() {
 }
 
 void Audio::processReceivedAudio(const QByteArray& audioByteArray) {
+
+    // parse sequence number for this packet
+    int numBytesPacketHeader = numBytesForPacketHeader(audioByteArray);
+    const char* sequenceAt = audioByteArray.constData() + numBytesPacketHeader;
+    quint16 sequence = *((quint16*)sequenceAt);
+    _incomingSequenceNumberStats.sequenceNumberReceived(sequence);
+
+    // parse audio data
     _ringBuffer.parseData(audioByteArray);
     
     float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float) _outputFormat.sampleRate())

interface/src/Audio.h

@@ -239,7 +239,8 @@ private:
 
     AudioMixerJitterBuffersStats _audioMixerJitterBufferStats;
 
-    quint16 _sequenceNumber;
+    quint16 _outgoingSequenceNumber;
+    SequenceNumberStats _incomingSequenceNumberStats;
 };
 
 

libraries/audio/src/AudioInjector.cpp

@@ -61,6 +61,11 @@ void AudioInjector::injectAudio() {
         QByteArray injectAudioPacket = byteArrayWithPopulatedHeader(PacketTypeInjectAudio);
         QDataStream packetStream(&injectAudioPacket, QIODevice::Append);
         
+        // skip sequence number for now
+        int numPreSequenceNumberBytes = injectAudioPacket.size();
+        packetStream.skipRawData(sizeof(quint16));
+
+        // pack stream identifier (a generated UUID)
         packetStream << QUuid::createUuid();
         
         // pack the flag for loopback
@@ -91,6 +96,7 @@ void AudioInjector::injectAudio() {
         bool shouldLoop = _options.getLoop();
         
         // loop to send off our audio in NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL byte chunks
+        quint16 outgoingSequenceNumber = 0;
         while (currentSendPosition < soundByteArray.size() && !_shouldStop) {
             
             int bytesToCopy = std::min(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL,
@@ -98,6 +104,9 @@ void AudioInjector::injectAudio() {
             
             // resize the QByteArray to the right size
             injectAudioPacket.resize(numPreAudioDataBytes + bytesToCopy);
+
+            // pack the sequence number
+            memcpy(injectAudioPacket.data() + numPreSequenceNumberBytes, &outgoingSequenceNumber, sizeof(quint16));
             
             // copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet
             memcpy(injectAudioPacket.data() + numPreAudioDataBytes, soundByteArray.data() + currentSendPosition, bytesToCopy);
@@ -107,6 +116,7 @@ void AudioInjector::injectAudio() {
             
             // send off this audio packet
             nodeList->writeDatagram(injectAudioPacket, audioMixer);
+            outgoingSequenceNumber++;
             
             currentSendPosition += bytesToCopy;
             

libraries/audio/src/AudioRingBuffer.cpp

@@ -66,6 +66,7 @@ void AudioRingBuffer::resizeForFrameSize(qint64 numFrameSamples) {
 }
 
 int AudioRingBuffer::parseData(const QByteArray& packet) {
+    // skip packet header and sequence number
     int numBytesBeforeAudioData = numBytesForPacketHeader(packet) + sizeof(quint16);
     return writeData(packet.data() + numBytesBeforeAudioData, packet.size() - numBytesBeforeAudioData);
 }

libraries/audio/src/PositionalAudioRingBuffer.cpp

@@ -107,6 +107,9 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) {
     
     // skip the packet header (includes the source UUID)
     int readBytes = numBytesForPacketHeader(packet);
+
+    // skip the sequence number
+    readBytes += sizeof(quint16);
     
     // hop over the channel flag that has already been read in AudioMixerClientData
     readBytes += sizeof(quint8);

libraries/networking/src/PacketHeaders.cpp

@@ -50,6 +50,8 @@ PacketVersion versionForPacketType(PacketType type) {
         case PacketTypeMicrophoneAudioNoEcho:
         case PacketTypeMicrophoneAudioWithEcho:
         case PacketTypeSilentAudioFrame:
+            return 2;
+        case PacketTypeMixedAudio:
             return 1;
         case PacketTypeAvatarData:
             return 3;

libraries/networking/src/SequenceNumbersStats.cpp

@@ -59,7 +59,7 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, const bool wa
             }
         } else if (absGap > MAX_REASONABLE_SEQUENCE_GAP) {
             // ignore packet if gap is unreasonable
-            qDebug() << "ignoring unreasonable packet... sequence:" << incoming
+            qDebug() << "ignoring unreasonable sequence number:" << incoming
                 << "previous:" << _lastReceived;
             _numUnreasonable++;
             return;

libraries/script-engine/src/ScriptEngine.cpp

@@ -459,32 +459,55 @@ void ScriptEngine::run() {
                     }
                 }
 
-                QByteArray audioPacket = byteArrayWithPopulatedHeader(silentFrame
+                char audioPacket[MAX_PACKET_SIZE];
-                                                                      ? PacketTypeSilentAudioFrame
-                                                                      : PacketTypeMicrophoneAudioNoEcho);
 
-                QDataStream packetStream(&audioPacket, QIODevice::Append);
+                // pack header
+                int numBytesPacketHeader = populatePacketHeader(audioPacket, silentFrame
+                                                                            ? PacketTypeSilentAudioFrame
+                                                                            : PacketTypeMicrophoneAudioNoEcho);
+                char* dataAt = audioPacket + numBytesPacketHeader;
+
+                // skip over sequence number for now; will be packed when destination node is known
+                char* sequenceAt = dataAt;
+                dataAt += sizeof(quint16);
 
                 // use the orientation and position of this avatar for the source of this audio
-                packetStream.writeRawData(reinterpret_cast<const char*>(&_avatarData->getPosition()), sizeof(glm::vec3));
+                memcpy(dataAt, &_avatarData->getPosition(), sizeof(glm::vec3));
+                dataAt += sizeof(glm::vec3);
+
                 glm::quat headOrientation = _avatarData->getHeadOrientation();
-                packetStream.writeRawData(reinterpret_cast<const char*>(&headOrientation), sizeof(glm::quat));
+                memcpy(dataAt, &headOrientation, sizeof(glm::quat));
+                dataAt += sizeof(glm::quat);
 
                 if (silentFrame) {
                     if (!_isListeningToAudioStream) {
                         // if we have a silent frame and we're not listening then just send nothing and break out of here
                         break;
                     }
-
                     // write the number of silent samples so the audio-mixer can uphold timing
-                    packetStream.writeRawData(reinterpret_cast<const char*>(&SCRIPT_AUDIO_BUFFER_SAMPLES), sizeof(int16_t));
+                    memcpy(dataAt, &SCRIPT_AUDIO_BUFFER_SAMPLES, sizeof(int16_t));
+                    dataAt += sizeof(int16_t);
                 } else if (nextSoundOutput) {
                     // write the raw audio data
-                    packetStream.writeRawData(reinterpret_cast<const char*>(nextSoundOutput),
+                    int numAvailableBytes = numAvailableSamples * sizeof(int16_t);
-                                              numAvailableSamples * sizeof(int16_t));
+                    memcpy(dataAt, nextSoundOutput, numAvailableBytes);
+                    dataAt += numAvailableBytes;
                 }
 
-                nodeList->broadcastToNodes(audioPacket, NodeSet() << NodeType::AudioMixer);
+                // write audio packet to AudioMixer nodes
+                int audioPacketSize = dataAt - audioPacket;
+                NodeList* nodeList = NodeList::getInstance();
+                foreach(const SharedNodePointer& node, nodeList->getNodeHash()) {
+                    // only send to nodes of type AudioMixer
+                    if (node->getType() == NodeType::AudioMixer) {
+                        // pack sequence number
+                        quint16 sequence = _outgoingSequenceNumbers[node->getUUID()]++;
+                        memcpy(sequenceAt, &sequence, sizeof(quint16));
+                        
+                        // send audio packet
+                        nodeList->writeDatagram(audioPacket, audioPacketSize, node);
+                    }
+                }
             }
         }
 
@@ -658,3 +681,7 @@ void ScriptEngine::include(const QString& includeFile) {
         emit errorMessage("Uncaught exception at (" + includeFile + ") line" + QString::number(line) + ":" + result.toString());
     }
 }
+
+void ScriptEngine::nodeKilled(SharedNodePointer node) {
+    _outgoingSequenceNumbers.remove(node->getUUID());
+}

libraries/script-engine/src/ScriptEngine.h

@@ -100,6 +100,8 @@ public slots:
     void include(const QString& includeFile);
     void print(const QString& message);
 
+    void nodeKilled(SharedNodePointer node);
+
 signals:
     void update(float deltaTime);
     void scriptEnding();
@@ -146,6 +148,7 @@ private:
     ScriptUUID _uuidLibrary;
     AnimationCache _animationCache;
 
+    QHash<QUuid, quint16> _outgoingSequenceNumbers;
 };
 
 #endif // hifi_ScriptEngine_h