From cb48825561c20f462a91f0916ffb7d5f83e9e7e1 Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 26 Jun 2014 16:52:23 -0700 Subject: [PATCH] added seq number to all Audio types, untested --- assignment-client/src/Agent.cpp | 2 + assignment-client/src/audio/AudioMixer.cpp | 26 +++++++--- .../src/audio/AudioMixerClientData.cpp | 12 ++++- .../src/audio/AudioMixerClientData.h | 10 +++- interface/src/Application.cpp | 3 ++ interface/src/Audio.cpp | 20 +++++--- interface/src/Audio.h | 3 +- libraries/audio/src/AudioInjector.cpp | 10 ++++ libraries/audio/src/AudioRingBuffer.cpp | 1 + .../audio/src/PositionalAudioRingBuffer.cpp | 3 ++ libraries/networking/src/PacketHeaders.cpp | 2 + .../networking/src/SequenceNumbersStats.cpp | 2 +- libraries/script-engine/src/ScriptEngine.cpp | 49 ++++++++++++++----- libraries/script-engine/src/ScriptEngine.h | 3 ++ 14 files changed, 117 insertions(+), 29 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index fcc2288356..561acc0eb0 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -147,6 +147,8 @@ void Agent::readPendingDatagrams() { } } else if (datagramPacketType == PacketTypeMixedAudio) { + // TODO: track sequence numbers for mixed audio??? + // parse the data and grab the average loudness _receivedAudioBuffer.parseData(receivedPacket); diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index f41c3392c7..6b43fb37e0 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -519,7 +519,7 @@ void AudioMixer::run() { QElapsedTimer timer; timer.start(); - char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO + char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO + sizeof(quint16) + numBytesForPacketHeaderGivenPacketType(PacketTypeMixedAudio)]; int usecToSleep = BUFFER_SEND_INTERVAL_USECS; @@ -602,18 +602,32 @@ void AudioMixer::run() { foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { if (node->getType() == NodeType::Agent && node->getActiveSocket() && node->getLinkedData() && ((AudioMixerClientData*) node->getLinkedData())->getAvatarAudioRingBuffer()) { + + AudioMixerClientData* nodeData = (AudioMixerClientData*)node->getLinkedData(); + prepareMixForListeningNode(node.data()); + // pack header int numBytesPacketHeader = populatePacketHeader(clientMixBuffer, PacketTypeMixedAudio); + char* dataAt = clientMixBuffer + numBytesPacketHeader; - memcpy(clientMixBuffer + numBytesPacketHeader, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); - nodeList->writeDatagram(clientMixBuffer, NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader, node); + // pack sequence number + quint16 sequence = nodeData->getOutgoingSequenceNumber(); + memcpy(dataAt, &sequence, sizeof(quint16)); + dataAt += sizeof(quint16); + + // pack mixed audio samples + memcpy(dataAt, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); + dataAt += NETWORK_BUFFER_LENGTH_BYTES_STEREO; + + // send mixed audio packet + nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); + nodeData->incrementOutgoingSequenceNumber(); // send an audio stream stats packet if it's time if (sendAudioStreamStats) { - int numBytesWritten = ((AudioMixerClientData*)node->getLinkedData()) - ->encodeAudioStreamStatsPacket(audioStreamStatsPacket); - nodeList->writeDatagram(audioStreamStatsPacket, numBytesWritten, node); + int numBytesAudioStreamStatsPacket = nodeData->encodeAudioStreamStatsPacket(audioStreamStatsPacket); + nodeList->writeDatagram(audioStreamStatsPacket, numBytesAudioStreamStatsPacket, node); } ++_sumListeners; diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index f35034eab3..8f50647463 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -20,7 +20,9 @@ #include "AudioMixerClientData.h" AudioMixerClientData::AudioMixerClientData() : - _ringBuffers() + _ringBuffers(), + _outgoingSequenceNumber(0), + _incomingAvatarSequenceNumberStats() { } @@ -49,13 +51,14 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { int numBytesPacketHeader = numBytesForPacketHeader(packet); const char* sequenceAt = packet.constData() + numBytesPacketHeader; quint16 sequence = *(reinterpret_cast(sequenceAt)); - _sequenceNumberStats.sequenceNumberReceived(sequence); PacketType packetType = packetTypeForPacket(packet); if (packetType == PacketTypeMicrophoneAudioWithEcho || packetType == PacketTypeMicrophoneAudioNoEcho || packetType == PacketTypeSilentAudioFrame) { + _incomingAvatarSequenceNumberStats.sequenceNumberReceived(sequence); + // grab the AvatarAudioRingBuffer from the vector (or create it if it doesn't exist) AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); @@ -85,6 +88,8 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { // grab the stream identifier for this injected audio QUuid streamIdentifier = QUuid::fromRfc4122(packet.mid(numBytesForPacketHeader(packet), NUM_BYTES_RFC4122_UUID)); + _incomingInjectedSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence); + InjectedAudioRingBuffer* matchingInjectedRingBuffer = NULL; for (int i = 0; i < _ringBuffers.size(); i++) { @@ -140,6 +145,9 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { } else if (audioBuffer->getType() == PositionalAudioRingBuffer::Injector && audioBuffer->hasStarted() && audioBuffer->isStarved()) { // this is an empty audio buffer that has starved, safe to delete + // also delete its sequence number stats + QUuid streamIdentifier = ((InjectedAudioRingBuffer*)audioBuffer)->getStreamIdentifier(); + _incomingInjectedSequenceNumberStatsMap.remove(streamIdentifier); delete audioBuffer; i = _ringBuffers.erase(i); continue; diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 508572db43..c418df7905 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -38,11 +38,17 @@ public: QString getJitterBufferStatsString() const; - const SequenceNumberStats& getSequenceNumberStats() const { return _sequenceNumberStats; } + void incrementOutgoingSequenceNumber() { _outgoingSequenceNumber++; } + + quint16 getOutgoingSequenceNumber() const { return _outgoingSequenceNumber; } + //const SequenceNumberStats& getIncomingSequenceNumberStats() const { return _incomingSequenceNumberStats; } private: QList _ringBuffers; - SequenceNumberStats _sequenceNumberStats; + + quint16 _outgoingSequenceNumber; + SequenceNumberStats _incomingAvatarSequenceNumberStats; + QHash _incomingInjectedSequenceNumberStatsMap; }; #endif // hifi_AudioMixerClientData_h diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index f40185f34f..80ef1ff6a1 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -3608,6 +3608,9 @@ ScriptEngine* Application::loadScript(const QString& scriptName, bool loadScript // when the application is about to quit, stop our script engine so it unwinds properly connect(this, SIGNAL(aboutToQuit()), scriptEngine, SLOT(stop())); + NodeList* nodeList = NodeList::getInstance(); + connect(nodeList, &NodeList::nodeKilled, scriptEngine, &ScriptEngine::nodeKilled); + scriptEngine->moveToThread(workerThread); // Starts an event loop, and emits workerThread->started() diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index ac7598908c..ff019712d7 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -104,7 +104,7 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _scopeOutputLeft(0), _scopeOutputRight(0), _audioMixerJitterBufferStats(), - _sequenceNumber(0) + _outgoingSequenceNumber(0) { // clear the array of locally injected samples memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); @@ -120,7 +120,7 @@ void Audio::init(QGLWidget *parent) { void Audio::reset() { _ringBuffer.reset(); - _sequenceNumber = 0; + _outgoingSequenceNumber = 0; } QAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName) { @@ -656,8 +656,8 @@ void Audio::handleAudioInput() { char* currentPacketPtr = audioDataPacket + populatePacketHeader(audioDataPacket, packetType); - // pack seq number - memcpy(currentPacketPtr, &_sequenceNumber, sizeof(quint16)); + // pack sequence number + memcpy(currentPacketPtr, &_outgoingSequenceNumber, sizeof(quint16)); currentPacketPtr += sizeof(quint16); // set the mono/stereo byte @@ -672,13 +672,13 @@ void Audio::handleAudioInput() { currentPacketPtr += sizeof(headOrientation); nodeList->writeDatagram(audioDataPacket, numAudioBytes + leadingBytes, audioMixer); - _sequenceNumber++; + _outgoingSequenceNumber++; Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) .updateValue(numAudioBytes + leadingBytes); } else { // reset seq numbers if there's no connection with an audiomixer - _sequenceNumber = 0; + _outgoingSequenceNumber = 0; } delete[] inputAudioSamples; } @@ -827,6 +827,14 @@ void Audio::toggleStereoInput() { } void Audio::processReceivedAudio(const QByteArray& audioByteArray) { + + // parse sequence number for this packet + int numBytesPacketHeader = numBytesForPacketHeader(audioByteArray); + const char* sequenceAt = audioByteArray.constData() + numBytesPacketHeader; + quint16 sequence = *((quint16*)sequenceAt); + _incomingSequenceNumberStats.sequenceNumberReceived(sequence); + + // parse audio data _ringBuffer.parseData(audioByteArray); float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float) _outputFormat.sampleRate()) diff --git a/interface/src/Audio.h b/interface/src/Audio.h index c6fde709b8..cb04418eaf 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -239,7 +239,8 @@ private: AudioMixerJitterBuffersStats _audioMixerJitterBufferStats; - quint16 _sequenceNumber; + quint16 _outgoingSequenceNumber; + SequenceNumberStats _incomingSequenceNumberStats; }; diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index 129dc47bd0..d9d4dc6d3a 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -61,6 +61,11 @@ void AudioInjector::injectAudio() { QByteArray injectAudioPacket = byteArrayWithPopulatedHeader(PacketTypeInjectAudio); QDataStream packetStream(&injectAudioPacket, QIODevice::Append); + // skip sequence number for now + int numPreSequenceNumberBytes = injectAudioPacket.size(); + packetStream.skipRawData(sizeof(quint16)); + + // pack stream identifier (a generated UUID) packetStream << QUuid::createUuid(); // pack the flag for loopback @@ -91,6 +96,7 @@ void AudioInjector::injectAudio() { bool shouldLoop = _options.getLoop(); // loop to send off our audio in NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL byte chunks + quint16 outgoingSequenceNumber = 0; while (currentSendPosition < soundByteArray.size() && !_shouldStop) { int bytesToCopy = std::min(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL, @@ -98,6 +104,9 @@ void AudioInjector::injectAudio() { // resize the QByteArray to the right size injectAudioPacket.resize(numPreAudioDataBytes + bytesToCopy); + + // pack the sequence number + memcpy(injectAudioPacket.data() + numPreSequenceNumberBytes, &outgoingSequenceNumber, sizeof(quint16)); // copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet memcpy(injectAudioPacket.data() + numPreAudioDataBytes, soundByteArray.data() + currentSendPosition, bytesToCopy); @@ -107,6 +116,7 @@ void AudioInjector::injectAudio() { // send off this audio packet nodeList->writeDatagram(injectAudioPacket, audioMixer); + outgoingSequenceNumber++; currentSendPosition += bytesToCopy; diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index d6b9624695..10aa6f47cf 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -66,6 +66,7 @@ void AudioRingBuffer::resizeForFrameSize(qint64 numFrameSamples) { } int AudioRingBuffer::parseData(const QByteArray& packet) { + // skip packet header and sequence number int numBytesBeforeAudioData = numBytesForPacketHeader(packet) + sizeof(quint16); return writeData(packet.data() + numBytesBeforeAudioData, packet.size() - numBytesBeforeAudioData); } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 1e26c708ca..e11d73358c 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -107,6 +107,9 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { // skip the packet header (includes the source UUID) int readBytes = numBytesForPacketHeader(packet); + + // skip the sequence number + readBytes += sizeof(quint16); // hop over the channel flag that has already been read in AudioMixerClientData readBytes += sizeof(quint8); diff --git a/libraries/networking/src/PacketHeaders.cpp b/libraries/networking/src/PacketHeaders.cpp index e2bc46b3be..a5c05a6ae9 100644 --- a/libraries/networking/src/PacketHeaders.cpp +++ b/libraries/networking/src/PacketHeaders.cpp @@ -50,6 +50,8 @@ PacketVersion versionForPacketType(PacketType type) { case PacketTypeMicrophoneAudioNoEcho: case PacketTypeMicrophoneAudioWithEcho: case PacketTypeSilentAudioFrame: + return 2; + case PacketTypeMixedAudio: return 1; case PacketTypeAvatarData: return 3; diff --git a/libraries/networking/src/SequenceNumbersStats.cpp b/libraries/networking/src/SequenceNumbersStats.cpp index b1b6adc9ac..eadf08e378 100644 --- a/libraries/networking/src/SequenceNumbersStats.cpp +++ b/libraries/networking/src/SequenceNumbersStats.cpp @@ -59,7 +59,7 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, const bool wa } } else if (absGap > MAX_REASONABLE_SEQUENCE_GAP) { // ignore packet if gap is unreasonable - qDebug() << "ignoring unreasonable packet... sequence:" << incoming + qDebug() << "ignoring unreasonable sequence number:" << incoming << "previous:" << _lastReceived; _numUnreasonable++; return; diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index b0cce114a9..b142819260 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -459,32 +459,55 @@ void ScriptEngine::run() { } } - QByteArray audioPacket = byteArrayWithPopulatedHeader(silentFrame - ? PacketTypeSilentAudioFrame - : PacketTypeMicrophoneAudioNoEcho); + char audioPacket[MAX_PACKET_SIZE]; - QDataStream packetStream(&audioPacket, QIODevice::Append); + // pack header + int numBytesPacketHeader = populatePacketHeader(audioPacket, silentFrame + ? PacketTypeSilentAudioFrame + : PacketTypeMicrophoneAudioNoEcho); + char* dataAt = audioPacket + numBytesPacketHeader; + + // skip over sequence number for now; will be packed when destination node is known + char* sequenceAt = dataAt; + dataAt += sizeof(quint16); // use the orientation and position of this avatar for the source of this audio - packetStream.writeRawData(reinterpret_cast(&_avatarData->getPosition()), sizeof(glm::vec3)); + memcpy(dataAt, &_avatarData->getPosition(), sizeof(glm::vec3)); + dataAt += sizeof(glm::vec3); + glm::quat headOrientation = _avatarData->getHeadOrientation(); - packetStream.writeRawData(reinterpret_cast(&headOrientation), sizeof(glm::quat)); + memcpy(dataAt, &headOrientation, sizeof(glm::quat)); + dataAt += sizeof(glm::quat); if (silentFrame) { if (!_isListeningToAudioStream) { // if we have a silent frame and we're not listening then just send nothing and break out of here break; } - // write the number of silent samples so the audio-mixer can uphold timing - packetStream.writeRawData(reinterpret_cast(&SCRIPT_AUDIO_BUFFER_SAMPLES), sizeof(int16_t)); + memcpy(dataAt, &SCRIPT_AUDIO_BUFFER_SAMPLES, sizeof(int16_t)); + dataAt += sizeof(int16_t); } else if (nextSoundOutput) { // write the raw audio data - packetStream.writeRawData(reinterpret_cast(nextSoundOutput), - numAvailableSamples * sizeof(int16_t)); + int numAvailableBytes = numAvailableSamples * sizeof(int16_t); + memcpy(dataAt, nextSoundOutput, numAvailableBytes); + dataAt += numAvailableBytes; } - nodeList->broadcastToNodes(audioPacket, NodeSet() << NodeType::AudioMixer); + // write audio packet to AudioMixer nodes + int audioPacketSize = dataAt - audioPacket; + NodeList* nodeList = NodeList::getInstance(); + foreach(const SharedNodePointer& node, nodeList->getNodeHash()) { + // only send to nodes of type AudioMixer + if (node->getType() == NodeType::AudioMixer) { + // pack sequence number + quint16 sequence = _outgoingSequenceNumbers[node->getUUID()]++; + memcpy(sequenceAt, &sequence, sizeof(quint16)); + + // send audio packet + nodeList->writeDatagram(audioPacket, audioPacketSize, node); + } + } } } @@ -658,3 +681,7 @@ void ScriptEngine::include(const QString& includeFile) { emit errorMessage("Uncaught exception at (" + includeFile + ") line" + QString::number(line) + ":" + result.toString()); } } + +void ScriptEngine::nodeKilled(SharedNodePointer node) { + _outgoingSequenceNumbers.remove(node->getUUID()); +} diff --git a/libraries/script-engine/src/ScriptEngine.h b/libraries/script-engine/src/ScriptEngine.h index bf2ac40568..f1aaf6f0bb 100644 --- a/libraries/script-engine/src/ScriptEngine.h +++ b/libraries/script-engine/src/ScriptEngine.h @@ -100,6 +100,8 @@ public slots: void include(const QString& includeFile); void print(const QString& message); + void nodeKilled(SharedNodePointer node); + signals: void update(float deltaTime); void scriptEnding(); @@ -146,6 +148,7 @@ private: ScriptUUID _uuidLibrary; AnimationCache _animationCache; + QHash _outgoingSequenceNumbers; }; #endif // hifi_ScriptEngine_h