From c9b6879ca82fef076fa0844bd6863b1ca8963f74 Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 24 Jul 2014 10:48:27 -0700 Subject: [PATCH] audiomixer code complete; need to test --- assignment-client/src/audio/AudioMixer.cpp | 51 +++-- .../src/audio/AudioMixerClientData.cpp | 189 ++++++----------- .../src/audio/AudioMixerClientData.h | 20 +- .../src/audio/AvatarAudioRingBuffer.cpp | 71 +++---- .../src/audio/AvatarAudioRingBuffer.h | 6 +- libraries/audio/src/AudioRingBuffer.cpp | 15 ++ libraries/audio/src/AudioRingBuffer.h | 82 ++++++++ libraries/audio/src/InboundAudioStream.cpp | 89 ++++++-- libraries/audio/src/InboundAudioStream.h | 7 +- .../audio/src/InjectedAudioRingBuffer.cpp | 54 +++-- libraries/audio/src/InjectedAudioRingBuffer.h | 16 +- .../audio/src/PositionalAudioRingBuffer.cpp | 190 +++--------------- .../audio/src/PositionalAudioRingBuffer.h | 89 ++------ 13 files changed, 394 insertions(+), 485 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index ad4787b407..1a436fc9bf 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -95,6 +95,19 @@ const float ATTENUATION_EPSILON_DISTANCE = 0.1f; void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuffer* bufferToAdd, AvatarAudioRingBuffer* listeningNodeBuffer) { + // if the frame to be mixed is silent, don't mix it + if (bufferToAdd->getNextOutputTrailingLoudness() == 0.0f) { + bufferToAdd->popFrames(1); + return; + } + + // get pointer to frame to be mixed. If the stream cannot provide a frame (is starved), bail + AudioRingBuffer::ConstIterator nextOutputStart; + if (!bufferToAdd->popFrames(&nextOutputStart, 1)) { + return; + } + + float bearingRelativeAngleToSource = 0.0f; float attenuationCoefficient = 1.0f; int numSamplesDelay = 0; @@ -203,7 +216,7 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf } } - const int16_t* nextOutputStart = bufferToAdd->getNextOutput(); + if (!bufferToAdd->isStereo() && shouldAttenuate) { // this is a mono buffer, which means it gets full attenuation and spatialization @@ -212,8 +225,8 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf int delayedChannelOffset = (bearingRelativeAngleToSource > 0.0f) ? 1 : 0; int goodChannelOffset = delayedChannelOffset == 0 ? 1 : 0; - const int16_t* bufferStart = bufferToAdd->getBuffer(); - int ringBufferSampleCapacity = bufferToAdd->getSampleCapacity(); + //const int16_t* bufferStart = bufferToAdd->getBuffer(); + //int ringBufferSampleCapacity = bufferToAdd->getSampleCapacity(); int16_t correctBufferSample[2], delayBufferSample[2]; int delayedChannelIndex = 0; @@ -241,14 +254,15 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf // if there was a sample delay for this buffer, we need to pull samples prior to the nextOutput // to stick at the beginning float attenuationAndWeakChannelRatio = attenuationCoefficient * weakChannelAmplitudeRatio; - const int16_t* delayNextOutputStart = nextOutputStart - numSamplesDelay; - if (delayNextOutputStart < bufferStart) { - delayNextOutputStart = bufferStart + ringBufferSampleCapacity - numSamplesDelay; - } + AudioRingBuffer::ConstIterator delayNextOutputStart = nextOutputStart - numSamplesDelay; + //if (delayNextOutputStart < bufferStart) { + //delayNextOutputStart = bufferStart + ringBufferSampleCapacity - numSamplesDelay; + //} for (int i = 0; i < numSamplesDelay; i++) { int parentIndex = i * 2; - _clientSamples[parentIndex + delayedChannelOffset] += delayNextOutputStart[i] * attenuationAndWeakChannelRatio; + _clientSamples[parentIndex + delayedChannelOffset] += *delayNextOutputStart * attenuationAndWeakChannelRatio; + ++delayNextOutputStart; } } } else { @@ -293,13 +307,13 @@ void AudioMixer::prepareMixForListeningNode(Node* node) { AudioMixerClientData* otherNodeClientData = (AudioMixerClientData*) otherNode->getLinkedData(); // enumerate the ARBs attached to the otherNode and add all that should be added to mix - for (int i = 0; i < otherNodeClientData->getRingBuffers().size(); i++) { - PositionalAudioRingBuffer* otherNodeBuffer = otherNodeClientData->getRingBuffers()[i]; + + const QHash& otherNodeRingBuffers = otherNodeClientData->getRingBuffers(); + QHash::ConstIterator i, end = otherNodeRingBuffers.constEnd(); + for (i = otherNodeRingBuffers.begin(); i != end; i++) { + PositionalAudioRingBuffer* otherNodeBuffer = i.value(); - if ((*otherNode != *node - || otherNodeBuffer->shouldLoopbackForNode()) - && otherNodeBuffer->willBeAddedToMix() - && otherNodeBuffer->getNextOutputTrailingLoudness() > 0.0f) { + if (*otherNode != *node || otherNodeBuffer->shouldLoopbackForNode()) { addBufferToMixForListeningNodeWithBuffer(otherNodeBuffer, nodeRingBuffer); } } @@ -307,7 +321,6 @@ void AudioMixer::prepareMixForListeningNode(Node* node) { } } - void AudioMixer::readPendingDatagrams() { QByteArray receivedPacket; HifiSockAddr senderSockAddr; @@ -500,12 +513,12 @@ void AudioMixer::run() { while (!_isFinished) { - foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { + /*foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { if (node->getLinkedData()) { ((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(_sourceUnattenuatedZone, _listenerUnattenuatedZone); } - } + }*/ const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; @@ -599,13 +612,13 @@ void AudioMixer::run() { ++_sumListeners; } } - + /* // push forward the next output pointers for any audio buffers we used foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { if (node->getLinkedData()) { ((AudioMixerClientData*) node->getLinkedData())->pushBuffersAfterFrameSend(); } - } + }*/ ++_numStatFrames; diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index a4983e6a95..31d0612a98 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -18,34 +18,28 @@ #include "AudioMixer.h" #include "AudioMixerClientData.h" -#include "MovingMinMaxAvg.h" -const int INCOMING_SEQ_STATS_HISTORY_LENGTH = INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS / - (TOO_LONG_SINCE_LAST_SEND_AUDIO_STREAM_STATS / USECS_PER_SECOND); AudioMixerClientData::AudioMixerClientData() : _ringBuffers(), - _outgoingMixedAudioSequenceNumber(0), - _incomingAvatarAudioSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH) + _outgoingMixedAudioSequenceNumber(0) { - } AudioMixerClientData::~AudioMixerClientData() { - for (int i = 0; i < _ringBuffers.size(); i++) { - // delete this attached PositionalAudioRingBuffer - delete _ringBuffers[i]; + QHash::ConstIterator i, end = _ringBuffers.constEnd(); + for (i = _ringBuffers.begin(); i != end; i++) { + // delete this attached InboundAudioStream + delete i.value(); } } AvatarAudioRingBuffer* AudioMixerClientData::getAvatarAudioRingBuffer() const { - for (int i = 0; i < _ringBuffers.size(); i++) { - if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Microphone) { - return (AvatarAudioRingBuffer*) _ringBuffers[i]; - } + if (_ringBuffers.contains(QUuid())) { + return (AvatarAudioRingBuffer*)_ringBuffers.value(QUuid()); } - // no AvatarAudioRingBuffer found - return NULL + // no mic stream found - return NULL return NULL; } @@ -57,96 +51,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { quint16 sequence = *(reinterpret_cast(sequenceAt)); PacketType packetType = packetTypeForPacket(packet); - if (packetType == PacketTypeMicrophoneAudioWithEcho - || packetType == PacketTypeMicrophoneAudioNoEcho - || packetType == PacketTypeSilentAudioFrame) { - - SequenceNumberStats::ArrivalInfo packetArrivalInfo = _incomingAvatarAudioSequenceNumberStats.sequenceNumberReceived(sequence); - - // grab the AvatarAudioRingBuffer from the vector (or create it if it doesn't exist) - AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); - - // read the first byte after the header to see if this is a stereo or mono buffer - quint8 channelFlag = packet.at(numBytesForPacketHeader(packet) + sizeof(quint16)); - bool isStereo = channelFlag == 1; - - if (avatarRingBuffer && avatarRingBuffer->isStereo() != isStereo) { - // there's a mismatch in the buffer channels for the incoming and current buffer - // so delete our current buffer and create a new one - _ringBuffers.removeOne(avatarRingBuffer); - avatarRingBuffer->deleteLater(); - avatarRingBuffer = NULL; - } - - if (!avatarRingBuffer) { - // we don't have an AvatarAudioRingBuffer yet, so add it - avatarRingBuffer = new AvatarAudioRingBuffer(isStereo, AudioMixer::getUseDynamicJitterBuffers()); - _ringBuffers.push_back(avatarRingBuffer); - } - - - // for now, late packets are simply discarded. In the future, it may be good to insert them into their correct place - // in the ring buffer (if that frame hasn't been mixed yet) - switch (packetArrivalInfo._status) { - case SequenceNumberStats::Early: { - int packetsLost = packetArrivalInfo._seqDiffFromExpected; - avatarRingBuffer->parseDataAndHandleDroppedPackets(packet, packetsLost); - break; - } - case SequenceNumberStats::OnTime: { - // ask the AvatarAudioRingBuffer instance to parse the data - avatarRingBuffer->parseDataAndHandleDroppedPackets(packet, 0); - break; - } - default: { - break; - } - } - } else if (packetType == PacketTypeInjectAudio) { - // this is injected audio - - // grab the stream identifier for this injected audio - QUuid streamIdentifier = QUuid::fromRfc4122(packet.mid(numBytesForPacketHeader(packet) + sizeof(quint16), NUM_BYTES_RFC4122_UUID)); - - if (!_incomingInjectedAudioSequenceNumberStatsMap.contains(streamIdentifier)) { - _incomingInjectedAudioSequenceNumberStatsMap.insert(streamIdentifier, SequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH)); - } - SequenceNumberStats::ArrivalInfo packetArrivalInfo = - _incomingInjectedAudioSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence); - - InjectedAudioRingBuffer* matchingInjectedRingBuffer = NULL; - - for (int i = 0; i < _ringBuffers.size(); i++) { - if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector - && ((InjectedAudioRingBuffer*) _ringBuffers[i])->getStreamIdentifier() == streamIdentifier) { - matchingInjectedRingBuffer = (InjectedAudioRingBuffer*) _ringBuffers[i]; - } - } - - if (!matchingInjectedRingBuffer) { - // we don't have a matching injected audio ring buffer, so add it - matchingInjectedRingBuffer = new InjectedAudioRingBuffer(streamIdentifier, AudioMixer::getUseDynamicJitterBuffers()); - _ringBuffers.push_back(matchingInjectedRingBuffer); - } - - // for now, late packets are simply discarded. In the future, it may be good to insert them into their correct place - // in the ring buffer (if that frame hasn't been mixed yet) - switch (packetArrivalInfo._status) { - case SequenceNumberStats::Early: { - int packetsLost = packetArrivalInfo._seqDiffFromExpected; - matchingInjectedRingBuffer->parseDataAndHandleDroppedPackets(packet, packetsLost); - break; - } - case SequenceNumberStats::OnTime: { - // ask the AvatarAudioRingBuffer instance to parse the data - matchingInjectedRingBuffer->parseDataAndHandleDroppedPackets(packet, 0); - break; - } - default: { - break; - } - } - } else if (packetType == PacketTypeAudioStreamStats) { + if (packetType == PacketTypeAudioStreamStats) { const char* dataAt = packet.data(); @@ -155,12 +60,52 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { // read the downstream audio stream stats memcpy(&_downstreamAudioStreamStats, dataAt, sizeof(AudioStreamStats)); - } + dataAt += sizeof(AudioStreamStats); + return dataAt - packet.data(); + + } else { + PositionalAudioRingBuffer* matchingStream = NULL; + + if (packetType == PacketTypeMicrophoneAudioWithEcho + || packetType == PacketTypeMicrophoneAudioNoEcho + || packetType == PacketTypeSilentAudioFrame) { + + QUuid nullUUID = QUuid(); + if (!_ringBuffers.contains(nullUUID)) { + // we don't have a mic stream yet, so add it + + // read the channel flag to see if our stream is stereo or not + const char* channelFlagAt = packet.constData() + numBytesForPacketHeader(packet) + sizeof(quint16); + quint8 channelFlag = *(reinterpret_cast(channelFlagAt)); + bool isStereo = channelFlag == 1; + + _ringBuffers.insert(nullUUID, + matchingStream = new AvatarAudioRingBuffer(isStereo, AudioMixer::getUseDynamicJitterBuffers())); + } else { + matchingStream = _ringBuffers.value(nullUUID); + } + } else if (packetType == PacketTypeInjectAudio) { + // this is injected audio + + // grab the stream identifier for this injected audio + int bytesBeforeStreamIdentifier = numBytesForPacketHeader(packet) + sizeof(quint16); + QUuid streamIdentifier = QUuid::fromRfc4122(packet.mid(bytesBeforeStreamIdentifier, NUM_BYTES_RFC4122_UUID)); + + if (!_ringBuffers.contains(streamIdentifier)) { + _ringBuffers.insert(streamIdentifier, + matchingStream = new InjectedAudioRingBuffer(streamIdentifier, AudioMixer::getUseDynamicJitterBuffers())); + } else { + matchingStream = _ringBuffers.value(streamIdentifier); + } + } + + return matchingStream->parseData(packet); + } return 0; } -void AudioMixerClientData::checkBuffersBeforeFrameSend(AABox* checkSourceZone, AABox* listenerZone) { +/*void AudioMixerClientData::checkBuffersBeforeFrameSend(AABox* checkSourceZone, AABox* listenerZone) { for (int i = 0; i < _ringBuffers.size(); i++) { if (_ringBuffers[i]->shouldBeAddedToMix()) { // this is a ring buffer that is ready to go @@ -205,9 +150,9 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { } i++; } -} +}*/ -AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer) const { +/*AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer) const { AudioStreamStats streamStats; @@ -239,20 +184,9 @@ AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const Positio streamStats._ringBufferSilentFramesDropped = ringBuffer->getSilentFramesDropped(); return streamStats; -} +}*/ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode) { - - // have all the seq number stats of each audio stream push their current stats into their history, - // which moves that history window 1 second forward (since that's how long since the last stats were pushed into history) - _incomingAvatarAudioSequenceNumberStats.pushStatsToHistory(); - QHash::Iterator i = _incomingInjectedAudioSequenceNumberStatsMap.begin(); - QHash::Iterator end = _incomingInjectedAudioSequenceNumberStatsMap.end(); - while (i != end) { - i.value().pushStatsToHistory(); - i++; - } - char packet[MAX_PACKET_SIZE]; NodeList* nodeList = NodeList::getInstance(); @@ -271,7 +205,7 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& // pack and send stream stats packets until all ring buffers' stats are sent int numStreamStatsRemaining = _ringBuffers.size(); - QList::ConstIterator ringBuffersIterator = _ringBuffers.constBegin(); + QHash::ConstIterator ringBuffersIterator = _ringBuffers.constBegin(); while (numStreamStatsRemaining > 0) { char* dataAt = headerEndAt; @@ -288,7 +222,7 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& // pack the calculated number of stream stats for (int i = 0; i < numStreamStatsToPack; i++) { - AudioStreamStats streamStats = getAudioStreamStatsOfStream(*ringBuffersIterator); + AudioStreamStats streamStats = ringBuffersIterator.value()->getAudioStreamStats(); memcpy(dataAt, &streamStats, sizeof(AudioStreamStats)); dataAt += sizeof(AudioStreamStats); @@ -322,7 +256,7 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); if (avatarRingBuffer) { - AudioStreamStats streamStats = getAudioStreamStatsOfStream(avatarRingBuffer); + AudioStreamStats streamStats = avatarRingBuffer->getAudioStreamStats(); result += " UPSTREAM.mic.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) + " desired_calc:" + QString::number(avatarRingBuffer->getCalculatedDesiredJitterBufferFrames()) + " available_avg_10s:" + QString::number(streamStats._ringBufferFramesAvailableAverage) @@ -343,11 +277,12 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { result = "mic unknown"; } - for (int i = 0; i < _ringBuffers.size(); i++) { - if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) { - AudioStreamStats streamStats = getAudioStreamStatsOfStream(_ringBuffers[i]); + QHash::ConstIterator i, end = _ringBuffers.end(); + for (i = _ringBuffers.begin(); i != end; i++) { + if (i.value()->getType() == PositionalAudioRingBuffer::Injector) { + AudioStreamStats streamStats = i.value()->getAudioStreamStats(); result += " UPSTREAM.inj.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) - + " desired_calc:" + QString::number(_ringBuffers[i]->getCalculatedDesiredJitterBufferFrames()) + + " desired_calc:" + QString::number(i.value()->getCalculatedDesiredJitterBufferFrames()) + " available_avg_10s:" + QString::number(streamStats._ringBufferFramesAvailableAverage) + " available:" + QString::number(streamStats._ringBufferFramesAvailable) + " starves:" + QString::number(streamStats._ringBufferStarveCount) diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 7475c0a60e..4baa7c2f3b 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -13,29 +13,23 @@ #define hifi_AudioMixerClientData_h #include -#include -#include +#include "PositionalAudioRingBuffer.h" #include "AvatarAudioRingBuffer.h" -#include "AudioStreamStats.h" -#include "SequenceNumberStats.h" - - -const int INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS = 30; class AudioMixerClientData : public NodeData { public: AudioMixerClientData(); ~AudioMixerClientData(); - const QList getRingBuffers() const { return _ringBuffers; } + const QHash& getRingBuffers() const { return _ringBuffers; } AvatarAudioRingBuffer* getAvatarAudioRingBuffer() const; int parseData(const QByteArray& packet); - void checkBuffersBeforeFrameSend(AABox* checkSourceZone = NULL, AABox* listenerZone = NULL); - void pushBuffersAfterFrameSend(); + //void checkBuffersBeforeFrameSend(AABox* checkSourceZone = NULL, AABox* listenerZone = NULL); + //void pushBuffersAfterFrameSend(); - AudioStreamStats getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer) const; + //AudioStreamStats getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer) const; QString getAudioStreamStatsString() const; void sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode); @@ -44,11 +38,9 @@ public: quint16 getOutgoingSequenceNumber() const { return _outgoingMixedAudioSequenceNumber; } private: - QList _ringBuffers; + QHash _ringBuffers; // mic stream stored under key of null UUID quint16 _outgoingMixedAudioSequenceNumber; - SequenceNumberStats _incomingAvatarAudioSequenceNumberStats; - QHash _incomingInjectedAudioSequenceNumberStatsMap; AudioStreamStats _downstreamAudioStreamStats; }; diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp index 382e8de68b..94a95ef177 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp @@ -14,57 +14,50 @@ #include "AvatarAudioRingBuffer.h" AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo, bool dynamicJitterBuffer) : - PositionalAudioRingBuffer(PositionalAudioRingBuffer::Microphone, isStereo, dynamicJitterBuffer) { - +PositionalAudioRingBuffer(PositionalAudioRingBuffer::Microphone, isStereo, dynamicJitterBuffer) { + } -int AvatarAudioRingBuffer::parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped) { - frameReceivedUpdateTimingStats(); +int AvatarAudioRingBuffer::parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) { - _shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho); + _shouldLoopbackForNode = (type == PacketTypeMicrophoneAudioWithEcho); - // skip the packet header (includes the source UUID) - int readBytes = numBytesForPacketHeader(packet); + int readBytes = 0; - // skip the sequence number - readBytes += sizeof(quint16); - - // hop over the channel flag that has already been read in AudioMixerClientData + // read the channel flag + quint8 channelFlag = packetAfterSeqNum.at(readBytes); + bool isStereo = channelFlag == 1; readBytes += sizeof(quint8); + + // if isStereo value has changed, restart the ring buffer with new frame size + if (isStereo != _isStereo) { + _ringBuffer.resizeForFrameSize(isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); + _isStereo = isStereo; + } + // read the positional data - readBytes += parsePositionalData(packet.mid(readBytes)); + readBytes += parsePositionalData(packetAfterSeqNum.mid(readBytes)); - if (packetTypeForPacket(packet) == PacketTypeSilentAudioFrame) { - // this source had no audio to send us, but this counts as a packet - // write silence equivalent to the number of silent samples they just sent us + if (type == PacketTypeSilentAudioFrame) { int16_t numSilentSamples; - - memcpy(&numSilentSamples, packet.data() + readBytes, sizeof(int16_t)); + memcpy(&numSilentSamples, packetAfterSeqNum.data() + readBytes, sizeof(int16_t)); readBytes += sizeof(int16_t); - // add silent samples for the dropped packets as well. - // ASSUME that each dropped packet had same number of silent samples as this one - numSilentSamples *= (packetsSkipped + 1); - - // NOTE: fixes a bug in old clients that would send garbage for their number of silentSamples - // CAN'T DO THIS because ScriptEngine.cpp sends frames of different size due to having a different sending interval - // (every 16.667ms) than Audio.cpp (every 10.667ms) - //numSilentSamples = getSamplesPerFrame(); - - addDroppableSilentSamples(numSilentSamples); - + numAudioSamples = numSilentSamples; } else { - int numAudioBytes = packet.size() - readBytes; - int numAudioSamples = numAudioBytes / sizeof(int16_t); - - // add silent samples for the dropped packets. - // ASSUME that each dropped packet had same number of samples as this one - if (packetsSkipped > 0) { - addDroppableSilentSamples(packetsSkipped * numAudioSamples); - } - - // there is audio data to read - readBytes += writeData(packet.data() + readBytes, numAudioBytes); + int numAudioBytes = packetAfterSeqNum.size() - readBytes; + numAudioSamples = numAudioBytes / sizeof(int16_t); + } + return readBytes; +} + +int AvatarAudioRingBuffer::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) { + int readBytes = 0; + if (type == PacketTypeSilentAudioFrame) { + writeDroppableSilentSamples(numAudioSamples); + } else { + // there is audio data to read + readBytes += _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t)); } return readBytes; } diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.h b/assignment-client/src/audio/AvatarAudioRingBuffer.h index 96233220cd..d846748aff 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.h +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.h @@ -19,12 +19,14 @@ class AvatarAudioRingBuffer : public PositionalAudioRingBuffer { public: AvatarAudioRingBuffer(bool isStereo = false, bool dynamicJitterBuffer = false); - - int parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped); + private: // disallow copying of AvatarAudioRingBuffer objects AvatarAudioRingBuffer(const AvatarAudioRingBuffer&); AvatarAudioRingBuffer& operator= (const AvatarAudioRingBuffer&); + + int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples); + int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples); }; #endif // hifi_AvatarAudioRingBuffer_h diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 0defa2ea33..8b289d7c52 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -231,3 +231,18 @@ int16_t* AudioRingBuffer::shiftedPositionAccomodatingWrap(int16_t* position, int return position + numSamplesShift; } } + +float AudioRingBuffer::getNextOutputFrameLoudness() const { + float loudness = 0.0f; + int16_t* sampleAt = _nextOutput; + int16_t* _bufferLastAt = _buffer + _sampleCapacity - 1; + if (samplesAvailable() >= _numFrameSamples) { + for (int i = 0; i < _numFrameSamples; ++i) { + loudness += fabsf(*sampleAt); + sampleAt = sampleAt == _bufferLastAt ? _buffer : sampleAt + 1; + } + loudness /= _numFrameSamples; + loudness /= MAX_SAMPLE_VALUE; + } + return loudness; +} diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 0ec9213db2..97ffa7e6c8 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -65,6 +65,8 @@ public: const int16_t& operator[] (const int index) const; void shiftReadPosition(unsigned int numSamples); + + float getNextOutputFrameLoudness() const; int samplesAvailable() const; int framesAvailable() const { return samplesAvailable() / _numFrameSamples; } @@ -99,6 +101,86 @@ protected: bool _isStarved; bool _hasStarted; bool _randomAccessMode; /// will this ringbuffer be used for random access? if so, do some special processing + +public: + class ConstIterator { //public std::iterator < std::forward_iterator_tag, int16_t > { + public: + ConstIterator() + : _capacity(0), + _bufferFirst(NULL), + _bufferLast(NULL), + _at(NULL) {} + + ConstIterator(int16_t* bufferFirst, int capacity, int16_t* at) + : _capacity(capacity), + _bufferFirst(bufferFirst), + _bufferLast(bufferFirst + capacity - 1), + _at(at) {} + + bool operator==(const ConstIterator& rhs) { return _at == rhs._at; } + bool operator!=(const ConstIterator& rhs) { return _at != rhs._at; } + int16_t operator*() { return *_at; } + + ConstIterator& operator=(const ConstIterator& rhs) { + _capacity = rhs._capacity; + _bufferFirst = rhs._bufferFirst; + _bufferLast = rhs._bufferLast; + _at = rhs._at; + return *this; + } + + ConstIterator& operator++() { + _at = (_at == _bufferLast) ? _bufferFirst : _at + 1; + return *this; + } + + ConstIterator operator++(int) { + ConstIterator tmp(*this); + ++(*this); + return tmp; + } + + ConstIterator& operator--() { + _at = (_at == _bufferFirst) ? _bufferLast : _at - 1; + return *this; + } + + ConstIterator operator--(int) { + ConstIterator tmp(*this); + --(*this); + return tmp; + } + + int16_t operator[] (int i) { + return *atShiftedBy(i); + } + + ConstIterator operator+(int i) { + return ConstIterator(_bufferFirst, _capacity, atShiftedBy(i)); + } + + ConstIterator operator-(int i) { + return ConstIterator(_bufferFirst, _capacity, atShiftedBy(-i)); + } + + private: + int16_t* atShiftedBy(int i) { + i = (_at - _bufferFirst + i) % _capacity; + if (i < 0) { + i += _capacity; + } + return _bufferFirst + i; + } + + private: + int _capacity; + int16_t* _bufferFirst; + int16_t* _bufferLast; + int16_t* _at; + }; + + + ConstIterator nextOutput() const { return ConstIterator(_buffer, _sampleCapacity, _nextOutput); } }; #endif // hifi_AudioRingBuffer_h diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index ad7830655d..727cb5c554 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -13,19 +13,19 @@ #include "PacketHeaders.h" InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacity, bool dynamicJitterBuffers) : -_ringBuffer(numFrameSamples, false, numFramesCapacity), -_dynamicJitterBuffers(dynamicJitterBuffers), -_desiredJitterBufferFrames(1), -_isStarved(true), -_hasStarted(false), -_consecutiveNotMixedCount(0), -_starveCount(0), -_silentFramesDropped(0), -_incomingSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS), -_lastFrameReceivedTime(0), -_interframeTimeGapStatsForJitterCalc(TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES, TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS), -_interframeTimeGapStatsForStatsPacket(TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES, TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS), -_framesAvailableStats(FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES, FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS) + _ringBuffer(numFrameSamples, false, numFramesCapacity), + _dynamicJitterBuffers(dynamicJitterBuffers), + _desiredJitterBufferFrames(1), + _isStarved(true), + _hasStarted(false), + _consecutiveNotMixedCount(0), + _starveCount(0), + _silentFramesDropped(0), + _incomingSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS), + _lastFrameReceivedTime(0), + _interframeTimeGapStatsForJitterCalc(TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES, TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS), + _interframeTimeGapStatsForStatsPacket(TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES, TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS), + _framesAvailableStats(FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES, FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS) { } @@ -94,19 +94,19 @@ int InboundAudioStream::parseData(const QByteArray& packet) { return readBytes; } -bool InboundAudioStream::popFrames(int16_t* dest, int numFrames, bool starveOnFail) { +bool InboundAudioStream::popFrames(int numFrames, bool starveOnFail) { if (_isStarved) { _consecutiveNotMixedCount++; return false; } - bool framesPopped = false; + bool popped = false; int numSamplesRequested = numFrames * _ringBuffer.getNumFrameSamples(); - if (_ringBuffer.samplesAvailable >= numSamplesRequested) { - _ringBuffer.readSamples(dest, numSamplesRequested); + if (_ringBuffer.samplesAvailable() >= numSamplesRequested) { + _ringBuffer.shiftReadPosition(numSamplesRequested); _hasStarted = true; - framesPopped = true; + popped = true; } else { if (starveOnFail) { setToStarved(); @@ -116,7 +116,58 @@ bool InboundAudioStream::popFrames(int16_t* dest, int numFrames, bool starveOnFa _framesAvailableStats.update(_ringBuffer.framesAvailable()); - return framesPopped; + return popped; +} + +bool InboundAudioStream::popFrames(int16_t* dest, int numFrames, bool starveOnFail) { + if (_isStarved) { + _consecutiveNotMixedCount++; + return false; + } + + bool popped = false; + + int numSamplesRequested = numFrames * _ringBuffer.getNumFrameSamples(); + if (_ringBuffer.samplesAvailable() >= numSamplesRequested) { + _ringBuffer.readSamples(dest, numSamplesRequested); + _hasStarted = true; + popped = true; + } else { + if (starveOnFail) { + setToStarved(); + _consecutiveNotMixedCount++; + } + } + + _framesAvailableStats.update(_ringBuffer.framesAvailable()); + + return popped; +} + +bool InboundAudioStream::popFrames(AudioRingBuffer::ConstIterator* nextOutput, int numFrames, bool starveOnFail) { + if (_isStarved) { + _consecutiveNotMixedCount++; + return false; + } + + bool popped = false; + + int numSamplesRequested = numFrames * _ringBuffer.getNumFrameSamples(); + if (_ringBuffer.samplesAvailable() >= numSamplesRequested) { + *nextOutput = _ringBuffer.nextOutput(); + _ringBuffer.shiftReadPosition(numSamplesRequested); + _hasStarted = true; + popped = true; + } else { + if (starveOnFail) { + setToStarved(); + _consecutiveNotMixedCount++; + } + } + + _framesAvailableStats.update(_ringBuffer.framesAvailable()); + + return popped; } void InboundAudioStream::setToStarved() { diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index 7b24f01077..65a4f07918 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -52,14 +52,15 @@ public: int parseData(const QByteArray& packet); - - bool popFrames(int16_t* dest, int numFrames, bool starveOnFail = true); + bool popFrames(int numFrames, bool starveOnFail = true); + bool popFrames(int16_t* dest, int numFrames, bool starveOnFail = true); + bool popFrames(AudioRingBuffer::ConstIterator* nextOutput, int numFrames, bool starveOnFail = true); void setToStarved(); - + /// this function should be called once per second to ensure the seq num stats history spans ~30 seconds AudioStreamStats updateSeqHistoryAndGetAudioStreamStats(); virtual AudioStreamStats getAudioStreamStats() const; diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index 4723bca906..e074d51bd9 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -20,52 +20,50 @@ #include "InjectedAudioRingBuffer.h" InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier, bool dynamicJitterBuffer) : - PositionalAudioRingBuffer(PositionalAudioRingBuffer::Injector, false, dynamicJitterBuffer), - _streamIdentifier(streamIdentifier), - _radius(0.0f), - _attenuationRatio(0) +PositionalAudioRingBuffer(PositionalAudioRingBuffer::Injector, false, dynamicJitterBuffer), +_streamIdentifier(streamIdentifier), +_radius(0.0f), +_attenuationRatio(0) { - + } const uchar MAX_INJECTOR_VOLUME = 255; -int InjectedAudioRingBuffer::parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped) { - frameReceivedUpdateTimingStats(); - +int InjectedAudioRingBuffer::parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) { // setup a data stream to read from this packet - QDataStream packetStream(packet); - packetStream.skipRawData(numBytesForPacketHeader(packet)); - - // push past the sequence number - packetStream.skipRawData(sizeof(quint16)); + QDataStream packetStream(packetAfterSeqNum); - // push past the stream identifier + // skip the stream identifier packetStream.skipRawData(NUM_BYTES_RFC4122_UUID); - + // pull the loopback flag and set our boolean uchar shouldLoopback; packetStream >> shouldLoopback; _shouldLoopbackForNode = (shouldLoopback == 1); - + // use parsePositionalData in parent PostionalAudioRingBuffer class to pull common positional data - packetStream.skipRawData(parsePositionalData(packet.mid(packetStream.device()->pos()))); - + packetStream.skipRawData(parsePositionalData(packetAfterSeqNum.mid(packetStream.device()->pos()))); + // pull out the radius for this injected source - if it's zero this is a point source packetStream >> _radius; - + quint8 attenuationByte = 0; packetStream >> attenuationByte; - _attenuationRatio = attenuationByte / (float) MAX_INJECTOR_VOLUME; - - int numAudioBytes = packet.size() - packetStream.device()->pos(); - int numAudioSamples = numAudioBytes / sizeof(int16_t); + _attenuationRatio = attenuationByte / (float)MAX_INJECTOR_VOLUME; - // add silent samples for the dropped packets. - // ASSUME that each dropped packet had same number of samples as this one - addDroppableSilentSamples(numAudioSamples * packetsSkipped); + int numAudioBytes = packetAfterSeqNum.size() - packetStream.device()->pos(); + numAudioSamples = numAudioBytes / sizeof(int16_t); - packetStream.skipRawData(writeData(packet.data() + packetStream.device()->pos(), numAudioBytes)); - return packetStream.device()->pos(); } + +int InjectedAudioRingBuffer::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) { + return _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t)); +} + +AudioStreamStats InjectedAudioRingBuffer::getAudioStreamStats() const { + AudioStreamStats streamStats = PositionalAudioRingBuffer::getAudioStreamStats(); + streamStats._streamIdentifier = _streamIdentifier; + return streamStats; +} diff --git a/libraries/audio/src/InjectedAudioRingBuffer.h b/libraries/audio/src/InjectedAudioRingBuffer.h index 4a1f8b5292..0f7c621baa 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.h +++ b/libraries/audio/src/InjectedAudioRingBuffer.h @@ -19,18 +19,22 @@ class InjectedAudioRingBuffer : public PositionalAudioRingBuffer { public: InjectedAudioRingBuffer(const QUuid& streamIdentifier = QUuid(), bool dynamicJitterBuffer = false); - - int parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped); - - const QUuid& getStreamIdentifier() const { return _streamIdentifier; } + float getRadius() const { return _radius; } float getAttenuationRatio() const { return _attenuationRatio; } + + QUuid getStreamIdentifier() const { return _streamIdentifier; } + private: // disallow copying of InjectedAudioRingBuffer objects InjectedAudioRingBuffer(const InjectedAudioRingBuffer&); InjectedAudioRingBuffer& operator= (const InjectedAudioRingBuffer&); - - QUuid _streamIdentifier; + + AudioStreamStats getAudioStreamStats() const; + int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples); + int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples); + + const QUuid _streamIdentifier; float _radius; float _attenuationRatio; }; diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 14af8c6a4a..519190d70d 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -9,6 +9,9 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +#include "PositionalAudioRingBuffer.h" +#include "SharedUtil.h" + #include #include @@ -18,66 +21,27 @@ #include #include -#include "PositionalAudioRingBuffer.h" -#include "SharedUtil.h" - PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo, bool dynamicJitterBuffers) : - - AudioRingBuffer(isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, - false, AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY), - _type(type), - _position(0.0f, 0.0f, 0.0f), - _orientation(0.0f, 0.0f, 0.0f, 0.0f), - _willBeAddedToMix(false), - _shouldLoopbackForNode(false), - _shouldOutputStarveDebug(true), - _isStereo(isStereo), - _nextOutputTrailingLoudness(0.0f), - _listenerUnattenuatedZone(NULL), - _lastFrameReceivedTime(0), - _interframeTimeGapStatsForJitterCalc(TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES, TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS), - _interframeTimeGapStatsForStatsPacket(TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES, TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS), - _framesAvailableStats(FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES, FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS), - _desiredJitterBufferFrames(1), - _dynamicJitterBuffers(dynamicJitterBuffers), - _consecutiveNotMixedCount(0), - _starveCount(0), - _silentFramesDropped(0) +InboundAudioStream(isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, +AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY, dynamicJitterBuffers), +_type(type), +_position(0.0f, 0.0f, 0.0f), +_orientation(0.0f, 0.0f, 0.0f, 0.0f), +_shouldLoopbackForNode(false), +_isStereo(isStereo), +_nextOutputTrailingLoudness(0.0f), +_listenerUnattenuatedZone(NULL) { } -int PositionalAudioRingBuffer::parsePositionalData(const QByteArray& positionalByteArray) { - QDataStream packetStream(positionalByteArray); - - packetStream.readRawData(reinterpret_cast(&_position), sizeof(_position)); - packetStream.readRawData(reinterpret_cast(&_orientation), sizeof(_orientation)); - - // if this node sent us a NaN for first float in orientation then don't consider this good audio and bail - if (glm::isnan(_orientation.x)) { - reset(); - return 0; - } - - return packetStream.device()->pos(); -} - void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() { - // ForBoundarySamples means that we expect the number of samples not to roll of the end of the ring buffer - float nextLoudness = 0; - - if (samplesAvailable() >= _numFrameSamples) { - for (int i = 0; i < _numFrameSamples; ++i) { - nextLoudness += fabsf(_nextOutput[i]); - } - nextLoudness /= _numFrameSamples; - nextLoudness /= MAX_SAMPLE_VALUE; - } - + float nextLoudness = _ringBuffer.getNextOutputFrameLoudness(); + const int TRAILING_AVERAGE_FRAMES = 100; const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; const float LOUDNESS_EPSILON = 0.000001f; - + if (nextLoudness >= _nextOutputTrailingLoudness) { _nextOutputTrailingLoudness = nextLoudness; } else { @@ -89,120 +53,24 @@ void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() { } } -bool PositionalAudioRingBuffer::shouldBeAddedToMix() { - int desiredJitterBufferSamples = _desiredJitterBufferFrames * _numFrameSamples; - - if (!isNotStarvedOrHasMinimumSamples(_numFrameSamples + desiredJitterBufferSamples)) { - // if the buffer was starved, allow it to accrue at least the desired number of - // jitter buffer frames before we start taking frames from it for mixing +int PositionalAudioRingBuffer::parsePositionalData(const QByteArray& positionalByteArray) { + QDataStream packetStream(positionalByteArray); - if (_shouldOutputStarveDebug) { - _shouldOutputStarveDebug = false; - } + packetStream.readRawData(reinterpret_cast(&_position), sizeof(_position)); + packetStream.readRawData(reinterpret_cast(&_orientation), sizeof(_orientation)); - _consecutiveNotMixedCount++; - return false; - } else if (samplesAvailable() < _numFrameSamples) { - // if the buffer doesn't have a full frame of samples to take for mixing, it is starved - _isStarved = true; - _starveCount++; - - _framesAvailableStats.reset(); - - // reset our _shouldOutputStarveDebug to true so the next is printed - _shouldOutputStarveDebug = true; - - _consecutiveNotMixedCount = 1; - return false; + // if this node sent us a NaN for first float in orientation then don't consider this good audio and bail + if (glm::isnan(_orientation.x)) { + // NOTE: why would we reset the ring buffer here? + _ringBuffer.reset(); + return 0; } - - // good buffer, add this to the mix - // if we just finished refilling after a starve, we have a new jitter buffer length. - // reset the frames available stats. - - _isStarved = false; - - _framesAvailableStats.update(framesAvailable()); - - // since we've read data from ring buffer at least once - we've started - _hasStarted = true; - - return true; + return packetStream.device()->pos(); } -int PositionalAudioRingBuffer::getCalculatedDesiredJitterBufferFrames() const { - const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; - - int calculatedDesiredJitterBufferFrames = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME); - if (calculatedDesiredJitterBufferFrames < 1) { - calculatedDesiredJitterBufferFrames = 1; - } - return calculatedDesiredJitterBufferFrames; -} - - -void PositionalAudioRingBuffer::frameReceivedUpdateTimingStats() { - // update the two time gap stats we're keeping - quint64 now = usecTimestampNow(); - if (_lastFrameReceivedTime != 0) { - quint64 gap = now - _lastFrameReceivedTime; - _interframeTimeGapStatsForJitterCalc.update(gap); - _interframeTimeGapStatsForStatsPacket.update(gap); - } - _lastFrameReceivedTime = now; - - // recalculate the _desiredJitterBufferFrames if _interframeTimeGapStatsForJitterCalc has updated stats for us - if (_interframeTimeGapStatsForJitterCalc.getNewStatsAvailableFlag()) { - if (!_dynamicJitterBuffers) { - _desiredJitterBufferFrames = 1; // HACK to see if this fixes the audio silence - } else { - const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; - - _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME); - if (_desiredJitterBufferFrames < 1) { - _desiredJitterBufferFrames = 1; - } - const int maxDesired = _frameCapacity - 1; - if (_desiredJitterBufferFrames > maxDesired) { - _desiredJitterBufferFrames = maxDesired; - } - } - _interframeTimeGapStatsForJitterCalc.clearNewStatsAvailableFlag(); - } -} - -void PositionalAudioRingBuffer::addDroppableSilentSamples(int numSilentSamples) { - - // This adds some number of frames to the desired jitter buffer frames target we use. - // The larger this value is, the less aggressive we are about reducing the jitter buffer length. - // Setting this to 0 will try to get the jitter buffer to be exactly _desiredJitterBufferFrames long, - // which could lead immediately to a starve. - const int DESIRED_JITTER_BUFFER_FRAMES_PADDING = 1; - - // calculate how many silent frames we should drop. We only drop silent frames if - // the running avg num frames available has stabilized and it's more than - // our desired number of frames by the margin defined above. - int numSilentFramesToDrop = 0; - if (_framesAvailableStats.getNewStatsAvailableFlag() && _framesAvailableStats.isWindowFilled() - && numSilentSamples >= _numFrameSamples) { - _framesAvailableStats.clearNewStatsAvailableFlag(); - int averageJitterBufferFrames = (int)_framesAvailableStats.getWindowAverage(); - int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING; - - if (averageJitterBufferFrames > desiredJitterBufferFramesPlusPadding) { - // our avg jitter buffer size exceeds its desired value, so ignore some silent - // frames to get that size as close to desired as possible - int numSilentFramesToDropDesired = averageJitterBufferFrames - desiredJitterBufferFramesPlusPadding; - int numSilentFramesReceived = numSilentSamples / _numFrameSamples; - numSilentFramesToDrop = std::min(numSilentFramesToDropDesired, numSilentFramesReceived); - - // since we now have a new jitter buffer length, reset the frames available stats. - _framesAvailableStats.reset(); - - _silentFramesDropped += numSilentFramesToDrop; - } - } - - addSilentFrame(numSilentSamples - numSilentFramesToDrop * _numFrameSamples); +AudioStreamStats PositionalAudioRingBuffer::getAudioStreamStats() const { + AudioStreamStats streamStats = InboundAudioStream::getAudioStreamStats(); + streamStats._streamType = _type; + return streamStats; } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 0b14a12858..edc6266613 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -13,105 +13,60 @@ #define hifi_PositionalAudioRingBuffer_h #include - #include -#include "AudioRingBuffer.h" -#include "MovingMinMaxAvg.h" - -// the time gaps stats for _desiredJitterBufferFrames calculation -// will recalculate the max for the past 5000 samples every 500 samples -const int TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES = 500; -const int TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS = 10; - -// the time gap stats for constructing AudioStreamStats will -// recalculate min/max/avg every ~1 second for the past ~30 seconds of time gap data -const int TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS; -const int TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS = 30; - -// the stats for calculating the average frames available will recalculate every ~1 second -// and will include data for the past ~10 seconds -const int FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS; -const int FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS = 10; +#include "InboundAudioStream.h" const int AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY = 100; -class PositionalAudioRingBuffer : public AudioRingBuffer { +class PositionalAudioRingBuffer : public InboundAudioStream { + Q_OBJECT public: enum Type { Microphone, Injector }; - - PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo = false, bool dynamicJitterBuffers = false); - - virtual int parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped) = 0; - int parsePositionalData(const QByteArray& positionalByteArray); - int parseListenModeData(const QByteArray& listenModeByteArray); - + PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo = false, bool dynamicJitterBuffers = false); + + virtual AudioStreamStats getAudioStreamStats() const; + void updateNextOutputTrailingLoudness(); float getNextOutputTrailingLoudness() const { return _nextOutputTrailingLoudness; } - - bool shouldBeAddedToMix(); - - bool willBeAddedToMix() const { return _willBeAddedToMix; } - void setWillBeAddedToMix(bool willBeAddedToMix) { _willBeAddedToMix = willBeAddedToMix; } - + bool shouldLoopbackForNode() const { return _shouldLoopbackForNode; } - bool isStereo() const { return _isStereo; } - PositionalAudioRingBuffer::Type getType() const { return _type; } const glm::vec3& getPosition() const { return _position; } const glm::quat& getOrientation() const { return _orientation; } - AABox* getListenerUnattenuatedZone() const { return _listenerUnattenuatedZone; } + void setListenerUnattenuatedZone(AABox* listenerUnattenuatedZone) { _listenerUnattenuatedZone = listenerUnattenuatedZone; } - - int getSamplesPerFrame() const { return _isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; } - - const MovingMinMaxAvg& getInterframeTimeGapStatsForStatsPacket() const { return _interframeTimeGapStatsForStatsPacket; } - - int getCalculatedDesiredJitterBufferFrames() const; /// returns what we would calculate our desired as if asked - int getDesiredJitterBufferFrames() const { return _desiredJitterBufferFrames; } - double getFramesAvailableAverage() const { return _framesAvailableStats.getWindowAverage(); } - - int getConsecutiveNotMixedCount() const { return _consecutiveNotMixedCount; } - int getStarveCount() const { return _starveCount; } - int getSilentFramesDropped() const { return _silentFramesDropped; } protected: // disallow copying of PositionalAudioRingBuffer objects PositionalAudioRingBuffer(const PositionalAudioRingBuffer&); PositionalAudioRingBuffer& operator= (const PositionalAudioRingBuffer&); - void frameReceivedUpdateTimingStats(); - void addDroppableSilentSamples(int numSilentSamples); - - PositionalAudioRingBuffer::Type _type; + /// parses the info between the seq num and the audio data in the network packet and calculates + /// how many audio samples this packet contains + virtual int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) = 0; + + /// parses the audio data in the network packet + virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) = 0; + + int parsePositionalData(const QByteArray& positionalByteArray); + +protected: + Type _type; glm::vec3 _position; glm::quat _orientation; - bool _willBeAddedToMix; + bool _shouldLoopbackForNode; - bool _shouldOutputStarveDebug; bool _isStereo; - + float _nextOutputTrailingLoudness; AABox* _listenerUnattenuatedZone; - - quint64 _lastFrameReceivedTime; - MovingMinMaxAvg _interframeTimeGapStatsForJitterCalc; - MovingMinMaxAvg _interframeTimeGapStatsForStatsPacket; - MovingMinMaxAvg _framesAvailableStats; - - int _desiredJitterBufferFrames; - bool _dynamicJitterBuffers; - - // extra stats - int _consecutiveNotMixedCount; - int _starveCount; - int _silentFramesDropped; }; #endif // hifi_PositionalAudioRingBuffer_h