From 972a611d0302a042e9c8b9b5d4bc172296722dee Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 4 Oct 2016 17:50:34 -0700 Subject: [PATCH 1/5] retain stereo on dead audio resampling --- libraries/audio-client/src/AudioClient.cpp | 6 +-- .../audio/src/MixedProcessedAudioStream.cpp | 40 +++++++++---------- .../audio/src/MixedProcessedAudioStream.h | 9 +++-- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 16f4c35d21..d70730380f 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1152,9 +1152,9 @@ bool AudioClient::outputLocalInjector(bool isStereo, AudioInjector* injector) { } void AudioClient::outputFormatChanged() { - int outputFormatChannelCountTimesSampleRate = _outputFormat.channelCount() * _outputFormat.sampleRate(); - _outputFrameSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * outputFormatChannelCountTimesSampleRate / _desiredOutputFormat.sampleRate(); - _receivedAudioStream.outputFormatChanged(outputFormatChannelCountTimesSampleRate); + _outputFrameSize = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * _outputFormat.channelCount() * _outputFormat.sampleRate()) / + _desiredOutputFormat.sampleRate(); + _receivedAudioStream.outputFormatChanged(_outputFormat.sampleRate(), _outputFormat.channelCount()); } bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceInfo) { diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 88a1e071ec..6820a17a47 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -17,27 +17,27 @@ static const int STEREO_FACTOR = 2; MixedProcessedAudioStream::MixedProcessedAudioStream(int numFrameSamples, int numFramesCapacity, int numStaticJitterFrames) : InboundAudioStream(numFrameSamples, numFramesCapacity, numStaticJitterFrames) {} -void MixedProcessedAudioStream::outputFormatChanged(int outputFormatChannelCountTimesSampleRate) { - _outputFormatChannelsTimesSampleRate = outputFormatChannelCountTimesSampleRate; - int deviceOutputFrameSize = networkToDeviceSamples(AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); - _ringBuffer.resizeForFrameSize(deviceOutputFrameSize); +void MixedProcessedAudioStream::outputFormatChanged(int sampleRate, int channelCount) { + _outputSampleRate = sampleRate; + _outputChannelCount = channelCount; + int deviceOutputFrameFrames = networkToDeviceFrames(AudioConstants::NETWORK_FRAME_SAMPLES_STEREO / STEREO_FACTOR); + int deviceOutputFrameSamples = deviceOutputFrameFrames * STEREO_FACTOR; + _ringBuffer.resizeForFrameSize(deviceOutputFrameSamples); } int MixedProcessedAudioStream::writeDroppableSilentSamples(int silentSamples) { - - int deviceSilentSamplesWritten = InboundAudioStream::writeDroppableSilentSamples(networkToDeviceSamples(silentSamples)); - - emit addedSilence(deviceToNetworkSamples(deviceSilentSamplesWritten) / STEREO_FACTOR); - + int deviceSilentFrames = networkToDeviceFrames(silentSamples / STEREO_FACTOR); + int deviceSilentSamples = deviceSilentFrames * STEREO_FACTOR; + int deviceSilentSamplesWritten = InboundAudioStream::writeDroppableSilentSamples(deviceSilentSamples); + emit addedSilence(deviceToNetworkFrames(deviceSilentSamplesWritten / STEREO_FACTOR)); return deviceSilentSamplesWritten; } int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) { - - int deviceSamplesWritten = InboundAudioStream::writeLastFrameRepeatedWithFade(networkToDeviceSamples(samples)); - - emit addedLastFrameRepeatedWithFade(deviceToNetworkSamples(deviceSamplesWritten) / STEREO_FACTOR); - + int deviceFrames = networkToDeviceFrames(samples / STEREO_FACTOR); + int deviceSamples = deviceFrames * STEREO_FACTOR; + int deviceSamplesWritten = InboundAudioStream::writeLastFrameRepeatedWithFade(deviceSamples); + emit addedLastFrameRepeatedWithFade(deviceToNetworkFrames(deviceSamplesWritten / STEREO_FACTOR)); return deviceSamplesWritten; } @@ -60,12 +60,12 @@ int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& return packetAfterStreamProperties.size(); } -int MixedProcessedAudioStream::networkToDeviceSamples(int networkSamples) { - return (quint64)networkSamples * (quint64)_outputFormatChannelsTimesSampleRate / (quint64)(STEREO_FACTOR - * AudioConstants::SAMPLE_RATE); +int MixedProcessedAudioStream::networkToDeviceFrames(int networkFrames) { + return ((quint64)networkFrames * _outputChannelCount * _outputSampleRate) / + (quint64)(STEREO_FACTOR * AudioConstants::SAMPLE_RATE); } -int MixedProcessedAudioStream::deviceToNetworkSamples(int deviceSamples) { - return (quint64)deviceSamples * (quint64)(STEREO_FACTOR * AudioConstants::SAMPLE_RATE) - / (quint64)_outputFormatChannelsTimesSampleRate; +int MixedProcessedAudioStream::deviceToNetworkFrames(int deviceFrames) { + return (quint64)deviceFrames * (quint64)(STEREO_FACTOR * AudioConstants::SAMPLE_RATE) / + (_outputSampleRate * _outputChannelCount); } diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index f28586f484..d89ce9f7e3 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -30,7 +30,7 @@ signals: void processSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer); public: - void outputFormatChanged(int outputFormatChannelCountTimesSampleRate); + void outputFormatChanged(int sampleRate, int channelCount); protected: int writeDroppableSilentSamples(int silentSamples) override; @@ -38,11 +38,12 @@ protected: int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) override; private: - int networkToDeviceSamples(int networkSamples); - int deviceToNetworkSamples(int deviceSamples); + int networkToDeviceFrames(int networkFrames); + int deviceToNetworkFrames(int deviceFrames); private: - int _outputFormatChannelsTimesSampleRate; + quint64 _outputSampleRate; + quint64 _outputChannelCount; }; #endif // hifi_MixedProcessedAudioStream_h From 00fabb77b42383a31355d33938b96a0a6e114e21 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 4 Oct 2016 18:20:01 -0700 Subject: [PATCH 2/5] use proper audio frame naming --- libraries/audio/src/InboundAudioStream.cpp | 26 +++++++++---------- libraries/audio/src/InboundAudioStream.h | 8 +++--- .../audio/src/MixedProcessedAudioStream.cpp | 22 +++++++--------- .../audio/src/MixedProcessedAudioStream.h | 4 +-- 4 files changed, 29 insertions(+), 31 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index a3fabad70f..71e8cb12ab 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -121,11 +121,11 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { packetReceivedUpdateTimingStats(); - int networkSamples; - + int networkFrames; + // parse the info after the seq number and before the audio data (the stream properties) int prePropertyPosition = message.getPosition(); - int propertyBytes = parseStreamProperties(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()), networkSamples); + int propertyBytes = parseStreamProperties(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()), networkFrames); message.seek(prePropertyPosition + propertyBytes); // handle this packet based on its arrival status. @@ -135,7 +135,7 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { // NOTE: we assume that each dropped packet contains the same number of samples // as the packet we just received. int packetsDropped = arrivalInfo._seqDiffFromExpected; - writeSamplesForDroppedPackets(packetsDropped * networkSamples); + writeFramesForDroppedPackets(packetsDropped * networkFrames); // fall through to OnTime case } @@ -143,7 +143,7 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { // Packet is on time; parse its data to the ringbuffer if (message.getType() == PacketType::SilentAudioFrame) { // FIXME - Some codecs need to know about these silent frames... and can produce better output - writeDroppableSilentSamples(networkSamples); + writeDroppableSilentFrames(networkFrames); } else { // note: PCM and no codec are identical bool selectedPCM = _selectedCodecName == "pcm" || _selectedCodecName == ""; @@ -153,7 +153,7 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { parseAudioData(message.getType(), afterProperties); } else { qDebug() << "Codec mismatch: expected" << _selectedCodecName << "got" << codecInPacket << "writing silence"; - writeDroppableSilentSamples(networkSamples); + writeDroppableSilentFrames(networkFrames); // inform others of the mismatch auto sendingNode = DependencyManager::get()->nodeWithUUID(message.getSourceID()); emit mismatchedAudioCodec(sendingNode, _selectedCodecName, codecInPacket); @@ -218,9 +218,9 @@ int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packet return _ringBuffer.writeData(decodedBuffer.data(), actualSize); } -int InboundAudioStream::writeDroppableSilentSamples(int silentSamples) { +int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) { if (_decoder) { - _decoder->trackLostFrames(silentSamples); + _decoder->trackLostFrames(silentFrames); } // calculate how many silent frames we should drop. @@ -228,12 +228,12 @@ int InboundAudioStream::writeDroppableSilentSamples(int silentSamples) { int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING; int numSilentFramesToDrop = 0; - if (silentSamples >= samplesPerFrame && _currentJitterBufferFrames > desiredJitterBufferFramesPlusPadding) { + if (silentFrames >= samplesPerFrame && _currentJitterBufferFrames > desiredJitterBufferFramesPlusPadding) { // our avg jitter buffer size exceeds its desired value, so ignore some silent // frames to get that size as close to desired as possible int numSilentFramesToDropDesired = _currentJitterBufferFrames - desiredJitterBufferFramesPlusPadding; - int numSilentFramesReceived = silentSamples / samplesPerFrame; + int numSilentFramesReceived = silentFrames / samplesPerFrame; numSilentFramesToDrop = std::min(numSilentFramesToDropDesired, numSilentFramesReceived); // dont reset _currentJitterBufferFrames here; we want to be able to drop further silent frames @@ -247,7 +247,7 @@ int InboundAudioStream::writeDroppableSilentSamples(int silentSamples) { _framesAvailableStat.reset(); } - int ret = _ringBuffer.addSilentSamples(silentSamples - numSilentFramesToDrop * samplesPerFrame); + int ret = _ringBuffer.addSilentSamples(silentFrames - numSilentFramesToDrop * samplesPerFrame); return ret; } @@ -414,8 +414,8 @@ void InboundAudioStream::packetReceivedUpdateTimingStats() { _lastPacketReceivedTime = now; } -int InboundAudioStream::writeSamplesForDroppedPackets(int networkSamples) { - return writeLastFrameRepeatedWithFade(networkSamples); +int InboundAudioStream::writeFramesForDroppedPackets(int networkFrames) { + return writeLastFrameRepeatedWithFade(networkFrames); } int InboundAudioStream::writeLastFrameRepeatedWithFade(int samples) { diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index 4c10c8c789..e72a45a01b 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -115,7 +115,7 @@ public slots: private: void packetReceivedUpdateTimingStats(); - int writeSamplesForDroppedPackets(int networkSamples); + int writeFramesForDroppedPackets(int networkFrames); void popSamplesNoCheck(int samples); void framesAvailableChanged(); @@ -134,12 +134,12 @@ protected: /// default implementation assumes packet contains raw audio samples after stream properties virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties); - /// writes silent samples to the buffer that may be dropped to reduce latency caused by the buffer - virtual int writeDroppableSilentSamples(int silentSamples); + /// writes silent frames to the buffer that may be dropped to reduce latency caused by the buffer + virtual int writeDroppableSilentFrames(int silentFrames); /// writes the last written frame repeatedly, gradually fading to silence. /// used for writing samples for dropped packets. - virtual int writeLastFrameRepeatedWithFade(int samples); + virtual int writeLastFrameRepeatedWithFade(int frames); protected: diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 6820a17a47..ccaf35f1c4 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -25,20 +25,18 @@ void MixedProcessedAudioStream::outputFormatChanged(int sampleRate, int channelC _ringBuffer.resizeForFrameSize(deviceOutputFrameSamples); } -int MixedProcessedAudioStream::writeDroppableSilentSamples(int silentSamples) { - int deviceSilentFrames = networkToDeviceFrames(silentSamples / STEREO_FACTOR); - int deviceSilentSamples = deviceSilentFrames * STEREO_FACTOR; - int deviceSilentSamplesWritten = InboundAudioStream::writeDroppableSilentSamples(deviceSilentSamples); - emit addedSilence(deviceToNetworkFrames(deviceSilentSamplesWritten / STEREO_FACTOR)); - return deviceSilentSamplesWritten; +int MixedProcessedAudioStream::writeDroppableSilentFrames(int silentFrames) { + int deviceSilentFrames = networkToDeviceFrames(silentFrames); + int deviceSilentFramesWritten = InboundAudioStream::writeDroppableSilentFrames(deviceSilentFrames); + emit addedSilence(deviceToNetworkFrames(deviceSilentFramesWritten)); + return deviceSilentFramesWritten; } -int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) { - int deviceFrames = networkToDeviceFrames(samples / STEREO_FACTOR); - int deviceSamples = deviceFrames * STEREO_FACTOR; - int deviceSamplesWritten = InboundAudioStream::writeLastFrameRepeatedWithFade(deviceSamples); - emit addedLastFrameRepeatedWithFade(deviceToNetworkFrames(deviceSamplesWritten / STEREO_FACTOR)); - return deviceSamplesWritten; +int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int frames) { + int deviceFrames = networkToDeviceFrames(frames); + int deviceFramesWritten = InboundAudioStream::writeLastFrameRepeatedWithFade(deviceFrames); + emit addedLastFrameRepeatedWithFade(deviceToNetworkFrames(deviceFramesWritten)); + return deviceFramesWritten; } int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index d89ce9f7e3..7071cf0e5a 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -33,8 +33,8 @@ public: void outputFormatChanged(int sampleRate, int channelCount); protected: - int writeDroppableSilentSamples(int silentSamples) override; - int writeLastFrameRepeatedWithFade(int samples) override; + int writeDroppableSilentFrames(int silentFrames) override; + int writeLastFrameRepeatedWithFade(int frames) override; int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) override; private: From 05aded5c9d7d645c8ada7425d313dd0a2a48235c Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 4 Oct 2016 18:45:03 -0700 Subject: [PATCH 3/5] differentiate smaples/frames from dropped audio --- libraries/audio/src/InboundAudioStream.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 71e8cb12ab..35c90e51bd 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -224,16 +224,17 @@ int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) { } // calculate how many silent frames we should drop. + int silentSamples = silentFrames * 2; int samplesPerFrame = _ringBuffer.getNumFrameSamples(); int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING; int numSilentFramesToDrop = 0; - if (silentFrames >= samplesPerFrame && _currentJitterBufferFrames > desiredJitterBufferFramesPlusPadding) { + if (silentSamples >= samplesPerFrame && _currentJitterBufferFrames > desiredJitterBufferFramesPlusPadding) { // our avg jitter buffer size exceeds its desired value, so ignore some silent // frames to get that size as close to desired as possible int numSilentFramesToDropDesired = _currentJitterBufferFrames - desiredJitterBufferFramesPlusPadding; - int numSilentFramesReceived = silentFrames / samplesPerFrame; + int numSilentFramesReceived = silentSamples / samplesPerFrame; numSilentFramesToDrop = std::min(numSilentFramesToDropDesired, numSilentFramesReceived); // dont reset _currentJitterBufferFrames here; we want to be able to drop further silent frames @@ -247,7 +248,7 @@ int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) { _framesAvailableStat.reset(); } - int ret = _ringBuffer.addSilentSamples(silentFrames - numSilentFramesToDrop * samplesPerFrame); + int ret = _ringBuffer.addSilentSamples(silentSamples - numSilentFramesToDrop * samplesPerFrame); return ret; } @@ -418,10 +419,10 @@ int InboundAudioStream::writeFramesForDroppedPackets(int networkFrames) { return writeLastFrameRepeatedWithFade(networkFrames); } -int InboundAudioStream::writeLastFrameRepeatedWithFade(int samples) { +int InboundAudioStream::writeLastFrameRepeatedWithFade(int frames) { AudioRingBuffer::ConstIterator frameToRepeat = _ringBuffer.lastFrameWritten(); int frameSize = _ringBuffer.getNumFrameSamples(); - int samplesToWrite = samples; + int samplesToWrite = frames * 2; int indexOfRepeat = 0; do { int samplesToWriteThisIteration = std::min(samplesToWrite, frameSize); @@ -434,7 +435,7 @@ int InboundAudioStream::writeLastFrameRepeatedWithFade(int samples) { indexOfRepeat++; } while (samplesToWrite > 0); - return samples; + return frames; } AudioStreamStats InboundAudioStream::getAudioStreamStats() const { From 5af95c6062bc80abd6ed7f5ad223c4be0fd61970 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 5 Oct 2016 11:33:12 -0700 Subject: [PATCH 4/5] store channels on audio stream --- assignment-client/src/Agent.cpp | 3 +-- libraries/audio-client/src/AudioClient.cpp | 2 +- libraries/audio/src/InboundAudioStream.cpp | 13 +++++++------ libraries/audio/src/InboundAudioStream.h | 3 ++- libraries/audio/src/MixedAudioStream.cpp | 9 +++++++-- libraries/audio/src/MixedAudioStream.h | 2 +- libraries/audio/src/MixedProcessedAudioStream.cpp | 5 +++-- libraries/audio/src/MixedProcessedAudioStream.h | 2 +- libraries/audio/src/PositionalAudioStream.cpp | 11 +++++++---- 9 files changed, 30 insertions(+), 20 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index c6a2a3d5e8..65443b0574 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -48,8 +48,7 @@ static const int RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES = 10; Agent::Agent(ReceivedMessage& message) : ThreadedAssignment(message), _entityEditSender(), - _receivedAudioStream(AudioConstants::NETWORK_FRAME_SAMPLES_STEREO, - RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES, RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES) { + _receivedAudioStream(RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES, RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES) { DependencyManager::get()->setPacketSender(&_entityEditSender); ResourceManager::init(); diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index d70730380f..6fbd93e386 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -115,7 +115,7 @@ AudioClient::AudioClient() : _loopbackAudioOutput(NULL), _loopbackOutputDevice(NULL), _inputRingBuffer(0), - _receivedAudioStream(0, RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES), + _receivedAudioStream(RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES), _isStereoInput(false), _outputStarveDetectionStartTimeMsec(0), _outputStarveDetectionCount(0), diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 35c90e51bd..177a5ddcef 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -46,10 +46,11 @@ static const int STATS_FOR_STATS_PACKET_WINDOW_SECONDS = 30; // _currentJitterBufferFrames is updated with the time-weighted avg and the running time-weighted avg is reset. static const quint64 FRAMES_AVAILABLE_STAT_WINDOW_USECS = 10 * USECS_PER_SECOND; -InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacity, int numStaticJitterFrames) : - _ringBuffer(numFrameSamples, numFramesCapacity), - _dynamicJitterBufferEnabled(numStaticJitterFrames == -1), - _staticJitterBufferFrames(std::max(numStaticJitterFrames, DEFAULT_STATIC_JITTER_FRAMES)), +InboundAudioStream::InboundAudioStream(int numChannels, int numFrames, int numBlocks, int numStaticJitterBlocks) : + _ringBuffer(numChannels * numFrames, numBlocks), + _numChannels(numChannels), + _dynamicJitterBufferEnabled(numStaticJitterBlocks == -1), + _staticJitterBufferFrames(std::max(numStaticJitterBlocks, DEFAULT_STATIC_JITTER_FRAMES)), _desiredJitterBufferFrames(_dynamicJitterBufferEnabled ? 1 : _staticJitterBufferFrames), _incomingSequenceNumberStats(STATS_FOR_STATS_PACKET_WINDOW_SECONDS), _starveHistory(STARVE_HISTORY_CAPACITY), @@ -224,7 +225,7 @@ int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) { } // calculate how many silent frames we should drop. - int silentSamples = silentFrames * 2; + int silentSamples = silentFrames * _numChannels; int samplesPerFrame = _ringBuffer.getNumFrameSamples(); int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING; int numSilentFramesToDrop = 0; @@ -422,7 +423,7 @@ int InboundAudioStream::writeFramesForDroppedPackets(int networkFrames) { int InboundAudioStream::writeLastFrameRepeatedWithFade(int frames) { AudioRingBuffer::ConstIterator frameToRepeat = _ringBuffer.lastFrameWritten(); int frameSize = _ringBuffer.getNumFrameSamples(); - int samplesToWrite = frames * 2; + int samplesToWrite = frames * _numChannels; int indexOfRepeat = 0; do { int samplesToWriteThisIteration = std::min(samplesToWrite, frameSize); diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index e72a45a01b..f7b79ab136 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -47,7 +47,7 @@ public: static const bool REPETITION_WITH_FADE; InboundAudioStream() = delete; - InboundAudioStream(int numFrameSamples, int numFramesCapacity, int numStaticJitterFrames = -1); + InboundAudioStream(int numChannels, int numFrames, int numBlocks, int numStaticJitterBlocks); ~InboundAudioStream(); void reset(); @@ -144,6 +144,7 @@ protected: protected: AudioRingBuffer _ringBuffer; + int _numChannels; bool _lastPopSucceeded { false }; AudioRingBuffer::ConstIterator _lastPopOutput; diff --git a/libraries/audio/src/MixedAudioStream.cpp b/libraries/audio/src/MixedAudioStream.cpp index 8a3091f596..0f28b23b24 100644 --- a/libraries/audio/src/MixedAudioStream.cpp +++ b/libraries/audio/src/MixedAudioStream.cpp @@ -11,5 +11,10 @@ #include "MixedAudioStream.h" -MixedAudioStream::MixedAudioStream(int numFrameSamples, int numFramesCapacity, int numStaticJitterFrames) : - InboundAudioStream(numFrameSamples, numFramesCapacity, numStaticJitterFrames) {} +#include "AudioConstants.h" + +static const int STEREO_FACTOR = 2; + +MixedAudioStream::MixedAudioStream(int numFramesCapacity, int numStaticJitterFrames) : + InboundAudioStream(STEREO_FACTOR, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, + numFramesCapacity, numStaticJitterFrames) {} diff --git a/libraries/audio/src/MixedAudioStream.h b/libraries/audio/src/MixedAudioStream.h index f86ebaef56..6f2716203e 100644 --- a/libraries/audio/src/MixedAudioStream.h +++ b/libraries/audio/src/MixedAudioStream.h @@ -16,7 +16,7 @@ class MixedAudioStream : public InboundAudioStream { public: - MixedAudioStream(int numFrameSamples, int numFramesCapacity, int numStaticJitterFrames = -1); + MixedAudioStream(int numFramesCapacity, int numStaticJitterFrames = -1); float getNextOutputFrameLoudness() const { return _ringBuffer.getNextOutputFrameLoudness(); } }; diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index ccaf35f1c4..26ae6f7928 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -14,8 +14,9 @@ static const int STEREO_FACTOR = 2; -MixedProcessedAudioStream::MixedProcessedAudioStream(int numFrameSamples, int numFramesCapacity, int numStaticJitterFrames) - : InboundAudioStream(numFrameSamples, numFramesCapacity, numStaticJitterFrames) {} +MixedProcessedAudioStream::MixedProcessedAudioStream(int numFramesCapacity, int numStaticJitterFrames) + : InboundAudioStream(STEREO_FACTOR, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, + numFramesCapacity, numStaticJitterFrames) {} void MixedProcessedAudioStream::outputFormatChanged(int sampleRate, int channelCount) { _outputSampleRate = sampleRate; diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index 7071cf0e5a..d536163d2d 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -19,7 +19,7 @@ class AudioClient; class MixedProcessedAudioStream : public InboundAudioStream { Q_OBJECT public: - MixedProcessedAudioStream(int numFrameSamples, int numFramesCapacity, int numStaticJitterFrames = -1); + MixedProcessedAudioStream(int numFramesCapacity, int numStaticJitterFrames = -1); signals: diff --git a/libraries/audio/src/PositionalAudioStream.cpp b/libraries/audio/src/PositionalAudioStream.cpp index bb14044312..92573ced6f 100644 --- a/libraries/audio/src/PositionalAudioStream.cpp +++ b/libraries/audio/src/PositionalAudioStream.cpp @@ -21,11 +21,14 @@ #include #include +static const int MONO_FACTOR = 1; +static const int STEREO_FACTOR = 2; + PositionalAudioStream::PositionalAudioStream(PositionalAudioStream::Type type, bool isStereo, int numStaticJitterFrames) : - InboundAudioStream(isStereo - ? AudioConstants::NETWORK_FRAME_SAMPLES_STEREO - : AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, - AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY, numStaticJitterFrames), + InboundAudioStream(isStereo ? STEREO_FACTOR : MONO_FACTOR, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, + AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY, + numStaticJitterFrames), _type(type), _position(0.0f, 0.0f, 0.0f), _orientation(0.0f, 0.0f, 0.0f, 0.0f), From afce8c4a458afbc17c606c7dacaba6625ae6c760 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 5 Oct 2016 13:02:28 -0700 Subject: [PATCH 5/5] STEREO_FACTOR -> AudioConstants::STEREO --- interface/src/audio/AudioScope.cpp | 12 +++++------- libraries/audio/src/MixedAudioStream.cpp | 4 +--- libraries/audio/src/MixedProcessedAudioStream.cpp | 14 ++++++-------- libraries/audio/src/PositionalAudioStream.cpp | 5 +---- 4 files changed, 13 insertions(+), 22 deletions(-) diff --git a/interface/src/audio/AudioScope.cpp b/interface/src/audio/AudioScope.cpp index 1946d216ff..346fbd11f4 100644 --- a/interface/src/audio/AudioScope.cpp +++ b/interface/src/audio/AudioScope.cpp @@ -250,8 +250,6 @@ int AudioScope::addSilenceToScope(QByteArray* byteArray, int frameOffset, int si } -const int STEREO_FACTOR = 2; - void AudioScope::addStereoSilenceToScope(int silentSamplesPerChannel) { if (!_isEnabled || _isPaused) { return; @@ -265,10 +263,10 @@ void AudioScope::addStereoSamplesToScope(const QByteArray& samples) { return; } const int16_t* samplesData = reinterpret_cast(samples.data()); - int samplesPerChannel = samples.size() / sizeof(int16_t) / STEREO_FACTOR; + int samplesPerChannel = samples.size() / sizeof(int16_t) / AudioConstants::STEREO; - addBufferToScope(_scopeOutputLeft, _scopeOutputOffset, samplesData, samplesPerChannel, 0, STEREO_FACTOR); - _scopeOutputOffset = addBufferToScope(_scopeOutputRight, _scopeOutputOffset, samplesData, samplesPerChannel, 1, STEREO_FACTOR); + addBufferToScope(_scopeOutputLeft, _scopeOutputOffset, samplesData, samplesPerChannel, 0, AudioConstants::STEREO); + _scopeOutputOffset = addBufferToScope(_scopeOutputRight, _scopeOutputOffset, samplesData, samplesPerChannel, 1, AudioConstants::STEREO); _scopeLastFrame = samples.right(AudioConstants::NETWORK_FRAME_BYTES_STEREO); } @@ -282,9 +280,9 @@ void AudioScope::addLastFrameRepeatedWithFadeToScope(int samplesPerChannel) { int samplesToWriteThisIteration = std::min(samplesRemaining, (int) AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); float fade = calculateRepeatedFrameFadeFactor(indexOfRepeat); addBufferToScope(_scopeOutputLeft, _scopeOutputOffset, lastFrameData, - samplesToWriteThisIteration, 0, STEREO_FACTOR, fade); + samplesToWriteThisIteration, 0, AudioConstants::STEREO, fade); _scopeOutputOffset = addBufferToScope(_scopeOutputRight, _scopeOutputOffset, - lastFrameData, samplesToWriteThisIteration, 1, STEREO_FACTOR, fade); + lastFrameData, samplesToWriteThisIteration, 1, AudioConstants::STEREO, fade); samplesRemaining -= samplesToWriteThisIteration; indexOfRepeat++; diff --git a/libraries/audio/src/MixedAudioStream.cpp b/libraries/audio/src/MixedAudioStream.cpp index 0f28b23b24..e9af6933b6 100644 --- a/libraries/audio/src/MixedAudioStream.cpp +++ b/libraries/audio/src/MixedAudioStream.cpp @@ -13,8 +13,6 @@ #include "AudioConstants.h" -static const int STEREO_FACTOR = 2; - MixedAudioStream::MixedAudioStream(int numFramesCapacity, int numStaticJitterFrames) : - InboundAudioStream(STEREO_FACTOR, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, + InboundAudioStream(AudioConstants::STEREO, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, numFramesCapacity, numStaticJitterFrames) {} diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 26ae6f7928..671d3a9d60 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -12,17 +12,15 @@ #include "MixedProcessedAudioStream.h" #include "AudioLogging.h" -static const int STEREO_FACTOR = 2; - MixedProcessedAudioStream::MixedProcessedAudioStream(int numFramesCapacity, int numStaticJitterFrames) - : InboundAudioStream(STEREO_FACTOR, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, + : InboundAudioStream(AudioConstants::STEREO, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, numFramesCapacity, numStaticJitterFrames) {} void MixedProcessedAudioStream::outputFormatChanged(int sampleRate, int channelCount) { _outputSampleRate = sampleRate; _outputChannelCount = channelCount; - int deviceOutputFrameFrames = networkToDeviceFrames(AudioConstants::NETWORK_FRAME_SAMPLES_STEREO / STEREO_FACTOR); - int deviceOutputFrameSamples = deviceOutputFrameFrames * STEREO_FACTOR; + int deviceOutputFrameFrames = networkToDeviceFrames(AudioConstants::NETWORK_FRAME_SAMPLES_STEREO / AudioConstants::STEREO); + int deviceOutputFrameSamples = deviceOutputFrameFrames * AudioConstants::STEREO; _ringBuffer.resizeForFrameSize(deviceOutputFrameSamples); } @@ -55,16 +53,16 @@ int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& _ringBuffer.writeData(outputBuffer.data(), outputBuffer.size()); qCDebug(audiostream, "Wrote %d samples to buffer (%d available)", outputBuffer.size() / (int)sizeof(int16_t), getSamplesAvailable()); - + return packetAfterStreamProperties.size(); } int MixedProcessedAudioStream::networkToDeviceFrames(int networkFrames) { return ((quint64)networkFrames * _outputChannelCount * _outputSampleRate) / - (quint64)(STEREO_FACTOR * AudioConstants::SAMPLE_RATE); + (quint64)(AudioConstants::STEREO * AudioConstants::SAMPLE_RATE); } int MixedProcessedAudioStream::deviceToNetworkFrames(int deviceFrames) { - return (quint64)deviceFrames * (quint64)(STEREO_FACTOR * AudioConstants::SAMPLE_RATE) / + return (quint64)deviceFrames * (quint64)(AudioConstants::STEREO * AudioConstants::SAMPLE_RATE) / (_outputSampleRate * _outputChannelCount); } diff --git a/libraries/audio/src/PositionalAudioStream.cpp b/libraries/audio/src/PositionalAudioStream.cpp index 92573ced6f..e7bf72c955 100644 --- a/libraries/audio/src/PositionalAudioStream.cpp +++ b/libraries/audio/src/PositionalAudioStream.cpp @@ -21,11 +21,8 @@ #include #include -static const int MONO_FACTOR = 1; -static const int STEREO_FACTOR = 2; - PositionalAudioStream::PositionalAudioStream(PositionalAudioStream::Type type, bool isStereo, int numStaticJitterFrames) : - InboundAudioStream(isStereo ? STEREO_FACTOR : MONO_FACTOR, + InboundAudioStream(isStereo ? AudioConstants::STEREO : AudioConstants::MONO, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY, numStaticJitterFrames),