From e276d15ed4a82070cdedd9cdeef870ca4ef92d1d Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 11 Aug 2014 16:25:43 -0700 Subject: [PATCH] repetition-with-fade implemented; testing interface crash --- assignment-client/src/audio/AudioMixer.cpp | 37 ++++-- .../resources/web/settings/describe.json | 4 +- interface/src/Audio.cpp | 111 ++++++++++++++---- interface/src/Audio.h | 17 ++- libraries/audio/src/AudioRingBuffer.cpp | 42 +++++++ libraries/audio/src/AudioRingBuffer.h | 6 +- libraries/audio/src/InboundAudioStream.cpp | 46 +++++++- libraries/audio/src/InboundAudioStream.h | 11 +- .../audio/src/MixedProcessedAudioStream.cpp | 42 ++++--- .../audio/src/MixedProcessedAudioStream.h | 20 ++-- 10 files changed, 261 insertions(+), 75 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index bd8877a128..509a965bf4 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -95,6 +95,26 @@ const float ATTENUATION_EPSILON_DISTANCE = 0.1f; int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* streamToAdd, AvatarAudioStream* listeningNodeStream) { + // If repetition with fade is enabled: + // If streamToAdd could not provide a frame (it was starved), then we'll mix its previously-mixed frame + // This is preferable to not mixing it at all since that's equivalent to inserting silence. + // Basically, we'll repeat that last frame until it has a frame to mix. Depending on how many times + // we've repeated that frame in a row, we'll gradually fade that repeated frame into silence. + // This improves the perceived quality of the audio slightly. + + float repeatedFrameFadeFactor = 1.0f; + + if (!streamToAdd->lastPopSucceeded()) { + if (_streamSettings._repetitionWithFade) { + repeatedFrameFadeFactor = calculateRepeatedFrameFadeFactor(streamToAdd->getConsecutiveNotMixedCount() - 1); + if (repeatedFrameFadeFactor == 0.0f) { + return 0; + } + } else { + return 0; + } + } + float bearingRelativeAngleToSource = 0.0f; float attenuationCoefficient = 1.0f; int numSamplesDelay = 0; @@ -216,12 +236,13 @@ int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* int delayedChannelIndex = 0; const int SINGLE_STEREO_OFFSET = 2; + float attenuationAndFade = attenuationCoefficient * repeatedFrameFadeFactor; for (int s = 0; s < NETWORK_BUFFER_LENGTH_SAMPLES_STEREO; s += 4) { // setup the int16_t variables for the two sample sets - correctStreamSample[0] = streamPopOutput[s / 2] * attenuationCoefficient; - correctStreamSample[1] = streamPopOutput[(s / 2) + 1] * attenuationCoefficient; + correctStreamSample[0] = streamPopOutput[s / 2] * attenuationAndFade; + correctStreamSample[1] = streamPopOutput[(s / 2) + 1] * attenuationAndFade; delayedChannelIndex = s + (numSamplesDelay * 2) + delayedChannelOffset; @@ -237,7 +258,7 @@ int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* if (numSamplesDelay > 0) { // if there was a sample delay for this stream, we need to pull samples prior to the popped output // to stick at the beginning - float attenuationAndWeakChannelRatio = attenuationCoefficient * weakChannelAmplitudeRatio; + float attenuationAndWeakChannelRatioAndFade = attenuationCoefficient * weakChannelAmplitudeRatio * repeatedFrameFadeFactor; AudioRingBuffer::ConstIterator delayStreamPopOutput = streamPopOutput - numSamplesDelay; // TODO: delayStreamPopOutput may be inside the last frame written if the ringbuffer is completely full @@ -245,7 +266,7 @@ int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* for (int i = 0; i < numSamplesDelay; i++) { int parentIndex = i * 2; - _clientSamples[parentIndex + delayedChannelOffset] += *delayStreamPopOutput * attenuationAndWeakChannelRatio; + _clientSamples[parentIndex + delayedChannelOffset] += *delayStreamPopOutput * attenuationAndWeakChannelRatioAndFade; ++delayStreamPopOutput; } } @@ -256,8 +277,10 @@ int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* attenuationCoefficient = 1.0f; } + float attenuationAndFade = attenuationCoefficient * repeatedFrameFadeFactor; + for (int s = 0; s < NETWORK_BUFFER_LENGTH_SAMPLES_STEREO; s++) { - _clientSamples[s] = glm::clamp(_clientSamples[s] + (int)(streamPopOutput[s / stereoDivider] * attenuationCoefficient), + _clientSamples[s] = glm::clamp(_clientSamples[s] + (int)(streamPopOutput[s / stereoDivider] * attenuationAndFade), MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); } } @@ -285,7 +308,6 @@ int AudioMixer::prepareMixForListeningNode(Node* node) { PositionalAudioStream* otherNodeStream = i.value(); if ((*otherNode != *node || otherNodeStream->shouldLoopbackForNode()) - && otherNodeStream->lastPopSucceeded() && otherNodeStream->getLastPopOutputFrameLoudness() > 0.0f) { //&& otherNodeStream->getLastPopOutputTrailingLoudness() > 0.0f) { @@ -627,6 +649,7 @@ void AudioMixer::run() { } // send mixed audio packet + if (nodeData->getOutgoingSequenceNumber() % 100 < 50) nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); nodeData->incrementOutgoingMixedAudioSequenceNumber(); @@ -654,6 +677,4 @@ void AudioMixer::run() { usleep(usecToSleep); } } - - delete[] clientMixBuffer; } diff --git a/domain-server/resources/web/settings/describe.json b/domain-server/resources/web/settings/describe.json index db2809bffb..cfb7e1ed79 100644 --- a/domain-server/resources/web/settings/describe.json +++ b/domain-server/resources/web/settings/describe.json @@ -7,7 +7,7 @@ "type": "checkbox", "label": "Dynamic Jitter Buffers", "help": "Dynamically buffer client audio based on perceived jitter in packet receipt timing", - "default": true + "default": false }, "B-static-desired-jitter-buffer-frames": { "label": "Static Desired Jitter Buffer Frames", @@ -49,7 +49,7 @@ "type": "checkbox", "label": "Repetition with Fade:", "help": "If enabled, dropped frames and mixing during starves will repeat the last frame, eventually fading to silence", - "default": true + "default": false }, "Z-unattenuated-zone": { "label": "Unattenuated Zone", diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 8aac32849e..0e481e15c2 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -105,6 +105,7 @@ Audio::Audio(QObject* parent) : _scopeInput(0), _scopeOutputLeft(0), _scopeOutputRight(0), + _scopeLastFrame(), _statsEnabled(false), _statsShowInjectedStreams(false), _outgoingAvatarAudioSequenceNumber(0), @@ -113,15 +114,17 @@ Audio::Audio(QObject* parent) : _audioOutputMsecsUnplayedStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), _lastSentAudioPacket(0), _packetSentTimeGaps(1, APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS), - _audioOutputIODevice(*this) + _audioOutputIODevice(_receivedAudioStream) { // clear the array of locally injected samples memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); // Create the noise sample array _noiseSampleFrames = new float[NUMBER_OF_NOISE_SAMPLE_FRAMES]; - connect(&_receivedAudioStream, &MixedProcessedAudioStream::processSamples, this, &Audio::processReceivedAudioStreamSamples, Qt::DirectConnection); - connect(&_receivedAudioStream, &MixedProcessedAudioStream::dataParsed, this, &Audio::updateScopeBuffers, Qt::DirectConnection); + connect(&_receivedAudioStream, &MixedProcessedAudioStream::addedSilence, this, &Audio::addStereoSilenceToScope, Qt::DirectConnection); + connect(&_receivedAudioStream, &MixedProcessedAudioStream::addedLastFrameRepeatedWithFade, this, &Audio::addLastFrameRepeatedWithFadeToScope, Qt::DirectConnection); + connect(&_receivedAudioStream, &MixedProcessedAudioStream::addedStereoSamples, this, &Audio::addStereoSamplesToScope, Qt::DirectConnection); + connect(&_receivedAudioStream, &MixedProcessedAudioStream::processSamples, this, &Audio::processReceivedSamples, Qt::DirectConnection); } void Audio::init(QGLWidget *parent) { @@ -657,9 +660,7 @@ void Audio::handleAudioInput() { if (!_isStereoInput && _scopeEnabled && !_scopeEnabledPause) { unsigned int numMonoAudioChannels = 1; unsigned int monoAudioChannel = 0; - addBufferToScope(_scopeInput, _scopeInputOffset, networkAudioSamples, monoAudioChannel, numMonoAudioChannels); - _scopeInputOffset += NETWORK_SAMPLES_PER_FRAME; - _scopeInputOffset %= _samplesPerScope; + _scopeInputOffset = addBufferToScope(_scopeInput, _scopeInputOffset, networkAudioSamples, NETWORK_SAMPLES_PER_FRAME, monoAudioChannel, numMonoAudioChannels); } NodeList* nodeList = NodeList::getInstance(); @@ -733,7 +734,48 @@ void Audio::handleAudioInput() { } } -void Audio::processReceivedAudioStreamSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) { +const int STEREO_FACTOR = 2; + +void Audio::addStereoSilenceToScope(int silentSamplesPerChannel) { + if (!_scopeEnabled || _scopeEnabledPause) { + return; + } + addSilenceToScope(_scopeOutputLeft, _scopeOutputOffset, silentSamplesPerChannel); + _scopeOutputOffset = addSilenceToScope(_scopeOutputRight, _scopeOutputOffset, silentSamplesPerChannel); +} + +void Audio::addStereoSamplesToScope(const QByteArray& samples) { + if (!_scopeEnabled || _scopeEnabledPause) { + return; + } + const int16_t* samplesData = reinterpret_cast(samples.data()); + int samplesPerChannel = samples.size() / sizeof(int16_t) / STEREO_FACTOR; + + addBufferToScope(_scopeOutputLeft, _scopeOutputOffset, samplesData, samplesPerChannel, 0, STEREO_FACTOR); + _scopeOutputOffset = addBufferToScope(_scopeOutputRight, _scopeOutputOffset, samplesData, samplesPerChannel, 1, STEREO_FACTOR); + + _scopeLastFrame = samples.right(NETWORK_BUFFER_LENGTH_BYTES_STEREO); +} + +void Audio::addLastFrameRepeatedWithFadeToScope(int samplesPerChannel) { + printf("addLastFrameRepeatedWithFadeToScope"); + const int16_t* lastFrameData = reinterpret_cast(_scopeLastFrame.data()); + + int samplesRemaining = samplesPerChannel; + int indexOfRepeat = 0; + do { + int samplesToWriteThisIteration = std::min(samplesRemaining, (int)NETWORK_SAMPLES_PER_FRAME); + float fade = calculateRepeatedFrameFadeFactor(indexOfRepeat); + printf("%f ", fade); + addBufferToScope(_scopeOutputLeft, _scopeOutputOffset, lastFrameData, samplesToWriteThisIteration, 0, STEREO_FACTOR, fade); + _scopeOutputOffset = addBufferToScope(_scopeOutputRight, _scopeOutputOffset, lastFrameData, samplesToWriteThisIteration, 1, STEREO_FACTOR, fade); + + samplesRemaining -= samplesToWriteThisIteration; + } while (samplesRemaining > 0); + printf("\n"); +} + +void Audio::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) { const int numNetworkOutputSamples = inputBuffer.size() / sizeof(int16_t); const int numDeviceOutputSamples = numNetworkOutputSamples * (_outputFormat.sampleRate() * _outputFormat.channelCount()) @@ -780,7 +822,7 @@ void Audio::processReceivedAudioStreamSamples(const QByteArray& inputBuffer, QBy _desiredOutputFormat, _outputFormat); } -void Audio::updateScopeBuffers() { +/*void Audio::updateScopeBuffers() { if (_scopeEnabled && !_scopeEnabledPause) { unsigned int numAudioChannels = _desiredOutputFormat.channelCount(); const int16_t* samples = _receivedAudioStream.getNetworkSamples(); @@ -794,7 +836,7 @@ void Audio::updateScopeBuffers() { samples, audioChannel, numAudioChannels); audioChannel = 1; - addBufferToScope( + _scopeOutputOffset = addBufferToScope( _scopeOutputRight, _scopeOutputOffset, samples, audioChannel, numAudioChannels); @@ -806,7 +848,7 @@ void Audio::updateScopeBuffers() { } _receivedAudioStream.clearNetworkSamples(); -} +}*/ void Audio::addReceivedAudioToStream(const QByteArray& audioByteArray) { @@ -1259,12 +1301,15 @@ void Audio::freeScope() { } } -void Audio::addBufferToScope( - QByteArray* byteArray, unsigned int frameOffset, const int16_t* source, unsigned int sourceChannel, unsigned int sourceNumberOfChannels) { +int Audio::addBufferToScope(QByteArray* byteArray, int frameOffset, const int16_t* source, int sourceSamplesPerChannel, + unsigned int sourceChannel, unsigned int sourceNumberOfChannels, float fade) { // Constant multiplier to map sample value to vertical size of scope float multiplier = (float)MULTIPLIER_SCOPE_HEIGHT / logf(2.0f); + // Used to scale each sample. (logf(sample) + fadeOffset) is same as logf(sample * fade). + float fadeOffset = logf(fade); + // Temporary variable receives sample value float sample; @@ -1275,17 +1320,41 @@ void Audio::addBufferToScope( // Short int pointer to mapped samples in byte array int16_t* destination = (int16_t*) byteArray->data(); - for (unsigned int i = 0; i < NETWORK_SAMPLES_PER_FRAME; i++) { + for (int i = 0; i < sourceSamplesPerChannel; i++) { sample = (float)source[i * sourceNumberOfChannels + sourceChannel]; - if (sample > 0) { - value = (int16_t)(multiplier * logf(sample)); - } else if (sample < 0) { - value = (int16_t)(-multiplier * logf(-sample)); + if (sample > 1) { + value = (int16_t)(multiplier * (logf(sample) + fadeOffset)); + } else if (sample < -1) { + value = (int16_t)(-multiplier * (logf(-sample) + fadeOffset)); } else { value = 0; } - destination[i + frameOffset] = value; + destination[frameOffset] = value; + frameOffset = (frameOffset == _samplesPerScope - 1) ? 0 : frameOffset + 1; } + return frameOffset; +} + +int Audio::addSilenceToScope(QByteArray* byteArray, int frameOffset, int silentSamples) { + + QMutexLocker lock(&_guard); + // Short int pointer to mapped samples in byte array + int16_t* destination = (int16_t*)byteArray->data(); + + if (silentSamples >= _samplesPerScope) { + memset(destination, 0, byteArray->size()); + return frameOffset; + } + + int samplesToBufferEnd = _samplesPerScope - frameOffset; + if (silentSamples > samplesToBufferEnd) { + memset(destination + frameOffset, 0, samplesToBufferEnd * sizeof(int16_t)); + memset(destination, 0, silentSamples - samplesToBufferEnd * sizeof(int16_t)); + } else { + memset(destination + frameOffset, 0, silentSamples * sizeof(int16_t)); + } + + return (frameOffset + silentSamples) % _samplesPerScope; } void Audio::renderStats(const float* color, int width, int height) { @@ -1761,13 +1830,11 @@ float Audio::getInputRingBufferMsecsAvailable() const { } qint64 Audio::AudioOutputIODevice::readData(char * data, qint64 maxSize) { - MixedProcessedAudioStream& receivedAUdioStream = _parent._receivedAudioStream; - int samplesRequested = maxSize / sizeof(int16_t); int samplesPopped; int bytesWritten; - if ((samplesPopped = receivedAUdioStream.popSamples(samplesRequested, false)) > 0) { - AudioRingBuffer::ConstIterator lastPopOutput = receivedAUdioStream.getLastPopOutput(); + if ((samplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) { + AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput(); lastPopOutput.readSamples((int16_t*)data, samplesPopped); bytesWritten = samplesPopped * sizeof(int16_t); } else { diff --git a/interface/src/Audio.h b/interface/src/Audio.h index c080557576..a8ba46a35e 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -48,14 +48,14 @@ public: class AudioOutputIODevice : public QIODevice { public: - AudioOutputIODevice(Audio& parent) : _parent(parent) {}; + AudioOutputIODevice(MixedProcessedAudioStream& receivedAudioStream) : _receivedAudioStream(receivedAudioStream) {}; void start() { open(QIODevice::ReadOnly); } void stop() { close(); } qint64 readData(char * data, qint64 maxSize); qint64 writeData(const char * data, qint64 maxSize) { return 0; } private: - Audio& _parent; + MixedProcessedAudioStream& _receivedAudioStream; }; @@ -105,8 +105,6 @@ public slots: void addReceivedAudioToStream(const QByteArray& audioByteArray); void parseAudioStreamStatsPacket(const QByteArray& packet); void addSpatialAudioToBuffer(unsigned int sampleTime, const QByteArray& spatialAudio, unsigned int numSamples); - void processReceivedAudioStreamSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer); - void updateScopeBuffers(); void handleAudioInput(); void reset(); void resetStats(); @@ -123,6 +121,11 @@ public slots: void selectAudioScopeFiveFrames(); void selectAudioScopeTwentyFrames(); void selectAudioScopeFiftyFrames(); + + void addStereoSilenceToScope(int silentSamplesPerChannel); + void addLastFrameRepeatedWithFadeToScope(int samplesPerChannel); + void addStereoSamplesToScope(const QByteArray& samples); + void processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer); virtual void handleAudioByteArray(const QByteArray& audioByteArray); @@ -244,8 +247,9 @@ private: void reallocateScope(int frames); // Audio scope methods for data acquisition - void addBufferToScope( - QByteArray* byteArray, unsigned int frameOffset, const int16_t* source, unsigned int sourceChannel, unsigned int sourceNumberOfChannels); + int addBufferToScope(QByteArray* byteArray, int frameOffset, const int16_t* source, int sourceSamples, + unsigned int sourceChannel, unsigned int sourceNumberOfChannels, float fade = 1.0f); + int addSilenceToScope(QByteArray* byteArray, int frameOffset, int silentSamples); // Audio scope methods for rendering void renderBackground(const float* color, int x, int y, int width, int height); @@ -272,6 +276,7 @@ private: QByteArray* _scopeInput; QByteArray* _scopeOutputLeft; QByteArray* _scopeOutputRight; + QByteArray _scopeLastFrame; #ifdef _WIN32 static const unsigned int STATS_WIDTH = 1500; #else diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index d9cb34ac1b..31c714a1cc 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -224,3 +224,45 @@ float AudioRingBuffer::getFrameLoudness(ConstIterator frameStart) const { float AudioRingBuffer::getNextOutputFrameLoudness() const { return getFrameLoudness(_nextOutput); } + +int AudioRingBuffer::writeSamples(ConstIterator source, int maxSamples) { + int samplesToCopy = std::min(maxSamples, _sampleCapacity); + int samplesRoomFor = _sampleCapacity - samplesAvailable(); + if (samplesToCopy > samplesRoomFor) { + // there's not enough room for this write. erase old data to make room for this new data + int samplesToDelete = samplesToCopy - samplesRoomFor; + _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, samplesToDelete); + _overflowCount++; + qDebug() << "Overflowed ring buffer! Overwriting old data"; + } + + int16_t* bufferLast = _buffer + _bufferLength - 1; + for (int i = 0; i < samplesToCopy; i++) { + *_endOfLastWrite = *source; + _endOfLastWrite = (_endOfLastWrite == bufferLast) ? _buffer : _endOfLastWrite + 1; + ++source; + } + + return samplesToCopy; +} + +int AudioRingBuffer::writeSamplesWithFade(ConstIterator source, int maxSamples, float fade) { + int samplesToCopy = std::min(maxSamples, _sampleCapacity); + int samplesRoomFor = _sampleCapacity - samplesAvailable(); + if (samplesToCopy > samplesRoomFor) { + // there's not enough room for this write. erase old data to make room for this new data + int samplesToDelete = samplesToCopy - samplesRoomFor; + _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, samplesToDelete); + _overflowCount++; + qDebug() << "Overflowed ring buffer! Overwriting old data"; + } + + int16_t* bufferLast = _buffer + _bufferLength - 1; + for (int i = 0; i < samplesToCopy; i++) { + *_endOfLastWrite = (int16_t)((float)(*source) * fade); + _endOfLastWrite = (_endOfLastWrite == bufferLast) ? _buffer : _endOfLastWrite + 1; + ++source; + } + + return samplesToCopy; +} diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index be4dcaf545..65e6947115 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -189,8 +189,12 @@ public: }; ConstIterator nextOutput() const { return ConstIterator(_buffer, _bufferLength, _nextOutput); } - + ConstIterator lastFrameWritten() const { return ConstIterator(_buffer, _bufferLength, _endOfLastWrite) - _numFrameSamples; } + float getFrameLoudness(ConstIterator frameStart) const; + + int writeSamples(ConstIterator source, int maxSamples); + int writeSamplesWithFade(ConstIterator source, int maxSamples, float fade); }; #endif // hifi_AudioRingBuffer_h \ No newline at end of file diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 39cd544b15..70a4086eb0 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -162,8 +162,6 @@ int InboundAudioStream::parseData(const QByteArray& packet) { framesAvailableChanged(); - emit dataParsed(); - return readBytes; } @@ -418,9 +416,31 @@ void InboundAudioStream::packetReceivedUpdateTimingStats() { } int InboundAudioStream::writeSamplesForDroppedPackets(int networkSamples) { + if (_repetitionWithFade) { + return writeLastFrameRepeatedWithFade(networkSamples); + } return writeDroppableSilentSamples(networkSamples); } +int InboundAudioStream::writeLastFrameRepeatedWithFade(int samples) { + AudioRingBuffer::ConstIterator frameToRepeat = _ringBuffer.lastFrameWritten(); + int frameSize = _ringBuffer.getNumFrameSamples(); + int samplesToWrite = samples; + int indexOfRepeat = 0; + do { + int samplesToWriteThisIteration = std::min(samplesToWrite, frameSize); + float fade = calculateRepeatedFrameFadeFactor(indexOfRepeat); + if (fade == 1.0f) { + samplesToWrite -= _ringBuffer.writeSamples(frameToRepeat, samplesToWriteThisIteration); + } else { + samplesToWrite -= _ringBuffer.writeSamplesWithFade(frameToRepeat, samplesToWriteThisIteration, fade); + } + indexOfRepeat++; + } while (samplesToWrite > 0); + + return samples; +} + float InboundAudioStream::getLastPopOutputFrameLoudness() const { return _ringBuffer.getFrameLoudness(_lastPopOutput); } @@ -448,3 +468,25 @@ AudioStreamStats InboundAudioStream::getAudioStreamStats() const { return streamStats; } + +float calculateRepeatedFrameFadeFactor(int indexOfRepeat) { + // fade factor scheme is from this paper: + // http://inst.eecs.berkeley.edu/~ee290t/sp04/lectures/packet_loss_recov_paper11.pdf + + const float INITIAL_MSECS_NO_FADE = 20.0f; + const float MSECS_FADE_TO_ZERO = 320.0f; + + const float INITIAL_FRAMES_NO_FADE = INITIAL_MSECS_NO_FADE * (float)USECS_PER_MSEC / (float)BUFFER_SEND_INTERVAL_USECS; + const float FRAMES_FADE_TO_ZERO = MSECS_FADE_TO_ZERO * (float)USECS_PER_MSEC / (float)BUFFER_SEND_INTERVAL_USECS; + + const float SAMPLE_RANGE = std::numeric_limits::max(); + + if (indexOfRepeat <= INITIAL_FRAMES_NO_FADE) { + return 1.0f; + } else if (indexOfRepeat <= INITIAL_FRAMES_NO_FADE + FRAMES_FADE_TO_ZERO) { + return pow(SAMPLE_RANGE, -(indexOfRepeat - INITIAL_FRAMES_NO_FADE) / FRAMES_FADE_TO_ZERO); + + //return 1.0f - ((indexOfRepeat - INITIAL_FRAMES_NO_FADE) / FRAMES_FADE_TO_ZERO); + } + return 0.0f; +} diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index f41d9255ff..f8413f8d75 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -33,7 +33,7 @@ const int STATS_FOR_STATS_PACKET_WINDOW_SECONDS = 30; // this controls the window size of the time-weighted avg of frames available. Every time the window fills up, // _currentJitterBufferFrames is updated with the time-weighted avg and the running time-weighted avg is reset. -const int FRAMES_AVAILABLE_STAT_WINDOW_USECS = 2 * USECS_PER_SECOND; +const int FRAMES_AVAILABLE_STAT_WINDOW_USECS = 10 * USECS_PER_SECOND; // default values for members of the Settings struct const int DEFAULT_MAX_FRAMES_OVER_DESIRED = 10; @@ -157,9 +157,6 @@ public: int getPacketsReceived() const { return _incomingSequenceNumberStats.getReceived(); } -signals: - void dataParsed(); - public slots: /// This function should be called every second for all the stats to function properly. If dynamic jitter buffers /// is enabled, those stats are used to calculate _desiredJitterBufferFrames. @@ -191,6 +188,10 @@ protected: /// writes silent samples to the buffer that may be dropped to reduce latency caused by the buffer virtual int writeDroppableSilentSamples(int silentSamples); + + /// writes the last written frame repeatedly, gradually fading to silence. + /// used for writing samples for dropped packets. + virtual int writeLastFrameRepeatedWithFade(int samples); protected: @@ -246,4 +247,6 @@ protected: bool _repetitionWithFade; }; +float calculateRepeatedFrameFadeFactor(int indexOfRepeat); + #endif // hifi_InboundAudioStream_h diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 5693af7c6e..4b28e2f2c1 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -11,9 +11,10 @@ #include "MixedProcessedAudioStream.h" +static const int STEREO_FACTOR = 2; + MixedProcessedAudioStream::MixedProcessedAudioStream(int numFrameSamples, int numFramesCapacity, const InboundAudioStream::Settings& settings) - : InboundAudioStream(numFrameSamples, numFramesCapacity, settings), - _networkSamplesWritten(0) + : InboundAudioStream(numFrameSamples, numFramesCapacity, settings) { } @@ -23,30 +24,33 @@ void MixedProcessedAudioStream::outputFormatChanged(int outputFormatChannelCount _ringBuffer.resizeForFrameSize(deviceOutputFrameSize); } -int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples) { - - memcpy(&_networkSamples[_networkSamplesWritten], packetAfterStreamProperties.data(), packetAfterStreamProperties.size()); - _networkSamplesWritten += packetAfterStreamProperties.size() / sizeof(int16_t); - - QByteArray outputBuffer; - emit processSamples(packetAfterStreamProperties, outputBuffer); - - _ringBuffer.writeData(outputBuffer.data(), outputBuffer.size()); - - return packetAfterStreamProperties.size(); -} - int MixedProcessedAudioStream::writeDroppableSilentSamples(int silentSamples) { int deviceSilentSamplesWritten = InboundAudioStream::writeDroppableSilentSamples(networkToDeviceSamples(silentSamples)); - int networkSilentSamplesWritten = deviceToNetworkSamples(deviceSilentSamplesWritten); - memset(&_networkSamples[_networkSamplesWritten], 0, networkSilentSamplesWritten * sizeof(int16_t)); - _networkSamplesWritten += networkSilentSamplesWritten; + emit addedSilence(deviceToNetworkSamples(deviceSilentSamplesWritten) / STEREO_FACTOR); return deviceSilentSamplesWritten; } -static const int STEREO_FACTOR = 2; +int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) { + int deviceSamplesWritten = InboundAudioStream::writeLastFrameRepeatedWithFade(networkToDeviceSamples(samples)); + + emit addedLastFrameRepeatedWithFade(deviceToNetworkSamples(deviceSamplesWritten) / STEREO_FACTOR); + + return deviceSamplesWritten; +} + +int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples) { + + emit addedStereoSamples(packetAfterStreamProperties); + + QByteArray outputBuffer; + emit processSamples(packetAfterStreamProperties, outputBuffer); + + _ringBuffer.writeData(outputBuffer.data(), outputBuffer.size()); + + return packetAfterStreamProperties.size(); +} int MixedProcessedAudioStream::networkToDeviceSamples(int networkSamples) { return networkSamples * _outputFormatChannelsTimesSampleRate / (STEREO_FACTOR * SAMPLE_RATE); diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index b85637a288..fd1f93a6a1 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -14,6 +14,8 @@ #include "InboundAudioStream.h" +class Audio; + class MixedProcessedAudioStream : public InboundAudioStream { Q_OBJECT public: @@ -21,30 +23,26 @@ public: signals: + void addedSilence(int silentSamplesPerChannel); + void addedLastFrameRepeatedWithFade(int samplesPerChannel); + void addedStereoSamples(const QByteArray& samples); + void processSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer); public: void outputFormatChanged(int outputFormatChannelCountTimesSampleRate); - const int16_t* getNetworkSamples() const { return _networkSamples; } - int getNetworkSamplesWritten() const { return _networkSamplesWritten; } - - void clearNetworkSamples() { _networkSamplesWritten = 0; } - protected: - int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples); int writeDroppableSilentSamples(int silentSamples); + int writeLastFrameRepeatedWithFade(int samples); + int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples); private: int networkToDeviceSamples(int networkSamples); int deviceToNetworkSamples(int deviceSamples); + private: int _outputFormatChannelsTimesSampleRate; - - // this buffer keeps a copy of the network samples written during parseData() for the sole purpose - // of passing it on to the audio scope - int16_t _networkSamples[10 * NETWORK_BUFFER_LENGTH_SAMPLES_STEREO]; - int _networkSamplesWritten; }; #endif // hifi_MixedProcessedAudioStream_h