From 641a6e9c30423159ae2a9b1d7d674f7f04a1123e Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 26 Jan 2016 09:29:02 -0800 Subject: [PATCH 1/7] Fix injector handling for when it falls behind --- libraries/audio/src/AudioConstants.h | 2 +- libraries/audio/src/AudioInjector.cpp | 87 +++++++++++--------- libraries/audio/src/AudioInjector.h | 2 +- libraries/audio/src/AudioInjectorManager.cpp | 2 +- 4 files changed, 53 insertions(+), 40 deletions(-) diff --git a/libraries/audio/src/AudioConstants.h b/libraries/audio/src/AudioConstants.h index b1c810710e..91080017a5 100644 --- a/libraries/audio/src/AudioConstants.h +++ b/libraries/audio/src/AudioConstants.h @@ -29,7 +29,7 @@ namespace AudioConstants { const int NETWORK_FRAME_SAMPLES_PER_CHANNEL = NETWORK_FRAME_BYTES_PER_CHANNEL / sizeof(AudioSample); const float NETWORK_FRAME_MSECS = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL / (float)AudioConstants::SAMPLE_RATE) * 1000.0f; - const unsigned int NETWORK_FRAME_USECS = (unsigned int)floorf(NETWORK_FRAME_MSECS * 1000.0f); + const int NETWORK_FRAME_USECS = floorf(NETWORK_FRAME_MSECS * 1000.0f); const int MIN_SAMPLE_VALUE = std::numeric_limits::min(); const int MAX_SAMPLE_VALUE = std::numeric_limits::max(); } diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index 1f15007339..bec4921675 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -158,10 +158,10 @@ bool AudioInjector::injectLocally() { } const uchar MAX_INJECTOR_VOLUME = 0xFF; -static const uint64_t NEXT_FRAME_DELTA_ERROR_OR_FINISHED = 0; -static const uint64_t NEXT_FRAME_DELTA_IMMEDIATELY = 1; +static const int64_t NEXT_FRAME_DELTA_ERROR_OR_FINISHED = -1; +static const int64_t NEXT_FRAME_DELTA_IMMEDIATELY = 0; -uint64_t AudioInjector::injectNextFrame() { +int64_t AudioInjector::injectNextFrame() { if (_state == AudioInjector::State::Finished) { qDebug() << "AudioInjector::injectNextFrame called but AudioInjector has finished and was not restarted. Returning."; @@ -198,110 +198,123 @@ uint64_t AudioInjector::injectNextFrame() { // pack some placeholder sequence number for now audioPacketStream << (quint16) 0; - + // pack stream identifier (a generated UUID) audioPacketStream << QUuid::createUuid(); - + // pack the stereo/mono type of the stream audioPacketStream << _options.stereo; - + // pack the flag for loopback - uchar loopbackFlag = (uchar) true; + uchar loopbackFlag = (uchar)true; audioPacketStream << loopbackFlag; - + // pack the position for injected audio positionOptionOffset = _currentPacket->pos(); audioPacketStream.writeRawData(reinterpret_cast(&_options.position), - sizeof(_options.position)); - + sizeof(_options.position)); + // pack our orientation for injected audio audioPacketStream.writeRawData(reinterpret_cast(&_options.orientation), - sizeof(_options.orientation)); - + sizeof(_options.orientation)); + // pack zero for radius float radius = 0; audioPacketStream << radius; - + // pack 255 for attenuation byte volumeOptionOffset = _currentPacket->pos(); quint8 volume = MAX_INJECTOR_VOLUME; audioPacketStream << volume; - + audioPacketStream << _options.ignorePenumbra; - + audioDataOffset = _currentPacket->pos(); - - } else { + + } + else { // no samples to inject, return immediately qDebug() << "AudioInjector::injectNextFrame() called with no samples to inject. Returning."; return NEXT_FRAME_DELTA_ERROR_OR_FINISHED; } } - + int bytesToCopy = std::min((_options.stereo ? 2 : 1) * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL, - _audioData.size() - _currentSendOffset); - + _audioData.size() - _currentSendOffset); + // Measure the loudness of this frame _loudness = 0.0f; for (int i = 0; i < bytesToCopy; i += sizeof(int16_t)) { _loudness += abs(*reinterpret_cast(_audioData.data() + _currentSendOffset + i)) / - (AudioConstants::MAX_SAMPLE_VALUE / 2.0f); + (AudioConstants::MAX_SAMPLE_VALUE / 2.0f); } _loudness /= (float)(bytesToCopy / sizeof(int16_t)); - + _currentPacket->seek(0); - + // pack the sequence number _currentPacket->writePrimitive(_outgoingSequenceNumber); - + _currentPacket->seek(positionOptionOffset); _currentPacket->writePrimitive(_options.position); _currentPacket->writePrimitive(_options.orientation); - + quint8 volume = MAX_INJECTOR_VOLUME * _options.volume; _currentPacket->seek(volumeOptionOffset); _currentPacket->writePrimitive(volume); - + _currentPacket->seek(audioDataOffset); - + // copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet _currentPacket->write(_audioData.data() + _currentSendOffset, bytesToCopy); - + // set the correct size used for this packet _currentPacket->setPayloadSize(_currentPacket->pos()); - + // grab our audio mixer from the NodeList, if it exists auto nodeList = DependencyManager::get(); SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); - + if (audioMixer) { // send off this audio packet nodeList->sendUnreliablePacket(*_currentPacket, *audioMixer); _outgoingSequenceNumber++; } - + _currentSendOffset += bytesToCopy; - + if (_currentSendOffset >= _audioData.size()) { // we're at the end of the audio data to send if (_options.loop) { // we were asked to loop, set our send offset to 0 _currentSendOffset = 0; - } else { + } + else { // we weren't to loop, say that we're done now finish(); return NEXT_FRAME_DELTA_ERROR_OR_FINISHED; } } - + if (_currentSendOffset == bytesToCopy) { // ask AudioInjectorManager to call us right away again to // immediately send the first two frames so the mixer can start using the audio right away return NEXT_FRAME_DELTA_IMMEDIATELY; - } else { - return (++_nextFrame * AudioConstants::NETWORK_FRAME_USECS) - _frameTimer->nsecsElapsed() / 1000; } - + + const int MAX_ALLOWED_FRAMES_TO_FALL_BEHIND = 7; + int64_t currentTime = _frameTimer->nsecsElapsed() / 1000; + auto currentFrameBasedOnElapsedTime = currentTime / AudioConstants::NETWORK_FRAME_USECS; + + if (currentFrameBasedOnElapsedTime - _nextFrame > MAX_ALLOWED_FRAMES_TO_FALL_BEHIND) { + // If we are falling behind by more frames than our threshold, let's skip the frames ahead + _nextFrame = currentFrameBasedOnElapsedTime - MAX_ALLOWED_FRAMES_TO_FALL_BEHIND; + } else { + ++_nextFrame; + } + + int64_t playNextFrameAt = _nextFrame * AudioConstants::NETWORK_FRAME_USECS; + return std::max(0LL, playNextFrameAt - currentTime); } void AudioInjector::stop() { diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index f815b6fe3a..5155b87a74 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -84,7 +84,7 @@ private slots: private: void setupInjection(); - uint64_t injectNextFrame(); + int64_t injectNextFrame(); bool injectLocally(); QByteArray _audioData; diff --git a/libraries/audio/src/AudioInjectorManager.cpp b/libraries/audio/src/AudioInjectorManager.cpp index f504b31907..b91bddc553 100644 --- a/libraries/audio/src/AudioInjectorManager.cpp +++ b/libraries/audio/src/AudioInjectorManager.cpp @@ -88,7 +88,7 @@ void AudioInjectorManager::run() { // this is an injector that's ready to go, have it send a frame now auto nextCallDelta = injector->injectNextFrame(); - if (nextCallDelta > 0 && !injector->isFinished()) { + if (nextCallDelta >= 0 && !injector->isFinished()) { // re-enqueue the injector with the correct timing _injectors.emplace(usecTimestampNow() + nextCallDelta, injector); } From 0e1742588bce3dd6bdec5e580f8b480096118525 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 26 Jan 2016 09:35:35 -0800 Subject: [PATCH 2/7] Fix indentation --- libraries/audio/src/AudioInjector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index bec4921675..660effddeb 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -240,7 +240,7 @@ int64_t AudioInjector::injectNextFrame() { } int bytesToCopy = std::min((_options.stereo ? 2 : 1) * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL, - _audioData.size() - _currentSendOffset); + _audioData.size() - _currentSendOffset); // Measure the loudness of this frame _loudness = 0.0f; From 5268ef8ac21f723e39654d0a7d6dd9735ed333f9 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 26 Jan 2016 09:36:40 -0800 Subject: [PATCH 3/7] Fix else indentation --- libraries/audio/src/AudioInjector.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index 660effddeb..e058a87636 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -288,8 +288,7 @@ int64_t AudioInjector::injectNextFrame() { if (_options.loop) { // we were asked to loop, set our send offset to 0 _currentSendOffset = 0; - } - else { + } else { // we weren't to loop, say that we're done now finish(); return NEXT_FRAME_DELTA_ERROR_OR_FINISHED; From 3160aa306867d791e2fb858b80ba74d37165f9a9 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 26 Jan 2016 09:36:52 -0800 Subject: [PATCH 4/7] Change 0LL to INT64_C(0) --- libraries/audio/src/AudioInjector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index e058a87636..f32b96257d 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -313,7 +313,7 @@ int64_t AudioInjector::injectNextFrame() { } int64_t playNextFrameAt = _nextFrame * AudioConstants::NETWORK_FRAME_USECS; - return std::max(0LL, playNextFrameAt - currentTime); + return std::max(INT64_C(0), playNextFrameAt - currentTime); } void AudioInjector::stop() { From b56668892e82c4bd81bb3973e8cf8532de45722b Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 26 Jan 2016 09:39:11 -0800 Subject: [PATCH 5/7] Fix else style --- libraries/audio/src/AudioInjector.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index f32b96257d..b203800703 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -231,8 +231,7 @@ int64_t AudioInjector::injectNextFrame() { audioDataOffset = _currentPacket->pos(); - } - else { + } else { // no samples to inject, return immediately qDebug() << "AudioInjector::injectNextFrame() called with no samples to inject. Returning."; return NEXT_FRAME_DELTA_ERROR_OR_FINISHED; From a3b2dd2295b3709ae9461ce1eec5cd8eb23f2d7e Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 27 Jan 2016 10:14:05 -0800 Subject: [PATCH 6/7] Add proper loop handling in injector --- libraries/audio/src/AudioInjector.cpp | 52 ++++++++++++++------------- libraries/audio/src/AudioInjector.h | 9 ++--- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index b203800703..fe6c1475b5 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -238,16 +238,19 @@ int64_t AudioInjector::injectNextFrame() { } } - int bytesToCopy = std::min((_options.stereo ? 2 : 1) * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL, - _audioData.size() - _currentSendOffset); + int totalBytesLeftToCopy = (_options.stereo ? 2 : 1) * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; + if (!_options.loop) { + // If we aren't looping, let's make sure we don't read past the end + totalBytesLeftToCopy = std::min(totalBytesLeftToCopy, _audioData.size() - _currentSendOffset); + } // Measure the loudness of this frame _loudness = 0.0f; - for (int i = 0; i < bytesToCopy; i += sizeof(int16_t)) { - _loudness += abs(*reinterpret_cast(_audioData.data() + _currentSendOffset + i)) / + for (int i = 0; i < totalBytesLeftToCopy; i += sizeof(int16_t)) { + _loudness += abs(*reinterpret_cast(_audioData.data() + ((_currentSendOffset + i) % _audioData.size()))) / (AudioConstants::MAX_SAMPLE_VALUE / 2.0f); } - _loudness /= (float)(bytesToCopy / sizeof(int16_t)); + _loudness /= (float)(totalBytesLeftToCopy/ sizeof(int16_t)); _currentPacket->seek(0); @@ -264,8 +267,16 @@ int64_t AudioInjector::injectNextFrame() { _currentPacket->seek(audioDataOffset); - // copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet - _currentPacket->write(_audioData.data() + _currentSendOffset, bytesToCopy); + while (totalBytesLeftToCopy > 0) { + int bytesToCopy = std::min(totalBytesLeftToCopy, _audioData.size() - _currentSendOffset); + + _currentPacket->write(_audioData.data() + _currentSendOffset, bytesToCopy); + _currentSendOffset += bytesToCopy; + totalBytesLeftToCopy -= bytesToCopy; + if (_currentSendOffset >= _audioData.size()) { + _currentSendOffset = 0; + } + } // set the correct size used for this packet _currentPacket->setPayloadSize(_currentPacket->pos()); @@ -280,21 +291,13 @@ int64_t AudioInjector::injectNextFrame() { _outgoingSequenceNumber++; } - _currentSendOffset += bytesToCopy; - - if (_currentSendOffset >= _audioData.size()) { - // we're at the end of the audio data to send - if (_options.loop) { - // we were asked to loop, set our send offset to 0 - _currentSendOffset = 0; - } else { - // we weren't to loop, say that we're done now - finish(); - return NEXT_FRAME_DELTA_ERROR_OR_FINISHED; - } + if (_currentSendOffset >= _audioData.size() && !_options.loop) { + finish(); + return NEXT_FRAME_DELTA_ERROR_OR_FINISHED; } - if (_currentSendOffset == bytesToCopy) { + if (!_hasSentFirstFrame) { + _hasSentFirstFrame = true; // ask AudioInjectorManager to call us right away again to // immediately send the first two frames so the mixer can start using the audio right away return NEXT_FRAME_DELTA_IMMEDIATELY; @@ -303,15 +306,14 @@ int64_t AudioInjector::injectNextFrame() { const int MAX_ALLOWED_FRAMES_TO_FALL_BEHIND = 7; int64_t currentTime = _frameTimer->nsecsElapsed() / 1000; auto currentFrameBasedOnElapsedTime = currentTime / AudioConstants::NETWORK_FRAME_USECS; - if (currentFrameBasedOnElapsedTime - _nextFrame > MAX_ALLOWED_FRAMES_TO_FALL_BEHIND) { // If we are falling behind by more frames than our threshold, let's skip the frames ahead - _nextFrame = currentFrameBasedOnElapsedTime - MAX_ALLOWED_FRAMES_TO_FALL_BEHIND; - } else { - ++_nextFrame; + qDebug() << "AudioInjector::injectNextFrame() skipping ahead, fell behind by " << (currentFrameBasedOnElapsedTime - _nextFrame) << " frames"; + _nextFrame = currentFrameBasedOnElapsedTime; + _currentSendOffset = _nextFrame * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL * (_options.stereo ? 2 : 1) % _audioData.size(); } - int64_t playNextFrameAt = _nextFrame * AudioConstants::NETWORK_FRAME_USECS; + int64_t playNextFrameAt = ++_nextFrame * AudioConstants::NETWORK_FRAME_USECS; return std::max(INT64_C(0), playNextFrameAt - currentTime); } diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index 5155b87a74..acfbc2b04a 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -90,10 +90,11 @@ private: QByteArray _audioData; AudioInjectorOptions _options; State _state { State::NotFinished }; - bool _hasSetup = false; - bool _shouldStop = false; - float _loudness = 0.0f; - int _currentSendOffset = 0; + bool _hasSentFirstFrame { false }; + bool _hasSetup { false }; + bool _shouldStop { false }; + float _loudness { 0.0f }; + int _currentSendOffset { 0 }; std::unique_ptr _currentPacket { nullptr }; AbstractAudioInterface* _localAudioInterface { nullptr }; AudioInjectorLocalBuffer* _localBuffer { nullptr }; From 607f23618d93bdd21e5dfbfde39c9c6aac08a6cb Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 27 Jan 2016 10:36:44 -0800 Subject: [PATCH 7/7] Fix compiler warning for float->int conversion --- libraries/audio/src/AudioConstants.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio/src/AudioConstants.h b/libraries/audio/src/AudioConstants.h index 91080017a5..6c12c6aa15 100644 --- a/libraries/audio/src/AudioConstants.h +++ b/libraries/audio/src/AudioConstants.h @@ -29,7 +29,7 @@ namespace AudioConstants { const int NETWORK_FRAME_SAMPLES_PER_CHANNEL = NETWORK_FRAME_BYTES_PER_CHANNEL / sizeof(AudioSample); const float NETWORK_FRAME_MSECS = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL / (float)AudioConstants::SAMPLE_RATE) * 1000.0f; - const int NETWORK_FRAME_USECS = floorf(NETWORK_FRAME_MSECS * 1000.0f); + const int NETWORK_FRAME_USECS = static_cast(NETWORK_FRAME_MSECS * 1000.0f); const int MIN_SAMPLE_VALUE = std::numeric_limits::min(); const int MAX_SAMPLE_VALUE = std::numeric_limits::max(); }