From 1f9ca00317019017bc97880838e8da77b23835f3 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Dec 2013 11:58:23 -0800 Subject: [PATCH] drive input from buffer callback and output from network --- assignment-client/src/audio/AudioMixer.cpp | 46 ++-- assignment-client/src/audio/AudioMixer.h | 2 +- .../src/audio/AudioMixerClientData.cpp | 2 +- interface/src/Audio.cpp | 255 +++++++++--------- interface/src/Audio.h | 4 +- libraries/audio/src/AudioRingBuffer.cpp | 127 +++++---- libraries/audio/src/AudioRingBuffer.h | 32 ++- .../audio/src/InjectedAudioRingBuffer.cpp | 2 +- .../audio/src/PositionalAudioRingBuffer.cpp | 31 +-- 9 files changed, 270 insertions(+), 231 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 0b89f5a83c..463746c8f2 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -54,7 +54,7 @@ const short JITTER_BUFFER_MSECS = 12; const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_MSECS * (SAMPLE_RATE / 1000.0); -const unsigned int BUFFER_SEND_INTERVAL_USECS = floorf((BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float) SAMPLE_RATE) * 1000 * 1000); +const unsigned int BUFFER_SEND_INTERVAL_USECS = floorf((NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float) SAMPLE_RATE) * 1000 * 1000); const int MAX_SAMPLE_VALUE = std::numeric_limits::max(); const int MIN_SAMPLE_VALUE = std::numeric_limits::min(); @@ -164,27 +164,29 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf int delayedChannelOffset = (bearingRelativeAngleToSource > 0.0f) ? 1 : 0; int goodChannelOffset = delayedChannelOffset == 0 ? 1 : 0; - for (int s = 0; s < BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2; s += 2) { - if (s < numSamplesDelay) { - // pull the earlier sample for the delayed channel - int earlierSample = (*bufferToAdd)[(s / 2) - numSamplesDelay] * attenuationCoefficient * weakChannelAmplitudeRatio; - - _clientSamples[s + delayedChannelOffset] = glm::clamp(_clientSamples[s + delayedChannelOffset] + earlierSample, - MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); - } + for (int s = 0; s < NETWORK_BUFFER_LENGTH_SAMPLES_STEREO; s += 2) { +// if (s < numSamplesDelay) { +// // pull the earlier sample for the delayed channel +// int earlierSample = (*bufferToAdd)[(s / 2) - numSamplesDelay] * attenuationCoefficient * weakChannelAmplitudeRatio; +// +// _clientSamples[s + delayedChannelOffset] = glm::clamp(_clientSamples[s + delayedChannelOffset] + earlierSample, +// MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); +// } +// +// // pull the current sample for the good channel +// int16_t currentSample = (*bufferToAdd)[s / 2] * attenuationCoefficient; +// _clientSamples[s + goodChannelOffset] = glm::clamp(_clientSamples[s + goodChannelOffset] + currentSample, +// MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); +// +// if (s + numSamplesDelay < NETWORK_BUFFER_LENGTH_SAMPLES_STEREO) { +// // place the curernt sample at the right spot in the delayed channel +// int16_t clampedSample = glm::clamp((int) (_clientSamples[s + numSamplesDelay + delayedChannelOffset] +// + (currentSample * weakChannelAmplitudeRatio)), +// MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); +// _clientSamples[s + numSamplesDelay + delayedChannelOffset] = clampedSample; +// } - // pull the current sample for the good channel - int16_t currentSample = (*bufferToAdd)[s / 2] * attenuationCoefficient; - _clientSamples[s + goodChannelOffset] = glm::clamp(_clientSamples[s + goodChannelOffset] + currentSample, - MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); - - if (s + numSamplesDelay < BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { - // place the curernt sample at the right spot in the delayed channel - int16_t clampedSample = glm::clamp((int) (_clientSamples[s + numSamplesDelay + delayedChannelOffset] - + (currentSample * weakChannelAmplitudeRatio)), - MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); - _clientSamples[s + numSamplesDelay + delayedChannelOffset] = clampedSample; - } + _clientSamples[s] = _clientSamples[s + 1] = (*bufferToAdd)[s / 2] * attenuationCoefficient; } } @@ -277,7 +279,7 @@ void AudioMixer::run() { gettimeofday(&startTime, NULL); int numBytesPacketHeader = numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_MIXED_AUDIO); - unsigned char clientPacket[BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader]; + unsigned char clientPacket[NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader]; populateTypeAndVersion(clientPacket, PACKET_TYPE_MIXED_AUDIO); while (!_isFinished) { diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 2c07e8747a..7326e1a161 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -35,7 +35,7 @@ private: void prepareMixForListeningNode(Node* node); - int16_t _clientSamples[BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2]; + int16_t _clientSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO]; }; #endif /* defined(__hifi__AudioMixer__) */ diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index b7f0aeac5f..4827fbc918 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -91,7 +91,7 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { PositionalAudioRingBuffer* audioBuffer = _ringBuffers[i]; if (audioBuffer->willBeAddedToMix()) { - audioBuffer->shiftReadPosition(BUFFER_LENGTH_SAMPLES_PER_CHANNEL); + audioBuffer->shiftReadPosition(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); audioBuffer->setWillBeAddedToMix(false); } else if (audioBuffer->isStarved()) { diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 5c359ce898..9c6b66edc5 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -13,6 +13,7 @@ #include #endif +#include #include #include #include @@ -33,7 +34,7 @@ static const float JITTER_BUFFER_LENGTH_MSECS = 12; static const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_LENGTH_MSECS * NUM_AUDIO_CHANNELS * (SAMPLE_RATE / 1000.0); -static const float AUDIO_CALLBACK_MSECS = (float)BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; +static const float AUDIO_CALLBACK_MSECS = (float) NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; // Mute icon configration static const int ICON_SIZE = 24; @@ -45,17 +46,15 @@ Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* p _audioInput(NULL), _desiredInputFormat(), _inputFormat(), - _inputDevice(NULL), _inputBuffer(), _numInputCallbackBytes(0), _audioOutput(NULL), _desiredOutputFormat(), _outputFormat(), _outputDevice(NULL), - _outputBuffer(), _numOutputCallbackBytes(0), - _nextOutputSamples(NULL), - _ringBuffer(true), + _inputRingBuffer(0), + _ringBuffer(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2), _scope(scope), _averagedLatency(0.0), _measuredJitter(0), @@ -249,7 +248,7 @@ void Audio::start() { qDebug() << "The format to be used for audio input is" << _inputFormat << "\n"; _audioInput = new QAudioInput(inputDeviceInfo, _inputFormat, this); - _numInputCallbackBytes = BUFFER_LENGTH_BYTES_PER_CHANNEL * _inputFormat.channelCount() + _numInputCallbackBytes = NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL * _inputFormat.channelCount() * (_inputFormat.sampleRate() / SAMPLE_RATE) / CALLBACK_ACCELERATOR_RATIO; _audioInput->setBufferSize(_numInputCallbackBytes); @@ -261,14 +260,11 @@ void Audio::start() { if (adjustedFormatForAudioDevice(outputDeviceInfo, _desiredOutputFormat, _outputFormat)) { qDebug() << "The format to be used for audio output is" << _outputFormat << "\n"; + _inputRingBuffer.resizeForFrameSize(_numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO / sizeof(int16_t)); _inputDevice = _audioInput->start(); - connect(_inputDevice, SIGNAL(readyRead()), SLOT(handleAudioInput())); + connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleAudioInput())); _audioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this); - _numOutputCallbackBytes = BUFFER_LENGTH_BYTES_PER_CHANNEL * _outputFormat.channelCount() - * (_outputFormat.sampleRate() / SAMPLE_RATE) - / CALLBACK_ACCELERATOR_RATIO; - _audioOutput->setBufferSize(_numOutputCallbackBytes); _outputDevice = _audioOutput->start(); gettimeofday(&_lastReceiveTime, NULL); @@ -281,50 +277,66 @@ void Audio::start() { } void Audio::handleAudioInput() { - static char monoAudioDataPacket[MAX_PACKET_SIZE]; static int numBytesPacketHeader = numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO); static int leadingBytes = numBytesPacketHeader + sizeof(glm::vec3) + sizeof(glm::quat) + NUM_BYTES_RFC4122_UUID; static int16_t* monoAudioSamples = (int16_t*) (monoAudioDataPacket + leadingBytes); - static float inputToOutputRatio = _numOutputCallbackBytes / _numInputCallbackBytes; - static float inputToNetworkInputRatio = _numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO / BUFFER_LENGTH_BYTES_PER_CHANNEL; + static float inputToNetworkInputRatio = _numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO + / NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; + + static int inputSamplesRequired = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * inputToNetworkInputRatio; QByteArray inputByteArray = _inputDevice->readAll(); + + _inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size()); + + while (_inputRingBuffer.samplesAvailable() > inputSamplesRequired) { + + int16_t inputAudioSamples[inputSamplesRequired]; + _inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired); + + // zero out the monoAudioSamples array + memset(monoAudioSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); + + if (!_muted) { + // we aren't muted, downsample the input audio + linearResampling((int16_t*) inputAudioSamples, + monoAudioSamples, + inputSamplesRequired, + NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, + _inputFormat, _desiredInputFormat); + + // add input data just written to the scope + // QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection, + // Q_ARG(QByteArray, inputByteArray), Q_ARG(bool, true)); + } + +// if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio)) { +// // if local loopback enabled, copy input to output +// QByteArray samplesForOutput; +// samplesForOutput.resize(inputSamplesRequired * outputToInputRatio * sizeof(int16_t)); +// +// linearResampling(monoAudioSamples, (int16_t*) samplesForOutput.data(), +// NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, +// inputSamplesRequired, +// _desiredInputFormat, _outputFormat); +// +// _outputDevice->write(samplesForOutput); +// } - int numResampledNetworkInputBytes = inputByteArray.size() / inputToNetworkInputRatio; - int numResampledNetworkInputSamples = numResampledNetworkInputBytes / sizeof(int16_t); - - // zero out the monoAudioSamples array - memset(monoAudioSamples, 0, numResampledNetworkInputBytes); - - if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio) && !_muted) { - _outputBuffer.resize(inputByteArray.size()); - // if local loopback enabled, copy input to output - linearResampling((int16_t*) inputByteArray.data(), (int16_t*) _outputBuffer.data(), - inputByteArray.size() / sizeof(int16_t), - inputByteArray.size() * inputToOutputRatio / sizeof(int16_t), - _inputFormat, _outputFormat); - } else { - _outputBuffer.fill(0, inputByteArray.size()); - } - - // add input data just written to the scope - // QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection, - // Q_ARG(QByteArray, inputByteArray), Q_ARG(bool, true)); - - // add procedural effects to the appropriate input samples - // addProceduralSounds(monoAudioSamples + (_isBufferSendCallback - // ? BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO : 0), - // (int16_t*) stereoOutputBuffer.data(), - // BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO); - - NodeList* nodeList = NodeList::getInstance(); - Node* audioMixer = nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER); - - if (false) { - if (audioMixer->getActiveSocket()) { + + // add procedural effects to the appropriate input samples + // addProceduralSounds(monoAudioSamples + (_isBufferSendCallback + // ? BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO : 0), + // (int16_t*) stereoOutputBuffer.data(), + // BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO); + + NodeList* nodeList = NodeList::getInstance(); + Node* audioMixer = nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER); + + if (audioMixer && nodeList->getNodeActiveSocketOrPing(audioMixer)) { MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar(); glm::vec3 headPosition = interfaceAvatar->getHeadJointPosition(); @@ -334,7 +346,7 @@ void Audio::handleAudioInput() { // + 12 for 3 floats for position + float for bearing + 1 attenuation byte PACKET_TYPE packetType = Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio) - ? PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO : PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO; + ? PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO : PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO; char* currentPacketPtr = monoAudioDataPacket + populateTypeAndVersion((unsigned char*) monoAudioDataPacket, packetType); @@ -357,94 +369,19 @@ void Audio::handleAudioInput() { // loudness /= BUFFER_LENGTH_SAMPLES_PER_CHANNEL; _lastInputLoudness = loudness; - // we aren't muted - pull our input audio to send off to the mixer - linearResampling((int16_t*) inputByteArray.data(), - monoAudioSamples, - inputByteArray.size() / sizeof(int16_t), - numResampledNetworkInputSamples, - _inputFormat, _desiredInputFormat); - } else { _lastInputLoudness = 0; } nodeList->getNodeSocket().writeDatagram(monoAudioDataPacket, - numResampledNetworkInputBytes + leadingBytes, + NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes, audioMixer->getActiveSocket()->getAddress(), audioMixer->getActiveSocket()->getPort()); Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) - .updateValue(BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes); - } else { - nodeList->pingPublicAndLocalSocketsForInactiveNode(audioMixer); + .updateValue(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes); } } - - if (_outputDevice) { - - int numRequiredNetworkOutputSamples = numResampledNetworkInputSamples - * (_desiredOutputFormat.channelCount() / _desiredInputFormat.channelCount()); - - int numResampledOutputBytes = _inputBuffer.size() * inputToOutputRatio; - - // linearResampling((int16_t*) inputByteArray.data(), - // monoAudioSamples, - // inputByteArray.size() / sizeof(int16_t), - // numResampledNetworkInputSamples, - // _inputFormat, _desiredInputFormat); - - // copy the packet from the RB to the output - // linearResampling(monoAudioSamples, - // (int16_t*) _outputBuffer.data(), - // numResampledNetworkInputSamples, - // numResampledOutputBytes / sizeof(int16_t), - // _desiredInputFormat, _outputFormat); - - - // if there is anything in the ring buffer, decide what to do - if (false) { - - - - if (!_ringBuffer.isNotStarvedOrHasMinimumSamples(numRequiredNetworkOutputSamples)) { - // starved and we don't have enough to start, keep waiting - qDebug() << "Buffer is starved and doesn't have enough samples to start. Held back.\n"; - } else { - // We are either already playing back, or we have enough audio to start playing back. - if (_ringBuffer.isStarved()) { - _ringBuffer.setIsStarved(false); - } - - int numResampledOutputBytes = inputByteArray.size() * inputToOutputRatio; - - // copy the samples we'll resample from the ring buffer - this also - // pushes the read pointer of the ring buffer forwards - int16_t ringBufferSamples[numRequiredNetworkOutputSamples]; - _ringBuffer.read(ringBufferSamples, numRequiredNetworkOutputSamples); - - // copy the packet from the RB to the output - linearResampling(ringBufferSamples, - (int16_t*) _outputBuffer.data(), - numRequiredNetworkOutputSamples, - numResampledOutputBytes / sizeof(int16_t), - _desiredOutputFormat, _outputFormat); - - } - } else if (_audioOutput->bytesFree() == _audioOutput->bufferSize()) { - // we don't have any audio data left in the output buffer, and the ring buffer from - // the network has nothing in it either - we just starved - _ringBuffer.setIsStarved(true); - _numFramesDisplayStarve = 10; - } - - // add output (@speakers) data just written to the scope - // QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection, - // Q_ARG(QByteArray, stereoOutputBuffer), Q_ARG(bool, false)); - - _outputDevice->write(_outputBuffer); - } - - gettimeofday(&_lastCallbackTime, NULL); } void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { @@ -466,7 +403,7 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { _measuredJitter = _stdev.getStDev(); _stdev.reset(); // Set jitter buffer to be a multiple of the measured standard deviation - const int MAX_JITTER_BUFFER_SAMPLES = RING_BUFFER_LENGTH_SAMPLES / 2; + const int MAX_JITTER_BUFFER_SAMPLES = _ringBuffer.getSampleCapacity() / 2; const float NUM_STANDARD_DEVIATIONS = 3.f; if (Menu::getInstance()->getAudioJitterBufferSamples() == 0) { float newJitterBufferSamples = (NUM_STANDARD_DEVIATIONS * _measuredJitter) / 1000.f * SAMPLE_RATE; @@ -476,8 +413,70 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { _ringBuffer.parseData((unsigned char*) audioByteArray.data(), audioByteArray.size()); - Application::getInstance()->getBandwidthMeter()->inputStream(BandwidthMeter::AUDIO).updateValue(PACKET_LENGTH_BYTES - + sizeof(PACKET_TYPE)); + static float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float) _outputFormat.sampleRate()) + * (_desiredOutputFormat.channelCount() / (float) _outputFormat.channelCount()); + + static int numRequiredOutputSamples = NETWORK_BUFFER_LENGTH_SAMPLES_STEREO / networkOutputToOutputRatio; + + int16_t outputBuffer[numRequiredOutputSamples]; + + // linearResampling((int16_t*) inputByteArray.data(), + // monoAudioSamples, + // inputByteArray.size() / sizeof(int16_t), + // numResampledNetworkInputSamples, + // _inputFormat, _desiredInputFormat); + + // copy the packet from the RB to the output + // linearResampling(monoAudioSamples, + // (int16_t*) _outputBuffer.data(), + // numResampledNetworkInputSamples, + // numResampledOutputBytes / sizeof(int16_t), + // _desiredInputFormat, _outputFormat); + + + // if there is anything in the ring buffer, decide what to do + if (_ringBuffer.samplesAvailable() > 0) { + if (!_ringBuffer.isNotStarvedOrHasMinimumSamples(NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + + (_jitterBufferSamples * 2))) { + // starved and we don't have enough to start, keep waiting + qDebug() << "Buffer is starved and doesn't have enough samples to start. Held back.\n"; + } else { + // We are either already playing back, or we have enough audio to start playing back. + if (_ringBuffer.isStarved()) { + _ringBuffer.setIsStarved(false); + } + + // copy the samples we'll resample from the ring buffer - this also + // pushes the read pointer of the ring buffer forwards + int16_t ringBufferSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO]; + _ringBuffer.readSamples(ringBufferSamples, NETWORK_BUFFER_LENGTH_SAMPLES_STEREO); + + // copy the packet from the RB to the output + linearResampling(ringBufferSamples, + outputBuffer, + NETWORK_BUFFER_LENGTH_SAMPLES_STEREO, + numRequiredOutputSamples, + _desiredOutputFormat, _outputFormat); + + if (_outputDevice) { + _outputDevice->write((char*) outputBuffer, numRequiredOutputSamples * sizeof(int16_t)); + } + } + + } else if (_audioOutput->bytesFree() == _audioOutput->bufferSize()) { + // we don't have any audio data left in the output buffer, and the ring buffer from + // the network has nothing in it either - we just starved + _ringBuffer.setIsStarved(true); + _numFramesDisplayStarve = 10; + } + + // add output (@speakers) data just written to the scope + // QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection, + // Q_ARG(QByteArray, stereoOutputBuffer), Q_ARG(bool, false)); + + + + Application::getInstance()->getBandwidthMeter()->inputStream(BandwidthMeter::AUDIO).updateValue(audioByteArray.size()); _lastReceiveTime = currentReceiveTime; } @@ -508,7 +507,7 @@ void Audio::render(int screenWidth, int screenHeight) { glVertex2f(currentX, topY); glVertex2f(currentX, bottomY); - for (int i = 0; i < RING_BUFFER_LENGTH_FRAMES / 2; i++) { + for (int i = 0; i < _ringBuffer.getSampleCapacity() / 2; i++) { glVertex2f(currentX, halfY); glVertex2f(currentX + frameWidth, halfY); currentX += frameWidth; diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 844c44c6dc..8557120ab7 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -70,6 +70,7 @@ public slots: void reset(); private: + QByteArray firstInputFrame; QAudioInput* _audioInput; QAudioFormat _desiredInputFormat; QAudioFormat _inputFormat; @@ -80,9 +81,8 @@ private: QAudioFormat _desiredOutputFormat; QAudioFormat _outputFormat; QIODevice* _outputDevice; - QByteArray _outputBuffer; int _numOutputCallbackBytes; - int16_t* _nextOutputSamples; + AudioRingBuffer _inputRingBuffer; AudioRingBuffer _ringBuffer; Oscilloscope* _scope; StDev _stdev; diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index e0c4ebb32a..83c2fe59a0 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -15,13 +15,23 @@ #include "AudioRingBuffer.h" -AudioRingBuffer::AudioRingBuffer(bool isStereo) : +const short RING_BUFFER_LENGTH_FRAMES = 10; + +AudioRingBuffer::AudioRingBuffer(int numFrameSamples) : NodeData(NULL), - _endOfLastWrite(NULL), - _isStarved(true) + _sampleCapacity(numFrameSamples * RING_BUFFER_LENGTH_FRAMES), + _isStarved(true), + _hasStarted(false) { - _buffer = new int16_t[RING_BUFFER_LENGTH_SAMPLES]; - _nextOutput = _buffer; + if (numFrameSamples) { + _buffer = new int16_t[_sampleCapacity]; + _nextOutput = _buffer; + _endOfLastWrite = _buffer; + } else { + _buffer = NULL; + _nextOutput = NULL; + _endOfLastWrite = NULL; + } }; AudioRingBuffer::~AudioRingBuffer() { @@ -34,20 +44,64 @@ void AudioRingBuffer::reset() { _isStarved = true; } -int AudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes) { - int numBytesPacketHeader = numBytesForPacketHeader(sourceBuffer); - return parseAudioSamples(sourceBuffer + numBytesPacketHeader, numBytes - numBytesPacketHeader); +void AudioRingBuffer::resizeForFrameSize(qint64 numFrameSamples) { + delete[] _buffer; + _sampleCapacity = numFrameSamples * RING_BUFFER_LENGTH_FRAMES; + _buffer = new int16_t[_sampleCapacity]; + _nextOutput = _buffer; + _endOfLastWrite = _buffer; } -int AudioRingBuffer::parseAudioSamples(unsigned char* sourceBuffer, int numBytes) { +int AudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes) { + int numBytesPacketHeader = numBytesForPacketHeader(sourceBuffer); + return writeData((char*) sourceBuffer + numBytesPacketHeader, numBytes - numBytesPacketHeader); +} + +qint64 AudioRingBuffer::readSamples(int16_t* destination, qint64 maxSamples) { + return readData((char*) destination, maxSamples * sizeof(int16_t)); +} + +qint64 AudioRingBuffer::readData(char *data, qint64 maxSize) { + + // only copy up to the number of samples we have available + int numReadSamples = std::min((unsigned) (maxSize / sizeof(int16_t)), samplesAvailable()); + + if (_nextOutput + numReadSamples > _buffer + _sampleCapacity) { + // we're going to need to do two reads to get this data, it wraps around the edge + + // read to the end of the buffer + int numSamplesToEnd = (_buffer + _sampleCapacity) - _nextOutput; + memcpy(data, _nextOutput, numSamplesToEnd * sizeof(int16_t)); + + // read the rest from the beginning of the buffer + memcpy(data + numSamplesToEnd, _buffer, (numReadSamples - numSamplesToEnd) * sizeof(int16_t)); + } else { + // read the data + memcpy(data, _nextOutput, numReadSamples * sizeof(int16_t)); + } + + // push the position of _nextOutput by the number of samples read + _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numReadSamples); + + return numReadSamples * sizeof(int16_t); +} + +qint64 AudioRingBuffer::writeSamples(const int16_t* source, qint64 maxSamples) { + return writeData((const char*) source, maxSamples * sizeof(int16_t)); +} + +qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { // make sure we have enough bytes left for this to be the right amount of audio // otherwise we should not copy that data, and leave the buffer pointers where they are - int samplesToCopy = numBytes / sizeof(int16_t); + int samplesToCopy = std::min(maxSize / sizeof(int16_t), (quint64) _sampleCapacity); - if (!_endOfLastWrite) { - _endOfLastWrite = _buffer; - } else if (samplesToCopy > RING_BUFFER_LENGTH_SAMPLES - samplesAvailable()) { + std::less less; + std::less_equal lessEqual; + + if (_hasStarted + && (less(_endOfLastWrite, _nextOutput) + && lessEqual(_nextOutput, shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy)))) { // this read will cross the next output, so call us starved and reset the buffer qDebug() << "Filled the ring buffer. Resetting.\n"; _endOfLastWrite = _buffer; @@ -55,49 +109,28 @@ int AudioRingBuffer::parseAudioSamples(unsigned char* sourceBuffer, int numBytes _isStarved = true; } - if (_endOfLastWrite + samplesToCopy <= _buffer + RING_BUFFER_LENGTH_SAMPLES) { - memcpy(_endOfLastWrite, sourceBuffer, numBytes); + _hasStarted = true; + + if (_endOfLastWrite + samplesToCopy <= _buffer + _sampleCapacity) { + memcpy(_endOfLastWrite, data, samplesToCopy * sizeof(int16_t)); } else { - int numSamplesToEnd = (_buffer + RING_BUFFER_LENGTH_SAMPLES) - _endOfLastWrite; - memcpy(_endOfLastWrite, sourceBuffer, numSamplesToEnd * sizeof(int16_t)); - memcpy(_buffer, sourceBuffer + (numSamplesToEnd * sizeof(int16_t)), (samplesToCopy - numSamplesToEnd) * sizeof(int16_t)); + int numSamplesToEnd = (_buffer + _sampleCapacity) - _endOfLastWrite; + memcpy(_endOfLastWrite, data, numSamplesToEnd * sizeof(int16_t)); + memcpy(_buffer, data + (numSamplesToEnd * sizeof(int16_t)), (samplesToCopy - numSamplesToEnd) * sizeof(int16_t)); } _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy); - return numBytes; + return samplesToCopy * sizeof(int16_t); } int16_t& AudioRingBuffer::operator[](const int index) { // make sure this is a valid index - assert(index > -RING_BUFFER_LENGTH_SAMPLES && index < RING_BUFFER_LENGTH_SAMPLES); + assert(index > -_sampleCapacity && index < _sampleCapacity); return *shiftedPositionAccomodatingWrap(_nextOutput, index); } -void AudioRingBuffer::read(int16_t* destination, unsigned int maxSamples) { - - // only copy up to the number of samples we have available - int numReadSamples = std::min(maxSamples, samplesAvailable()); - - if (_nextOutput + numReadSamples > _buffer + RING_BUFFER_LENGTH_SAMPLES) { - // we're going to need to do two reads to get this data, it wraps around the edge - - // read to the end of the buffer - int numSamplesToEnd = (_buffer + RING_BUFFER_LENGTH_SAMPLES) - _nextOutput; - memcpy(destination, _nextOutput, numSamplesToEnd * sizeof(int16_t)); - - // read the rest from the beginning of the buffer - memcpy(destination + numSamplesToEnd, _buffer, (numReadSamples - numSamplesToEnd) * sizeof(int16_t)); - } else { - // read the data - memcpy(destination, _nextOutput, numReadSamples * sizeof(int16_t)); - } - - // push the position of _nextOutput by the number of samples read - _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numReadSamples); -} - void AudioRingBuffer::shiftReadPosition(unsigned int numSamples) { _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); } @@ -109,7 +142,7 @@ unsigned int AudioRingBuffer::samplesAvailable() const { int sampleDifference = _endOfLastWrite - _nextOutput; if (sampleDifference < 0) { - sampleDifference += RING_BUFFER_LENGTH_SAMPLES; + sampleDifference += _sampleCapacity; } return sampleDifference; @@ -126,12 +159,12 @@ bool AudioRingBuffer::isNotStarvedOrHasMinimumSamples(unsigned int numRequiredSa int16_t* AudioRingBuffer::shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const { - if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + RING_BUFFER_LENGTH_SAMPLES) { + if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + _sampleCapacity) { // this shift will wrap the position around to the beginning of the ring - return position + numSamplesShift - RING_BUFFER_LENGTH_SAMPLES; + return position + numSamplesShift - _sampleCapacity; } else if (numSamplesShift < 0 && position + numSamplesShift < _buffer) { // this shift will go around to the end of the ring - return position + numSamplesShift - RING_BUFFER_LENGTH_SAMPLES; + return position + numSamplesShift - _sampleCapacity; } else { return position + numSamplesShift; } diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index aafc3f20f6..d715e63a97 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -10,35 +10,41 @@ #define __interface__AudioRingBuffer__ #include -#include #include +#include + #include "NodeData.h" const int SAMPLE_RATE = 24000; -const int BUFFER_LENGTH_BYTES_STEREO = 1024; -const int BUFFER_LENGTH_BYTES_PER_CHANNEL = 512; -const int BUFFER_LENGTH_SAMPLES_PER_CHANNEL = BUFFER_LENGTH_BYTES_PER_CHANNEL / sizeof(int16_t); - -const short RING_BUFFER_LENGTH_FRAMES = 20; -const short RING_BUFFER_LENGTH_SAMPLES = RING_BUFFER_LENGTH_FRAMES * BUFFER_LENGTH_SAMPLES_PER_CHANNEL; +const int NETWORK_BUFFER_LENGTH_BYTES_STEREO = 1024; +const int NETWORK_BUFFER_LENGTH_SAMPLES_STEREO = NETWORK_BUFFER_LENGTH_BYTES_STEREO / sizeof(int16_t); +const int NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL = 512; +const int NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL = NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL / sizeof(int16_t); class AudioRingBuffer : public NodeData { + Q_OBJECT public: - AudioRingBuffer(bool isStereo); + AudioRingBuffer(int numFrameSamples); ~AudioRingBuffer(); void reset(); - + void resizeForFrameSize(qint64 numFrameSamples); + + int getSampleCapacity() const { return _sampleCapacity; } + int parseData(unsigned char* sourceBuffer, int numBytes); - int parseAudioSamples(unsigned char* sourceBuffer, int numBytes); + + qint64 readSamples(int16_t* destination, qint64 maxSamples); + qint64 writeSamples(const int16_t* source, qint64 maxSamples); + + qint64 readData(char* data, qint64 maxSize); + qint64 writeData(const char* data, qint64 maxSize); int16_t& operator[](const int index); - void read(int16_t* destination, unsigned int numSamples); - void shiftReadPosition(unsigned int numSamples); unsigned int samplesAvailable() const; @@ -54,10 +60,12 @@ protected: int16_t* shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const; + int _sampleCapacity; int16_t* _nextOutput; int16_t* _endOfLastWrite; int16_t* _buffer; bool _isStarved; + bool _hasStarted; }; #endif /* defined(__interface__AudioRingBuffer__) */ diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index 9adb525b93..d66a24672a 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -42,7 +42,7 @@ int InjectedAudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes unsigned int attenuationByte = *(currentBuffer++); _attenuationRatio = attenuationByte / (float) MAX_INJECTOR_VOLUME; - currentBuffer += parseAudioSamples(currentBuffer, numBytes - (currentBuffer - sourceBuffer)); + currentBuffer += writeData((char*) currentBuffer, numBytes - (currentBuffer - sourceBuffer)); return currentBuffer - sourceBuffer; } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index c84444516e..0b7c26dc7d 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -15,7 +15,7 @@ #include "PositionalAudioRingBuffer.h" PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type) : - AudioRingBuffer(false), + AudioRingBuffer(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL), _type(type), _position(0.0f, 0.0f, 0.0f), _orientation(0.0f, 0.0f, 0.0f, 0.0f), @@ -31,7 +31,7 @@ int PositionalAudioRingBuffer::parseData(unsigned char* sourceBuffer, int numByt unsigned char* currentBuffer = sourceBuffer + numBytesForPacketHeader(sourceBuffer); currentBuffer += NUM_BYTES_RFC4122_UUID; // the source UUID currentBuffer += parsePositionalData(currentBuffer, numBytes - (currentBuffer - sourceBuffer)); - currentBuffer += parseAudioSamples(currentBuffer, numBytes - (currentBuffer - sourceBuffer)); + currentBuffer += writeData((char*) currentBuffer, numBytes - (currentBuffer - sourceBuffer)); return currentBuffer - sourceBuffer; } @@ -47,8 +47,7 @@ int PositionalAudioRingBuffer::parsePositionalData(unsigned char* sourceBuffer, // if this node sent us a NaN for first float in orientation then don't consider this good audio and bail if (std::isnan(_orientation.x)) { - _endOfLastWrite = _nextOutput = _buffer; - _isStarved = true; + reset(); return 0; } @@ -56,19 +55,17 @@ int PositionalAudioRingBuffer::parsePositionalData(unsigned char* sourceBuffer, } bool PositionalAudioRingBuffer::shouldBeAddedToMix(int numJitterBufferSamples) { - if (_endOfLastWrite) { - if (!isNotStarvedOrHasMinimumSamples(BUFFER_LENGTH_SAMPLES_PER_CHANNEL + numJitterBufferSamples)) { - qDebug() << "Starved and do not have minimum samples to start. Buffer held back.\n"; - return false; - } else if (samplesAvailable() < BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { - qDebug() << "Do not have number of samples needed for interval. Buffer starved.\n"; - _isStarved = true; - return false; - } else { - // good buffer, add this to the mix - _isStarved = false; - return true; - } + if (!isNotStarvedOrHasMinimumSamples(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL + numJitterBufferSamples)) { + qDebug() << "Starved and do not have minimum samples to start. Buffer held back.\n"; + return false; + } else if (samplesAvailable() < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { + qDebug() << "Do not have number of samples needed for interval. Buffer starved.\n"; + _isStarved = true; + return false; + } else { + // good buffer, add this to the mix + _isStarved = false; + return true; } return false;