diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 81b7d5e614..700642bf59 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -54,10 +53,7 @@ const short JITTER_BUFFER_MSECS = 12; const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_MSECS * (SAMPLE_RATE / 1000.0); -const unsigned int BUFFER_SEND_INTERVAL_USECS = floorf((BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float) SAMPLE_RATE) * 1000 * 1000); - -const int MAX_SAMPLE_VALUE = std::numeric_limits::max(); -const int MIN_SAMPLE_VALUE = std::numeric_limits::min(); +const unsigned int BUFFER_SEND_INTERVAL_USECS = floorf((NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float) SAMPLE_RATE) * 1000 * 1000); const char AUDIO_MIXER_LOGGING_TARGET_NAME[] = "audio-mixer"; @@ -160,35 +156,30 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf } } - int16_t* sourceBuffer = bufferToAdd->getNextOutput(); + // if the bearing relative angle to source is > 0 then the delayed channel is the right one + int delayedChannelOffset = (bearingRelativeAngleToSource > 0.0f) ? 1 : 0; + int goodChannelOffset = delayedChannelOffset == 0 ? 1 : 0; - int16_t* goodChannel = (bearingRelativeAngleToSource > 0.0f) - ? _clientSamples - : _clientSamples + BUFFER_LENGTH_SAMPLES_PER_CHANNEL; - int16_t* delayedChannel = (bearingRelativeAngleToSource > 0.0f) - ? _clientSamples + BUFFER_LENGTH_SAMPLES_PER_CHANNEL - : _clientSamples; - - int16_t* delaySamplePointer = bufferToAdd->getNextOutput() == bufferToAdd->getBuffer() - ? bufferToAdd->getBuffer() + RING_BUFFER_LENGTH_SAMPLES - numSamplesDelay - : bufferToAdd->getNextOutput() - numSamplesDelay; - - for (int s = 0; s < BUFFER_LENGTH_SAMPLES_PER_CHANNEL; s++) { - if (s < numSamplesDelay) { + for (int s = 0; s < NETWORK_BUFFER_LENGTH_SAMPLES_STEREO; s += 2) { + if ((s / 2) < numSamplesDelay) { // pull the earlier sample for the delayed channel - int earlierSample = delaySamplePointer[s] * attenuationCoefficient * weakChannelAmplitudeRatio; + int earlierSample = (*bufferToAdd)[(s / 2) - numSamplesDelay] * attenuationCoefficient * weakChannelAmplitudeRatio; - delayedChannel[s] = glm::clamp(delayedChannel[s] + earlierSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + _clientSamples[s + delayedChannelOffset] = glm::clamp(_clientSamples[s + delayedChannelOffset] + earlierSample, + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); } // pull the current sample for the good channel - int16_t currentSample = sourceBuffer[s] * attenuationCoefficient; - goodChannel[s] = glm::clamp(goodChannel[s] + currentSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + int16_t currentSample = (*bufferToAdd)[s / 2] * attenuationCoefficient; + _clientSamples[s + goodChannelOffset] = glm::clamp(_clientSamples[s + goodChannelOffset] + currentSample, + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); - if (s + numSamplesDelay < BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { + if ((s / 2) + numSamplesDelay < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { // place the curernt sample at the right spot in the delayed channel - int sumSample = delayedChannel[s + numSamplesDelay] + (currentSample * weakChannelAmplitudeRatio); - delayedChannel[s + numSamplesDelay] = glm::clamp(sumSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + int16_t clampedSample = glm::clamp((int) (_clientSamples[s + numSamplesDelay + delayedChannelOffset] + + (currentSample * weakChannelAmplitudeRatio)), + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + _clientSamples[s + numSamplesDelay + delayedChannelOffset] = clampedSample; } } } @@ -282,7 +273,7 @@ void AudioMixer::run() { gettimeofday(&startTime, NULL); int numBytesPacketHeader = numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_MIXED_AUDIO); - unsigned char clientPacket[BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader]; + unsigned char clientPacket[NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader]; populateTypeAndVersion(clientPacket, PACKET_TYPE_MIXED_AUDIO); while (!_isFinished) { diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 2c07e8747a..7326e1a161 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -35,7 +35,7 @@ private: void prepareMixForListeningNode(Node* node); - int16_t _clientSamples[BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2]; + int16_t _clientSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO]; }; #endif /* defined(__hifi__AudioMixer__) */ diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 627bf4ec11..4827fbc918 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -90,17 +90,15 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { // this was a used buffer, push the output pointer forwards PositionalAudioRingBuffer* audioBuffer = _ringBuffers[i]; - if (audioBuffer->willBeAddedToMix()) { - audioBuffer->setNextOutput(audioBuffer->getNextOutput() + BUFFER_LENGTH_SAMPLES_PER_CHANNEL); - - if (audioBuffer->getNextOutput() >= audioBuffer->getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { - audioBuffer->setNextOutput(audioBuffer->getBuffer()); - } + if (audioBuffer->willBeAddedToMix()) { + audioBuffer->shiftReadPosition(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); audioBuffer->setWillBeAddedToMix(false); - } else if (audioBuffer->hasStarted() && audioBuffer->isStarved()) { - delete audioBuffer; - _ringBuffers.erase(_ringBuffers.begin() + i); + } else if (audioBuffer->isStarved()) { + // this was previously the kill for injected audio from a client + // fix when that is added back + // delete audioBuffer; + // _ringBuffers.erase(_ringBuffers.begin() + i); } } } diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 92ea597234..4d964407ca 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -72,7 +72,7 @@ const int IDLE_SIMULATE_MSECS = 16; // How often should call simul // in the idle loop? (60 FPS is default) static QTimer* idleTimer = NULL; -const int STARTUP_JITTER_SAMPLES = PACKET_LENGTH_SAMPLES_PER_CHANNEL / 2; +const int STARTUP_JITTER_SAMPLES = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / 2; // Startup optimistically with small jitter buffer that // will start playback on the second received audio packet. diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index c3565a4eb0..4129973bb7 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -13,6 +13,7 @@ #include #endif +#include #include #include #include @@ -33,7 +34,7 @@ static const float JITTER_BUFFER_LENGTH_MSECS = 12; static const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_LENGTH_MSECS * NUM_AUDIO_CHANNELS * (SAMPLE_RATE / 1000.0); -static const float AUDIO_CALLBACK_MSECS = (float)BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; +static const float AUDIO_CALLBACK_MSECS = (float) NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; // Mute icon configration static const int ICON_SIZE = 24; @@ -43,12 +44,16 @@ static const int BOTTOM_PADDING = 110; Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* parent) : QObject(parent), _audioInput(NULL), - _inputDevice(NULL), + _desiredInputFormat(), + _inputFormat(), + _numInputCallbackBytes(0), _audioOutput(NULL), + _desiredOutputFormat(), + _outputFormat(), _outputDevice(NULL), - _isBufferSendCallback(false), - _nextOutputSamples(NULL), - _ringBuffer(true), + _numOutputCallbackBytes(0), + _inputRingBuffer(0), + _ringBuffer(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL), _scope(scope), _averagedLatency(0.0), _measuredJitter(0), @@ -65,7 +70,8 @@ Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* p _numFramesDisplayStarve(0), _muted(false) { - + // clear the array of locally injected samples + memset(_localInjectedSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); } void Audio::init(QGLWidget *parent) { @@ -124,242 +130,254 @@ QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { return (mode == QAudio::AudioInput) ? QAudioDeviceInfo::defaultInputDevice() : QAudioDeviceInfo::defaultOutputDevice(); } -const int QT_SAMPLE_RATE = 44100; -const int SAMPLE_RATE_RATIO = QT_SAMPLE_RATE / SAMPLE_RATE; +bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice, + const QAudioFormat& desiredAudioFormat, + QAudioFormat& adjustedAudioFormat) { + if (!audioDevice.isFormatSupported(desiredAudioFormat)) { + qDebug() << "The desired format for audio I/O is" << desiredAudioFormat << "\n"; + qDebug() << "The desired audio format is not supported by this device.\n"; + + if (desiredAudioFormat.channelCount() == 1) { + adjustedAudioFormat = desiredAudioFormat; + adjustedAudioFormat.setChannelCount(2); + + if (audioDevice.isFormatSupported(adjustedAudioFormat)) { + return true; + } else { + adjustedAudioFormat.setChannelCount(1); + } + } + + if (audioDevice.supportedSampleRates().contains(SAMPLE_RATE * 2)) { + // use 48, which is a sample downsample, upsample + adjustedAudioFormat = desiredAudioFormat; + adjustedAudioFormat.setSampleRate(SAMPLE_RATE * 2); + + // return the nearest in case it needs 2 channels + adjustedAudioFormat = audioDevice.nearestFormat(adjustedAudioFormat); + return true; + } + + return false; + } else { + // set the adjustedAudioFormat to the desiredAudioFormat, since it will work + adjustedAudioFormat = desiredAudioFormat; + return true; + } +} + +void linearResampling(int16_t* sourceSamples, int16_t* destinationSamples, + unsigned int numSourceSamples, unsigned int numDestinationSamples, + const QAudioFormat& sourceAudioFormat, const QAudioFormat& destinationAudioFormat) { + if (sourceAudioFormat == destinationAudioFormat) { + memcpy(destinationSamples, sourceSamples, numSourceSamples * sizeof(int16_t)); + } else { + int destinationChannels = (destinationAudioFormat.channelCount() >= 2) ? 2 : destinationAudioFormat.channelCount(); + float sourceToDestinationFactor = (sourceAudioFormat.sampleRate() / (float) destinationAudioFormat.sampleRate()) + * (sourceAudioFormat.channelCount() / (float) destinationChannels); + + // take into account the number of channels in source and destination + // accomodate for the case where have an output with > 2 channels + // this is the case with our HDMI capture + + if (sourceToDestinationFactor >= 2) { + // we need to downsample from 48 to 24 + // for now this only supports a mono output - this would be the case for audio input + + for (int i = sourceAudioFormat.channelCount(); i < numSourceSamples; i += 2 * sourceAudioFormat.channelCount()) { + if (i + (sourceAudioFormat.channelCount()) >= numSourceSamples) { + destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] = + (sourceSamples[i - sourceAudioFormat.channelCount()] / 2) + + (sourceSamples[i] / 2); + } else { + destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] = + (sourceSamples[i - sourceAudioFormat.channelCount()] / 4) + + (sourceSamples[i] / 2) + + (sourceSamples[i + sourceAudioFormat.channelCount()] / 4); + } + } + + } else { + // upsample from 24 to 48 + // for now this only supports a stereo to stereo conversion - this is our case for network audio to output + int sourceIndex = 0; + int destinationToSourceFactor = (1 / sourceToDestinationFactor); + + for (int i = 0; i < numDestinationSamples; i += destinationAudioFormat.channelCount() * destinationToSourceFactor) { + sourceIndex = (i / destinationToSourceFactor); + + // fill the L/R channels and make the rest silent + for (int j = i; j < i + (destinationToSourceFactor * destinationAudioFormat.channelCount()); j++) { + if (j % destinationAudioFormat.channelCount() == 0) { + // left channel + destinationSamples[j] = sourceSamples[sourceIndex]; + } else if (j % destinationAudioFormat.channelCount() == 1) { + // right channel + destinationSamples[j] = sourceSamples[sourceIndex + 1]; + } else { + // channels above 2, fill with silence + destinationSamples[j] = 0; + } + } + } + } + } +} + const int CALLBACK_ACCELERATOR_RATIO = 2; -const int CALLBACK_IO_BUFFER_SIZE = BUFFER_LENGTH_BYTES_STEREO * SAMPLE_RATE_RATIO / CALLBACK_ACCELERATOR_RATIO; void Audio::start() { - QAudioFormat audioFormat; // set up the desired audio format - audioFormat.setSampleRate(QT_SAMPLE_RATE); - audioFormat.setSampleSize(16); - audioFormat.setCodec("audio/pcm"); - audioFormat.setSampleType(QAudioFormat::SignedInt); - audioFormat.setByteOrder(QAudioFormat::LittleEndian); - audioFormat.setChannelCount(2); + _desiredInputFormat.setSampleRate(SAMPLE_RATE); + _desiredInputFormat.setSampleSize(16); + _desiredInputFormat.setCodec("audio/pcm"); + _desiredInputFormat.setSampleType(QAudioFormat::SignedInt); + _desiredInputFormat.setByteOrder(QAudioFormat::LittleEndian); + _desiredInputFormat.setChannelCount(1); - qDebug() << "The format for audio I/O is" << audioFormat << "\n"; + _desiredOutputFormat = _desiredInputFormat; + _desiredOutputFormat.setChannelCount(2); - QAudioDeviceInfo inputAudioDevice = defaultAudioDeviceForMode(QAudio::AudioInput); + QAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput); - qDebug() << "Audio input device is" << inputAudioDevice.deviceName() << "\n"; - if (!inputAudioDevice.isFormatSupported(audioFormat)) { - qDebug() << "The desired audio input format is not supported by this device. Not starting audio input.\n"; + qDebug() << "The audio input device is" << inputDeviceInfo.deviceName() << "\n"; + + if (adjustedFormatForAudioDevice(inputDeviceInfo, _desiredInputFormat, _inputFormat)) { + qDebug() << "The format to be used for audio input is" << _inputFormat << "\n"; + + _audioInput = new QAudioInput(inputDeviceInfo, _inputFormat, this); + _numInputCallbackBytes = NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL * _inputFormat.channelCount() + * (_inputFormat.sampleRate() / SAMPLE_RATE) + / CALLBACK_ACCELERATOR_RATIO; + _audioInput->setBufferSize(_numInputCallbackBytes); + + QAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput); + + qDebug() << "The audio output device is" << outputDeviceInfo.deviceName() << "\n"; + + if (adjustedFormatForAudioDevice(outputDeviceInfo, _desiredOutputFormat, _outputFormat)) { + qDebug() << "The format to be used for audio output is" << _outputFormat << "\n"; + + _inputRingBuffer.resizeForFrameSize(_numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO / sizeof(int16_t)); + _inputDevice = _audioInput->start(); + connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleAudioInput())); + + _audioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this); + _outputDevice = _audioOutput->start(); + + gettimeofday(&_lastReceiveTime, NULL); + } + return; } - _audioInput = new QAudioInput(inputAudioDevice, audioFormat, this); - _audioInput->setBufferSize(CALLBACK_IO_BUFFER_SIZE); - _inputDevice = _audioInput->start(); - - connect(_inputDevice, SIGNAL(readyRead()), SLOT(handleAudioInput())); - - QAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput); - - qDebug() << outputDeviceInfo.supportedSampleRates() << "\n"; - - qDebug() << "Audio output device is" << outputDeviceInfo.deviceName() << "\n"; - - if (!outputDeviceInfo.isFormatSupported(audioFormat)) { - qDebug() << "The desired audio output format is not supported by this device.\n"; - return; - } - - _audioOutput = new QAudioOutput(outputDeviceInfo, audioFormat, this); - _audioOutput->setBufferSize(CALLBACK_IO_BUFFER_SIZE); - _outputDevice = _audioOutput->start(); - - gettimeofday(&_lastReceiveTime, NULL); + qDebug() << "Unable to set up audio I/O because of a problem with input or output formats.\n"; } void Audio::handleAudioInput() { - static int16_t stereoInputBuffer[CALLBACK_IO_BUFFER_SIZE * 2]; static char monoAudioDataPacket[MAX_PACKET_SIZE]; - static int bufferSizeSamples = _audioInput->bufferSize() / sizeof(int16_t); static int numBytesPacketHeader = numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO); static int leadingBytes = numBytesPacketHeader + sizeof(glm::vec3) + sizeof(glm::quat) + NUM_BYTES_RFC4122_UUID; + static int16_t* monoAudioSamples = (int16_t*) (monoAudioDataPacket + leadingBytes); - QByteArray inputByteArray = _inputDevice->read(CALLBACK_IO_BUFFER_SIZE); + static float inputToNetworkInputRatio = _numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO + / NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; - if (_isBufferSendCallback) { - // copy samples from the inputByteArray to the stereoInputBuffer - memcpy((char*) (stereoInputBuffer + bufferSizeSamples), inputByteArray.data(), inputByteArray.size()); - - // Measure the loudness of the signal from the microphone and store in audio object - float loudness = 0; - for (int i = 0; i < BUFFER_LENGTH_SAMPLES_PER_CHANNEL * SAMPLE_RATE_RATIO; i += 2) { - loudness += abs(stereoInputBuffer[i]); - } - - loudness /= BUFFER_LENGTH_SAMPLES_PER_CHANNEL * SAMPLE_RATE_RATIO; - _lastInputLoudness = loudness; - - } else { - // this is the first half of a full buffer of data - // zero out the monoAudioSamples array - memset(monoAudioSamples, 0, BUFFER_LENGTH_BYTES_PER_CHANNEL); - - // take samples we have in this callback and store them in the first half of the static buffer - // to send off in the next callback - memcpy((char*) stereoInputBuffer, inputByteArray.data(), inputByteArray.size()); - } + static int inputSamplesRequired = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * inputToNetworkInputRatio; - // add input data just written to the scope - QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection, - Q_ARG(QByteArray, inputByteArray), Q_ARG(bool, true)); + QByteArray inputByteArray = _inputDevice->readAll(); - QByteArray stereoOutputBuffer; + _inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size()); - if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio) && !_muted) { - // if local loopback enabled, copy input to output - if (_isBufferSendCallback) { - stereoOutputBuffer.append((char*) (stereoInputBuffer + bufferSizeSamples), CALLBACK_IO_BUFFER_SIZE); + while (_inputRingBuffer.samplesAvailable() > inputSamplesRequired) { + + int16_t inputAudioSamples[inputSamplesRequired]; + _inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired); + + // zero out the monoAudioSamples array and the locally injected audio + memset(monoAudioSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); + + // zero out the locally injected audio in preparation for audio procedural sounds + memset(_localInjectedSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); + + if (!_muted) { + // we aren't muted, downsample the input audio + linearResampling((int16_t*) inputAudioSamples, + monoAudioSamples, + inputSamplesRequired, + NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, + _inputFormat, _desiredInputFormat); + + float loudness = 0; + + for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { + loudness += fabsf(monoAudioSamples[i]); + } + + _lastInputLoudness = loudness / NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + + // add input data just written to the scope + QMetaObject::invokeMethod(_scope, "addSamples", Qt::QueuedConnection, + Q_ARG(QByteArray, QByteArray((char*) monoAudioSamples, + NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL)), + Q_ARG(bool, false), Q_ARG(bool, true)); + + if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio)) { + // if this person wants local loopback add that to the locally injected audio + memcpy(_localInjectedSamples, monoAudioSamples, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); + } } else { - stereoOutputBuffer.append((char*) stereoInputBuffer, CALLBACK_IO_BUFFER_SIZE); + // our input loudness is 0, since we're muted + _lastInputLoudness = 0; } - } else { - // zero out the stereoOutputBuffer - stereoOutputBuffer = QByteArray(CALLBACK_IO_BUFFER_SIZE, 0); - } - - // add procedural effects to the appropriate input samples - addProceduralSounds(monoAudioSamples + (_isBufferSendCallback - ? BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO : 0), - (int16_t*) stereoOutputBuffer.data(), - BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO); - - if (_isBufferSendCallback) { + + // add procedural effects to the appropriate input samples + addProceduralSounds(monoAudioSamples, + NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); + NodeList* nodeList = NodeList::getInstance(); Node* audioMixer = nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER); - if (audioMixer) { - if (audioMixer->getActiveSocket()) { - MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar(); - - glm::vec3 headPosition = interfaceAvatar->getHeadJointPosition(); - glm::quat headOrientation = interfaceAvatar->getHead().getOrientation(); - - // we need the amount of bytes in the buffer + 1 for type - // + 12 for 3 floats for position + float for bearing + 1 attenuation byte - - PACKET_TYPE packetType = Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio) - ? PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO : PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO; - - char* currentPacketPtr = monoAudioDataPacket + populateTypeAndVersion((unsigned char*) monoAudioDataPacket, - packetType); - - // pack Source Data - QByteArray rfcUUID = NodeList::getInstance()->getOwnerUUID().toRfc4122(); - memcpy(currentPacketPtr, rfcUUID.constData(), rfcUUID.size()); - currentPacketPtr += rfcUUID.size(); - - // memcpy the three float positions - memcpy(currentPacketPtr, &headPosition, sizeof(headPosition)); - currentPacketPtr += (sizeof(headPosition)); - - // memcpy our orientation - memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation)); - currentPacketPtr += sizeof(headOrientation); - - if (!_muted) { - // we aren't muted, average each set of four samples together to set up the mono input buffers - for (int i = 2; i < BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2 * SAMPLE_RATE_RATIO; i += 4) { - - int16_t averagedSample = 0; - if (i + 2 == BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2 * SAMPLE_RATE_RATIO) { - averagedSample = (stereoInputBuffer[i - 2] / 2) + (stereoInputBuffer[i] / 2); - } else { - averagedSample = (stereoInputBuffer[i - 2] / 4) + (stereoInputBuffer[i] / 2) - + (stereoInputBuffer[i + 2] / 4); - } - - // add the averaged sample to our array of audio samples - monoAudioSamples[(i - 2) / 4] += averagedSample; - } - } - - nodeList->getNodeSocket().writeDatagram(monoAudioDataPacket, BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes, - audioMixer->getActiveSocket()->getAddress(), - audioMixer->getActiveSocket()->getPort()); - - Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) - .updateValue(BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes); - } else { - nodeList->pingPublicAndLocalSocketsForInactiveNode(audioMixer); - } + if (audioMixer && nodeList->getNodeActiveSocketOrPing(audioMixer)) { + MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar(); + + glm::vec3 headPosition = interfaceAvatar->getHeadJointPosition(); + glm::quat headOrientation = interfaceAvatar->getHead().getOrientation(); + + // we need the amount of bytes in the buffer + 1 for type + // + 12 for 3 floats for position + float for bearing + 1 attenuation byte + + PACKET_TYPE packetType = Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio) + ? PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO : PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO; + + char* currentPacketPtr = monoAudioDataPacket + populateTypeAndVersion((unsigned char*) monoAudioDataPacket, + packetType); + + // pack Source Data + QByteArray rfcUUID = NodeList::getInstance()->getOwnerUUID().toRfc4122(); + memcpy(currentPacketPtr, rfcUUID.constData(), rfcUUID.size()); + currentPacketPtr += rfcUUID.size(); + + // memcpy the three float positions + memcpy(currentPacketPtr, &headPosition, sizeof(headPosition)); + currentPacketPtr += (sizeof(headPosition)); + + // memcpy our orientation + memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation)); + currentPacketPtr += sizeof(headOrientation); + + nodeList->getNodeSocket().writeDatagram(monoAudioDataPacket, + NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes, + audioMixer->getActiveSocket()->getAddress(), + audioMixer->getActiveSocket()->getPort()); + + Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) + .updateValue(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes); } } - - // if there is anything in the ring buffer, decide what to do - - if (!_nextOutputSamples) { - if (_ringBuffer.getEndOfLastWrite()) { - if (_ringBuffer.isStarved() && _ringBuffer.diffLastWriteNextOutput() < - (PACKET_LENGTH_SAMPLES + _jitterBufferSamples * (_ringBuffer.isStereo() ? 2 : 1))) { - // If not enough audio has arrived to start playback, keep waiting - } else if (!_ringBuffer.isStarved() && _ringBuffer.diffLastWriteNextOutput() == 0) { - // If we have started and now have run out of audio to send to the audio device, - // this means we've starved and should restart. - _ringBuffer.setIsStarved(true); - - // show a starve in the GUI for 10 frames - _numFramesDisplayStarve = 10; - - } else { - // We are either already playing back, or we have enough audio to start playing back. - if (_ringBuffer.isStarved()) { - _ringBuffer.setIsStarved(false); - _ringBuffer.setHasStarted(true); - } - - _nextOutputSamples = _ringBuffer.getNextOutput(); - } - } - } - - if (_nextOutputSamples) { - - int16_t* stereoOutputBufferSamples = (int16_t*) stereoOutputBuffer.data(); - - // play whatever we have in the audio buffer - for (int s = 0; s < PACKET_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO; s++) { - int16_t leftSample = _nextOutputSamples[s]; - int16_t rightSample = _nextOutputSamples[s + PACKET_LENGTH_SAMPLES_PER_CHANNEL]; - - stereoOutputBufferSamples[(s * 4)] += leftSample; - stereoOutputBufferSamples[(s * 4) + 2] += leftSample; - - stereoOutputBufferSamples[(s * 4) + 1] += rightSample; - stereoOutputBufferSamples[(s * 4) + 3] += rightSample; - } - - if (_isBufferSendCallback) { - _ringBuffer.setNextOutput(_ringBuffer.getNextOutput() + PACKET_LENGTH_SAMPLES); - - if (_ringBuffer.getNextOutput() == _ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { - _ringBuffer.setNextOutput(_ringBuffer.getBuffer()); - } - - _nextOutputSamples = NULL; - } else { - _nextOutputSamples += PACKET_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO; - } - } - - _outputDevice->write(stereoOutputBuffer); - - - // add output (@speakers) data just written to the scope - QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection, - Q_ARG(QByteArray, stereoOutputBuffer), Q_ARG(bool, false)); - - _isBufferSendCallback = !_isBufferSendCallback; - - gettimeofday(&_lastCallbackTime, NULL); } void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { @@ -381,7 +399,7 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { _measuredJitter = _stdev.getStDev(); _stdev.reset(); // Set jitter buffer to be a multiple of the measured standard deviation - const int MAX_JITTER_BUFFER_SAMPLES = RING_BUFFER_LENGTH_SAMPLES / 2; + const int MAX_JITTER_BUFFER_SAMPLES = _ringBuffer.getSampleCapacity() / 2; const float NUM_STANDARD_DEVIATIONS = 3.f; if (Menu::getInstance()->getAudioJitterBufferSamples() == 0) { float newJitterBufferSamples = (NUM_STANDARD_DEVIATIONS * _measuredJitter) / 1000.f * SAMPLE_RATE; @@ -389,22 +407,69 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { } } - if (_ringBuffer.diffLastWriteNextOutput() + PACKET_LENGTH_SAMPLES > - PACKET_LENGTH_SAMPLES + (ceilf((float) (_jitterBufferSamples * 2) / PACKET_LENGTH_SAMPLES) * PACKET_LENGTH_SAMPLES)) { - // this packet would give us more than the required amount for play out - // discard the first packet in the buffer - - _ringBuffer.setNextOutput(_ringBuffer.getNextOutput() + PACKET_LENGTH_SAMPLES); - - if (_ringBuffer.getNextOutput() == _ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { - _ringBuffer.setNextOutput(_ringBuffer.getBuffer()); - } - } - _ringBuffer.parseData((unsigned char*) audioByteArray.data(), audioByteArray.size()); - Application::getInstance()->getBandwidthMeter()->inputStream(BandwidthMeter::AUDIO).updateValue(PACKET_LENGTH_BYTES - + sizeof(PACKET_TYPE)); + static float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float) _outputFormat.sampleRate()) + * (_desiredOutputFormat.channelCount() / (float) _outputFormat.channelCount()); + + static int numRequiredOutputSamples = NETWORK_BUFFER_LENGTH_SAMPLES_STEREO / networkOutputToOutputRatio; + + QByteArray outputBuffer; + outputBuffer.resize(numRequiredOutputSamples * sizeof(int16_t)); + + // if there is anything in the ring buffer, decide what to do + if (_ringBuffer.samplesAvailable() > 0) { + if (!_ringBuffer.isNotStarvedOrHasMinimumSamples(NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + + (_jitterBufferSamples * 2))) { + // starved and we don't have enough to start, keep waiting + qDebug() << "Buffer is starved and doesn't have enough samples to start. Held back.\n"; + } else { + // We are either already playing back, or we have enough audio to start playing back. + _ringBuffer.setIsStarved(false); + + // copy the samples we'll resample from the ring buffer - this also + // pushes the read pointer of the ring buffer forwards + int16_t ringBufferSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO]; + _ringBuffer.readSamples(ringBufferSamples, NETWORK_BUFFER_LENGTH_SAMPLES_STEREO); + + // add to the output samples whatever is in the _localAudioOutput byte array + // that lets this user hear sound effects and loopback (if enabled) + + for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { + ringBufferSamples[i * 2] = glm::clamp(ringBufferSamples[i * 2] + _localInjectedSamples[i], + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + ringBufferSamples[(i * 2) + 1] += glm::clamp(ringBufferSamples[(i * 2) + 1] + _localInjectedSamples[i], + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + } + + // copy the packet from the RB to the output + linearResampling(ringBufferSamples, + (int16_t*) outputBuffer.data(), + NETWORK_BUFFER_LENGTH_SAMPLES_STEREO, + numRequiredOutputSamples, + _desiredOutputFormat, _outputFormat); + + if (_outputDevice) { + + _outputDevice->write(outputBuffer); + + // add output (@speakers) data just written to the scope + QMetaObject::invokeMethod(_scope, "addSamples", Qt::QueuedConnection, + Q_ARG(QByteArray, QByteArray((char*) ringBufferSamples, + NETWORK_BUFFER_LENGTH_BYTES_STEREO)), + Q_ARG(bool, true), Q_ARG(bool, false)); + } + } + + } else if (_audioOutput->bytesFree() == _audioOutput->bufferSize()) { + // we don't have any audio data left in the output buffer, and the ring buffer from + // the network has nothing in it either - we just starved + qDebug() << "Audio output just starved.\n"; + _ringBuffer.setIsStarved(true); + _numFramesDisplayStarve = 10; + } + + Application::getInstance()->getBandwidthMeter()->inputStream(BandwidthMeter::AUDIO).updateValue(audioByteArray.size()); _lastReceiveTime = currentReceiveTime; } @@ -435,7 +500,7 @@ void Audio::render(int screenWidth, int screenHeight) { glVertex2f(currentX, topY); glVertex2f(currentX, bottomY); - for (int i = 0; i < RING_BUFFER_LENGTH_FRAMES / 2; i++) { + for (int i = 0; i < RING_BUFFER_LENGTH_FRAMES; i++) { glVertex2f(currentX, halfY); glVertex2f(currentX + frameWidth, halfY); currentX += frameWidth; @@ -445,17 +510,15 @@ void Audio::render(int screenWidth, int screenHeight) { } glEnd(); - // Show a bar with the amount of audio remaining in ring buffer beyond current playback - float remainingBuffer = 0; - timeval currentTime; - gettimeofday(¤tTime, NULL); - float timeLeftInCurrentBuffer = 0; - if (_lastCallbackTime.tv_usec > 0) { - timeLeftInCurrentBuffer = AUDIO_CALLBACK_MSECS - diffclock(&_lastCallbackTime, ¤tTime); - } + // show a bar with the amount of audio remaining in ring buffer and output device + // beyond the current playback - if (_ringBuffer.getEndOfLastWrite() != NULL) - remainingBuffer = _ringBuffer.diffLastWriteNextOutput() / PACKET_LENGTH_SAMPLES * AUDIO_CALLBACK_MSECS; + int bytesLeftInAudioOutput = _audioOutput->bufferSize() - _audioOutput->bytesFree(); + float secondsLeftForAudioOutput = (bytesLeftInAudioOutput / sizeof(int16_t)) + / ((float) _outputFormat.sampleRate() * _outputFormat.channelCount()); + float secondsLeftForRingBuffer = _ringBuffer.samplesAvailable() + / ((float) _desiredOutputFormat.sampleRate() * _desiredOutputFormat.channelCount()); + float msLeftForAudioOutput = (secondsLeftForAudioOutput + secondsLeftForRingBuffer) * 1000; if (_numFramesDisplayStarve == 0) { glColor3f(0, 1, 0); @@ -464,19 +527,19 @@ void Audio::render(int screenWidth, int screenHeight) { _numFramesDisplayStarve--; } + if (_averagedLatency == 0.0) { + _averagedLatency = msLeftForAudioOutput; + } else { + _averagedLatency = 0.99f * _averagedLatency + 0.01f * (msLeftForAudioOutput); + } + glBegin(GL_QUADS); glVertex2f(startX, topY + 2); - glVertex2f(startX + (remainingBuffer + timeLeftInCurrentBuffer) / AUDIO_CALLBACK_MSECS * frameWidth, topY + 2); - glVertex2f(startX + (remainingBuffer + timeLeftInCurrentBuffer) / AUDIO_CALLBACK_MSECS * frameWidth, bottomY - 2); + glVertex2f(startX + _averagedLatency / AUDIO_CALLBACK_MSECS * frameWidth, topY + 2); + glVertex2f(startX + _averagedLatency / AUDIO_CALLBACK_MSECS * frameWidth, bottomY - 2); glVertex2f(startX, bottomY - 2); glEnd(); - if (_averagedLatency == 0.0) { - _averagedLatency = remainingBuffer + timeLeftInCurrentBuffer; - } else { - _averagedLatency = 0.99f * _averagedLatency + 0.01f * (remainingBuffer + timeLeftInCurrentBuffer); - } - // Show a yellow bar with the averaged msecs latency you are hearing (from time of packet receipt) glColor3f(1,1,0); glBegin(GL_QUADS); @@ -493,7 +556,8 @@ void Audio::render(int screenWidth, int screenHeight) { // Show a red bar with the 'start' point of one frame plus the jitter buffer glColor3f(1, 0, 0); - int jitterBufferPels = (1.f + (float)getJitterBufferSamples() / (float) PACKET_LENGTH_SAMPLES_PER_CHANNEL) * frameWidth; + int jitterBufferPels = (1.f + (float)getJitterBufferSamples() + / (float) NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL) * frameWidth; sprintf(out, "%.0f\n", getJitterBufferSamples() / SAMPLE_RATE * 1000.f); drawtext(startX + jitterBufferPels - 5, topY - 9, 0.10, 0, 1, 0, out, 1, 0, 0); sprintf(out, "j %.1f\n", _measuredJitter); @@ -515,7 +579,7 @@ void Audio::render(int screenWidth, int screenHeight) { } // Take a pointer to the acquired microphone input samples and add procedural sounds -void Audio::addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutput, int numSamples) { +void Audio::addProceduralSounds(int16_t* monoInput, int numSamples) { const float MAX_AUDIBLE_VELOCITY = 6.0; const float MIN_AUDIBLE_VELOCITY = 0.1; const int VOLUME_BASELINE = 400; @@ -551,11 +615,9 @@ void Audio::addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutp int16_t collisionSample = (int16_t) sample; - monoInput[i] += collisionSample; - - for (int j = (i * 4); j < (i * 4) + 4; j++) { - stereoUpsampledOutput[j] += collisionSample; - } + monoInput[i] = glm::clamp(monoInput[i] + collisionSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + _localInjectedSamples[i] = glm::clamp(_localInjectedSamples[i] + collisionSample, + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); _collisionSoundMagnitude *= _collisionSoundDuration; } @@ -577,11 +639,9 @@ void Audio::addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutp int16_t collisionSample = (int16_t) sample; - monoInput[i] += collisionSample; - - for (int j = (i * 4); j < (i * 4) + 4; j++) { - stereoUpsampledOutput[j] += collisionSample; - } + monoInput[i] = glm::clamp(monoInput[i] + collisionSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + _localInjectedSamples[i] = glm::clamp(_localInjectedSamples[i] + collisionSample, + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); _drumSoundVolume *= (1.f - _drumSoundDecay); } diff --git a/interface/src/Audio.h b/interface/src/Audio.h index ed9472bcf1..8c5706d08a 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -15,6 +15,7 @@ #include "InterfaceConfig.h" #include +#include #include #include @@ -26,11 +27,6 @@ static const int NUM_AUDIO_CHANNELS = 2; -static const int PACKET_LENGTH_BYTES = 1024; -static const int PACKET_LENGTH_BYTES_PER_CHANNEL = PACKET_LENGTH_BYTES / 2; -static const int PACKET_LENGTH_SAMPLES = PACKET_LENGTH_BYTES / sizeof(int16_t); -static const int PACKET_LENGTH_SAMPLES_PER_CHANNEL = PACKET_LENGTH_SAMPLES / 2; - class QAudioInput; class QAudioOutput; class QIODevice; @@ -70,16 +66,22 @@ public slots: void reset(); private: + QByteArray firstInputFrame; QAudioInput* _audioInput; + QAudioFormat _desiredInputFormat; + QAudioFormat _inputFormat; QIODevice* _inputDevice; + int16_t _localInjectedSamples[NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL]; + int _numInputCallbackBytes; QAudioOutput* _audioOutput; + QAudioFormat _desiredOutputFormat; + QAudioFormat _outputFormat; QIODevice* _outputDevice; - bool _isBufferSendCallback; - int16_t* _nextOutputSamples; + int _numOutputCallbackBytes; + AudioRingBuffer _inputRingBuffer; AudioRingBuffer _ringBuffer; Oscilloscope* _scope; StDev _stdev; - timeval _lastCallbackTime; timeval _lastReceiveTime; float _averagedLatency; float _measuredJitter; @@ -114,7 +116,7 @@ private: inline void performIO(int16_t* inputLeft, int16_t* outputLeft, int16_t* outputRight); // Add sounds that we want the user to not hear themselves, by adding on top of mic input signal - void addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutput, int numSamples); + void addProceduralSounds(int16_t* monoInput, int numSamples); void renderToolIcon(int screenHeight); }; diff --git a/interface/src/Oscilloscope.cpp b/interface/src/Oscilloscope.cpp index 2c65f2a07e..5b3cfb9f09 100644 --- a/interface/src/Oscilloscope.cpp +++ b/interface/src/Oscilloscope.cpp @@ -68,24 +68,18 @@ Oscilloscope::~Oscilloscope() { delete[] _samples; } -void Oscilloscope::addStereoSamples(const QByteArray& audioByteArray, bool isInput) { +void Oscilloscope::addSamples(const QByteArray& audioByteArray, bool isStereo, bool isInput) { if (! enabled || inputPaused) { return; } - unsigned int numSamplesPerChannel = audioByteArray.size() / (sizeof(int16_t) * 2); - int16_t samples[numSamplesPerChannel]; - const int16_t* stereoSamples = (int16_t*) audioByteArray.constData(); + int numSamplesPerChannel = audioByteArray.size() / (sizeof(int16_t) * (isStereo ? 2 : 1)); + int16_t* samples = (int16_t*) audioByteArray.data(); - for (int channel = 0; channel < (isInput ? 1 : 2); channel++) { + for (int channel = 0; channel < (isStereo ? 2 : 1); channel++) { // add samples for each of the channels - - // enumerate the interleaved stereoSamples array and pull out the samples for this channel - for (int i = 0; i < audioByteArray.size() / sizeof(int16_t); i += 2) { - samples[i / 2] = stereoSamples[i + channel]; - } - + // determine start/end offset of this channel's region unsigned baseOffs = MAX_SAMPLES_PER_CHANNEL * (channel + !isInput); unsigned endOffs = baseOffs + MAX_SAMPLES_PER_CHANNEL; @@ -103,10 +97,21 @@ void Oscilloscope::addStereoSamples(const QByteArray& audioByteArray, bool isInp numSamplesPerChannel -= n2; } - // copy data - memcpy(_samples + writePos, samples, numSamplesPerChannel * sizeof(int16_t)); - if (n2 > 0) { - memcpy(_samples + baseOffs, samples + numSamplesPerChannel, n2 * sizeof(int16_t)); + if (!isStereo) { + // copy data + memcpy(_samples + writePos, samples, numSamplesPerChannel * sizeof(int16_t)); + if (n2 > 0) { + memcpy(_samples + baseOffs, samples + numSamplesPerChannel, n2 * sizeof(int16_t)); + } + } else { + // we have interleaved samples we need to separate into two channels + for (int i = 0; i < numSamplesPerChannel + n2; i++) { + if (i < numSamplesPerChannel - n2) { + _samples[writePos] = samples[(i * 2) + channel]; + } else { + _samples[baseOffs] = samples[(i * 2) + channel]; + } + } } // set new write position for this channel diff --git a/interface/src/Oscilloscope.h b/interface/src/Oscilloscope.h index f17976d4e4..a245f79f05 100644 --- a/interface/src/Oscilloscope.h +++ b/interface/src/Oscilloscope.h @@ -59,7 +59,7 @@ public: // just uses every nTh sample. void setDownsampleRatio(unsigned n) { assert(n > 0); _downsampleRatio = n; } public slots: - void addStereoSamples(const QByteArray& audioByteArray, bool isInput); + void addSamples(const QByteArray& audioByteArray, bool isStereo, bool isInput); private: // don't copy/assign Oscilloscope(Oscilloscope const&); // = delete; diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index b18ff01b95..5e9abf38b7 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -9,19 +9,27 @@ #include #include +#include + #include "PacketHeaders.h" #include "AudioRingBuffer.h" -AudioRingBuffer::AudioRingBuffer(bool isStereo) : +AudioRingBuffer::AudioRingBuffer(int numFrameSamples) : NodeData(NULL), - _endOfLastWrite(NULL), + _sampleCapacity(numFrameSamples * RING_BUFFER_LENGTH_FRAMES), _isStarved(true), - _hasStarted(false), - _isStereo(isStereo) + _hasStarted(false) { - _buffer = new int16_t[RING_BUFFER_LENGTH_SAMPLES]; - _nextOutput = _buffer; + if (numFrameSamples) { + _buffer = new int16_t[_sampleCapacity]; + _nextOutput = _buffer; + _endOfLastWrite = _buffer; + } else { + _buffer = NULL; + _nextOutput = NULL; + _endOfLastWrite = NULL; + } }; AudioRingBuffer::~AudioRingBuffer() { @@ -32,53 +40,130 @@ void AudioRingBuffer::reset() { _endOfLastWrite = _buffer; _nextOutput = _buffer; _isStarved = true; - _hasStarted = false; +} + +void AudioRingBuffer::resizeForFrameSize(qint64 numFrameSamples) { + delete[] _buffer; + _sampleCapacity = numFrameSamples * RING_BUFFER_LENGTH_FRAMES; + _buffer = new int16_t[_sampleCapacity]; + _nextOutput = _buffer; + _endOfLastWrite = _buffer; } int AudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes) { int numBytesPacketHeader = numBytesForPacketHeader(sourceBuffer); - return parseAudioSamples(sourceBuffer + numBytesPacketHeader, numBytes - numBytesPacketHeader); + return writeData((char*) sourceBuffer + numBytesPacketHeader, numBytes - numBytesPacketHeader); } -int AudioRingBuffer::parseAudioSamples(unsigned char* sourceBuffer, int numBytes) { +qint64 AudioRingBuffer::readSamples(int16_t* destination, qint64 maxSamples) { + return readData((char*) destination, maxSamples * sizeof(int16_t)); +} + +qint64 AudioRingBuffer::readData(char *data, qint64 maxSize) { + + // only copy up to the number of samples we have available + int numReadSamples = std::min((unsigned) (maxSize / sizeof(int16_t)), samplesAvailable()); + + if (_nextOutput + numReadSamples > _buffer + _sampleCapacity) { + // we're going to need to do two reads to get this data, it wraps around the edge + + // read to the end of the buffer + int numSamplesToEnd = (_buffer + _sampleCapacity) - _nextOutput; + memcpy(data, _nextOutput, numSamplesToEnd * sizeof(int16_t)); + + // read the rest from the beginning of the buffer + memcpy(data + numSamplesToEnd, _buffer, (numReadSamples - numSamplesToEnd) * sizeof(int16_t)); + } else { + // read the data + memcpy(data, _nextOutput, numReadSamples * sizeof(int16_t)); + } + + // push the position of _nextOutput by the number of samples read + _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numReadSamples); + + return numReadSamples * sizeof(int16_t); +} + +qint64 AudioRingBuffer::writeSamples(const int16_t* source, qint64 maxSamples) { + return writeData((const char*) source, maxSamples * sizeof(int16_t)); +} + +qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { // make sure we have enough bytes left for this to be the right amount of audio // otherwise we should not copy that data, and leave the buffer pointers where they are - int samplesToCopy = BUFFER_LENGTH_SAMPLES_PER_CHANNEL * (_isStereo ? 2 : 1); - if (numBytes == samplesToCopy * sizeof(int16_t)) { - - if (!_endOfLastWrite) { - _endOfLastWrite = _buffer; - } else if (diffLastWriteNextOutput() > RING_BUFFER_LENGTH_SAMPLES - samplesToCopy) { - _endOfLastWrite = _buffer; - _nextOutput = _buffer; - _isStarved = true; - } - - memcpy(_endOfLastWrite, sourceBuffer, numBytes); - - _endOfLastWrite += samplesToCopy; - - if (_endOfLastWrite >= _buffer + RING_BUFFER_LENGTH_SAMPLES) { - _endOfLastWrite = _buffer; - } - - return numBytes; + int samplesToCopy = std::min(maxSize / sizeof(int16_t), (quint64) _sampleCapacity); + + std::less less; + std::less_equal lessEqual; + + if (_hasStarted + && (less(_endOfLastWrite, _nextOutput) + && lessEqual(_nextOutput, shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy)))) { + // this read will cross the next output, so call us starved and reset the buffer + qDebug() << "Filled the ring buffer. Resetting.\n"; + _endOfLastWrite = _buffer; + _nextOutput = _buffer; + _isStarved = true; + } + + _hasStarted = true; + + if (_endOfLastWrite + samplesToCopy <= _buffer + _sampleCapacity) { + memcpy(_endOfLastWrite, data, samplesToCopy * sizeof(int16_t)); } else { - return 0; - } + int numSamplesToEnd = (_buffer + _sampleCapacity) - _endOfLastWrite; + memcpy(_endOfLastWrite, data, numSamplesToEnd * sizeof(int16_t)); + memcpy(_buffer, data + (numSamplesToEnd * sizeof(int16_t)), (samplesToCopy - numSamplesToEnd) * sizeof(int16_t)); + } + + _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy); + + return samplesToCopy * sizeof(int16_t); } -int AudioRingBuffer::diffLastWriteNextOutput() const { +int16_t& AudioRingBuffer::operator[](const int index) { + // make sure this is a valid index + assert(index > -_sampleCapacity && index < _sampleCapacity); + + return *shiftedPositionAccomodatingWrap(_nextOutput, index); +} + +void AudioRingBuffer::shiftReadPosition(unsigned int numSamples) { + _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); +} + +unsigned int AudioRingBuffer::samplesAvailable() const { if (!_endOfLastWrite) { return 0; } else { int sampleDifference = _endOfLastWrite - _nextOutput; if (sampleDifference < 0) { - sampleDifference += RING_BUFFER_LENGTH_SAMPLES; + sampleDifference += _sampleCapacity; } return sampleDifference; } } + +bool AudioRingBuffer::isNotStarvedOrHasMinimumSamples(unsigned int numRequiredSamples) const { + if (!_isStarved) { + return true; + } else { + return samplesAvailable() >= numRequiredSamples; + } +} + +int16_t* AudioRingBuffer::shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const { + + if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + _sampleCapacity) { + // this shift will wrap the position around to the beginning of the ring + return position + numSamplesShift - _sampleCapacity; + } else if (numSamplesShift < 0 && position + numSamplesShift < _buffer) { + // this shift will go around to the end of the ring + return position + numSamplesShift - _sampleCapacity; + } else { + return position + numSamplesShift; + } +} diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index f31fa780fb..addad13146 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -9,61 +9,69 @@ #ifndef __interface__AudioRingBuffer__ #define __interface__AudioRingBuffer__ +#include #include -#include #include +#include + #include "NodeData.h" -const int SAMPLE_RATE = 22050; +const int SAMPLE_RATE = 24000; -const int BUFFER_LENGTH_BYTES_STEREO = 1024; -const int BUFFER_LENGTH_BYTES_PER_CHANNEL = 512; -const int BUFFER_LENGTH_SAMPLES_PER_CHANNEL = BUFFER_LENGTH_BYTES_PER_CHANNEL / sizeof(int16_t); +const int NETWORK_BUFFER_LENGTH_BYTES_STEREO = 1024; +const int NETWORK_BUFFER_LENGTH_SAMPLES_STEREO = NETWORK_BUFFER_LENGTH_BYTES_STEREO / sizeof(int16_t); +const int NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL = 512; +const int NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL = NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL / sizeof(int16_t); -const short RING_BUFFER_LENGTH_FRAMES = 20; -const short RING_BUFFER_LENGTH_SAMPLES = RING_BUFFER_LENGTH_FRAMES * BUFFER_LENGTH_SAMPLES_PER_CHANNEL; +const short RING_BUFFER_LENGTH_FRAMES = 10; + +const int MAX_SAMPLE_VALUE = std::numeric_limits::max(); +const int MIN_SAMPLE_VALUE = std::numeric_limits::min(); class AudioRingBuffer : public NodeData { + Q_OBJECT public: - AudioRingBuffer(bool isStereo); + AudioRingBuffer(int numFrameSamples); ~AudioRingBuffer(); void reset(); - + void resizeForFrameSize(qint64 numFrameSamples); + + int getSampleCapacity() const { return _sampleCapacity; } + int parseData(unsigned char* sourceBuffer, int numBytes); - int parseAudioSamples(unsigned char* sourceBuffer, int numBytes); - int16_t* getNextOutput() const { return _nextOutput; } - void setNextOutput(int16_t* nextOutput) { _nextOutput = nextOutput; } + qint64 readSamples(int16_t* destination, qint64 maxSamples); + qint64 writeSamples(const int16_t* source, qint64 maxSamples); - int16_t* getEndOfLastWrite() const { return _endOfLastWrite; } - void setEndOfLastWrite(int16_t* endOfLastWrite) { _endOfLastWrite = endOfLastWrite; } + qint64 readData(char* data, qint64 maxSize); + qint64 writeData(const char* data, qint64 maxSize); - int16_t* getBuffer() const { return _buffer; } + int16_t& operator[](const int index); + + void shiftReadPosition(unsigned int numSamples); + + unsigned int samplesAvailable() const; + + bool isNotStarvedOrHasMinimumSamples(unsigned int numRequiredSamples) const; bool isStarved() const { return _isStarved; } void setIsStarved(bool isStarved) { _isStarved = isStarved; } - - bool hasStarted() const { return _hasStarted; } - void setHasStarted(bool hasStarted) { _hasStarted = hasStarted; } - - int diffLastWriteNextOutput() const; - - bool isStereo() const { return _isStereo; } - protected: // disallow copying of AudioRingBuffer objects AudioRingBuffer(const AudioRingBuffer&); AudioRingBuffer& operator= (const AudioRingBuffer&); + int16_t* shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const; + + int _sampleCapacity; int16_t* _nextOutput; int16_t* _endOfLastWrite; int16_t* _buffer; bool _isStarved; bool _hasStarted; - bool _isStereo; }; #endif /* defined(__interface__AudioRingBuffer__) */ diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index 9adb525b93..d66a24672a 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -42,7 +42,7 @@ int InjectedAudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes unsigned int attenuationByte = *(currentBuffer++); _attenuationRatio = attenuationByte / (float) MAX_INJECTOR_VOLUME; - currentBuffer += parseAudioSamples(currentBuffer, numBytes - (currentBuffer - sourceBuffer)); + currentBuffer += writeData((char*) currentBuffer, numBytes - (currentBuffer - sourceBuffer)); return currentBuffer - sourceBuffer; } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index ea8d2aca4a..0b7c26dc7d 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -15,7 +15,7 @@ #include "PositionalAudioRingBuffer.h" PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type) : - AudioRingBuffer(false), + AudioRingBuffer(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL), _type(type), _position(0.0f, 0.0f, 0.0f), _orientation(0.0f, 0.0f, 0.0f, 0.0f), @@ -31,7 +31,7 @@ int PositionalAudioRingBuffer::parseData(unsigned char* sourceBuffer, int numByt unsigned char* currentBuffer = sourceBuffer + numBytesForPacketHeader(sourceBuffer); currentBuffer += NUM_BYTES_RFC4122_UUID; // the source UUID currentBuffer += parsePositionalData(currentBuffer, numBytes - (currentBuffer - sourceBuffer)); - currentBuffer += parseAudioSamples(currentBuffer, numBytes - (currentBuffer - sourceBuffer)); + currentBuffer += writeData((char*) currentBuffer, numBytes - (currentBuffer - sourceBuffer)); return currentBuffer - sourceBuffer; } @@ -47,8 +47,7 @@ int PositionalAudioRingBuffer::parsePositionalData(unsigned char* sourceBuffer, // if this node sent us a NaN for first float in orientation then don't consider this good audio and bail if (std::isnan(_orientation.x)) { - _endOfLastWrite = _nextOutput = _buffer; - _isStarved = true; + reset(); return 0; } @@ -56,20 +55,17 @@ int PositionalAudioRingBuffer::parsePositionalData(unsigned char* sourceBuffer, } bool PositionalAudioRingBuffer::shouldBeAddedToMix(int numJitterBufferSamples) { - if (_endOfLastWrite) { - if (_isStarved && diffLastWriteNextOutput() <= BUFFER_LENGTH_SAMPLES_PER_CHANNEL + numJitterBufferSamples) { - printf("Buffer held back\n"); - return false; - } else if (diffLastWriteNextOutput() < BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { - printf("Buffer starved.\n"); - _isStarved = true; - return false; - } else { - // good buffer, add this to the mix - _isStarved = false; - _hasStarted = true; - return true; - } + if (!isNotStarvedOrHasMinimumSamples(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL + numJitterBufferSamples)) { + qDebug() << "Starved and do not have minimum samples to start. Buffer held back.\n"; + return false; + } else if (samplesAvailable() < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { + qDebug() << "Do not have number of samples needed for interval. Buffer starved.\n"; + _isStarved = true; + return false; + } else { + // good buffer, add this to the mix + _isStarved = false; + return true; } return false;