From ebe96f9b25dcc4423b088618dfd68bdd0f9a42c4 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Sat, 7 Jan 2017 17:54:45 -0500 Subject: [PATCH 01/45] rm injectors from mixed audio processing --- libraries/audio-client/src/AudioClient.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 1e3dc11338..342c7b282e 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1177,11 +1177,6 @@ void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteA _mixBuffer[i] = (float)decodedSamples[i] * (1/32768.0f); } - // mix in active injectors - if (getActiveLocalAudioInjectors().size() > 0) { - mixLocalAudioInjectors(_mixBuffer); - } - // apply stereo reverb bool hasReverb = _reverb || _receivedAudioStream.hasReverb(); if (hasReverb) { From 24d53ea13c6248d28a66709df3a7216ab64f6972 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Sun, 8 Jan 2017 18:30:57 -0500 Subject: [PATCH 02/45] rm audio output limiting --- libraries/audio-client/src/AudioClient.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 342c7b282e..6477449366 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1172,7 +1172,7 @@ void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteA outputBuffer.resize(_outputFrameSize * AudioConstants::SAMPLE_SIZE); int16_t* outputSamples = reinterpret_cast(outputBuffer.data()); - // convert network audio to float + // convert network audio (int16_t) to mix audio (float) for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) { _mixBuffer[i] = (float)decodedSamples[i] * (1/32768.0f); } @@ -1184,16 +1184,16 @@ void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteA _listenerReverb.render(_mixBuffer, _mixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } + // convert mix audio (float) to network audio (int16_t) + for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) { + _scratchBuffer[i] = (int16_t)(_mixBuffer[i] * 32768.0f); + } + + // resample to output sample rate if (_networkToOutputResampler) { - - // resample to output sample rate - _audioLimiter.render(_mixBuffer, _scratchBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); _networkToOutputResampler->render(_scratchBuffer, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - } else { - - // no resampling needed - _audioLimiter.render(_mixBuffer, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + memcpy(outputBuffer.data(), _scratchBuffer, AudioConstants::NETWORK_FRAME_BYTES_STEREO); } } From a7ecf41a426dab17fb2c5064477a7dd909b0cdc5 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Sun, 8 Jan 2017 23:24:23 -0500 Subject: [PATCH 03/45] add audio limiting to device callback --- libraries/audio-client/src/AudioClient.cpp | 44 ++++++++++++++++++---- libraries/audio-client/src/AudioClient.h | 5 +++ 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 6477449366..1e97de8dca 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1386,6 +1386,12 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice _loopbackOutputDevice = NULL; delete _loopbackAudioOutput; _loopbackAudioOutput = NULL; + + delete[] _limitMixBuffer; + _limitMixBuffer = NULL; + + delete[] _limitScratchBuffer; + _limitScratchBuffer = NULL; } if (_networkToOutputResampler) { @@ -1436,6 +1442,12 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice _audioOutput->start(&_audioOutputIODevice); lock.unlock(); + int periodSampleSize = _audioOutput->periodSize() / AudioConstants::SAMPLE_SIZE; + // device callback is not restricted to periodSampleSize, so double the mix/scratch buffer sizes + _limitPeriod = periodSampleSize * 2; + _limitMixBuffer = new float[_limitPeriod]; + _limitScratchBuffer = new int16_t[_limitPeriod]; + qCDebug(audioclient) << "Output Buffer capacity in frames: " << _audioOutput->bufferSize() / AudioConstants::SAMPLE_SIZE / (float)deviceFrameSize << "requested bytes:" << requestedSize << "actual bytes:" << _audioOutput->bufferSize() << "os default:" << osDefaultBufferSize << "period size:" << _audioOutput->periodSize(); @@ -1545,6 +1557,8 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { // samples requested from OUTPUT_CHANNEL_COUNT int deviceChannelCount = _audio->_outputFormat.channelCount(); int samplesRequested = (int)(maxSize / AudioConstants::SAMPLE_SIZE) * OUTPUT_CHANNEL_COUNT / deviceChannelCount; + // restrict samplesRequested to the size of our mix/scratch buffers + samplesRequested = std::min(samplesRequested, _audio->_limitPeriod); int samplesPopped; int bytesWritten; @@ -1553,14 +1567,30 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { qCDebug(audiostream, "Read %d samples from buffer (%d available)", samplesPopped, _receivedAudioStream.getSamplesAvailable()); AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput(); - // if required, upmix or downmix to deviceChannelCount - if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) { - lastPopOutput.readSamples((int16_t*)data, samplesPopped); - } else if (deviceChannelCount > OUTPUT_CHANNEL_COUNT) { - lastPopOutput.readSamplesWithUpmix((int16_t*)data, samplesPopped, deviceChannelCount - OUTPUT_CHANNEL_COUNT); - } else { - lastPopOutput.readSamplesWithDownmix((int16_t*)data, samplesPopped); + int16_t* scratchBuffer = _audio->_limitScratchBuffer; + lastPopOutput.readSamples(scratchBuffer, samplesPopped); + + float* mixBuffer = _audio->_limitMixBuffer; + for (int i = 0; i < samplesPopped; i++) { + mixBuffer[i] = (float)scratchBuffer[i] * (1/32768.0f); } + + int framesPopped = samplesPopped / AudioConstants::STEREO; + if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) { + // limit the audio + _audio->_audioLimiter.render(mixBuffer, (int16_t*)data, framesPopped); + } else { + _audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped); + + // upmix or downmix to deviceChannelCount + if (deviceChannelCount > OUTPUT_CHANNEL_COUNT) { + int extraChannels = deviceChannelCount - OUTPUT_CHANNEL_COUNT; + channelUpmix(scratchBuffer, (int16_t*)data, samplesPopped, extraChannels); + } else { + channelDownmix(scratchBuffer, (int16_t*)data, samplesPopped); + } + } + bytesWritten = (samplesPopped * AudioConstants::SAMPLE_SIZE) * deviceChannelCount / OUTPUT_CHANNEL_COUNT; } else { // nothing on network, don't grab anything from injectors, and just return 0s diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 123da35319..0320b5db2b 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -300,6 +300,11 @@ private: // for local hrtf-ing float _mixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _scratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; + + // for limiting + int _limitPeriod { 0 }; + float* _limitMixBuffer { NULL }; + int16_t* _limitScratchBuffer { NULL }; AudioLimiter _audioLimiter; // Adds Reverb From 3a0d874bb5557b9815d05f7f071b0e9d7f894f80 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 9 Jan 2017 13:21:20 -0500 Subject: [PATCH 04/45] add injector ring buffer to audio client --- libraries/audio-client/src/AudioClient.cpp | 3 ++- libraries/audio-client/src/AudioClient.h | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 1e97de8dca..8e73c1a983 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -127,6 +127,7 @@ AudioClient::AudioClient() : _loopbackAudioOutput(NULL), _loopbackOutputDevice(NULL), _inputRingBuffer(0), + _localInjectorsBuffer(0), _receivedAudioStream(RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES), _isStereoInput(false), _outputStarveDetectionStartTimeMsec(0), @@ -146,7 +147,7 @@ AudioClient::AudioClient() : _networkToOutputResampler(NULL), _audioLimiter(AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT), _outgoingAvatarAudioSequenceNumber(0), - _audioOutputIODevice(_receivedAudioStream, this), + _audioOutputIODevice(_localInjectorsBuffer, _receivedAudioStream, this), _stats(&_receivedAudioStream), _inputGate(), _positionGetter(DEFAULT_POSITION_GETTER), diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 0320b5db2b..0733bba47c 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -84,8 +84,10 @@ public: class AudioOutputIODevice : public QIODevice { public: - AudioOutputIODevice(MixedProcessedAudioStream& receivedAudioStream, AudioClient* audio) : - _receivedAudioStream(receivedAudioStream), _audio(audio), _unfulfilledReads(0) {}; + AudioOutputIODevice(AudioRingBuffer& localInjectorsBuffer, MixedProcessedAudioStream& receivedAudioStream, + AudioClient* audio) : + _localInjectorsBuffer(localInjectorsBuffer), _receivedAudioStream(receivedAudioStream), + _audio(audio), _unfulfilledReads(0) {} void start() { open(QIODevice::ReadOnly | QIODevice::Unbuffered); } void stop() { close(); } @@ -93,6 +95,7 @@ public: qint64 writeData(const char * data, qint64 maxSize) override { return 0; } int getRecentUnfulfilledReads() { int unfulfilledReads = _unfulfilledReads; _unfulfilledReads = 0; return unfulfilledReads; } private: + AudioRingBuffer& _localInjectorsBuffer; MixedProcessedAudioStream& _receivedAudioStream; AudioClient* _audio; int _unfulfilledReads; @@ -262,6 +265,7 @@ private: QAudioOutput* _loopbackAudioOutput; QIODevice* _loopbackOutputDevice; AudioRingBuffer _inputRingBuffer; + AudioRingBuffer _localInjectorsBuffer; MixedProcessedAudioStream _receivedAudioStream; bool _isStereoInput; From 0f1ec63b177385746d40f2661b30de4449f53df5 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 9 Jan 2017 13:41:10 -0500 Subject: [PATCH 05/45] enable injectors in audio device callback --- libraries/audio-client/src/AudioClient.cpp | 42 ++++++++++++++++------ 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 8e73c1a983..9056ee0bea 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1561,22 +1561,44 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { // restrict samplesRequested to the size of our mix/scratch buffers samplesRequested = std::min(samplesRequested, _audio->_limitPeriod); - int samplesPopped; - int bytesWritten; + int16_t* scratchBuffer = _audio->_limitScratchBuffer; + float* mixBuffer = _audio->_limitMixBuffer; - if ((samplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) { - qCDebug(audiostream, "Read %d samples from buffer (%d available)", samplesPopped, _receivedAudioStream.getSamplesAvailable()); + int networkSamplesPopped; + if ((networkSamplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) { + qCDebug(audiostream, "Read %d samples from buffer (%d available)", networkSamplesPopped, _receivedAudioStream.getSamplesAvailable()); AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput(); + lastPopOutput.readSamples(scratchBuffer, networkSamplesPopped); - int16_t* scratchBuffer = _audio->_limitScratchBuffer; - lastPopOutput.readSamples(scratchBuffer, samplesPopped); - - float* mixBuffer = _audio->_limitMixBuffer; - for (int i = 0; i < samplesPopped; i++) { + // convert to mix buffer (float) + for (int i = 0; i < networkSamplesPopped; i++) { mixBuffer[i] = (float)scratchBuffer[i] * (1/32768.0f); } - int framesPopped = samplesPopped / AudioConstants::STEREO; + samplesRequested = networkSamplesPopped; + } + + int injectorSamplesPopped; + if ((injectorSamplesPopped = _localInjectorsBuffer.readSamples(scratchBuffer, samplesRequested)) > 0) { + qCDebug(audiostream, "Read %d samples from injectors (%d available)", injectorSamplesPopped, _localInjectorsBuffer.samplesAvailable()); + + if (networkSamplesPopped == 0) { + // convert to mix buffer (float) + for (int i = 0; i < injectorSamplesPopped; i++) { + mixBuffer[i] = (float)scratchBuffer[i] * (1/32768.0f); + } + } else { + // add to mix buffer (float) + for (int i = 0; i < injectorSamplesPopped; i++) { + mixBuffer[i] += (float)scratchBuffer[i] * (1/32768.0f); + } + } + } + + int samplesPopped = std::max(networkSamplesPopped, networkSamplesPopped); + int framesPopped = samplesPopped / AudioConstants::STEREO; + int bytesWritten; + if (samplesPopped > 0) { if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) { // limit the audio _audio->_audioLimiter.render(mixBuffer, (int16_t*)data, framesPopped); From dee5f6303739e62a6996d1670ef38597c76cf70f Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 10 Jan 2017 18:19:18 -0500 Subject: [PATCH 06/45] rename audio mix/scratch buffers Conflicts: libraries/audio-client/src/AudioClient.cpp libraries/audio-client/src/AudioClient.h --- libraries/audio-client/src/AudioClient.cpp | 30 +++++++++++----------- libraries/audio-client/src/AudioClient.h | 15 ++++++----- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 9056ee0bea..63c4139d84 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1175,26 +1175,26 @@ void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteA // convert network audio (int16_t) to mix audio (float) for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) { - _mixBuffer[i] = (float)decodedSamples[i] * (1/32768.0f); + _networkMixBuffer[i] = (float)decodedSamples[i] * (1/32768.0f); } // apply stereo reverb bool hasReverb = _reverb || _receivedAudioStream.hasReverb(); if (hasReverb) { updateReverbOptions(); - _listenerReverb.render(_mixBuffer, _mixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + _listenerReverb.render(_networkMixBuffer, _networkMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } // convert mix audio (float) to network audio (int16_t) for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) { - _scratchBuffer[i] = (int16_t)(_mixBuffer[i] * 32768.0f); + _networkScratchBuffer[i] = (int16_t)(_networkMixBuffer[i] * 32768.0f); } // resample to output sample rate if (_networkToOutputResampler) { - _networkToOutputResampler->render(_scratchBuffer, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + _networkToOutputResampler->render(_networkScratchBuffer, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } else { - memcpy(outputBuffer.data(), _scratchBuffer, AudioConstants::NETWORK_FRAME_BYTES_STEREO); + memcpy(outputBuffer.data(), _networkScratchBuffer, AudioConstants::NETWORK_FRAME_BYTES_STEREO); } } @@ -1388,11 +1388,11 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice delete _loopbackAudioOutput; _loopbackAudioOutput = NULL; - delete[] _limitMixBuffer; - _limitMixBuffer = NULL; + delete _outputMixBuffer; + _outputMixBuffer = NULL; - delete[] _limitScratchBuffer; - _limitScratchBuffer = NULL; + delete _outputScratchBuffer; + _outputScratchBuffer = NULL; } if (_networkToOutputResampler) { @@ -1445,9 +1445,9 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice int periodSampleSize = _audioOutput->periodSize() / AudioConstants::SAMPLE_SIZE; // device callback is not restricted to periodSampleSize, so double the mix/scratch buffer sizes - _limitPeriod = periodSampleSize * 2; - _limitMixBuffer = new float[_limitPeriod]; - _limitScratchBuffer = new int16_t[_limitPeriod]; + _outputPeriod = periodSampleSize * 2; + _outputMixBuffer = new float[_outputPeriod]; + _outputScratchBuffer = new int16_t[_outputPeriod]; qCDebug(audioclient) << "Output Buffer capacity in frames: " << _audioOutput->bufferSize() / AudioConstants::SAMPLE_SIZE / (float)deviceFrameSize << "requested bytes:" << requestedSize << "actual bytes:" << _audioOutput->bufferSize() << @@ -1559,10 +1559,10 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { int deviceChannelCount = _audio->_outputFormat.channelCount(); int samplesRequested = (int)(maxSize / AudioConstants::SAMPLE_SIZE) * OUTPUT_CHANNEL_COUNT / deviceChannelCount; // restrict samplesRequested to the size of our mix/scratch buffers - samplesRequested = std::min(samplesRequested, _audio->_limitPeriod); + samplesRequested = std::min(samplesRequested, _audio->_outputPeriod); - int16_t* scratchBuffer = _audio->_limitScratchBuffer; - float* mixBuffer = _audio->_limitMixBuffer; + int16_t* scratchBuffer = _audio->_outputScratchBuffer; + float* mixBuffer = _audio->_outputMixBuffer; int networkSamplesPopped; if ((networkSamplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) { diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 0733bba47c..e245277d61 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -301,14 +301,15 @@ private: AudioSRC* _inputToNetworkResampler; AudioSRC* _networkToOutputResampler; - // for local hrtf-ing - float _mixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; - int16_t _scratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; + // for network audio (used by network audio threads) + float _networkMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + int16_t _networkScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; + + // for output audio (used by this thread only) + int _outputPeriod { 0 }; + float* _outputMixBuffer { NULL }; + int16_t* _outputScratchBuffer { NULL }; - // for limiting - int _limitPeriod { 0 }; - float* _limitMixBuffer { NULL }; - int16_t* _limitScratchBuffer { NULL }; AudioLimiter _audioLimiter; // Adds Reverb From 969d776e1f0a824013f2cebd341a1ee8b663c902 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 10 Jan 2017 18:31:17 -0500 Subject: [PATCH 07/45] queue injector audio after device callbacks Conflicts: libraries/audio-client/src/AudioClient.cpp --- libraries/audio-client/src/AudioClient.cpp | 70 +++++++++++++++++++--- libraries/audio-client/src/AudioClient.h | 9 ++- 2 files changed, 68 insertions(+), 11 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 63c4139d84..5e52953949 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1083,14 +1083,62 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { PacketType::MicrophoneAudioWithEcho, _selectedCodecName); } -void AudioClient::mixLocalAudioInjectors(float* mixBuffer) { +void AudioClient::prepareLocalAudioInjectors() { + if (_outputPeriod == 0) { + return; + } + + int periodSamples = _audioOutput->periodSize() / AudioConstants::SAMPLE_SIZE; + int samplesNeeded; + if ((samplesNeeded = periodSamples - _localInjectorsBuffer.samplesAvailable()) > 0) { + while (samplesNeeded > 0) { + // get a network frame of local injectors' audio + if (!mixLocalAudioInjectors(_localMixBuffer)) { + return; + } + + // reverb + if (_reverb) { + updateReverbOptions(); + _listenerReverb.render(_localMixBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + } + + // convert mix audio (float) to network audio (int16_t) + for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) { + _localScratchBuffer[i] = (int16_t)(_localMixBuffer[i] * 32768.0f); + } + + // resample to output sample rate + int samples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; + int16_t* scratchBuffer = _localScratchBuffer; + if (_networkToOutputResampler) { + int frames = _networkToOutputResampler->render(_localScratchBuffer, _outputScratchBuffer, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + samples = frames * AudioConstants::STEREO; + scratchBuffer = _outputScratchBuffer; + } + + // write to local injectors' ring buffer + _localInjectorsBuffer.writeSamples(scratchBuffer, samples); + samplesNeeded -= samples; + } + } +} + +bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) { QVector injectorsToRemove; // lock the injector vector Lock lock(_injectorsMutex); - for (AudioInjector* injector : getActiveLocalAudioInjectors()) { + if (_activeLocalAudioInjectors.size() == 0) { + return false; + } + + memset(mixBuffer, 0, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO * sizeof(float)); + + for (AudioInjector* injector : _activeLocalAudioInjectors) { if (injector->getLocalBuffer()) { static const int HRTF_DATASET_INDEX = 1; @@ -1099,8 +1147,8 @@ void AudioClient::mixLocalAudioInjectors(float* mixBuffer) { qint64 bytesToRead = numChannels * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; // get one frame from the injector - memset(_scratchBuffer, 0, bytesToRead); - if (0 < injector->getLocalBuffer()->readData((char*)_scratchBuffer, bytesToRead)) { + memset(_localScratchBuffer, 0, bytesToRead); + if (0 < injector->getLocalBuffer()->readData((char*)_localScratchBuffer, bytesToRead)) { if (injector->isAmbisonic()) { @@ -1120,7 +1168,7 @@ void AudioClient::mixLocalAudioInjectors(float* mixBuffer) { float qz = relativeOrientation.y; // Ambisonic gets spatialized into mixBuffer - injector->getLocalFOA().render(_scratchBuffer, mixBuffer, HRTF_DATASET_INDEX, + injector->getLocalFOA().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, qw, qx, qy, qz, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } else if (injector->isStereo()) { @@ -1128,7 +1176,7 @@ void AudioClient::mixLocalAudioInjectors(float* mixBuffer) { // stereo gets directly mixed into mixBuffer float gain = injector->getVolume(); for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) { - mixBuffer[i] += (float)_scratchBuffer[i] * (1/32768.0f) * gain; + mixBuffer[i] += (float)_localScratchBuffer[i] * (1/32768.0f) * gain; } } else { @@ -1140,7 +1188,7 @@ void AudioClient::mixLocalAudioInjectors(float* mixBuffer) { float azimuth = azimuthForSource(relativePosition); // mono gets spatialized into mixBuffer - injector->getLocalHRTF().render(_scratchBuffer, mixBuffer, HRTF_DATASET_INDEX, + injector->getLocalHRTF().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, azimuth, distance, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } @@ -1161,8 +1209,10 @@ void AudioClient::mixLocalAudioInjectors(float* mixBuffer) { for (AudioInjector* injector : injectorsToRemove) { qCDebug(audioclient) << "removing injector"; - getActiveLocalAudioInjectors().removeOne(injector); + _activeLocalAudioInjectors.removeOne(injector); } + + return true; } void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteArray& outputBuffer) { @@ -1448,6 +1498,7 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice _outputPeriod = periodSampleSize * 2; _outputMixBuffer = new float[_outputPeriod]; _outputScratchBuffer = new int16_t[_outputPeriod]; + _localInjectorsBuffer.resizeForFrameSize(_outputPeriod); qCDebug(audioclient) << "Output Buffer capacity in frames: " << _audioOutput->bufferSize() / AudioConstants::SAMPLE_SIZE / (float)deviceFrameSize << "requested bytes:" << requestedSize << "actual bytes:" << _audioOutput->bufferSize() << @@ -1595,6 +1646,9 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { } } + // prepare injectors for the next callback + QMetaObject::invokeMethod(_audio, "prepareLocalAudioInjectors", Qt::QueuedConnection); + int samplesPopped = std::max(networkSamplesPopped, networkSamplesPopped); int framesPopped = samplesPopped / AudioConstants::STEREO; int bytesWritten; diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index e245277d61..322a27d71e 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -132,8 +132,6 @@ public: Q_INVOKABLE void setAvatarBoundingBoxParameters(glm::vec3 corner, glm::vec3 scale); - QVector& getActiveLocalAudioInjectors() { return _activeLocalAudioInjectors; } - void checkDevices(); static const float CALLBACK_ACCELERATOR_RATIO; @@ -174,6 +172,7 @@ public slots: int setOutputBufferSize(int numFrames, bool persist = true); + void prepareLocalAudioInjectors(); bool outputLocalInjector(AudioInjector* injector) override; bool shouldLoopbackInjectors() override { return _shouldEchoToServer; } @@ -221,7 +220,7 @@ protected: private: void outputFormatChanged(); - void mixLocalAudioInjectors(float* mixBuffer); + bool mixLocalAudioInjectors(float* mixBuffer); float azimuthForSource(const glm::vec3& relativePosition); float gainForSource(float distance, float volume); @@ -305,6 +304,10 @@ private: float _networkMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _networkScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; + // for local audio (used by this thread only) + float _localMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; + // for output audio (used by this thread only) int _outputPeriod { 0 }; float* _outputMixBuffer { NULL }; From 42f5af7c39a0f9a07f21eed2f6234379c7e89342 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 11 Jan 2017 11:31:44 -0500 Subject: [PATCH 08/45] improve audiostream debug logs --- libraries/audio-client/src/AudioClient.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 5e52953949..8b89430a76 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1617,7 +1617,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { int networkSamplesPopped; if ((networkSamplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) { - qCDebug(audiostream, "Read %d samples from buffer (%d available)", networkSamplesPopped, _receivedAudioStream.getSamplesAvailable()); + qCDebug(audiostream, "Read %d samples from buffer (%d available, %d requested)", networkSamplesPopped, _receivedAudioStream.getSamplesAvailable(), samplesRequested); AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput(); lastPopOutput.readSamples(scratchBuffer, networkSamplesPopped); @@ -1631,7 +1631,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { int injectorSamplesPopped; if ((injectorSamplesPopped = _localInjectorsBuffer.readSamples(scratchBuffer, samplesRequested)) > 0) { - qCDebug(audiostream, "Read %d samples from injectors (%d available)", injectorSamplesPopped, _localInjectorsBuffer.samplesAvailable()); + qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsBuffer.samplesAvailable(), samplesRequested); if (networkSamplesPopped == 0) { // convert to mix buffer (float) From 61f7f72c5e5dd3a4a6b1c1904ae3aacccbd0e727 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 11 Jan 2017 12:29:04 -0500 Subject: [PATCH 09/45] simplify resampling --- libraries/audio-client/src/AudioClient.cpp | 35 ++++++++++++++-------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 8b89430a76..a85a6c4282 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1108,18 +1108,23 @@ void AudioClient::prepareLocalAudioInjectors() { _localScratchBuffer[i] = (int16_t)(_localMixBuffer[i] * 32768.0f); } - // resample to output sample rate - int samples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; - int16_t* scratchBuffer = _localScratchBuffer; + int samples; if (_networkToOutputResampler) { + // resample to output sample rate int frames = _networkToOutputResampler->render(_localScratchBuffer, _outputScratchBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + // write to local injectors' ring buffer samples = frames * AudioConstants::STEREO; - scratchBuffer = _outputScratchBuffer; + _localInjectorsBuffer.writeSamples(_outputScratchBuffer, samples); + + } else { + // write to local injectors' ring buffer + samples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; + _localInjectorsBuffer.writeSamples(_localScratchBuffer, + AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); } - // write to local injectors' ring buffer - _localInjectorsBuffer.writeSamples(scratchBuffer, samples); samplesNeeded -= samples; } } @@ -1235,16 +1240,20 @@ void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteA _listenerReverb.render(_networkMixBuffer, _networkMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } - // convert mix audio (float) to network audio (int16_t) - for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) { - _networkScratchBuffer[i] = (int16_t)(_networkMixBuffer[i] * 32768.0f); - } - - // resample to output sample rate if (_networkToOutputResampler) { + // convert mix audio (float) to network audio (int16_t) + for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) { + _networkScratchBuffer[i] = (int16_t)(_networkMixBuffer[i] * 32768.0f); + } + + // resample to output sample rate _networkToOutputResampler->render(_networkScratchBuffer, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + } else { - memcpy(outputBuffer.data(), _networkScratchBuffer, AudioConstants::NETWORK_FRAME_BYTES_STEREO); + // convert mix audio (float) to network audio (int16_t) + for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) { + outputSamples[i] = (int16_t)(_networkMixBuffer[i] * 32768.0f); + } } } From 0f08abfa1421fa17a9f9bb59c89693e4ae5dd893 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 11 Jan 2017 12:29:28 -0500 Subject: [PATCH 10/45] fix bug in audio samples popped check --- libraries/audio-client/src/AudioClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index a85a6c4282..e0b36891a7 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1658,7 +1658,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { // prepare injectors for the next callback QMetaObject::invokeMethod(_audio, "prepareLocalAudioInjectors", Qt::QueuedConnection); - int samplesPopped = std::max(networkSamplesPopped, networkSamplesPopped); + int samplesPopped = std::max(networkSamplesPopped, injectorSamplesPopped); int framesPopped = samplesPopped / AudioConstants::STEREO; int bytesWritten; if (samplesPopped > 0) { From d1673e554ff289b5723996a117398c68667bea34 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 11 Jan 2017 12:29:45 -0500 Subject: [PATCH 11/45] simplify audio bytesWritten computation --- libraries/audio-client/src/AudioClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index e0b36891a7..ddc5bf47e5 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1677,7 +1677,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { } } - bytesWritten = (samplesPopped * AudioConstants::SAMPLE_SIZE) * deviceChannelCount / OUTPUT_CHANNEL_COUNT; + bytesWritten = framesPopped * AudioConstants::SAMPLE_SIZE * deviceChannelCount; } else { // nothing on network, don't grab anything from injectors, and just return 0s // this will flood the log: qCDebug(audioclient, "empty/partial network buffer"); From d7085ec6851b7fbf58898f99232d51a8dc7fbba6 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 11 Jan 2017 12:39:43 -0500 Subject: [PATCH 12/45] add audio helpers convertToMix/Scratch --- libraries/audio-client/src/AudioClient.cpp | 61 +++------------------- libraries/shared/src/AudioHelpers.h | 42 +++++++++++++++ 2 files changed, 49 insertions(+), 54 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index ddc5bf47e5..92e3b4dcf2 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -46,6 +46,7 @@ #include #include "PositionalAudioStream.h" +#include "AudioHelpers.h" #include "AudioClientLogging.h" #include "AudioLogging.h" @@ -842,36 +843,6 @@ void AudioClient::setReverbOptions(const AudioEffectOptions* options) { } } -static void channelUpmix(int16_t* source, int16_t* dest, int numSamples, int numExtraChannels) { - - for (int i = 0; i < numSamples/2; i++) { - - // read 2 samples - int16_t left = *source++; - int16_t right = *source++; - - // write 2 + N samples - *dest++ = left; - *dest++ = right; - for (int n = 0; n < numExtraChannels; n++) { - *dest++ = 0; - } - } -} - -static void channelDownmix(int16_t* source, int16_t* dest, int numSamples) { - - for (int i = 0; i < numSamples/2; i++) { - - // read 2 samples - int16_t left = *source++; - int16_t right = *source++; - - // write 1 sample - *dest++ = (int16_t)((left + right) / 2); - } -} - void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { // If there is server echo, reverb will be applied to the recieved audio stream so no need to have it here. bool hasReverb = _reverb || _receivedAudioStream.hasReverb(); @@ -1103,10 +1074,7 @@ void AudioClient::prepareLocalAudioInjectors() { _listenerReverb.render(_localMixBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } - // convert mix audio (float) to network audio (int16_t) - for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) { - _localScratchBuffer[i] = (int16_t)(_localMixBuffer[i] * 32768.0f); - } + convertToScratch(_localScratchBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); int samples; if (_networkToOutputResampler) { @@ -1228,10 +1196,7 @@ void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteA outputBuffer.resize(_outputFrameSize * AudioConstants::SAMPLE_SIZE); int16_t* outputSamples = reinterpret_cast(outputBuffer.data()); - // convert network audio (int16_t) to mix audio (float) - for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) { - _networkMixBuffer[i] = (float)decodedSamples[i] * (1/32768.0f); - } + convertToMix(_networkMixBuffer, decodedSamples, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); // apply stereo reverb bool hasReverb = _reverb || _receivedAudioStream.hasReverb(); @@ -1241,19 +1206,13 @@ void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteA } if (_networkToOutputResampler) { - // convert mix audio (float) to network audio (int16_t) - for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) { - _networkScratchBuffer[i] = (int16_t)(_networkMixBuffer[i] * 32768.0f); - } + convertToScratch(_networkScratchBuffer, _networkMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); // resample to output sample rate _networkToOutputResampler->render(_networkScratchBuffer, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } else { - // convert mix audio (float) to network audio (int16_t) - for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) { - outputSamples[i] = (int16_t)(_networkMixBuffer[i] * 32768.0f); - } + convertToScratch(outputSamples, _networkMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); } } @@ -1630,10 +1589,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput(); lastPopOutput.readSamples(scratchBuffer, networkSamplesPopped); - // convert to mix buffer (float) - for (int i = 0; i < networkSamplesPopped; i++) { - mixBuffer[i] = (float)scratchBuffer[i] * (1/32768.0f); - } + convertToMix(mixBuffer, scratchBuffer, networkSamplesPopped); samplesRequested = networkSamplesPopped; } @@ -1643,10 +1599,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsBuffer.samplesAvailable(), samplesRequested); if (networkSamplesPopped == 0) { - // convert to mix buffer (float) - for (int i = 0; i < injectorSamplesPopped; i++) { - mixBuffer[i] = (float)scratchBuffer[i] * (1/32768.0f); - } + convertToMix(mixBuffer, scratchBuffer, injectorSamplesPopped); } else { // add to mix buffer (float) for (int i = 0; i < injectorSamplesPopped; i++) { diff --git a/libraries/shared/src/AudioHelpers.h b/libraries/shared/src/AudioHelpers.h index b43764ef5d..b0fcb7248d 100644 --- a/libraries/shared/src/AudioHelpers.h +++ b/libraries/shared/src/AudioHelpers.h @@ -91,4 +91,46 @@ static inline float unpackFloatGainFromByte(uint8_t byte) { return gain; } +static inline void convertToMix(float* mixBuffer, const int16_t* scratchBuffer, int numSamples) { + for (int i = 0; i < numSamples; i++) { + mixBuffer[i] = (float)scratchBuffer[i] * (1/32768.0f); + } +} + +static inline void convertToScratch(int16_t* scratchBuffer, const float* mixBuffer, int numSamples) { + for (int i = 0; i < numSamples; i++) { + scratchBuffer[i] = (int16_t)(mixBuffer[i] * 32768.0f); + } +} + +static void channelUpmix(int16_t* source, int16_t* dest, int numSamples, int numExtraChannels) { + + for (int i = 0; i < numSamples/2; i++) { + + // read 2 samples + int16_t left = *source++; + int16_t right = *source++; + + // write 2 + N samples + *dest++ = left; + *dest++ = right; + for (int n = 0; n < numExtraChannels; n++) { + *dest++ = 0; + } + } +} + +static void channelDownmix(int16_t* source, int16_t* dest, int numSamples) { + + for (int i = 0; i < numSamples/2; i++) { + + // read 2 samples + int16_t left = *source++; + int16_t right = *source++; + + // write 1 sample + *dest++ = (int16_t)((left + right) / 2); + } +} + #endif // hifi_AudioHelpers_h From 4c7c7ee3ccfdacc8c6fa3e33194b95de5014b14a Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 11 Jan 2017 15:25:31 -0500 Subject: [PATCH 13/45] mv audio injector preparation to own thread --- libraries/audio-client/src/AudioClient.cpp | 134 ++++++++++++++------- libraries/audio-client/src/AudioClient.h | 24 +++- 2 files changed, 109 insertions(+), 49 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 92e3b4dcf2..eb2b0f87e0 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -113,6 +113,10 @@ private: bool _quit { false }; }; +void AudioInjectorsThread::prepare() { + _audio->prepareLocalAudioInjectors(); +} + AudioClient::AudioClient() : AbstractAudioInterface(), _gate(this), @@ -146,6 +150,8 @@ AudioClient::AudioClient() : _reverbOptions(&_scriptReverbOptions), _inputToNetworkResampler(NULL), _networkToOutputResampler(NULL), + _localToOutputResampler(NULL), + _localAudioThread(this), _audioLimiter(AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT), _outgoingAvatarAudioSequenceNumber(0), _audioOutputIODevice(_localInjectorsBuffer, _receivedAudioStream, this), @@ -178,6 +184,10 @@ AudioClient::AudioClient() : _checkDevicesThread->setPriority(QThread::LowPriority); _checkDevicesThread->start(); + // start a thread to process local injectors + _localAudioThread.setObjectName("LocalAudio Thread"); + _localAudioThread.start(); + configureReverb(); auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); @@ -215,6 +225,7 @@ void AudioClient::reset() { _stats.reset(); _sourceReverb.reset(); _listenerReverb.reset(); + _localReverb.reset(); } void AudioClient::audioMixerKilled() { @@ -764,6 +775,7 @@ void AudioClient::configureReverb() { p.wetDryMix = _reverbOptions->getWetDryMix(); _listenerReverb.setParameters(&p); + _localReverb.setParameters(&p); // used only for adding self-reverb to loopback audio p.sampleRate = _outputFormat.sampleRate(); @@ -810,6 +822,7 @@ void AudioClient::setReverb(bool reverb) { if (!_reverb) { _sourceReverb.reset(); _listenerReverb.reset(); + _localReverb.reset(); } } @@ -1059,42 +1072,56 @@ void AudioClient::prepareLocalAudioInjectors() { return; } - int periodSamples = _audioOutput->periodSize() / AudioConstants::SAMPLE_SIZE; - int samplesNeeded; - if ((samplesNeeded = periodSamples - _localInjectorsBuffer.samplesAvailable()) > 0) { - while (samplesNeeded > 0) { - // get a network frame of local injectors' audio - if (!mixLocalAudioInjectors(_localMixBuffer)) { - return; - } + int bufferCapacity = _localInjectorsBuffer.getSampleCapacity(); + if (_localToOutputResampler) { + // avoid overwriting the buffer + bufferCapacity -= + _localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) * + AudioConstants::STEREO; + bufferCapacity += 1; + } - // reverb - if (_reverb) { - updateReverbOptions(); - _listenerReverb.render(_localMixBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - } + int samplesNeeded = std::numeric_limits::max(); + while (samplesNeeded > 0) { + // lock for every write to avoid locking out the device callback + Lock lock(_localAudioMutex); - convertToScratch(_localScratchBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); - - int samples; - if (_networkToOutputResampler) { - // resample to output sample rate - int frames = _networkToOutputResampler->render(_localScratchBuffer, _outputScratchBuffer, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - // write to local injectors' ring buffer - samples = frames * AudioConstants::STEREO; - _localInjectorsBuffer.writeSamples(_outputScratchBuffer, samples); - - } else { - // write to local injectors' ring buffer - samples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; - _localInjectorsBuffer.writeSamples(_localScratchBuffer, - AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); - } - - samplesNeeded -= samples; + samplesNeeded = bufferCapacity - _localInjectorsBuffer.samplesAvailable(); + if (samplesNeeded <= 0) { + break; } + + // get a network frame of local injectors' audio + if (!mixLocalAudioInjectors(_localMixBuffer)) { + break; + } + + // reverb + if (_reverb) { + _localReverb.render(_localMixBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + } + + convertToScratch(_localScratchBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); + + int samples; + if (_localToOutputResampler) { + // resample to output sample rate + int frames = _localToOutputResampler->render(_localScratchBuffer, _localOutputScratchBuffer, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + // write to local injectors' ring buffer + samples = frames * AudioConstants::STEREO; + _localInjectorsBuffer.writeSamples(_localOutputScratchBuffer, samples); + + } + else { + // write to local injectors' ring buffer + samples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; + _localInjectorsBuffer.writeSamples(_localScratchBuffer, + AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); + } + + samplesNeeded -= samples; } } @@ -1158,7 +1185,7 @@ bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) { glm::vec3 relativePosition = injector->getPosition() - _positionGetter(); float distance = glm::max(glm::length(relativePosition), EPSILON); float gain = gainForSource(distance, injector->getVolume()); - float azimuth = azimuthForSource(relativePosition); + float azimuth = azimuthForSource(relativePosition); // mono gets spatialized into mixBuffer injector->getLocalHRTF().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, @@ -1395,6 +1422,8 @@ void AudioClient::outputNotify() { bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo) { bool supportedFormat = false; + Lock lock(_localAudioMutex); + // cleanup any previously initialized device if (_audioOutput) { _audioOutput->stop(); @@ -1406,17 +1435,23 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice delete _loopbackAudioOutput; _loopbackAudioOutput = NULL; - delete _outputMixBuffer; + delete[] _outputMixBuffer; _outputMixBuffer = NULL; - delete _outputScratchBuffer; + delete[] _outputScratchBuffer; _outputScratchBuffer = NULL; + + delete[] _localOutputScratchBuffer; + _localOutputScratchBuffer = NULL; } if (_networkToOutputResampler) { // if we were using an input to network resampler, delete it here delete _networkToOutputResampler; _networkToOutputResampler = NULL; + + delete _localToOutputResampler; + _localToOutputResampler = NULL; } if (!outputDeviceInfo.isNull()) { @@ -1436,6 +1471,7 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice assert(_outputFormat.sampleSize() == 16); _networkToOutputResampler = new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT); + _localToOutputResampler = new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT); } else { qCDebug(audioclient) << "No resampling required for network output to match actual output format."; @@ -1466,7 +1502,8 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice _outputPeriod = periodSampleSize * 2; _outputMixBuffer = new float[_outputPeriod]; _outputScratchBuffer = new int16_t[_outputPeriod]; - _localInjectorsBuffer.resizeForFrameSize(_outputPeriod); + _localOutputScratchBuffer = new int16_t[_outputPeriod]; + _localInjectorsBuffer.resizeForFrameSize(_outputPeriod * 2); qCDebug(audioclient) << "Output Buffer capacity in frames: " << _audioOutput->bufferSize() / AudioConstants::SAMPLE_SIZE / (float)deviceFrameSize << "requested bytes:" << requestedSize << "actual bytes:" << _audioOutput->bufferSize() << @@ -1595,21 +1632,26 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { } int injectorSamplesPopped; - if ((injectorSamplesPopped = _localInjectorsBuffer.readSamples(scratchBuffer, samplesRequested)) > 0) { - qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsBuffer.samplesAvailable(), samplesRequested); + { + Lock lock(_audio->_localAudioMutex); + if ((injectorSamplesPopped = _localInjectorsBuffer.readSamples(scratchBuffer, samplesRequested)) > 0) { + qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsBuffer.samplesAvailable(), samplesRequested); - if (networkSamplesPopped == 0) { - convertToMix(mixBuffer, scratchBuffer, injectorSamplesPopped); - } else { - // add to mix buffer (float) - for (int i = 0; i < injectorSamplesPopped; i++) { - mixBuffer[i] += (float)scratchBuffer[i] * (1/32768.0f); + if (_audio->_shouldEchoToServer) { + // omit local audio, it should be echoed + } else if (networkSamplesPopped == 0) { + convertToMix(mixBuffer, scratchBuffer, injectorSamplesPopped); + } else { + // add to mix buffer (float) + for (int i = 0; i < injectorSamplesPopped; i++) { + mixBuffer[i] += (float)scratchBuffer[i] * (1 / 32768.0f); + } } } } // prepare injectors for the next callback - QMetaObject::invokeMethod(_audio, "prepareLocalAudioInjectors", Qt::QueuedConnection); + QMetaObject::invokeMethod(&_audio->_localAudioThread, "prepare", Qt::QueuedConnection); int samplesPopped = std::max(networkSamplesPopped, injectorSamplesPopped); int framesPopped = samplesPopped / AudioConstants::STEREO; diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 322a27d71e..78fd97abb2 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -69,6 +69,19 @@ class QIODevice; class Transform; class NLPacket; +class AudioInjectorsThread : public QThread { + Q_OBJECT + +public: + AudioInjectorsThread(AudioClient* audio) : _audio(audio) {} + +public slots : + void prepare(); + +private: + AudioClient* _audio; +}; + class AudioClient : public AbstractAudioInterface, public Dependency { Q_OBJECT SINGLETON_DEPENDENCY @@ -295,20 +308,25 @@ private: AudioEffectOptions* _reverbOptions; AudioReverb _sourceReverb { AudioConstants::SAMPLE_RATE }; AudioReverb _listenerReverb { AudioConstants::SAMPLE_RATE }; + AudioReverb _localReverb { AudioConstants::SAMPLE_RATE }; // possible streams needed for resample AudioSRC* _inputToNetworkResampler; AudioSRC* _networkToOutputResampler; + AudioSRC* _localToOutputResampler; - // for network audio (used by network audio threads) + // for network audio (used by network audio thread) float _networkMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _networkScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; - // for local audio (used by this thread only) + // for local audio (used by audio injectors thread) float _localMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; + int16_t* _localOutputScratchBuffer { NULL }; + AudioInjectorsThread _localAudioThread; + Mutex _localAudioMutex; - // for output audio (used by this thread only) + // for output audio (used by this thread) int _outputPeriod { 0 }; float* _outputMixBuffer { NULL }; int16_t* _outputScratchBuffer { NULL }; From 492795f7e5a7ef7c54e2e6717c414d30c6c2c8a1 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 19 Jan 2017 18:08:45 -0500 Subject: [PATCH 14/45] audio client cosmetics --- libraries/audio-client/src/AudioClient.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index eb2b0f87e0..2358c20a2b 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1113,8 +1113,7 @@ void AudioClient::prepareLocalAudioInjectors() { samples = frames * AudioConstants::STEREO; _localInjectorsBuffer.writeSamples(_localOutputScratchBuffer, samples); - } - else { + } else { // write to local injectors' ring buffer samples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; _localInjectorsBuffer.writeSamples(_localScratchBuffer, @@ -1675,7 +1674,6 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { bytesWritten = framesPopped * AudioConstants::SAMPLE_SIZE * deviceChannelCount; } else { // nothing on network, don't grab anything from injectors, and just return 0s - // this will flood the log: qCDebug(audioclient, "empty/partial network buffer"); memset(data, 0, maxSize); bytesWritten = maxSize; } From 7261b5285ee35f714ffba843d882109143c2787d Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 19 Jan 2017 18:09:07 -0500 Subject: [PATCH 15/45] omit all injector samples on server echo --- libraries/audio-client/src/AudioClient.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 2358c20a2b..b0d8619482 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1638,6 +1638,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { if (_audio->_shouldEchoToServer) { // omit local audio, it should be echoed + injectorSamplesPopped = 0; } else if (networkSamplesPopped == 0) { convertToMix(mixBuffer, scratchBuffer, injectorSamplesPopped); } else { From c5415f96240696037278eeaa8dda336a3927a8d0 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 19 Jan 2017 14:36:49 -0500 Subject: [PATCH 16/45] clean audio client logs --- libraries/audio-client/src/AudioClient.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index b0d8619482..eadbe08b20 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -378,7 +378,7 @@ QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { CoUninitialize(); } - qCDebug(audioclient) << "DEBUG [" << deviceName << "] [" << getNamedAudioDeviceForMode(mode, deviceName).deviceName() << "]"; + qCDebug(audioclient) << "[" << deviceName << "] [" << getNamedAudioDeviceForMode(mode, deviceName).deviceName() << "]"; return getNamedAudioDeviceForMode(mode, deviceName); #endif @@ -400,12 +400,12 @@ bool nativeFormatForAudioDevice(const QAudioDeviceInfo& audioDevice, audioFormat.setByteOrder(QAudioFormat::LittleEndian); if (!audioDevice.isFormatSupported(audioFormat)) { - qCDebug(audioclient) << "WARNING: The native format is" << audioFormat << "but isFormatSupported() failed."; + qCWarning(audioclient) << "The native format is" << audioFormat << "but isFormatSupported() failed."; return false; } // converting to/from this rate must produce an integral number of samples if (audioFormat.sampleRate() * AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL % AudioConstants::SAMPLE_RATE != 0) { - qCDebug(audioclient) << "WARNING: The native sample rate [" << audioFormat.sampleRate() << "] is not supported."; + qCWarning(audioclient) << "The native sample rate [" << audioFormat.sampleRate() << "] is not supported."; return false; } return true; @@ -739,12 +739,12 @@ QVector AudioClient::getDeviceNames(QAudio::Mode mode) { } bool AudioClient::switchInputToAudioDevice(const QString& inputDeviceName) { - qCDebug(audioclient) << "DEBUG [" << inputDeviceName << "] [" << getNamedAudioDeviceForMode(QAudio::AudioInput, inputDeviceName).deviceName() << "]"; + qCDebug(audioclient) << "[" << inputDeviceName << "] [" << getNamedAudioDeviceForMode(QAudio::AudioInput, inputDeviceName).deviceName() << "]"; return switchInputToAudioDevice(getNamedAudioDeviceForMode(QAudio::AudioInput, inputDeviceName)); } bool AudioClient::switchOutputToAudioDevice(const QString& outputDeviceName) { - qCDebug(audioclient) << "DEBUG [" << outputDeviceName << "] [" << getNamedAudioDeviceForMode(QAudio::AudioOutput, outputDeviceName).deviceName() << "]"; + qCDebug(audioclient) << "[" << outputDeviceName << "] [" << getNamedAudioDeviceForMode(QAudio::AudioOutput, outputDeviceName).deviceName() << "]"; return switchOutputToAudioDevice(getNamedAudioDeviceForMode(QAudio::AudioOutput, outputDeviceName)); } From 4f7f3c2a609a897db8bc64f1cf1ffcc3cd36196c Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 19 Jan 2017 15:17:07 -0500 Subject: [PATCH 17/45] mv localInjectorsBuffer to float-based localInjectorsStream The localInjectorsBuffer is based on AudioRingBuffer, which only accounts for int16_t. Local injectors are mixed, and so they can exceed std::numeric_limits before limiting. This will allow them to remain as float until limiting (in the device callback) - once the new stream is implemented. --- libraries/audio-client/src/AudioClient.cpp | 46 +++++++++------------- libraries/audio-client/src/AudioClient.h | 21 +++++++--- 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index eadbe08b20..491c4e341b 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -132,7 +132,7 @@ AudioClient::AudioClient() : _loopbackAudioOutput(NULL), _loopbackOutputDevice(NULL), _inputRingBuffer(0), - _localInjectorsBuffer(0), + _localInjectorsStream(0), _receivedAudioStream(RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES), _isStereoInput(false), _outputStarveDetectionStartTimeMsec(0), @@ -154,7 +154,7 @@ AudioClient::AudioClient() : _localAudioThread(this), _audioLimiter(AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT), _outgoingAvatarAudioSequenceNumber(0), - _audioOutputIODevice(_localInjectorsBuffer, _receivedAudioStream, this), + _audioOutputIODevice(_localInjectorsStream, _receivedAudioStream, this), _stats(&_receivedAudioStream), _inputGate(), _positionGetter(DEFAULT_POSITION_GETTER), @@ -1072,7 +1072,7 @@ void AudioClient::prepareLocalAudioInjectors() { return; } - int bufferCapacity = _localInjectorsBuffer.getSampleCapacity(); + int bufferCapacity = _localInjectorsStream.getSampleCapacity(); if (_localToOutputResampler) { // avoid overwriting the buffer bufferCapacity -= @@ -1086,7 +1086,7 @@ void AudioClient::prepareLocalAudioInjectors() { // lock for every write to avoid locking out the device callback Lock lock(_localAudioMutex); - samplesNeeded = bufferCapacity - _localInjectorsBuffer.samplesAvailable(); + samplesNeeded = bufferCapacity - _localInjectorsStream.samplesAvailable(); if (samplesNeeded <= 0) { break; } @@ -1101,22 +1101,20 @@ void AudioClient::prepareLocalAudioInjectors() { _localReverb.render(_localMixBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } - convertToScratch(_localScratchBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); - int samples; if (_localToOutputResampler) { // resample to output sample rate - int frames = _localToOutputResampler->render(_localScratchBuffer, _localOutputScratchBuffer, + int frames = _localToOutputResampler->render(_localMixBuffer, _localOutputMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); // write to local injectors' ring buffer samples = frames * AudioConstants::STEREO; - _localInjectorsBuffer.writeSamples(_localOutputScratchBuffer, samples); + _localInjectorsStream.writeSamples(_localOutputMixBuffer, samples); } else { // write to local injectors' ring buffer samples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; - _localInjectorsBuffer.writeSamples(_localScratchBuffer, + _localInjectorsStream.writeSamples(_localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); } @@ -1440,8 +1438,8 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice delete[] _outputScratchBuffer; _outputScratchBuffer = NULL; - delete[] _localOutputScratchBuffer; - _localOutputScratchBuffer = NULL; + delete[] _localOutputMixBuffer; + _localOutputMixBuffer = NULL; } if (_networkToOutputResampler) { @@ -1501,8 +1499,8 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice _outputPeriod = periodSampleSize * 2; _outputMixBuffer = new float[_outputPeriod]; _outputScratchBuffer = new int16_t[_outputPeriod]; - _localOutputScratchBuffer = new int16_t[_outputPeriod]; - _localInjectorsBuffer.resizeForFrameSize(_outputPeriod * 2); + _localOutputMixBuffer = new float[_outputPeriod]; + _localInjectorsStream.resizeForFrameSize(_outputPeriod * 2); qCDebug(audioclient) << "Output Buffer capacity in frames: " << _audioOutput->bufferSize() / AudioConstants::SAMPLE_SIZE / (float)deviceFrameSize << "requested bytes:" << requestedSize << "actual bytes:" << _audioOutput->bufferSize() << @@ -1630,22 +1628,16 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { samplesRequested = networkSamplesPopped; } - int injectorSamplesPopped; + int injectorSamplesPopped = 0; { Lock lock(_audio->_localAudioMutex); - if ((injectorSamplesPopped = _localInjectorsBuffer.readSamples(scratchBuffer, samplesRequested)) > 0) { - qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsBuffer.samplesAvailable(), samplesRequested); - - if (_audio->_shouldEchoToServer) { - // omit local audio, it should be echoed - injectorSamplesPopped = 0; - } else if (networkSamplesPopped == 0) { - convertToMix(mixBuffer, scratchBuffer, injectorSamplesPopped); - } else { - // add to mix buffer (float) - for (int i = 0; i < injectorSamplesPopped; i++) { - mixBuffer[i] += (float)scratchBuffer[i] * (1 / 32768.0f); - } + if (_audio->_shouldEchoToServer) { + // omit local audio, it should be echoed + _localInjectorsStream.skipSamples(samplesRequested); + } else { + bool append = networkSamplesPopped > 0; + if ((injectorSamplesPopped = _localInjectorsStream.readSamples(mixBuffer, samplesRequested, append)) > 0) { + qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsStream.samplesAvailable(), samplesRequested); } } } diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 78fd97abb2..8befd86f26 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -69,6 +69,17 @@ class QIODevice; class Transform; class NLPacket; +class LocalInjectorsStream { +public: + LocalInjectorsStream(int numFrameSamples); + int getSampleCapacity() { return 0; }; + int samplesAvailable() { return 0; } + int writeSamples(const float*, int numSamples) { return 0; } + void resizeForFrameSize(int numFrameSamples) {} + int skipSamples(int numSamples) { return 0; } + int readSamples(float* mixBuffer, int numSamples, bool append) { return 0; } +}; + class AudioInjectorsThread : public QThread { Q_OBJECT @@ -97,9 +108,9 @@ public: class AudioOutputIODevice : public QIODevice { public: - AudioOutputIODevice(AudioRingBuffer& localInjectorsBuffer, MixedProcessedAudioStream& receivedAudioStream, + AudioOutputIODevice(LocalInjectorsStream& localInjectorsStream, MixedProcessedAudioStream& receivedAudioStream, AudioClient* audio) : - _localInjectorsBuffer(localInjectorsBuffer), _receivedAudioStream(receivedAudioStream), + _localInjectorsStream(localInjectorsStream), _receivedAudioStream(receivedAudioStream), _audio(audio), _unfulfilledReads(0) {} void start() { open(QIODevice::ReadOnly | QIODevice::Unbuffered); } @@ -108,7 +119,7 @@ public: qint64 writeData(const char * data, qint64 maxSize) override { return 0; } int getRecentUnfulfilledReads() { int unfulfilledReads = _unfulfilledReads; _unfulfilledReads = 0; return unfulfilledReads; } private: - AudioRingBuffer& _localInjectorsBuffer; + LocalInjectorsStream& _localInjectorsStream; MixedProcessedAudioStream& _receivedAudioStream; AudioClient* _audio; int _unfulfilledReads; @@ -277,7 +288,7 @@ private: QAudioOutput* _loopbackAudioOutput; QIODevice* _loopbackOutputDevice; AudioRingBuffer _inputRingBuffer; - AudioRingBuffer _localInjectorsBuffer; + LocalInjectorsStream _localInjectorsStream; MixedProcessedAudioStream _receivedAudioStream; bool _isStereoInput; @@ -322,7 +333,7 @@ private: // for local audio (used by audio injectors thread) float _localMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; - int16_t* _localOutputScratchBuffer { NULL }; + float* _localOutputMixBuffer { NULL }; AudioInjectorsThread _localAudioThread; Mutex _localAudioMutex; From 95a7b38ea44241ea27cf8b7633511ceb36a995aa Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 19 Jan 2017 15:46:24 -0500 Subject: [PATCH 18/45] templatize AudioRingBuffer --- libraries/audio/src/AudioRingBuffer.cpp | 120 ++++++++++++------------ libraries/audio/src/AudioRingBuffer.h | 115 ++++++++++++----------- 2 files changed, 121 insertions(+), 114 deletions(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 260c682cde..c52d8f1447 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -26,15 +26,15 @@ static const QString RING_BUFFER_OVERFLOW_DEBUG { "AudioRingBuffer::writeData has overflown the buffer. Overwriting old data." }; static const QString DROPPED_SILENT_DEBUG { "AudioRingBuffer::addSilentSamples dropping silent samples to prevent overflow." }; -AudioRingBuffer::AudioRingBuffer(int numFrameSamples, int numFramesCapacity) : +AudioRingBufferTemplate::AudioRingBufferTemplate(int numFrameSamples, int numFramesCapacity) : _numFrameSamples(numFrameSamples), _frameCapacity(numFramesCapacity), _sampleCapacity(numFrameSamples * numFramesCapacity), _bufferLength(numFrameSamples * (numFramesCapacity + 1)) { if (numFrameSamples) { - _buffer = new int16_t[_bufferLength]; - memset(_buffer, 0, _bufferLength * sizeof(int16_t)); + _buffer = new Sample[_bufferLength]; + memset(_buffer, 0, _bufferLength * SampleSize); _nextOutput = _buffer; _endOfLastWrite = _buffer; } @@ -43,29 +43,29 @@ AudioRingBuffer::AudioRingBuffer(int numFrameSamples, int numFramesCapacity) : static QString repeatedDroppedMessage = LogHandler::getInstance().addRepeatedMessageRegex(DROPPED_SILENT_DEBUG); }; -AudioRingBuffer::~AudioRingBuffer() { +AudioRingBufferTemplate::~AudioRingBufferTemplate() { delete[] _buffer; } -void AudioRingBuffer::clear() { +void AudioRingBufferTemplate::clear() { _endOfLastWrite = _buffer; _nextOutput = _buffer; } -void AudioRingBuffer::reset() { +void AudioRingBufferTemplate::reset() { clear(); _overflowCount = 0; } -void AudioRingBuffer::resizeForFrameSize(int numFrameSamples) { +void AudioRingBufferTemplate::resizeForFrameSize(int numFrameSamples) { delete[] _buffer; _numFrameSamples = numFrameSamples; _sampleCapacity = numFrameSamples * _frameCapacity; _bufferLength = numFrameSamples * (_frameCapacity + 1); if (numFrameSamples) { - _buffer = new int16_t[_bufferLength]; - memset(_buffer, 0, _bufferLength * sizeof(int16_t)); + _buffer = new Sample[_bufferLength]; + memset(_buffer, 0, _bufferLength * SampleSize); } else { _buffer = nullptr; } @@ -73,17 +73,17 @@ void AudioRingBuffer::resizeForFrameSize(int numFrameSamples) { reset(); } -int AudioRingBuffer::readSamples(int16_t* destination, int maxSamples) { - return readData((char*)destination, maxSamples * sizeof(int16_t)) / sizeof(int16_t); +int AudioRingBufferTemplate::readSamples(Sample* destination, int maxSamples) { + return readData((char*)destination, maxSamples * SampleSize) / SampleSize; } -int AudioRingBuffer::writeSamples(const int16_t* source, int maxSamples) { - return writeData((char*)source, maxSamples * sizeof(int16_t)) / sizeof(int16_t); +int AudioRingBufferTemplate::writeSamples(const Sample* source, int maxSamples) { + return writeData((char*)source, maxSamples * SampleSize) / SampleSize; } -int AudioRingBuffer::readData(char *data, int maxSize) { +int AudioRingBufferTemplate::readData(char *data, int maxSize) { // only copy up to the number of samples we have available - int maxSamples = maxSize / sizeof(int16_t); + int maxSamples = maxSize / SampleSize; int numReadSamples = std::min(maxSamples, samplesAvailable()); if (_nextOutput + numReadSamples > _buffer + _bufferLength) { @@ -91,22 +91,22 @@ int AudioRingBuffer::readData(char *data, int maxSize) { int numSamplesToEnd = (_buffer + _bufferLength) - _nextOutput; // read to the end of the buffer - memcpy(data, _nextOutput, numSamplesToEnd * sizeof(int16_t)); + memcpy(data, _nextOutput, numSamplesToEnd * SampleSize); // read the rest from the beginning of the buffer - memcpy(data + (numSamplesToEnd * sizeof(int16_t)), _buffer, (numReadSamples - numSamplesToEnd) * sizeof(int16_t)); + memcpy(data + (numSamplesToEnd * SampleSize), _buffer, (numReadSamples - numSamplesToEnd) * SampleSize); } else { - memcpy(data, _nextOutput, numReadSamples * sizeof(int16_t)); + memcpy(data, _nextOutput, numReadSamples * SampleSize); } shiftReadPosition(numReadSamples); - return numReadSamples * sizeof(int16_t); + return numReadSamples * SampleSize; } -int AudioRingBuffer::writeData(const char* data, int maxSize) { +int AudioRingBufferTemplate::writeData(const char* data, int maxSize) { // only copy up to the number of samples we have capacity for - int maxSamples = maxSize / sizeof(int16_t); + int maxSamples = maxSize / SampleSize; int numWriteSamples = std::min(maxSamples, _sampleCapacity); int samplesRoomFor = _sampleCapacity - samplesAvailable(); @@ -124,20 +124,20 @@ int AudioRingBuffer::writeData(const char* data, int maxSize) { int numSamplesToEnd = (_buffer + _bufferLength) - _endOfLastWrite; // write to the end of the buffer - memcpy(_endOfLastWrite, data, numSamplesToEnd * sizeof(int16_t)); + memcpy(_endOfLastWrite, data, numSamplesToEnd * SampleSize); // write the rest to the beginning of the buffer - memcpy(_buffer, data + (numSamplesToEnd * sizeof(int16_t)), (numWriteSamples - numSamplesToEnd) * sizeof(int16_t)); + memcpy(_buffer, data + (numSamplesToEnd * SampleSize), (numWriteSamples - numSamplesToEnd) * SampleSize); } else { - memcpy(_endOfLastWrite, data, numWriteSamples * sizeof(int16_t)); + memcpy(_endOfLastWrite, data, numWriteSamples * SampleSize); } _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, numWriteSamples); - return numWriteSamples * sizeof(int16_t); + return numWriteSamples * SampleSize; } -int AudioRingBuffer::samplesAvailable() const { +int AudioRingBufferTemplate::samplesAvailable() const { if (!_endOfLastWrite) { return 0; } @@ -149,31 +149,7 @@ int AudioRingBuffer::samplesAvailable() const { return sampleDifference; } -int AudioRingBuffer::addSilentSamples(int silentSamples) { - // NOTE: This implementation is nearly identical to writeData save for s/memcpy/memset, refer to comments there - int numWriteSamples = std::min(silentSamples, _sampleCapacity); - int samplesRoomFor = _sampleCapacity - samplesAvailable(); - - if (numWriteSamples > samplesRoomFor) { - numWriteSamples = samplesRoomFor; - - qCDebug(audio) << qPrintable(DROPPED_SILENT_DEBUG); - } - - if (_endOfLastWrite + numWriteSamples > _buffer + _bufferLength) { - int numSamplesToEnd = (_buffer + _bufferLength) - _endOfLastWrite; - memset(_endOfLastWrite, 0, numSamplesToEnd * sizeof(int16_t)); - memset(_buffer, 0, (numWriteSamples - numSamplesToEnd) * sizeof(int16_t)); - } else { - memset(_endOfLastWrite, 0, numWriteSamples * sizeof(int16_t)); - } - - _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, numWriteSamples); - - return numWriteSamples; -} - -int16_t* AudioRingBuffer::shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const { +int16_t* AudioRingBufferTemplate::shiftedPositionAccomodatingWrap(Sample* position, int numSamplesShift) const { // NOTE: It is possible to shift out-of-bounds if (|numSamplesShift| > 2 * _bufferLength), but this should not occur if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + _bufferLength) { // this shift will wrap the position around to the beginning of the ring @@ -186,11 +162,35 @@ int16_t* AudioRingBuffer::shiftedPositionAccomodatingWrap(int16_t* position, int } } -float AudioRingBuffer::getFrameLoudness(const int16_t* frameStart) const { +int AudioRingBufferTemplate::addSilentSamples(int silentSamples) { + // NOTE: This implementation is nearly identical to writeData save for s/memcpy/memset, refer to comments there + int numWriteSamples = std::min(silentSamples, _sampleCapacity); + int samplesRoomFor = _sampleCapacity - samplesAvailable(); + + if (numWriteSamples > samplesRoomFor) { + numWriteSamples = samplesRoomFor; + + qCDebug(audio) << qPrintable(DROPPED_SILENT_DEBUG); + } + + if (_endOfLastWrite + numWriteSamples > _buffer + _bufferLength) { + int numSamplesToEnd = (_buffer + _bufferLength) - _endOfLastWrite; + memset(_endOfLastWrite, 0, numSamplesToEnd * SampleSize); + memset(_buffer, 0, (numWriteSamples - numSamplesToEnd) * SampleSize); + } else { + memset(_endOfLastWrite, 0, numWriteSamples * SampleSize); + } + + _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, numWriteSamples); + + return numWriteSamples; +} + +float AudioRingBufferTemplate::getFrameLoudness(const Sample* frameStart) const { // FIXME: This is a bad measure of loudness - normal estimation uses sqrt(sum(x*x)) float loudness = 0.0f; - const int16_t* sampleAt = frameStart; - const int16_t* bufferLastAt = _buffer + _bufferLength - 1; + const Sample* sampleAt = frameStart; + const Sample* bufferLastAt = _buffer + _bufferLength - 1; for (int i = 0; i < _numFrameSamples; ++i) { loudness += (float) std::abs(*sampleAt); @@ -203,14 +203,14 @@ float AudioRingBuffer::getFrameLoudness(const int16_t* frameStart) const { return loudness; } -float AudioRingBuffer::getFrameLoudness(ConstIterator frameStart) const { +float AudioRingBufferTemplate::getFrameLoudness(ConstIterator frameStart) const { if (frameStart.isNull()) { return 0.0f; } return getFrameLoudness(&(*frameStart)); } -int AudioRingBuffer::writeSamples(ConstIterator source, int maxSamples) { +int AudioRingBufferTemplate::writeSamples(ConstIterator source, int maxSamples) { int samplesToCopy = std::min(maxSamples, _sampleCapacity); int samplesRoomFor = _sampleCapacity - samplesAvailable(); if (samplesToCopy > samplesRoomFor) { @@ -221,7 +221,7 @@ int AudioRingBuffer::writeSamples(ConstIterator source, int maxSamples) { qCDebug(audio) << qPrintable(RING_BUFFER_OVERFLOW_DEBUG); } - int16_t* bufferLast = _buffer + _bufferLength - 1; + Sample* bufferLast = _buffer + _bufferLength - 1; for (int i = 0; i < samplesToCopy; i++) { *_endOfLastWrite = *source; _endOfLastWrite = (_endOfLastWrite == bufferLast) ? _buffer : _endOfLastWrite + 1; @@ -231,7 +231,7 @@ int AudioRingBuffer::writeSamples(ConstIterator source, int maxSamples) { return samplesToCopy; } -int AudioRingBuffer::writeSamplesWithFade(ConstIterator source, int maxSamples, float fade) { +int AudioRingBufferTemplate::writeSamplesWithFade(ConstIterator source, int maxSamples, float fade) { int samplesToCopy = std::min(maxSamples, _sampleCapacity); int samplesRoomFor = _sampleCapacity - samplesAvailable(); if (samplesToCopy > samplesRoomFor) { @@ -242,9 +242,9 @@ int AudioRingBuffer::writeSamplesWithFade(ConstIterator source, int maxSamples, qCDebug(audio) << qPrintable(RING_BUFFER_OVERFLOW_DEBUG); } - int16_t* bufferLast = _buffer + _bufferLength - 1; + Sample* bufferLast = _buffer + _bufferLength - 1; for (int i = 0; i < samplesToCopy; i++) { - *_endOfLastWrite = (int16_t)((float)(*source) * fade); + *_endOfLastWrite = (Sample)((float)(*source) * fade); _endOfLastWrite = (_endOfLastWrite == bufferLast) ? _buffer : _endOfLastWrite + 1; ++source; } diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 29e7a9e998..b1f65b8a8d 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -21,15 +21,19 @@ const int DEFAULT_RING_BUFFER_FRAME_CAPACITY = 10; -class AudioRingBuffer { +template +class AudioRingBufferTemplate { + using Sample = T; + static size_t SampleSize = sizeof(Sample); + public: - AudioRingBuffer(int numFrameSamples, int numFramesCapacity = DEFAULT_RING_BUFFER_FRAME_CAPACITY); - ~AudioRingBuffer(); + AudioRingBufferTemplate(int numFrameSamples, int numFramesCapacity = DEFAULT_RING_BUFFER_FRAME_CAPACITY); + ~AudioRingBufferTemplate(); // disallow copying - AudioRingBuffer(const AudioRingBuffer&) = delete; - AudioRingBuffer(AudioRingBuffer&&) = delete; - AudioRingBuffer& operator=(const AudioRingBuffer&) = delete; + AudioRingBufferTemplate(const AudioRingBufferTemplate&) = delete; + AudioRingBufferTemplate(AudioRingBufferTemplate&&) = delete; + AudioRingBufferTemplate& operator=(const AudioRingBufferTemplate&) = delete; /// Invalidate any data in the buffer void clear(); @@ -43,11 +47,11 @@ public: /// Read up to maxSamples into destination (will only read up to samplesAvailable()) /// Returns number of read samples - int readSamples(int16_t* destination, int maxSamples); + int readSamples(Sample* destination, int maxSamples); /// Write up to maxSamples from source (will only write up to sample capacity) /// Returns number of written samples - int writeSamples(const int16_t* source, int maxSamples); + int writeSamples(const Sample* source, int maxSamples); /// Write up to maxSamples silent samples (will only write until other data exists in the buffer) /// This method will not overwrite existing data in the buffer, instead dropping silent samples that would overflow @@ -63,8 +67,8 @@ public: int writeData(const char* source, int maxSize); /// Returns a reference to the index-th sample offset from the current read sample - int16_t& operator[](const int index) { return *shiftedPositionAccomodatingWrap(_nextOutput, index); } - const int16_t& operator[] (const int index) const { return *shiftedPositionAccomodatingWrap(_nextOutput, index); } + Sample& operator[](const int index) { return *shiftedPositionAccomodatingWrap(_nextOutput, index); } + const Sample& operator[] (const int index) const { return *shiftedPositionAccomodatingWrap(_nextOutput, index); } /// Essentially discards the next numSamples from the ring buffer /// NOTE: This is not checked - it is possible to shift past written data @@ -85,36 +89,36 @@ public: class ConstIterator { public: ConstIterator(); - ConstIterator(int16_t* bufferFirst, int capacity, int16_t* at); + ConstIterator(Sample* bufferFirst, int capacity, Sample* at); ConstIterator(const ConstIterator& rhs) = default; bool isNull() const { return _at == NULL; } bool operator==(const ConstIterator& rhs) { return _at == rhs._at; } bool operator!=(const ConstIterator& rhs) { return _at != rhs._at; } - const int16_t& operator*() { return *_at; } + const Sample& operator*() { return *_at; } ConstIterator& operator=(const ConstIterator& rhs); ConstIterator& operator++(); ConstIterator operator++(int); ConstIterator& operator--(); ConstIterator operator--(int); - const int16_t& operator[] (int i); + const Sample& operator[] (int i); ConstIterator operator+(int i); ConstIterator operator-(int i); - void readSamples(int16_t* dest, int numSamples); - void readSamplesWithFade(int16_t* dest, int numSamples, float fade); - void readSamplesWithUpmix(int16_t* dest, int numSamples, int numExtraChannels); - void readSamplesWithDownmix(int16_t* dest, int numSamples); + void readSamples(Sample* dest, int numSamples); + void readSamplesWithFade(Sample* dest, int numSamples, float fade); + void readSamplesWithUpmix(Sample* dest, int numSamples, int numExtraChannels); + void readSamplesWithDownmix(Sample* dest, int numSamples); private: - int16_t* atShiftedBy(int i); + Sample* atShiftedBy(int i); int _bufferLength; - int16_t* _bufferFirst; - int16_t* _bufferLast; - int16_t* _at; + Sample* _bufferFirst; + Sample* _bufferLast; + Sample* _at; }; ConstIterator nextOutput() const; @@ -126,8 +130,8 @@ public: float getFrameLoudness(ConstIterator frameStart) const; protected: - int16_t* shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const; - float getFrameLoudness(const int16_t* frameStart) const; + Sample* shiftedPositionAccomodatingWrap(Sample* position, int numSamplesShift) const; + float getFrameLoudness(const Sample* frameStart) const; int _numFrameSamples; int _frameCapacity; @@ -135,25 +139,28 @@ protected: int _bufferLength; // actual _buffer length (_sampleCapacity + 1) int _overflowCount{ 0 }; // times the ring buffer has overwritten data - int16_t* _nextOutput{ nullptr }; - int16_t* _endOfLastWrite{ nullptr }; - int16_t* _buffer{ nullptr }; + Sample* _nextOutput{ nullptr }; + Sample* _endOfLastWrite{ nullptr }; + Sample* _buffer{ nullptr }; }; +using AudioRingBuffer = AudioRingBufferTemplate; +using AudioRingMixBuffer = AudioRingBufferTemplate; + // inline the iterator: -inline AudioRingBuffer::ConstIterator::ConstIterator() : +inline AudioRingBufferTemplate::ConstIterator::ConstIterator() : _bufferLength(0), _bufferFirst(NULL), _bufferLast(NULL), _at(NULL) {} -inline AudioRingBuffer::ConstIterator::ConstIterator(int16_t* bufferFirst, int capacity, int16_t* at) : +inline AudioRingBufferTemplate::ConstIterator::ConstIterator(Sample* bufferFirst, int capacity, Sample* at) : _bufferLength(capacity), _bufferFirst(bufferFirst), _bufferLast(bufferFirst + capacity - 1), _at(at) {} -inline AudioRingBuffer::ConstIterator& AudioRingBuffer::ConstIterator::operator=(const ConstIterator& rhs) { +inline AudioRingBufferTemplate::ConstIterator& AudioRingBufferTemplate::ConstIterator::operator=(const ConstIterator& rhs) { _bufferLength = rhs._bufferLength; _bufferFirst = rhs._bufferFirst; _bufferLast = rhs._bufferLast; @@ -161,41 +168,41 @@ inline AudioRingBuffer::ConstIterator& AudioRingBuffer::ConstIterator::operator= return *this; } -inline AudioRingBuffer::ConstIterator& AudioRingBuffer::ConstIterator::operator++() { +inline AudioRingBufferTemplate::ConstIterator& AudioRingBufferTemplate::ConstIterator::operator++() { _at = (_at == _bufferLast) ? _bufferFirst : _at + 1; return *this; } -inline AudioRingBuffer::ConstIterator AudioRingBuffer::ConstIterator::operator++(int) { +inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::ConstIterator::operator++(int) { ConstIterator tmp(*this); ++(*this); return tmp; } -inline AudioRingBuffer::ConstIterator& AudioRingBuffer::ConstIterator::operator--() { +inline AudioRingBufferTemplate::ConstIterator& AudioRingBufferTemplate::ConstIterator::operator--() { _at = (_at == _bufferFirst) ? _bufferLast : _at - 1; return *this; } -inline AudioRingBuffer::ConstIterator AudioRingBuffer::ConstIterator::operator--(int) { +inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::ConstIterator::operator--(int) { ConstIterator tmp(*this); --(*this); return tmp; } -inline const int16_t& AudioRingBuffer::ConstIterator::operator[] (int i) { +inline const int16_t& AudioRingBufferTemplate::ConstIterator::operator[] (int i) { return *atShiftedBy(i); } -inline AudioRingBuffer::ConstIterator AudioRingBuffer::ConstIterator::operator+(int i) { +inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::ConstIterator::operator+(int i) { return ConstIterator(_bufferFirst, _bufferLength, atShiftedBy(i)); } -inline AudioRingBuffer::ConstIterator AudioRingBuffer::ConstIterator::operator-(int i) { +inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::ConstIterator::operator-(int i) { return ConstIterator(_bufferFirst, _bufferLength, atShiftedBy(-i)); } -inline int16_t* AudioRingBuffer::ConstIterator::atShiftedBy(int i) { +inline int16_t* AudioRingBufferTemplate::ConstIterator::atShiftedBy(int i) { i = (_at - _bufferFirst + i) % _bufferLength; if (i < 0) { i += _bufferLength; @@ -203,23 +210,23 @@ inline int16_t* AudioRingBuffer::ConstIterator::atShiftedBy(int i) { return _bufferFirst + i; } -inline void AudioRingBuffer::ConstIterator::readSamples(int16_t* dest, int numSamples) { +inline void AudioRingBufferTemplate::ConstIterator::readSamples(Sample* dest, int numSamples) { auto samplesToEnd = _bufferLast - _at + 1; if (samplesToEnd >= numSamples) { - memcpy(dest, _at, numSamples * sizeof(int16_t)); + memcpy(dest, _at, numSamples * SampleSize); _at += numSamples; } else { auto samplesFromStart = numSamples - samplesToEnd; - memcpy(dest, _at, samplesToEnd * sizeof(int16_t)); - memcpy(dest + samplesToEnd, _bufferFirst, samplesFromStart * sizeof(int16_t)); + memcpy(dest, _at, samplesToEnd * SampleSize); + memcpy(dest + samplesToEnd, _bufferFirst, samplesFromStart * SampleSize); _at = _bufferFirst + samplesFromStart; } } -inline void AudioRingBuffer::ConstIterator::readSamplesWithFade(int16_t* dest, int numSamples, float fade) { - int16_t* at = _at; +inline void AudioRingBufferTemplate::ConstIterator::readSamplesWithFade(Sample* dest, int numSamples, float fade) { + Sample* at = _at; for (int i = 0; i < numSamples; i++) { *dest = (float)*at * fade; ++dest; @@ -227,14 +234,14 @@ inline void AudioRingBuffer::ConstIterator::readSamplesWithFade(int16_t* dest, i } } -inline void AudioRingBuffer::ConstIterator::readSamplesWithUpmix(int16_t* dest, int numSamples, int numExtraChannels) { - int16_t* at = _at; +inline void AudioRingBufferTemplate::ConstIterator::readSamplesWithUpmix(Sample* dest, int numSamples, int numExtraChannels) { + Sample* at = _at; for (int i = 0; i < numSamples/2; i++) { // read 2 samples - int16_t left = *at; + Sample left = *at; at = (at == _bufferLast) ? _bufferFirst : at + 1; - int16_t right = *at; + Sample right = *at; at = (at == _bufferLast) ? _bufferFirst : at + 1; // write 2 + N samples @@ -246,26 +253,26 @@ inline void AudioRingBuffer::ConstIterator::readSamplesWithUpmix(int16_t* dest, } } -inline void AudioRingBuffer::ConstIterator::readSamplesWithDownmix(int16_t* dest, int numSamples) { - int16_t* at = _at; +inline void AudioRingBufferTemplate::ConstIterator::readSamplesWithDownmix(Sample* dest, int numSamples) { + Sample* at = _at; for (int i = 0; i < numSamples/2; i++) { // read 2 samples - int16_t left = *at; + Sample left = *at; at = (at == _bufferLast) ? _bufferFirst : at + 1; - int16_t right = *at; + Sample right = *at; at = (at == _bufferLast) ? _bufferFirst : at + 1; // write 1 sample - *dest++ = (int16_t)((left + right) / 2); + *dest++ = (Sample)((left + right) / 2); } } -inline AudioRingBuffer::ConstIterator AudioRingBuffer::nextOutput() const { +inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::nextOutput() const { return ConstIterator(_buffer, _bufferLength, _nextOutput); } -inline AudioRingBuffer::ConstIterator AudioRingBuffer::lastFrameWritten() const { +inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::lastFrameWritten() const { return ConstIterator(_buffer, _bufferLength, _endOfLastWrite) - _numFrameSamples; } From 02e62938a46d10f2bb0ca752e6489b6b80c3f90c Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 19 Jan 2017 16:37:06 -0500 Subject: [PATCH 19/45] add AudioRingMixBuffer --- libraries/audio/src/AudioRingBuffer.cpp | 207 +++++++++++++++++++++++- libraries/audio/src/AudioRingBuffer.h | 48 +++--- 2 files changed, 236 insertions(+), 19 deletions(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index c52d8f1447..ab0948e328 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -26,6 +26,7 @@ static const QString RING_BUFFER_OVERFLOW_DEBUG { "AudioRingBuffer::writeData has overflown the buffer. Overwriting old data." }; static const QString DROPPED_SILENT_DEBUG { "AudioRingBuffer::addSilentSamples dropping silent samples to prevent overflow." }; +template<> AudioRingBufferTemplate::AudioRingBufferTemplate(int numFrameSamples, int numFramesCapacity) : _numFrameSamples(numFrameSamples), _frameCapacity(numFramesCapacity), @@ -41,22 +42,26 @@ AudioRingBufferTemplate::AudioRingBufferTemplate(int numFrameSamples, i static QString repeatedOverflowMessage = LogHandler::getInstance().addRepeatedMessageRegex(RING_BUFFER_OVERFLOW_DEBUG); static QString repeatedDroppedMessage = LogHandler::getInstance().addRepeatedMessageRegex(DROPPED_SILENT_DEBUG); -}; +} +template<> AudioRingBufferTemplate::~AudioRingBufferTemplate() { delete[] _buffer; } +template<> void AudioRingBufferTemplate::clear() { _endOfLastWrite = _buffer; _nextOutput = _buffer; } +template<> void AudioRingBufferTemplate::reset() { clear(); _overflowCount = 0; } +template<> void AudioRingBufferTemplate::resizeForFrameSize(int numFrameSamples) { delete[] _buffer; _numFrameSamples = numFrameSamples; @@ -73,14 +78,17 @@ void AudioRingBufferTemplate::resizeForFrameSize(int numFrameSamples) { reset(); } +template<> int AudioRingBufferTemplate::readSamples(Sample* destination, int maxSamples) { return readData((char*)destination, maxSamples * SampleSize) / SampleSize; } +template<> int AudioRingBufferTemplate::writeSamples(const Sample* source, int maxSamples) { return writeData((char*)source, maxSamples * SampleSize) / SampleSize; } +template<> int AudioRingBufferTemplate::readData(char *data, int maxSize) { // only copy up to the number of samples we have available int maxSamples = maxSize / SampleSize; @@ -104,6 +112,7 @@ int AudioRingBufferTemplate::readData(char *data, int maxSize) { return numReadSamples * SampleSize; } +template<> int AudioRingBufferTemplate::writeData(const char* data, int maxSize) { // only copy up to the number of samples we have capacity for int maxSamples = maxSize / SampleSize; @@ -137,6 +146,7 @@ int AudioRingBufferTemplate::writeData(const char* data, int maxSize) { return numWriteSamples * SampleSize; } +template<> int AudioRingBufferTemplate::samplesAvailable() const { if (!_endOfLastWrite) { return 0; @@ -149,6 +159,7 @@ int AudioRingBufferTemplate::samplesAvailable() const { return sampleDifference; } +template<> int16_t* AudioRingBufferTemplate::shiftedPositionAccomodatingWrap(Sample* position, int numSamplesShift) const { // NOTE: It is possible to shift out-of-bounds if (|numSamplesShift| > 2 * _bufferLength), but this should not occur if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + _bufferLength) { @@ -162,6 +173,7 @@ int16_t* AudioRingBufferTemplate::shiftedPositionAccomodatingWrap(Sampl } } +template<> int AudioRingBufferTemplate::addSilentSamples(int silentSamples) { // NOTE: This implementation is nearly identical to writeData save for s/memcpy/memset, refer to comments there int numWriteSamples = std::min(silentSamples, _sampleCapacity); @@ -186,6 +198,7 @@ int AudioRingBufferTemplate::addSilentSamples(int silentSamples) { return numWriteSamples; } +template<> float AudioRingBufferTemplate::getFrameLoudness(const Sample* frameStart) const { // FIXME: This is a bad measure of loudness - normal estimation uses sqrt(sum(x*x)) float loudness = 0.0f; @@ -203,6 +216,7 @@ float AudioRingBufferTemplate::getFrameLoudness(const Sample* frameStar return loudness; } +template<> float AudioRingBufferTemplate::getFrameLoudness(ConstIterator frameStart) const { if (frameStart.isNull()) { return 0.0f; @@ -210,6 +224,7 @@ float AudioRingBufferTemplate::getFrameLoudness(ConstIterator frameStar return getFrameLoudness(&(*frameStart)); } +template<> int AudioRingBufferTemplate::writeSamples(ConstIterator source, int maxSamples) { int samplesToCopy = std::min(maxSamples, _sampleCapacity); int samplesRoomFor = _sampleCapacity - samplesAvailable(); @@ -231,6 +246,7 @@ int AudioRingBufferTemplate::writeSamples(ConstIterator source, int max return samplesToCopy; } +template<> int AudioRingBufferTemplate::writeSamplesWithFade(ConstIterator source, int maxSamples, float fade) { int samplesToCopy = std::min(maxSamples, _sampleCapacity); int samplesRoomFor = _sampleCapacity - samplesAvailable(); @@ -251,3 +267,192 @@ int AudioRingBufferTemplate::writeSamplesWithFade(ConstIterator source, return samplesToCopy; } + +template<> +AudioRingBufferTemplate::AudioRingBufferTemplate(int numFrameSamples, int numFramesCapacity) : + _numFrameSamples(numFrameSamples), + _frameCapacity(numFramesCapacity), + _sampleCapacity(numFrameSamples * numFramesCapacity), + _bufferLength(numFrameSamples * (numFramesCapacity + 1)) +{ + if (numFrameSamples) { + _buffer = new Sample[_bufferLength]; + memset(_buffer, 0, _bufferLength * SampleSize); + _nextOutput = _buffer; + _endOfLastWrite = _buffer; + } + + static QString repeatedOverflowMessage = LogHandler::getInstance().addRepeatedMessageRegex(RING_BUFFER_OVERFLOW_DEBUG); + static QString repeatedDroppedMessage = LogHandler::getInstance().addRepeatedMessageRegex(DROPPED_SILENT_DEBUG); +} + +template<> +AudioRingBufferTemplate::~AudioRingBufferTemplate() { + delete[] _buffer; +} + +template<> +void AudioRingBufferTemplate::clear() { + _endOfLastWrite = _buffer; + _nextOutput = _buffer; +} + +template<> +void AudioRingBufferTemplate::reset() { + clear(); + _overflowCount = 0; +} + +template<> +void AudioRingBufferTemplate::resizeForFrameSize(int numFrameSamples) { + delete[] _buffer; + _numFrameSamples = numFrameSamples; + _sampleCapacity = numFrameSamples * _frameCapacity; + _bufferLength = numFrameSamples * (_frameCapacity + 1); + + if (numFrameSamples) { + _buffer = new Sample[_bufferLength]; + memset(_buffer, 0, _bufferLength * SampleSize); + } else { + _buffer = nullptr; + } + + reset(); +} + +template<> +int AudioRingBufferTemplate::readSamples(Sample* destination, int maxSamples) { + return readData((char*)destination, maxSamples * SampleSize) / SampleSize; +} + +template<> +int AudioRingBufferTemplate::appendSamples(Sample* destination, int maxSamples, bool append) { + if (append) { + return appendData((char*)destination, maxSamples * SampleSize) / SampleSize; + } else { + return readData((char*)destination, maxSamples * SampleSize) / SampleSize; + } +} + +template<> +int AudioRingBufferTemplate::writeSamples(const Sample* source, int maxSamples) { + return writeData((char*)source, maxSamples * SampleSize) / SampleSize; +} + +template<> +int AudioRingBufferTemplate::readData(char *data, int maxSize) { + // only copy up to the number of samples we have available + int maxSamples = maxSize / SampleSize; + int numReadSamples = std::min(maxSamples, samplesAvailable()); + + if (_nextOutput + numReadSamples > _buffer + _bufferLength) { + // we're going to need to do two reads to get this data, it wraps around the edge + int numSamplesToEnd = (_buffer + _bufferLength) - _nextOutput; + + // read to the end of the buffer + memcpy(data, _nextOutput, numSamplesToEnd * SampleSize); + + // read the rest from the beginning of the buffer + memcpy(data + (numSamplesToEnd * SampleSize), _buffer, (numReadSamples - numSamplesToEnd) * SampleSize); + } else { + memcpy(data, _nextOutput, numReadSamples * SampleSize); + } + + shiftReadPosition(numReadSamples); + + return numReadSamples * SampleSize; +} + +template<> +int AudioRingBufferTemplate::appendData(char *data, int maxSize) { + // only copy up to the number of samples we have available + int maxSamples = maxSize / SampleSize; + int numReadSamples = std::min(maxSamples, samplesAvailable()); + + Sample* dest = reinterpret_cast(data); + Sample* output = _nextOutput; + if (_nextOutput + numReadSamples > _buffer + _bufferLength) { + // we're going to need to do two reads to get this data, it wraps around the edge + int numSamplesToEnd = (_buffer + _bufferLength) - _nextOutput; + + // read to the end of the buffer + for (int i = 0; i < numSamplesToEnd; i++) { + *dest++ = *output++; + } + + // read the rest from the beginning of the buffer + output = _buffer; + for (int i = 0; i < (numReadSamples - numSamplesToEnd); i++) { + *dest++ = *output++; + } + } else { + for (int i = 0; i < numReadSamples; i++) { + *dest++ = *output++; + } + } + + shiftReadPosition(numReadSamples); + + return numReadSamples * SampleSize; +} + +template<> +int AudioRingBufferTemplate::writeData(const char* data, int maxSize) { + // only copy up to the number of samples we have capacity for + int maxSamples = maxSize / SampleSize; + int numWriteSamples = std::min(maxSamples, _sampleCapacity); + int samplesRoomFor = _sampleCapacity - samplesAvailable(); + + if (numWriteSamples > samplesRoomFor) { + // there's not enough room for this write. erase old data to make room for this new data + int samplesToDelete = numWriteSamples - samplesRoomFor; + _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, samplesToDelete); + _overflowCount++; + + qCDebug(audio) << qPrintable(RING_BUFFER_OVERFLOW_DEBUG); + } + + if (_endOfLastWrite + numWriteSamples > _buffer + _bufferLength) { + // we're going to need to do two writes to set this data, it wraps around the edge + int numSamplesToEnd = (_buffer + _bufferLength) - _endOfLastWrite; + + // write to the end of the buffer + memcpy(_endOfLastWrite, data, numSamplesToEnd * SampleSize); + + // write the rest to the beginning of the buffer + memcpy(_buffer, data + (numSamplesToEnd * SampleSize), (numWriteSamples - numSamplesToEnd) * SampleSize); + } else { + memcpy(_endOfLastWrite, data, numWriteSamples * SampleSize); + } + + _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, numWriteSamples); + + return numWriteSamples * SampleSize; +} + +template<> +int AudioRingBufferTemplate::samplesAvailable() const { + if (!_endOfLastWrite) { + return 0; + } + + int sampleDifference = _endOfLastWrite - _nextOutput; + if (sampleDifference < 0) { + sampleDifference += _bufferLength; + } + return sampleDifference; +} + +template<> +float* AudioRingBufferTemplate::shiftedPositionAccomodatingWrap(Sample* position, int numSamplesShift) const { + // NOTE: It is possible to shift out-of-bounds if (|numSamplesShift| > 2 * _bufferLength), but this should not occur + if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + _bufferLength) { + // this shift will wrap the position around to the beginning of the ring + return position + numSamplesShift - _bufferLength; + } else if (numSamplesShift < 0 && position + numSamplesShift < _buffer) { + // this shift will go around to the end of the ring + return position + numSamplesShift + _bufferLength; + } else { + return position + numSamplesShift; + } +} \ No newline at end of file diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index b1f65b8a8d..92c6dcc336 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -24,7 +24,7 @@ const int DEFAULT_RING_BUFFER_FRAME_CAPACITY = 10; template class AudioRingBufferTemplate { using Sample = T; - static size_t SampleSize = sizeof(Sample); + static const int SampleSize = sizeof(Sample); public: AudioRingBufferTemplate(int numFrameSamples, int numFramesCapacity = DEFAULT_RING_BUFFER_FRAME_CAPACITY); @@ -49,6 +49,14 @@ public: /// Returns number of read samples int readSamples(Sample* destination, int maxSamples); + /// Append up to maxSamples into destination (will only read up to samplesAvailable()) + /// If append == false, behaves as readSamples + /// Returns number of appended samples + int appendSamples(Sample* destination, int maxSamples, bool append = true); + + /// Skip up to maxSamples (will only skip up to samplesAvailable()) + void skipSamples(int maxSamples) { shiftReadPosition(std::min(maxSamples, samplesAvailable())); } + /// Write up to maxSamples from source (will only write up to sample capacity) /// Returns number of written samples int writeSamples(const Sample* source, int maxSamples); @@ -62,6 +70,10 @@ public: /// Returns number of read bytes int readData(char* destination, int maxSize); + /// Append up to maxSize into destination + /// Returns number of read bytes + int appendData(char* destination, int maxSize); + /// Write up to maxSize from source /// Returns number of written bytes int writeData(const char* source, int maxSize); @@ -148,19 +160,19 @@ using AudioRingBuffer = AudioRingBufferTemplate; using AudioRingMixBuffer = AudioRingBufferTemplate; // inline the iterator: -inline AudioRingBufferTemplate::ConstIterator::ConstIterator() : +template<> inline AudioRingBufferTemplate::ConstIterator::ConstIterator() : _bufferLength(0), _bufferFirst(NULL), _bufferLast(NULL), _at(NULL) {} -inline AudioRingBufferTemplate::ConstIterator::ConstIterator(Sample* bufferFirst, int capacity, Sample* at) : +template<> inline AudioRingBufferTemplate::ConstIterator::ConstIterator(Sample* bufferFirst, int capacity, Sample* at) : _bufferLength(capacity), _bufferFirst(bufferFirst), _bufferLast(bufferFirst + capacity - 1), _at(at) {} -inline AudioRingBufferTemplate::ConstIterator& AudioRingBufferTemplate::ConstIterator::operator=(const ConstIterator& rhs) { +template<> inline AudioRingBufferTemplate::ConstIterator& AudioRingBufferTemplate::ConstIterator::operator=(const ConstIterator& rhs) { _bufferLength = rhs._bufferLength; _bufferFirst = rhs._bufferFirst; _bufferLast = rhs._bufferLast; @@ -168,41 +180,41 @@ inline AudioRingBufferTemplate::ConstIterator& AudioRingBufferTemplate< return *this; } -inline AudioRingBufferTemplate::ConstIterator& AudioRingBufferTemplate::ConstIterator::operator++() { +template<> inline AudioRingBufferTemplate::ConstIterator& AudioRingBufferTemplate::ConstIterator::operator++() { _at = (_at == _bufferLast) ? _bufferFirst : _at + 1; return *this; } -inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::ConstIterator::operator++(int) { +template<> inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::ConstIterator::operator++(int) { ConstIterator tmp(*this); ++(*this); return tmp; } -inline AudioRingBufferTemplate::ConstIterator& AudioRingBufferTemplate::ConstIterator::operator--() { +template<> inline AudioRingBufferTemplate::ConstIterator& AudioRingBufferTemplate::ConstIterator::operator--() { _at = (_at == _bufferFirst) ? _bufferLast : _at - 1; return *this; } -inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::ConstIterator::operator--(int) { +template<> inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::ConstIterator::operator--(int) { ConstIterator tmp(*this); --(*this); return tmp; } -inline const int16_t& AudioRingBufferTemplate::ConstIterator::operator[] (int i) { +template<> inline const int16_t& AudioRingBufferTemplate::ConstIterator::operator[] (int i) { return *atShiftedBy(i); } -inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::ConstIterator::operator+(int i) { +template<> inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::ConstIterator::operator+(int i) { return ConstIterator(_bufferFirst, _bufferLength, atShiftedBy(i)); } -inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::ConstIterator::operator-(int i) { +template<> inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::ConstIterator::operator-(int i) { return ConstIterator(_bufferFirst, _bufferLength, atShiftedBy(-i)); } -inline int16_t* AudioRingBufferTemplate::ConstIterator::atShiftedBy(int i) { +template<> inline int16_t* AudioRingBufferTemplate::ConstIterator::atShiftedBy(int i) { i = (_at - _bufferFirst + i) % _bufferLength; if (i < 0) { i += _bufferLength; @@ -210,7 +222,7 @@ inline int16_t* AudioRingBufferTemplate::ConstIterator::atShiftedBy(int return _bufferFirst + i; } -inline void AudioRingBufferTemplate::ConstIterator::readSamples(Sample* dest, int numSamples) { +template<> inline void AudioRingBufferTemplate::ConstIterator::readSamples(Sample* dest, int numSamples) { auto samplesToEnd = _bufferLast - _at + 1; if (samplesToEnd >= numSamples) { @@ -225,7 +237,7 @@ inline void AudioRingBufferTemplate::ConstIterator::readSamples(Sample* } } -inline void AudioRingBufferTemplate::ConstIterator::readSamplesWithFade(Sample* dest, int numSamples, float fade) { +template<> inline void AudioRingBufferTemplate::ConstIterator::readSamplesWithFade(Sample* dest, int numSamples, float fade) { Sample* at = _at; for (int i = 0; i < numSamples; i++) { *dest = (float)*at * fade; @@ -234,7 +246,7 @@ inline void AudioRingBufferTemplate::ConstIterator::readSamplesWithFade } } -inline void AudioRingBufferTemplate::ConstIterator::readSamplesWithUpmix(Sample* dest, int numSamples, int numExtraChannels) { +template<> inline void AudioRingBufferTemplate::ConstIterator::readSamplesWithUpmix(Sample* dest, int numSamples, int numExtraChannels) { Sample* at = _at; for (int i = 0; i < numSamples/2; i++) { @@ -253,7 +265,7 @@ inline void AudioRingBufferTemplate::ConstIterator::readSamplesWithUpmi } } -inline void AudioRingBufferTemplate::ConstIterator::readSamplesWithDownmix(Sample* dest, int numSamples) { +template<> inline void AudioRingBufferTemplate::ConstIterator::readSamplesWithDownmix(Sample* dest, int numSamples) { Sample* at = _at; for (int i = 0; i < numSamples/2; i++) { @@ -268,11 +280,11 @@ inline void AudioRingBufferTemplate::ConstIterator::readSamplesWithDown } } -inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::nextOutput() const { +template<> inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::nextOutput() const { return ConstIterator(_buffer, _bufferLength, _nextOutput); } -inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::lastFrameWritten() const { +template<> inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::lastFrameWritten() const { return ConstIterator(_buffer, _bufferLength, _endOfLastWrite) - _numFrameSamples; } From 5927c089ac1368ce4e7bf94266eb8c8c8b7ba5b1 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 19 Jan 2017 16:37:31 -0500 Subject: [PATCH 20/45] use AudioRingMixBuffer as LocalInjectorsStream --- libraries/audio-client/src/AudioClient.cpp | 2 +- libraries/audio-client/src/AudioClient.h | 13 ++----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 491c4e341b..1fa6bd2d41 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1636,7 +1636,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { _localInjectorsStream.skipSamples(samplesRequested); } else { bool append = networkSamplesPopped > 0; - if ((injectorSamplesPopped = _localInjectorsStream.readSamples(mixBuffer, samplesRequested, append)) > 0) { + if ((injectorSamplesPopped = _localInjectorsStream.appendSamples(mixBuffer, samplesRequested, append)) > 0) { qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsStream.samplesAvailable(), samplesRequested); } } diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 8befd86f26..103d8a0892 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -69,17 +69,6 @@ class QIODevice; class Transform; class NLPacket; -class LocalInjectorsStream { -public: - LocalInjectorsStream(int numFrameSamples); - int getSampleCapacity() { return 0; }; - int samplesAvailable() { return 0; } - int writeSamples(const float*, int numSamples) { return 0; } - void resizeForFrameSize(int numFrameSamples) {} - int skipSamples(int numSamples) { return 0; } - int readSamples(float* mixBuffer, int numSamples, bool append) { return 0; } -}; - class AudioInjectorsThread : public QThread { Q_OBJECT @@ -96,6 +85,8 @@ private: class AudioClient : public AbstractAudioInterface, public Dependency { Q_OBJECT SINGLETON_DEPENDENCY + + using LocalInjectorsStream = AudioRingMixBuffer; public: static const int MIN_BUFFER_FRAMES; static const int MAX_BUFFER_FRAMES; From 75281099bd480298d65fc1fddc0a12bea9df3452 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Fri, 20 Jan 2017 14:33:54 -0500 Subject: [PATCH 21/45] add conformant explicit instantiation for AudioRingBuffers --- libraries/audio/src/AudioRingBuffer.cpp | 395 ++++++++---------------- libraries/audio/src/AudioRingBuffer.h | 227 +++++--------- 2 files changed, 207 insertions(+), 415 deletions(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index ab0948e328..4f64d4a4b0 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -26,8 +26,8 @@ static const QString RING_BUFFER_OVERFLOW_DEBUG { "AudioRingBuffer::writeData has overflown the buffer. Overwriting old data." }; static const QString DROPPED_SILENT_DEBUG { "AudioRingBuffer::addSilentSamples dropping silent samples to prevent overflow." }; -template<> -AudioRingBufferTemplate::AudioRingBufferTemplate(int numFrameSamples, int numFramesCapacity) : +template +AudioRingBufferTemplate::AudioRingBufferTemplate(int numFrameSamples, int numFramesCapacity) : _numFrameSamples(numFrameSamples), _frameCapacity(numFramesCapacity), _sampleCapacity(numFrameSamples * numFramesCapacity), @@ -44,25 +44,25 @@ AudioRingBufferTemplate::AudioRingBufferTemplate(int numFrameSamples, i static QString repeatedDroppedMessage = LogHandler::getInstance().addRepeatedMessageRegex(DROPPED_SILENT_DEBUG); } -template<> -AudioRingBufferTemplate::~AudioRingBufferTemplate() { +template +AudioRingBufferTemplate::~AudioRingBufferTemplate() { delete[] _buffer; } -template<> -void AudioRingBufferTemplate::clear() { +template +void AudioRingBufferTemplate::clear() { _endOfLastWrite = _buffer; _nextOutput = _buffer; } -template<> -void AudioRingBufferTemplate::reset() { +template +void AudioRingBufferTemplate::reset() { clear(); _overflowCount = 0; } -template<> -void AudioRingBufferTemplate::resizeForFrameSize(int numFrameSamples) { +template +void AudioRingBufferTemplate::resizeForFrameSize(int numFrameSamples) { delete[] _buffer; _numFrameSamples = numFrameSamples; _sampleCapacity = numFrameSamples * _frameCapacity; @@ -78,255 +78,13 @@ void AudioRingBufferTemplate::resizeForFrameSize(int numFrameSamples) { reset(); } -template<> -int AudioRingBufferTemplate::readSamples(Sample* destination, int maxSamples) { +template +int AudioRingBufferTemplate::readSamples(Sample* destination, int maxSamples) { return readData((char*)destination, maxSamples * SampleSize) / SampleSize; } -template<> -int AudioRingBufferTemplate::writeSamples(const Sample* source, int maxSamples) { - return writeData((char*)source, maxSamples * SampleSize) / SampleSize; -} - -template<> -int AudioRingBufferTemplate::readData(char *data, int maxSize) { - // only copy up to the number of samples we have available - int maxSamples = maxSize / SampleSize; - int numReadSamples = std::min(maxSamples, samplesAvailable()); - - if (_nextOutput + numReadSamples > _buffer + _bufferLength) { - // we're going to need to do two reads to get this data, it wraps around the edge - int numSamplesToEnd = (_buffer + _bufferLength) - _nextOutput; - - // read to the end of the buffer - memcpy(data, _nextOutput, numSamplesToEnd * SampleSize); - - // read the rest from the beginning of the buffer - memcpy(data + (numSamplesToEnd * SampleSize), _buffer, (numReadSamples - numSamplesToEnd) * SampleSize); - } else { - memcpy(data, _nextOutput, numReadSamples * SampleSize); - } - - shiftReadPosition(numReadSamples); - - return numReadSamples * SampleSize; -} - -template<> -int AudioRingBufferTemplate::writeData(const char* data, int maxSize) { - // only copy up to the number of samples we have capacity for - int maxSamples = maxSize / SampleSize; - int numWriteSamples = std::min(maxSamples, _sampleCapacity); - int samplesRoomFor = _sampleCapacity - samplesAvailable(); - - if (numWriteSamples > samplesRoomFor) { - // there's not enough room for this write. erase old data to make room for this new data - int samplesToDelete = numWriteSamples - samplesRoomFor; - _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, samplesToDelete); - _overflowCount++; - - qCDebug(audio) << qPrintable(RING_BUFFER_OVERFLOW_DEBUG); - } - - if (_endOfLastWrite + numWriteSamples > _buffer + _bufferLength) { - // we're going to need to do two writes to set this data, it wraps around the edge - int numSamplesToEnd = (_buffer + _bufferLength) - _endOfLastWrite; - - // write to the end of the buffer - memcpy(_endOfLastWrite, data, numSamplesToEnd * SampleSize); - - // write the rest to the beginning of the buffer - memcpy(_buffer, data + (numSamplesToEnd * SampleSize), (numWriteSamples - numSamplesToEnd) * SampleSize); - } else { - memcpy(_endOfLastWrite, data, numWriteSamples * SampleSize); - } - - _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, numWriteSamples); - - return numWriteSamples * SampleSize; -} - -template<> -int AudioRingBufferTemplate::samplesAvailable() const { - if (!_endOfLastWrite) { - return 0; - } - - int sampleDifference = _endOfLastWrite - _nextOutput; - if (sampleDifference < 0) { - sampleDifference += _bufferLength; - } - return sampleDifference; -} - -template<> -int16_t* AudioRingBufferTemplate::shiftedPositionAccomodatingWrap(Sample* position, int numSamplesShift) const { - // NOTE: It is possible to shift out-of-bounds if (|numSamplesShift| > 2 * _bufferLength), but this should not occur - if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + _bufferLength) { - // this shift will wrap the position around to the beginning of the ring - return position + numSamplesShift - _bufferLength; - } else if (numSamplesShift < 0 && position + numSamplesShift < _buffer) { - // this shift will go around to the end of the ring - return position + numSamplesShift + _bufferLength; - } else { - return position + numSamplesShift; - } -} - -template<> -int AudioRingBufferTemplate::addSilentSamples(int silentSamples) { - // NOTE: This implementation is nearly identical to writeData save for s/memcpy/memset, refer to comments there - int numWriteSamples = std::min(silentSamples, _sampleCapacity); - int samplesRoomFor = _sampleCapacity - samplesAvailable(); - - if (numWriteSamples > samplesRoomFor) { - numWriteSamples = samplesRoomFor; - - qCDebug(audio) << qPrintable(DROPPED_SILENT_DEBUG); - } - - if (_endOfLastWrite + numWriteSamples > _buffer + _bufferLength) { - int numSamplesToEnd = (_buffer + _bufferLength) - _endOfLastWrite; - memset(_endOfLastWrite, 0, numSamplesToEnd * SampleSize); - memset(_buffer, 0, (numWriteSamples - numSamplesToEnd) * SampleSize); - } else { - memset(_endOfLastWrite, 0, numWriteSamples * SampleSize); - } - - _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, numWriteSamples); - - return numWriteSamples; -} - -template<> -float AudioRingBufferTemplate::getFrameLoudness(const Sample* frameStart) const { - // FIXME: This is a bad measure of loudness - normal estimation uses sqrt(sum(x*x)) - float loudness = 0.0f; - const Sample* sampleAt = frameStart; - const Sample* bufferLastAt = _buffer + _bufferLength - 1; - - for (int i = 0; i < _numFrameSamples; ++i) { - loudness += (float) std::abs(*sampleAt); - // wrap if necessary - sampleAt = sampleAt == bufferLastAt ? _buffer : sampleAt + 1; - } - loudness /= _numFrameSamples; - loudness /= AudioConstants::MAX_SAMPLE_VALUE; - - return loudness; -} - -template<> -float AudioRingBufferTemplate::getFrameLoudness(ConstIterator frameStart) const { - if (frameStart.isNull()) { - return 0.0f; - } - return getFrameLoudness(&(*frameStart)); -} - -template<> -int AudioRingBufferTemplate::writeSamples(ConstIterator source, int maxSamples) { - int samplesToCopy = std::min(maxSamples, _sampleCapacity); - int samplesRoomFor = _sampleCapacity - samplesAvailable(); - if (samplesToCopy > samplesRoomFor) { - // there's not enough room for this write. erase old data to make room for this new data - int samplesToDelete = samplesToCopy - samplesRoomFor; - _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, samplesToDelete); - _overflowCount++; - qCDebug(audio) << qPrintable(RING_BUFFER_OVERFLOW_DEBUG); - } - - Sample* bufferLast = _buffer + _bufferLength - 1; - for (int i = 0; i < samplesToCopy; i++) { - *_endOfLastWrite = *source; - _endOfLastWrite = (_endOfLastWrite == bufferLast) ? _buffer : _endOfLastWrite + 1; - ++source; - } - - return samplesToCopy; -} - -template<> -int AudioRingBufferTemplate::writeSamplesWithFade(ConstIterator source, int maxSamples, float fade) { - int samplesToCopy = std::min(maxSamples, _sampleCapacity); - int samplesRoomFor = _sampleCapacity - samplesAvailable(); - if (samplesToCopy > samplesRoomFor) { - // there's not enough room for this write. erase old data to make room for this new data - int samplesToDelete = samplesToCopy - samplesRoomFor; - _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, samplesToDelete); - _overflowCount++; - qCDebug(audio) << qPrintable(RING_BUFFER_OVERFLOW_DEBUG); - } - - Sample* bufferLast = _buffer + _bufferLength - 1; - for (int i = 0; i < samplesToCopy; i++) { - *_endOfLastWrite = (Sample)((float)(*source) * fade); - _endOfLastWrite = (_endOfLastWrite == bufferLast) ? _buffer : _endOfLastWrite + 1; - ++source; - } - - return samplesToCopy; -} - -template<> -AudioRingBufferTemplate::AudioRingBufferTemplate(int numFrameSamples, int numFramesCapacity) : - _numFrameSamples(numFrameSamples), - _frameCapacity(numFramesCapacity), - _sampleCapacity(numFrameSamples * numFramesCapacity), - _bufferLength(numFrameSamples * (numFramesCapacity + 1)) -{ - if (numFrameSamples) { - _buffer = new Sample[_bufferLength]; - memset(_buffer, 0, _bufferLength * SampleSize); - _nextOutput = _buffer; - _endOfLastWrite = _buffer; - } - - static QString repeatedOverflowMessage = LogHandler::getInstance().addRepeatedMessageRegex(RING_BUFFER_OVERFLOW_DEBUG); - static QString repeatedDroppedMessage = LogHandler::getInstance().addRepeatedMessageRegex(DROPPED_SILENT_DEBUG); -} - -template<> -AudioRingBufferTemplate::~AudioRingBufferTemplate() { - delete[] _buffer; -} - -template<> -void AudioRingBufferTemplate::clear() { - _endOfLastWrite = _buffer; - _nextOutput = _buffer; -} - -template<> -void AudioRingBufferTemplate::reset() { - clear(); - _overflowCount = 0; -} - -template<> -void AudioRingBufferTemplate::resizeForFrameSize(int numFrameSamples) { - delete[] _buffer; - _numFrameSamples = numFrameSamples; - _sampleCapacity = numFrameSamples * _frameCapacity; - _bufferLength = numFrameSamples * (_frameCapacity + 1); - - if (numFrameSamples) { - _buffer = new Sample[_bufferLength]; - memset(_buffer, 0, _bufferLength * SampleSize); - } else { - _buffer = nullptr; - } - - reset(); -} - -template<> -int AudioRingBufferTemplate::readSamples(Sample* destination, int maxSamples) { - return readData((char*)destination, maxSamples * SampleSize) / SampleSize; -} - -template<> -int AudioRingBufferTemplate::appendSamples(Sample* destination, int maxSamples, bool append) { +template +int AudioRingBufferTemplate::appendSamples(Sample* destination, int maxSamples, bool append) { if (append) { return appendData((char*)destination, maxSamples * SampleSize) / SampleSize; } else { @@ -334,13 +92,13 @@ int AudioRingBufferTemplate::appendSamples(Sample* destination, int maxSa } } -template<> -int AudioRingBufferTemplate::writeSamples(const Sample* source, int maxSamples) { +template +int AudioRingBufferTemplate::writeSamples(const Sample* source, int maxSamples) { return writeData((char*)source, maxSamples * SampleSize) / SampleSize; } -template<> -int AudioRingBufferTemplate::readData(char *data, int maxSize) { +template +int AudioRingBufferTemplate::readData(char *data, int maxSize) { // only copy up to the number of samples we have available int maxSamples = maxSize / SampleSize; int numReadSamples = std::min(maxSamples, samplesAvailable()); @@ -363,8 +121,8 @@ int AudioRingBufferTemplate::readData(char *data, int maxSize) { return numReadSamples * SampleSize; } -template<> -int AudioRingBufferTemplate::appendData(char *data, int maxSize) { +template +int AudioRingBufferTemplate::appendData(char *data, int maxSize) { // only copy up to the number of samples we have available int maxSamples = maxSize / SampleSize; int numReadSamples = std::min(maxSamples, samplesAvailable()); @@ -396,8 +154,8 @@ int AudioRingBufferTemplate::appendData(char *data, int maxSize) { return numReadSamples * SampleSize; } -template<> -int AudioRingBufferTemplate::writeData(const char* data, int maxSize) { +template +int AudioRingBufferTemplate::writeData(const char* data, int maxSize) { // only copy up to the number of samples we have capacity for int maxSamples = maxSize / SampleSize; int numWriteSamples = std::min(maxSamples, _sampleCapacity); @@ -430,8 +188,8 @@ int AudioRingBufferTemplate::writeData(const char* data, int maxSize) { return numWriteSamples * SampleSize; } -template<> -int AudioRingBufferTemplate::samplesAvailable() const { +template +int AudioRingBufferTemplate::samplesAvailable() const { if (!_endOfLastWrite) { return 0; } @@ -443,8 +201,8 @@ int AudioRingBufferTemplate::samplesAvailable() const { return sampleDifference; } -template<> -float* AudioRingBufferTemplate::shiftedPositionAccomodatingWrap(Sample* position, int numSamplesShift) const { +template +typename AudioRingBufferTemplate::Sample* AudioRingBufferTemplate::shiftedPositionAccomodatingWrap(Sample* position, int numSamplesShift) const { // NOTE: It is possible to shift out-of-bounds if (|numSamplesShift| > 2 * _bufferLength), but this should not occur if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + _bufferLength) { // this shift will wrap the position around to the beginning of the ring @@ -455,4 +213,103 @@ float* AudioRingBufferTemplate::shiftedPositionAccomodatingWrap(Sample* p } else { return position + numSamplesShift; } -} \ No newline at end of file +} + +template +int AudioRingBufferTemplate::addSilentSamples(int silentSamples) { + // NOTE: This implementation is nearly identical to writeData save for s/memcpy/memset, refer to comments there + int numWriteSamples = std::min(silentSamples, _sampleCapacity); + int samplesRoomFor = _sampleCapacity - samplesAvailable(); + + if (numWriteSamples > samplesRoomFor) { + numWriteSamples = samplesRoomFor; + + qCDebug(audio) << qPrintable(DROPPED_SILENT_DEBUG); + } + + if (_endOfLastWrite + numWriteSamples > _buffer + _bufferLength) { + int numSamplesToEnd = (_buffer + _bufferLength) - _endOfLastWrite; + memset(_endOfLastWrite, 0, numSamplesToEnd * SampleSize); + memset(_buffer, 0, (numWriteSamples - numSamplesToEnd) * SampleSize); + } else { + memset(_endOfLastWrite, 0, numWriteSamples * SampleSize); + } + + _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, numWriteSamples); + + return numWriteSamples; +} + +template +float AudioRingBufferTemplate::getFrameLoudness(const Sample* frameStart) const { + // FIXME: This is a bad measure of loudness - normal estimation uses sqrt(sum(x*x)) + float loudness = 0.0f; + const Sample* sampleAt = frameStart; + const Sample* bufferLastAt = _buffer + _bufferLength - 1; + + for (int i = 0; i < _numFrameSamples; ++i) { + loudness += (float) std::abs(*sampleAt); + // wrap if necessary + sampleAt = sampleAt == bufferLastAt ? _buffer : sampleAt + 1; + } + loudness /= _numFrameSamples; + loudness /= AudioConstants::MAX_SAMPLE_VALUE; + + return loudness; +} + +template +float AudioRingBufferTemplate::getFrameLoudness(ConstIterator frameStart) const { + if (frameStart.isNull()) { + return 0.0f; + } + return getFrameLoudness(&(*frameStart)); +} + +template +int AudioRingBufferTemplate::writeSamples(ConstIterator source, int maxSamples) { + int samplesToCopy = std::min(maxSamples, _sampleCapacity); + int samplesRoomFor = _sampleCapacity - samplesAvailable(); + if (samplesToCopy > samplesRoomFor) { + // there's not enough room for this write. erase old data to make room for this new data + int samplesToDelete = samplesToCopy - samplesRoomFor; + _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, samplesToDelete); + _overflowCount++; + qCDebug(audio) << qPrintable(RING_BUFFER_OVERFLOW_DEBUG); + } + + Sample* bufferLast = _buffer + _bufferLength - 1; + for (int i = 0; i < samplesToCopy; i++) { + *_endOfLastWrite = *source; + _endOfLastWrite = (_endOfLastWrite == bufferLast) ? _buffer : _endOfLastWrite + 1; + ++source; + } + + return samplesToCopy; +} + +template +int AudioRingBufferTemplate::writeSamplesWithFade(ConstIterator source, int maxSamples, float fade) { + int samplesToCopy = std::min(maxSamples, _sampleCapacity); + int samplesRoomFor = _sampleCapacity - samplesAvailable(); + if (samplesToCopy > samplesRoomFor) { + // there's not enough room for this write. erase old data to make room for this new data + int samplesToDelete = samplesToCopy - samplesRoomFor; + _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, samplesToDelete); + _overflowCount++; + qCDebug(audio) << qPrintable(RING_BUFFER_OVERFLOW_DEBUG); + } + + Sample* bufferLast = _buffer + _bufferLength - 1; + for (int i = 0; i < samplesToCopy; i++) { + *_endOfLastWrite = (Sample)((float)(*source) * fade); + _endOfLastWrite = (_endOfLastWrite == bufferLast) ? _buffer : _endOfLastWrite + 1; + ++source; + } + + return samplesToCopy; +} + +// explicit instantiations for scratch/mix buffers +template class AudioRingBufferTemplate; +template class AudioRingBufferTemplate; diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 92c6dcc336..0208c8686e 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -100,8 +100,16 @@ public: class ConstIterator { public: - ConstIterator(); - ConstIterator(Sample* bufferFirst, int capacity, Sample* at); + ConstIterator() : + _bufferLength(0), + _bufferFirst(NULL), + _bufferLast(NULL), + _at(NULL) {} + ConstIterator(Sample* bufferFirst, int capacity, Sample* at) : + _bufferLength(capacity), + _bufferFirst(bufferFirst), + _bufferLast(bufferFirst + capacity - 1), + _at(at) {} ConstIterator(const ConstIterator& rhs) = default; bool isNull() const { return _at == NULL; } @@ -110,22 +118,73 @@ public: bool operator!=(const ConstIterator& rhs) { return _at != rhs._at; } const Sample& operator*() { return *_at; } - ConstIterator& operator=(const ConstIterator& rhs); - ConstIterator& operator++(); - ConstIterator operator++(int); - ConstIterator& operator--(); - ConstIterator operator--(int); - const Sample& operator[] (int i); - ConstIterator operator+(int i); - ConstIterator operator-(int i); + ConstIterator& operator=(const ConstIterator& rhs) { + _bufferLength = rhs._bufferLength; + _bufferFirst = rhs._bufferFirst; + _bufferLast = rhs._bufferLast; + _at = rhs._at; + return *this; + } + ConstIterator& operator++() { + _at = (_at == _bufferLast) ? _bufferFirst : _at + 1; + return *this; + } + ConstIterator operator++(int) { + ConstIterator tmp(*this); + ++(*this); + return tmp; + } + ConstIterator& operator--() { + _at = (_at == _bufferFirst) ? _bufferLast : _at - 1; + return *this; + } + ConstIterator operator--(int) { + ConstIterator tmp(*this); + --(*this); + return tmp; + } + const Sample& operator[] (int i) { + return *atShiftedBy(i); + } + ConstIterator operator+(int i) { + return ConstIterator(_bufferFirst, _bufferLength, atShiftedBy(i)); + } + ConstIterator operator-(int i) { + return ConstIterator(_bufferFirst, _bufferLength, atShiftedBy(-i)); + } + + void readSamples(Sample* dest, int numSamples) { + auto samplesToEnd = _bufferLast - _at + 1; + + if (samplesToEnd >= numSamples) { + memcpy(dest, _at, numSamples * SampleSize); + _at += numSamples; + } else { + auto samplesFromStart = numSamples - samplesToEnd; + memcpy(dest, _at, samplesToEnd * SampleSize); + memcpy(dest + samplesToEnd, _bufferFirst, samplesFromStart * SampleSize); + + _at = _bufferFirst + samplesFromStart; + } + } + void readSamplesWithFade(Sample* dest, int numSamples, float fade) { + Sample* at = _at; + for (int i = 0; i < numSamples; i++) { + *dest = (float)*at * fade; + ++dest; + at = (at == _bufferLast) ? _bufferFirst : at + 1; + } + } - void readSamples(Sample* dest, int numSamples); - void readSamplesWithFade(Sample* dest, int numSamples, float fade); - void readSamplesWithUpmix(Sample* dest, int numSamples, int numExtraChannels); - void readSamplesWithDownmix(Sample* dest, int numSamples); private: - Sample* atShiftedBy(int i); + Sample* atShiftedBy(int i) { + i = (_at - _bufferFirst + i) % _bufferLength; + if (i < 0) { + i += _bufferLength; + } + return _bufferFirst + i; + } int _bufferLength; Sample* _bufferFirst; @@ -133,8 +192,12 @@ public: Sample* _at; }; - ConstIterator nextOutput() const; - ConstIterator lastFrameWritten() const; + ConstIterator nextOutput() const { + return ConstIterator(_buffer, _bufferLength, _nextOutput); + } + ConstIterator lastFrameWritten() const { + return ConstIterator(_buffer, _bufferLength, _endOfLastWrite) - _numFrameSamples; + } int writeSamples(ConstIterator source, int maxSamples); int writeSamplesWithFade(ConstIterator source, int maxSamples, float fade); @@ -156,136 +219,8 @@ protected: Sample* _buffer{ nullptr }; }; +// expose explicit instantiations for scratch/mix buffers using AudioRingBuffer = AudioRingBufferTemplate; using AudioRingMixBuffer = AudioRingBufferTemplate; -// inline the iterator: -template<> inline AudioRingBufferTemplate::ConstIterator::ConstIterator() : - _bufferLength(0), - _bufferFirst(NULL), - _bufferLast(NULL), - _at(NULL) {} - -template<> inline AudioRingBufferTemplate::ConstIterator::ConstIterator(Sample* bufferFirst, int capacity, Sample* at) : - _bufferLength(capacity), - _bufferFirst(bufferFirst), - _bufferLast(bufferFirst + capacity - 1), - _at(at) {} - -template<> inline AudioRingBufferTemplate::ConstIterator& AudioRingBufferTemplate::ConstIterator::operator=(const ConstIterator& rhs) { - _bufferLength = rhs._bufferLength; - _bufferFirst = rhs._bufferFirst; - _bufferLast = rhs._bufferLast; - _at = rhs._at; - return *this; -} - -template<> inline AudioRingBufferTemplate::ConstIterator& AudioRingBufferTemplate::ConstIterator::operator++() { - _at = (_at == _bufferLast) ? _bufferFirst : _at + 1; - return *this; -} - -template<> inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::ConstIterator::operator++(int) { - ConstIterator tmp(*this); - ++(*this); - return tmp; -} - -template<> inline AudioRingBufferTemplate::ConstIterator& AudioRingBufferTemplate::ConstIterator::operator--() { - _at = (_at == _bufferFirst) ? _bufferLast : _at - 1; - return *this; -} - -template<> inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::ConstIterator::operator--(int) { - ConstIterator tmp(*this); - --(*this); - return tmp; -} - -template<> inline const int16_t& AudioRingBufferTemplate::ConstIterator::operator[] (int i) { - return *atShiftedBy(i); -} - -template<> inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::ConstIterator::operator+(int i) { - return ConstIterator(_bufferFirst, _bufferLength, atShiftedBy(i)); -} - -template<> inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::ConstIterator::operator-(int i) { - return ConstIterator(_bufferFirst, _bufferLength, atShiftedBy(-i)); -} - -template<> inline int16_t* AudioRingBufferTemplate::ConstIterator::atShiftedBy(int i) { - i = (_at - _bufferFirst + i) % _bufferLength; - if (i < 0) { - i += _bufferLength; - } - return _bufferFirst + i; -} - -template<> inline void AudioRingBufferTemplate::ConstIterator::readSamples(Sample* dest, int numSamples) { - auto samplesToEnd = _bufferLast - _at + 1; - - if (samplesToEnd >= numSamples) { - memcpy(dest, _at, numSamples * SampleSize); - _at += numSamples; - } else { - auto samplesFromStart = numSamples - samplesToEnd; - memcpy(dest, _at, samplesToEnd * SampleSize); - memcpy(dest + samplesToEnd, _bufferFirst, samplesFromStart * SampleSize); - - _at = _bufferFirst + samplesFromStart; - } -} - -template<> inline void AudioRingBufferTemplate::ConstIterator::readSamplesWithFade(Sample* dest, int numSamples, float fade) { - Sample* at = _at; - for (int i = 0; i < numSamples; i++) { - *dest = (float)*at * fade; - ++dest; - at = (at == _bufferLast) ? _bufferFirst : at + 1; - } -} - -template<> inline void AudioRingBufferTemplate::ConstIterator::readSamplesWithUpmix(Sample* dest, int numSamples, int numExtraChannels) { - Sample* at = _at; - for (int i = 0; i < numSamples/2; i++) { - - // read 2 samples - Sample left = *at; - at = (at == _bufferLast) ? _bufferFirst : at + 1; - Sample right = *at; - at = (at == _bufferLast) ? _bufferFirst : at + 1; - - // write 2 + N samples - *dest++ = left; - *dest++ = right; - for (int n = 0; n < numExtraChannels; n++) { - *dest++ = 0; - } - } -} - -template<> inline void AudioRingBufferTemplate::ConstIterator::readSamplesWithDownmix(Sample* dest, int numSamples) { - Sample* at = _at; - for (int i = 0; i < numSamples/2; i++) { - - // read 2 samples - Sample left = *at; - at = (at == _bufferLast) ? _bufferFirst : at + 1; - Sample right = *at; - at = (at == _bufferLast) ? _bufferFirst : at + 1; - - // write 1 sample - *dest++ = (Sample)((left + right) / 2); - } -} - -template<> inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::nextOutput() const { - return ConstIterator(_buffer, _bufferLength, _nextOutput); -} - -template<> inline AudioRingBufferTemplate::ConstIterator AudioRingBufferTemplate::lastFrameWritten() const { - return ConstIterator(_buffer, _bufferLength, _endOfLastWrite) - _numFrameSamples; -} - #endif // hifi_AudioRingBuffer_h From 280ed04f74fe386d619f6a3e737c50850731c35b Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Fri, 20 Jan 2017 14:35:43 -0500 Subject: [PATCH 22/45] fix unused warning for channel up/downmix --- libraries/audio-client/src/AudioClient.cpp | 28 ++++++++++++++++++++ libraries/shared/src/AudioHelpers.h | 30 ---------------------- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 1fa6bd2d41..e94e1f4385 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -117,6 +117,34 @@ void AudioInjectorsThread::prepare() { _audio->prepareLocalAudioInjectors(); } +static void channelUpmix(int16_t* source, int16_t* dest, int numSamples, int numExtraChannels) { + for (int i = 0; i < numSamples/2; i++) { + + // read 2 samples + int16_t left = *source++; + int16_t right = *source++; + + // write 2 + N samples + *dest++ = left; + *dest++ = right; + for (int n = 0; n < numExtraChannels; n++) { + *dest++ = 0; + } + } +} + +static void channelDownmix(int16_t* source, int16_t* dest, int numSamples) { + for (int i = 0; i < numSamples/2; i++) { + + // read 2 samples + int16_t left = *source++; + int16_t right = *source++; + + // write 1 sample + *dest++ = (int16_t)((left + right) / 2); + } +} + AudioClient::AudioClient() : AbstractAudioInterface(), _gate(this), diff --git a/libraries/shared/src/AudioHelpers.h b/libraries/shared/src/AudioHelpers.h index b0fcb7248d..c891876648 100644 --- a/libraries/shared/src/AudioHelpers.h +++ b/libraries/shared/src/AudioHelpers.h @@ -103,34 +103,4 @@ static inline void convertToScratch(int16_t* scratchBuffer, const float* mixBuff } } -static void channelUpmix(int16_t* source, int16_t* dest, int numSamples, int numExtraChannels) { - - for (int i = 0; i < numSamples/2; i++) { - - // read 2 samples - int16_t left = *source++; - int16_t right = *source++; - - // write 2 + N samples - *dest++ = left; - *dest++ = right; - for (int n = 0; n < numExtraChannels; n++) { - *dest++ = 0; - } - } -} - -static void channelDownmix(int16_t* source, int16_t* dest, int numSamples) { - - for (int i = 0; i < numSamples/2; i++) { - - // read 2 samples - int16_t left = *source++; - int16_t right = *source++; - - // write 1 sample - *dest++ = (int16_t)((left + right) / 2); - } -} - #endif // hifi_AudioHelpers_h From bb247fe8a39339bd2c4f68aed589a6b835c83aaa Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Fri, 20 Jan 2017 14:48:54 -0500 Subject: [PATCH 23/45] rename AudioMixRingBuffer --- libraries/audio-client/src/AudioClient.h | 2 +- libraries/audio/src/AudioRingBuffer.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 103d8a0892..b4ae84754a 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -86,7 +86,7 @@ class AudioClient : public AbstractAudioInterface, public Dependency { Q_OBJECT SINGLETON_DEPENDENCY - using LocalInjectorsStream = AudioRingMixBuffer; + using LocalInjectorsStream = AudioMixRingBuffer; public: static const int MIN_BUFFER_FRAMES; static const int MAX_BUFFER_FRAMES; diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 0208c8686e..e43fd22548 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -221,6 +221,6 @@ protected: // expose explicit instantiations for scratch/mix buffers using AudioRingBuffer = AudioRingBufferTemplate; -using AudioRingMixBuffer = AudioRingBufferTemplate; +using AudioMixRingBuffer = AudioRingBufferTemplate; #endif // hifi_AudioRingBuffer_h From 685483b924d0658288788e7336f0ab82445ce009 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Fri, 20 Jan 2017 17:52:03 -0500 Subject: [PATCH 24/45] do not omit local audio when echoing server audio --- libraries/audio-client/src/AudioClient.cpp | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index e94e1f4385..62dc5c527b 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1659,14 +1659,9 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { int injectorSamplesPopped = 0; { Lock lock(_audio->_localAudioMutex); - if (_audio->_shouldEchoToServer) { - // omit local audio, it should be echoed - _localInjectorsStream.skipSamples(samplesRequested); - } else { - bool append = networkSamplesPopped > 0; - if ((injectorSamplesPopped = _localInjectorsStream.appendSamples(mixBuffer, samplesRequested, append)) > 0) { - qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsStream.samplesAvailable(), samplesRequested); - } + bool append = networkSamplesPopped > 0; + if ((injectorSamplesPopped = _localInjectorsStream.appendSamples(mixBuffer, samplesRequested, append)) > 0) { + qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsStream.samplesAvailable(), samplesRequested); } } From df051ff8df4a3b4bbb165361e7f3f29b2709ecfb Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Fri, 20 Jan 2017 18:37:46 -0500 Subject: [PATCH 25/45] maintain network audio in int16_t --- libraries/audio-client/src/AudioClient.cpp | 21 +++++++++++---------- libraries/audio-client/src/AudioClient.h | 1 - 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 62dc5c527b..aff8406897 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1248,23 +1248,24 @@ void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteA outputBuffer.resize(_outputFrameSize * AudioConstants::SAMPLE_SIZE); int16_t* outputSamples = reinterpret_cast(outputBuffer.data()); - convertToMix(_networkMixBuffer, decodedSamples, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); - - // apply stereo reverb bool hasReverb = _reverb || _receivedAudioStream.hasReverb(); + + // apply stereo reverb if (hasReverb) { updateReverbOptions(); - _listenerReverb.render(_networkMixBuffer, _networkMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + int16_t* reverbSamples = _networkToOutputResampler ? _networkScratchBuffer : outputSamples; + _listenerReverb.render(decodedSamples, reverbSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } + // resample to output sample rate if (_networkToOutputResampler) { - convertToScratch(_networkScratchBuffer, _networkMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); + const int16_t* inputSamples = hasReverb ? _networkScratchBuffer : decodedSamples; + _networkToOutputResampler->render(inputSamples, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + } - // resample to output sample rate - _networkToOutputResampler->render(_networkScratchBuffer, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - } else { - convertToScratch(outputSamples, _networkMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); + // if no transformations were applied, we still need to copy the buffer + if (!hasReverb && !_networkToOutputResampler) { + memcpy(outputSamples, decodedSamples, decodedBuffer.size()); } } diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index b4ae84754a..b0eaa2dd34 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -318,7 +318,6 @@ private: AudioSRC* _localToOutputResampler; // for network audio (used by network audio thread) - float _networkMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _networkScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; // for local audio (used by audio injectors thread) From 72f8fa49f9f7febaf66ca6cb358d8cdbc63c1d1c Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Fri, 20 Jan 2017 18:38:21 -0500 Subject: [PATCH 26/45] inline audio convertToFloat --- libraries/audio-client/src/AudioClient.cpp | 11 ++++++++--- libraries/shared/src/AudioHelpers.h | 12 ------------ 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index aff8406897..b12c48ef82 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -46,7 +46,6 @@ #include #include "PositionalAudioStream.h" -#include "AudioHelpers.h" #include "AudioClientLogging.h" #include "AudioLogging.h" @@ -145,6 +144,10 @@ static void channelDownmix(int16_t* source, int16_t* dest, int numSamples) { } } +static inline float convertToFloat(int16_t sample) { + return (float)sample * (1 / 32768.0f); +} + AudioClient::AudioClient() : AbstractAudioInterface(), _gate(this), @@ -1201,7 +1204,7 @@ bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) { // stereo gets directly mixed into mixBuffer float gain = injector->getVolume(); for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) { - mixBuffer[i] += (float)_localScratchBuffer[i] * (1/32768.0f) * gain; + mixBuffer[i] += convertToFloat(_localScratchBuffer[i]) * gain; } } else { @@ -1652,7 +1655,9 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput(); lastPopOutput.readSamples(scratchBuffer, networkSamplesPopped); - convertToMix(mixBuffer, scratchBuffer, networkSamplesPopped); + for (int i = 0; i < networkSamplesPopped; i++) { + mixBuffer[i] = convertToFloat(scratchBuffer[i]); + } samplesRequested = networkSamplesPopped; } diff --git a/libraries/shared/src/AudioHelpers.h b/libraries/shared/src/AudioHelpers.h index c891876648..b43764ef5d 100644 --- a/libraries/shared/src/AudioHelpers.h +++ b/libraries/shared/src/AudioHelpers.h @@ -91,16 +91,4 @@ static inline float unpackFloatGainFromByte(uint8_t byte) { return gain; } -static inline void convertToMix(float* mixBuffer, const int16_t* scratchBuffer, int numSamples) { - for (int i = 0; i < numSamples; i++) { - mixBuffer[i] = (float)scratchBuffer[i] * (1/32768.0f); - } -} - -static inline void convertToScratch(int16_t* scratchBuffer, const float* mixBuffer, int numSamples) { - for (int i = 0; i < numSamples; i++) { - scratchBuffer[i] = (int16_t)(mixBuffer[i] * 32768.0f); - } -} - #endif // hifi_AudioHelpers_h From 061668cba41bb76a77d0147cec536a724ba960ef Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Sat, 21 Jan 2017 19:10:34 -0500 Subject: [PATCH 27/45] use lock-free pipe for local audio in device callback --- libraries/audio-client/src/AudioClient.cpp | 13 +++++++++++-- libraries/audio-client/src/AudioClient.h | 3 +++ libraries/audio/src/AudioRingBuffer.h | 6 ++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index b12c48ef82..2e532d67bf 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -190,6 +190,9 @@ AudioClient::AudioClient() : _inputGate(), _positionGetter(DEFAULT_POSITION_GETTER), _orientationGetter(DEFAULT_ORIENTATION_GETTER) { + // avoid putting a lock in the device callback + assert(_localSamplesAvailable.is_lock_free()); + // deprecate legacy settings { Setting::Handle::Deprecated("maxFramesOverDesired", InboundAudioStream::MAX_FRAMES_OVER_DESIRED); @@ -1105,7 +1108,8 @@ void AudioClient::prepareLocalAudioInjectors() { int bufferCapacity = _localInjectorsStream.getSampleCapacity(); if (_localToOutputResampler) { - // avoid overwriting the buffer + // avoid overwriting the buffer, + // instead of failing on writes because the buffer is used as a lock-free pipe bufferCapacity -= _localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) * AudioConstants::STEREO; @@ -1115,9 +1119,10 @@ void AudioClient::prepareLocalAudioInjectors() { int samplesNeeded = std::numeric_limits::max(); while (samplesNeeded > 0) { // lock for every write to avoid locking out the device callback + // this lock is intentional - the buffer is only lock-free in its use in the device callback Lock lock(_localAudioMutex); - samplesNeeded = bufferCapacity - _localInjectorsStream.samplesAvailable(); + samplesNeeded = bufferCapacity - _localSamplesAvailable.load(std::memory_order_relaxed); if (samplesNeeded <= 0) { break; } @@ -1149,6 +1154,7 @@ void AudioClient::prepareLocalAudioInjectors() { AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); } + _localSamplesAvailable.fetch_add(samples, std::memory_order_release); samplesNeeded -= samples; } } @@ -1452,6 +1458,7 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice bool supportedFormat = false; Lock lock(_localAudioMutex); + _localSamplesAvailable.exchange(0, std::memory_order_release); // cleanup any previously initialized device if (_audioOutput) { @@ -1666,7 +1673,9 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { { Lock lock(_audio->_localAudioMutex); bool append = networkSamplesPopped > 0; + samplesRequested = std::min(samplesRequested, _audio->_localSamplesAvailable.load(std::memory_order_acquire)); if ((injectorSamplesPopped = _localInjectorsStream.appendSamples(mixBuffer, samplesRequested, append)) > 0) { + _audio->_localSamplesAvailable.fetch_sub(injectorSamplesPopped, std::memory_order_release); qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsStream.samplesAvailable(), samplesRequested); } } diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index b0eaa2dd34..699ba71ef7 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -280,6 +280,9 @@ private: QIODevice* _loopbackOutputDevice; AudioRingBuffer _inputRingBuffer; LocalInjectorsStream _localInjectorsStream; + // In order to use _localInjectorsStream as a lock-free pipe, + // use it with a single producer/consumer, and track available samples + std::atomic _localSamplesAvailable { 0 }; MixedProcessedAudioStream _receivedAudioStream; bool _isStereoInput; diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index e43fd22548..bb32df19a2 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -45,6 +45,12 @@ public: // FIXME: discards any data in the buffer void resizeForFrameSize(int numFrameSamples); + // Reading and writing to the buffer uses minimal shared data, such that + // in cases that avoid overwriting the buffer, a single producer/consumer + // may use this as a lock-free pipe (see audio-client/src/AudioClient.cpp). + // IMPORTANT: Avoid changes to the implementation that touch shared data unless you can + // maintain this behavior. + /// Read up to maxSamples into destination (will only read up to samplesAvailable()) /// Returns number of read samples int readSamples(Sample* destination, int maxSamples); From 1c5228cb6d35a48d4b231f11504535c1b98adf51 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 26 Jan 2017 19:16:02 -0500 Subject: [PATCH 28/45] fix AudioRingBuffer::appendData --- libraries/audio/src/AudioRingBuffer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 4f64d4a4b0..59b3e874d7 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -135,17 +135,17 @@ int AudioRingBufferTemplate::appendData(char *data, int maxSize) { // read to the end of the buffer for (int i = 0; i < numSamplesToEnd; i++) { - *dest++ = *output++; + *dest++ += *output++; } // read the rest from the beginning of the buffer output = _buffer; for (int i = 0; i < (numReadSamples - numSamplesToEnd); i++) { - *dest++ = *output++; + *dest++ += *output++; } } else { for (int i = 0; i < numReadSamples; i++) { - *dest++ = *output++; + *dest++ += *output++; } } From 725cfa804553e8642ff5a0a6b935d59de7d08c95 Mon Sep 17 00:00:00 2001 From: Zach Fox Date: Mon, 30 Jan 2017 11:52:08 -0800 Subject: [PATCH 29/45] Fix it! --- .../entity-server-filter-example.js | 52 ++++++++++++++----- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/script-archive/entity-server-filter-example.js b/script-archive/entity-server-filter-example.js index 4d4f7273f1..424e6e9668 100644 --- a/script-archive/entity-server-filter-example.js +++ b/script-archive/entity-server-filter-example.js @@ -4,51 +4,75 @@ function filter(p) { /* Simple example: if someone specifies name, add an 'x' to it. Note that print is ok to use. */ if (p.name) {p.name += 'x'; print('fixme name', p. name);} + /* This example clamps y. A better filter would probably zero y component of velocity and acceleration. */ if (p.position) {p.position.y = Math.min(1, p.position.y); print('fixme p.y', p.position.y);} + /* Can also reject altogether */ if (p.userData) { return false; } + /* Reject if modifications made to Model properties */ if (p.modelURL || p.compoundShapeURL || p.shape || p.shapeType || p.url || p.fps || p.currentFrame || p.running || p.loop || p.firstFrame || p.lastFrame || p.hold || p.textures || p.xTextureURL || p.yTextureURL || p.zTextureURL) { return false; } + /* Clamp velocity to maxVelocity units/second. Zeroing each component of acceleration keeps us from slamming.*/ var maxVelocity = 5; + function sign(val) { + if (val > 0) { + return 1; + } else if (val < 0) { + return -1; + } else { + return 0; + } + } + /* Random near-zero value used as "zero" to prevent two sequential updates from being + exactly the same (which would cause them to be ignored) */ + var nearZero = 0.0001 * Math.random() + 0.001; if (p.velocity) { if (Math.abs(p.velocity.x) > maxVelocity) { - p.velocity.x = Math.sign(p.velocity.x) * maxVelocity; - p.acceleration.x = 0; + p.velocity.x = sign(p.velocity.x) * (maxVelocity + nearZero); + p.acceleration.x = nearZero; } if (Math.abs(p.velocity.y) > maxVelocity) { - p.velocity.y = Math.sign(p.velocity.y) * maxVelocity; - p.acceleration.y = 0; + p.velocity.y = sign(p.velocity.y) * (maxVelocity + nearZero); + p.acceleration.y = nearZero; } if (Math.abs(p.velocity.z) > maxVelocity) { - p.velocity.z = Math.sign(p.velocity.z) * maxVelocity; - p.acceleration.z = 0; + p.velocity.z = sign(p.velocity.z) * (maxVelocity + nearZero); + p.acceleration.z = nearZero; } } + /* Define an axis-aligned zone in which entities are not allowed to enter. */ /* This example zone corresponds to an area to the right of the spawnpoint in your Sandbox. It's an area near the big rock to the right. If an entity enters the zone, it'll move behind the rock.*/ - var boxMin = {x: 25.5, y: -0.48, z: -9.9}; - var boxMax = {x: 31.1, y: 4, z: -3.79}; - var zero = {x: 0.0, y: 0.0, z: 0.0}; - if (p.position) { + /* Random near-zero value used as "zero" to prevent two sequential updates from being + exactly the same (which would cause them to be ignored) */ + var nearZero = 0.0001 * Math.random() + 0.001; + /* Define the points that create the "NO ENTITIES ALLOWED" box */ + var boxMin = {x: 25.5, y: -0.48, z: -9.9}; + var boxMax = {x: 31.1, y: 4, z: -3.79}; + /* Define the point that you want entites that enter the box to appear */ + var resetPoint = {x: 29.5, y: 0.37 + nearZero, z: -2}; var x = p.position.x; var y = p.position.y; var z = p.position.z; if ((x > boxMin.x && x < boxMax.x) && (y > boxMin.y && y < boxMax.y) && (z > boxMin.z && z < boxMax.z)) { - /* Move it to the origin of the zone */ - p.position = boxMin; - p.velocity = zero; - p.acceleration = zero; + p.position = resetPoint; + if (p.velocity) { + p.velocity = {x: 0, y: nearZero, z: 0}; + } + if (p.acceleration) { + p.acceleration = {x: 0, y: nearZero, z: 0}; + } } } From 54b4612ee3cf267f6edb386b49c4280d1f4d1e26 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Fri, 23 Dec 2016 10:36:13 -0800 Subject: [PATCH 30/45] fix for missed START collision events --- libraries/physics/src/ContactInfo.cpp | 10 +++++----- libraries/physics/src/ContactInfo.h | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/libraries/physics/src/ContactInfo.cpp b/libraries/physics/src/ContactInfo.cpp index c2ea6e8671..59948db671 100644 --- a/libraries/physics/src/ContactInfo.cpp +++ b/libraries/physics/src/ContactInfo.cpp @@ -13,15 +13,15 @@ void ContactInfo::update(uint32_t currentStep, const btManifoldPoint& p) { _lastStep = currentStep; - ++_numSteps; positionWorldOnB = p.m_positionWorldOnB; normalWorldOnB = p.m_normalWorldOnB; distance = p.m_distance1; -} +} ContactEventType ContactInfo::computeType(uint32_t thisStep) { - if (_lastStep != thisStep) { - return CONTACT_EVENT_TYPE_END; + ++_numChecks; + if (_numChecks == 1) { + return CONTACT_EVENT_TYPE_START; } - return (_numSteps == 1) ? CONTACT_EVENT_TYPE_START : CONTACT_EVENT_TYPE_CONTINUE; + return (_lastStep == thisStep) ? CONTACT_EVENT_TYPE_CONTINUE : CONTACT_EVENT_TYPE_END; } diff --git a/libraries/physics/src/ContactInfo.h b/libraries/physics/src/ContactInfo.h index 11c908a414..17356969d1 100644 --- a/libraries/physics/src/ContactInfo.h +++ b/libraries/physics/src/ContactInfo.h @@ -19,7 +19,7 @@ class ContactInfo { -public: +public: void update(uint32_t currentStep, const btManifoldPoint& p); ContactEventType computeType(uint32_t thisStep); @@ -30,9 +30,9 @@ public: btVector3 normalWorldOnB; btScalar distance; private: - uint32_t _lastStep = 0; - uint32_t _numSteps = 0; -}; + uint32_t _lastStep { 0 }; + uint32_t _numChecks { 0 }; +}; #endif // hifi_ContactEvent_h From ed17c4fa16a1417a493dba09ed0945cc217a9953 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Fri, 23 Dec 2016 16:55:39 -0800 Subject: [PATCH 31/45] fix an old typo about when to send collision event --- libraries/entities-renderer/src/EntityTreeRenderer.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libraries/entities-renderer/src/EntityTreeRenderer.cpp b/libraries/entities-renderer/src/EntityTreeRenderer.cpp index d277fd540f..9a3b66ff1f 100644 --- a/libraries/entities-renderer/src/EntityTreeRenderer.cpp +++ b/libraries/entities-renderer/src/EntityTreeRenderer.cpp @@ -957,7 +957,8 @@ void EntityTreeRenderer::checkAndCallPreload(const EntityItemID& entityID, const } bool EntityTreeRenderer::isCollisionOwner(const QUuid& myNodeID, EntityTreePointer entityTree, - const EntityItemID& id, const Collision& collision) { + const EntityItemID& id, const Collision& collision) { + // BUG: this method is poorly named. It should be called like: isOwnerOfObjectOrOwnerOfOtherIfObjectIsUnowned() EntityItemPointer entity = entityTree->findEntityByEntityItemID(id); if (!entity) { return false; @@ -965,7 +966,7 @@ bool EntityTreeRenderer::isCollisionOwner(const QUuid& myNodeID, EntityTreePoint QUuid simulatorID = entity->getSimulatorID(); if (simulatorID.isNull()) { // Can be null if it has never moved since being created or coming out of persistence. - // However, for there to be a collission, one of the two objects must be moving. + // However, for there to be a collision, one of the two objects must be moving. const EntityItemID& otherID = (id == collision.idA) ? collision.idB : collision.idA; EntityItemPointer otherEntity = entityTree->findEntityByEntityItemID(otherID); if (!otherEntity) { @@ -1054,6 +1055,8 @@ void EntityTreeRenderer::entityCollisionWithEntity(const EntityItemID& idA, cons playEntityCollisionSound(myNodeID, entityTree, idB, collision); // And now the entity scripts + // BUG! scripts don't get the final COLLISION_EVENT_TYPE_END event in a timely manner because + // by the time it gets here the object has been deactivated and local ownership is relenquished. if (isCollisionOwner(myNodeID, entityTree, idA, collision)) { emit collisionWithEntity(idA, idB, collision); if (_entitiesScriptEngine) { From b7cd8827f93e752bc512243c33b6e8f7562fcaaf Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Fri, 30 Dec 2016 14:11:24 -0800 Subject: [PATCH 32/45] collision events for owned objects only also: fewer entityID lookups for scripted collision sounds and events --- .../src/EntityTreeRenderer.cpp | 63 +++++++++---------- .../src/EntityTreeRenderer.h | 6 +- 2 files changed, 30 insertions(+), 39 deletions(-) diff --git a/libraries/entities-renderer/src/EntityTreeRenderer.cpp b/libraries/entities-renderer/src/EntityTreeRenderer.cpp index 9a3b66ff1f..0e7fbee5c1 100644 --- a/libraries/entities-renderer/src/EntityTreeRenderer.cpp +++ b/libraries/entities-renderer/src/EntityTreeRenderer.cpp @@ -956,6 +956,7 @@ void EntityTreeRenderer::checkAndCallPreload(const EntityItemID& entityID, const } } +/* bool EntityTreeRenderer::isCollisionOwner(const QUuid& myNodeID, EntityTreePointer entityTree, const EntityItemID& id, const Collision& collision) { // BUG: this method is poorly named. It should be called like: isOwnerOfObjectOrOwnerOfOtherIfObjectIsUnowned() @@ -982,43 +983,35 @@ bool EntityTreeRenderer::isCollisionOwner(const QUuid& myNodeID, EntityTreePoint return true; } -void EntityTreeRenderer::playEntityCollisionSound(const QUuid& myNodeID, EntityTreePointer entityTree, - const EntityItemID& id, const Collision& collision) { +bool EntityTreeRenderer::isCollisionOwner(const QUuid& myNodeID, EntityTreePointer entityTree, const EntityItemID& id) { + EntityItemPointer entity = entityTree->findEntityByEntityItemID(id); + return ((bool)entity && myNodeID == entity->getSimulatorID()); +} +*/ - if (!isCollisionOwner(myNodeID, entityTree, id, collision)) { - return; - } - - SharedSoundPointer collisionSound; - float mass = 1.0; // value doesn't get used, but set it so compiler is quiet - AACube minAACube; - bool success = false; - _tree->withReadLock([&] { - EntityItemPointer entity = entityTree->findEntityByEntityItemID(id); - if (entity) { - collisionSound = entity->getCollisionSound(); - mass = entity->computeMass(); - minAACube = entity->getMinimumAACube(success); - } - }); - if (!success) { - return; - } +void EntityTreeRenderer::playEntityCollisionSound(EntityItemPointer entity, const Collision& collision) { + assert((bool)entity); + SharedSoundPointer collisionSound = entity->getCollisionSound(); if (!collisionSound) { return; } + bool success = false; + AACube minAACube = entity->getMinimumAACube(success); + if (!success) { + return; + } + float mass = entity->computeMass(); - const float COLLISION_PENETRATION_TO_VELOCITY = 50; // as a subsitute for RELATIVE entity->getVelocity() + const float COLLISION_PENETRATION_TO_VELOCITY = 50.0f; // as a subsitute for RELATIVE entity->getVelocity() // The collision.penetration is a pretty good indicator of changed velocity AFTER the initial contact, // but that first contact depends on exactly where we hit in the physics step. // We can get a more consistent initial-contact energy reading by using the changed velocity. // Note that velocityChange is not a good indicator for continuing collisions, because it does not distinguish // between bounce and sliding along a surface. - const float linearVelocity = (collision.type == CONTACT_EVENT_TYPE_START) ? - glm::length(collision.velocityChange) : - glm::length(collision.penetration) * COLLISION_PENETRATION_TO_VELOCITY; - const float energy = mass * linearVelocity * linearVelocity / 2.0f; - const glm::vec3 position = collision.contactPoint; + const float speedSquared = (collision.type == CONTACT_EVENT_TYPE_START) ? + glm::length2(collision.velocityChange) : + glm::length2(collision.penetration) * COLLISION_PENETRATION_TO_VELOCITY; + const float energy = mass * speedSquared / 2.0f; const float COLLISION_ENERGY_AT_FULL_VOLUME = (collision.type == CONTACT_EVENT_TYPE_START) ? 150.0f : 5.0f; const float COLLISION_MINIMUM_VOLUME = 0.005f; const float energyFactorOfFull = fmin(1.0f, energy / COLLISION_ENERGY_AT_FULL_VOLUME); @@ -1032,7 +1025,7 @@ void EntityTreeRenderer::playEntityCollisionSound(const QUuid& myNodeID, EntityT // Shift the pitch down by ln(1 + (size / COLLISION_SIZE_FOR_STANDARD_PITCH)) / ln(2) const float COLLISION_SIZE_FOR_STANDARD_PITCH = 0.2f; const float stretchFactor = log(1.0f + (minAACube.getLargestDimension() / COLLISION_SIZE_FOR_STANDARD_PITCH)) / log(2); - AudioInjector::playSound(collisionSound, volume, stretchFactor, position); + AudioInjector::playSound(collisionSound, volume, stretchFactor, collision.contactPoint); } void EntityTreeRenderer::entityCollisionWithEntity(const EntityItemID& idA, const EntityItemID& idB, @@ -1048,23 +1041,23 @@ void EntityTreeRenderer::entityCollisionWithEntity(const EntityItemID& idA, cons return; } - // See if we should play sounds EntityTreePointer entityTree = std::static_pointer_cast(_tree); const QUuid& myNodeID = DependencyManager::get()->getSessionUUID(); - playEntityCollisionSound(myNodeID, entityTree, idA, collision); - playEntityCollisionSound(myNodeID, entityTree, idB, collision); - // And now the entity scripts + // trigger scripted collision sounds and events for locally owned objects // BUG! scripts don't get the final COLLISION_EVENT_TYPE_END event in a timely manner because // by the time it gets here the object has been deactivated and local ownership is relenquished. - if (isCollisionOwner(myNodeID, entityTree, idA, collision)) { + EntityItemPointer entityA = entityTree->findEntityByEntityItemID(idA); + if ((bool)entityA && myNodeID == entityA->getSimulatorID()) { + playEntityCollisionSound(entityA, collision); emit collisionWithEntity(idA, idB, collision); if (_entitiesScriptEngine) { _entitiesScriptEngine->callEntityScriptMethod(idA, "collisionWithEntity", idB, collision); } } - - if (isCollisionOwner(myNodeID, entityTree, idB, collision)) { + EntityItemPointer entityB = entityTree->findEntityByEntityItemID(idB); + if ((bool)entityB && myNodeID == entityB->getSimulatorID()) { + playEntityCollisionSound(entityB, collision); emit collisionWithEntity(idB, idA, collision); if (_entitiesScriptEngine) { _entitiesScriptEngine->callEntityScriptMethod(idB, "collisionWithEntity", idA, collision); diff --git a/libraries/entities-renderer/src/EntityTreeRenderer.h b/libraries/entities-renderer/src/EntityTreeRenderer.h index 8c021ad184..a0673207f9 100644 --- a/libraries/entities-renderer/src/EntityTreeRenderer.h +++ b/libraries/entities-renderer/src/EntityTreeRenderer.h @@ -170,11 +170,9 @@ private: bool _wantScripts; QSharedPointer _entitiesScriptEngine; - bool isCollisionOwner(const QUuid& myNodeID, EntityTreePointer entityTree, - const EntityItemID& id, const Collision& collision); + //static bool isCollisionOwner(const QUuid& myNodeID, EntityTreePointer entityTree, const EntityItemID& id); - void playEntityCollisionSound(const QUuid& myNodeID, EntityTreePointer entityTree, - const EntityItemID& id, const Collision& collision); + static void playEntityCollisionSound(EntityItemPointer entity, const Collision& collision); bool _lastPointerEventValid; PointerEvent _lastPointerEvent; From 8cf7aee0092d576c4520c6f5e6c6e7c5e5606edf Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Fri, 30 Dec 2016 14:13:11 -0800 Subject: [PATCH 33/45] fix bug: second collision event with bad data --- libraries/entities-renderer/src/EntityTreeRenderer.cpp | 5 ++++- libraries/shared/src/RegisteredMetaTypes.cpp | 6 ++++++ libraries/shared/src/RegisteredMetaTypes.h | 6 ++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/libraries/entities-renderer/src/EntityTreeRenderer.cpp b/libraries/entities-renderer/src/EntityTreeRenderer.cpp index 0e7fbee5c1..da06a07552 100644 --- a/libraries/entities-renderer/src/EntityTreeRenderer.cpp +++ b/libraries/entities-renderer/src/EntityTreeRenderer.cpp @@ -1060,7 +1060,10 @@ void EntityTreeRenderer::entityCollisionWithEntity(const EntityItemID& idA, cons playEntityCollisionSound(entityB, collision); emit collisionWithEntity(idB, idA, collision); if (_entitiesScriptEngine) { - _entitiesScriptEngine->callEntityScriptMethod(idB, "collisionWithEntity", idA, collision); + // since we're swapping A and B we need to send the inverted collision + Collision invertedCollision(collision); + invertedCollision.invert(); + _entitiesScriptEngine->callEntityScriptMethod(idB, "collisionWithEntity", idA, invertedCollision); } } } diff --git a/libraries/shared/src/RegisteredMetaTypes.cpp b/libraries/shared/src/RegisteredMetaTypes.cpp index 984529c4ba..7f12d6cc00 100644 --- a/libraries/shared/src/RegisteredMetaTypes.cpp +++ b/libraries/shared/src/RegisteredMetaTypes.cpp @@ -742,6 +742,12 @@ void collisionFromScriptValue(const QScriptValue &object, Collision& collision) // TODO: implement this when we know what it means to accept collision events from JS } +void Collision::invert() { + std::swap(idA, idB); + contactPoint += penetration; + penetration *= -1.0f; +} + QScriptValue quuidToScriptValue(QScriptEngine* engine, const QUuid& uuid) { if (uuid.isNull()) { return QScriptValue::NullValue; diff --git a/libraries/shared/src/RegisteredMetaTypes.h b/libraries/shared/src/RegisteredMetaTypes.h index 2aefd3aa47..498a8b3b3a 100644 --- a/libraries/shared/src/RegisteredMetaTypes.h +++ b/libraries/shared/src/RegisteredMetaTypes.h @@ -142,11 +142,13 @@ public: const glm::vec3& cPenetration, const glm::vec3& velocityChange) : type(cType), idA(cIdA), idB(cIdB), contactPoint(cPoint), penetration(cPenetration), velocityChange(velocityChange) { } + void invert(); // swap A and B + ContactEventType type; QUuid idA; QUuid idB; - glm::vec3 contactPoint; - glm::vec3 penetration; + glm::vec3 contactPoint; // on B in world-frame + glm::vec3 penetration; // from B towards A in world-frame glm::vec3 velocityChange; }; Q_DECLARE_METATYPE(Collision) From 2162a364a93faadd564df779c5e0e7efcbeecf83 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Fri, 30 Dec 2016 16:04:28 -0800 Subject: [PATCH 34/45] minor cleanup --- libraries/physics/src/PhysicsEngine.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/physics/src/PhysicsEngine.cpp b/libraries/physics/src/PhysicsEngine.cpp index ba002d925c..f8e02a14b8 100644 --- a/libraries/physics/src/PhysicsEngine.cpp +++ b/libraries/physics/src/PhysicsEngine.cpp @@ -402,7 +402,7 @@ const CollisionEvents& PhysicsEngine::getCollisionEvents() { while (contactItr != _contactMap.end()) { ContactInfo& contact = contactItr->second; ContactEventType type = contact.computeType(_numContactFrames); - if(type != CONTACT_EVENT_TYPE_CONTINUE || _numSubsteps % CONTINUE_EVENT_FILTER_FREQUENCY == 0) { + if (type != CONTACT_EVENT_TYPE_CONTINUE || _numSubsteps % CONTINUE_EVENT_FILTER_FREQUENCY == 0) { ObjectMotionState* motionStateA = static_cast(contactItr->first._a); ObjectMotionState* motionStateB = static_cast(contactItr->first._b); glm::vec3 velocityChange = (motionStateA ? motionStateA->getObjectLinearVelocityChange() : glm::vec3(0.0f)) + @@ -421,7 +421,7 @@ const CollisionEvents& PhysicsEngine::getCollisionEvents() { QUuid idB = motionStateB->getObjectID(); glm::vec3 position = bulletToGLM(contact.getPositionWorldOnA()) + _originOffset; // NOTE: we're flipping the order of A and B (so that the first objectID is never NULL) - // hence we must negate the penetration. + // hence we must negate the penetration (because penetration always points from B to A). glm::vec3 penetration = - bulletToGLM(contact.distance * contact.normalWorldOnB); _collisionEvents.push_back(Collision(type, idB, QUuid(), position, penetration, velocityChange)); } From 0809149a8c9ba30ada618da126d745727b293cda Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Tue, 10 Jan 2017 19:36:06 -0800 Subject: [PATCH 35/45] harvest collision events before disowning --- interface/src/Application.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 576241ddc2..77f5487986 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -4377,6 +4377,10 @@ void Application::update(float deltaTime) { PROFILE_RANGE_EX(simulation_physics, "HarvestChanges", 0xffffff00, (uint64_t)getActiveDisplayPlugin()->presentCount()); PerformanceTimer perfTimer("harvestChanges"); if (_physicsEngine->hasOutgoingChanges()) { + // grab the collision events BEFORE handleOutgoingChanges() because at this point + // we have a better idea of which objects we own or should own. + auto& collisionEvents = _physicsEngine->getCollisionEvents(); + getEntities()->getTree()->withWriteLock([&] { PerformanceTimer perfTimer("handleOutgoingChanges"); const VectorOfMotionStates& outgoingChanges = _physicsEngine->getOutgoingChanges(); @@ -4384,11 +4388,10 @@ void Application::update(float deltaTime) { avatarManager->handleOutgoingChanges(outgoingChanges); }); - auto collisionEvents = _physicsEngine->getCollisionEvents(); - avatarManager->handleCollisionEvents(collisionEvents); - if (!_aboutToQuit) { + // handleCollisionEvents() AFTER handleOutgoinChanges() PerformanceTimer perfTimer("entities"); + avatarManager->handleCollisionEvents(collisionEvents); // Collision events (and their scripts) must not be handled when we're locked, above. (That would risk // deadlock.) _entitySimulation->handleCollisionEvents(collisionEvents); From 2541bfb1a8fdf4a81a1ccf83e6b426e6bbbe6d0b Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Tue, 10 Jan 2017 20:19:09 -0800 Subject: [PATCH 36/45] only create collision events for owned entities --- .../src/EntityTreeRenderer.cpp | 2 -- libraries/physics/src/EntityMotionState.cpp | 5 ++++ libraries/physics/src/EntityMotionState.h | 2 ++ libraries/physics/src/ObjectMotionState.h | 2 ++ libraries/physics/src/PhysicsEngine.cpp | 23 ++++++++++++++----- 5 files changed, 26 insertions(+), 8 deletions(-) diff --git a/libraries/entities-renderer/src/EntityTreeRenderer.cpp b/libraries/entities-renderer/src/EntityTreeRenderer.cpp index da06a07552..c56d55286e 100644 --- a/libraries/entities-renderer/src/EntityTreeRenderer.cpp +++ b/libraries/entities-renderer/src/EntityTreeRenderer.cpp @@ -1045,8 +1045,6 @@ void EntityTreeRenderer::entityCollisionWithEntity(const EntityItemID& idA, cons const QUuid& myNodeID = DependencyManager::get()->getSessionUUID(); // trigger scripted collision sounds and events for locally owned objects - // BUG! scripts don't get the final COLLISION_EVENT_TYPE_END event in a timely manner because - // by the time it gets here the object has been deactivated and local ownership is relenquished. EntityItemPointer entityA = entityTree->findEntityByEntityItemID(idA); if ((bool)entityA && myNodeID == entityA->getSimulatorID()) { playEntityCollisionSound(entityA, collision); diff --git a/libraries/physics/src/EntityMotionState.cpp b/libraries/physics/src/EntityMotionState.cpp index 1833d0aba4..b0bdc34b52 100644 --- a/libraries/physics/src/EntityMotionState.cpp +++ b/libraries/physics/src/EntityMotionState.cpp @@ -762,6 +762,11 @@ void EntityMotionState::computeCollisionGroupAndMask(int16_t& group, int16_t& ma _entity->computeCollisionGroupAndFinalMask(group, mask); } +bool EntityMotionState::shouldBeLocallyOwned() const { + return (_outgoingPriority > VOLUNTEER_SIMULATION_PRIORITY && _outgoingPriority > _entity->getSimulationPriority()) || + _entity->getSimulatorID() == Physics::getSessionUUID(); +} + void EntityMotionState::upgradeOutgoingPriority(uint8_t priority) { _outgoingPriority = glm::max(_outgoingPriority, priority); } diff --git a/libraries/physics/src/EntityMotionState.h b/libraries/physics/src/EntityMotionState.h index 194d82805f..feac47d8ec 100644 --- a/libraries/physics/src/EntityMotionState.h +++ b/libraries/physics/src/EntityMotionState.h @@ -78,6 +78,8 @@ public: virtual void computeCollisionGroupAndMask(int16_t& group, int16_t& mask) const override; + bool shouldBeLocallyOwned() const override; + friend class PhysicalEntitySimulation; protected: diff --git a/libraries/physics/src/ObjectMotionState.h b/libraries/physics/src/ObjectMotionState.h index a7894998a8..1d258560c3 100644 --- a/libraries/physics/src/ObjectMotionState.h +++ b/libraries/physics/src/ObjectMotionState.h @@ -146,6 +146,8 @@ public: void dirtyInternalKinematicChanges() { _hasInternalKinematicChanges = true; } void clearInternalKinematicChanges() { _hasInternalKinematicChanges = false; } + virtual bool shouldBeLocallyOwned() const { return false; } + friend class PhysicsEngine; protected: diff --git a/libraries/physics/src/PhysicsEngine.cpp b/libraries/physics/src/PhysicsEngine.cpp index f8e02a14b8..5b638b9a98 100644 --- a/libraries/physics/src/PhysicsEngine.cpp +++ b/libraries/physics/src/PhysicsEngine.cpp @@ -405,25 +405,36 @@ const CollisionEvents& PhysicsEngine::getCollisionEvents() { if (type != CONTACT_EVENT_TYPE_CONTINUE || _numSubsteps % CONTINUE_EVENT_FILTER_FREQUENCY == 0) { ObjectMotionState* motionStateA = static_cast(contactItr->first._a); ObjectMotionState* motionStateB = static_cast(contactItr->first._b); - glm::vec3 velocityChange = (motionStateA ? motionStateA->getObjectLinearVelocityChange() : glm::vec3(0.0f)) + - (motionStateB ? motionStateB->getObjectLinearVelocityChange() : glm::vec3(0.0f)); - if (motionStateA) { + // NOTE: the MyAvatar RigidBody is the only object in the simulation that does NOT have a MotionState + // which means should we ever want to report ALL collision events against the avatar we can + // modify the logic below. + // + // We only create events when at least one of the objects is (or should be) owned in the local simulation. + if (motionStateA && (motionStateA->shouldBeLocallyOwned())) { QUuid idA = motionStateA->getObjectID(); QUuid idB; if (motionStateB) { idB = motionStateB->getObjectID(); } glm::vec3 position = bulletToGLM(contact.getPositionWorldOnB()) + _originOffset; + glm::vec3 velocityChange = motionStateA->getObjectLinearVelocityChange() + + (motionStateB ? motionStateB->getObjectLinearVelocityChange() : glm::vec3(0.0f)); glm::vec3 penetration = bulletToGLM(contact.distance * contact.normalWorldOnB); _collisionEvents.push_back(Collision(type, idA, idB, position, penetration, velocityChange)); - } else if (motionStateB) { + } else if (motionStateB && (motionStateB->shouldBeLocallyOwned())) { QUuid idB = motionStateB->getObjectID(); + QUuid idA; + if (motionStateA) { + idA = motionStateA->getObjectID(); + } glm::vec3 position = bulletToGLM(contact.getPositionWorldOnA()) + _originOffset; + glm::vec3 velocityChange = motionStateB->getObjectLinearVelocityChange() + + (motionStateA ? motionStateA->getObjectLinearVelocityChange() : glm::vec3(0.0f)); // NOTE: we're flipping the order of A and B (so that the first objectID is never NULL) - // hence we must negate the penetration (because penetration always points from B to A). + // hence we negate the penetration (because penetration always points from B to A). glm::vec3 penetration = - bulletToGLM(contact.distance * contact.normalWorldOnB); - _collisionEvents.push_back(Collision(type, idB, QUuid(), position, penetration, velocityChange)); + _collisionEvents.push_back(Collision(type, idB, idA, position, penetration, velocityChange)); } } From b5537304a3366c08ee30bb69cacbb0b2cba57852 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Tue, 10 Jan 2017 21:29:08 -0800 Subject: [PATCH 37/45] more correct CONTINUE collision event filter --- libraries/physics/src/ContactInfo.cpp | 14 ++++++++++++-- libraries/physics/src/ContactInfo.h | 4 +++- libraries/physics/src/PhysicsEngine.cpp | 5 ++--- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/libraries/physics/src/ContactInfo.cpp b/libraries/physics/src/ContactInfo.cpp index 59948db671..085f746a73 100644 --- a/libraries/physics/src/ContactInfo.cpp +++ b/libraries/physics/src/ContactInfo.cpp @@ -18,10 +18,20 @@ void ContactInfo::update(uint32_t currentStep, const btManifoldPoint& p) { distance = p.m_distance1; } +const uint32_t STEPS_BETWEEN_CONTINUE_EVENTS = 9; + ContactEventType ContactInfo::computeType(uint32_t thisStep) { - ++_numChecks; - if (_numChecks == 1) { + if (_continueExpiry == 0) { + _continueExpiry = thisStep + STEPS_BETWEEN_CONTINUE_EVENTS; return CONTACT_EVENT_TYPE_START; } return (_lastStep == thisStep) ? CONTACT_EVENT_TYPE_CONTINUE : CONTACT_EVENT_TYPE_END; } + +bool ContactInfo::readyForContinue(uint32_t thisStep) { + if (thisStep > _continueExpiry) { + _continueExpiry = thisStep + STEPS_BETWEEN_CONTINUE_EVENTS; + return true; + } + return false; +} diff --git a/libraries/physics/src/ContactInfo.h b/libraries/physics/src/ContactInfo.h index 17356969d1..8d05f73b61 100644 --- a/libraries/physics/src/ContactInfo.h +++ b/libraries/physics/src/ContactInfo.h @@ -26,12 +26,14 @@ public: const btVector3& getPositionWorldOnB() const { return positionWorldOnB; } btVector3 getPositionWorldOnA() const { return positionWorldOnB + normalWorldOnB * distance; } + bool readyForContinue(uint32_t thisStep); + btVector3 positionWorldOnB; btVector3 normalWorldOnB; btScalar distance; private: uint32_t _lastStep { 0 }; - uint32_t _numChecks { 0 }; + uint32_t _continueExpiry { 0 }; }; diff --git a/libraries/physics/src/PhysicsEngine.cpp b/libraries/physics/src/PhysicsEngine.cpp index 5b638b9a98..72596cb599 100644 --- a/libraries/physics/src/PhysicsEngine.cpp +++ b/libraries/physics/src/PhysicsEngine.cpp @@ -270,7 +270,7 @@ void PhysicsEngine::stepSimulation() { } auto onSubStep = [this]() { - updateContactMap(); + this->updateContactMap(); }; int numSubsteps = _dynamicsWorld->stepSimulationWithSubstepCallback(timeStep, PHYSICS_ENGINE_MAX_NUM_SUBSTEPS, @@ -393,7 +393,6 @@ void PhysicsEngine::updateContactMap() { } const CollisionEvents& PhysicsEngine::getCollisionEvents() { - const uint32_t CONTINUE_EVENT_FILTER_FREQUENCY = 10; _collisionEvents.clear(); // scan known contacts and trigger events @@ -402,7 +401,7 @@ const CollisionEvents& PhysicsEngine::getCollisionEvents() { while (contactItr != _contactMap.end()) { ContactInfo& contact = contactItr->second; ContactEventType type = contact.computeType(_numContactFrames); - if (type != CONTACT_EVENT_TYPE_CONTINUE || _numSubsteps % CONTINUE_EVENT_FILTER_FREQUENCY == 0) { + if (type != CONTACT_EVENT_TYPE_CONTINUE || contact.readyForContinue(_numContactFrames)) { ObjectMotionState* motionStateA = static_cast(contactItr->first._a); ObjectMotionState* motionStateB = static_cast(contactItr->first._b); From aa8e7d27dbdf654c7394faa45fa87667fd961c62 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Tue, 10 Jan 2017 21:53:26 -0800 Subject: [PATCH 38/45] move depth filtering closer to source --- libraries/entities-renderer/src/EntityTreeRenderer.cpp | 5 ----- libraries/physics/src/PhysicsEngine.cpp | 5 ++++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/libraries/entities-renderer/src/EntityTreeRenderer.cpp b/libraries/entities-renderer/src/EntityTreeRenderer.cpp index c56d55286e..1a9b327adc 100644 --- a/libraries/entities-renderer/src/EntityTreeRenderer.cpp +++ b/libraries/entities-renderer/src/EntityTreeRenderer.cpp @@ -1035,11 +1035,6 @@ void EntityTreeRenderer::entityCollisionWithEntity(const EntityItemID& idA, cons if (!_tree || _shuttingDown) { return; } - // Don't respond to small continuous contacts. - const float COLLISION_MINUMUM_PENETRATION = 0.002f; - if ((collision.type == CONTACT_EVENT_TYPE_CONTINUE) && (glm::length(collision.penetration) < COLLISION_MINUMUM_PENETRATION)) { - return; - } EntityTreePointer entityTree = std::static_pointer_cast(_tree); const QUuid& myNodeID = DependencyManager::get()->getSessionUUID(); diff --git a/libraries/physics/src/PhysicsEngine.cpp b/libraries/physics/src/PhysicsEngine.cpp index 72596cb599..f57be4eab3 100644 --- a/libraries/physics/src/PhysicsEngine.cpp +++ b/libraries/physics/src/PhysicsEngine.cpp @@ -401,7 +401,10 @@ const CollisionEvents& PhysicsEngine::getCollisionEvents() { while (contactItr != _contactMap.end()) { ContactInfo& contact = contactItr->second; ContactEventType type = contact.computeType(_numContactFrames); - if (type != CONTACT_EVENT_TYPE_CONTINUE || contact.readyForContinue(_numContactFrames)) { + const btScalar SIGNIFICANT_DEPTH = -0.002f; // penetrations have negative distance + if (type != CONTACT_EVENT_TYPE_CONTINUE || + (contact.distance < SIGNIFICANT_DEPTH && + contact.readyForContinue(_numContactFrames))) { ObjectMotionState* motionStateA = static_cast(contactItr->first._a); ObjectMotionState* motionStateB = static_cast(contactItr->first._b); From 31861d31920631583494daa9e64215ad9eb74f15 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Wed, 11 Jan 2017 08:24:53 -0800 Subject: [PATCH 39/45] use inverted collision for B-A event --- libraries/entities-renderer/src/EntityTreeRenderer.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/entities-renderer/src/EntityTreeRenderer.cpp b/libraries/entities-renderer/src/EntityTreeRenderer.cpp index 1a9b327adc..a1f9d6d414 100644 --- a/libraries/entities-renderer/src/EntityTreeRenderer.cpp +++ b/libraries/entities-renderer/src/EntityTreeRenderer.cpp @@ -1051,11 +1051,11 @@ void EntityTreeRenderer::entityCollisionWithEntity(const EntityItemID& idA, cons EntityItemPointer entityB = entityTree->findEntityByEntityItemID(idB); if ((bool)entityB && myNodeID == entityB->getSimulatorID()) { playEntityCollisionSound(entityB, collision); - emit collisionWithEntity(idB, idA, collision); + // since we're swapping A and B we need to send the inverted collision + Collision invertedCollision(collision); + invertedCollision.invert(); + emit collisionWithEntity(idB, idA, invertedCollision); if (_entitiesScriptEngine) { - // since we're swapping A and B we need to send the inverted collision - Collision invertedCollision(collision); - invertedCollision.invert(); _entitiesScriptEngine->callEntityScriptMethod(idB, "collisionWithEntity", idA, invertedCollision); } } From 6ef4420f3738da8d089c98c850b12734897826c6 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Wed, 11 Jan 2017 08:26:49 -0800 Subject: [PATCH 40/45] remove commented out cruft --- .../src/EntityTreeRenderer.cpp | 33 ------------------- .../src/EntityTreeRenderer.h | 2 -- 2 files changed, 35 deletions(-) diff --git a/libraries/entities-renderer/src/EntityTreeRenderer.cpp b/libraries/entities-renderer/src/EntityTreeRenderer.cpp index a1f9d6d414..60bb29f85f 100644 --- a/libraries/entities-renderer/src/EntityTreeRenderer.cpp +++ b/libraries/entities-renderer/src/EntityTreeRenderer.cpp @@ -956,39 +956,6 @@ void EntityTreeRenderer::checkAndCallPreload(const EntityItemID& entityID, const } } -/* -bool EntityTreeRenderer::isCollisionOwner(const QUuid& myNodeID, EntityTreePointer entityTree, - const EntityItemID& id, const Collision& collision) { - // BUG: this method is poorly named. It should be called like: isOwnerOfObjectOrOwnerOfOtherIfObjectIsUnowned() - EntityItemPointer entity = entityTree->findEntityByEntityItemID(id); - if (!entity) { - return false; - } - QUuid simulatorID = entity->getSimulatorID(); - if (simulatorID.isNull()) { - // Can be null if it has never moved since being created or coming out of persistence. - // However, for there to be a collision, one of the two objects must be moving. - const EntityItemID& otherID = (id == collision.idA) ? collision.idB : collision.idA; - EntityItemPointer otherEntity = entityTree->findEntityByEntityItemID(otherID); - if (!otherEntity) { - return false; - } - simulatorID = otherEntity->getSimulatorID(); - } - - if (simulatorID.isNull() || (simulatorID != myNodeID)) { - return false; - } - - return true; -} - -bool EntityTreeRenderer::isCollisionOwner(const QUuid& myNodeID, EntityTreePointer entityTree, const EntityItemID& id) { - EntityItemPointer entity = entityTree->findEntityByEntityItemID(id); - return ((bool)entity && myNodeID == entity->getSimulatorID()); -} -*/ - void EntityTreeRenderer::playEntityCollisionSound(EntityItemPointer entity, const Collision& collision) { assert((bool)entity); SharedSoundPointer collisionSound = entity->getCollisionSound(); diff --git a/libraries/entities-renderer/src/EntityTreeRenderer.h b/libraries/entities-renderer/src/EntityTreeRenderer.h index a0673207f9..29d463b915 100644 --- a/libraries/entities-renderer/src/EntityTreeRenderer.h +++ b/libraries/entities-renderer/src/EntityTreeRenderer.h @@ -170,8 +170,6 @@ private: bool _wantScripts; QSharedPointer _entitiesScriptEngine; - //static bool isCollisionOwner(const QUuid& myNodeID, EntityTreePointer entityTree, const EntityItemID& id); - static void playEntityCollisionSound(EntityItemPointer entity, const Collision& collision); bool _lastPointerEventValid; From f553656e36d51090dc1ec096a6a7056c01eb051d Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 30 Jan 2017 13:33:42 -0800 Subject: [PATCH 41/45] Fix importing of PolyVox entities PolyVox entities reference neighboring PolyVox entities in their entity properties so that they can be stitched together. When importing, a new ID is generated for each entity. When importing PolyVox entities, the neighboring entity IDs were not updated to reflect the newly generated IDs. This commit fixes that. --- libraries/entities/src/EntityTree.cpp | 40 ++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/libraries/entities/src/EntityTree.cpp b/libraries/entities/src/EntityTree.cpp index 7c3eb7bec3..29c1f1af86 100644 --- a/libraries/entities/src/EntityTree.cpp +++ b/libraries/entities/src/EntityTree.cpp @@ -1552,6 +1552,8 @@ bool EntityTree::sendEntitiesOperation(OctreeElementPointer element, void* extra return args->map->value(oldID); } EntityItemID newID = QUuid::createUuid(); + args->map->insert(oldID, newID); + EntityItemProperties properties = item->getProperties(); EntityItemID oldParentID = properties.getParentID(); if (oldParentID.isInvalidID()) { // no parent @@ -1567,6 +1569,43 @@ bool EntityTree::sendEntitiesOperation(OctreeElementPointer element, void* extra } } + if (!properties.getXNNeighborID().isInvalidID()) { + auto neighborEntity = args->ourTree->findEntityByEntityItemID(properties.getXNNeighborID()); + if (neighborEntity) { + properties.setXNNeighborID(getMapped(neighborEntity)); + } + } + if (!properties.getXPNeighborID().isInvalidID()) { + auto neighborEntity = args->ourTree->findEntityByEntityItemID(properties.getXPNeighborID()); + if (neighborEntity) { + properties.setXPNeighborID(getMapped(neighborEntity)); + } + } + if (!properties.getYNNeighborID().isInvalidID()) { + auto neighborEntity = args->ourTree->findEntityByEntityItemID(properties.getYNNeighborID()); + if (neighborEntity) { + properties.setYNNeighborID(getMapped(neighborEntity)); + } + } + if (!properties.getYPNeighborID().isInvalidID()) { + auto neighborEntity = args->ourTree->findEntityByEntityItemID(properties.getYPNeighborID()); + if (neighborEntity) { + properties.setYPNeighborID(getMapped(neighborEntity)); + } + } + if (!properties.getZNNeighborID().isInvalidID()) { + auto neighborEntity = args->ourTree->findEntityByEntityItemID(properties.getZNNeighborID()); + if (neighborEntity) { + properties.setZNNeighborID(getMapped(neighborEntity)); + } + } + if (!properties.getZPNeighborID().isInvalidID()) { + auto neighborEntity = args->ourTree->findEntityByEntityItemID(properties.getZPNeighborID()); + if (neighborEntity) { + properties.setZPNeighborID(getMapped(neighborEntity)); + } + } + // set creation time to "now" for imported entities properties.setCreated(usecTimestampNow()); @@ -1584,7 +1623,6 @@ bool EntityTree::sendEntitiesOperation(OctreeElementPointer element, void* extra args->otherTree->addEntity(newID, properties); }); } - args->map->insert(oldID, newID); return newID; }; From e58c9326a0f454c715607e34802c5191131582e5 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 30 Jan 2017 13:37:00 -0800 Subject: [PATCH 42/45] Fix Entities.isChildOfParent crashing if given unknown ID If an entity was not in the local tree a null deref crash would occur. This commit makes sure the entity pointer is checked for null before it is used. --- .../entities/src/EntityScriptingInterface.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/libraries/entities/src/EntityScriptingInterface.cpp b/libraries/entities/src/EntityScriptingInterface.cpp index 26754335a0..9ee5e991fb 100644 --- a/libraries/entities/src/EntityScriptingInterface.cpp +++ b/libraries/entities/src/EntityScriptingInterface.cpp @@ -1366,12 +1366,14 @@ bool EntityScriptingInterface::isChildOfParent(QUuid childID, QUuid parentID) { _entityTree->withReadLock([&] { EntityItemPointer parent = _entityTree->findEntityByEntityItemID(parentID); - parent->forEachDescendant([&](SpatiallyNestablePointer descendant) { - if(descendant->getID() == childID) { - isChild = true; - return; - } - }); + if (parent) { + parent->forEachDescendant([&](SpatiallyNestablePointer descendant) { + if (descendant->getID() == childID) { + isChild = true; + return; + } + }); + } }); return isChild; From 740a0add8ad062b8833d38862798fe82a8b5bafb Mon Sep 17 00:00:00 2001 From: Zach Fox Date: Mon, 30 Jan 2017 15:47:04 -0800 Subject: [PATCH 43/45] Updates --- .../entity-server-filter-example.js | 57 +++++++++++++------ 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/script-archive/entity-server-filter-example.js b/script-archive/entity-server-filter-example.js index 424e6e9668..a82670c53d 100644 --- a/script-archive/entity-server-filter-example.js +++ b/script-archive/entity-server-filter-example.js @@ -1,6 +1,17 @@ function filter(p) { - /* block comments are ok, but not double-slash end-of-line-comments */ - + /******************************************************/ + /* General Filter Comments + /* + - Custom filters must be named "filter" and must be global + - Block comments are ok, but not double-slash end-of-line-comments + - Certain JavaScript functions are not available, like Math.sign(), as they are undefined in QT's non-conforming JS + - HiFi's scripting interface is unavailable here. That means you can't call, for example, Users.*() + */ + /******************************************************/ + + /******************************************************/ + /* Simple Filter Examples + /******************************************************/ /* Simple example: if someone specifies name, add an 'x' to it. Note that print is ok to use. */ if (p.name) {p.name += 'x'; print('fixme name', p. name);} @@ -9,7 +20,7 @@ function filter(p) { if (p.position) {p.position.y = Math.min(1, p.position.y); print('fixme p.y', p.position.y);} - /* Can also reject altogether */ + /* Can also reject new properties altogether by returning false */ if (p.userData) { return false; } @@ -17,21 +28,35 @@ function filter(p) { if (p.modelURL || p.compoundShapeURL || p.shape || p.shapeType || p.url || p.fps || p.currentFrame || p.running || p.loop || p.firstFrame || p.lastFrame || p.hold || p.textures || p.xTextureURL || p.yTextureURL || p.zTextureURL) { return false; } + /******************************************************/ + /* Physical Property Filter Examples + /* + NOTES about filtering physical properties: + - For now, ensure you always supply a new value for the filtered physical property + (instead of simply removing the property) + - Ensure you always specify a slightly different value for physical properties every + time your filter returns. Look to "var nearZero" below for an example). + This is necessary because Interface checks if a physical property has changed + when deciding whether to apply or reject the server's physical properties. + If a physical property's value doesn't change, Interface will reject the server's property value, + and Bullet will continue simulating the entity with stale physical properties. + */ + /******************************************************/ /* Clamp velocity to maxVelocity units/second. Zeroing each component of acceleration keeps us from slamming.*/ - var maxVelocity = 5; - function sign(val) { - if (val > 0) { - return 1; - } else if (val < 0) { - return -1; - } else { - return 0; - } - } - /* Random near-zero value used as "zero" to prevent two sequential updates from being - exactly the same (which would cause them to be ignored) */ - var nearZero = 0.0001 * Math.random() + 0.001; if (p.velocity) { + var maxVelocity = 5; + /* Random near-zero value used as "zero" to prevent two sequential updates from being + exactly the same (which would cause them to be ignored) */ + var nearZero = 0.0001 * Math.random() + 0.001; + function sign(val) { + if (val > 0) { + return 1; + } else if (val < 0) { + return -1; + } else { + return 0; + } + } if (Math.abs(p.velocity.x) > maxVelocity) { p.velocity.x = sign(p.velocity.x) * (maxVelocity + nearZero); p.acceleration.x = nearZero; From 18fd965c3369ae313a82cc75ded0591928a8b4d5 Mon Sep 17 00:00:00 2001 From: Zach Fox Date: Mon, 30 Jan 2017 16:07:43 -0800 Subject: [PATCH 44/45] Add note about floats? --- script-archive/entity-server-filter-example.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script-archive/entity-server-filter-example.js b/script-archive/entity-server-filter-example.js index a82670c53d..ad44bf1583 100644 --- a/script-archive/entity-server-filter-example.js +++ b/script-archive/entity-server-filter-example.js @@ -40,6 +40,9 @@ function filter(p) { when deciding whether to apply or reject the server's physical properties. If a physical property's value doesn't change, Interface will reject the server's property value, and Bullet will continue simulating the entity with stale physical properties. + Ensure that this value is not changed by such a small amount such that new values + fall within floating point precision boundaries. If you accidentally do this, prepare for many + hours of frustrating debugging :). */ /******************************************************/ /* Clamp velocity to maxVelocity units/second. Zeroing each component of acceleration keeps us from slamming.*/ From eaf033107c7c7022840d7f4f86ca835b257dfd75 Mon Sep 17 00:00:00 2001 From: David Kelly Date: Mon, 30 Jan 2017 17:13:13 -0700 Subject: [PATCH 45/45] Require password verification in domain-server settings If you modify the security settings to use a username/password for access to the domain server settings, we now have a second password field which must match the first one you entered. --- domain-server/resources/describe-settings.json | 7 +++++++ .../resources/web/settings/js/settings.js | 14 ++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index 58b1df08c1..20d2711743 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -372,6 +372,13 @@ "help": "Password used for basic HTTP authentication. Leave this blank if you do not want to change it.", "value-hidden": true }, + { + "name": "verify_http_password", + "label": "Verify HTTP Password", + "type": "password", + "help": "Must match the password entered above for change to be saved.", + "value-hidden": true + }, { "name": "maximum_user_capacity", "label": "Maximum User Capacity", diff --git a/domain-server/resources/web/settings/js/settings.js b/domain-server/resources/web/settings/js/settings.js index c31d6e2dfc..659372267c 100644 --- a/domain-server/resources/web/settings/js/settings.js +++ b/domain-server/resources/web/settings/js/settings.js @@ -904,10 +904,18 @@ function saveSettings() { var formJSON = form2js('settings-form', ".", false, cleanupFormValues, true); // check if we've set the basic http password - if so convert it to base64 + var canPost = true; if (formJSON["security"]) { var password = formJSON["security"]["http_password"]; + var verify_password = formJSON["security"]["verify_http_password"]; if (password && password.length > 0) { - formJSON["security"]["http_password"] = sha256_digest(password); + if (password != verify_password) { + bootbox.alert({"message": "Passwords must match!", "title":"Password Error"}); + canPost = false; + } else { + formJSON["security"]["http_password"] = sha256_digest(password); + delete formJSON["security"]["verify_http_password"]; + } } } @@ -923,7 +931,9 @@ function saveSettings() { $(this).blur(); // POST the form JSON to the domain-server settings.json endpoint so the settings are saved - postSettings(formJSON); + if (canPost) { + postSettings(formJSON); + } } $('body').on('click', '.save-button', function(e){