Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-08-08 13:38:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

mv audio injector preparation to own thread

Browse source
This commit is contained in:
Zach Pomerantz 2017-01-11 15:25:31 -05:00
parent d7085ec685
commit 4c7c7ee3cc
2 changed files with 109 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

134
libraries/audio-client/src/AudioClient.cpp
View file

@ -113,6 +113,10 @@ private:
bool _quit { false }; bool _quit { false };
}; };
void AudioInjectorsThread::prepare() {
_audio->prepareLocalAudioInjectors();
}
AudioClient::AudioClient() : AudioClient::AudioClient() :
AbstractAudioInterface(), AbstractAudioInterface(),
_gate(this), _gate(this),
@ -146,6 +150,8 @@ AudioClient::AudioClient() :
_reverbOptions(&_scriptReverbOptions), _reverbOptions(&_scriptReverbOptions),
_inputToNetworkResampler(NULL), _inputToNetworkResampler(NULL),
_networkToOutputResampler(NULL), _networkToOutputResampler(NULL),
_localToOutputResampler(NULL),
_localAudioThread(this),
_audioLimiter(AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT), _audioLimiter(AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT),
_outgoingAvatarAudioSequenceNumber(0), _outgoingAvatarAudioSequenceNumber(0),
_audioOutputIODevice(_localInjectorsBuffer, _receivedAudioStream, this), _audioOutputIODevice(_localInjectorsBuffer, _receivedAudioStream, this),
@ -178,6 +184,10 @@ AudioClient::AudioClient() :
_checkDevicesThread->setPriority(QThread::LowPriority); _checkDevicesThread->setPriority(QThread::LowPriority);
_checkDevicesThread->start(); _checkDevicesThread->start();
// start a thread to process local injectors
_localAudioThread.setObjectName("LocalAudio Thread");
_localAudioThread.start();
configureReverb(); configureReverb();
auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver(); auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
@ -215,6 +225,7 @@ void AudioClient::reset() {
_stats.reset(); _stats.reset();
_sourceReverb.reset(); _sourceReverb.reset();
_listenerReverb.reset(); _listenerReverb.reset();
_localReverb.reset();
} }
void AudioClient::audioMixerKilled() { void AudioClient::audioMixerKilled() {
@ -764,6 +775,7 @@ void AudioClient::configureReverb() {
p.wetDryMix = _reverbOptions->getWetDryMix(); p.wetDryMix = _reverbOptions->getWetDryMix();
_listenerReverb.setParameters(&p); _listenerReverb.setParameters(&p);
_localReverb.setParameters(&p);
// used only for adding self-reverb to loopback audio // used only for adding self-reverb to loopback audio
p.sampleRate = _outputFormat.sampleRate(); p.sampleRate = _outputFormat.sampleRate();
@ -810,6 +822,7 @@ void AudioClient::setReverb(bool reverb) {
if (!_reverb) { if (!_reverb) {
_sourceReverb.reset(); _sourceReverb.reset();
_listenerReverb.reset(); _listenerReverb.reset();
_localReverb.reset();
} }
} }
@ -1059,42 +1072,56 @@ void AudioClient::prepareLocalAudioInjectors() {
return; return;
} }
int periodSamples = _audioOutput->periodSize() / AudioConstants::SAMPLE_SIZE; int bufferCapacity = _localInjectorsBuffer.getSampleCapacity();
int samplesNeeded; if (_localToOutputResampler) {
if ((samplesNeeded = periodSamples - _localInjectorsBuffer.samplesAvailable()) > 0) { // avoid overwriting the buffer
while (samplesNeeded > 0) { bufferCapacity -=
// get a network frame of local injectors' audio _localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) *
if (!mixLocalAudioInjectors(_localMixBuffer)) { AudioConstants::STEREO;
return; bufferCapacity += 1;
} }
// reverb int samplesNeeded = std::numeric_limits<int>::max();
if (_reverb) { while (samplesNeeded > 0) {
updateReverbOptions(); // lock for every write to avoid locking out the device callback
_listenerReverb.render(_localMixBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); Lock lock(_localAudioMutex);
}
convertToScratch(_localScratchBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); samplesNeeded = bufferCapacity - _localInjectorsBuffer.samplesAvailable();
if (samplesNeeded <= 0) {
int samples; break;
if (_networkToOutputResampler) {
// resample to output sample rate
int frames = _networkToOutputResampler->render(_localScratchBuffer, _outputScratchBuffer,
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
// write to local injectors' ring buffer
samples = frames * AudioConstants::STEREO;
_localInjectorsBuffer.writeSamples(_outputScratchBuffer, samples);
} else {
// write to local injectors' ring buffer
samples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
_localInjectorsBuffer.writeSamples(_localScratchBuffer,
AudioConstants::NETWORK_FRAME_SAMPLES_STEREO);
}
samplesNeeded -= samples;
} }
// get a network frame of local injectors' audio
if (!mixLocalAudioInjectors(_localMixBuffer)) {
break;
}
// reverb
if (_reverb) {
_localReverb.render(_localMixBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
}
convertToScratch(_localScratchBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO);
int samples;
if (_localToOutputResampler) {
// resample to output sample rate
int frames = _localToOutputResampler->render(_localScratchBuffer, _localOutputScratchBuffer,
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
// write to local injectors' ring buffer
samples = frames * AudioConstants::STEREO;
_localInjectorsBuffer.writeSamples(_localOutputScratchBuffer, samples);
}
else {
// write to local injectors' ring buffer
samples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
_localInjectorsBuffer.writeSamples(_localScratchBuffer,
AudioConstants::NETWORK_FRAME_SAMPLES_STEREO);
}
samplesNeeded -= samples;
} }
} }
@ -1158,7 +1185,7 @@ bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) {
glm::vec3 relativePosition = injector->getPosition() - _positionGetter(); glm::vec3 relativePosition = injector->getPosition() - _positionGetter();
float distance = glm::max(glm::length(relativePosition), EPSILON); float distance = glm::max(glm::length(relativePosition), EPSILON);
float gain = gainForSource(distance, injector->getVolume()); float gain = gainForSource(distance, injector->getVolume());
float azimuth = azimuthForSource(relativePosition); float azimuth = azimuthForSource(relativePosition);
// mono gets spatialized into mixBuffer // mono gets spatialized into mixBuffer
injector->getLocalHRTF().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, injector->getLocalHRTF().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX,
@ -1395,6 +1422,8 @@ void AudioClient::outputNotify() {
bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo) { bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo) {
bool supportedFormat = false; bool supportedFormat = false;
Lock lock(_localAudioMutex);
// cleanup any previously initialized device // cleanup any previously initialized device
if (_audioOutput) { if (_audioOutput) {
_audioOutput->stop(); _audioOutput->stop();
@ -1406,17 +1435,23 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice
delete _loopbackAudioOutput; delete _loopbackAudioOutput;
_loopbackAudioOutput = NULL; _loopbackAudioOutput = NULL;
delete _outputMixBuffer; delete[] _outputMixBuffer;
_outputMixBuffer = NULL; _outputMixBuffer = NULL;
delete _outputScratchBuffer; delete[] _outputScratchBuffer;
_outputScratchBuffer = NULL; _outputScratchBuffer = NULL;
delete[] _localOutputScratchBuffer;
_localOutputScratchBuffer = NULL;
} }
if (_networkToOutputResampler) { if (_networkToOutputResampler) {
// if we were using an input to network resampler, delete it here // if we were using an input to network resampler, delete it here
delete _networkToOutputResampler; delete _networkToOutputResampler;
_networkToOutputResampler = NULL; _networkToOutputResampler = NULL;
delete _localToOutputResampler;
_localToOutputResampler = NULL;
} }
if (!outputDeviceInfo.isNull()) { if (!outputDeviceInfo.isNull()) {
@ -1436,6 +1471,7 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice
assert(_outputFormat.sampleSize() == 16); assert(_outputFormat.sampleSize() == 16);
_networkToOutputResampler = new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT); _networkToOutputResampler = new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);
_localToOutputResampler = new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);
} else { } else {
qCDebug(audioclient) << "No resampling required for network output to match actual output format."; qCDebug(audioclient) << "No resampling required for network output to match actual output format.";
@ -1466,7 +1502,8 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice
_outputPeriod = periodSampleSize * 2; _outputPeriod = periodSampleSize * 2;
_outputMixBuffer = new float[_outputPeriod]; _outputMixBuffer = new float[_outputPeriod];
_outputScratchBuffer = new int16_t[_outputPeriod]; _outputScratchBuffer = new int16_t[_outputPeriod];
_localInjectorsBuffer.resizeForFrameSize(_outputPeriod); _localOutputScratchBuffer = new int16_t[_outputPeriod];
_localInjectorsBuffer.resizeForFrameSize(_outputPeriod * 2);
qCDebug(audioclient) << "Output Buffer capacity in frames: " << _audioOutput->bufferSize() / AudioConstants::SAMPLE_SIZE / (float)deviceFrameSize << qCDebug(audioclient) << "Output Buffer capacity in frames: " << _audioOutput->bufferSize() / AudioConstants::SAMPLE_SIZE / (float)deviceFrameSize <<
"requested bytes:" << requestedSize << "actual bytes:" << _audioOutput->bufferSize() << "requested bytes:" << requestedSize << "actual bytes:" << _audioOutput->bufferSize() <<
@ -1595,21 +1632,26 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
} }
int injectorSamplesPopped; int injectorSamplesPopped;
if ((injectorSamplesPopped = _localInjectorsBuffer.readSamples(scratchBuffer, samplesRequested)) > 0) { {
qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsBuffer.samplesAvailable(), samplesRequested); Lock lock(_audio->_localAudioMutex);
if ((injectorSamplesPopped = _localInjectorsBuffer.readSamples(scratchBuffer, samplesRequested)) > 0) {
qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsBuffer.samplesAvailable(), samplesRequested);
if (networkSamplesPopped == 0) { if (_audio->_shouldEchoToServer) {
convertToMix(mixBuffer, scratchBuffer, injectorSamplesPopped); // omit local audio, it should be echoed
} else { } else if (networkSamplesPopped == 0) {
// add to mix buffer (float) convertToMix(mixBuffer, scratchBuffer, injectorSamplesPopped);
for (int i = 0; i < injectorSamplesPopped; i++) { } else {
mixBuffer[i] += (float)scratchBuffer[i] * (1/32768.0f); // add to mix buffer (float)
for (int i = 0; i < injectorSamplesPopped; i++) {
mixBuffer[i] += (float)scratchBuffer[i] * (1 / 32768.0f);
}
} }
} }
} }
// prepare injectors for the next callback // prepare injectors for the next callback
QMetaObject::invokeMethod(_audio, "prepareLocalAudioInjectors", Qt::QueuedConnection); QMetaObject::invokeMethod(&_audio->_localAudioThread, "prepare", Qt::QueuedConnection);
int samplesPopped = std::max(networkSamplesPopped, injectorSamplesPopped); int samplesPopped = std::max(networkSamplesPopped, injectorSamplesPopped);
int framesPopped = samplesPopped / AudioConstants::STEREO; int framesPopped = samplesPopped / AudioConstants::STEREO;

24
libraries/audio-client/src/AudioClient.h
View file

@ -69,6 +69,19 @@ class QIODevice;
class Transform; class Transform;
class NLPacket; class NLPacket;
class AudioInjectorsThread : public QThread {
Q_OBJECT
public:
AudioInjectorsThread(AudioClient* audio) : _audio(audio) {}
public slots :
void prepare();
private:
AudioClient* _audio;
};
class AudioClient : public AbstractAudioInterface, public Dependency { class AudioClient : public AbstractAudioInterface, public Dependency {
Q_OBJECT Q_OBJECT
SINGLETON_DEPENDENCY SINGLETON_DEPENDENCY
@ -295,20 +308,25 @@ private:
AudioEffectOptions* _reverbOptions; AudioEffectOptions* _reverbOptions;
AudioReverb _sourceReverb { AudioConstants::SAMPLE_RATE }; AudioReverb _sourceReverb { AudioConstants::SAMPLE_RATE };
AudioReverb _listenerReverb { AudioConstants::SAMPLE_RATE }; AudioReverb _listenerReverb { AudioConstants::SAMPLE_RATE };
AudioReverb _localReverb { AudioConstants::SAMPLE_RATE };
// possible streams needed for resample // possible streams needed for resample
AudioSRC* _inputToNetworkResampler; AudioSRC* _inputToNetworkResampler;
AudioSRC* _networkToOutputResampler; AudioSRC* _networkToOutputResampler;
AudioSRC* _localToOutputResampler;
// for network audio (used by network audio threads) // for network audio (used by network audio thread)
float _networkMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; float _networkMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
int16_t _networkScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; int16_t _networkScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC];
// for local audio (used by this thread only) // for local audio (used by audio injectors thread)
float _localMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; float _localMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC];
int16_t* _localOutputScratchBuffer { NULL };
AudioInjectorsThread _localAudioThread;
Mutex _localAudioMutex;
// for output audio (used by this thread only) // for output audio (used by this thread)
int _outputPeriod { 0 }; int _outputPeriod { 0 };
float* _outputMixBuffer { NULL }; float* _outputMixBuffer { NULL };
int16_t* _outputScratchBuffer { NULL }; int16_t* _outputScratchBuffer { NULL };