mirror of
https://github.com/overte-org/overte.git
synced 2025-08-08 16:18:05 +02:00
mv audio injector preparation to own thread
This commit is contained in:
parent
d7085ec685
commit
4c7c7ee3cc
2 changed files with 109 additions and 49 deletions
|
@ -113,6 +113,10 @@ private:
|
||||||
bool _quit { false };
|
bool _quit { false };
|
||||||
};
|
};
|
||||||
|
|
||||||
|
void AudioInjectorsThread::prepare() {
|
||||||
|
_audio->prepareLocalAudioInjectors();
|
||||||
|
}
|
||||||
|
|
||||||
AudioClient::AudioClient() :
|
AudioClient::AudioClient() :
|
||||||
AbstractAudioInterface(),
|
AbstractAudioInterface(),
|
||||||
_gate(this),
|
_gate(this),
|
||||||
|
@ -146,6 +150,8 @@ AudioClient::AudioClient() :
|
||||||
_reverbOptions(&_scriptReverbOptions),
|
_reverbOptions(&_scriptReverbOptions),
|
||||||
_inputToNetworkResampler(NULL),
|
_inputToNetworkResampler(NULL),
|
||||||
_networkToOutputResampler(NULL),
|
_networkToOutputResampler(NULL),
|
||||||
|
_localToOutputResampler(NULL),
|
||||||
|
_localAudioThread(this),
|
||||||
_audioLimiter(AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT),
|
_audioLimiter(AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT),
|
||||||
_outgoingAvatarAudioSequenceNumber(0),
|
_outgoingAvatarAudioSequenceNumber(0),
|
||||||
_audioOutputIODevice(_localInjectorsBuffer, _receivedAudioStream, this),
|
_audioOutputIODevice(_localInjectorsBuffer, _receivedAudioStream, this),
|
||||||
|
@ -178,6 +184,10 @@ AudioClient::AudioClient() :
|
||||||
_checkDevicesThread->setPriority(QThread::LowPriority);
|
_checkDevicesThread->setPriority(QThread::LowPriority);
|
||||||
_checkDevicesThread->start();
|
_checkDevicesThread->start();
|
||||||
|
|
||||||
|
// start a thread to process local injectors
|
||||||
|
_localAudioThread.setObjectName("LocalAudio Thread");
|
||||||
|
_localAudioThread.start();
|
||||||
|
|
||||||
configureReverb();
|
configureReverb();
|
||||||
|
|
||||||
auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
|
auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
|
||||||
|
@ -215,6 +225,7 @@ void AudioClient::reset() {
|
||||||
_stats.reset();
|
_stats.reset();
|
||||||
_sourceReverb.reset();
|
_sourceReverb.reset();
|
||||||
_listenerReverb.reset();
|
_listenerReverb.reset();
|
||||||
|
_localReverb.reset();
|
||||||
}
|
}
|
||||||
|
|
||||||
void AudioClient::audioMixerKilled() {
|
void AudioClient::audioMixerKilled() {
|
||||||
|
@ -764,6 +775,7 @@ void AudioClient::configureReverb() {
|
||||||
p.wetDryMix = _reverbOptions->getWetDryMix();
|
p.wetDryMix = _reverbOptions->getWetDryMix();
|
||||||
|
|
||||||
_listenerReverb.setParameters(&p);
|
_listenerReverb.setParameters(&p);
|
||||||
|
_localReverb.setParameters(&p);
|
||||||
|
|
||||||
// used only for adding self-reverb to loopback audio
|
// used only for adding self-reverb to loopback audio
|
||||||
p.sampleRate = _outputFormat.sampleRate();
|
p.sampleRate = _outputFormat.sampleRate();
|
||||||
|
@ -810,6 +822,7 @@ void AudioClient::setReverb(bool reverb) {
|
||||||
if (!_reverb) {
|
if (!_reverb) {
|
||||||
_sourceReverb.reset();
|
_sourceReverb.reset();
|
||||||
_listenerReverb.reset();
|
_listenerReverb.reset();
|
||||||
|
_localReverb.reset();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1059,42 +1072,56 @@ void AudioClient::prepareLocalAudioInjectors() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
int periodSamples = _audioOutput->periodSize() / AudioConstants::SAMPLE_SIZE;
|
int bufferCapacity = _localInjectorsBuffer.getSampleCapacity();
|
||||||
int samplesNeeded;
|
if (_localToOutputResampler) {
|
||||||
if ((samplesNeeded = periodSamples - _localInjectorsBuffer.samplesAvailable()) > 0) {
|
// avoid overwriting the buffer
|
||||||
while (samplesNeeded > 0) {
|
bufferCapacity -=
|
||||||
// get a network frame of local injectors' audio
|
_localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) *
|
||||||
if (!mixLocalAudioInjectors(_localMixBuffer)) {
|
AudioConstants::STEREO;
|
||||||
return;
|
bufferCapacity += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
// reverb
|
int samplesNeeded = std::numeric_limits<int>::max();
|
||||||
if (_reverb) {
|
while (samplesNeeded > 0) {
|
||||||
updateReverbOptions();
|
// lock for every write to avoid locking out the device callback
|
||||||
_listenerReverb.render(_localMixBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
|
Lock lock(_localAudioMutex);
|
||||||
}
|
|
||||||
|
|
||||||
convertToScratch(_localScratchBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO);
|
samplesNeeded = bufferCapacity - _localInjectorsBuffer.samplesAvailable();
|
||||||
|
if (samplesNeeded <= 0) {
|
||||||
int samples;
|
break;
|
||||||
if (_networkToOutputResampler) {
|
|
||||||
// resample to output sample rate
|
|
||||||
int frames = _networkToOutputResampler->render(_localScratchBuffer, _outputScratchBuffer,
|
|
||||||
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
|
|
||||||
|
|
||||||
// write to local injectors' ring buffer
|
|
||||||
samples = frames * AudioConstants::STEREO;
|
|
||||||
_localInjectorsBuffer.writeSamples(_outputScratchBuffer, samples);
|
|
||||||
|
|
||||||
} else {
|
|
||||||
// write to local injectors' ring buffer
|
|
||||||
samples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
|
|
||||||
_localInjectorsBuffer.writeSamples(_localScratchBuffer,
|
|
||||||
AudioConstants::NETWORK_FRAME_SAMPLES_STEREO);
|
|
||||||
}
|
|
||||||
|
|
||||||
samplesNeeded -= samples;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// get a network frame of local injectors' audio
|
||||||
|
if (!mixLocalAudioInjectors(_localMixBuffer)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// reverb
|
||||||
|
if (_reverb) {
|
||||||
|
_localReverb.render(_localMixBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
|
||||||
|
}
|
||||||
|
|
||||||
|
convertToScratch(_localScratchBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO);
|
||||||
|
|
||||||
|
int samples;
|
||||||
|
if (_localToOutputResampler) {
|
||||||
|
// resample to output sample rate
|
||||||
|
int frames = _localToOutputResampler->render(_localScratchBuffer, _localOutputScratchBuffer,
|
||||||
|
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
|
||||||
|
|
||||||
|
// write to local injectors' ring buffer
|
||||||
|
samples = frames * AudioConstants::STEREO;
|
||||||
|
_localInjectorsBuffer.writeSamples(_localOutputScratchBuffer, samples);
|
||||||
|
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
// write to local injectors' ring buffer
|
||||||
|
samples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
|
||||||
|
_localInjectorsBuffer.writeSamples(_localScratchBuffer,
|
||||||
|
AudioConstants::NETWORK_FRAME_SAMPLES_STEREO);
|
||||||
|
}
|
||||||
|
|
||||||
|
samplesNeeded -= samples;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1158,7 +1185,7 @@ bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) {
|
||||||
glm::vec3 relativePosition = injector->getPosition() - _positionGetter();
|
glm::vec3 relativePosition = injector->getPosition() - _positionGetter();
|
||||||
float distance = glm::max(glm::length(relativePosition), EPSILON);
|
float distance = glm::max(glm::length(relativePosition), EPSILON);
|
||||||
float gain = gainForSource(distance, injector->getVolume());
|
float gain = gainForSource(distance, injector->getVolume());
|
||||||
float azimuth = azimuthForSource(relativePosition);
|
float azimuth = azimuthForSource(relativePosition);
|
||||||
|
|
||||||
// mono gets spatialized into mixBuffer
|
// mono gets spatialized into mixBuffer
|
||||||
injector->getLocalHRTF().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX,
|
injector->getLocalHRTF().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX,
|
||||||
|
@ -1395,6 +1422,8 @@ void AudioClient::outputNotify() {
|
||||||
bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo) {
|
bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo) {
|
||||||
bool supportedFormat = false;
|
bool supportedFormat = false;
|
||||||
|
|
||||||
|
Lock lock(_localAudioMutex);
|
||||||
|
|
||||||
// cleanup any previously initialized device
|
// cleanup any previously initialized device
|
||||||
if (_audioOutput) {
|
if (_audioOutput) {
|
||||||
_audioOutput->stop();
|
_audioOutput->stop();
|
||||||
|
@ -1406,17 +1435,23 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice
|
||||||
delete _loopbackAudioOutput;
|
delete _loopbackAudioOutput;
|
||||||
_loopbackAudioOutput = NULL;
|
_loopbackAudioOutput = NULL;
|
||||||
|
|
||||||
delete _outputMixBuffer;
|
delete[] _outputMixBuffer;
|
||||||
_outputMixBuffer = NULL;
|
_outputMixBuffer = NULL;
|
||||||
|
|
||||||
delete _outputScratchBuffer;
|
delete[] _outputScratchBuffer;
|
||||||
_outputScratchBuffer = NULL;
|
_outputScratchBuffer = NULL;
|
||||||
|
|
||||||
|
delete[] _localOutputScratchBuffer;
|
||||||
|
_localOutputScratchBuffer = NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (_networkToOutputResampler) {
|
if (_networkToOutputResampler) {
|
||||||
// if we were using an input to network resampler, delete it here
|
// if we were using an input to network resampler, delete it here
|
||||||
delete _networkToOutputResampler;
|
delete _networkToOutputResampler;
|
||||||
_networkToOutputResampler = NULL;
|
_networkToOutputResampler = NULL;
|
||||||
|
|
||||||
|
delete _localToOutputResampler;
|
||||||
|
_localToOutputResampler = NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!outputDeviceInfo.isNull()) {
|
if (!outputDeviceInfo.isNull()) {
|
||||||
|
@ -1436,6 +1471,7 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice
|
||||||
assert(_outputFormat.sampleSize() == 16);
|
assert(_outputFormat.sampleSize() == 16);
|
||||||
|
|
||||||
_networkToOutputResampler = new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);
|
_networkToOutputResampler = new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);
|
||||||
|
_localToOutputResampler = new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
qCDebug(audioclient) << "No resampling required for network output to match actual output format.";
|
qCDebug(audioclient) << "No resampling required for network output to match actual output format.";
|
||||||
|
@ -1466,7 +1502,8 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice
|
||||||
_outputPeriod = periodSampleSize * 2;
|
_outputPeriod = periodSampleSize * 2;
|
||||||
_outputMixBuffer = new float[_outputPeriod];
|
_outputMixBuffer = new float[_outputPeriod];
|
||||||
_outputScratchBuffer = new int16_t[_outputPeriod];
|
_outputScratchBuffer = new int16_t[_outputPeriod];
|
||||||
_localInjectorsBuffer.resizeForFrameSize(_outputPeriod);
|
_localOutputScratchBuffer = new int16_t[_outputPeriod];
|
||||||
|
_localInjectorsBuffer.resizeForFrameSize(_outputPeriod * 2);
|
||||||
|
|
||||||
qCDebug(audioclient) << "Output Buffer capacity in frames: " << _audioOutput->bufferSize() / AudioConstants::SAMPLE_SIZE / (float)deviceFrameSize <<
|
qCDebug(audioclient) << "Output Buffer capacity in frames: " << _audioOutput->bufferSize() / AudioConstants::SAMPLE_SIZE / (float)deviceFrameSize <<
|
||||||
"requested bytes:" << requestedSize << "actual bytes:" << _audioOutput->bufferSize() <<
|
"requested bytes:" << requestedSize << "actual bytes:" << _audioOutput->bufferSize() <<
|
||||||
|
@ -1595,21 +1632,26 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
|
||||||
}
|
}
|
||||||
|
|
||||||
int injectorSamplesPopped;
|
int injectorSamplesPopped;
|
||||||
if ((injectorSamplesPopped = _localInjectorsBuffer.readSamples(scratchBuffer, samplesRequested)) > 0) {
|
{
|
||||||
qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsBuffer.samplesAvailable(), samplesRequested);
|
Lock lock(_audio->_localAudioMutex);
|
||||||
|
if ((injectorSamplesPopped = _localInjectorsBuffer.readSamples(scratchBuffer, samplesRequested)) > 0) {
|
||||||
|
qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsBuffer.samplesAvailable(), samplesRequested);
|
||||||
|
|
||||||
if (networkSamplesPopped == 0) {
|
if (_audio->_shouldEchoToServer) {
|
||||||
convertToMix(mixBuffer, scratchBuffer, injectorSamplesPopped);
|
// omit local audio, it should be echoed
|
||||||
} else {
|
} else if (networkSamplesPopped == 0) {
|
||||||
// add to mix buffer (float)
|
convertToMix(mixBuffer, scratchBuffer, injectorSamplesPopped);
|
||||||
for (int i = 0; i < injectorSamplesPopped; i++) {
|
} else {
|
||||||
mixBuffer[i] += (float)scratchBuffer[i] * (1/32768.0f);
|
// add to mix buffer (float)
|
||||||
|
for (int i = 0; i < injectorSamplesPopped; i++) {
|
||||||
|
mixBuffer[i] += (float)scratchBuffer[i] * (1 / 32768.0f);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// prepare injectors for the next callback
|
// prepare injectors for the next callback
|
||||||
QMetaObject::invokeMethod(_audio, "prepareLocalAudioInjectors", Qt::QueuedConnection);
|
QMetaObject::invokeMethod(&_audio->_localAudioThread, "prepare", Qt::QueuedConnection);
|
||||||
|
|
||||||
int samplesPopped = std::max(networkSamplesPopped, injectorSamplesPopped);
|
int samplesPopped = std::max(networkSamplesPopped, injectorSamplesPopped);
|
||||||
int framesPopped = samplesPopped / AudioConstants::STEREO;
|
int framesPopped = samplesPopped / AudioConstants::STEREO;
|
||||||
|
|
|
@ -69,6 +69,19 @@ class QIODevice;
|
||||||
class Transform;
|
class Transform;
|
||||||
class NLPacket;
|
class NLPacket;
|
||||||
|
|
||||||
|
class AudioInjectorsThread : public QThread {
|
||||||
|
Q_OBJECT
|
||||||
|
|
||||||
|
public:
|
||||||
|
AudioInjectorsThread(AudioClient* audio) : _audio(audio) {}
|
||||||
|
|
||||||
|
public slots :
|
||||||
|
void prepare();
|
||||||
|
|
||||||
|
private:
|
||||||
|
AudioClient* _audio;
|
||||||
|
};
|
||||||
|
|
||||||
class AudioClient : public AbstractAudioInterface, public Dependency {
|
class AudioClient : public AbstractAudioInterface, public Dependency {
|
||||||
Q_OBJECT
|
Q_OBJECT
|
||||||
SINGLETON_DEPENDENCY
|
SINGLETON_DEPENDENCY
|
||||||
|
@ -295,20 +308,25 @@ private:
|
||||||
AudioEffectOptions* _reverbOptions;
|
AudioEffectOptions* _reverbOptions;
|
||||||
AudioReverb _sourceReverb { AudioConstants::SAMPLE_RATE };
|
AudioReverb _sourceReverb { AudioConstants::SAMPLE_RATE };
|
||||||
AudioReverb _listenerReverb { AudioConstants::SAMPLE_RATE };
|
AudioReverb _listenerReverb { AudioConstants::SAMPLE_RATE };
|
||||||
|
AudioReverb _localReverb { AudioConstants::SAMPLE_RATE };
|
||||||
|
|
||||||
// possible streams needed for resample
|
// possible streams needed for resample
|
||||||
AudioSRC* _inputToNetworkResampler;
|
AudioSRC* _inputToNetworkResampler;
|
||||||
AudioSRC* _networkToOutputResampler;
|
AudioSRC* _networkToOutputResampler;
|
||||||
|
AudioSRC* _localToOutputResampler;
|
||||||
|
|
||||||
// for network audio (used by network audio threads)
|
// for network audio (used by network audio thread)
|
||||||
float _networkMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
|
float _networkMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
|
||||||
int16_t _networkScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC];
|
int16_t _networkScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC];
|
||||||
|
|
||||||
// for local audio (used by this thread only)
|
// for local audio (used by audio injectors thread)
|
||||||
float _localMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
|
float _localMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
|
||||||
int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC];
|
int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC];
|
||||||
|
int16_t* _localOutputScratchBuffer { NULL };
|
||||||
|
AudioInjectorsThread _localAudioThread;
|
||||||
|
Mutex _localAudioMutex;
|
||||||
|
|
||||||
// for output audio (used by this thread only)
|
// for output audio (used by this thread)
|
||||||
int _outputPeriod { 0 };
|
int _outputPeriod { 0 };
|
||||||
float* _outputMixBuffer { NULL };
|
float* _outputMixBuffer { NULL };
|
||||||
int16_t* _outputScratchBuffer { NULL };
|
int16_t* _outputScratchBuffer { NULL };
|
||||||
|
|
Loading…
Reference in a new issue