From 33718684e6166f6a582351f1347b265ff3c728b0 Mon Sep 17 00:00:00 2001 From: matsukaze Date: Fri, 4 Apr 2014 16:09:25 -0700 Subject: [PATCH 1/2] More audio fixes. --- interface/src/Audio.cpp | 42 +++++++++++++++++++------------- interface/src/Audio.h | 2 +- interface/src/AudioReflector.cpp | 18 +++----------- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 374976c691..4a501f74b7 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -623,9 +623,9 @@ unsigned int Audio::timeValToSampleTick(const quint64 time, int sampleRate) { return sample; } -void Audio::addSpatialAudioToBuffer(unsigned int sampleTime, const AudioRingBuffer& spatialAudio) { +void Audio::addSpatialAudioToBuffer(unsigned int sampleTime, QByteArray& spatialAudio, unsigned int numSamples) { - // Calculate the number of remaining samples available, the source spatial audio buffer will get + // Calculate the number of remaining samples available. The source spatial audio buffer will get // clipped if there are insufficient samples available in the accumulation buffer. unsigned int remaining = _spatialAudioRingBuffer.getSampleCapacity() - _spatialAudioRingBuffer.samplesAvailable(); @@ -635,9 +635,9 @@ void Audio::addSpatialAudioToBuffer(unsigned int sampleTime, const AudioRingBuff // Nothing in the spatial audio ring buffer yet // Just do a straight copy, clipping if necessary - unsigned int sampleCt = (remaining < spatialAudio.samplesAvailable()) ? remaining : spatialAudio.samplesAvailable(); + unsigned int sampleCt = (remaining < numSamples) ? remaining : numSamples; if (sampleCt) { - _spatialAudioRingBuffer.writeSamples(spatialAudio.getNextOutput(), sampleCt); + _spatialAudioRingBuffer.writeSamples((int16_t*)spatialAudio.data(), sampleCt); } _spatialAudioFinish = _spatialAudioStart + sampleCt / _desiredOutputFormat.channelCount(); @@ -654,35 +654,37 @@ void Audio::addSpatialAudioToBuffer(unsigned int sampleTime, const AudioRingBuff // Recalculate the number of remaining samples remaining -= silentCt; - unsigned int sampleCt = (remaining < spatialAudio.samplesAvailable()) ? remaining : spatialAudio.samplesAvailable(); + unsigned int sampleCt = (remaining < numSamples) ? remaining : numSamples; // Copy the new spatial audio to the accumulation ring buffer if (sampleCt) { - _spatialAudioRingBuffer.writeSamples(spatialAudio.getNextOutput(), sampleCt); + _spatialAudioRingBuffer.writeSamples((int16_t*)spatialAudio.data(), sampleCt); } _spatialAudioFinish += (sampleCt + silentCt) / _desiredOutputFormat.channelCount(); } } else { - // There is overlap between the spatial audio buffer and the new sample, // acumulate the overlap + + // Calculate the offset from the buffer's current read position, which should be located at _spatialAudioStart unsigned int offset = (sampleTime - _spatialAudioStart) * _desiredOutputFormat.channelCount(); unsigned int accumulationCt = (_spatialAudioFinish - sampleTime) * _desiredOutputFormat.channelCount(); - accumulationCt = (accumulationCt < spatialAudio.samplesAvailable()) ? accumulationCt : spatialAudio.samplesAvailable(); + accumulationCt = (accumulationCt < numSamples) ? accumulationCt : numSamples; + int16_t* spatial = reinterpret_cast(spatialAudio.data()); int j = 0; for (int i = accumulationCt; --i >= 0; j++) { - int tmp = _spatialAudioRingBuffer[j + offset] + spatialAudio[j]; + int tmp = _spatialAudioRingBuffer[j + offset] + spatial[j]; _spatialAudioRingBuffer[j + offset] = static_cast(glm::clamp(tmp, std::numeric_limits::min(), std::numeric_limits::max())); } // Copy the remaining unoverlapped spatial audio to the accumulation buffer, if any - unsigned int sampleCt = spatialAudio.samplesAvailable() - accumulationCt; + unsigned int sampleCt = numSamples - accumulationCt; sampleCt = (remaining < sampleCt) ? remaining : sampleCt; if (sampleCt) { - _spatialAudioRingBuffer.writeSamples(spatialAudio.getNextOutput() + accumulationCt, sampleCt); + _spatialAudioRingBuffer.writeSamples((int16_t*)spatialAudio.data() + accumulationCt, sampleCt); _spatialAudioFinish += sampleCt / _desiredOutputFormat.channelCount(); } } @@ -737,23 +739,29 @@ void Audio::processReceivedAudio(unsigned int sampleTime, AudioRingBuffer& ringB //qDebug() << "pushing " << numNetworkOutputSamples; ringBuffer.setIsStarved(false); - int16_t* ringBufferSamples= new int16_t[numNetworkOutputSamples]; + int16_t* ringBufferSamples = new int16_t[numNetworkOutputSamples]; if (_processSpatialAudio) { unsigned int sampleTime = _spatialAudioStart; + QByteArray buffer; + buffer.resize(numDeviceOutputSamples * sizeof(int16_t)); + + ringBuffer.readSamples((int16_t*)buffer.data(), numDeviceOutputSamples); // Accumulate direct transmission of audio from sender to receiver - addSpatialAudioToBuffer(sampleTime, ringBuffer); + addSpatialAudioToBuffer(sampleTime, buffer, numNetworkOutputSamples); //addSpatialAudioToBuffer(sampleTime + 48000, ringBuffer); // Send audio off for spatial processing - emit processSpatialAudio(sampleTime, QByteArray((char*)ringBuffer.getBuffer(), numNetworkOutputSamples), _desiredOutputFormat); + emit processSpatialAudio(sampleTime, buffer, _desiredOutputFormat); // copy the samples we'll resample from the spatial audio ring buffer - this also // pushes the read pointer of the spatial audio ring buffer forwards _spatialAudioRingBuffer.readSamples(ringBufferSamples, numNetworkOutputSamples); - int samples = ringBuffer.samplesAvailable(); - _spatialAudioStart += samples / _desiredOutputFormat.channelCount(); + + // Advance the start point for the next packet of audio to arrive + _spatialAudioStart += numNetworkOutputSamples; - ringBuffer.reset(); + // Advance the read position by the same amount + //ringBuffer.shiftReadPosition(numNetworkOutputSamples); } else { diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 052eb06bdd..80f2e93688 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -78,7 +78,7 @@ public slots: void start(); void stop(); void addReceivedAudioToBuffer(const QByteArray& audioByteArray); - void addSpatialAudioToBuffer(unsigned int sampleTime, const AudioRingBuffer& spatialAudio); + void addSpatialAudioToBuffer(unsigned int sampleTime, QByteArray& spatialAudio, unsigned int numSamples); void handleAudioInput(); void reset(); void toggleMute(); diff --git a/interface/src/AudioReflector.cpp b/interface/src/AudioReflector.cpp index e7d5c29e25..d9471ab796 100644 --- a/interface/src/AudioReflector.cpp +++ b/interface/src/AudioReflector.cpp @@ -130,10 +130,6 @@ void AudioReflector::calculateReflections(const glm::vec3& origin, const glm::ve int16_t* attenuatedLeftSamplesData = (int16_t*)attenuatedLeftSamples.data(); int16_t* attenuatedRightSamplesData = (int16_t*)attenuatedRightSamples.data(); - AudioRingBuffer attenuatedLeftBuffer(totalNumberOfSamples); - AudioRingBuffer attenuatedRightBuffer(totalNumberOfSamples); - - for (int bounceNumber = 1; bounceNumber <= bounces; bounceNumber++) { if (_voxels->findRayIntersection(start, direction, elementHit, distance, face)) { glm::vec3 end = start + (direction * (distance * SLIGHTLY_SHORT)); @@ -179,13 +175,8 @@ void AudioReflector::calculateReflections(const glm::vec3& origin, const glm::ve //qDebug() << "sampleTimeLeft=" << sampleTimeLeft << "sampleTimeRight=" << sampleTimeRight; - attenuatedLeftBuffer.writeSamples(attenuatedLeftSamplesData, totalNumberOfSamples); - attenuatedRightBuffer.writeSamples(attenuatedRightSamplesData, totalNumberOfSamples); - - _audio->addSpatialAudioToBuffer(sampleTimeLeft, attenuatedLeftBuffer); - _audio->addSpatialAudioToBuffer(sampleTimeRight, attenuatedRightBuffer); - attenuatedLeftBuffer.reset(); - attenuatedRightBuffer.reset(); + _audio->addSpatialAudioToBuffer(sampleTimeLeft, attenuatedLeftSamples, totalNumberOfSamples); + _audio->addSpatialAudioToBuffer(sampleTimeRight, attenuatedRightSamples, totalNumberOfSamples); } } } @@ -195,7 +186,6 @@ void AudioReflector::processSpatialAudio(unsigned int sampleTime, const QByteArr //qDebug() << "AudioReflector::processSpatialAudio()...sampleTime=" << sampleTime << " threadID=" << QThread::currentThreadId(); - /* int totalNumberOfSamples = samples.size() / (sizeof(int16_t)); int numFrameSamples = format.sampleRate() * format.channelCount(); @@ -205,6 +195,7 @@ void AudioReflector::processSpatialAudio(unsigned int sampleTime, const QByteArr qDebug() << " sizeof(int16_t)=" << sizeof(int16_t); + /* AudioRingBuffer samplesRingBuffer(totalNumberOfSamples); qint64 bytesCopied = samplesRingBuffer.writeData(samples.constData(),samples.size()); for(int i = 0; i < totalNumberOfSamples; i++) { @@ -212,8 +203,7 @@ void AudioReflector::processSpatialAudio(unsigned int sampleTime, const QByteArr } qDebug() << " bytesCopied=" << bytesCopied; - - _audio->addSpatialAudioToBuffer(sampleTime + 12000, samplesRingBuffer); + _audio->addSpatialAudioToBuffer(sampleTime + 12000, samples, totalNumberOfSamples); return; */ From 9e157ff1c5bdd48013ae6e5f8c808b875b0f3019 Mon Sep 17 00:00:00 2001 From: matsukaze Date: Fri, 4 Apr 2014 16:17:49 -0700 Subject: [PATCH 2/2] More audio fixes. --- interface/src/Audio.cpp | 22 +++++++++++++++------- interface/src/Audio.h | 2 +- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 33fc6ec4d2..5efb120793 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -624,7 +624,7 @@ unsigned int Audio::timeValToSampleTick(const quint64 time, int sampleRate) { return sample; } -void Audio::addSpatialAudioToBuffer(unsigned int sampleTime, QByteArray& spatialAudio, unsigned int numSamples) { +void Audio::addSpatialAudioToBuffer(unsigned int sampleTime, const QByteArray& spatialAudio, unsigned int numSamples) { // Calculate the number of remaining samples available. The source spatial audio buffer will get // clipped if there are insufficient samples available in the accumulation buffer. @@ -673,10 +673,12 @@ void Audio::addSpatialAudioToBuffer(unsigned int sampleTime, QByteArray& spatial unsigned int accumulationCt = (_spatialAudioFinish - sampleTime) * _desiredOutputFormat.channelCount(); accumulationCt = (accumulationCt < numSamples) ? accumulationCt : numSamples; - int16_t* spatial = reinterpret_cast(spatialAudio.data()); + const int16_t* spatial = reinterpret_cast(spatialAudio.data()); int j = 0; for (int i = accumulationCt; --i >= 0; j++) { - int tmp = _spatialAudioRingBuffer[j + offset] + spatial[j]; + int t1 = _spatialAudioRingBuffer[j + offset]; + int t2 = spatial[j]; + int tmp = t1 + t2; _spatialAudioRingBuffer[j + offset] = static_cast(glm::clamp(tmp, std::numeric_limits::min(), std::numeric_limits::max())); } @@ -686,6 +688,7 @@ void Audio::addSpatialAudioToBuffer(unsigned int sampleTime, QByteArray& spatial sampleCt = (remaining < sampleCt) ? remaining : sampleCt; if (sampleCt) { _spatialAudioRingBuffer.writeSamples((int16_t*)spatialAudio.data() + accumulationCt, sampleCt); + // Extend the finish time by the amount of unoverlapped samples _spatialAudioFinish += sampleCt / _desiredOutputFormat.channelCount(); } } @@ -744,12 +747,12 @@ void Audio::processReceivedAudio(unsigned int sampleTime, AudioRingBuffer& ringB if (_processSpatialAudio) { unsigned int sampleTime = _spatialAudioStart; QByteArray buffer; - buffer.resize(numDeviceOutputSamples * sizeof(int16_t)); + buffer.resize(numNetworkOutputSamples * sizeof(int16_t)); - ringBuffer.readSamples((int16_t*)buffer.data(), numDeviceOutputSamples); + ringBuffer.readSamples((int16_t*)buffer.data(), numNetworkOutputSamples); // Accumulate direct transmission of audio from sender to receiver addSpatialAudioToBuffer(sampleTime, buffer, numNetworkOutputSamples); - //addSpatialAudioToBuffer(sampleTime + 48000, ringBuffer); + //addSpatialAudioToBuffer(sampleTime + 48000, buffer, numNetworkOutputSamples); // Send audio off for spatial processing emit processSpatialAudio(sampleTime, buffer, _desiredOutputFormat); @@ -759,7 +762,7 @@ void Audio::processReceivedAudio(unsigned int sampleTime, AudioRingBuffer& ringB _spatialAudioRingBuffer.readSamples(ringBufferSamples, numNetworkOutputSamples); // Advance the start point for the next packet of audio to arrive - _spatialAudioStart += numNetworkOutputSamples; + _spatialAudioStart += numNetworkOutputSamples / _desiredOutputFormat.channelCount(); // Advance the read position by the same amount //ringBuffer.shiftReadPosition(numNetworkOutputSamples); @@ -838,6 +841,11 @@ void Audio::toggleToneInjection() { void Audio::toggleAudioSpatialProcessing() { _processSpatialAudio = !_processSpatialAudio; + if (_processSpatialAudio) { + _spatialAudioStart = 0; + _spatialAudioFinish = 0; + _spatialAudioRingBuffer.reset(); + } } // Take a pointer to the acquired microphone input samples and add procedural sounds diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 7520d97c0b..11ad235289 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -78,7 +78,7 @@ public slots: void start(); void stop(); void addReceivedAudioToBuffer(const QByteArray& audioByteArray); - void addSpatialAudioToBuffer(unsigned int sampleTime, QByteArray& spatialAudio, unsigned int numSamples); + void addSpatialAudioToBuffer(unsigned int sampleTime, const QByteArray& spatialAudio, unsigned int numSamples); void handleAudioInput(); void reset(); void toggleMute();