From 84588e6e35c3693cd6aa6e03541c15afa8e41a9b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 1 Feb 2013 18:19:57 -0800 Subject: [PATCH] add AudioRingBuffer class for new ring buffer algo --- Source/Audio.cpp | 241 ++++++++++++++++------------ Source/AudioData.cpp | 13 +- Source/AudioData.h | 5 + Source/AudioRingBuffer.cpp | 44 +++++ Source/AudioRingBuffer.h | 28 ++++ interface.xcodeproj/project.pbxproj | 6 + 6 files changed, 234 insertions(+), 103 deletions(-) create mode 100644 Source/AudioRingBuffer.cpp create mode 100644 Source/AudioRingBuffer.h diff --git a/Source/Audio.cpp b/Source/Audio.cpp index a790b04139..74638ad5c5 100644 --- a/Source/Audio.cpp +++ b/Source/Audio.cpp @@ -15,21 +15,26 @@ #include "AudioSource.h" #include "UDPSocket.h" -const int BUFFER_LENGTH_BYTES = 1024; -const int BUFFER_LENGTH_SAMPLES = BUFFER_LENGTH_BYTES / sizeof(int16_t); +const short BUFFER_LENGTH_BYTES = 1024; +const short BUFFER_LENGTH_SAMPLES = BUFFER_LENGTH_BYTES / sizeof(int16_t); + +const short PACKET_LENGTH_BYTES = 1024; +const short PACKET_LENGTH_SAMPLES = PACKET_LENGTH_BYTES / sizeof(int16_t); const int PHASE_DELAY_AT_90 = 20; -const int AMPLITUDE_RATIO_AT_90 = 0.5; +const float AMPLITUDE_RATIO_AT_90 = 0.5; -const int JITTER_BUFFER_MSECS = 5; +const short RING_BUFFER_FRAMES = 4; +const short RING_BUFFER_SIZE_SAMPLES = RING_BUFFER_FRAMES * BUFFER_LENGTH_SAMPLES; -const int NUM_AUDIO_SOURCES = 1; -const int ECHO_SERVER_TEST = 1; +const short JITTER_BUFFER_LENGTH_MSECS = 3; +const int SAMPLE_RATE = 22050; + +const short NUM_AUDIO_SOURCES = 2; +const short ECHO_SERVER_TEST = 1; const int AUDIO_UDP_LISTEN_PORT = 55444; -pthread_mutex_t jitterMutex; - #define LOG_SAMPLE_DELAY 1 bool Audio::initialized; @@ -80,89 +85,98 @@ int audioCallback (const void *inputBuffer, memset(outputLeft, 0, BUFFER_LENGTH_BYTES); memset(outputRight, 0, BUFFER_LENGTH_BYTES); - for (int s = 0; s < NUM_AUDIO_SOURCES; s++) { + if (ECHO_SERVER_TEST) { + AudioRingBuffer *ringBuffer = data->ringBuffer; + + int16_t *queueBuffer = data->samplesToQueue; + memset(queueBuffer, 0, BUFFER_LENGTH_BYTES); - AudioSource *source = data->sources[s]; + // if we've been reset, and there isn't any new packets yet + // just play some silence - if (ECHO_SERVER_TEST) { - AudioSource::JitterBuffer *bufferToCopy = NULL; + if (ringBuffer->endOfLastWrite != NULL) { - timeval sendTime; - gettimeofday(&sendTime, NULL); + // play whatever we have in the audio buffer + short silentTail = 0; - pthread_mutex_lock(&jitterMutex); - - // copy whatever the oldest data to the left and right output channels - // as long as it came in at least JITTER_BUFFER_MSECS ago - if (source->oldestData != NULL) { - bufferToCopy = source->oldestData; - } else if (source->newestData != NULL) { - bufferToCopy = source->newestData; + // if the end of the last write to the ring is in front of the current output pointer + // AND the difference between the two is less than a full output buffer + // we need to add some silence after the audio data, to avoid replaying old data + if ((ringBuffer->endOfLastWrite - ringBuffer->buffer) > (ringBuffer->nextOutput - ringBuffer->buffer) + && (ringBuffer->endOfLastWrite - ringBuffer->nextOutput) < BUFFER_LENGTH_SAMPLES) { + silentTail = BUFFER_LENGTH_SAMPLES - (ringBuffer->endOfLastWrite - ringBuffer->nextOutput); } - if (bufferToCopy != NULL && diffclock(bufferToCopy->receiveTime, sendTime) > JITTER_BUFFER_MSECS) { - memcpy(outputLeft, bufferToCopy->audioData, BUFFER_LENGTH_BYTES); - memcpy(outputRight, bufferToCopy->audioData, BUFFER_LENGTH_BYTES); + // no sample overlap, either a direct copy of the audio data, or a copy with some appended silence + memcpy(queueBuffer, ringBuffer->nextOutput, (BUFFER_LENGTH_SAMPLES - silentTail) * sizeof(int16_t)); + + ringBuffer->nextOutput += BUFFER_LENGTH_SAMPLES; + + if (ringBuffer->nextOutput == ringBuffer->buffer + RING_BUFFER_SIZE_SAMPLES) { + ringBuffer->nextOutput = ringBuffer->buffer; + } + + if (ringBuffer->diffLastWriteNextOutput() < BUFFER_LENGTH_SAMPLES) { + std::cout << "Starved\n"; + ringBuffer->endOfLastWrite = NULL; + } + } + + // copy whatever is in the queueBuffer to the outputLeft and outputRight buffers + memcpy(outputLeft, queueBuffer, BUFFER_LENGTH_BYTES); + memcpy(outputRight, queueBuffer, BUFFER_LENGTH_BYTES); + + } else { + + for (int s = 0; s < NUM_AUDIO_SOURCES; s++) { + AudioSource *source = data->sources[s]; + + glm::vec3 headPos = data->linkedHead->getPos(); + glm::vec3 sourcePos = source->position; + + int startPointer = source->samplePointer; + int wrapAroundSamples = (BUFFER_LENGTH_SAMPLES) - (source->lengthInSamples - source->samplePointer); + + if (wrapAroundSamples <= 0) { + memcpy(data->samplesToQueue, source->sourceData + source->samplePointer, BUFFER_LENGTH_BYTES); + source->samplePointer += (BUFFER_LENGTH_SAMPLES); + } else { + memcpy(data->samplesToQueue, source->sourceData + source->samplePointer, (source->lengthInSamples - source->samplePointer) * sizeof(int16_t)); + memcpy(data->samplesToQueue + (source->lengthInSamples - source->samplePointer), source->sourceData, wrapAroundSamples * sizeof(int16_t)); + source->samplePointer = wrapAroundSamples; + } + + float distance = sqrtf(powf(-headPos[0] - sourcePos[0], 2) + powf(-headPos[2] - sourcePos[2], 2)); + float distanceAmpRatio = powf(0.5, cbrtf(distance * 10)); + + float angleToSource = angle_to(headPos * -1.f, sourcePos, data->linkedHead->getRenderYaw(), data->linkedHead->getYaw()) * M_PI/180; + float sinRatio = sqrt(fabsf(sinf(angleToSource))); + int numSamplesDelay = PHASE_DELAY_AT_90 * sinRatio; + + float phaseAmpRatio = 1.f - (AMPLITUDE_RATIO_AT_90 * sinRatio); + + // std::cout << "S: " << numSamplesDelay << " A: " << angleToSource << " S: " << sinRatio << " AR: " << phaseAmpRatio << "\n"; + + int16_t *leadingOutput = angleToSource > 0 ? outputLeft : outputRight; + int16_t *trailingOutput = angleToSource > 0 ? outputRight : outputLeft; + + for (int i = 0; i < BUFFER_LENGTH_SAMPLES; i++) { + data->samplesToQueue[i] *= distanceAmpRatio / NUM_AUDIO_SOURCES; + leadingOutput[i] += data->samplesToQueue[i]; - delete bufferToCopy; - - if (bufferToCopy == source->oldestData) { - source->oldestData = NULL; + if (i >= numSamplesDelay) { + trailingOutput[i] += data->samplesToQueue[i - numSamplesDelay]; } else { - source->newestData = NULL; + int sampleIndex = startPointer - numSamplesDelay + i; + + if (sampleIndex < 0) { + sampleIndex += source->lengthInSamples; + } + + trailingOutput[i] += source->sourceData[sampleIndex] * (distanceAmpRatio * phaseAmpRatio / NUM_AUDIO_SOURCES); } } - - pthread_mutex_unlock(&jitterMutex); } -// } else { -// glm::vec3 headPos = data->linkedHead->getPos(); -// glm::vec3 sourcePos = source->position; -// -// int startPointer = source->samplePointer; -// int wrapAroundSamples = (BUFFER_LENGTH_SAMPLES) - (source->lengthInSamples - source->samplePointer); -// -// if (wrapAroundSamples <= 0) { -// memcpy(data->samplesToQueue, source->sourceData + source->samplePointer, BUFFER_LENGTH_BYTES); -// source->samplePointer += (BUFFER_LENGTH_SAMPLES); -// } else { -// memcpy(data->samplesToQueue, source->sourceData + source->samplePointer, (source->lengthInSamples - source->samplePointer) * sizeof(int16_t)); -// memcpy(data->samplesToQueue + (source->lengthInSamples - source->samplePointer), source->sourceData, wrapAroundSamples * sizeof(int16_t)); -// source->samplePointer = wrapAroundSamples; -// } -// -// float distance = sqrtf(powf(-headPos[0] - sourcePos[0], 2) + powf(-headPos[2] - sourcePos[2], 2)); -// float distanceAmpRatio = powf(0.5, cbrtf(distance * 10)); -// -// float angleToSource = angle_to(headPos * -1.f, sourcePos, data->linkedHead->getRenderYaw(), data->linkedHead->getYaw()) * M_PI/180; -// float sinRatio = sqrt(fabsf(sinf(angleToSource))); -// int numSamplesDelay = PHASE_DELAY_AT_90 * sinRatio; -// -// float phaseAmpRatio = 1.f - (AMPLITUDE_RATIO_AT_90 * sinRatio); -// -// // std::cout << "S: " << numSamplesDelay << " A: " << angleToSource << " S: " << sinRatio << " AR: " << phaseAmpRatio << "\n"; -// -// int16_t *leadingOutput = angleToSource > 0 ? outputLeft : outputRight; -// int16_t *trailingOutput = angleToSource > 0 ? outputRight : outputLeft; -// -// for (int i = 0; i < BUFFER_LENGTH_SAMPLES; i++) { -// data->samplesToQueue[i] *= distanceAmpRatio / NUM_AUDIO_SOURCES; -// leadingOutput[i] += data->samplesToQueue[i]; -// -// if (i >= numSamplesDelay) { -// trailingOutput[i] += data->samplesToQueue[i - numSamplesDelay]; -// } else { -// int sampleIndex = startPointer - numSamplesDelay + i; -// -// if (sampleIndex < 0) { -// sampleIndex += source->lengthInSamples; -// } -// -// trailingOutput[i] += source->sourceData[sampleIndex] * (distanceAmpRatio * phaseAmpRatio / NUM_AUDIO_SOURCES); -// } -// } -// } -// } } return paContinue; @@ -202,30 +216,58 @@ void *receiveAudioViaUDP(void *args) { logFile << timeDiff << std::endl; } - AudioSource *inputSource = sharedAudioData->sources[0]; + AudioRingBuffer *ringBuffer = sharedAudioData->ringBuffer; - pthread_mutex_lock(&jitterMutex); + int16_t *copyToPointer; + bool needsJitterBuffer = ringBuffer->endOfLastWrite == NULL; + short bufferSampleOverlap = 0; - if (inputSource->newestData != NULL) { - if (inputSource->oldestData != NULL) { - delete inputSource->oldestData; - } - - inputSource->oldestData = inputSource->newestData; - inputSource->newestData = NULL; + if (!needsJitterBuffer && ringBuffer->diffLastWriteNextOutput() > RING_BUFFER_SIZE_SAMPLES - PACKET_LENGTH_SAMPLES) { + needsJitterBuffer = true; } - - inputSource->newestData = new AudioSource::JitterBuffer(); - inputSource->newestData->audioData = new int16_t[BUFFER_LENGTH_SAMPLES]; - memcpy(inputSource->newestData->audioData, receivedData, BUFFER_LENGTH_BYTES); - inputSource->newestData->receiveTime = currentReceiveTime; - pthread_mutex_unlock(&jitterMutex); + if (needsJitterBuffer) { + // we'll need a jitter buffer + // reset the ring buffer and write + copyToPointer = ringBuffer->buffer; + } else { + copyToPointer = ringBuffer->endOfLastWrite; + + // check for possibility of overlap + bufferSampleOverlap = ringBuffer->bufferOverlap(copyToPointer, PACKET_LENGTH_SAMPLES); + } + + if (!bufferSampleOverlap) { + if (needsJitterBuffer) { + // we need to inject a jitter buffer + short jitterBufferSamples = JITTER_BUFFER_LENGTH_MSECS * (SAMPLE_RATE / 1000); + + // add silence for jitter buffer and then the received packet + memset(copyToPointer, 0, jitterBufferSamples * sizeof(int16_t)); + memcpy(copyToPointer + jitterBufferSamples, receivedData, PACKET_LENGTH_BYTES); + + // the end of the write is the pointer to the buffer + packet + jitter buffer + ringBuffer->endOfLastWrite = ringBuffer->buffer + PACKET_LENGTH_SAMPLES + jitterBufferSamples; + } else { + // no jitter buffer, no overlap + // just copy the recieved data to the right spot and then add packet length to previous pointer + memcpy(copyToPointer, receivedData, PACKET_LENGTH_BYTES); + ringBuffer->endOfLastWrite += PACKET_LENGTH_SAMPLES; + } + } else { + // no jitter buffer, but overlap + // copy to the end, and then from the begining to the overlap + memcpy(copyToPointer, receivedData, (PACKET_LENGTH_SAMPLES - bufferSampleOverlap) * sizeof(int16_t)); + memcpy(ringBuffer->buffer, receivedData + bufferSampleOverlap, bufferSampleOverlap * sizeof(int16_t)); + + // the end of the write is the amount of overlap + ringBuffer->endOfLastWrite = ringBuffer->buffer + bufferSampleOverlap; + } if (LOG_SAMPLE_DELAY) { gettimeofday(&previousReceiveTime, NULL); } - } + } } } @@ -248,18 +290,17 @@ bool Audio::init(Head *mainHead) if (err != paNoError) goto error; if (ECHO_SERVER_TEST) { - data = new AudioData(1, BUFFER_LENGTH_BYTES); + data = new AudioData(BUFFER_LENGTH_BYTES); // setup a UDPSocket data->audioSocket = new UDPSocket(AUDIO_UDP_LISTEN_PORT); + data->ringBuffer = new AudioRingBuffer(RING_BUFFER_SIZE_SAMPLES); pthread_t audioReceiveThread; AudioRecThreadStruct threadArgs; threadArgs.sharedAudioData = data; - pthread_mutex_init(&jitterMutex, NULL); - pthread_create(&audioReceiveThread, NULL, receiveAudioViaUDP, (void *) &threadArgs); } else { data = new AudioData(NUM_AUDIO_SOURCES, BUFFER_LENGTH_BYTES); @@ -337,8 +378,6 @@ bool Audio::terminate () logFile.close(); } - pthread_mutex_destroy(&jitterMutex); - return true; error: diff --git a/Source/AudioData.cpp b/Source/AudioData.cpp index a9c7de60bc..00f550af92 100644 --- a/Source/AudioData.cpp +++ b/Source/AudioData.cpp @@ -8,6 +8,12 @@ #include "AudioData.h" +AudioData::AudioData(int bufferLength) { + sources = NULL; + + samplesToQueue = new int16_t[bufferLength / sizeof(int16_t)]; +} + AudioData::AudioData(int numberOfSources, int bufferLength) { _numberOfSources = numberOfSources; @@ -21,9 +27,12 @@ AudioData::AudioData(int numberOfSources, int bufferLength) { } AudioData::~AudioData() { - for (int s = 0; s < _numberOfSources; s++) { - delete sources[s]; + if (sources != NULL) { + for (int s = 0; s < _numberOfSources; s++) { + delete sources[s]; + } } + delete[] samplesToQueue; } \ No newline at end of file diff --git a/Source/AudioData.h b/Source/AudioData.h index 127ef91503..7e926f1049 100644 --- a/Source/AudioData.h +++ b/Source/AudioData.h @@ -10,6 +10,7 @@ #define __interface__AudioData__ #include +#include "AudioRingBuffer.h" #include "AudioSource.h" #include "Head.h" #include "UDPSocket.h" @@ -17,11 +18,15 @@ class AudioData { public: Head *linkedHead; + + AudioRingBuffer *ringBuffer; AudioSource **sources; + UDPSocket *audioSocket; int16_t *samplesToQueue; + AudioData(int bufferLength); AudioData(int numberOfSources, int bufferLength); ~AudioData(); diff --git a/Source/AudioRingBuffer.cpp b/Source/AudioRingBuffer.cpp new file mode 100644 index 0000000000..6cb617457d --- /dev/null +++ b/Source/AudioRingBuffer.cpp @@ -0,0 +1,44 @@ +// +// AudioRingBuffer.cpp +// interface +// +// Created by Stephen Birarda on 2/1/13. +// Copyright (c) 2013 HighFidelity, Inc. All rights reserved. +// + +#include "AudioRingBuffer.h" + +AudioRingBuffer::AudioRingBuffer(short ringBufferSamples) { + ringBufferLengthSamples = ringBufferSamples; + + endOfLastWrite = NULL; + + buffer = new int16_t[ringBufferLengthSamples]; + nextOutput = buffer; +}; + +AudioRingBuffer::~AudioRingBuffer() { + delete[] buffer; +}; + +short AudioRingBuffer::diffLastWriteNextOutput() +{ + short sampleDifference = endOfLastWrite - nextOutput; + + if (sampleDifference < 0) { + sampleDifference += ringBufferLengthSamples; + } + + return sampleDifference; +} + +short AudioRingBuffer::bufferOverlap(int16_t *pointer, short addedDistance) +{ + short samplesLeft = (buffer + ringBufferLengthSamples) - pointer; + + if (samplesLeft <= addedDistance) { + return addedDistance - samplesLeft; + } else { + return 0; + } +} diff --git a/Source/AudioRingBuffer.h b/Source/AudioRingBuffer.h new file mode 100644 index 0000000000..c78038afb3 --- /dev/null +++ b/Source/AudioRingBuffer.h @@ -0,0 +1,28 @@ +// +// AudioRingBuffer.h +// interface +// +// Created by Stephen Birarda on 2/1/13. +// Copyright (c) 2013 HighFidelity, Inc. All rights reserved. +// + +#ifndef __interface__AudioRingBuffer__ +#define __interface__AudioRingBuffer__ + +#include + +class AudioRingBuffer { + public: + int16_t *nextOutput; + int16_t *endOfLastWrite; + int16_t *buffer; + short ringBufferLengthSamples; + + short diffLastWriteNextOutput(); + short bufferOverlap(int16_t *pointer, short addedDistance); + + AudioRingBuffer(short ringBufferSamples); + ~AudioRingBuffer(); +}; + +#endif /* defined(__interface__AudioRingBuffer__) */ diff --git a/interface.xcodeproj/project.pbxproj b/interface.xcodeproj/project.pbxproj index 339fa60896..e89ebccbc4 100644 --- a/interface.xcodeproj/project.pbxproj +++ b/interface.xcodeproj/project.pbxproj @@ -20,6 +20,7 @@ 533BF9D516B31A4700AC31BB /* jeska.raw in CopyFiles */ = {isa = PBXBuildFile; fileRef = 533BF9D316B31A3B00AC31BB /* jeska.raw */; }; 535B821116B9BED400D18440 /* lodepng.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 535B820F16B9BED400D18440 /* lodepng.cpp */; }; 538BA8A316B1B71E000BF99C /* love.raw in CopyFiles */ = {isa = PBXBuildFile; fileRef = 538BA8A216B1B719000BF99C /* love.raw */; }; + 53A4EAB216BC770A00F07F4C /* AudioRingBuffer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 53A4EAB016BC770A00F07F4C /* AudioRingBuffer.cpp */; }; 53ACC39816B9C59500ABD227 /* Oscilloscope.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 53ACC39616B9C59500ABD227 /* Oscilloscope.cpp */; }; 53CF371716B9C039001FCB05 /* Agent.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 53CF36EC16B9C039001FCB05 /* Agent.cpp */; }; 53CF371816B9C039001FCB05 /* Audio.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 53CF36EE16B9C039001FCB05 /* Audio.cpp */; }; @@ -308,6 +309,8 @@ 535B820F16B9BED400D18440 /* lodepng.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = lodepng.cpp; sourceTree = ""; }; 535B821016B9BED400D18440 /* lodepng.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = lodepng.h; sourceTree = ""; }; 538BA8A216B1B719000BF99C /* love.raw */ = {isa = PBXFileReference; lastKnownFileType = file; path = love.raw; sourceTree = ""; }; + 53A4EAB016BC770A00F07F4C /* AudioRingBuffer.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = AudioRingBuffer.cpp; sourceTree = ""; }; + 53A4EAB116BC770A00F07F4C /* AudioRingBuffer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AudioRingBuffer.h; sourceTree = ""; }; 53ACC39616B9C59500ABD227 /* Oscilloscope.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = Oscilloscope.cpp; sourceTree = ""; }; 53ACC39716B9C59500ABD227 /* Oscilloscope.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Oscilloscope.h; sourceTree = ""; }; 53CF36EC16B9C039001FCB05 /* Agent.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = Agent.cpp; sourceTree = ""; }; @@ -451,6 +454,8 @@ 53CF371416B9C039001FCB05 /* Util.cpp */, 53CF371516B9C039001FCB05 /* Util.h */, 53CF371616B9C039001FCB05 /* world.h */, + 53A4EAB016BC770A00F07F4C /* AudioRingBuffer.cpp */, + 53A4EAB116BC770A00F07F4C /* AudioRingBuffer.h */, ); path = Source; sourceTree = ""; @@ -868,6 +873,7 @@ 53CF372916B9C039001FCB05 /* UDPSocket.cpp in Sources */, 53CF372A16B9C039001FCB05 /* Util.cpp in Sources */, 53ACC39816B9C59500ABD227 /* Oscilloscope.cpp in Sources */, + 53A4EAB216BC770A00F07F4C /* AudioRingBuffer.cpp in Sources */, ); runOnlyForDeploymentPostprocessing = 0; };