From 4693082db05daf90ce24df64124d5dcbd2d6b02a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 15 May 2013 10:54:36 -0700 Subject: [PATCH 01/14] refactor Audio to remove requirement of AudioData --- interface/src/Application.cpp | 20 +- interface/src/Application.h | 5 +- interface/src/Audio.cpp | 359 ++++++++++++----------------- interface/src/Audio.h | 55 ++--- interface/src/AudioData.cpp | 48 ---- interface/src/AudioData.h | 54 ----- libraries/shared/src/AgentList.cpp | 12 +- libraries/shared/src/AgentList.h | 1 - 8 files changed, 182 insertions(+), 372 deletions(-) delete mode 100644 interface/src/AudioData.cpp delete mode 100644 interface/src/AudioData.h diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 1cf45ac750..81ab975fe8 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -154,7 +154,7 @@ Application::Application(int& argc, char** argv) : _oculusProgram(0), _oculusDistortionScale(1.25), #ifndef _WIN32 - _audio(&_audioScope, &_myAvatar), + _audio(&_audioScope), #endif _stopNetworkReceiveThread(false), _packetCount(0), @@ -202,10 +202,6 @@ Application::Application(int& argc, char** argv) : // the callback for our instance of AgentList is attachNewHeadToAgent AgentList::getInstance()->linkedDataCreateCallback = &attachNewHeadToAgent; - #ifndef _WIN32 - AgentList::getInstance()->audioMixerSocketUpdate = &audioMixerUpdate; - #endif - #ifdef _WIN32 WSADATA WsaData; int wsaresult = WSAStartup(MAKEWORD(2,2), &WsaData); @@ -913,11 +909,7 @@ void Application::terminate() { // Close serial port // close(serial_fd); - _myAvatar.writeAvatarDataToFile(); - - #ifndef _WIN32 - _audio.terminate(); - #endif + _myAvatar.writeAvatarDataToFile(); if (_enableNetworkThread) { _stopNetworkReceiveThread = true; @@ -1278,7 +1270,7 @@ void Application::updateAvatar(float deltaTime) { // Get audio loudness data from audio input device #ifndef _WIN32 - _myAvatar.setLoudness(_audio.getInputLoudness()); + _myAvatar.setLoudness(_audio.getLastInputLoudness()); #endif // Update Avatar with latest camera and view frustum data... @@ -1972,12 +1964,6 @@ void Application::attachNewHeadToAgent(Agent *newAgent) { } } -#ifndef _WIN32 -void Application::audioMixerUpdate(in_addr_t newMixerAddress, in_port_t newMixerPort) { - static_cast(QCoreApplication::instance())->_audio.updateMixerParams(newMixerAddress, newMixerPort); -} -#endif - // Receive packets from other agents/servers and decide what to do with them! void* Application::networkReceive(void* args) { sockaddr senderAddress; diff --git a/interface/src/Application.h b/interface/src/Application.h index 5418d0740e..d6b0c45ecb 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -57,6 +57,8 @@ public: void mouseReleaseEvent(QMouseEvent* event); void wheelEvent(QWheelEvent* event); + + const Avatar& getAvatar() const { return _myAvatar; } private slots: @@ -119,9 +121,6 @@ private: void setMenuShortcutsEnabled(bool enabled); static void attachNewHeadToAgent(Agent *newAgent); - #ifndef _WIN32 - static void audioMixerUpdate(in_addr_t newMixerAddress, in_port_t newMixerPort); - #endif static void* networkReceive(void* args); QMainWindow* _window; diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 8a455ec419..9b9ebb222b 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -10,19 +10,21 @@ #include #include #include - #include #include + #include #include #include #include +#include +#include + +#include "Application.h" #include "Audio.h" #include "Util.h" #include "Log.h" -Oscilloscope * scope; - const int NUM_AUDIO_CHANNELS = 2; const int PACKET_LENGTH_BYTES = 1024; @@ -55,15 +57,8 @@ const float AUDIO_CALLBACK_MSECS = (float)BUFFER_LENGTH_SAMPLES / (float)SAMPLE_ const int AGENT_LOOPBACK_MODIFIER = 307; -const char LOCALHOST_MIXER[] = "0.0.0.0"; -const char WORKCLUB_MIXER[] = "192.168.1.19"; -const char EC2_WEST_MIXER[] = "54.241.92.53"; - -const int AUDIO_UDP_LISTEN_PORT = 55444; - int starve_counter = 0; StDev stdev; -bool stopAudioReceiveThread = false; int samplesLeftForFlange = 0; int lastYawMeasuredMaximum = 0; @@ -71,8 +66,6 @@ float flangeIntensity = 0; float flangeRate = 0; float flangeWeight = 0; -timeval firstPlaybackTimer; -int packetsReceivedThisPlayback = 0; float usecsAtStartup = 0; /** @@ -94,20 +87,23 @@ float usecsAtStartup = 0; * @return Should be of type PaStreamCallbackResult. Return paComplete to end the stream, or paContinue to continue (default). Can be used to end the stream from within the callback. */ - -int audioCallback (const void *inputBuffer, - void *outputBuffer, +int audioCallback (const void* inputBuffer, + void* outputBuffer, unsigned long frames, const PaStreamCallbackTimeInfo *timeInfo, PaStreamCallbackFlags statusFlags, - void *userData) -{ - AudioData *data = (AudioData *) userData; + void* userData) { + + Audio* parentAudio = (Audio*) userData; + AgentList* agentList = AgentList::getInstance(); + + Application* interface = (Application*) QCoreApplication::instance(); + Avatar interfaceAvatar = interface->getAvatar(); int16_t *inputLeft = ((int16_t **) inputBuffer)[0]; // Add Procedural effects to input samples - data->addProceduralSounds(inputLeft, BUFFER_LENGTH_SAMPLES); + parentAudio->addProceduralSounds(inputLeft, BUFFER_LENGTH_SAMPLES); if (inputLeft != NULL) { @@ -118,17 +114,14 @@ int audioCallback (const void *inputBuffer, } loudness /= BUFFER_LENGTH_SAMPLES; - data->lastInputLoudness = loudness; + parentAudio->_lastInputLoudness = loudness; // add data to the scope - scope->addSamples(0, inputLeft, BUFFER_LENGTH_SAMPLES); + parentAudio->_scope->addSamples(0, inputLeft, BUFFER_LENGTH_SAMPLES); - if (data->mixerAddress != 0) { - sockaddr_in audioMixerSocket; - audioMixerSocket.sin_family = AF_INET; - audioMixerSocket.sin_addr.s_addr = data->mixerAddress; - audioMixerSocket.sin_port = data->mixerPort; - + Agent* audioMixer = agentList->soloAgentOfType(AGENT_TYPE_AUDIO_MIXER); + + if (audioMixer) { int leadingBytes = 2 + (sizeof(float) * 4); // we need the amount of bytes in the buffer + 1 for type @@ -139,14 +132,14 @@ int audioCallback (const void *inputBuffer, unsigned char *currentPacketPtr = dataPacket + 1; // memcpy the three float positions - memcpy(currentPacketPtr, &data->linkedAvatar->getHeadPosition(), sizeof(float) * 3); + memcpy(currentPacketPtr, &interfaceAvatar.getHeadPosition(), sizeof(float) * 3); currentPacketPtr += (sizeof(float) * 3); // tell the mixer not to add additional attenuation to our source *(currentPacketPtr++) = 255; // memcpy the corrected render yaw - float correctedYaw = fmodf(-1 * data->linkedAvatar->getAbsoluteHeadYaw(), 360); + float correctedYaw = fmodf(-1 * interfaceAvatar.getAbsoluteHeadYaw(), 360); if (correctedYaw > 180) { correctedYaw -= 360; @@ -154,29 +147,30 @@ int audioCallback (const void *inputBuffer, correctedYaw += 360; } - if (data->mixerLoopbackFlag) { + if (parentAudio->_mixerLoopbackFlag) { correctedYaw = correctedYaw > 0 - ? correctedYaw + AGENT_LOOPBACK_MODIFIER - : correctedYaw - AGENT_LOOPBACK_MODIFIER; + ? correctedYaw + AGENT_LOOPBACK_MODIFIER + : correctedYaw - AGENT_LOOPBACK_MODIFIER; } memcpy(currentPacketPtr, &correctedYaw, sizeof(float)); - currentPacketPtr += sizeof(float); + currentPacketPtr += sizeof(float); // copy the audio data to the last BUFFER_LENGTH_BYTES bytes of the data packet memcpy(currentPacketPtr, inputLeft, BUFFER_LENGTH_BYTES); - data->audioSocket->send((sockaddr *)&audioMixerSocket, dataPacket, BUFFER_LENGTH_BYTES + leadingBytes); + agentList->getAgentSocket().send(audioMixer->getActiveSocket(), dataPacket, BUFFER_LENGTH_BYTES + leadingBytes); } + } - int16_t *outputLeft = ((int16_t **) outputBuffer)[0]; - int16_t *outputRight = ((int16_t **) outputBuffer)[1]; + int16_t* outputLeft = ((int16_t**) outputBuffer)[0]; + int16_t* outputRight = ((int16_t**) outputBuffer)[1]; memset(outputLeft, 0, PACKET_LENGTH_BYTES_PER_CHANNEL); memset(outputRight, 0, PACKET_LENGTH_BYTES_PER_CHANNEL); - - AudioRingBuffer *ringBuffer = data->ringBuffer; + + AudioRingBuffer* ringBuffer = &parentAudio->_ringBuffer; // if we've been reset, and there isn't any new packets yet // just play some silence @@ -184,15 +178,16 @@ int audioCallback (const void *inputBuffer, if (ringBuffer->getEndOfLastWrite() != NULL) { if (!ringBuffer->isStarted() && ringBuffer->diffLastWriteNextOutput() < PACKET_LENGTH_SAMPLES + JITTER_BUFFER_SAMPLES) { - //printLog("Held back, buffer has %d of %d samples required.\n", ringBuffer->diffLastWriteNextOutput(), PACKET_LENGTH_SAMPLES + JITTER_BUFFER_SAMPLES); + //printLog("Held back, buffer has %d of %d samples required.\n", + // ringBuffer->diffLastWriteNextOutput(), PACKET_LENGTH_SAMPLES + JITTER_BUFFER_SAMPLES); } else if (ringBuffer->diffLastWriteNextOutput() < PACKET_LENGTH_SAMPLES) { ringBuffer->setStarted(false); starve_counter++; - packetsReceivedThisPlayback = 0; - + parentAudio->_packetsReceivedThisPlayback = 0; + // printLog("Starved #%d\n", starve_counter); - data->wasStarved = 10; // Frames to render the indication that the system was starved. + parentAudio->_wasStarved = 10; // Frames to render the indication that the system was starved. } else { if (!ringBuffer->isStarted()) { ringBuffer->setStarted(true); @@ -206,7 +201,7 @@ int audioCallback (const void *inputBuffer, // if we haven't fired off the flange effect, check if we should // TODO: lastMeasuredHeadYaw is now relative to body - check if this still works. - int lastYawMeasured = fabsf(data->linkedAvatar->getLastMeasuredHeadYaw()); + int lastYawMeasured = fabsf(interfaceAvatar.getLastMeasuredHeadYaw()); if (!samplesLeftForFlange && lastYawMeasured > MIN_FLANGE_EFFECT_THRESHOLD) { // we should flange for one second @@ -216,8 +211,8 @@ int audioCallback (const void *inputBuffer, samplesLeftForFlange = SAMPLE_RATE; flangeIntensity = MIN_FLANGE_INTENSITY + - ((lastYawMeasuredMaximum - MIN_FLANGE_EFFECT_THRESHOLD) / (float)(MAX_FLANGE_EFFECT_THRESHOLD - MIN_FLANGE_EFFECT_THRESHOLD)) * - (1 - MIN_FLANGE_INTENSITY); + ((lastYawMeasuredMaximum - MIN_FLANGE_EFFECT_THRESHOLD) / (float)(MAX_FLANGE_EFFECT_THRESHOLD - MIN_FLANGE_EFFECT_THRESHOLD)) * + (1 - MIN_FLANGE_INTENSITY); flangeRate = FLANGE_BASE_RATE * flangeIntensity; flangeWeight = MAX_FLANGE_SAMPLE_WEIGHT * flangeIntensity; @@ -235,7 +230,7 @@ int audioCallback (const void *inputBuffer, if (samplesLeftForFlange != SAMPLE_RATE || s >= (SAMPLE_RATE / 2000)) { // we have a delayed sample to add to this sample - + int16_t *flangeFrame = ringBuffer->getNextOutput(); int flangeIndex = s - sampleFlangeDelay; @@ -267,8 +262,8 @@ int audioCallback (const void *inputBuffer, } // add data to the scope - scope->addSamples(1, outputLeft, PACKET_LENGTH_SAMPLES_PER_CHANNEL); - scope->addSamples(2, outputRight, PACKET_LENGTH_SAMPLES_PER_CHANNEL); + parentAudio->_scope->addSamples(1, outputLeft, PACKET_LENGTH_SAMPLES_PER_CHANNEL); + parentAudio->_scope->addSamples(2, outputRight, PACKET_LENGTH_SAMPLES_PER_CHANNEL); ringBuffer->setNextOutput(ringBuffer->getNextOutput() + PACKET_LENGTH_SAMPLES); @@ -278,140 +273,99 @@ int audioCallback (const void *inputBuffer, } } - gettimeofday(&data->lastCallback, NULL); + gettimeofday(&parentAudio->_lastCallbackTime, NULL); return paContinue; } -void Audio::updateMixerParams(in_addr_t newMixerAddress, in_port_t newMixerPort) { - audioData->mixerAddress = newMixerAddress; - audioData->mixerPort = newMixerPort; -} - -struct AudioRecThreadStruct { - AudioData *sharedAudioData; -}; - -void *receiveAudioViaUDP(void *args) { - AudioRecThreadStruct *threadArgs = (AudioRecThreadStruct *) args; - AudioData *sharedAudioData = threadArgs->sharedAudioData; - - int16_t *receivedData = new int16_t[PACKET_LENGTH_SAMPLES]; - ssize_t receivedBytes; - - // Init Jitter timer values - timeval previousReceiveTime, currentReceiveTime = {}; - gettimeofday(&previousReceiveTime, NULL); - gettimeofday(¤tReceiveTime, NULL); - - int totalPacketsReceived = 0; - - stdev.reset(); - - while (!stopAudioReceiveThread) { - - if (sharedAudioData->audioSocket->receive((void *)receivedData, &receivedBytes)) { - - gettimeofday(¤tReceiveTime, NULL); - totalPacketsReceived++; - - double tDiff = diffclock(&previousReceiveTime, ¤tReceiveTime); - //printLog("tDiff %4.1f\n", tDiff); - // Discard first few received packets for computing jitter (often they pile up on start) - if (totalPacketsReceived > 3) stdev.addValue(tDiff); - if (stdev.getSamples() > 500) { - sharedAudioData->measuredJitter = stdev.getStDev(); - //printLog("Avg: %4.2f, Stdev: %4.2f\n", stdev.getAverage(), sharedAudioData->measuredJitter); - stdev.reset(); - } - - AudioRingBuffer *ringBuffer = sharedAudioData->ringBuffer; - - - if (!ringBuffer->isStarted()) { - packetsReceivedThisPlayback++; - } - else { - //printLog("Audio packet received at %6.0f\n", usecTimestampNow()/1000); - } - if (packetsReceivedThisPlayback == 1) gettimeofday(&firstPlaybackTimer, NULL); - - ringBuffer->parseData((unsigned char *)receivedData, PACKET_LENGTH_BYTES); - - previousReceiveTime = currentReceiveTime; - } +void outputPortAudioError(PaError error) { + if (error != paNoError) { + printLog("-- portaudio termination error --\n"); + printLog("PortAudio error (%d): %s\n", error, Pa_GetErrorText(error)); } - - pthread_exit(0); } -void Audio::setMixerLoopbackFlag(bool newMixerLoopbackFlag) { - audioData->mixerLoopbackFlag = newMixerLoopbackFlag; -} - -bool Audio::getMixerLoopbackFlag() { - return audioData->mixerLoopbackFlag; -} - -/** - * Initialize portaudio and start an audio stream. - * Should be called at the beginning of program exection. - * @seealso Audio::terminate - * @return Returns true if successful or false if an error occurred. -Use Audio::getError() to retrieve the error code. - */ -Audio::Audio(Oscilloscope* s, Avatar* linkedAvatar) { - paError = Pa_Initialize(); - if (paError != paNoError) goto error; +Audio::Audio(Oscilloscope* scope) : + _ringBuffer(RING_BUFFER_SAMPLES, PACKET_LENGTH_SAMPLES), + _scope(scope), + _averagedLatency(0.0), + _measuredJitter(0), + _wasStarved(0), + _totalPacketsReceived(0) { - scope = s; + outputPortAudioError(Pa_Initialize()); - audioData = new AudioData(); + outputPortAudioError(Pa_OpenDefaultStream(&_stream, + 2, + 2, + (paInt16 | paNonInterleaved), + SAMPLE_RATE, + BUFFER_LENGTH_SAMPLES, + audioCallback, + (void*) this)); - audioData->linkedAvatar = linkedAvatar; - - // setup a UDPSocket - audioData->audioSocket = new UDPSocket(AUDIO_UDP_LISTEN_PORT); - audioData->ringBuffer = new AudioRingBuffer(RING_BUFFER_SAMPLES, PACKET_LENGTH_SAMPLES); - - AudioRecThreadStruct threadArgs; - threadArgs.sharedAudioData = audioData; - - pthread_create(&audioReceiveThread, NULL, receiveAudioViaUDP, (void *) &threadArgs); - - paError = Pa_OpenDefaultStream(&stream, - 2, // input channels - 2, // output channels - (paInt16 | paNonInterleaved), // sample format - SAMPLE_RATE, // sample rate (hz) - BUFFER_LENGTH_SAMPLES, // frames per buffer - audioCallback, // callback function - (void *) audioData); // user data to be passed to callback - if (paError != paNoError) goto error; - - initialized = true; + _initialized = true; // start the stream now that sources are good to go - Pa_StartStream(stream); - if (paError != paNoError) goto error; - - - return; - -error: - printLog("-- Failed to initialize portaudio --\n"); - printLog("PortAudio error (%d): %s\n", paError, Pa_GetErrorText(paError)); - initialized = false; - delete[] audioData; + outputPortAudioError(Pa_StartStream(_stream)); + + gettimeofday(&_lastReceiveTime, NULL); } - -float Audio::getInputLoudness() const { - return audioData->lastInputLoudness; +Audio::~Audio() { + if (_initialized) { + outputPortAudioError(Pa_CloseStream(_stream)); + outputPortAudioError(Pa_Terminate()); + } } -void Audio::render(int screenWidth, int screenHeight) -{ - if (initialized) { +// Take a pointer to the acquired microphone input samples and add procedural sounds +void Audio::addProceduralSounds(int16_t* inputBuffer, int numSamples) { + const float MAX_AUDIBLE_VELOCITY = 6.0; + const float MIN_AUDIBLE_VELOCITY = 0.1; + float speed = glm::length(_lastVelocity); + float volume = 400 * (1.f - speed/MAX_AUDIBLE_VELOCITY); + + // Add a noise-modulated sinewave with volume that tapers off with speed increasing + if ((speed > MIN_AUDIBLE_VELOCITY) && (speed < MAX_AUDIBLE_VELOCITY)) { + for (int i = 0; i < numSamples; i++) { + inputBuffer[i] += (int16_t) ((cosf((float)i / 8.f * speed) * randFloat()) * volume * speed) ; + } + } +} + +void Audio::addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBytes) { + timeval currentReceiveTime; + gettimeofday(¤tReceiveTime, NULL); + _totalPacketsReceived++; + + double timeDiff = diffclock(&_lastReceiveTime, ¤tReceiveTime); + + // Discard first few received packets for computing jitter (often they pile up on start) + if (_totalPacketsReceived > 3) { + stdev.addValue(timeDiff); + } + + if (stdev.getSamples() > 500) { + _measuredJitter = stdev.getStDev(); + //printLog("Avg: %4.2f, Stdev: %4.2f\n", stdev.getAverage(), sharedAudioData->measuredJitter); + stdev.reset(); + } + + if (!_ringBuffer.isStarted()) { + _packetsReceivedThisPlayback++; + } + + if (_packetsReceivedThisPlayback == 1) { + gettimeofday(&_firstPlaybackTime, NULL); + } + + _ringBuffer.parseData((unsigned char *)receivedData, PACKET_LENGTH_BYTES); + + _lastReceiveTime = currentReceiveTime; +} + +void Audio::render(int screenWidth, int screenHeight) { + if (_initialized) { glLineWidth(2.0); glBegin(GL_LINES); glColor3f(1,1,1); @@ -444,19 +398,20 @@ void Audio::render(int screenWidth, int screenHeight) timeval currentTime; gettimeofday(¤tTime, NULL); float timeLeftInCurrentBuffer = 0; - if (audioData->lastCallback.tv_usec > 0) { - timeLeftInCurrentBuffer = AUDIO_CALLBACK_MSECS - diffclock(&audioData->lastCallback, ¤tTime); + if (_lastCallbackTime.tv_usec > 0) { + timeLeftInCurrentBuffer = AUDIO_CALLBACK_MSECS - diffclock(&_lastCallbackTime, ¤tTime); } // /(1000.0*(float)BUFFER_LENGTH_SAMPLES/(float)SAMPLE_RATE) * frameWidth - if (audioData->ringBuffer->getEndOfLastWrite() != NULL) - remainingBuffer = audioData->ringBuffer->diffLastWriteNextOutput() / PACKET_LENGTH_SAMPLES * AUDIO_CALLBACK_MSECS; + if (_ringBuffer.getEndOfLastWrite() != NULL) + remainingBuffer = _ringBuffer.diffLastWriteNextOutput() / PACKET_LENGTH_SAMPLES * AUDIO_CALLBACK_MSECS; - if (audioData->wasStarved == 0) glColor3f(0, 1, 0); - else { - glColor3f(0.5 + (float)audioData->wasStarved/20.0, 0, 0); - audioData->wasStarved--; + if (_wasStarved == 0) { + glColor3f(0, 1, 0); + } else { + glColor3f(0.5 + (_wasStarved / 20.0f), 0, 0); + _wasStarved--; } glBegin(GL_QUADS); @@ -466,26 +421,30 @@ void Audio::render(int screenWidth, int screenHeight) glVertex2f(startX, bottomY - 2); glEnd(); - if (audioData->averagedLatency == 0.0) audioData->averagedLatency = remainingBuffer + timeLeftInCurrentBuffer; - else audioData->averagedLatency = 0.99*audioData->averagedLatency + 0.01*((float)remainingBuffer + (float)timeLeftInCurrentBuffer); + if (_averagedLatency == 0.0) { + _averagedLatency = remainingBuffer + timeLeftInCurrentBuffer; + } + else { + _averagedLatency = 0.99f * _averagedLatency + 0.01f * (remainingBuffer + timeLeftInCurrentBuffer); + } // Show a yellow bar with the averaged msecs latency you are hearing (from time of packet receipt) glColor3f(1,1,0); glBegin(GL_QUADS); - glVertex2f(startX + audioData->averagedLatency/AUDIO_CALLBACK_MSECS*frameWidth - 2, topY - 2); - glVertex2f(startX + audioData->averagedLatency/AUDIO_CALLBACK_MSECS*frameWidth + 2, topY - 2); - glVertex2f(startX + audioData->averagedLatency/AUDIO_CALLBACK_MSECS*frameWidth + 2, bottomY + 2); - glVertex2f(startX + audioData->averagedLatency/AUDIO_CALLBACK_MSECS*frameWidth - 2, bottomY + 2); + glVertex2f(startX + _averagedLatency / AUDIO_CALLBACK_MSECS * frameWidth - 2, topY - 2); + glVertex2f(startX + _averagedLatency / AUDIO_CALLBACK_MSECS * frameWidth + 2, topY - 2); + glVertex2f(startX + _averagedLatency / AUDIO_CALLBACK_MSECS * frameWidth + 2, bottomY + 2); + glVertex2f(startX + _averagedLatency / AUDIO_CALLBACK_MSECS * frameWidth - 2, bottomY + 2); glEnd(); char out[40]; - sprintf(out, "%3.0f\n", audioData->averagedLatency); - drawtext(startX + audioData->averagedLatency/AUDIO_CALLBACK_MSECS*frameWidth - 10, topY-10, 0.10, 0, 1, 0, out, 1,1,0); + sprintf(out, "%3.0f\n", _averagedLatency); + drawtext(startX + _averagedLatency / AUDIO_CALLBACK_MSECS * frameWidth - 10, topY-10, 0.10, 0, 1, 0, out, 1,1,0); //drawtext(startX + 0, topY-10, 0.08, 0, 1, 0, out, 1,1,0); // Show a Cyan bar with the most recently measured jitter stdev - int jitterPels = (float) audioData->measuredJitter/ ((1000.0*(float)PACKET_LENGTH_SAMPLES/(float)SAMPLE_RATE)) * (float)frameWidth; + int jitterPels = _measuredJitter / ((1000.0f * PACKET_LENGTH_SAMPLES / SAMPLE_RATE)) * frameWidth; glColor3f(0,1,1); glBegin(GL_QUADS); @@ -495,7 +454,7 @@ void Audio::render(int screenWidth, int screenHeight) glVertex2f(startX + jitterPels - 2, bottomY + 2); glEnd(); - sprintf(out,"%3.1f\n", audioData->measuredJitter); + sprintf(out,"%3.1f\n", _measuredJitter); drawtext(startX + jitterPels - 5, topY-10, 0.10, 0, 1, 0, out, 0,1,1); sprintf(out, "%3.1fms\n", JITTER_BUFFER_LENGTH_MSECS); @@ -503,34 +462,4 @@ void Audio::render(int screenWidth, int screenHeight) } } -/** - * Close the running audio stream, and deinitialize portaudio. - * Should be called at the end of program execution. - * @return Returns true if the initialization was successful, or false if an error occured. - The error code may be retrieved by Audio::getError(). - */ -bool Audio::terminate() { - stopAudioReceiveThread = true; - pthread_join(audioReceiveThread, NULL); - - if (initialized) { - initialized = false; - - paError = Pa_CloseStream(stream); - if (paError != paNoError) goto error; - - paError = Pa_Terminate(); - if (paError != paNoError) goto error; - } - - delete audioData; - - return true; - -error: - printLog("-- portaudio termination error --\n"); - printLog("PortAudio error (%d): %s\n", paError, Pa_GetErrorText(paError)); - return false; -} - #endif diff --git a/interface/src/Audio.h b/interface/src/Audio.h index d2ad9073be..e005682b06 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -10,44 +10,47 @@ #define __interface__Audio__ #include -#include "AudioData.h" + +#include + #include "Oscilloscope.h" #include "Avatar.h" class Audio { public: // initializes audio I/O - Audio(Oscilloscope *s, Avatar *linkedAvatar); - - void render(); + Audio(Oscilloscope* scope); + ~Audio(); + void render(int screenWidth, int screenHeight); - bool getMixerLoopbackFlag(); - void setMixerLoopbackFlag(bool newMixerLoopbackFlag); + void setMixerLoopbackFlag(bool mixerLoopbackFlag) { _mixerLoopbackFlag = mixerLoopbackFlag; } - float getInputLoudness() const; - void updateMixerParams(in_addr_t mixerAddress, in_port_t mixerPort); + float getLastInputLoudness() const { return _lastInputLoudness; }; - void setLastAcceleration(glm::vec3 a) { audioData->setLastAcceleration(a); }; - void setLastVelocity(glm::vec3 v) { audioData->setLastVelocity(v); }; + void setLastAcceleration(glm::vec3 lastAcceleration) { _lastAcceleration = lastAcceleration; }; + void setLastVelocity(glm::vec3 lastVelocity) { _lastVelocity = lastVelocity; }; - // terminates audio I/O - bool terminate(); + void addProceduralSounds(int16_t* inputBuffer, int numSamples); + + void addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBytes); private: - bool initialized; - AudioData *audioData; - - // protects constructor so that public init method is used - Audio(); - - // hold potential error returned from PortAudio functions - PaError paError; - - // audio stream handle - PaStream *stream; - - // audio receive thread - pthread_t audioReceiveThread; + bool _initialized; + PaStream* _stream; + AudioRingBuffer _ringBuffer; + Oscilloscope* _scope; + timeval _lastCallbackTime; + timeval _lastReceiveTime; + float _averagedLatency; + float _measuredJitter; + int _wasStarved; + float _lastInputLoudness; + bool _mixerLoopbackFlag; + glm::vec3 _lastVelocity; + glm::vec3 _lastAcceleration; + int _totalPacketsReceived; + timeval _firstPlaybackTime; + int _packetsReceivedThisPlayback; // give access to AudioData class from audioCallback friend int audioCallback (const void*, void*, unsigned long, const PaStreamCallbackTimeInfo*, PaStreamCallbackFlags, void*); diff --git a/interface/src/AudioData.cpp b/interface/src/AudioData.cpp deleted file mode 100644 index 83b53b0ea3..0000000000 --- a/interface/src/AudioData.cpp +++ /dev/null @@ -1,48 +0,0 @@ -// -// AudioData.cpp -// interface -// -// Created by Stephen Birarda on 1/29/13. -// Copyright (c) 2013 HighFidelity, Inc. All rights reserved. -// -#ifndef _WIN32 - -#include "AudioData.h" - -AudioData::AudioData() { - mixerAddress = 0; - mixerPort = 0; - - averagedLatency = 0.0; - lastCallback.tv_usec = 0; - wasStarved = 0; - measuredJitter = 0; - jitterBuffer = 0; - - mixerLoopbackFlag = false; - audioSocket = NULL; -} - - -AudioData::~AudioData() { - delete audioSocket; -} - -// Take a pointer to the acquired microphone input samples and add procedural sounds -void AudioData::addProceduralSounds(int16_t* inputBuffer, int numSamples) { - const float MAX_AUDIBLE_VELOCITY = 6.0; - const float MIN_AUDIBLE_VELOCITY = 0.1; - float speed = glm::length(_lastVelocity); - float volume = 400 * (1.f - speed/MAX_AUDIBLE_VELOCITY); - // Add a noise-modulated sinewave with volume that tapers off with speed increasing - if ((speed > MIN_AUDIBLE_VELOCITY) && (speed < MAX_AUDIBLE_VELOCITY)) { - for (int i = 0; i < numSamples; i++) { - inputBuffer[i] += (int16_t) ((cosf((float)i / 8.f * speed) * randFloat()) * volume * speed) ; - } - } - - return; -} - - -#endif diff --git a/interface/src/AudioData.h b/interface/src/AudioData.h deleted file mode 100644 index 80de3df6dc..0000000000 --- a/interface/src/AudioData.h +++ /dev/null @@ -1,54 +0,0 @@ -// -// AudioData.h -// interface -// -// Created by Stephen Birarda on 1/29/13. -// Copyright (c) 2013 HighFidelity, Inc. All rights reserved. -// - -#ifndef __interface__AudioData__ -#define __interface__AudioData__ - -#include -#include -#include "AudioRingBuffer.h" -#include "UDPSocket.h" -#include "Avatar.h" - -class AudioData { - public: - AudioData(); - ~AudioData(); - AudioRingBuffer *ringBuffer; - - UDPSocket *audioSocket; - - Avatar *linkedAvatar; - - // store current mixer address and port - in_addr_t mixerAddress; - in_port_t mixerPort; - - timeval lastCallback; - float averagedLatency; - float measuredJitter; - float jitterBuffer; - int wasStarved; - - float lastInputLoudness; - - bool mixerLoopbackFlag; - - // Added avatar acceleration and velocity for procedural effects sounds from client - void setLastVelocity(glm::vec3 v) { _lastVelocity = v; }; - void setLastAcceleration(glm::vec3 a) { _lastAcceleration = a; }; - void addProceduralSounds(int16_t* inputBuffer, int numSamples); - - private: - glm::vec3 _lastVelocity; - glm::vec3 _lastAcceleration; - - -}; - -#endif /* defined(__interface__AudioData__) */ diff --git a/libraries/shared/src/AgentList.cpp b/libraries/shared/src/AgentList.cpp index 44364bfc88..3bca0fa12b 100644 --- a/libraries/shared/src/AgentList.cpp +++ b/libraries/shared/src/AgentList.cpp @@ -243,14 +243,10 @@ bool AgentList::addOrUpdateAgent(sockaddr *publicSocket, sockaddr *localSocket, // set the agent active right away newAgent->activatePublicSocket(); } - - if (newAgent->getType() == AGENT_TYPE_AUDIO_MIXER && audioMixerSocketUpdate != NULL) { - // this is an audio mixer - // for now that means we need to tell the audio class - // to use the local socket information the domain server gave us - sockaddr_in *publicSocketIn = (sockaddr_in *)publicSocket; - audioMixerSocketUpdate(publicSocketIn->sin_addr.s_addr, publicSocketIn->sin_port); - } else if (newAgent->getType() == AGENT_TYPE_VOXEL || newAgent->getType() == AGENT_TYPE_AVATAR_MIXER) { + + if (newAgent->getType() == AGENT_TYPE_VOXEL || + newAgent->getType() == AGENT_TYPE_AVATAR_MIXER || + newAgent->getType() == AGENT_TYPE_AUDIO_MIXER) { // this is currently the cheat we use to talk directly to our test servers on EC2 // to be removed when we have a proper identification strategy newAgent->activatePublicSocket(); diff --git a/libraries/shared/src/AgentList.h b/libraries/shared/src/AgentList.h index db71357114..399085404d 100644 --- a/libraries/shared/src/AgentList.h +++ b/libraries/shared/src/AgentList.h @@ -46,7 +46,6 @@ public: AgentListIterator end() const; void(*linkedDataCreateCallback)(Agent *); - void(*audioMixerSocketUpdate)(in_addr_t, in_port_t); int size() { return _numAgents; } From 26c34bb86c36cd573f332ff2f8e471f4831f7d18 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 15 May 2013 11:29:52 -0700 Subject: [PATCH 02/14] sensible initialization for Audio member variables --- interface/src/Audio.cpp | 15 ++++++++++----- interface/src/Audio.h | 1 - 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 9b9ebb222b..68aeb5b3ef 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -285,12 +285,19 @@ void outputPortAudioError(PaError error) { } Audio::Audio(Oscilloscope* scope) : + _stream(NULL), _ringBuffer(RING_BUFFER_SAMPLES, PACKET_LENGTH_SAMPLES), _scope(scope), _averagedLatency(0.0), _measuredJitter(0), _wasStarved(0), - _totalPacketsReceived(0) { + _lastInputLoudness(0), + _mixerLoopbackFlag(false), + _lastVelocity(0), + _lastAcceleration(0), + _totalPacketsReceived(0), + _firstPlaybackTime(), + _packetsReceivedThisPlayback(0) { outputPortAudioError(Pa_Initialize()); @@ -303,8 +310,6 @@ Audio::Audio(Oscilloscope* scope) : audioCallback, (void*) this)); - _initialized = true; - // start the stream now that sources are good to go outputPortAudioError(Pa_StartStream(_stream)); @@ -312,7 +317,7 @@ Audio::Audio(Oscilloscope* scope) : } Audio::~Audio() { - if (_initialized) { + if (_stream) { outputPortAudioError(Pa_CloseStream(_stream)); outputPortAudioError(Pa_Terminate()); } @@ -365,7 +370,7 @@ void Audio::addReceivedAudioToBuffer(unsigned char* receivedData, int receivedB } void Audio::render(int screenWidth, int screenHeight) { - if (_initialized) { + if (_stream) { glLineWidth(2.0); glBegin(GL_LINES); glColor3f(1,1,1); diff --git a/interface/src/Audio.h b/interface/src/Audio.h index e005682b06..4a2b2ec974 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -35,7 +35,6 @@ public: void addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBytes); private: - bool _initialized; PaStream* _stream; AudioRingBuffer _ringBuffer; Oscilloscope* _scope; From ce34a8f3e097565f2e4e6db0c27ef2ce98912ef2 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 15 May 2013 11:43:49 -0700 Subject: [PATCH 03/14] add packet headers and agent types for audio clarity --- audio-mixer/src/main.cpp | 7 +++++-- interface/src/Audio.cpp | 2 +- libraries/shared/src/AgentTypes.h | 3 ++- libraries/shared/src/PacketHeaders.h | 2 ++ 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/audio-mixer/src/main.cpp b/audio-mixer/src/main.cpp index 6b63075da5..eecd1ae59d 100644 --- a/audio-mixer/src/main.cpp +++ b/audio-mixer/src/main.cpp @@ -268,9 +268,12 @@ int main(int argc, const char* argv[]) { // pull any new audio data from agents off of the network stack while (agentList->getAgentSocket().receive(agentAddress, packetData, &receivedBytes)) { - if (packetData[0] == PACKET_HEADER_INJECT_AUDIO) { + if (packetData[0] == PACKET_HEADER_INJECT_AUDIO || packetData[0] == PACKET_HEADER_MICROPHONE_AUDIO) { + char agentType = (packetData[0] == PACKET_HEADER_MICROPHONE_AUDIO) + ? AGENT_TYPE_AVATAR + : AGENT_TYPE_AUDIO_INJECTOR; - if (agentList->addOrUpdateAgent(agentAddress, agentAddress, packetData[0], agentList->getLastAgentID())) { + if (agentList->addOrUpdateAgent(agentAddress, agentAddress, agentType, agentList->getLastAgentID())) { agentList->increaseAgentID(); } diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 68aeb5b3ef..407c520fa8 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -128,7 +128,7 @@ int audioCallback (const void* inputBuffer, // + 12 for 3 floats for position + float for bearing + 1 attenuation byte unsigned char dataPacket[BUFFER_LENGTH_BYTES + leadingBytes]; - dataPacket[0] = PACKET_HEADER_INJECT_AUDIO; + dataPacket[0] = PACKET_HEADER_MICROPHONE_AUDIO; unsigned char *currentPacketPtr = dataPacket + 1; // memcpy the three float positions diff --git a/libraries/shared/src/AgentTypes.h b/libraries/shared/src/AgentTypes.h index c43af79446..e095cb1035 100644 --- a/libraries/shared/src/AgentTypes.h +++ b/libraries/shared/src/AgentTypes.h @@ -20,8 +20,9 @@ // Agent Type Codes const char AGENT_TYPE_DOMAIN = 'D'; const char AGENT_TYPE_VOXEL = 'V'; -const char AGENT_TYPE_AVATAR = 'I'; // could also be injector??? +const char AGENT_TYPE_AVATAR = 'I'; const char AGENT_TYPE_AUDIO_MIXER = 'M'; const char AGENT_TYPE_AVATAR_MIXER = 'W'; +const char AGENT_TYPE_AUDIO_INJECTOR = 'A'; #endif diff --git a/libraries/shared/src/PacketHeaders.h b/libraries/shared/src/PacketHeaders.h index 1b385a68f7..8d9c4ae880 100644 --- a/libraries/shared/src/PacketHeaders.h +++ b/libraries/shared/src/PacketHeaders.h @@ -20,6 +20,8 @@ const PACKET_HEADER PACKET_HEADER_PING_REPLY = 'R'; const PACKET_HEADER PACKET_HEADER_HEAD_DATA = 'H'; const PACKET_HEADER PACKET_HEADER_Z_COMMAND = 'Z'; const PACKET_HEADER PACKET_HEADER_INJECT_AUDIO = 'I'; +const PACKET_HEADER PACKET_HEADER_MIXED_AUDIO = 'A'; +const PACKET_HEADER PACKET_HEADER_MICROPHONE_AUDIO = 'M'; const PACKET_HEADER PACKET_HEADER_SET_VOXEL = 'S'; const PACKET_HEADER PACKET_HEADER_SET_VOXEL_DESTRUCTIVE = 'O'; const PACKET_HEADER PACKET_HEADER_ERASE_VOXEL = 'E'; From e83710e45ef862d08152932ebf61a7d4db7abb19 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 15 May 2013 12:00:04 -0700 Subject: [PATCH 04/14] prepend audio packets from the mixer with a packet header --- audio-mixer/src/main.cpp | 21 ++++++++++++------ interface/src/Application.cpp | 3 +++ libraries/shared/src/AudioRingBuffer.cpp | 27 ++++++++++++++---------- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/audio-mixer/src/main.cpp b/audio-mixer/src/main.cpp index eecd1ae59d..cf649b6a52 100644 --- a/audio-mixer/src/main.cpp +++ b/audio-mixer/src/main.cpp @@ -99,6 +99,11 @@ int main(int argc, const char* argv[]) { int nextFrame = 0; timeval startTime; + unsigned char clientPacket[BUFFER_LENGTH_BYTES + 1]; + clientPacket[0] = PACKET_HEADER_MIXED_AUDIO; + + int16_t clientSamples[BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2] = {}; + gettimeofday(&startTime, NULL); while (true) { @@ -129,8 +134,9 @@ int main(int argc, const char* argv[]) { for (AgentList::iterator agent = agentList->begin(); agent != agentList->end(); agent++) { AudioRingBuffer* agentRingBuffer = (AudioRingBuffer*) agent->getLinkedData(); - - int16_t clientMix[BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2] = {}; + + // zero out the client mix for this agent + memset(clientSamples, 0, sizeof(clientSamples)); for (AgentList::iterator otherAgent = agentList->begin(); otherAgent != agentList->end(); otherAgent++) { if (otherAgent != agent || (otherAgent == agent && agentRingBuffer->shouldLoopbackForAgent())) { @@ -219,11 +225,11 @@ int main(int argc, const char* argv[]) { } int16_t* goodChannel = bearingRelativeAngleToSource > 0.0f - ? clientMix + BUFFER_LENGTH_SAMPLES_PER_CHANNEL - : clientMix; + ? clientSamples + BUFFER_LENGTH_SAMPLES_PER_CHANNEL + : clientSamples; int16_t* delayedChannel = bearingRelativeAngleToSource > 0.0f - ? clientMix - : clientMix + BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + ? clientSamples + : clientSamples + BUFFER_LENGTH_SAMPLES_PER_CHANNEL; int16_t* delaySamplePointer = otherAgentBuffer->getNextOutput() == otherAgentBuffer->getBuffer() ? otherAgentBuffer->getBuffer() + RING_BUFFER_SAMPLES - numSamplesDelay @@ -249,7 +255,8 @@ int main(int argc, const char* argv[]) { } } - agentList->getAgentSocket().send(agent->getPublicSocket(), clientMix, BUFFER_LENGTH_BYTES); + memcpy(clientPacket + 1, clientSamples, sizeof(clientSamples)); + agentList->getAgentSocket().send(agent->getPublicSocket(), clientSamples, BUFFER_LENGTH_BYTES + 1); } // push forward the next output pointers for any audio buffers we used diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 81ab975fe8..b2c4ab364f 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1995,6 +1995,9 @@ void* Application::networkReceive(void* args) { printf("The rotation: %f, %f, %f\n", rotationRates[0], rotationRates[1], rotationRates[2]); break; + case PACKET_HEADER_MIXED_AUDIO: + app->_audio.addReceivedAudioToBuffer(app->_incomingPacket, bytesReceived); + break; case PACKET_HEADER_VOXEL_DATA: case PACKET_HEADER_VOXEL_DATA_MONOCHROME: case PACKET_HEADER_Z_COMMAND: diff --git a/libraries/shared/src/AudioRingBuffer.cpp b/libraries/shared/src/AudioRingBuffer.cpp index b49b65945c..746ff523f8 100644 --- a/libraries/shared/src/AudioRingBuffer.cpp +++ b/libraries/shared/src/AudioRingBuffer.cpp @@ -7,6 +7,9 @@ // #include + +#include "PacketHeaders.h" + #include "AudioRingBuffer.h" AudioRingBuffer::AudioRingBuffer(int ringSamples, int bufferSamples) : @@ -46,18 +49,22 @@ AudioRingBuffer* AudioRingBuffer::clone() const { const int AGENT_LOOPBACK_MODIFIER = 307; int AudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes) { - if (numBytes > (_bufferLengthSamples * sizeof(int16_t))) { + + unsigned char* dataBuffer = sourceBuffer + 1; + + if (sourceBuffer[0] == PACKET_HEADER_INJECT_AUDIO || + sourceBuffer[0] == PACKET_HEADER_MICROPHONE_AUDIO) { + // if this came from an injector or interface client + // there's data required for spatialization to pull out - unsigned char *dataPtr = sourceBuffer + 1; + memcpy(&_position, dataBuffer, sizeof(_position)); + dataBuffer += (sizeof(_position)); - memcpy(&_position, dataPtr, sizeof(_position)); - dataPtr += (sizeof(_position)); - - unsigned int attenuationByte = *(dataPtr++); + unsigned int attenuationByte = *(dataBuffer++); _attenuationRatio = attenuationByte / 255.0f; - memcpy(&_bearing, dataPtr, sizeof(float)); - dataPtr += sizeof(_bearing); + memcpy(&_bearing, dataBuffer, sizeof(float)); + dataBuffer += sizeof(_bearing); if (_bearing > 180 || _bearing < -180) { // we were passed an invalid bearing because this agent wants loopback (pressed the H key) @@ -70,8 +77,6 @@ int AudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes) { } else { _shouldLoopbackForAgent = false; } - - sourceBuffer = dataPtr; } if (!_endOfLastWrite) { @@ -82,7 +87,7 @@ int AudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes) { _started = false; } - memcpy(_endOfLastWrite, sourceBuffer, _bufferLengthSamples * sizeof(int16_t)); + memcpy(_endOfLastWrite, dataBuffer, _bufferLengthSamples * sizeof(int16_t)); _endOfLastWrite += _bufferLengthSamples; From e4a734fc78a470c89ae984a2d9519f16242e9dcb Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 15 May 2013 12:01:32 -0700 Subject: [PATCH 05/14] refactor constructor in AudioInjector --- libraries/shared/src/AudioInjector.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/libraries/shared/src/AudioInjector.cpp b/libraries/shared/src/AudioInjector.cpp index 6e65c168d0..e08100130e 100644 --- a/libraries/shared/src/AudioInjector.cpp +++ b/libraries/shared/src/AudioInjector.cpp @@ -22,12 +22,9 @@ const float BUFFER_SEND_INTERVAL_USECS = (BUFFER_LENGTH_SAMPLES / SAMPLE_RATE) * AudioInjector::AudioInjector(const char* filename) : _numTotalBytesAudio(0), + _position(), _bearing(0), - _attenuationModifier(255) -{ - _position[0] = 0.0f; - _position[1] = 0.0f; - _position[2] = 0.0f; + _attenuationModifier(255) { std::fstream sourceFile; From 182b0b4fa16030a50f29d58cebd3eed0c3879870 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 15 May 2013 12:13:39 -0700 Subject: [PATCH 06/14] data sent to client from mixer must include packet header --- audio-mixer/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/audio-mixer/src/main.cpp b/audio-mixer/src/main.cpp index cf649b6a52..88cfb9d1bf 100644 --- a/audio-mixer/src/main.cpp +++ b/audio-mixer/src/main.cpp @@ -256,7 +256,7 @@ int main(int argc, const char* argv[]) { } memcpy(clientPacket + 1, clientSamples, sizeof(clientSamples)); - agentList->getAgentSocket().send(agent->getPublicSocket(), clientSamples, BUFFER_LENGTH_BYTES + 1); + agentList->getAgentSocket().send(agent->getPublicSocket(), clientPacket, BUFFER_LENGTH_BYTES + 1); } // push forward the next output pointers for any audio buffers we used From fb42e8df04a2b4c06e399857cfafe35020d878d3 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 15 May 2013 12:19:40 -0700 Subject: [PATCH 07/14] correct some global references in Audio.cpp --- interface/src/Audio.cpp | 50 +++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 407c520fa8..04d918a500 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -57,7 +57,7 @@ const float AUDIO_CALLBACK_MSECS = (float)BUFFER_LENGTH_SAMPLES / (float)SAMPLE_ const int AGENT_LOOPBACK_MODIFIER = 307; -int starve_counter = 0; +int numStarves = 0; StDev stdev; int samplesLeftForFlange = 0; @@ -183,7 +183,7 @@ int audioCallback (const void* inputBuffer, } else if (ringBuffer->diffLastWriteNextOutput() < PACKET_LENGTH_SAMPLES) { ringBuffer->setStarted(false); - starve_counter++; + ::numStarves++; parentAudio->_packetsReceivedThisPlayback = 0; // printLog("Starved #%d\n", starve_counter); @@ -203,19 +203,20 @@ int audioCallback (const void* inputBuffer, int lastYawMeasured = fabsf(interfaceAvatar.getLastMeasuredHeadYaw()); - if (!samplesLeftForFlange && lastYawMeasured > MIN_FLANGE_EFFECT_THRESHOLD) { + if (!::samplesLeftForFlange && lastYawMeasured > MIN_FLANGE_EFFECT_THRESHOLD) { // we should flange for one second - if ((lastYawMeasuredMaximum = std::max(lastYawMeasuredMaximum, lastYawMeasured)) != lastYawMeasured) { - lastYawMeasuredMaximum = std::min(lastYawMeasuredMaximum, MIN_FLANGE_EFFECT_THRESHOLD); + if ((::lastYawMeasuredMaximum = std::max(::lastYawMeasuredMaximum, lastYawMeasured)) != lastYawMeasured) { + ::lastYawMeasuredMaximum = std::min(::lastYawMeasuredMaximum, MIN_FLANGE_EFFECT_THRESHOLD); - samplesLeftForFlange = SAMPLE_RATE; + ::samplesLeftForFlange = SAMPLE_RATE; - flangeIntensity = MIN_FLANGE_INTENSITY + - ((lastYawMeasuredMaximum - MIN_FLANGE_EFFECT_THRESHOLD) / (float)(MAX_FLANGE_EFFECT_THRESHOLD - MIN_FLANGE_EFFECT_THRESHOLD)) * - (1 - MIN_FLANGE_INTENSITY); + ::flangeIntensity = MIN_FLANGE_INTENSITY + + ((::lastYawMeasuredMaximum - MIN_FLANGE_EFFECT_THRESHOLD) / + (float)(MAX_FLANGE_EFFECT_THRESHOLD - MIN_FLANGE_EFFECT_THRESHOLD)) * + (1 - MIN_FLANGE_INTENSITY); - flangeRate = FLANGE_BASE_RATE * flangeIntensity; - flangeWeight = MAX_FLANGE_SAMPLE_WEIGHT * flangeIntensity; + ::flangeRate = FLANGE_BASE_RATE * ::flangeIntensity; + ::flangeWeight = MAX_FLANGE_SAMPLE_WEIGHT * ::flangeIntensity; } } @@ -224,11 +225,12 @@ int audioCallback (const void* inputBuffer, int leftSample = ringBuffer->getNextOutput()[s]; int rightSample = ringBuffer->getNextOutput()[s + PACKET_LENGTH_SAMPLES_PER_CHANNEL]; - if (samplesLeftForFlange > 0) { - float exponent = (SAMPLE_RATE - samplesLeftForFlange - (SAMPLE_RATE / flangeRate)) / (SAMPLE_RATE / flangeRate); - int sampleFlangeDelay = (SAMPLE_RATE / (1000 * flangeIntensity)) * powf(2, exponent); + if (::samplesLeftForFlange > 0) { + float exponent = (SAMPLE_RATE - ::samplesLeftForFlange - (SAMPLE_RATE / ::flangeRate)) / + (SAMPLE_RATE / ::flangeRate); + int sampleFlangeDelay = (SAMPLE_RATE / (1000 * ::flangeIntensity)) * powf(2, exponent); - if (samplesLeftForFlange != SAMPLE_RATE || s >= (SAMPLE_RATE / 2000)) { + if (::samplesLeftForFlange != SAMPLE_RATE || s >= (SAMPLE_RATE / 2000)) { // we have a delayed sample to add to this sample int16_t *flangeFrame = ringBuffer->getNextOutput(); @@ -246,13 +248,13 @@ int audioCallback (const void* inputBuffer, int16_t leftFlangeSample = flangeFrame[flangeIndex]; int16_t rightFlangeSample = flangeFrame[flangeIndex + PACKET_LENGTH_SAMPLES_PER_CHANNEL]; - leftSample = (1 - flangeWeight) * leftSample + (flangeWeight * leftFlangeSample); - rightSample = (1 - flangeWeight) * rightSample + (flangeWeight * rightFlangeSample); + leftSample = (1 - ::flangeWeight) * leftSample + (::flangeWeight * leftFlangeSample); + rightSample = (1 - ::flangeWeight) * rightSample + (::flangeWeight * rightFlangeSample); - samplesLeftForFlange--; + ::samplesLeftForFlange--; - if (samplesLeftForFlange == 0) { - lastYawMeasuredMaximum = 0; + if (::samplesLeftForFlange == 0) { + ::lastYawMeasuredMaximum = 0; } } } @@ -347,13 +349,13 @@ void Audio::addReceivedAudioToBuffer(unsigned char* receivedData, int receivedB // Discard first few received packets for computing jitter (often they pile up on start) if (_totalPacketsReceived > 3) { - stdev.addValue(timeDiff); + ::stdev.addValue(timeDiff); } - if (stdev.getSamples() > 500) { - _measuredJitter = stdev.getStDev(); + if (::stdev.getSamples() > 500) { + _measuredJitter = ::stdev.getStDev(); //printLog("Avg: %4.2f, Stdev: %4.2f\n", stdev.getAverage(), sharedAudioData->measuredJitter); - stdev.reset(); + ::stdev.reset(); } if (!_ringBuffer.isStarted()) { From 44b12cb59398f637e06cdd700bacf47ca8eac608 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 15 May 2013 12:21:43 -0700 Subject: [PATCH 08/14] spacing changes --- interface/src/Audio.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 04d918a500..a33b8e5854 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -302,7 +302,6 @@ Audio::Audio(Oscilloscope* scope) : _packetsReceivedThisPlayback(0) { outputPortAudioError(Pa_Initialize()); - outputPortAudioError(Pa_OpenDefaultStream(&_stream, 2, 2, @@ -330,7 +329,7 @@ void Audio::addProceduralSounds(int16_t* inputBuffer, int numSamples) { const float MAX_AUDIBLE_VELOCITY = 6.0; const float MIN_AUDIBLE_VELOCITY = 0.1; float speed = glm::length(_lastVelocity); - float volume = 400 * (1.f - speed/MAX_AUDIBLE_VELOCITY); + float volume = 400 * (1.f - speed / MAX_AUDIBLE_VELOCITY); // Add a noise-modulated sinewave with volume that tapers off with speed increasing if ((speed > MIN_AUDIBLE_VELOCITY) && (speed < MAX_AUDIBLE_VELOCITY)) { From c5fdfd64e7adbfd4a7c98f9eb137e3a7b516ddf9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 15 May 2013 12:22:11 -0700 Subject: [PATCH 09/14] remove extra space --- interface/src/Audio.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index a33b8e5854..17d14720ec 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -339,7 +339,7 @@ void Audio::addProceduralSounds(int16_t* inputBuffer, int numSamples) { } } -void Audio::addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBytes) { +void Audio::addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBytes) { timeval currentReceiveTime; gettimeofday(¤tReceiveTime, NULL); _totalPacketsReceived++; From b186dd0f6520a52394e80fbe28c19c47805581d9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 15 May 2013 12:24:58 -0700 Subject: [PATCH 10/14] comment cleanup for audioCallback --- interface/src/Audio.cpp | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 17d14720ec..938db374cc 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -68,25 +68,12 @@ float flangeWeight = 0; float usecsAtStartup = 0; -/** - * Audio callback used by portaudio. - * Communicates with Audio via a shared pointer to Audio::data. - * Writes input audio channels (if they exist) into Audio::data->buffer, - multiplied by Audio::data->inputGain. - * Then writes Audio::data->buffer into output audio channels, and clears - the portion of Audio::data->buffer that has been read from for reuse. - * - * @param[in] inputBuffer A pointer to an internal portaudio data buffer containing data read by portaudio. - * @param[out] outputBuffer A pointer to an internal portaudio data buffer to be read by the configured output device. - * @param[in] frames Number of frames that portaudio requests to be read/written. - (Valid size of input/output buffers = frames * number of channels (2) * sizeof data type (float)). - * @param[in] timeInfo Portaudio time info. Currently unused. - * @param[in] statusFlags Portaudio status flags. Currently unused. - * @param[in] userData Pointer to supplied user data (in this case, a pointer to Audio::data). - Used to communicate with external code (since portaudio calls this function from another thread). - * @return Should be of type PaStreamCallbackResult. Return paComplete to end the stream, or paContinue to continue (default). - Can be used to end the stream from within the callback. - */ +// inputBuffer A pointer to an internal portaudio data buffer containing data read by portaudio. +// outputBuffer A pointer to an internal portaudio data buffer to be read by the configured output device. +// frames Number of frames that portaudio requests to be read/written. +// timeInfo Portaudio time info. Currently unused. +// statusFlags Portaudio status flags. Currently unused. +// userData Pointer to supplied user data (in this case, a pointer to the parent Audio object int audioCallback (const void* inputBuffer, void* outputBuffer, unsigned long frames, From 274c810dec99b1e58e532474e7733af4974f20a5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 15 May 2013 12:35:41 -0700 Subject: [PATCH 11/14] constantize the number of packets to discard for stats --- interface/src/Audio.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 938db374cc..5f46465346 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -286,8 +286,8 @@ Audio::Audio(Oscilloscope* scope) : _lastAcceleration(0), _totalPacketsReceived(0), _firstPlaybackTime(), - _packetsReceivedThisPlayback(0) { - + _packetsReceivedThisPlayback(0) +{ outputPortAudioError(Pa_Initialize()); outputPortAudioError(Pa_OpenDefaultStream(&_stream, 2, @@ -315,18 +315,21 @@ Audio::~Audio() { void Audio::addProceduralSounds(int16_t* inputBuffer, int numSamples) { const float MAX_AUDIBLE_VELOCITY = 6.0; const float MIN_AUDIBLE_VELOCITY = 0.1; + float speed = glm::length(_lastVelocity); float volume = 400 * (1.f - speed / MAX_AUDIBLE_VELOCITY); // Add a noise-modulated sinewave with volume that tapers off with speed increasing if ((speed > MIN_AUDIBLE_VELOCITY) && (speed < MAX_AUDIBLE_VELOCITY)) { for (int i = 0; i < numSamples; i++) { - inputBuffer[i] += (int16_t) ((cosf((float)i / 8.f * speed) * randFloat()) * volume * speed) ; + inputBuffer[i] += (int16_t)((cosf((float) i / 8.f * speed) * randFloat()) * volume * speed); } } } void Audio::addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBytes) { + const int NUM_INITIAL_PACKETS_DISCARD = 3; + timeval currentReceiveTime; gettimeofday(¤tReceiveTime, NULL); _totalPacketsReceived++; @@ -334,7 +337,7 @@ void Audio::addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBy double timeDiff = diffclock(&_lastReceiveTime, ¤tReceiveTime); // Discard first few received packets for computing jitter (often they pile up on start) - if (_totalPacketsReceived > 3) { + if (_totalPacketsReceived > NUM_INITIAL_PACKETS_DISCARD) { ::stdev.addValue(timeDiff); } From 63a2b203d913fc4cbc086d30172ffa695e339b41 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 15 May 2013 12:36:19 -0700 Subject: [PATCH 12/14] constantize VOLUME_BASELINE in addProceduralSounds --- interface/src/Audio.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 5f46465346..774a2945ff 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -315,9 +315,10 @@ Audio::~Audio() { void Audio::addProceduralSounds(int16_t* inputBuffer, int numSamples) { const float MAX_AUDIBLE_VELOCITY = 6.0; const float MIN_AUDIBLE_VELOCITY = 0.1; + const int VOLUME_BASELINE = 400; float speed = glm::length(_lastVelocity); - float volume = 400 * (1.f - speed / MAX_AUDIBLE_VELOCITY); + float volume = VOLUME_BASELINE * (1.f - speed / MAX_AUDIBLE_VELOCITY); // Add a noise-modulated sinewave with volume that tapers off with speed increasing if ((speed > MIN_AUDIBLE_VELOCITY) && (speed < MAX_AUDIBLE_VELOCITY)) { From 351412f6e8e541f9398211b8e69c9179cc7e2d35 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 15 May 2013 12:36:53 -0700 Subject: [PATCH 13/14] constantize SOUND_PITCH in addProceduralSounds --- interface/src/Audio.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 774a2945ff..348978002d 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -316,6 +316,7 @@ void Audio::addProceduralSounds(int16_t* inputBuffer, int numSamples) { const float MAX_AUDIBLE_VELOCITY = 6.0; const float MIN_AUDIBLE_VELOCITY = 0.1; const int VOLUME_BASELINE = 400; + const float SOUND_PITCH = 8.f; float speed = glm::length(_lastVelocity); float volume = VOLUME_BASELINE * (1.f - speed / MAX_AUDIBLE_VELOCITY); @@ -323,7 +324,7 @@ void Audio::addProceduralSounds(int16_t* inputBuffer, int numSamples) { // Add a noise-modulated sinewave with volume that tapers off with speed increasing if ((speed > MIN_AUDIBLE_VELOCITY) && (speed < MAX_AUDIBLE_VELOCITY)) { for (int i = 0; i < numSamples; i++) { - inputBuffer[i] += (int16_t)((cosf((float) i / 8.f * speed) * randFloat()) * volume * speed); + inputBuffer[i] += (int16_t)((cosf((float) i / SOUND_PITCH * speed) * randFloat()) * volume * speed); } } } From e0ee50fa9c1cd22b930558d234dfeddfaf730bb6 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 15 May 2013 12:38:08 -0700 Subject: [PATCH 14/14] spacing changes in Audio render --- interface/src/Audio.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 348978002d..ceaac13dda 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -390,7 +390,6 @@ void Audio::render(int screenWidth, int screenHeight) { } glEnd(); - // Show a bar with the amount of audio remaining in ring buffer beyond current playback float remainingBuffer = 0; timeval currentTime; @@ -399,8 +398,6 @@ void Audio::render(int screenWidth, int screenHeight) { if (_lastCallbackTime.tv_usec > 0) { timeLeftInCurrentBuffer = AUDIO_CALLBACK_MSECS - diffclock(&_lastCallbackTime, ¤tTime); } - - // /(1000.0*(float)BUFFER_LENGTH_SAMPLES/(float)SAMPLE_RATE) * frameWidth if (_ringBuffer.getEndOfLastWrite() != NULL) remainingBuffer = _ringBuffer.diffLastWriteNextOutput() / PACKET_LENGTH_SAMPLES * AUDIO_CALLBACK_MSECS; @@ -421,8 +418,7 @@ void Audio::render(int screenWidth, int screenHeight) { if (_averagedLatency == 0.0) { _averagedLatency = remainingBuffer + timeLeftInCurrentBuffer; - } - else { + } else { _averagedLatency = 0.99f * _averagedLatency + 0.01f * (remainingBuffer + timeLeftInCurrentBuffer); } @@ -437,7 +433,7 @@ void Audio::render(int screenWidth, int screenHeight) { char out[40]; sprintf(out, "%3.0f\n", _averagedLatency); - drawtext(startX + _averagedLatency / AUDIO_CALLBACK_MSECS * frameWidth - 10, topY-10, 0.10, 0, 1, 0, out, 1,1,0); + drawtext(startX + _averagedLatency / AUDIO_CALLBACK_MSECS * frameWidth - 10, topY - 10, 0.10, 0, 1, 0, out, 1,1,0); //drawtext(startX + 0, topY-10, 0.08, 0, 1, 0, out, 1,1,0); // Show a Cyan bar with the most recently measured jitter stdev