From 1a283c3ac061ae4a926a174600972c3c9ebcfc3f Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Wed, 15 May 2013 21:15:55 -0600 Subject: [PATCH] Improvements to scope display, added framework for echo analysis over multiple frames --- interface/src/Application.cpp | 5 ++ interface/src/Audio.cpp | 96 ++++++++++++++++++++++++++-------- interface/src/Audio.h | 11 ++++ interface/src/AudioData.cpp | 79 ---------------------------- interface/src/AudioData.h | 55 ------------------- interface/src/Oscilloscope.cpp | 9 ++-- 6 files changed, 96 insertions(+), 159 deletions(-) delete mode 100644 interface/src/AudioData.cpp delete mode 100644 interface/src/AudioData.h diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index faebf55df3..b161f635b8 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -534,6 +534,10 @@ void Application::keyPressEvent(QKeyEvent* event) { sendVoxelServerAddScene(); break; + case Qt::Key_Semicolon: + _audio.startEchoTest(); + break; + case Qt::Key_L: _displayLevels = !_displayLevels; break; @@ -1641,6 +1645,7 @@ void Application::displayOverlay() { #ifndef _WIN32 _audio.render(_glWidget->width(), _glWidget->height()); _audioScope.render(20, _glWidget->height() - 200); + //_audio.renderEchoCompare(); // PER: Will turn back on to further test echo #endif //noiseTest(_glWidget->width(), _glWidget->height()); diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index df3fab75ed..f9504b9ed7 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -87,22 +87,26 @@ int audioCallback (const void* inputBuffer, Application* interface = (Application*) QCoreApplication::instance(); Avatar interfaceAvatar = interface->getAvatar(); - bool addPing = (randFloat() < 0.005f); - int16_t *inputLeft = ((int16_t **) inputBuffer)[0]; int16_t *outputLeft = ((int16_t **) outputBuffer)[0]; int16_t *outputRight = ((int16_t **) outputBuffer)[1]; - // Compare the input and output streams to look for correlation - parentAudio->analyzeEcho(inputLeft, outputLeft, BUFFER_LENGTH_SAMPLES); - // Add Procedural effects to input samples parentAudio->addProceduralSounds(inputLeft, BUFFER_LENGTH_SAMPLES); - // add data to the scope + // add output (@speakers) data to the scope parentAudio->_scope->addSamples(1, outputLeft, PACKET_LENGTH_SAMPLES_PER_CHANNEL); parentAudio->_scope->addSamples(2, outputRight, PACKET_LENGTH_SAMPLES_PER_CHANNEL); - + + // if needed, add input/output data to echo analysis buffers + if (parentAudio->_gatheringEchoFrames) { + memcpy(parentAudio->_echoInputSamples, inputLeft, + PACKET_LENGTH_SAMPLES_PER_CHANNEL * sizeof(int16_t)); + memcpy(parentAudio->_echoOutputSamples, outputLeft, + PACKET_LENGTH_SAMPLES_PER_CHANNEL * sizeof(int16_t)); + parentAudio->addedPingFrame(); + } + if (inputLeft != NULL) { // Measure the loudness of the signal from the microphone and store in audio object @@ -114,7 +118,7 @@ int audioCallback (const void* inputBuffer, loudness /= BUFFER_LENGTH_SAMPLES; parentAudio->_lastInputLoudness = loudness; - // add data to the scope + // add input (@microphone) data to the scope parentAudio->_scope->addSamples(0, inputLeft, BUFFER_LENGTH_SAMPLES); Agent* audioMixer = agentList->soloAgentOfType(AGENT_TYPE_AUDIO_MIXER); @@ -192,7 +196,6 @@ int audioCallback (const void* inputBuffer, } // play whatever we have in the audio buffer - // if we haven't fired off the flange effect, check if we should // TODO: lastMeasuredHeadYaw is now relative to body - check if this still works. @@ -254,12 +257,8 @@ int audioCallback (const void* inputBuffer, } } - if (!addPing) { - outputLeft[s] = leftSample; - outputRight[s] = rightSample; - } else { - outputLeft[s] = outputRight[s] = (int16_t)(sinf((float) s / 15.f) * 8000.f); - } + outputLeft[s] = leftSample; + outputRight[s] = rightSample; } ringBuffer->setNextOutput(ringBuffer->getNextOutput() + PACKET_LENGTH_SAMPLES); @@ -268,11 +267,19 @@ int audioCallback (const void* inputBuffer, } } } - + if (parentAudio->_sendingEchoPing) { + const float PING_PITCH = 4.f; + const float PING_VOLUME = 32000.f; + for (int s = 0; s < PACKET_LENGTH_SAMPLES_PER_CHANNEL; s++) { + outputLeft[s] = outputRight[s] = (int16_t)(sinf((float) s / PING_PITCH) * PING_VOLUME); + } + parentAudio->_gatheringEchoFrames = true; + } gettimeofday(&parentAudio->_lastCallbackTime, NULL); return paContinue; } + void outputPortAudioError(PaError error) { if (error != paNoError) { printLog("-- portaudio termination error --\n"); @@ -293,8 +300,12 @@ Audio::Audio(Oscilloscope* scope) : _lastAcceleration(0), _totalPacketsReceived(0), _firstPlaybackTime(), - _packetsReceivedThisPlayback(0) -{ + _packetsReceivedThisPlayback(0), + _startEcho(false), + _sendingEchoPing(false), + _echoPingFrameCount(0), + _gatheringEchoFrames(false) +{ outputPortAudioError(Pa_Initialize()); outputPortAudioError(Pa_OpenDefaultStream(&_stream, 2, @@ -307,7 +318,12 @@ Audio::Audio(Oscilloscope* scope) : // start the stream now that sources are good to go outputPortAudioError(Pa_StartStream(_stream)); - + + _echoInputSamples = new int16_t[BUFFER_LENGTH_BYTES]; + _echoOutputSamples = new int16_t[BUFFER_LENGTH_BYTES]; + memset(_echoInputSamples, 0, BUFFER_LENGTH_SAMPLES * sizeof(int)); + memset(_echoOutputSamples, 0, BUFFER_LENGTH_SAMPLES * sizeof(int)); + gettimeofday(&_lastReceiveTime, NULL); } @@ -318,6 +334,28 @@ Audio::~Audio() { } } +void Audio::renderEchoCompare() { + const int XPOS = 0; + const int YPOS = 500; + const int YSCALE = 500; + const int XSCALE = 2; + glPointSize(1.0); + glLineWidth(1.0); + glDisable(GL_LINE_SMOOTH); + glColor3f(1,1,1); + glBegin(GL_LINE_STRIP); + for (int i = 0; i < BUFFER_LENGTH_SAMPLES; i++) { + glVertex2f(XPOS + i * XSCALE, YPOS + _echoInputSamples[i]/YSCALE); + } + glEnd(); + glColor3f(0,1,1); + glBegin(GL_LINE_STRIP); + for (int i = 0; i < BUFFER_LENGTH_SAMPLES; i++) { + glVertex2f(XPOS + i * XSCALE, YPOS + _echoOutputSamples[i]/YSCALE); + } + glEnd(); +} + // Take a pointer to the acquired microphone input samples and add procedural sounds void Audio::addProceduralSounds(int16_t* inputBuffer, int numSamples) { const float MAX_AUDIBLE_VELOCITY = 6.0; @@ -331,10 +369,27 @@ void Audio::addProceduralSounds(int16_t* inputBuffer, int numSamples) { // Add a noise-modulated sinewave with volume that tapers off with speed increasing if ((speed > MIN_AUDIBLE_VELOCITY) && (speed < MAX_AUDIBLE_VELOCITY)) { for (int i = 0; i < numSamples; i++) { - inputBuffer[i] += (int16_t)((cosf((float) i / SOUND_PITCH * speed) * randFloat()) * volume * speed); + inputBuffer[i] += (int16_t)((sinf((float) i / SOUND_PITCH * speed) * randFloat()) * volume * speed); } } } + +void Audio::startEchoTest() { + _startEcho = true; + _echoPingFrameCount = 0; + _sendingEchoPing = true; + _gatheringEchoFrames = false; +} + +void Audio::addedPingFrame() { + const int ECHO_PING_FRAMES = 1; + _echoPingFrameCount++; + if (_echoPingFrameCount == ECHO_PING_FRAMES) { + _gatheringEchoFrames = false; + _sendingEchoPing = false; + //startEchoTest(); + } +} void Audio::analyzeEcho(int16_t* inputBuffer, int16_t* outputBuffer, int numSamples) { // Compare output and input streams, looking for evidence of correlation needing echo cancellation // @@ -385,7 +440,6 @@ void Audio::addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBy if (::stdev.getSamples() > 500) { _measuredJitter = ::stdev.getStDev(); - //printLog("Avg: %4.2f, Stdev: %4.2f\n", stdev.getAverage(), sharedAudioData->measuredJitter); ::stdev.reset(); } diff --git a/interface/src/Audio.h b/interface/src/Audio.h index eb5021d10e..28d47a68d8 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -36,6 +36,11 @@ public: void addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBytes); + + void startEchoTest(); + void addedPingFrame(); + void renderEchoCompare(); + private: PaStream* _stream; AudioRingBuffer _ringBuffer; @@ -52,6 +57,12 @@ private: int _totalPacketsReceived; timeval _firstPlaybackTime; int _packetsReceivedThisPlayback; + bool _startEcho; + bool _sendingEchoPing; + int _echoPingFrameCount; + int16_t* _echoInputSamples; + int16_t* _echoOutputSamples; + bool _gatheringEchoFrames; // give access to AudioData class from audioCallback friend int audioCallback (const void*, void*, unsigned long, const PaStreamCallbackTimeInfo*, PaStreamCallbackFlags, void*); diff --git a/interface/src/AudioData.cpp b/interface/src/AudioData.cpp deleted file mode 100644 index faa0a27a83..0000000000 --- a/interface/src/AudioData.cpp +++ /dev/null @@ -1,79 +0,0 @@ -// -// AudioData.cpp -// interface -// -// Created by Stephen Birarda on 1/29/13. -// Copyright (c) 2013 HighFidelity, Inc. All rights reserved. -// -#ifndef _WIN32 - -#include "AudioData.h" - -AudioData::AudioData() { - mixerAddress = 0; - mixerPort = 0; - - averagedLatency = 0.0; - lastCallback.tv_usec = 0; - wasStarved = 0; - measuredJitter = 0; - jitterBuffer = 0; - - mixerLoopbackFlag = false; - audioSocket = NULL; -} - - -AudioData::~AudioData() { - delete audioSocket; -} - -// Take a pointer to the acquired microphone input samples and add procedural sounds -void AudioData::addProceduralSounds(int16_t* inputBuffer, int numSamples) { - const float MAX_AUDIBLE_VELOCITY = 6.0; - const float MIN_AUDIBLE_VELOCITY = 0.1; - float speed = glm::length(_lastVelocity); - float volume = 400 * (1.f - speed/MAX_AUDIBLE_VELOCITY); - // Add a noise-modulated sinewave with volume that tapers off with speed increasing - if ((speed > MIN_AUDIBLE_VELOCITY) && (speed < MAX_AUDIBLE_VELOCITY)) { - for (int i = 0; i < numSamples; i++) { - inputBuffer[i] += (int16_t) ((cosf((float)i / 8.f * speed) * randFloat()) * volume * speed) ; - } - } - - return; -} - -void AudioData::analyzeEcho(int16_t* inputBuffer, int16_t* outputBuffer, int numSamples) { - // Compare output and input streams, looking for evidence of correlation needing echo cancellation - // - // OFFSET_RANGE tells us how many samples to vary the analysis window when looking for correlation, - // and should be equal to the largest physical distance between speaker and microphone, where - // OFFSET_RANGE = 1 / (speedOfSound (meters / sec) / SamplingRate (samples / sec)) * distance - // - const int OFFSET_RANGE = 10; - const int SIGNAL_FLOOR = 1000; - float correlation[2 * OFFSET_RANGE + 1]; - int numChecked = 0; - bool foundSignal = false; - for (int offset = -OFFSET_RANGE; offset <= OFFSET_RANGE; offset++) { - for (int i = 0; i < numSamples; i++) { - if ((i + offset >= 0) && (i + offset < numSamples)) { - correlation[offset + OFFSET_RANGE] += - (float) abs(inputBuffer[i] - outputBuffer[i + offset]); - numChecked++; - foundSignal |= (inputBuffer[i] > SIGNAL_FLOOR); - } - } - correlation[offset + OFFSET_RANGE] /= numChecked; - numChecked = 0; - if (foundSignal) { - printLog("%4.2f, ", correlation[offset + OFFSET_RANGE]); - } - } - if (foundSignal) printLog("\n"); -} - - - -#endif diff --git a/interface/src/AudioData.h b/interface/src/AudioData.h deleted file mode 100644 index a98c40b38a..0000000000 --- a/interface/src/AudioData.h +++ /dev/null @@ -1,55 +0,0 @@ -// -// AudioData.h -// interface -// -// Created by Stephen Birarda on 1/29/13. -// Copyright (c) 2013 HighFidelity, Inc. All rights reserved. -// - -#ifndef __interface__AudioData__ -#define __interface__AudioData__ - -#include -#include -#include "AudioRingBuffer.h" -#include "UDPSocket.h" -#include "Avatar.h" - -class AudioData { - public: - AudioData(); - ~AudioData(); - AudioRingBuffer *ringBuffer; - - UDPSocket *audioSocket; - - Avatar *linkedAvatar; - - // store current mixer address and port - in_addr_t mixerAddress; - in_port_t mixerPort; - - timeval lastCallback; - float averagedLatency; - float measuredJitter; - float jitterBuffer; - int wasStarved; - - float lastInputLoudness; - - bool mixerLoopbackFlag; - - // Added avatar acceleration and velocity for procedural effects sounds from client - void setLastVelocity(glm::vec3 v) { _lastVelocity = v; }; - void setLastAcceleration(glm::vec3 a) { _lastAcceleration = a; }; - void addProceduralSounds(int16_t* inputBuffer, int numSamples); - void analyzeEcho(int16_t* inputBuffer, int16_t* outputBuffer, int numSamples); - - private: - glm::vec3 _lastVelocity; - glm::vec3 _lastAcceleration; - - -}; - -#endif /* defined(__interface__AudioData__) */ diff --git a/interface/src/Oscilloscope.cpp b/interface/src/Oscilloscope.cpp index e34903b298..ae984ebe61 100644 --- a/interface/src/Oscilloscope.cpp +++ b/interface/src/Oscilloscope.cpp @@ -113,18 +113,19 @@ void Oscilloscope::render(int x, int y) { } } - glLineWidth(2.0); + glLineWidth(1.0); + glDisable(GL_LINE_SMOOTH); glPushMatrix(); glTranslatef((float)x + 0.0f, (float)y + _valHeight / 2.0f, 0.0f); glScaled(1.0f, _valHeight / 32767.0f, 1.0f); glVertexPointer(2, GL_SHORT, 0, _arrVertices); glEnableClientState(GL_VERTEX_ARRAY); glColor3f(1.0f, 1.0f, 1.0f); - glDrawArrays(GL_LINES, MAX_SAMPLES * 0, usedWidth); + glDrawArrays(GL_LINE_STRIP, MAX_SAMPLES * 0, usedWidth); glColor3f(0.0f, 1.0f ,1.0f); - glDrawArrays(GL_LINES, MAX_SAMPLES * 1, usedWidth); + glDrawArrays(GL_LINE_STRIP, MAX_SAMPLES * 1, usedWidth); glColor3f(0.0f, 1.0f ,1.0f); - glDrawArrays(GL_LINES, MAX_SAMPLES * 2, usedWidth); + glDrawArrays(GL_LINE_STRIP, MAX_SAMPLES * 2, usedWidth); glDisableClientState(GL_VERTEX_ARRAY); glPopMatrix(); }