Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-08-04 16:30:10 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improvements to scope display, added framework for echo analysis over multiple frames

Browse source
This commit is contained in:
Philip Rosedale 2013-05-15 21:15:55 -06:00
parent 2820323567
commit 1a283c3ac0
6 changed files with 96 additions and 159 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
interface/src/Application.cpp
View file

@ -534,6 +534,10 @@ void Application::keyPressEvent(QKeyEvent* event) {
sendVoxelServerAddScene();
break;
case Qt::Key_Semicolon:
_audio.startEchoTest();
break;
case Qt::Key_L:
_displayLevels = !_displayLevels;
break;
@ -1641,6 +1645,7 @@ void Application::displayOverlay() {
#ifndef _WIN32
_audio.render(_glWidget->width(), _glWidget->height());
_audioScope.render(20, _glWidget->height() - 200);
//_audio.renderEchoCompare(); // PER: Will turn back on to further test echo
#endif
//noiseTest(_glWidget->width(), _glWidget->height());

96
interface/src/Audio.cpp
View file

@ -87,22 +87,26 @@ int audioCallback (const void* inputBuffer,
Application* interface = (Application*) QCoreApplication::instance();
Avatar interfaceAvatar = interface->getAvatar();
bool addPing = (randFloat() < 0.005f);
int16_t *inputLeft = ((int16_t **) inputBuffer)[0];
int16_t *outputLeft = ((int16_t **) outputBuffer)[0];
int16_t *outputRight = ((int16_t **) outputBuffer)[1];
// Compare the input and output streams to look for correlation
parentAudio->analyzeEcho(inputLeft, outputLeft, BUFFER_LENGTH_SAMPLES);
// Add Procedural effects to input samples
parentAudio->addProceduralSounds(inputLeft, BUFFER_LENGTH_SAMPLES);
// add data to the scope
// add output (@speakers) data to the scope
parentAudio->_scope->addSamples(1, outputLeft, PACKET_LENGTH_SAMPLES_PER_CHANNEL);
parentAudio->_scope->addSamples(2, outputRight, PACKET_LENGTH_SAMPLES_PER_CHANNEL);
// if needed, add input/output data to echo analysis buffers
if (parentAudio->_gatheringEchoFrames) {
memcpy(parentAudio->_echoInputSamples, inputLeft,
PACKET_LENGTH_SAMPLES_PER_CHANNEL * sizeof(int16_t));
memcpy(parentAudio->_echoOutputSamples, outputLeft,
PACKET_LENGTH_SAMPLES_PER_CHANNEL * sizeof(int16_t));
parentAudio->addedPingFrame();
}
if (inputLeft != NULL) {
// Measure the loudness of the signal from the microphone and store in audio object
@ -114,7 +118,7 @@ int audioCallback (const void* inputBuffer,
loudness /= BUFFER_LENGTH_SAMPLES;
parentAudio->_lastInputLoudness = loudness;
// add data to the scope
// add input (@microphone) data to the scope
parentAudio->_scope->addSamples(0, inputLeft, BUFFER_LENGTH_SAMPLES);
Agent* audioMixer = agentList->soloAgentOfType(AGENT_TYPE_AUDIO_MIXER);
@ -192,7 +196,6 @@ int audioCallback (const void* inputBuffer,
}
// play whatever we have in the audio buffer
// if we haven't fired off the flange effect, check if we should
// TODO: lastMeasuredHeadYaw is now relative to body - check if this still works.
@ -254,12 +257,8 @@ int audioCallback (const void* inputBuffer,
}
}
if (!addPing) {
outputLeft[s] = leftSample;
outputRight[s] = rightSample;
} else {
outputLeft[s] = outputRight[s] = (int16_t)(sinf((float) s / 15.f) * 8000.f);
}
outputLeft[s] = leftSample;
outputRight[s] = rightSample;
}
ringBuffer->setNextOutput(ringBuffer->getNextOutput() + PACKET_LENGTH_SAMPLES);
@ -268,11 +267,19 @@ int audioCallback (const void* inputBuffer,
}
}
}
if (parentAudio->_sendingEchoPing) {
const float PING_PITCH = 4.f;
const float PING_VOLUME = 32000.f;
for (int s = 0; s < PACKET_LENGTH_SAMPLES_PER_CHANNEL; s++) {
outputLeft[s] = outputRight[s] = (int16_t)(sinf((float) s / PING_PITCH) * PING_VOLUME);
}
parentAudio->_gatheringEchoFrames = true;
}
gettimeofday(&parentAudio->_lastCallbackTime, NULL);
return paContinue;
}
void outputPortAudioError(PaError error) {
if (error != paNoError) {
printLog("-- portaudio termination error --\n");
@ -293,8 +300,12 @@ Audio::Audio(Oscilloscope* scope) :
_lastAcceleration(0),
_totalPacketsReceived(0),
_firstPlaybackTime(),
_packetsReceivedThisPlayback(0)
{
_packetsReceivedThisPlayback(0),
_startEcho(false),
_sendingEchoPing(false),
_echoPingFrameCount(0),
_gatheringEchoFrames(false)
{
outputPortAudioError(Pa_Initialize());
outputPortAudioError(Pa_OpenDefaultStream(&_stream,
2,
@ -307,7 +318,12 @@ Audio::Audio(Oscilloscope* scope) :
// start the stream now that sources are good to go
outputPortAudioError(Pa_StartStream(_stream));
_echoInputSamples = new int16_t[BUFFER_LENGTH_BYTES];
_echoOutputSamples = new int16_t[BUFFER_LENGTH_BYTES];
memset(_echoInputSamples, 0, BUFFER_LENGTH_SAMPLES * sizeof(int));
memset(_echoOutputSamples, 0, BUFFER_LENGTH_SAMPLES * sizeof(int));
gettimeofday(&_lastReceiveTime, NULL);
}
@ -318,6 +334,28 @@ Audio::~Audio() {
}
}
void Audio::renderEchoCompare() {
const int XPOS = 0;
const int YPOS = 500;
const int YSCALE = 500;
const int XSCALE = 2;
glPointSize(1.0);
glLineWidth(1.0);
glDisable(GL_LINE_SMOOTH);
glColor3f(1,1,1);
glBegin(GL_LINE_STRIP);
for (int i = 0; i < BUFFER_LENGTH_SAMPLES; i++) {
glVertex2f(XPOS + i * XSCALE, YPOS + _echoInputSamples[i]/YSCALE);
}
glEnd();
glColor3f(0,1,1);
glBegin(GL_LINE_STRIP);
for (int i = 0; i < BUFFER_LENGTH_SAMPLES; i++) {
glVertex2f(XPOS + i * XSCALE, YPOS + _echoOutputSamples[i]/YSCALE);
}
glEnd();
}
// Take a pointer to the acquired microphone input samples and add procedural sounds
void Audio::addProceduralSounds(int16_t* inputBuffer, int numSamples) {
const float MAX_AUDIBLE_VELOCITY = 6.0;
@ -331,10 +369,27 @@ void Audio::addProceduralSounds(int16_t* inputBuffer, int numSamples) {
// Add a noise-modulated sinewave with volume that tapers off with speed increasing
if ((speed > MIN_AUDIBLE_VELOCITY) && (speed < MAX_AUDIBLE_VELOCITY)) {
for (int i = 0; i < numSamples; i++) {
inputBuffer[i] += (int16_t)((cosf((float) i / SOUND_PITCH * speed) * randFloat()) * volume * speed);
inputBuffer[i] += (int16_t)((sinf((float) i / SOUND_PITCH * speed) * randFloat()) * volume * speed);
}
}
}
void Audio::startEchoTest() {
_startEcho = true;
_echoPingFrameCount = 0;
_sendingEchoPing = true;
_gatheringEchoFrames = false;
}
void Audio::addedPingFrame() {
const int ECHO_PING_FRAMES = 1;
_echoPingFrameCount++;
if (_echoPingFrameCount == ECHO_PING_FRAMES) {
_gatheringEchoFrames = false;
_sendingEchoPing = false;
//startEchoTest();
}
}
void Audio::analyzeEcho(int16_t* inputBuffer, int16_t* outputBuffer, int numSamples) {
// Compare output and input streams, looking for evidence of correlation needing echo cancellation
//
@ -385,7 +440,6 @@ void Audio::addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBy
if (::stdev.getSamples() > 500) {
_measuredJitter = ::stdev.getStDev();
//printLog("Avg: %4.2f, Stdev: %4.2f\n", stdev.getAverage(), sharedAudioData->measuredJitter);
::stdev.reset();
}

11
interface/src/Audio.h
View file

@ -36,6 +36,11 @@ public:
void addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBytes);
void startEchoTest();
void addedPingFrame();
void renderEchoCompare();
private:
PaStream* _stream;
AudioRingBuffer _ringBuffer;
@ -52,6 +57,12 @@ private:
int _totalPacketsReceived;
timeval _firstPlaybackTime;
int _packetsReceivedThisPlayback;
bool _startEcho;
bool _sendingEchoPing;
int _echoPingFrameCount;
int16_t* _echoInputSamples;
int16_t* _echoOutputSamples;
bool _gatheringEchoFrames;
// give access to AudioData class from audioCallback
friend int audioCallback (const void*, void*, unsigned long, const PaStreamCallbackTimeInfo*, PaStreamCallbackFlags, void*);

79
interface/src/AudioData.cpp
View file

@ -1,79 +0,0 @@
//
// AudioData.cpp
// interface
//
// Created by Stephen Birarda on 1/29/13.
// Copyright (c) 2013 HighFidelity, Inc. All rights reserved.
//
#ifndef _WIN32
#include "AudioData.h"
AudioData::AudioData() {
mixerAddress = 0;
mixerPort = 0;
averagedLatency = 0.0;
lastCallback.tv_usec = 0;
wasStarved = 0;
measuredJitter = 0;
jitterBuffer = 0;
mixerLoopbackFlag = false;
audioSocket = NULL;
}
AudioData::~AudioData() {
delete audioSocket;
}
// Take a pointer to the acquired microphone input samples and add procedural sounds
void AudioData::addProceduralSounds(int16_t* inputBuffer, int numSamples) {
const float MAX_AUDIBLE_VELOCITY = 6.0;
const float MIN_AUDIBLE_VELOCITY = 0.1;
float speed = glm::length(_lastVelocity);
float volume = 400 * (1.f - speed/MAX_AUDIBLE_VELOCITY);
// Add a noise-modulated sinewave with volume that tapers off with speed increasing
if ((speed > MIN_AUDIBLE_VELOCITY) && (speed < MAX_AUDIBLE_VELOCITY)) {
for (int i = 0; i < numSamples; i++) {
inputBuffer[i] += (int16_t) ((cosf((float)i / 8.f * speed) * randFloat()) * volume * speed) ;
}
}
return;
}
void AudioData::analyzeEcho(int16_t* inputBuffer, int16_t* outputBuffer, int numSamples) {
// Compare output and input streams, looking for evidence of correlation needing echo cancellation
//
// OFFSET_RANGE tells us how many samples to vary the analysis window when looking for correlation,
// and should be equal to the largest physical distance between speaker and microphone, where
// OFFSET_RANGE = 1 / (speedOfSound (meters / sec) / SamplingRate (samples / sec)) * distance
//
const int OFFSET_RANGE = 10;
const int SIGNAL_FLOOR = 1000;
float correlation[2 * OFFSET_RANGE + 1];
int numChecked = 0;
bool foundSignal = false;
for (int offset = -OFFSET_RANGE; offset <= OFFSET_RANGE; offset++) {
for (int i = 0; i < numSamples; i++) {
if ((i + offset >= 0) && (i + offset < numSamples)) {
correlation[offset + OFFSET_RANGE] +=
(float) abs(inputBuffer[i] - outputBuffer[i + offset]);
numChecked++;
foundSignal |= (inputBuffer[i] > SIGNAL_FLOOR);
}
}
correlation[offset + OFFSET_RANGE] /= numChecked;
numChecked = 0;
if (foundSignal) {
printLog("%4.2f, ", correlation[offset + OFFSET_RANGE]);
}
}
if (foundSignal) printLog("\n");
}
#endif

55
interface/src/AudioData.h
View file

@ -1,55 +0,0 @@
//
// AudioData.h
// interface
//
// Created by Stephen Birarda on 1/29/13.
// Copyright (c) 2013 HighFidelity, Inc. All rights reserved.
//
#ifndef __interface__AudioData__
#define __interface__AudioData__
#include <stdint.h>
#include <glm/glm.hpp>
#include "AudioRingBuffer.h"
#include "UDPSocket.h"
#include "Avatar.h"
class AudioData {
public:
AudioData();
~AudioData();
AudioRingBuffer *ringBuffer;
UDPSocket *audioSocket;
Avatar *linkedAvatar;
// store current mixer address and port
in_addr_t mixerAddress;
in_port_t mixerPort;
timeval lastCallback;
float averagedLatency;
float measuredJitter;
float jitterBuffer;
int wasStarved;
float lastInputLoudness;
bool mixerLoopbackFlag;
// Added avatar acceleration and velocity for procedural effects sounds from client
void setLastVelocity(glm::vec3 v) { _lastVelocity = v; };
void setLastAcceleration(glm::vec3 a) { _lastAcceleration = a; };
void addProceduralSounds(int16_t* inputBuffer, int numSamples);
void analyzeEcho(int16_t* inputBuffer, int16_t* outputBuffer, int numSamples);
private:
glm::vec3 _lastVelocity;
glm::vec3 _lastAcceleration;
};
#endif /* defined(__interface__AudioData__) */

9
interface/src/Oscilloscope.cpp
View file

@ -113,18 +113,19 @@ void Oscilloscope::render(int x, int y) {
}
}
glLineWidth(2.0);
glLineWidth(1.0);
glDisable(GL_LINE_SMOOTH);
glPushMatrix();
glTranslatef((float)x + 0.0f, (float)y + _valHeight / 2.0f, 0.0f);
glScaled(1.0f, _valHeight / 32767.0f, 1.0f);
glVertexPointer(2, GL_SHORT, 0, _arrVertices);
glEnableClientState(GL_VERTEX_ARRAY);
glColor3f(1.0f, 1.0f, 1.0f);
glDrawArrays(GL_LINES, MAX_SAMPLES * 0, usedWidth);
glDrawArrays(GL_LINE_STRIP, MAX_SAMPLES * 0, usedWidth);
glColor3f(0.0f, 1.0f ,1.0f);
glDrawArrays(GL_LINES, MAX_SAMPLES * 1, usedWidth);
glDrawArrays(GL_LINE_STRIP, MAX_SAMPLES * 1, usedWidth);
glColor3f(0.0f, 1.0f ,1.0f);
glDrawArrays(GL_LINES, MAX_SAMPLES * 2, usedWidth);
glDrawArrays(GL_LINE_STRIP, MAX_SAMPLES * 2, usedWidth);
glDisableClientState(GL_VERTEX_ARRAY);
glPopMatrix();
}