Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-08-09 03:17:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

fix last input loudness for mouth movement with no depth camera

Browse source
This commit is contained in:
Stephen Birarda 2013-12-16 16:01:03 -08:00
parent 89beff4253
commit e5d2647e70
2 changed files with 41 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

82
interface/src/Audio.cpp
View file

@ -46,8 +46,6 @@ Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* p
_audioInput(NULL), _audioInput(NULL),
_desiredInputFormat(), _desiredInputFormat(),
_inputFormat(), _inputFormat(),
_inputBuffer(),
_monoAudioSamples(NULL),
_numInputCallbackBytes(0), _numInputCallbackBytes(0),
_audioOutput(NULL), _audioOutput(NULL),
_desiredOutputFormat(), _desiredOutputFormat(),
@ -55,7 +53,7 @@ Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* p
_outputDevice(NULL), _outputDevice(NULL),
_numOutputCallbackBytes(0), _numOutputCallbackBytes(0),
_inputRingBuffer(0), _inputRingBuffer(0),
_ringBuffer(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2), _ringBuffer(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL),
_scope(scope), _scope(scope),
_averagedLatency(0.0), _averagedLatency(0.0),
_measuredJitter(0), _measuredJitter(0),
@ -72,7 +70,8 @@ Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* p
_numFramesDisplayStarve(0), _numFramesDisplayStarve(0),
_muted(false) _muted(false)
{ {
// clear the array of locally injected samples
memset(_localInjectedSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL);
} }
void Audio::init(QGLWidget *parent) { void Audio::init(QGLWidget *parent) {
@ -282,7 +281,7 @@ void Audio::handleAudioInput() {
static int numBytesPacketHeader = numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO); static int numBytesPacketHeader = numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO);
static int leadingBytes = numBytesPacketHeader + sizeof(glm::vec3) + sizeof(glm::quat) + NUM_BYTES_RFC4122_UUID; static int leadingBytes = numBytesPacketHeader + sizeof(glm::vec3) + sizeof(glm::quat) + NUM_BYTES_RFC4122_UUID;
_monoAudioSamples = (int16_t*) (monoAudioDataPacket + leadingBytes); static int16_t* monoAudioSamples = (int16_t*) (monoAudioDataPacket + leadingBytes);
static float inputToNetworkInputRatio = _numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO static float inputToNetworkInputRatio = _numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO
/ NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; / NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL;
@ -298,29 +297,46 @@ void Audio::handleAudioInput() {
int16_t inputAudioSamples[inputSamplesRequired]; int16_t inputAudioSamples[inputSamplesRequired];
_inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired); _inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired);
// zero out the monoAudioSamples array // zero out the monoAudioSamples array and the locally injected audio
memset(_monoAudioSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); memset(monoAudioSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL);
// zero out the locally injected audio in preparation for audio procedural sounds
memset(_localInjectedSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL);
if (!_muted) { if (!_muted) {
// we aren't muted, downsample the input audio // we aren't muted, downsample the input audio
linearResampling((int16_t*) inputAudioSamples, linearResampling((int16_t*) inputAudioSamples,
_monoAudioSamples, monoAudioSamples,
inputSamplesRequired, inputSamplesRequired,
NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL,
_inputFormat, _desiredInputFormat); _inputFormat, _desiredInputFormat);
float loudness = 0;
for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) {
loudness += fabsf(monoAudioSamples[i]);
}
_lastInputLoudness = loudness / NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL;
// add input data just written to the scope // add input data just written to the scope
QMetaObject::invokeMethod(_scope, "addSamples", Qt::QueuedConnection, QMetaObject::invokeMethod(_scope, "addSamples", Qt::QueuedConnection,
Q_ARG(QByteArray, QByteArray((char*) _monoAudioSamples, Q_ARG(QByteArray, QByteArray((char*) monoAudioSamples,
NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL)), NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL)),
Q_ARG(bool, false), Q_ARG(bool, true)); Q_ARG(bool, false), Q_ARG(bool, true));
if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio)) {
// if this person wants local loopback add that to the locally injected audio
memcpy(_localInjectedSamples, monoAudioSamples, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL);
}
} else {
// our input loudness is 0, since we're muted
_lastInputLoudness = 0;
} }
// add procedural effects to the appropriate input samples // add procedural effects to the appropriate input samples
// addProceduralSounds(monoAudioSamples + (_isBufferSendCallback addProceduralSounds(monoAudioSamples,
// ? BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO : 0), NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL);
// (int16_t*) stereoOutputBuffer.data(),
// BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO);
NodeList* nodeList = NodeList::getInstance(); NodeList* nodeList = NodeList::getInstance();
Node* audioMixer = nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER); Node* audioMixer = nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER);
@ -353,18 +369,6 @@ void Audio::handleAudioInput() {
memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation)); memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation));
currentPacketPtr += sizeof(headOrientation); currentPacketPtr += sizeof(headOrientation);
float loudness = 0;
if (!_muted) {
for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) {
loudness += fabsf(_monoAudioSamples[i]);
}
loudness /= NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL;
}
_lastInputLoudness = loudness;
nodeList->getNodeSocket().writeDatagram(monoAudioDataPacket, nodeList->getNodeSocket().writeDatagram(monoAudioDataPacket,
NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes,
audioMixer->getActiveSocket()->getAddress(), audioMixer->getActiveSocket()->getAddress(),
@ -430,15 +434,13 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) {
int16_t ringBufferSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO]; int16_t ringBufferSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO];
_ringBuffer.readSamples(ringBufferSamples, NETWORK_BUFFER_LENGTH_SAMPLES_STEREO); _ringBuffer.readSamples(ringBufferSamples, NETWORK_BUFFER_LENGTH_SAMPLES_STEREO);
if (!_muted && Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio)) { // add to the output samples whatever is in the _localAudioOutput byte array
// copy whatever is pointed to at _monoAudioSamples into our ringBufferSamples // that lets this user hear sound effects and loopback (if enabled)
// so that local audio is echoed back
for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) {
for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { ringBufferSamples[i * 2] = glm::clamp(ringBufferSamples[i * 2] + _localInjectedSamples[i],
ringBufferSamples[i * 2] = glm::clamp(ringBufferSamples[i * 2] + _monoAudioSamples[i], MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); ringBufferSamples[(i * 2) + 1] = ringBufferSamples[i * 2];
ringBufferSamples[(i * 2) + 1] = ringBufferSamples[i * 2];
}
} }
// copy the packet from the RB to the output // copy the packet from the RB to the output
@ -577,7 +579,7 @@ void Audio::render(int screenWidth, int screenHeight) {
} }
// Take a pointer to the acquired microphone input samples and add procedural sounds // Take a pointer to the acquired microphone input samples and add procedural sounds
void Audio::addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutput, int numSamples) { void Audio::addProceduralSounds(int16_t* monoInput, int numSamples) {
const float MAX_AUDIBLE_VELOCITY = 6.0; const float MAX_AUDIBLE_VELOCITY = 6.0;
const float MIN_AUDIBLE_VELOCITY = 0.1; const float MIN_AUDIBLE_VELOCITY = 0.1;
const int VOLUME_BASELINE = 400; const int VOLUME_BASELINE = 400;
@ -614,10 +616,7 @@ void Audio::addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutp
int16_t collisionSample = (int16_t) sample; int16_t collisionSample = (int16_t) sample;
monoInput[i] += collisionSample; monoInput[i] += collisionSample;
_localInjectedSamples[i] += collisionSample;
for (int j = (i * 4); j < (i * 4) + 4; j++) {
stereoUpsampledOutput[j] += collisionSample;
}
_collisionSoundMagnitude *= _collisionSoundDuration; _collisionSoundMagnitude *= _collisionSoundDuration;
} }
@ -640,10 +639,7 @@ void Audio::addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutp
int16_t collisionSample = (int16_t) sample; int16_t collisionSample = (int16_t) sample;
monoInput[i] += collisionSample; monoInput[i] += collisionSample;
_localInjectedSamples[i] += collisionSample;
for (int j = (i * 4); j < (i * 4) + 4; j++) {
stereoUpsampledOutput[j] += collisionSample;
}
_drumSoundVolume *= (1.f - _drumSoundDecay); _drumSoundVolume *= (1.f - _drumSoundDecay);
} }

5
interface/src/Audio.h
View file

@ -75,8 +75,7 @@ private:
QAudioFormat _desiredInputFormat; QAudioFormat _desiredInputFormat;
QAudioFormat _inputFormat; QAudioFormat _inputFormat;
QIODevice* _inputDevice; QIODevice* _inputDevice;
QByteArray _inputBuffer; int16_t _localInjectedSamples[NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL];
int16_t* _monoAudioSamples;
int _numInputCallbackBytes; int _numInputCallbackBytes;
QAudioOutput* _audioOutput; QAudioOutput* _audioOutput;
QAudioFormat _desiredOutputFormat; QAudioFormat _desiredOutputFormat;
@ -122,7 +121,7 @@ private:
inline void performIO(int16_t* inputLeft, int16_t* outputLeft, int16_t* outputRight); inline void performIO(int16_t* inputLeft, int16_t* outputLeft, int16_t* outputRight);
// Add sounds that we want the user to not hear themselves, by adding on top of mic input signal // Add sounds that we want the user to not hear themselves, by adding on top of mic input signal
void addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutput, int numSamples); void addProceduralSounds(int16_t* monoInput, int numSamples);
void renderToolIcon(int screenHeight); void renderToolIcon(int screenHeight);
}; };