Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-20 04:44:11 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #3750 from Atlante45/reverb_for_own_voice

Browse source 
Reverb for own voice
This commit is contained in:
Philip Rosedale 2014-11-11 10:52:26 -08:00
commit d9141b4010
5 changed files with 160 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

113
assignment-client/src/audio/AudioMixer.cpp
View file

@ -465,6 +465,63 @@ int AudioMixer::prepareMixForListeningNode(Node* node) {
return streamsMixed;
}
void AudioMixer::sendAudioEnvironmentPacket(SharedNodePointer node) {
static char clientEnvBuffer[MAX_PACKET_SIZE];
// Send stream properties
bool hasReverb = false;
float reverbTime, wetLevel;
// find reverb properties
for (int i = 0; i < _zoneReverbSettings.size(); ++i) {
AudioMixerClientData* data = static_cast<AudioMixerClientData*>(node->getLinkedData());
glm::vec3 streamPosition = data->getAvatarAudioStream()->getPosition();
if (_audioZones[_zoneReverbSettings[i].zone].contains(streamPosition)) {
hasReverb = true;
reverbTime = _zoneReverbSettings[i].reverbTime;
wetLevel = _zoneReverbSettings[i].wetLevel;
break;
}
}
AudioMixerClientData* nodeData = static_cast<AudioMixerClientData*>(node->getLinkedData());
AvatarAudioStream* stream = nodeData->getAvatarAudioStream();
bool dataChanged = (stream->hasReverb() != hasReverb) ||
(stream->hasReverb() && (stream->getRevebTime() != reverbTime ||
stream->getWetLevel() != wetLevel));
if (dataChanged) {
// Update stream
if (hasReverb) {
stream->setReverb(reverbTime, wetLevel);
} else {
stream->clearReverb();
}
}
// Send at change or every so often
float CHANCE_OF_SEND = 0.01f;
bool sendData = dataChanged || (randFloat() < CHANCE_OF_SEND);
if (sendData) {
int numBytesEnvPacketHeader = populatePacketHeader(clientEnvBuffer, PacketTypeAudioEnvironment);
char* envDataAt = clientEnvBuffer + numBytesEnvPacketHeader;
unsigned char bitset = 0;
if (hasReverb) {
setAtBit(bitset, HAS_REVERB_BIT);
}
memcpy(envDataAt, &bitset, sizeof(unsigned char));
envDataAt += sizeof(unsigned char);
if (hasReverb) {
memcpy(envDataAt, &reverbTime, sizeof(float));
envDataAt += sizeof(float);
memcpy(envDataAt, &wetLevel, sizeof(float));
envDataAt += sizeof(float);
}
NodeList::getInstance()->writeDatagram(clientEnvBuffer, envDataAt - clientEnvBuffer, node);
}
}
void AudioMixer::readPendingDatagram(const QByteArray& receivedPacket, const HifiSockAddr& senderSockAddr) {
NodeList* nodeList = NodeList::getInstance();
@ -642,7 +699,6 @@ void AudioMixer::run() {
timer.start();
char clientMixBuffer[MAX_PACKET_SIZE];
char clientEnvBuffer[MAX_PACKET_SIZE];
int usecToSleep = BUFFER_SEND_INTERVAL_USECS;
@ -759,58 +815,6 @@ void AudioMixer::run() {
// pack mixed audio samples
memcpy(mixDataAt, _mixSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO);
mixDataAt += NETWORK_BUFFER_LENGTH_BYTES_STEREO;
// Send stream properties
bool hasReverb = false;
float reverbTime, wetLevel;
// find reverb properties
for (int i = 0; i < _zoneReverbSettings.size(); ++i) {
AudioMixerClientData* data = static_cast<AudioMixerClientData*>(node->getLinkedData());
glm::vec3 streamPosition = data->getAvatarAudioStream()->getPosition();
if (_audioZones[_zoneReverbSettings[i].zone].contains(streamPosition)) {
hasReverb = true;
reverbTime = _zoneReverbSettings[i].reverbTime;
wetLevel = _zoneReverbSettings[i].wetLevel;
break;
}
}
AvatarAudioStream* stream = nodeData->getAvatarAudioStream();
bool dataChanged = (stream->hasReverb() != hasReverb) ||
(stream->hasReverb() && (stream->getRevebTime() != reverbTime ||
stream->getWetLevel() != wetLevel));
if (dataChanged) {
// Update stream
if (hasReverb) {
stream->setReverb(reverbTime, wetLevel);
} else {
stream->clearReverb();
}
}
// Send at change or every so often
float CHANCE_OF_SEND = 0.01f;
bool sendData = dataChanged || (randFloat() < CHANCE_OF_SEND);
if (sendData) {
int numBytesEnvPacketHeader = populatePacketHeader(clientEnvBuffer, PacketTypeAudioEnvironment);
char* envDataAt = clientEnvBuffer + numBytesEnvPacketHeader;
unsigned char bitset = 0;
if (hasReverb) {
setAtBit(bitset, HAS_REVERB_BIT);
}
memcpy(envDataAt, &bitset, sizeof(unsigned char));
envDataAt += sizeof(unsigned char);
if (hasReverb) {
memcpy(envDataAt, &reverbTime, sizeof(float));
envDataAt += sizeof(float);
memcpy(envDataAt, &wetLevel, sizeof(float));
envDataAt += sizeof(float);
}
nodeList->writeDatagram(clientEnvBuffer, envDataAt - clientEnvBuffer, node);
}
} else {
// pack header
int numBytesPacketHeader = populatePacketHeader(clientMixBuffer, PacketTypeSilentAudioFrame);
@ -826,6 +830,9 @@ void AudioMixer::run() {
memcpy(mixDataAt, &numSilentSamples, sizeof(quint16));
mixDataAt += sizeof(quint16);
}
// Send audio environment
sendAudioEnvironmentPacket(node);
// send mixed audio packet
nodeList->writeDatagram(clientMixBuffer, mixDataAt - clientMixBuffer, node);

3
assignment-client/src/audio/AudioMixer.h
View file

@ -49,6 +49,9 @@ private:
/// prepares and sends a mix to one Node
int prepareMixForListeningNode(Node* node);
/// Send Audio Environment packet for a single node
void sendAudioEnvironmentPacket(SharedNodePointer node);
// used on a per stream basis to run the filter on before mixing, large enough to handle the historical
// data from a phase delay as well as an entire network buffer

140
interface/src/Audio.cpp
View file

@ -516,6 +516,33 @@ void Audio::initGverb() {
gverb_set_taillevel(_gverb, DB_CO(_reverbOptions->getTailLevel()));
}
void Audio::updateGverbOptions() {
bool reverbChanged = false;
if (_receivedAudioStream.hasReverb()) {
if (_zoneReverbOptions.getReverbTime() != _receivedAudioStream.getRevebTime()) {
_zoneReverbOptions.setReverbTime(_receivedAudioStream.getRevebTime());
reverbChanged = true;
}
if (_zoneReverbOptions.getWetLevel() != _receivedAudioStream.getWetLevel()) {
_zoneReverbOptions.setWetLevel(_receivedAudioStream.getWetLevel());
reverbChanged = true;
}
if (_reverbOptions != &_zoneReverbOptions) {
_reverbOptions = &_zoneReverbOptions;
reverbChanged = true;
}
} else if (_reverbOptions != &_scriptReverbOptions) {
_reverbOptions = &_scriptReverbOptions;
reverbChanged = true;
}
if (reverbChanged) {
initGverb();
}
}
void Audio::setReverbOptions(const AudioEffectOptions* options) {
// Save the new options
_scriptReverbOptions.setMaxRoomSize(options->getMaxRoomSize());
@ -550,11 +577,11 @@ void Audio::addReverb(int16_t* samplesData, int numSamples, QAudioFormat& audioF
for (int j = sample; j < sample + audioFormat.channelCount(); j++) {
if (j == sample) {
// left channel
int lResult = glm::clamp((int)(samplesData[j] * dryFraction + lValue * wetFraction), -32768, 32767);
int lResult = glm::clamp((int)(samplesData[j] * dryFraction + lValue * wetFraction), MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
samplesData[j] = (int16_t)lResult;
} else if (j == (sample + 1)) {
// right channel
int rResult = glm::clamp((int)(samplesData[j] * dryFraction + rValue * wetFraction), -32768, 32767);
int rResult = glm::clamp((int)(samplesData[j] * dryFraction + rValue * wetFraction), MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
samplesData[j] = (int16_t)rResult;
} else {
// ignore channels above 2
@ -563,6 +590,60 @@ void Audio::addReverb(int16_t* samplesData, int numSamples, QAudioFormat& audioF
}
}
void Audio::handleLocalEchoAndReverb(QByteArray& inputByteArray) {
bool hasEcho = Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio);
// If there is server echo, reverb will be applied to the recieved audio stream so no need to have it here.
bool hasLocalReverb = (_reverb || _receivedAudioStream.hasReverb()) &&
!Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio);
if (_muted || !_audioOutput || (!hasEcho && !hasLocalReverb)) {
return;
}
// if this person wants local loopback add that to the locally injected audio
// if there is reverb apply it to local audio and substract the origin samples
if (!_loopbackOutputDevice && _loopbackAudioOutput) {
// we didn't have the loopback output device going so set that up now
_loopbackOutputDevice = _loopbackAudioOutput->start();
}
QByteArray loopBackByteArray(inputByteArray);
if (_inputFormat != _outputFormat) {
float loopbackOutputToInputRatio = (_outputFormat.sampleRate() / (float) _inputFormat.sampleRate()) *
(_outputFormat.channelCount() / _inputFormat.channelCount());
loopBackByteArray.resize(inputByteArray.size() * loopbackOutputToInputRatio);
loopBackByteArray.fill(0);
linearResampling(reinterpret_cast<int16_t*>(inputByteArray.data()),
reinterpret_cast<int16_t*>(loopBackByteArray.data()),
inputByteArray.size() / sizeof(int16_t), loopBackByteArray.size() / sizeof(int16_t),
_inputFormat, _outputFormat);
}
if (hasLocalReverb) {
QByteArray loopbackCopy;
if (!hasEcho) {
loopbackCopy = loopBackByteArray;
}
int16_t* loopbackSamples = reinterpret_cast<int16_t*>(loopBackByteArray.data());
int numLoopbackSamples = loopBackByteArray.size() / sizeof(int16_t);
updateGverbOptions();
addReverb(loopbackSamples, numLoopbackSamples, _outputFormat);
if (!hasEcho) {
int16_t* loopbackCopySamples = reinterpret_cast<int16_t*>(loopbackCopy.data());
for (int i = 0; i < numLoopbackSamples; ++i) {
loopbackSamples[i] = glm::clamp((int)loopbackSamples[i] - loopbackCopySamples[i],
MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
}
}
}
if (_loopbackOutputDevice) {
_loopbackOutputDevice->write(loopBackByteArray);
}
}
void Audio::handleAudioInput() {
static char audioDataPacket[MAX_PACKET_SIZE];
@ -607,34 +688,8 @@ void Audio::handleAudioInput() {
_inputFrameBuffer.copyFrames(1, inputFrameCount, inputFrameData, true /*copy out*/);
}
if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio) && !_muted && _audioOutput) {
// if this person wants local loopback add that to the locally injected audio
if (!_loopbackOutputDevice && _loopbackAudioOutput) {
// we didn't have the loopback output device going so set that up now
_loopbackOutputDevice = _loopbackAudioOutput->start();
}
if (_inputFormat == _outputFormat) {
if (_loopbackOutputDevice) {
_loopbackOutputDevice->write(inputByteArray);
}
} else {
float loopbackOutputToInputRatio = (_outputFormat.sampleRate() / (float) _inputFormat.sampleRate())
* (_outputFormat.channelCount() / _inputFormat.channelCount());
QByteArray loopBackByteArray(inputByteArray.size() * loopbackOutputToInputRatio, 0);
linearResampling((int16_t*) inputByteArray.data(), (int16_t*) loopBackByteArray.data(),
inputByteArray.size() / sizeof(int16_t),
loopBackByteArray.size() / sizeof(int16_t), _inputFormat, _outputFormat);
if (_loopbackOutputDevice) {
_loopbackOutputDevice->write(loopBackByteArray);
}
}
}
handleLocalEchoAndReverb(inputByteArray);
_inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size());
@ -971,30 +1026,7 @@ void Audio::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& ou
_desiredOutputFormat, _outputFormat);
if(_reverb || _receivedAudioStream.hasReverb()) {
bool reverbChanged = false;
if (_receivedAudioStream.hasReverb()) {
if (_zoneReverbOptions.getReverbTime() != _receivedAudioStream.getRevebTime()) {
_zoneReverbOptions.setReverbTime(_receivedAudioStream.getRevebTime());
reverbChanged = true;
}
if (_zoneReverbOptions.getWetLevel() != _receivedAudioStream.getWetLevel()) {
_zoneReverbOptions.setWetLevel(_receivedAudioStream.getWetLevel());
reverbChanged = true;
}
if (_reverbOptions != &_zoneReverbOptions) {
_reverbOptions = &_zoneReverbOptions;
reverbChanged = true;
}
} else if (_reverbOptions != &_scriptReverbOptions) {
_reverbOptions = &_scriptReverbOptions;
reverbChanged = true;
}
if (reverbChanged) {
initGverb();
}
updateGverbOptions();
addReverb((int16_t*)outputBuffer.data(), numDeviceOutputSamples, _outputFormat);
}
}

3
interface/src/Audio.h
View file

@ -272,8 +272,11 @@ private:
// Adds Reverb
void initGverb();
void updateGverbOptions();
void addReverb(int16_t* samples, int numSamples, QAudioFormat& format);
void handleLocalEchoAndReverb(QByteArray& inputByteArray);
// Add sounds that we want the user to not hear themselves, by adding on top of mic input signal
void addProceduralSounds(int16_t* monoInput, int numSamples);

9
libraries/networking/src/LimitedNodeList.cpp
View file

@ -209,8 +209,15 @@ bool LimitedNodeList::packetVersionAndHashMatch(const QByteArray& packet) {
if (hashFromPacketHeader(packet) == hashForPacketAndConnectionUUID(packet, sendingNode->getConnectionSecret())) {
return true;
} else {
qDebug() << "Packet hash mismatch on" << checkType << "- Sender"
static QMultiMap<QUuid, PacketType> hashDebugSuppressMap;
QUuid senderUUID = uuidFromPacketHeader(packet);
if (!hashDebugSuppressMap.contains(senderUUID, checkType)) {
qDebug() << "Packet hash mismatch on" << checkType << "- Sender"
<< uuidFromPacketHeader(packet);
hashDebugSuppressMap.insert(senderUUID, checkType);
}
}
} else {
static QString repeatedMessage