From ccfe6bda63eae782e602a9d9881ff886940450d9 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Mon, 5 Aug 2019 10:34:03 -0700 Subject: [PATCH 1/5] Detect clipping prior to echo cancellation (overload of the uncancelled raw input will cause echo leakage!) --- libraries/audio-client/src/AudioClient.cpp | 52 +++++++++++----------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index c16e297c28..99add0f797 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -170,27 +170,21 @@ static void channelDownmix(int16_t* source, int16_t* dest, int numSamples) { } } -static float computeLoudness(int16_t* samples, int numSamples, int numChannels, bool& isClipping) { +static bool detectClipping(int16_t* samples, int numSamples, int numChannels) { const int32_t CLIPPING_THRESHOLD = 32392; // -0.1 dBFS - const int32_t CLIPPING_DETECTION = 3; // consecutive samples over threshold + const int CLIPPING_DETECTION = 3; // consecutive samples over threshold - float scale = numSamples ? 1.0f / numSamples : 0.0f; - - int32_t loudness = 0; - isClipping = false; + bool isClipping = false; if (numChannels == 2) { - int32_t oversLeft = 0; - int32_t oversRight = 0; + int oversLeft = 0; + int oversRight = 0; for (int i = 0; i < numSamples/2; i++) { int32_t left = std::abs((int32_t)samples[2*i+0]); int32_t right = std::abs((int32_t)samples[2*i+1]); - loudness += left; - loudness += right; - if (left > CLIPPING_THRESHOLD) { isClipping |= (++oversLeft >= CLIPPING_DETECTION); } else { @@ -203,13 +197,11 @@ static float computeLoudness(int16_t* samples, int numSamples, int numChannels, } } } else { - int32_t overs = 0; + int overs = 0; for (int i = 0; i < numSamples; i++) { int32_t sample = std::abs((int32_t)samples[i]); - loudness += sample; - if (sample > CLIPPING_THRESHOLD) { isClipping |= (++overs >= CLIPPING_DETECTION); } else { @@ -218,6 +210,17 @@ static float computeLoudness(int16_t* samples, int numSamples, int numChannels, } } + return isClipping; +} + +static float computeLoudness(int16_t* samples, int numSamples) { + + float scale = numSamples ? 1.0f / numSamples : 0.0f; + + int32_t loudness = 0; + for (int i = 0; i < numSamples; i++) { + loudness += std::abs((int32_t)samples[i]); + } return (float)loudness * scale; } @@ -1393,6 +1396,15 @@ void AudioClient::handleMicAudioInput() { _inputRingBuffer.readSamples(inputAudioSamples.get(), inputSamplesRequired); + // detect clipping on the raw input + bool isClipping = detectClipping(inputAudioSamples.get(), inputSamplesRequired, _inputFormat.channelCount()); + if (isClipping) { + _timeSinceLastClip = 0.0f; + } else if (_timeSinceLastClip >= 0.0f) { + _timeSinceLastClip += AudioConstants::NETWORK_FRAME_SECS; + } + isClipping = (_timeSinceLastClip >= 0.0f) && (_timeSinceLastClip < 2.0f); // 2 second hold time + #if defined(WEBRTC_ENABLED) if (_isAECEnabled) { processWebrtcNearEnd(inputAudioSamples.get(), inputSamplesRequired / _inputFormat.channelCount(), @@ -1400,9 +1412,7 @@ void AudioClient::handleMicAudioInput() { } #endif - // detect loudness and clipping on the raw input - bool isClipping = false; - float loudness = computeLoudness(inputAudioSamples.get(), inputSamplesRequired, _inputFormat.channelCount(), isClipping); + float loudness = computeLoudness(inputAudioSamples.get(), inputSamplesRequired); _lastRawInputLoudness = loudness; // envelope detection @@ -1410,14 +1420,6 @@ void AudioClient::handleMicAudioInput() { loudness += tc * (_lastSmoothedRawInputLoudness - loudness); _lastSmoothedRawInputLoudness = loudness; - // clipping indicator - if (isClipping) { - _timeSinceLastClip = 0.0f; - } else if (_timeSinceLastClip >= 0.0f) { - _timeSinceLastClip += AudioConstants::NETWORK_FRAME_SECS; - } - isClipping = (_timeSinceLastClip >= 0.0f) && (_timeSinceLastClip < 2.0f); // 2 second hold time - emit inputLoudnessChanged(_lastSmoothedRawInputLoudness, isClipping); if (!_muted) { From 74eacf23460a3cef407863327bbe9fd7ac6b3d43 Mon Sep 17 00:00:00 2001 From: luiscuenca Date: Wed, 7 Aug 2019 15:08:44 -0700 Subject: [PATCH 2/5] Make sure random switch does not repeat previous state --- libraries/animation/src/AnimNodeLoader.cpp | 3 -- libraries/animation/src/AnimRandomSwitch.cpp | 34 ++++++++++++++------ libraries/animation/src/AnimRandomSwitch.h | 3 +- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/libraries/animation/src/AnimNodeLoader.cpp b/libraries/animation/src/AnimNodeLoader.cpp index 6828ed1c7e..d43351fff9 100644 --- a/libraries/animation/src/AnimNodeLoader.cpp +++ b/libraries/animation/src/AnimNodeLoader.cpp @@ -943,9 +943,6 @@ bool processRandomSwitchStateMachineNode(AnimNode::Pointer node, const QJsonObje } auto randomStatePtr = std::make_shared(id, iter->second, interpTarget, interpDuration, interpTypeEnum, easingTypeEnum, priority, resume); - if (priority > 0.0f) { - smNode->addToPrioritySum(priority); - } assert(randomStatePtr); if (!interpTargetVar.isEmpty()) { diff --git a/libraries/animation/src/AnimRandomSwitch.cpp b/libraries/animation/src/AnimRandomSwitch.cpp index edc8c8dd96..92d6edb36f 100644 --- a/libraries/animation/src/AnimRandomSwitch.cpp +++ b/libraries/animation/src/AnimRandomSwitch.cpp @@ -27,22 +27,34 @@ const AnimPoseVec& AnimRandomSwitch::evaluate(const AnimVariantMap& animVars, co AnimRandomSwitch::RandomSwitchState::Pointer desiredState = _currentState; if (abs(_randomSwitchEvaluationCount - context.getEvaluationCount()) > 1 || animVars.lookup(_triggerRandomSwitchVar, false)) { - // get a random number and decide which motion to choose. + // filter states different to the last random state and with priorities. bool currentStateHasPriority = false; - float dice = randFloatInRange(0.0f, 1.0f); - float lowerBound = 0.0f; - for (const RandomSwitchState::Pointer& randState : _randomStates) { + std::vector randomStatesToConsider; + float totalPriorities = 0.0f; + for (int i = 0; i < _randomStates.size(); i++) { + auto randState = _randomStates[i]; if (randState->getPriority() > 0.0f) { - float upperBound = lowerBound + (randState->getPriority() / _totalPriorities); - if ((dice > lowerBound) && (dice < upperBound)) { - desiredState = randState; + bool isRepeatingClip = _children[randState->getChildIndex()]->getID() == _lastPlayedState; + if (!isRepeatingClip) { + randomStatesToConsider.push_back(randState); + totalPriorities += randState->getPriority(); } - lowerBound = upperBound; - // this indicates if the curent state is one that can be selected randomly, or is one that was transitioned to by the random duration timer. currentStateHasPriority = currentStateHasPriority || (_currentState == randState); } } + // get a random number and decide which motion to choose. + float dice = randFloatInRange(0.0f, 1.0f); + float lowerBound = 0.0f; + for (int i = 0; i < randomStatesToConsider.size(); i++) { + auto randState = randomStatesToConsider[i]; + float upperBound = lowerBound + (randState->getPriority() / totalPriorities); + if ((dice > lowerBound) && (dice < upperBound)) { + desiredState = randState; + break; + } + lowerBound = upperBound; + } if (abs(_randomSwitchEvaluationCount - context.getEvaluationCount()) > 1) { _duringInterp = false; switchRandomState(animVars, context, desiredState, _duringInterp); @@ -155,8 +167,10 @@ void AnimRandomSwitch::addState(RandomSwitchState::Pointer randomState) { } void AnimRandomSwitch::switchRandomState(const AnimVariantMap& animVars, const AnimContext& context, RandomSwitchState::Pointer desiredState, bool shouldInterp) { - auto nextStateNode = _children[desiredState->getChildIndex()]; + if (nextStateNode->getType() == AnimNodeType::Clip) { + _lastPlayedState = nextStateNode->getID(); + } if (shouldInterp) { const float FRAMES_PER_SECOND = 30.0f; diff --git a/libraries/animation/src/AnimRandomSwitch.h b/libraries/animation/src/AnimRandomSwitch.h index 7c185dd7cb..888ed1f6b5 100644 --- a/libraries/animation/src/AnimRandomSwitch.h +++ b/libraries/animation/src/AnimRandomSwitch.h @@ -143,7 +143,6 @@ protected: void setTransitionVar(const QString& transitionVar) { _transitionVar = transitionVar; } void setTriggerTimeMin(float triggerTimeMin) { _triggerTimeMin = triggerTimeMin; } void setTriggerTimeMax(float triggerTimeMax) { _triggerTimeMax = triggerTimeMax; } - void addToPrioritySum(float priority) { _totalPriorities += priority; } void addState(RandomSwitchState::Pointer randomState); @@ -164,7 +163,6 @@ protected: float _alpha = 0.0f; AnimPoseVec _prevPoses; AnimPoseVec _nextPoses; - float _totalPriorities { 0.0f }; RandomSwitchState::Pointer _currentState; RandomSwitchState::Pointer _previousState; @@ -179,6 +177,7 @@ protected: float _randomSwitchTimeMin { 10.0f }; float _randomSwitchTimeMax { 20.0f }; float _randomSwitchTime { 0.0f }; + QString _lastPlayedState; private: // no copies From 473af995d6d371941d4d564696aebe5fb71f1e80 Mon Sep 17 00:00:00 2001 From: luiscuenca Date: Wed, 7 Aug 2019 16:53:08 -0700 Subject: [PATCH 3/5] Fix warnings --- libraries/animation/src/AnimRandomSwitch.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/animation/src/AnimRandomSwitch.cpp b/libraries/animation/src/AnimRandomSwitch.cpp index 92d6edb36f..45e6ae0050 100644 --- a/libraries/animation/src/AnimRandomSwitch.cpp +++ b/libraries/animation/src/AnimRandomSwitch.cpp @@ -31,7 +31,7 @@ const AnimPoseVec& AnimRandomSwitch::evaluate(const AnimVariantMap& animVars, co bool currentStateHasPriority = false; std::vector randomStatesToConsider; float totalPriorities = 0.0f; - for (int i = 0; i < _randomStates.size(); i++) { + for (size_t i = 0; i < _randomStates.size(); i++) { auto randState = _randomStates[i]; if (randState->getPriority() > 0.0f) { bool isRepeatingClip = _children[randState->getChildIndex()]->getID() == _lastPlayedState; @@ -46,7 +46,7 @@ const AnimPoseVec& AnimRandomSwitch::evaluate(const AnimVariantMap& animVars, co // get a random number and decide which motion to choose. float dice = randFloatInRange(0.0f, 1.0f); float lowerBound = 0.0f; - for (int i = 0; i < randomStatesToConsider.size(); i++) { + for (size_t i = 0; i < randomStatesToConsider.size(); i++) { auto randState = randomStatesToConsider[i]; float upperBound = lowerBound + (randState->getPriority() / totalPriorities); if ((dice > lowerBound) && (dice < upperBound)) { From 3a22db0a13bcb6f22249960d8d4dba09eb722ccb Mon Sep 17 00:00:00 2001 From: luiscuenca Date: Thu, 8 Aug 2019 06:25:04 -0700 Subject: [PATCH 4/5] Last state for all node types --- libraries/animation/src/AnimRandomSwitch.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libraries/animation/src/AnimRandomSwitch.cpp b/libraries/animation/src/AnimRandomSwitch.cpp index 45e6ae0050..8b99568f07 100644 --- a/libraries/animation/src/AnimRandomSwitch.cpp +++ b/libraries/animation/src/AnimRandomSwitch.cpp @@ -168,9 +168,7 @@ void AnimRandomSwitch::addState(RandomSwitchState::Pointer randomState) { void AnimRandomSwitch::switchRandomState(const AnimVariantMap& animVars, const AnimContext& context, RandomSwitchState::Pointer desiredState, bool shouldInterp) { auto nextStateNode = _children[desiredState->getChildIndex()]; - if (nextStateNode->getType() == AnimNodeType::Clip) { - _lastPlayedState = nextStateNode->getID(); - } + _lastPlayedState = nextStateNode->getID(); if (shouldInterp) { const float FRAMES_PER_SECOND = 30.0f; From 597cff335adfcc00a161f5f8db26647d033beaaa Mon Sep 17 00:00:00 2001 From: luiscuenca Date: Thu, 8 Aug 2019 08:40:00 -0700 Subject: [PATCH 5/5] Reserve vector size --- libraries/animation/src/AnimRandomSwitch.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/animation/src/AnimRandomSwitch.cpp b/libraries/animation/src/AnimRandomSwitch.cpp index 8b99568f07..3cf402cc14 100644 --- a/libraries/animation/src/AnimRandomSwitch.cpp +++ b/libraries/animation/src/AnimRandomSwitch.cpp @@ -30,6 +30,7 @@ const AnimPoseVec& AnimRandomSwitch::evaluate(const AnimVariantMap& animVars, co // filter states different to the last random state and with priorities. bool currentStateHasPriority = false; std::vector randomStatesToConsider; + randomStatesToConsider.reserve(_randomStates.size()); float totalPriorities = 0.0f; for (size_t i = 0; i < _randomStates.size(); i++) { auto randState = _randomStates[i];