From e076b32f9c29160a744d29e167428f52446e52e8 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Mon, 6 Mar 2017 10:15:53 -0800 Subject: [PATCH 01/14] don't automatically unhook overlays from hands unless they were grabbable overlays --- scripts/system/controllers/handControllerGrab.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js index d982a032cc..cd18d6f932 100644 --- a/scripts/system/controllers/handControllerGrab.js +++ b/scripts/system/controllers/handControllerGrab.js @@ -3213,13 +3213,21 @@ function MyController(hand) { } _this.previouslyUnhooked[childID] = now; - // we don't know if it's an entity or an overlay + if (Overlays.getProperty(childID, "grabbable")) { + // only auto-unhook overlays that were flagged as grabbable. this avoids unhooking overlays + // used in tutorial. + Overlays.editOverlay(childID, { + parentID: previousParentID, + parentJointIndex: previousParentJointIndex + }); + } Entities.editEntity(childID, { parentID: previousParentID, parentJointIndex: previousParentJointIndex }); - Overlays.editOverlay(childID, { parentID: previousParentID, parentJointIndex: previousParentJointIndex }); } else { Entities.editEntity(childID, { parentID: NULL_UUID }); - Overlays.editOverlay(childID, { parentID: NULL_UUID }); + if (Overlays.getProperty(childID, "grabbable")) { + Overlays.editOverlay(childID, { parentID: NULL_UUID }); + } } } }); From 74323cb1f0065e3e41c97b8c615cdb12de50f370 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Mon, 6 Mar 2017 11:45:49 -0800 Subject: [PATCH 02/14] remove debug print --- scripts/system/controllers/handControllerGrab.js | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js index cd18d6f932..5b99738a78 100644 --- a/scripts/system/controllers/handControllerGrab.js +++ b/scripts/system/controllers/handControllerGrab.js @@ -3195,7 +3195,6 @@ function MyController(hand) { // we appear to be holding something and this script isn't in a state that would be holding something. // unhook it. if we previously took note of this entity's parent, put it back where it was. This // works around some problems that happen when more than one hand or avatar is passing something around. - print("disconnecting stray child of hand: (" + _this.hand + ") " + childID); if (_this.previousParentID[childID]) { var previousParentID = _this.previousParentID[childID]; var previousParentJointIndex = _this.previousParentJointIndex[childID]; From c52c43e23d27142118fae920b37970f61f67cfc1 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 6 Mar 2017 19:35:00 -0800 Subject: [PATCH 03/14] Remove weighted offset, special case downward pressure --- .../animation/src/AnimInverseKinematics.cpp | 48 +++++++------------ 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/libraries/animation/src/AnimInverseKinematics.cpp b/libraries/animation/src/AnimInverseKinematics.cpp index 173af3fdf6..fa8e4654f6 100644 --- a/libraries/animation/src/AnimInverseKinematics.cpp +++ b/libraries/animation/src/AnimInverseKinematics.cpp @@ -488,13 +488,7 @@ const AnimPoseVec& AnimInverseKinematics::overlay(const AnimVariantMap& animVars // measure new _hipsOffset for next frame // by looking for discrepancies between where a targeted endEffector is // and where it wants to be (after IK solutions are done) - - // use weighted average between HMD and other targets - float HMD_WEIGHT = 10.0f; - float OTHER_WEIGHT = 1.0f; - float totalWeight = 0.0f; - - glm::vec3 additionalHipsOffset = Vectors::ZERO; + glm::vec3 newHipsOffset = Vectors::ZERO; for (auto& target: targets) { int targetIndex = target.getIndex(); if (targetIndex == _headIndex && _headIndex != -1) { @@ -505,42 +499,34 @@ const AnimPoseVec& AnimInverseKinematics::overlay(const AnimVariantMap& animVars glm::vec3 under = _skeleton->getAbsolutePose(_headIndex, underPoses).trans(); glm::vec3 actual = _skeleton->getAbsolutePose(_headIndex, _relativePoses).trans(); const float HEAD_OFFSET_SLAVE_FACTOR = 0.65f; - additionalHipsOffset += (OTHER_WEIGHT * HEAD_OFFSET_SLAVE_FACTOR) * (under- actual); - totalWeight += OTHER_WEIGHT; + newHipsOffset += HEAD_OFFSET_SLAVE_FACTOR * (actual - under); } else if (target.getType() == IKTarget::Type::HmdHead) { + // we want to shift the hips to bring the head to its designated position glm::vec3 actual = _skeleton->getAbsolutePose(_headIndex, _relativePoses).trans(); - glm::vec3 thisOffset = target.getTranslation() - actual; - glm::vec3 futureHipsOffset = _hipsOffset + thisOffset; - if (glm::length(glm::vec2(futureHipsOffset.x, futureHipsOffset.z)) < _maxHipsOffsetLength) { - // it is imperative to shift the hips and bring the head to its designated position - // so we slam newHipsOffset here and ignore all other targets - additionalHipsOffset = futureHipsOffset - _hipsOffset; - totalWeight = 0.0f; - break; - } else { - additionalHipsOffset += HMD_WEIGHT * (target.getTranslation() - actual); - totalWeight += HMD_WEIGHT; - } + _hipsOffset += target.getTranslation() - actual; + // and ignore all other targets + newHipsOffset = _hipsOffset; + break; + } else if (target.getType() == IKTarget::Type::RotationAndPosition) { + glm::vec3 actualPosition = _skeleton->getAbsolutePose(targetIndex, _relativePoses).trans(); + glm::vec3 targetPosition = target.getTranslation(); + newHipsOffset += targetPosition - actualPosition; + + // Add downward pressure on the hips + newHipsOffset *= 0.95f; + newHipsOffset -= 1.0f; } } else if (target.getType() == IKTarget::Type::RotationAndPosition) { glm::vec3 actualPosition = _skeleton->getAbsolutePose(targetIndex, _relativePoses).trans(); glm::vec3 targetPosition = target.getTranslation(); - additionalHipsOffset += OTHER_WEIGHT * (targetPosition - actualPosition); - totalWeight += OTHER_WEIGHT; + newHipsOffset += targetPosition - actualPosition; } } - if (totalWeight > 1.0f) { - additionalHipsOffset /= totalWeight; - } - - // Add downward pressure on the hips - additionalHipsOffset *= 0.95f; - additionalHipsOffset -= 1.0f; // smooth transitions by relaxing _hipsOffset toward the new value const float HIPS_OFFSET_SLAVE_TIMESCALE = 0.10f; float tau = dt < HIPS_OFFSET_SLAVE_TIMESCALE ? dt / HIPS_OFFSET_SLAVE_TIMESCALE : 1.0f; - _hipsOffset += additionalHipsOffset * tau; + _hipsOffset += (newHipsOffset - _hipsOffset) * tau; // clamp the hips offset float hipsOffsetLength = glm::length(_hipsOffset); From ffe69acb5cb0ec4acc53a3a2ab4d5c25c2b633dd Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Wed, 8 Mar 2017 19:26:12 -0800 Subject: [PATCH 04/14] fix rare physics crash --- libraries/physics/src/PhysicsEngine.cpp | 27 +++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/libraries/physics/src/PhysicsEngine.cpp b/libraries/physics/src/PhysicsEngine.cpp index f57be4eab3..1073f4b1cf 100644 --- a/libraries/physics/src/PhysicsEngine.cpp +++ b/libraries/physics/src/PhysicsEngine.cpp @@ -143,12 +143,35 @@ void PhysicsEngine::addObjectToDynamicsWorld(ObjectMotionState* motionState) { } void PhysicsEngine::removeObjects(const VectorOfMotionStates& objects) { - // first bump and prune contacts for all objects in the list + // bump and prune contacts for all objects in the list for (auto object : objects) { bumpAndPruneContacts(object); } - // then remove them + if (_activeStaticBodies.size() > 0) { + // very unlikely to get here but its theoretically possible: + // immediately after a static entity was moved we skipped a simulation step and never got around to + // updating the static entity's RigidBody AABB. Now we're deleting bodies ==> we must scan + // _activeStaticBodies for bodies being removed and clear any that we find. + for (auto object : objects) { + btRigidBody* body = object->getRigidBody(); + + std::vector::reverse_iterator itr = _activeStaticBodies.rbegin(); + while (itr != _activeStaticBodies.rend()) { + if (body == *itr) { + if (*itr != *(_activeStaticBodies.rbegin())) { + // swap with rbegin + *itr = *(_activeStaticBodies.rbegin()); + } + _activeStaticBodies.pop_back(); + break; + } + ++itr; + } + } + } + + // remove bodies for (auto object : objects) { btRigidBody* body = object->getRigidBody(); if (body) { From 2c141fd5552565a8c5e14fa6f2382b3080113c25 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Fri, 10 Mar 2017 09:18:29 -0800 Subject: [PATCH 05/14] more correct comment --- libraries/physics/src/PhysicsEngine.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/physics/src/PhysicsEngine.cpp b/libraries/physics/src/PhysicsEngine.cpp index 1073f4b1cf..363887de25 100644 --- a/libraries/physics/src/PhysicsEngine.cpp +++ b/libraries/physics/src/PhysicsEngine.cpp @@ -149,10 +149,10 @@ void PhysicsEngine::removeObjects(const VectorOfMotionStates& objects) { } if (_activeStaticBodies.size() > 0) { - // very unlikely to get here but its theoretically possible: - // immediately after a static entity was moved we skipped a simulation step and never got around to - // updating the static entity's RigidBody AABB. Now we're deleting bodies ==> we must scan - // _activeStaticBodies for bodies being removed and clear any that we find. + // _activeStaticBodies was not cleared last frame. + // The only way to get here is if a static object were moved but we did not actually step the simulation last + // frame (because the framerate is faster than our physics simulation rate). When this happens we must scan + // _activeStaticBodies for objects that were recently deleted so we don't try to access a dangling pointer. for (auto object : objects) { btRigidBody* body = object->getRigidBody(); From b67b17d3c01bbdfb957a8feba6506cbcea02b14b Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 13 Mar 2017 09:10:34 -0800 Subject: [PATCH 06/14] Fix JS exception messages formatting --- libraries/script-engine/src/ScriptEngine.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index 73a79f1bc6..d721d1c86f 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -142,7 +142,7 @@ QString encodeEntityIdIntoEntityUrl(const QString& url, const QString& entityID) QString ScriptEngine::logException(const QScriptValue& exception) { auto message = formatException(exception); - scriptErrorMessage(qPrintable(message)); + scriptErrorMessage(message); return message; } @@ -453,7 +453,7 @@ void ScriptEngine::loadURL(const QUrl& scriptURL, bool reload) { } void ScriptEngine::scriptErrorMessage(const QString& message) { - qCCritical(scriptengine) << message; + qCCritical(scriptengine) << qPrintable(message); emit errorMessage(message); } From 3e020ede78ba0584114a0c402f4d93af0ae0fc32 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 13 Mar 2017 11:58:43 -0700 Subject: [PATCH 07/14] CR feedback --- libraries/audio-client/src/AudioClient.cpp | 9 +- libraries/audio-client/src/AudioNoiseGate.cpp | 97 ++++++++++--------- libraries/audio-client/src/AudioNoiseGate.h | 24 ++--- 3 files changed, 68 insertions(+), 62 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 9fb8bae6ca..c32b5600d9 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1028,7 +1028,7 @@ void AudioClient::handleAudioInput() { // if we performed the noise gate we can get values from it instead of enumerating the samples again _lastInputLoudness = _inputGate.getLastLoudness(); - if (_inputGate.clippedInLastFrame()) { + if (_inputGate.clippedInLastBlock()) { _timeSinceLastClip = 0.0f; } @@ -1049,10 +1049,9 @@ void AudioClient::handleAudioInput() { emit inputReceived({ reinterpret_cast(networkAudioSamples), numNetworkBytes }); - if (_inputGate.openedInLastFrame()) { + if (_inputGate.openedInLastBlock()) { emit noiseGateOpened(); - } - if (_inputGate.closedInLastFrame()) { + } else if (_inputGate.closedInLastBlock()) { emit noiseGateClosed(); } @@ -1072,7 +1071,7 @@ void AudioClient::handleAudioInput() { // the output from the input gate (eventually, this could be crossfaded) // and allow the codec to properly encode down to silent/zero. If we still // have _lastInputLoudness of 0 in our NEXT frame, we will send a silent packet - if (_lastInputLoudness == 0 && !_inputGate.closedInLastFrame()) { + if (_lastInputLoudness == 0 && !_inputGate.closedInLastBlock()) { packetType = PacketType::SilentAudioFrame; _silentOutbound.increment(); } else { diff --git a/libraries/audio-client/src/AudioNoiseGate.cpp b/libraries/audio-client/src/AudioNoiseGate.cpp index 17bbd1a509..2acdea6f0c 100644 --- a/libraries/audio-client/src/AudioNoiseGate.cpp +++ b/libraries/audio-client/src/AudioNoiseGate.cpp @@ -19,16 +19,16 @@ const float AudioNoiseGate::CLIPPING_THRESHOLD = 0.90f; AudioNoiseGate::AudioNoiseGate() : - _inputFrameCounter(0), + _inputBlockCounter(0), _lastLoudness(0.0f), - _quietestFrame(std::numeric_limits::max()), - _loudestFrame(0.0f), - _didClipInLastFrame(false), + _quietestBlock(std::numeric_limits::max()), + _loudestBlock(0.0f), + _didClipInLastBlock(false), _dcOffset(0.0f), _measuredFloor(0.0f), _sampleCounter(0), _isOpen(false), - _framesToClose(0) + _buffersToClose(0) { } @@ -37,7 +37,7 @@ void AudioNoiseGate::removeDCOffset(int16_t* samples, int numSamples) { // // DC Offset correction // - // Measure the DC offset over a trailing number of frames, and remove it from the input signal. + // Measure the DC offset over a trailing number of buffers, and remove it from the input signal. // This causes the noise background measurements and server muting to be more accurate. Many off-board // ADC's have a noticeable DC offset. // @@ -51,7 +51,7 @@ void AudioNoiseGate::removeDCOffset(int16_t* samples, int numSamples) { // Update measured DC offset measuredDcOffset /= numSamples; if (_dcOffset == 0.0f) { - // On first frame, copy over measured offset + // On first buffer, copy over measured offset _dcOffset = measuredDcOffset; } else { _dcOffset = DC_OFFSET_AVERAGING * _dcOffset + (1.0f - DC_OFFSET_AVERAGING) * measuredDcOffset; @@ -69,13 +69,13 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { // // NOISE_GATE_HEIGHT: How loud you have to speak relative to noise background to open the gate. // Make this value lower for more sensitivity and less rejection of noise. - // NOISE_GATE_WIDTH: The number of samples in an audio frame for which the height must be exceeded + // NOISE_GATE_WIDTH: The number of samples in an audio buffer for which the height must be exceeded // to open the gate. - // NOISE_GATE_CLOSE_FRAME_DELAY: Once the noise is below the gate height for the frame, how many frames + // NOISE_GATE_CLOSE_BLOCK_DELAY: Once the noise is below the gate height for the buffer, how many buffers // will we wait before closing the gate. - // NOISE_GATE_FRAMES_TO_AVERAGE: How many audio frames should we average together to compute noise floor. + // NOISE_GATE_BLOCKSS_TO_AVERAGE: How many audio buffers should we average together to compute noise floor. // More means better rejection but also can reject continuous things like singing. - // NUMBER_OF_NOISE_SAMPLE_FRAMES: How often should we re-evaluate the noise floor? + // NUMBER_OF_NOISE_SAMPLE_BLOCKS: How often should we re-evaluate the noise floor? float loudness = 0; int thisSample = 0; @@ -83,16 +83,16 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { const float NOISE_GATE_HEIGHT = 7.0f; const int NOISE_GATE_WIDTH = 5; - const int NOISE_GATE_CLOSE_FRAME_DELAY = 5; - const int NOISE_GATE_FRAMES_TO_AVERAGE = 5; + const int NOISE_GATE_CLOSE_BLOCK_DELAY = 5; + const int NOISE_GATE_BLOCKS_TO_AVERAGE = 5; // Check clipping, and check if should open noise gate - _didClipInLastFrame = false; + _didClipInLastBlock = false; for (int i = 0; i < numSamples; i++) { thisSample = std::abs(samples[i]); if (thisSample >= ((float) AudioConstants::MAX_SAMPLE_VALUE * CLIPPING_THRESHOLD)) { - _didClipInLastFrame = true; + _didClipInLastBlock = true; } loudness += thisSample; @@ -104,61 +104,64 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { _lastLoudness = fabs(loudness / numSamples); - if (_quietestFrame > _lastLoudness) { - _quietestFrame = _lastLoudness; + if (_quietestBlock > _lastLoudness) { + _quietestBlock = _lastLoudness; } - if (_loudestFrame < _lastLoudness) { - _loudestFrame = _lastLoudness; + if (_loudestBlock < _lastLoudness) { + _loudestBlock = _lastLoudness; } const int FRAMES_FOR_NOISE_DETECTION = 400; - if (_inputFrameCounter++ > FRAMES_FOR_NOISE_DETECTION) { - _quietestFrame = std::numeric_limits::max(); - _loudestFrame = 0.0f; - _inputFrameCounter = 0; + if (_inputBlockCounter++ > FRAMES_FOR_NOISE_DETECTION) { + _quietestBlock = std::numeric_limits::max(); + _loudestBlock = 0.0f; + _inputBlockCounter = 0; } // If Noise Gate is enabled, check and turn the gate on and off - float averageOfAllSampleFrames = 0.0f; - _sampleFrames[_sampleCounter++] = _lastLoudness; - if (_sampleCounter == NUMBER_OF_NOISE_SAMPLE_FRAMES) { + float averageOfAllSampleBlocks = 0.0f; + _sampleBlocks[_sampleCounter++] = _lastLoudness; + if (_sampleCounter == NUMBER_OF_NOISE_SAMPLE_BLOCKS) { float smallestSample = std::numeric_limits::max(); - for (int i = 0; i <= NUMBER_OF_NOISE_SAMPLE_FRAMES - NOISE_GATE_FRAMES_TO_AVERAGE; i += NOISE_GATE_FRAMES_TO_AVERAGE) { + for (int i = 0; i <= NUMBER_OF_NOISE_SAMPLE_BLOCKS - NOISE_GATE_BLOCKS_TO_AVERAGE; i += NOISE_GATE_BLOCKS_TO_AVERAGE) { float thisAverage = 0.0f; - for (int j = i; j < i + NOISE_GATE_FRAMES_TO_AVERAGE; j++) { - thisAverage += _sampleFrames[j]; - averageOfAllSampleFrames += _sampleFrames[j]; + for (int j = i; j < i + NOISE_GATE_BLOCKS_TO_AVERAGE; j++) { + thisAverage += _sampleBlocks[j]; + averageOfAllSampleBlocks += _sampleBlocks[j]; } - thisAverage /= NOISE_GATE_FRAMES_TO_AVERAGE; + thisAverage /= NOISE_GATE_BLOCKS_TO_AVERAGE; if (thisAverage < smallestSample) { smallestSample = thisAverage; } } - averageOfAllSampleFrames /= NUMBER_OF_NOISE_SAMPLE_FRAMES; + averageOfAllSampleBlocks /= NUMBER_OF_NOISE_SAMPLE_BLOCKS; _measuredFloor = smallestSample; _sampleCounter = 0; } - _closedInLastFrame = false; - _openedInLastFrame = false; + _closedInLastBlock = false; + _openedInLastBlock = false; if (samplesOverNoiseGate > NOISE_GATE_WIDTH) { - _openedInLastFrame = !_isOpen; + _openedInLastBlock = !_isOpen; _isOpen = true; - _framesToClose = NOISE_GATE_CLOSE_FRAME_DELAY; + _buffersToClose = NOISE_GATE_CLOSE_BLOCK_DELAY; } else { - if (--_framesToClose == 0) { - _closedInLastFrame = _isOpen; + if (--_buffersToClose == 0) { + _closedInLastBlock = _isOpen; _isOpen = false; } } if (!_isOpen) { - if (_closedInLastFrame) { - // would be nice to do a little crossfade to silence + // First buffer after being closed gets faded to silence, we fade across + // the entire buffer on fading out. All subsequent buffers are muted by being slammed + // to zeros + if (_closedInLastBlock) { + float fadeSlope = (1.0f / numSamples); for (int i = 0; i < numSamples; i++) { - float fadedSample = (1.0f - (float)i / (float)numSamples) * (float)samples[i]; + float fadedSample = (1.0f - ((float)i * fadeSlope)) * (float)samples[i]; samples[i] = (int16_t)fadedSample; } } else { @@ -167,10 +170,14 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { _lastLoudness = 0; } - if (_openedInLastFrame) { - // would be nice to do a little crossfade from silence - for (int i = 0; i < numSamples; i++) { - float fadedSample = ((float)i / (float)numSamples) * (float)samples[i]; + if (_openedInLastBlock) { + // would be nice to do a little crossfade from silence, but we only want to fade + // across the first 1/10th of the buffer, because we don't want to miss early + // transients. + int fadeSamples = numSamples / 10; // fade over 1/10th of the samples + float fadeSlope = (1.0f / fadeSamples); + for (int i = 0; i < fadeSamples; i++) { + float fadedSample = (float)i * fadeSlope * (float)samples[i]; samples[i] = (int16_t)fadedSample; } } diff --git a/libraries/audio-client/src/AudioNoiseGate.h b/libraries/audio-client/src/AudioNoiseGate.h index fb31561994..81ef27664d 100644 --- a/libraries/audio-client/src/AudioNoiseGate.h +++ b/libraries/audio-client/src/AudioNoiseGate.h @@ -14,7 +14,7 @@ #include -const int NUMBER_OF_NOISE_SAMPLE_FRAMES = 300; +const int NUMBER_OF_NOISE_SAMPLE_BLOCKS = 300; class AudioNoiseGate { public: @@ -23,9 +23,9 @@ public: void gateSamples(int16_t* samples, int numSamples); void removeDCOffset(int16_t* samples, int numSamples); - bool clippedInLastFrame() const { return _didClipInLastFrame; } - bool closedInLastFrame() const { return _closedInLastFrame; } - bool openedInLastFrame() const { return _openedInLastFrame; } + bool clippedInLastBlock() const { return _didClipInLastBlock; } + bool closedInLastBlock() const { return _closedInLastBlock; } + bool openedInLastBlock() const { return _openedInLastBlock; } bool isOpen() const { return _isOpen; } float getMeasuredFloor() const { return _measuredFloor; } float getLastLoudness() const { return _lastLoudness; } @@ -33,19 +33,19 @@ public: static const float CLIPPING_THRESHOLD; private: - int _inputFrameCounter; + int _inputBlockCounter; float _lastLoudness; - float _quietestFrame; - float _loudestFrame; - bool _didClipInLastFrame; + float _quietestBlock; + float _loudestBlock; + bool _didClipInLastBlock; float _dcOffset; float _measuredFloor; - float _sampleFrames[NUMBER_OF_NOISE_SAMPLE_FRAMES]; + float _sampleBlocks[NUMBER_OF_NOISE_SAMPLE_BLOCKS]; int _sampleCounter; bool _isOpen; - bool _closedInLastFrame { false }; - bool _openedInLastFrame { false }; - int _framesToClose; + bool _closedInLastBlock { false }; + bool _openedInLastBlock { false }; + int _buffersToClose; }; #endif // hifi_AudioNoiseGate_h \ No newline at end of file From 099ee1db7f1ffda4248b3c3a4a4aabfd04b5a8be Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 13 Mar 2017 12:05:42 -0700 Subject: [PATCH 08/14] CR feedback --- libraries/audio-client/src/AudioNoiseGate.cpp | 22 +++++++++---------- libraries/audio-client/src/AudioNoiseGate.h | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/libraries/audio-client/src/AudioNoiseGate.cpp b/libraries/audio-client/src/AudioNoiseGate.cpp index 2acdea6f0c..7aef8086a0 100644 --- a/libraries/audio-client/src/AudioNoiseGate.cpp +++ b/libraries/audio-client/src/AudioNoiseGate.cpp @@ -28,7 +28,7 @@ AudioNoiseGate::AudioNoiseGate() : _measuredFloor(0.0f), _sampleCounter(0), _isOpen(false), - _buffersToClose(0) + _blocksToClose(0) { } @@ -37,7 +37,7 @@ void AudioNoiseGate::removeDCOffset(int16_t* samples, int numSamples) { // // DC Offset correction // - // Measure the DC offset over a trailing number of buffers, and remove it from the input signal. + // Measure the DC offset over a trailing number of blocks, and remove it from the input signal. // This causes the noise background measurements and server muting to be more accurate. Many off-board // ADC's have a noticeable DC offset. // @@ -51,7 +51,7 @@ void AudioNoiseGate::removeDCOffset(int16_t* samples, int numSamples) { // Update measured DC offset measuredDcOffset /= numSamples; if (_dcOffset == 0.0f) { - // On first buffer, copy over measured offset + // On first block, copy over measured offset _dcOffset = measuredDcOffset; } else { _dcOffset = DC_OFFSET_AVERAGING * _dcOffset + (1.0f - DC_OFFSET_AVERAGING) * measuredDcOffset; @@ -69,11 +69,11 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { // // NOISE_GATE_HEIGHT: How loud you have to speak relative to noise background to open the gate. // Make this value lower for more sensitivity and less rejection of noise. - // NOISE_GATE_WIDTH: The number of samples in an audio buffer for which the height must be exceeded + // NOISE_GATE_WIDTH: The number of samples in an audio block for which the height must be exceeded // to open the gate. - // NOISE_GATE_CLOSE_BLOCK_DELAY: Once the noise is below the gate height for the buffer, how many buffers + // NOISE_GATE_CLOSE_BLOCK_DELAY: Once the noise is below the gate height for the block, how many blocks // will we wait before closing the gate. - // NOISE_GATE_BLOCKSS_TO_AVERAGE: How many audio buffers should we average together to compute noise floor. + // NOISE_GATE_BLOCKSS_TO_AVERAGE: How many audio blocks should we average together to compute noise floor. // More means better rejection but also can reject continuous things like singing. // NUMBER_OF_NOISE_SAMPLE_BLOCKS: How often should we re-evaluate the noise floor? @@ -147,16 +147,16 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { if (samplesOverNoiseGate > NOISE_GATE_WIDTH) { _openedInLastBlock = !_isOpen; _isOpen = true; - _buffersToClose = NOISE_GATE_CLOSE_BLOCK_DELAY; + _blocksToClose = NOISE_GATE_CLOSE_BLOCK_DELAY; } else { - if (--_buffersToClose == 0) { + if (--_blocksToClose == 0) { _closedInLastBlock = _isOpen; _isOpen = false; } } if (!_isOpen) { - // First buffer after being closed gets faded to silence, we fade across - // the entire buffer on fading out. All subsequent buffers are muted by being slammed + // First block after being closed gets faded to silence, we fade across + // the entire block on fading out. All subsequent blocks are muted by being slammed // to zeros if (_closedInLastBlock) { float fadeSlope = (1.0f / numSamples); @@ -172,7 +172,7 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { if (_openedInLastBlock) { // would be nice to do a little crossfade from silence, but we only want to fade - // across the first 1/10th of the buffer, because we don't want to miss early + // across the first 1/10th of the block, because we don't want to miss early // transients. int fadeSamples = numSamples / 10; // fade over 1/10th of the samples float fadeSlope = (1.0f / fadeSamples); diff --git a/libraries/audio-client/src/AudioNoiseGate.h b/libraries/audio-client/src/AudioNoiseGate.h index 81ef27664d..f72e92b0d5 100644 --- a/libraries/audio-client/src/AudioNoiseGate.h +++ b/libraries/audio-client/src/AudioNoiseGate.h @@ -45,7 +45,7 @@ private: bool _isOpen; bool _closedInLastBlock { false }; bool _openedInLastBlock { false }; - int _buffersToClose; + int _blocksToClose; }; #endif // hifi_AudioNoiseGate_h \ No newline at end of file From 956fdd09339378ae25ecc17a3fbb262d3f0c59ad Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 13 Mar 2017 12:06:38 -0700 Subject: [PATCH 09/14] CR feedback --- libraries/audio-client/src/AudioNoiseGate.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio-client/src/AudioNoiseGate.cpp b/libraries/audio-client/src/AudioNoiseGate.cpp index 7aef8086a0..8a9134b5dc 100644 --- a/libraries/audio-client/src/AudioNoiseGate.cpp +++ b/libraries/audio-client/src/AudioNoiseGate.cpp @@ -73,7 +73,7 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { // to open the gate. // NOISE_GATE_CLOSE_BLOCK_DELAY: Once the noise is below the gate height for the block, how many blocks // will we wait before closing the gate. - // NOISE_GATE_BLOCKSS_TO_AVERAGE: How many audio blocks should we average together to compute noise floor. + // NOISE_GATE_BLOCKS_TO_AVERAGE: How many audio blocks should we average together to compute noise floor. // More means better rejection but also can reject continuous things like singing. // NUMBER_OF_NOISE_SAMPLE_BLOCKS: How often should we re-evaluate the noise floor? From efc7fbacb14a800c46982f9b870620f010930b97 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 7 Mar 2017 10:03:38 -0800 Subject: [PATCH 10/14] Bug fix for potential crash when getting DebugDraw::instance (cherry picked from commit b0ad9a8110b8206115700e30ddd48d26959218a7) --- interface/src/Application.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index f870bd9f83..e35dcc674d 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -608,6 +608,8 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo } } + // make sure the debug draw singleton is initialized on the main thread. + DebugDraw::getInstance().removeMarker(""); _runningMarker.startRunningMarker(); From d38b994f4c2a577b47adc9d5868ef098ec2d2d0e Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 7 Mar 2017 16:07:30 -0800 Subject: [PATCH 11/14] Fix for potential crash due to DebugDraw data race. (cherry picked from commit 1b8a624edb81e8f62e2cb257b74e588653daaab2) --- libraries/render-utils/src/AnimDebugDraw.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libraries/render-utils/src/AnimDebugDraw.cpp b/libraries/render-utils/src/AnimDebugDraw.cpp index c8746d5c60..6066385847 100644 --- a/libraries/render-utils/src/AnimDebugDraw.cpp +++ b/libraries/render-utils/src/AnimDebugDraw.cpp @@ -346,7 +346,9 @@ void AnimDebugDraw::update() { numVerts += (int)markerMap.size() * VERTICES_PER_BONE; auto myAvatarMarkerMap = DebugDraw::getInstance().getMyAvatarMarkerMap(); numVerts += (int)myAvatarMarkerMap.size() * VERTICES_PER_BONE; - numVerts += (int)DebugDraw::getInstance().getRays().size() * VERTICES_PER_RAY; + auto rays = DebugDraw::getInstance().getRays(); + DebugDraw::getInstance().clearRays(); + numVerts += (int)rays.size() * VERTICES_PER_RAY; // allocate verts! std::vector vertices; @@ -398,10 +400,9 @@ void AnimDebugDraw::update() { } // draw rays from shared DebugDraw singleton - for (auto& iter : DebugDraw::getInstance().getRays()) { + for (auto& iter : rays) { addLine(std::get<0>(iter), std::get<1>(iter), std::get<2>(iter), v); } - DebugDraw::getInstance().clearRays(); data._vertexBuffer->resize(sizeof(AnimDebugDrawData::Vertex) * numVerts); data._vertexBuffer->setSubData(0, vertices); From 6747f553bf507ec33e64e75d25d3422427b53970 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 7 Mar 2017 18:13:42 -0800 Subject: [PATCH 12/14] Add DebugDraw to .eslintrc config file (cherry picked from commit c7cd0fdc3821ac932ad4e86e02c2797c5800182c) --- .eslintrc.js | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc.js b/.eslintrc.js index c708decc51..9635142d1a 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -17,6 +17,7 @@ module.exports = { "Clipboard": false, "Controller": false, "DialogsManager": false, + "DebugDraw": false, "Entities": false, "FaceTracker": false, "GlobalServices": false, From 0abcda86a8b17cae2f8425cf2d24c0d725352fe3 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 13 Mar 2017 18:52:31 -0400 Subject: [PATCH 13/14] mv AudioNoiseGate to audio --- libraries/audio-client/src/AudioClient.h | 2 +- libraries/{audio-client => audio}/src/AudioNoiseGate.cpp | 8 ++++---- libraries/{audio-client => audio}/src/AudioNoiseGate.h | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) rename libraries/{audio-client => audio}/src/AudioNoiseGate.cpp (99%) rename libraries/{audio-client => audio}/src/AudioNoiseGate.h (96%) diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 512b4bb3c1..7e9acc0586 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -45,13 +45,13 @@ #include #include #include +#include #include #include #include "AudioIOStats.h" -#include "AudioNoiseGate.h" #ifdef _WIN32 #pragma warning( push ) diff --git a/libraries/audio-client/src/AudioNoiseGate.cpp b/libraries/audio/src/AudioNoiseGate.cpp similarity index 99% rename from libraries/audio-client/src/AudioNoiseGate.cpp rename to libraries/audio/src/AudioNoiseGate.cpp index 8a9134b5dc..c99b31b989 100644 --- a/libraries/audio-client/src/AudioNoiseGate.cpp +++ b/libraries/audio/src/AudioNoiseGate.cpp @@ -1,6 +1,6 @@ // // AudioNoiseGate.cpp -// interface/src/audio +// libraries/audio // // Created by Stephen Birarda on 2014-12-16. // Copyright 2014 High Fidelity, Inc. @@ -9,12 +9,12 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +#include "AudioNoiseGate.h" + #include #include -#include - -#include "AudioNoiseGate.h" +#include "AudioConstants.h" const float AudioNoiseGate::CLIPPING_THRESHOLD = 0.90f; diff --git a/libraries/audio-client/src/AudioNoiseGate.h b/libraries/audio/src/AudioNoiseGate.h similarity index 96% rename from libraries/audio-client/src/AudioNoiseGate.h rename to libraries/audio/src/AudioNoiseGate.h index f72e92b0d5..d47cebf853 100644 --- a/libraries/audio-client/src/AudioNoiseGate.h +++ b/libraries/audio/src/AudioNoiseGate.h @@ -1,6 +1,6 @@ // // AudioNoiseGate.h -// interface/src/audio +// libraries/audio // // Created by Stephen Birarda on 2014-12-16. // Copyright 2014 High Fidelity, Inc. @@ -48,4 +48,4 @@ private: int _blocksToClose; }; -#endif // hifi_AudioNoiseGate_h \ No newline at end of file +#endif // hifi_AudioNoiseGate_h From f43706dfe1f2f2ed87b5e8793b8f1b147b56b8e7 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 13 Mar 2017 18:56:54 -0400 Subject: [PATCH 14/14] clean AudioNoiseGate --- libraries/audio/src/AudioNoiseGate.cpp | 38 ++++++-------------------- libraries/audio/src/AudioNoiseGate.h | 11 +++----- 2 files changed, 13 insertions(+), 36 deletions(-) diff --git a/libraries/audio/src/AudioNoiseGate.cpp b/libraries/audio/src/AudioNoiseGate.cpp index c99b31b989..604897af8a 100644 --- a/libraries/audio/src/AudioNoiseGate.cpp +++ b/libraries/audio/src/AudioNoiseGate.cpp @@ -19,19 +19,13 @@ const float AudioNoiseGate::CLIPPING_THRESHOLD = 0.90f; AudioNoiseGate::AudioNoiseGate() : - _inputBlockCounter(0), _lastLoudness(0.0f), - _quietestBlock(std::numeric_limits::max()), - _loudestBlock(0.0f), _didClipInLastBlock(false), _dcOffset(0.0f), _measuredFloor(0.0f), _sampleCounter(0), _isOpen(false), - _blocksToClose(0) -{ - -} + _blocksToClose(0) {} void AudioNoiseGate::removeDCOffset(int16_t* samples, int numSamples) { // @@ -80,7 +74,7 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { float loudness = 0; int thisSample = 0; int samplesOverNoiseGate = 0; - + const float NOISE_GATE_HEIGHT = 7.0f; const int NOISE_GATE_WIDTH = 5; const int NOISE_GATE_CLOSE_BLOCK_DELAY = 5; @@ -88,36 +82,22 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { // Check clipping, and check if should open noise gate _didClipInLastBlock = false; - + for (int i = 0; i < numSamples; i++) { thisSample = std::abs(samples[i]); if (thisSample >= ((float) AudioConstants::MAX_SAMPLE_VALUE * CLIPPING_THRESHOLD)) { _didClipInLastBlock = true; } - + loudness += thisSample; // Noise Reduction: Count peaks above the average loudness if (thisSample > (_measuredFloor * NOISE_GATE_HEIGHT)) { samplesOverNoiseGate++; } } - + _lastLoudness = fabs(loudness / numSamples); - - if (_quietestBlock > _lastLoudness) { - _quietestBlock = _lastLoudness; - } - if (_loudestBlock < _lastLoudness) { - _loudestBlock = _lastLoudness; - } - - const int FRAMES_FOR_NOISE_DETECTION = 400; - if (_inputBlockCounter++ > FRAMES_FOR_NOISE_DETECTION) { - _quietestBlock = std::numeric_limits::max(); - _loudestBlock = 0.0f; - _inputBlockCounter = 0; - } - + // If Noise Gate is enabled, check and turn the gate on and off float averageOfAllSampleBlocks = 0.0f; _sampleBlocks[_sampleCounter++] = _lastLoudness; @@ -130,7 +110,7 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { averageOfAllSampleBlocks += _sampleBlocks[j]; } thisAverage /= NOISE_GATE_BLOCKS_TO_AVERAGE; - + if (thisAverage < smallestSample) { smallestSample = thisAverage; } @@ -138,7 +118,7 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { averageOfAllSampleBlocks /= NUMBER_OF_NOISE_SAMPLE_BLOCKS; _measuredFloor = smallestSample; _sampleCounter = 0; - + } _closedInLastBlock = false; @@ -156,7 +136,7 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { } if (!_isOpen) { // First block after being closed gets faded to silence, we fade across - // the entire block on fading out. All subsequent blocks are muted by being slammed + // the entire block on fading out. All subsequent blocks are muted by being slammed // to zeros if (_closedInLastBlock) { float fadeSlope = (1.0f / numSamples); diff --git a/libraries/audio/src/AudioNoiseGate.h b/libraries/audio/src/AudioNoiseGate.h index d47cebf853..8430f120e5 100644 --- a/libraries/audio/src/AudioNoiseGate.h +++ b/libraries/audio/src/AudioNoiseGate.h @@ -19,24 +19,21 @@ const int NUMBER_OF_NOISE_SAMPLE_BLOCKS = 300; class AudioNoiseGate { public: AudioNoiseGate(); - + void gateSamples(int16_t* samples, int numSamples); void removeDCOffset(int16_t* samples, int numSamples); - + bool clippedInLastBlock() const { return _didClipInLastBlock; } bool closedInLastBlock() const { return _closedInLastBlock; } bool openedInLastBlock() const { return _openedInLastBlock; } bool isOpen() const { return _isOpen; } float getMeasuredFloor() const { return _measuredFloor; } float getLastLoudness() const { return _lastLoudness; } - + static const float CLIPPING_THRESHOLD; - + private: - int _inputBlockCounter; float _lastLoudness; - float _quietestBlock; - float _loudestBlock; bool _didClipInLastBlock; float _dcOffset; float _measuredFloor;