From 4ed22ad7acd1e5268b513879a9f609735d398d48 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 7 Apr 2017 18:00:32 -0700 Subject: [PATCH] move audio loudness to AvatarData, change AvatarData to ScriptOwnership --- interface/src/avatar/Head.cpp | 9 ++++++--- interface/src/avatar/MyAvatar.cpp | 4 ++-- libraries/avatars/src/AvatarData.cpp | 4 ++-- libraries/avatars/src/AvatarData.h | 21 +++++++++++++++----- libraries/avatars/src/AvatarHashMap.cpp | 3 --- libraries/avatars/src/HeadData.cpp | 2 -- libraries/avatars/src/HeadData.h | 17 +--------------- libraries/script-engine/src/ScriptEngine.cpp | 2 +- scripts/system/pal.js | 2 +- 9 files changed, 29 insertions(+), 35 deletions(-) diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index 4e748037a4..d404a1eb93 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -72,11 +73,13 @@ void Head::reset() { void Head::simulate(float deltaTime, bool isMine) { const float NORMAL_HZ = 60.0f; // the update rate the constant values were tuned for + // grab the audio loudness from the audio client + float audioLoudness = DependencyManager::get()->getLastInputLoudness(); // Update audio trailing average for rendering facial animations const float AUDIO_AVERAGING_SECS = 0.05f; const float AUDIO_LONG_TERM_AVERAGING_SECS = 30.0f; - _averageLoudness = glm::mix(_averageLoudness, _audioLoudness, glm::min(deltaTime / AUDIO_AVERAGING_SECS, 1.0f)); + _averageLoudness = glm::mix(_averageLoudness, audioLoudness, glm::min(deltaTime / AUDIO_AVERAGING_SECS, 1.0f)); if (_longTermAverageLoudness == -1.0f) { _longTermAverageLoudness = _averageLoudness; @@ -154,8 +157,8 @@ void Head::simulate(float deltaTime, bool isMine) { // Update audio attack data for facial animation (eyebrows and mouth) float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz _audioAttack = audioAttackAveragingRate * _audioAttack + - (1.0f - audioAttackAveragingRate) * fabs((_audioLoudness - _longTermAverageLoudness) - _lastLoudness); - _lastLoudness = (_audioLoudness - _longTermAverageLoudness); + (1.0f - audioAttackAveragingRate) * fabs((audioLoudness - _longTermAverageLoudness) - _lastLoudness); + _lastLoudness = (audioLoudness - _longTermAverageLoudness); const float BROW_LIFT_THRESHOLD = 100.0f; if (_audioAttack > BROW_LIFT_THRESHOLD) { diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index d1edf9d44e..9acd4367dd 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -403,8 +403,8 @@ void MyAvatar::update(float deltaTime) { // Also get the AudioClient so we can update the avatar bounding box data // on the AudioClient side. auto audio = DependencyManager::get(); - head->setAudioLoudness(audio->getLastInputLoudness()); - head->setAudioAverageLoudness(audio->getAudioAverageInputLoudness()); + setAudioLoudness(audio->getLastInputLoudness()); + setAudioAverageLoudness(audio->getAudioAverageInputLoudness()); glm::vec3 halfBoundingBoxDimensions(_characterController.getCapsuleRadius(), _characterController.getCapsuleHalfHeight(), _characterController.getCapsuleRadius()); halfBoundingBoxDimensions += _characterController.getCapsuleLocalOffset(); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index a1ea103edb..c63edbc176 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -351,7 +351,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent if (hasAudioLoudness) { auto startSection = destinationBuffer; auto data = reinterpret_cast(destinationBuffer); - data->audioLoudness = packFloatGainToByte(_headData->getAudioLoudness() / AUDIO_LOUDNESS_SCALE); + data->audioLoudness = packFloatGainToByte(getAudioLoudness() / AUDIO_LOUDNESS_SCALE); destinationBuffer += sizeof(AvatarDataPacket::AudioLoudness); int numBytes = destinationBuffer - startSection; @@ -835,7 +835,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { } return buffer.size(); } - _headData->setAudioLoudness(audioLoudness); + setAudioLoudness(audioLoudness); int numBytesRead = sourceBuffer - startSection; _audioLoudnessRate.increment(numBytesRead); _audioLoudnessUpdateRate.increment(); diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 1327798a0a..5ade0c448e 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -448,10 +448,18 @@ public: void setHeadRoll(float value) { _headData->setBaseRoll(value); } // access to Head().set/getAverageLoudness - float getAudioLoudness() const { return _headData->getAudioLoudness(); } - void setAudioLoudness(float value) { _headData->setAudioLoudness(value); } - float getAudioAverageLoudness() const { return _headData->getAudioAverageLoudness(); } - void setAudioAverageLoudness(float value) { _headData->setAudioAverageLoudness(value); } + + float getAudioLoudness() const { return _audioLoudness; } + void setAudioLoudness(float audioLoudness) { + if (audioLoudness != _audioLoudness) { + _audioLoudnessChanged = usecTimestampNow(); + } + _audioLoudness = audioLoudness; + } + bool audioLoudnessChangedSince(quint64 time) const { return _audioLoudnessChanged >= time; } + + float getAudioAverageLoudness() const { return _audioAverageLoudness; } + void setAudioAverageLoudness(float audioAverageLoudness) { _audioAverageLoudness = audioAverageLoudness; } // Scale virtual void setTargetScale(float targetScale); @@ -642,7 +650,6 @@ protected: bool avatarBoundingBoxChangedSince(quint64 time) const { return _avatarBoundingBoxChanged >= time; } bool avatarScaleChangedSince(quint64 time) const { return _avatarScaleChanged >= time; } bool lookAtPositionChangedSince(quint64 time) const { return _headData->lookAtPositionChangedSince(time); } - bool audioLoudnessChangedSince(quint64 time) const { return _headData->audioLoudnessChangedSince(time); } bool sensorToWorldMatrixChangedSince(quint64 time) const { return _sensorToWorldMatrixChanged >= time; } bool additionalFlagsChangedSince(quint64 time) const { return _additionalFlagsChanged >= time; } bool parentInfoChangedSince(quint64 time) const { return _parentChanged >= time; } @@ -768,6 +775,10 @@ protected: int getFauxJointIndex(const QString& name) const; + float _audioLoudness { 0.0f }; + quint64 _audioLoudnessChanged { 0 }; + float _audioAverageLoudness { 0.0f }; + private: friend void avatarStateFromFrame(const QByteArray& frameData, AvatarData* _avatar); static QUrl _defaultFullAvatarModelUrl; diff --git a/libraries/avatars/src/AvatarHashMap.cpp b/libraries/avatars/src/AvatarHashMap.cpp index c3a1428c23..54c4cdc643 100644 --- a/libraries/avatars/src/AvatarHashMap.cpp +++ b/libraries/avatars/src/AvatarHashMap.cpp @@ -19,9 +19,6 @@ #include "AvatarLogging.h" #include "AvatarHashMap.h" - -int avatarDataPointerMetaTypeId = qRegisterMetaType>(); - AvatarHashMap::AvatarHashMap() { auto nodeList = DependencyManager::get(); diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index bf8593f1d9..909c6a613e 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -32,14 +32,12 @@ HeadData::HeadData(AvatarData* owningAvatar) : _basePitch(0.0f), _baseRoll(0.0f), _lookAtPosition(0.0f, 0.0f, 0.0f), - _audioLoudness(0.0f), _isFaceTrackerConnected(false), _isEyeTrackerConnected(false), _leftEyeBlink(0.0f), _rightEyeBlink(0.0f), _averageLoudness(0.0f), _browAudioLift(0.0f), - _audioAverageLoudness(0.0f), _owningAvatar(owningAvatar) { diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index cbf6c6bb32..6c468daefd 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -57,18 +57,6 @@ public: glm::quat getOrientation() const; void setOrientation(const glm::quat& orientation); - float getAudioLoudness() const { return _audioLoudness; } - void setAudioLoudness(float audioLoudness) { - if (audioLoudness != _audioLoudness) { - _audioLoudnessChanged = usecTimestampNow(); - } - _audioLoudness = audioLoudness; - } - bool audioLoudnessChangedSince(quint64 time) { return _audioLoudnessChanged >= time; } - - float getAudioAverageLoudness() const { return _audioAverageLoudness; } - void setAudioAverageLoudness(float audioAverageLoudness) { _audioAverageLoudness = audioAverageLoudness; } - void setBlendshape(QString name, float val); const QVector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } void setBlendshapeCoefficients(const QVector& blendshapeCoefficients) { _blendshapeCoefficients = blendshapeCoefficients; } @@ -96,16 +84,13 @@ protected: glm::vec3 _lookAtPosition; quint64 _lookAtPositionChanged { 0 }; - float _audioLoudness; - quint64 _audioLoudnessChanged { 0 }; - bool _isFaceTrackerConnected; bool _isEyeTrackerConnected; float _leftEyeBlink; float _rightEyeBlink; float _averageLoudness; float _browAudioLift; - float _audioAverageLoudness; + QVector _blendshapeCoefficients; AvatarData* _owningAvatar; diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index caa42bea5e..b8d4c6c4ec 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -112,7 +112,7 @@ static QScriptValue debugPrint(QScriptContext* context, QScriptEngine* engine) { } QScriptValue avatarDataToScriptValue(QScriptEngine* engine, AvatarData* const &in) { - return engine->newQObject(in, QScriptEngine::QtOwnership, DEFAULT_QOBJECT_WRAP_OPTIONS); + return engine->newQObject(in, QScriptEngine::ScriptOwnership, DEFAULT_QOBJECT_WRAP_OPTIONS); } void avatarDataFromScriptValue(const QScriptValue &object, AvatarData* &out) { diff --git a/scripts/system/pal.js b/scripts/system/pal.js index 80f51f56f9..17b0bf90ca 100644 --- a/scripts/system/pal.js +++ b/scripts/system/pal.js @@ -850,7 +850,7 @@ function getAudioLevel(id) { if (data) { // we will do exponential moving average by taking some the last loudness and averaging - data.accumulatedLevel = 0;//AVERAGING_RATIO * (data.accumulatedLevel || 0) + (1 - AVERAGING_RATIO) * (avatar.audioLoudness); + data.accumulatedLevel = AVERAGING_RATIO * (data.accumulatedLevel || 0) + (1 - AVERAGING_RATIO) * (avatar.audioLoudness); // add 1 to insure we don't go log() and hit -infinity. Math.log is // natural log, so to get log base 2, just divide by ln(2).