mirror of
https://github.com/overte-org/overte.git
synced 2025-08-09 00:17:25 +02:00
move audio loudness to AvatarData, change AvatarData to ScriptOwnership
This commit is contained in:
parent
1ed53113a9
commit
4ed22ad7ac
9 changed files with 29 additions and 35 deletions
|
@ -11,6 +11,7 @@
|
||||||
#include <glm/gtx/quaternion.hpp>
|
#include <glm/gtx/quaternion.hpp>
|
||||||
#include <gpu/Batch.h>
|
#include <gpu/Batch.h>
|
||||||
|
|
||||||
|
#include <AudioClient.h>
|
||||||
#include <NodeList.h>
|
#include <NodeList.h>
|
||||||
#include <recording/Deck.h>
|
#include <recording/Deck.h>
|
||||||
|
|
||||||
|
@ -72,11 +73,13 @@ void Head::reset() {
|
||||||
|
|
||||||
void Head::simulate(float deltaTime, bool isMine) {
|
void Head::simulate(float deltaTime, bool isMine) {
|
||||||
const float NORMAL_HZ = 60.0f; // the update rate the constant values were tuned for
|
const float NORMAL_HZ = 60.0f; // the update rate the constant values were tuned for
|
||||||
|
// grab the audio loudness from the audio client
|
||||||
|
float audioLoudness = DependencyManager::get<AudioClient>()->getLastInputLoudness();
|
||||||
|
|
||||||
// Update audio trailing average for rendering facial animations
|
// Update audio trailing average for rendering facial animations
|
||||||
const float AUDIO_AVERAGING_SECS = 0.05f;
|
const float AUDIO_AVERAGING_SECS = 0.05f;
|
||||||
const float AUDIO_LONG_TERM_AVERAGING_SECS = 30.0f;
|
const float AUDIO_LONG_TERM_AVERAGING_SECS = 30.0f;
|
||||||
_averageLoudness = glm::mix(_averageLoudness, _audioLoudness, glm::min(deltaTime / AUDIO_AVERAGING_SECS, 1.0f));
|
_averageLoudness = glm::mix(_averageLoudness, audioLoudness, glm::min(deltaTime / AUDIO_AVERAGING_SECS, 1.0f));
|
||||||
|
|
||||||
if (_longTermAverageLoudness == -1.0f) {
|
if (_longTermAverageLoudness == -1.0f) {
|
||||||
_longTermAverageLoudness = _averageLoudness;
|
_longTermAverageLoudness = _averageLoudness;
|
||||||
|
@ -154,8 +157,8 @@ void Head::simulate(float deltaTime, bool isMine) {
|
||||||
// Update audio attack data for facial animation (eyebrows and mouth)
|
// Update audio attack data for facial animation (eyebrows and mouth)
|
||||||
float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz
|
float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz
|
||||||
_audioAttack = audioAttackAveragingRate * _audioAttack +
|
_audioAttack = audioAttackAveragingRate * _audioAttack +
|
||||||
(1.0f - audioAttackAveragingRate) * fabs((_audioLoudness - _longTermAverageLoudness) - _lastLoudness);
|
(1.0f - audioAttackAveragingRate) * fabs((audioLoudness - _longTermAverageLoudness) - _lastLoudness);
|
||||||
_lastLoudness = (_audioLoudness - _longTermAverageLoudness);
|
_lastLoudness = (audioLoudness - _longTermAverageLoudness);
|
||||||
|
|
||||||
const float BROW_LIFT_THRESHOLD = 100.0f;
|
const float BROW_LIFT_THRESHOLD = 100.0f;
|
||||||
if (_audioAttack > BROW_LIFT_THRESHOLD) {
|
if (_audioAttack > BROW_LIFT_THRESHOLD) {
|
||||||
|
|
|
@ -403,8 +403,8 @@ void MyAvatar::update(float deltaTime) {
|
||||||
// Also get the AudioClient so we can update the avatar bounding box data
|
// Also get the AudioClient so we can update the avatar bounding box data
|
||||||
// on the AudioClient side.
|
// on the AudioClient side.
|
||||||
auto audio = DependencyManager::get<AudioClient>();
|
auto audio = DependencyManager::get<AudioClient>();
|
||||||
head->setAudioLoudness(audio->getLastInputLoudness());
|
setAudioLoudness(audio->getLastInputLoudness());
|
||||||
head->setAudioAverageLoudness(audio->getAudioAverageInputLoudness());
|
setAudioAverageLoudness(audio->getAudioAverageInputLoudness());
|
||||||
|
|
||||||
glm::vec3 halfBoundingBoxDimensions(_characterController.getCapsuleRadius(), _characterController.getCapsuleHalfHeight(), _characterController.getCapsuleRadius());
|
glm::vec3 halfBoundingBoxDimensions(_characterController.getCapsuleRadius(), _characterController.getCapsuleHalfHeight(), _characterController.getCapsuleRadius());
|
||||||
halfBoundingBoxDimensions += _characterController.getCapsuleLocalOffset();
|
halfBoundingBoxDimensions += _characterController.getCapsuleLocalOffset();
|
||||||
|
|
|
@ -351,7 +351,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
|
||||||
if (hasAudioLoudness) {
|
if (hasAudioLoudness) {
|
||||||
auto startSection = destinationBuffer;
|
auto startSection = destinationBuffer;
|
||||||
auto data = reinterpret_cast<AvatarDataPacket::AudioLoudness*>(destinationBuffer);
|
auto data = reinterpret_cast<AvatarDataPacket::AudioLoudness*>(destinationBuffer);
|
||||||
data->audioLoudness = packFloatGainToByte(_headData->getAudioLoudness() / AUDIO_LOUDNESS_SCALE);
|
data->audioLoudness = packFloatGainToByte(getAudioLoudness() / AUDIO_LOUDNESS_SCALE);
|
||||||
destinationBuffer += sizeof(AvatarDataPacket::AudioLoudness);
|
destinationBuffer += sizeof(AvatarDataPacket::AudioLoudness);
|
||||||
|
|
||||||
int numBytes = destinationBuffer - startSection;
|
int numBytes = destinationBuffer - startSection;
|
||||||
|
@ -835,7 +835,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) {
|
||||||
}
|
}
|
||||||
return buffer.size();
|
return buffer.size();
|
||||||
}
|
}
|
||||||
_headData->setAudioLoudness(audioLoudness);
|
setAudioLoudness(audioLoudness);
|
||||||
int numBytesRead = sourceBuffer - startSection;
|
int numBytesRead = sourceBuffer - startSection;
|
||||||
_audioLoudnessRate.increment(numBytesRead);
|
_audioLoudnessRate.increment(numBytesRead);
|
||||||
_audioLoudnessUpdateRate.increment();
|
_audioLoudnessUpdateRate.increment();
|
||||||
|
|
|
@ -448,10 +448,18 @@ public:
|
||||||
void setHeadRoll(float value) { _headData->setBaseRoll(value); }
|
void setHeadRoll(float value) { _headData->setBaseRoll(value); }
|
||||||
|
|
||||||
// access to Head().set/getAverageLoudness
|
// access to Head().set/getAverageLoudness
|
||||||
float getAudioLoudness() const { return _headData->getAudioLoudness(); }
|
|
||||||
void setAudioLoudness(float value) { _headData->setAudioLoudness(value); }
|
float getAudioLoudness() const { return _audioLoudness; }
|
||||||
float getAudioAverageLoudness() const { return _headData->getAudioAverageLoudness(); }
|
void setAudioLoudness(float audioLoudness) {
|
||||||
void setAudioAverageLoudness(float value) { _headData->setAudioAverageLoudness(value); }
|
if (audioLoudness != _audioLoudness) {
|
||||||
|
_audioLoudnessChanged = usecTimestampNow();
|
||||||
|
}
|
||||||
|
_audioLoudness = audioLoudness;
|
||||||
|
}
|
||||||
|
bool audioLoudnessChangedSince(quint64 time) const { return _audioLoudnessChanged >= time; }
|
||||||
|
|
||||||
|
float getAudioAverageLoudness() const { return _audioAverageLoudness; }
|
||||||
|
void setAudioAverageLoudness(float audioAverageLoudness) { _audioAverageLoudness = audioAverageLoudness; }
|
||||||
|
|
||||||
// Scale
|
// Scale
|
||||||
virtual void setTargetScale(float targetScale);
|
virtual void setTargetScale(float targetScale);
|
||||||
|
@ -642,7 +650,6 @@ protected:
|
||||||
bool avatarBoundingBoxChangedSince(quint64 time) const { return _avatarBoundingBoxChanged >= time; }
|
bool avatarBoundingBoxChangedSince(quint64 time) const { return _avatarBoundingBoxChanged >= time; }
|
||||||
bool avatarScaleChangedSince(quint64 time) const { return _avatarScaleChanged >= time; }
|
bool avatarScaleChangedSince(quint64 time) const { return _avatarScaleChanged >= time; }
|
||||||
bool lookAtPositionChangedSince(quint64 time) const { return _headData->lookAtPositionChangedSince(time); }
|
bool lookAtPositionChangedSince(quint64 time) const { return _headData->lookAtPositionChangedSince(time); }
|
||||||
bool audioLoudnessChangedSince(quint64 time) const { return _headData->audioLoudnessChangedSince(time); }
|
|
||||||
bool sensorToWorldMatrixChangedSince(quint64 time) const { return _sensorToWorldMatrixChanged >= time; }
|
bool sensorToWorldMatrixChangedSince(quint64 time) const { return _sensorToWorldMatrixChanged >= time; }
|
||||||
bool additionalFlagsChangedSince(quint64 time) const { return _additionalFlagsChanged >= time; }
|
bool additionalFlagsChangedSince(quint64 time) const { return _additionalFlagsChanged >= time; }
|
||||||
bool parentInfoChangedSince(quint64 time) const { return _parentChanged >= time; }
|
bool parentInfoChangedSince(quint64 time) const { return _parentChanged >= time; }
|
||||||
|
@ -768,6 +775,10 @@ protected:
|
||||||
|
|
||||||
int getFauxJointIndex(const QString& name) const;
|
int getFauxJointIndex(const QString& name) const;
|
||||||
|
|
||||||
|
float _audioLoudness { 0.0f };
|
||||||
|
quint64 _audioLoudnessChanged { 0 };
|
||||||
|
float _audioAverageLoudness { 0.0f };
|
||||||
|
|
||||||
private:
|
private:
|
||||||
friend void avatarStateFromFrame(const QByteArray& frameData, AvatarData* _avatar);
|
friend void avatarStateFromFrame(const QByteArray& frameData, AvatarData* _avatar);
|
||||||
static QUrl _defaultFullAvatarModelUrl;
|
static QUrl _defaultFullAvatarModelUrl;
|
||||||
|
|
|
@ -19,9 +19,6 @@
|
||||||
#include "AvatarLogging.h"
|
#include "AvatarLogging.h"
|
||||||
#include "AvatarHashMap.h"
|
#include "AvatarHashMap.h"
|
||||||
|
|
||||||
|
|
||||||
int avatarDataPointerMetaTypeId = qRegisterMetaType<QSharedPointer<AvatarData>>();
|
|
||||||
|
|
||||||
AvatarHashMap::AvatarHashMap() {
|
AvatarHashMap::AvatarHashMap() {
|
||||||
auto nodeList = DependencyManager::get<NodeList>();
|
auto nodeList = DependencyManager::get<NodeList>();
|
||||||
|
|
||||||
|
|
|
@ -32,14 +32,12 @@ HeadData::HeadData(AvatarData* owningAvatar) :
|
||||||
_basePitch(0.0f),
|
_basePitch(0.0f),
|
||||||
_baseRoll(0.0f),
|
_baseRoll(0.0f),
|
||||||
_lookAtPosition(0.0f, 0.0f, 0.0f),
|
_lookAtPosition(0.0f, 0.0f, 0.0f),
|
||||||
_audioLoudness(0.0f),
|
|
||||||
_isFaceTrackerConnected(false),
|
_isFaceTrackerConnected(false),
|
||||||
_isEyeTrackerConnected(false),
|
_isEyeTrackerConnected(false),
|
||||||
_leftEyeBlink(0.0f),
|
_leftEyeBlink(0.0f),
|
||||||
_rightEyeBlink(0.0f),
|
_rightEyeBlink(0.0f),
|
||||||
_averageLoudness(0.0f),
|
_averageLoudness(0.0f),
|
||||||
_browAudioLift(0.0f),
|
_browAudioLift(0.0f),
|
||||||
_audioAverageLoudness(0.0f),
|
|
||||||
_owningAvatar(owningAvatar)
|
_owningAvatar(owningAvatar)
|
||||||
{
|
{
|
||||||
|
|
||||||
|
|
|
@ -57,18 +57,6 @@ public:
|
||||||
glm::quat getOrientation() const;
|
glm::quat getOrientation() const;
|
||||||
void setOrientation(const glm::quat& orientation);
|
void setOrientation(const glm::quat& orientation);
|
||||||
|
|
||||||
float getAudioLoudness() const { return _audioLoudness; }
|
|
||||||
void setAudioLoudness(float audioLoudness) {
|
|
||||||
if (audioLoudness != _audioLoudness) {
|
|
||||||
_audioLoudnessChanged = usecTimestampNow();
|
|
||||||
}
|
|
||||||
_audioLoudness = audioLoudness;
|
|
||||||
}
|
|
||||||
bool audioLoudnessChangedSince(quint64 time) { return _audioLoudnessChanged >= time; }
|
|
||||||
|
|
||||||
float getAudioAverageLoudness() const { return _audioAverageLoudness; }
|
|
||||||
void setAudioAverageLoudness(float audioAverageLoudness) { _audioAverageLoudness = audioAverageLoudness; }
|
|
||||||
|
|
||||||
void setBlendshape(QString name, float val);
|
void setBlendshape(QString name, float val);
|
||||||
const QVector<float>& getBlendshapeCoefficients() const { return _blendshapeCoefficients; }
|
const QVector<float>& getBlendshapeCoefficients() const { return _blendshapeCoefficients; }
|
||||||
void setBlendshapeCoefficients(const QVector<float>& blendshapeCoefficients) { _blendshapeCoefficients = blendshapeCoefficients; }
|
void setBlendshapeCoefficients(const QVector<float>& blendshapeCoefficients) { _blendshapeCoefficients = blendshapeCoefficients; }
|
||||||
|
@ -96,16 +84,13 @@ protected:
|
||||||
glm::vec3 _lookAtPosition;
|
glm::vec3 _lookAtPosition;
|
||||||
quint64 _lookAtPositionChanged { 0 };
|
quint64 _lookAtPositionChanged { 0 };
|
||||||
|
|
||||||
float _audioLoudness;
|
|
||||||
quint64 _audioLoudnessChanged { 0 };
|
|
||||||
|
|
||||||
bool _isFaceTrackerConnected;
|
bool _isFaceTrackerConnected;
|
||||||
bool _isEyeTrackerConnected;
|
bool _isEyeTrackerConnected;
|
||||||
float _leftEyeBlink;
|
float _leftEyeBlink;
|
||||||
float _rightEyeBlink;
|
float _rightEyeBlink;
|
||||||
float _averageLoudness;
|
float _averageLoudness;
|
||||||
float _browAudioLift;
|
float _browAudioLift;
|
||||||
float _audioAverageLoudness;
|
|
||||||
QVector<float> _blendshapeCoefficients;
|
QVector<float> _blendshapeCoefficients;
|
||||||
AvatarData* _owningAvatar;
|
AvatarData* _owningAvatar;
|
||||||
|
|
||||||
|
|
|
@ -112,7 +112,7 @@ static QScriptValue debugPrint(QScriptContext* context, QScriptEngine* engine) {
|
||||||
}
|
}
|
||||||
|
|
||||||
QScriptValue avatarDataToScriptValue(QScriptEngine* engine, AvatarData* const &in) {
|
QScriptValue avatarDataToScriptValue(QScriptEngine* engine, AvatarData* const &in) {
|
||||||
return engine->newQObject(in, QScriptEngine::QtOwnership, DEFAULT_QOBJECT_WRAP_OPTIONS);
|
return engine->newQObject(in, QScriptEngine::ScriptOwnership, DEFAULT_QOBJECT_WRAP_OPTIONS);
|
||||||
}
|
}
|
||||||
|
|
||||||
void avatarDataFromScriptValue(const QScriptValue &object, AvatarData* &out) {
|
void avatarDataFromScriptValue(const QScriptValue &object, AvatarData* &out) {
|
||||||
|
|
|
@ -850,7 +850,7 @@ function getAudioLevel(id) {
|
||||||
if (data) {
|
if (data) {
|
||||||
|
|
||||||
// we will do exponential moving average by taking some the last loudness and averaging
|
// we will do exponential moving average by taking some the last loudness and averaging
|
||||||
data.accumulatedLevel = 0;//AVERAGING_RATIO * (data.accumulatedLevel || 0) + (1 - AVERAGING_RATIO) * (avatar.audioLoudness);
|
data.accumulatedLevel = AVERAGING_RATIO * (data.accumulatedLevel || 0) + (1 - AVERAGING_RATIO) * (avatar.audioLoudness);
|
||||||
|
|
||||||
// add 1 to insure we don't go log() and hit -infinity. Math.log is
|
// add 1 to insure we don't go log() and hit -infinity. Math.log is
|
||||||
// natural log, so to get log base 2, just divide by ln(2).
|
// natural log, so to get log base 2, just divide by ln(2).
|
||||||
|
|
Loading…
Reference in a new issue