Mirrors/overte-HifiExperiments
3
0
Fork 0
mirror of https://github.com/HifiExperiments/overte.git synced 2025-06-23 10:09:52 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #10443 from ctrlaltdavid/21326-2

Browse source 
Fix avatar recording playback mouth not moving
This commit is contained in:
Andrew Meadows 2017-05-11 10:41:41 -07:00 committed by GitHub
commit 0852852f39
5 changed files with 43 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
interface/src/avatar/MyHead.cpp
View file

@ -48,7 +48,7 @@ void MyHead::simulate(float deltaTime) {
FaceTracker* faceTracker = qApp->getActiveFaceTracker(); FaceTracker* faceTracker = qApp->getActiveFaceTracker();
_isFaceTrackerConnected = faceTracker != NULL && !faceTracker->isMuted(); _isFaceTrackerConnected = faceTracker != NULL && !faceTracker->isMuted();
if (_isFaceTrackerConnected) { if (_isFaceTrackerConnected) {
_blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); _transientBlendshapeCoefficients = faceTracker->getBlendshapeCoefficients();
if (typeid(*faceTracker) == typeid(DdeFaceTracker)) { if (typeid(*faceTracker) == typeid(DdeFaceTracker)) {
@ -60,11 +60,11 @@ void MyHead::simulate(float deltaTime) {
const int FUNNEL_BLENDSHAPE = 40; const int FUNNEL_BLENDSHAPE = 40;
const int SMILE_LEFT_BLENDSHAPE = 28; const int SMILE_LEFT_BLENDSHAPE = 28;
const int SMILE_RIGHT_BLENDSHAPE = 29; const int SMILE_RIGHT_BLENDSHAPE = 29;
_blendshapeCoefficients[JAW_OPEN_BLENDSHAPE] += _audioJawOpen; _transientBlendshapeCoefficients[JAW_OPEN_BLENDSHAPE] += _audioJawOpen;
_blendshapeCoefficients[SMILE_LEFT_BLENDSHAPE] += _mouth4; _transientBlendshapeCoefficients[SMILE_LEFT_BLENDSHAPE] += _mouth4;
_blendshapeCoefficients[SMILE_RIGHT_BLENDSHAPE] += _mouth4; _transientBlendshapeCoefficients[SMILE_RIGHT_BLENDSHAPE] += _mouth4;
_blendshapeCoefficients[MMMM_BLENDSHAPE] += _mouth2; _transientBlendshapeCoefficients[MMMM_BLENDSHAPE] += _mouth2;
_blendshapeCoefficients[FUNNEL_BLENDSHAPE] += _mouth3; _transientBlendshapeCoefficients[FUNNEL_BLENDSHAPE] += _mouth3;
} }
applyEyelidOffset(getFinalOrientationInWorldFrame()); applyEyelidOffset(getFinalOrientationInWorldFrame());
} }

12
libraries/avatars-renderer/src/avatars-renderer/Head.cpp
View file

@ -151,7 +151,7 @@ void Head::simulate(float deltaTime) {
_mouth2, _mouth2,
_mouth3, _mouth3,
_mouth4, _mouth4,
_blendshapeCoefficients); _transientBlendshapeCoefficients);
applyEyelidOffset(getOrientation()); applyEyelidOffset(getOrientation());
@ -234,15 +234,15 @@ void Head::applyEyelidOffset(glm::quat headOrientation) {
for (int i = 0; i < 2; i++) { for (int i = 0; i < 2; i++) {
const int LEFT_EYE = 8; const int LEFT_EYE = 8;
float eyeCoefficient = _blendshapeCoefficients[i] - _blendshapeCoefficients[LEFT_EYE + i]; // Raw value float eyeCoefficient = _transientBlendshapeCoefficients[i] - _transientBlendshapeCoefficients[LEFT_EYE + i];
eyeCoefficient = glm::clamp(eyelidOffset + eyeCoefficient * (1.0f - eyelidOffset), -1.0f, 1.0f); eyeCoefficient = glm::clamp(eyelidOffset + eyeCoefficient * (1.0f - eyelidOffset), -1.0f, 1.0f);
if (eyeCoefficient > 0.0f) { if (eyeCoefficient > 0.0f) {
_blendshapeCoefficients[i] = eyeCoefficient; _transientBlendshapeCoefficients[i] = eyeCoefficient;
_blendshapeCoefficients[LEFT_EYE + i] = 0.0f; _transientBlendshapeCoefficients[LEFT_EYE + i] = 0.0f;
} else { } else {
_blendshapeCoefficients[i] = 0.0f; _transientBlendshapeCoefficients[i] = 0.0f;
_blendshapeCoefficients[LEFT_EYE + i] = -eyeCoefficient; _transientBlendshapeCoefficients[LEFT_EYE + i] = -eyeCoefficient;
} }
} }
} }

10
libraries/avatars/src/AvatarData.cpp
View file

@ -445,17 +445,17 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
if (hasFaceTrackerInfo) { if (hasFaceTrackerInfo) {
auto startSection = destinationBuffer; auto startSection = destinationBuffer;
auto faceTrackerInfo = reinterpret_cast<AvatarDataPacket::FaceTrackerInfo*>(destinationBuffer); auto faceTrackerInfo = reinterpret_cast<AvatarDataPacket::FaceTrackerInfo*>(destinationBuffer);
auto blendshapeCoefficients = _headData->getSummedBlendshapeCoefficients();
faceTrackerInfo->leftEyeBlink = _headData->_leftEyeBlink; faceTrackerInfo->leftEyeBlink = _headData->_leftEyeBlink;
faceTrackerInfo->rightEyeBlink = _headData->_rightEyeBlink; faceTrackerInfo->rightEyeBlink = _headData->_rightEyeBlink;
faceTrackerInfo->averageLoudness = _headData->_averageLoudness; faceTrackerInfo->averageLoudness = _headData->_averageLoudness;
faceTrackerInfo->browAudioLift = _headData->_browAudioLift; faceTrackerInfo->browAudioLift = _headData->_browAudioLift;
faceTrackerInfo->numBlendshapeCoefficients = _headData->_blendshapeCoefficients.size(); faceTrackerInfo->numBlendshapeCoefficients = blendshapeCoefficients.size();
destinationBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo); destinationBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo);
// followed by a variable number of float coefficients memcpy(destinationBuffer, blendshapeCoefficients.data(), blendshapeCoefficients.size() * sizeof(float));
memcpy(destinationBuffer, _headData->_blendshapeCoefficients.data(), _headData->_blendshapeCoefficients.size() * sizeof(float)); destinationBuffer += blendshapeCoefficients.size() * sizeof(float);
destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float);
int numBytes = destinationBuffer - startSection; int numBytes = destinationBuffer - startSection;
if (outboundDataRateOut) { if (outboundDataRateOut) {
@ -965,7 +965,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) {
const int coefficientsSize = sizeof(float) * numCoefficients; const int coefficientsSize = sizeof(float) * numCoefficients;
PACKET_READ_CHECK(FaceTrackerCoefficients, coefficientsSize); PACKET_READ_CHECK(FaceTrackerCoefficients, coefficientsSize);
_headData->_blendshapeCoefficients.resize(numCoefficients); // make sure there's room for the copy! _headData->_blendshapeCoefficients.resize(numCoefficients); // make sure there's room for the copy!
_headData->_baseBlendshapeCoefficients.resize(numCoefficients); _headData->_transientBlendshapeCoefficients.resize(numCoefficients);
memcpy(_headData->_blendshapeCoefficients.data(), sourceBuffer, coefficientsSize); memcpy(_headData->_blendshapeCoefficients.data(), sourceBuffer, coefficientsSize);
sourceBuffer += coefficientsSize; sourceBuffer += coefficientsSize;
int numBytesRead = sourceBuffer - startSection; int numBytesRead = sourceBuffer - startSection;

45
libraries/avatars/src/HeadData.cpp
View file

@ -34,8 +34,9 @@ HeadData::HeadData(AvatarData* owningAvatar) :
_rightEyeBlink(0.0f), _rightEyeBlink(0.0f),
_averageLoudness(0.0f), _averageLoudness(0.0f),
_browAudioLift(0.0f), _browAudioLift(0.0f),
_baseBlendshapeCoefficients(QVector<float>(0, 0.0f)), _blendshapeCoefficients(QVector<float>(0, 0.0f)),
_currBlendShapeCoefficients(QVector<float>(0, 0.0f)), _transientBlendshapeCoefficients(QVector<float>(0, 0.0f)),
_summedBlendshapeCoefficients(QVector<float>(0, 0.0f)),
_owningAvatar(owningAvatar) _owningAvatar(owningAvatar)
{ {
@ -85,22 +86,22 @@ static const QMap<QString, int>& getBlendshapesLookupMap() {
} }
const QVector<float>& HeadData::getSummedBlendshapeCoefficients() { const QVector<float>& HeadData::getSummedBlendshapeCoefficients() {
int maxSize = std::max(_baseBlendshapeCoefficients.size(), _blendshapeCoefficients.size()); int maxSize = std::max(_blendshapeCoefficients.size(), _transientBlendshapeCoefficients.size());
if (_currBlendShapeCoefficients.size() != maxSize) { if (_summedBlendshapeCoefficients.size() != maxSize) {
_currBlendShapeCoefficients.resize(maxSize); _summedBlendshapeCoefficients.resize(maxSize);
} }
for (int i = 0; i < maxSize; i++) { for (int i = 0; i < maxSize; i++) {
if (i >= _baseBlendshapeCoefficients.size()) { if (i >= _blendshapeCoefficients.size()) {
_currBlendShapeCoefficients[i] = _blendshapeCoefficients[i]; _summedBlendshapeCoefficients[i] = _transientBlendshapeCoefficients[i];
} else if (i >= _blendshapeCoefficients.size()) { } else if (i >= _transientBlendshapeCoefficients.size()) {
_currBlendShapeCoefficients[i] = _baseBlendshapeCoefficients[i]; _summedBlendshapeCoefficients[i] = _blendshapeCoefficients[i];
} else { } else {
_currBlendShapeCoefficients[i] = _baseBlendshapeCoefficients[i] + _blendshapeCoefficients[i]; _summedBlendshapeCoefficients[i] = _blendshapeCoefficients[i] + _transientBlendshapeCoefficients[i];
} }
} }
return _currBlendShapeCoefficients; return _summedBlendshapeCoefficients;
} }
void HeadData::setBlendshape(QString name, float val) { void HeadData::setBlendshape(QString name, float val) {
@ -112,10 +113,10 @@ void HeadData::setBlendshape(QString name, float val) {
if (_blendshapeCoefficients.size() <= it.value()) { if (_blendshapeCoefficients.size() <= it.value()) {
_blendshapeCoefficients.resize(it.value() + 1); _blendshapeCoefficients.resize(it.value() + 1);
} }
if (_baseBlendshapeCoefficients.size() <= it.value()) { if (_transientBlendshapeCoefficients.size() <= it.value()) {
_baseBlendshapeCoefficients.resize(it.value() + 1); _transientBlendshapeCoefficients.resize(it.value() + 1);
} }
_baseBlendshapeCoefficients[it.value()] = val; _blendshapeCoefficients[it.value()] = val;
} }
} }
@ -131,14 +132,16 @@ QJsonObject HeadData::toJson() const {
QJsonObject blendshapesJson; QJsonObject blendshapesJson;
for (auto name : blendshapeLookupMap.keys()) { for (auto name : blendshapeLookupMap.keys()) {
auto index = blendshapeLookupMap[name]; auto index = blendshapeLookupMap[name];
if (index >= _blendshapeCoefficients.size()) { float value = 0.0f;
continue; if (index < _blendshapeCoefficients.size()) {
value += _blendshapeCoefficients[index];
} }
auto value = _blendshapeCoefficients[index]; if (index < _transientBlendshapeCoefficients.size()) {
if (value == 0.0f) { value += _transientBlendshapeCoefficients[index];
continue; }
if (value != 0.0f) {
blendshapesJson[name] = value;
} }
blendshapesJson[name] = value;
} }
if (!blendshapesJson.isEmpty()) { if (!blendshapesJson.isEmpty()) {
headJson[JSON_AVATAR_HEAD_BLENDSHAPE_COEFFICIENTS] = blendshapesJson; headJson[JSON_AVATAR_HEAD_BLENDSHAPE_COEFFICIENTS] = blendshapesJson;
@ -163,8 +166,8 @@ void HeadData::fromJson(const QJsonObject& json) {
QJsonArray blendshapeCoefficientsJson = jsonValue.toArray(); QJsonArray blendshapeCoefficientsJson = jsonValue.toArray();
for (const auto& blendshapeCoefficient : blendshapeCoefficientsJson) { for (const auto& blendshapeCoefficient : blendshapeCoefficientsJson) {
blendshapeCoefficients.push_back((float)blendshapeCoefficient.toDouble()); blendshapeCoefficients.push_back((float)blendshapeCoefficient.toDouble());
setBlendshapeCoefficients(blendshapeCoefficients);
} }
setBlendshapeCoefficients(blendshapeCoefficients);
} else if (jsonValue.isObject()) { } else if (jsonValue.isObject()) {
QJsonObject blendshapeCoefficientsJson = jsonValue.toObject(); QJsonObject blendshapeCoefficientsJson = jsonValue.toObject();
for (const QString& name : blendshapeCoefficientsJson.keys()) { for (const QString& name : blendshapeCoefficientsJson.keys()) {

4
libraries/avatars/src/HeadData.h
View file

@ -93,8 +93,8 @@ protected:
float _browAudioLift; float _browAudioLift;
QVector<float> _blendshapeCoefficients; QVector<float> _blendshapeCoefficients;
QVector<float> _baseBlendshapeCoefficients; QVector<float> _transientBlendshapeCoefficients;
QVector<float> _currBlendShapeCoefficients; QVector<float> _summedBlendshapeCoefficients;
AvatarData* _owningAvatar; AvatarData* _owningAvatar;
private: private: