Merge pull request #62 from HifiExperiments/blendshapes

Support blendshapes on model entities

libraries/avatars/src/HeadData.cpp

@@ -34,7 +34,6 @@ HeadData::HeadData(AvatarData* owningAvatar) :
 {
     _userProceduralAnimationFlags.assign((size_t)ProceduralAnimaitonTypeCount, true);
     _suppressProceduralAnimationFlags.assign((size_t)ProceduralAnimaitonTypeCount, false);
-    computeBlendshapesLookupMap();
 }
 
 glm::quat HeadData::getRawOrientation() const {
@@ -72,12 +71,6 @@ void HeadData::setOrientation(const glm::quat& orientation) {
     setHeadOrientation(orientation);
 }
 
-void HeadData::computeBlendshapesLookupMap(){
-    for (int i = 0; i < (int)Blendshapes::BlendshapeCount; i++) {
-        _blendshapeLookupMap[FACESHIFT_BLENDSHAPES[i]] = i;
-    }
-}
-
 int HeadData::getNumSummedBlendshapeCoefficients() const {
     int maxSize = std::max(_blendshapeCoefficients.size(), _transientBlendshapeCoefficients.size());
     return maxSize;
@@ -109,8 +102,8 @@ const QVector<float>& HeadData::getSummedBlendshapeCoefficients() {
 void HeadData::setBlendshape(QString name, float val) {
 
     // Check to see if the named blendshape exists, and then set its value if it does
-    auto it = _blendshapeLookupMap.find(name);
+    auto it = BLENDSHAPE_LOOKUP_MAP.find(name);
-    if (it != _blendshapeLookupMap.end()) {
+    if (it != BLENDSHAPE_LOOKUP_MAP.end()) {
         if (_blendshapeCoefficients.size() <= it.value()) {
             _blendshapeCoefficients.resize(it.value() + 1);
         }
@@ -135,8 +128,8 @@ void HeadData::setBlendshape(QString name, float val) {
 }
 
 int HeadData::getBlendshapeIndex(const QString& name) {
-    auto it = _blendshapeLookupMap.find(name);
+    auto it = BLENDSHAPE_LOOKUP_MAP.find(name);
-    int index = it != _blendshapeLookupMap.end() ? it.value() : -1;
+    int index = it != BLENDSHAPE_LOOKUP_MAP.end() ? it.value() : -1;
     return index;
 }
 
@@ -155,8 +148,8 @@ static const QString JSON_AVATAR_HEAD_LOOKAT = QStringLiteral("lookAt");
 QJsonObject HeadData::toJson() const {
     QJsonObject headJson;
     QJsonObject blendshapesJson;
-    for (auto name : _blendshapeLookupMap.keys()) {
+    for (auto name : BLENDSHAPE_LOOKUP_MAP.keys()) {
-        auto index = _blendshapeLookupMap[name];
+        auto index = BLENDSHAPE_LOOKUP_MAP[name];
         float value = 0.0f;
         if (index < _blendshapeCoefficients.size()) {
             value += _blendshapeCoefficients[index];

libraries/avatars/src/HeadData.h

@@ -125,7 +125,6 @@ protected:
     QVector<float> _blendshapeCoefficients;
     QVector<float> _transientBlendshapeCoefficients;
     QVector<float> _summedBlendshapeCoefficients;
-    QMap<QString, int> _blendshapeLookupMap;
     AvatarData* _owningAvatar;
 
 private:
@@ -134,7 +133,6 @@ private:
     HeadData& operator= (const HeadData&);
 
     void setHeadOrientation(const glm::quat& orientation);
-    void computeBlendshapesLookupMap();
 };
 
 #endif // hifi_HeadData_h

libraries/entities-renderer/src/RenderableModelEntityItem.cpp

@@ -1023,6 +1023,11 @@ uint32_t ModelEntityRenderer::metaFetchMetaSubItems(ItemIDs& subItems) const {
     return 0;
 }
 
+void ModelEntityRenderer::handleBlendedVertices(int blendshapeNumber, const QVector<BlendshapeOffset>& blendshapeOffsets,
+                                                const QVector<int>& blendedMeshSizes, const render::ItemIDs& subItemIDs) {
+    setBlendedVertices(blendshapeNumber, blendshapeOffsets, blendedMeshSizes, subItemIDs);
+}
+
 void ModelEntityRenderer::removeFromScene(const ScenePointer& scene, Transaction& transaction) {
     if (_model) {
         _model->removeFromScene(scene, transaction);
@@ -1191,7 +1196,11 @@ bool ModelEntityRenderer::needsRenderUpdateFromTypedEntity(const TypedEntityPoin
     if (model && model->isLoaded()) {
         if (!entity->_dimensionsInitialized || entity->_needsInitialSimulation || !entity->_originalTexturesRead) {
             return true;
-       } 
+        }
+
+        if (entity->blendshapesChanged()) {
+            return true;
+        }
 
         // Check to see if we need to update the model bounds
         if (entity->needsUpdateModelBounds()) {
@@ -1350,6 +1359,11 @@ void ModelEntityRenderer::doRenderUpdateSynchronousTyped(const ScenePointer& sce
         model->setTagMask(tagMask, scene);
     }
 
+    if (entity->blendshapesChanged()) {
+        model->setBlendshapeCoefficients(entity->getBlendshapeCoefficientVector());
+        model->updateBlendshapes();
+    }
+
     // TODO? early exit here when not visible?
 
     if (model->canCastShadow() != _canCastShadow) {
@@ -1370,7 +1384,8 @@ void ModelEntityRenderer::doRenderUpdateSynchronousTyped(const ScenePointer& sce
             model->removeFromScene(scene, transaction);
             render::Item::Status::Getters statusGetters;
             makeStatusGetters(entity, statusGetters);
-            model->addToScene(scene, transaction, statusGetters);
+            using namespace std::placeholders;
+            model->addToScene(scene, transaction, statusGetters, std::bind(&ModelEntityRenderer::metaBlendshapeOperator, _renderItemID, _1, _2, _3, _4));
             entity->bumpAncestorChainRenderableVersion();
             processMaterials();
         }
@@ -1529,3 +1544,12 @@ void ModelEntityRenderer::processMaterials() {
         }
     }
 }
+
+void ModelEntityRenderer::metaBlendshapeOperator(render::ItemID renderItemID, int blendshapeNumber, const QVector<BlendshapeOffset>& blendshapeOffsets,
+                                                 const QVector<int>& blendedMeshSizes, const render::ItemIDs& subItemIDs) {
+    render::Transaction transaction;
+    transaction.updateItem<PayloadProxyInterface>(renderItemID, [blendshapeNumber, blendshapeOffsets, blendedMeshSizes, subItemIDs](PayloadProxyInterface& self) {
+        self.handleBlendedVertices(blendshapeNumber, blendshapeOffsets, blendedMeshSizes, subItemIDs);
+    });
+    AbstractViewStateInterface::instance()->getMain3DScene()->enqueueTransaction(transaction);
+}

libraries/entities-renderer/src/RenderableModelEntityItem.h

@@ -21,6 +21,7 @@
 #include <AnimationCache.h>
 #include <Model.h>
 #include <model-networking/ModelCache.h>
+#include <MetaModelPayload.h>
 
 #include "RenderableEntityItem.h"
 
@@ -131,7 +132,7 @@ private:
 
 namespace render { namespace entities { 
 
-class ModelEntityRenderer : public TypedEntityRenderer<RenderableModelEntityItem> {
+class ModelEntityRenderer : public TypedEntityRenderer<RenderableModelEntityItem>, public MetaModelPayload {
     using Parent = TypedEntityRenderer<RenderableModelEntityItem>;
     friend class EntityRenderer;
     Q_OBJECT
@@ -155,6 +156,8 @@ protected:
     void setKey(bool didVisualGeometryRequestSucceed);
     virtual ItemKey getKey() override;
     virtual uint32_t metaFetchMetaSubItems(ItemIDs& subItems) const override;
+    virtual void handleBlendedVertices(int blendshapeNumber, const QVector<BlendshapeOffset>& blendshapeOffsets,
+                                       const QVector<int>& blendedMeshSizes, const render::ItemIDs& subItemIDs) override;
 
     virtual bool needsRenderUpdateFromTypedEntity(const TypedEntityPointer& entity) const override;
     virtual bool needsRenderUpdate() const override;
@@ -199,6 +202,10 @@ private:
     bool _prevModelLoaded { false };
 
     void processMaterials();
+
+    static void metaBlendshapeOperator(render::ItemID renderItemID, int blendshapeNumber, const QVector<BlendshapeOffset>& blendshapeOffsets,
+                                       const QVector<int>& blendedMeshSizes, const render::ItemIDs& subItemIDs);
+
 };
 
 } } // namespace 

libraries/entities/src/EntityItemProperties.cpp

@@ -536,6 +536,7 @@ EntityPropertyFlags EntityItemProperties::getChangedProperties() const {
     CHECK_PROPERTY_CHANGE(PROP_JOINT_TRANSLATIONS, jointTranslations);
     CHECK_PROPERTY_CHANGE(PROP_RELAY_PARENT_JOINTS, relayParentJoints);
     CHECK_PROPERTY_CHANGE(PROP_GROUP_CULLED, groupCulled);
+    CHECK_PROPERTY_CHANGE(PROP_BLENDSHAPE_COEFFICIENTS, blendshapeCoefficients);
     changedProperties += _animation.getChangedProperties();
 
     // Light
@@ -996,6 +997,9 @@ EntityPropertyFlags EntityItemProperties::getChangedProperties() const {
  *     compressed in GZ format, in which case the URL ends in ".gz".
  * @property {Vec3} modelScale - The scale factor applied to the model's dimensions.
  *     <p class="important">Deprecated: This property is deprecated and will be removed.</p>
+ * @property {string} blendshapeCoefficients - A JSON string of a map of blendshape names to values.  Only stores set values.
+ *     When editing this property, only coefficients that you are editing will change; it will not explicitly reset other
+ *     coefficients.
  * @property {string} textures="" - A JSON string of texture name, URL pairs used when rendering the model in place of the
  *     model's original textures. Use a texture name from the <code>originalTextures</code> property to override that texture. 
  *     Only the texture names and URLs to be overridden need be specified; original textures are used where there are no 
@@ -1723,6 +1727,7 @@ QScriptValue EntityItemProperties::copyToScriptValue(QScriptEngine* engine, bool
         COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_JOINT_TRANSLATIONS, jointTranslations);
         COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_RELAY_PARENT_JOINTS, relayParentJoints);
         COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_GROUP_CULLED, groupCulled);
+        COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_BLENDSHAPE_COEFFICIENTS, blendshapeCoefficients);
         if (!psuedoPropertyFlagsButDesiredEmpty) {
             _animation.copyToScriptValue(_desiredProperties, properties, engine, skipDefaults, defaultEntityProperties);
         }
@@ -2130,6 +2135,7 @@ void EntityItemProperties::copyFromScriptValue(const QScriptValue& object, bool
     COPY_PROPERTY_FROM_QSCRIPTVALUE(jointTranslations, qVectorVec3, setJointTranslations);
     COPY_PROPERTY_FROM_QSCRIPTVALUE(relayParentJoints, bool, setRelayParentJoints);
     COPY_PROPERTY_FROM_QSCRIPTVALUE(groupCulled, bool, setGroupCulled);
+    COPY_PROPERTY_FROM_QSCRIPTVALUE(blendshapeCoefficients, QString, setBlendshapeCoefficients);
     _animation.copyFromScriptValue(object, _defaultSettings);
 
     // Light
@@ -2420,6 +2426,7 @@ void EntityItemProperties::merge(const EntityItemProperties& other) {
     COPY_PROPERTY_IF_CHANGED(jointTranslations);
     COPY_PROPERTY_IF_CHANGED(relayParentJoints);
     COPY_PROPERTY_IF_CHANGED(groupCulled);
+    COPY_PROPERTY_IF_CHANGED(blendshapeCoefficients);
     _animation.merge(other._animation);
 
     // Light
@@ -2774,6 +2781,7 @@ bool EntityItemProperties::getPropertyInfo(const QString& propertyName, EntityPr
         ADD_PROPERTY_TO_MAP(PROP_JOINT_TRANSLATIONS, JointTranslations, jointTranslations, QVector<vec3>);
         ADD_PROPERTY_TO_MAP(PROP_RELAY_PARENT_JOINTS, RelayParentJoints, relayParentJoints, bool);
         ADD_PROPERTY_TO_MAP(PROP_GROUP_CULLED, GroupCulled, groupCulled, bool);
+        ADD_PROPERTY_TO_MAP(PROP_BLENDSHAPE_COEFFICIENTS, BlendshapeCoefficients, blendshapeCoefficients, QString);
         { // Animation
             ADD_GROUP_PROPERTY_TO_MAP(PROP_ANIMATION_URL, Animation, animation, URL, url);
             ADD_GROUP_PROPERTY_TO_MAP(PROP_ANIMATION_ALLOW_TRANSLATION, Animation, animation, AllowTranslation, allowTranslation);
@@ -3213,6 +3221,7 @@ OctreeElement::AppendState EntityItemProperties::encodeEntityEditPacket(PacketTy
                 APPEND_ENTITY_PROPERTY(PROP_JOINT_TRANSLATIONS, properties.getJointTranslations());
                 APPEND_ENTITY_PROPERTY(PROP_RELAY_PARENT_JOINTS, properties.getRelayParentJoints());
                 APPEND_ENTITY_PROPERTY(PROP_GROUP_CULLED, properties.getGroupCulled());
+                APPEND_ENTITY_PROPERTY(PROP_BLENDSHAPE_COEFFICIENTS, properties.getBlendshapeCoefficients());
 
                 _staticAnimation.setProperties(properties);
                 _staticAnimation.appendToEditPacket(packetData, requestedProperties, propertyFlags, propertiesDidntFit, propertyCount, appendState);
@@ -3700,6 +3709,7 @@ bool EntityItemProperties::decodeEntityEditPacket(const unsigned char* data, int
         READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_JOINT_TRANSLATIONS, QVector<vec3>, setJointTranslations);
         READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_RELAY_PARENT_JOINTS, bool, setRelayParentJoints);
         READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_GROUP_CULLED, bool, setGroupCulled);
+        READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_BLENDSHAPE_COEFFICIENTS, QString, setBlendshapeCoefficients);
 
         properties.getAnimation().decodeFromEditPacket(propertyFlags, dataAt, processedBytes);
     }
@@ -4107,6 +4117,7 @@ void EntityItemProperties::markAllChanged() {
     _jointTranslationsChanged = true;
     _relayParentJointsChanged = true;
     _groupCulledChanged = true;
+    _blendshapeCoefficientsChanged = true;
     _animation.markAllChanged();
 
     // Light
@@ -4675,6 +4686,9 @@ QList<QString> EntityItemProperties::listChangedProperties() {
     if (groupCulledChanged()) {
         out += "groupCulled";
     }
+    if (blendshapeCoefficientsChanged()) {
+        out += "blendshapeCoefficients";
+    }
     getAnimation().listChangedProperties(out);
 
     // Light

libraries/entities/src/EntityItemProperties.h

@@ -300,6 +300,7 @@ public:
     DEFINE_PROPERTY_REF(PROP_JOINT_TRANSLATIONS, JointTranslations, jointTranslations, QVector<glm::vec3>, ENTITY_ITEM_DEFAULT_EMPTY_VEC3_QVEC);
     DEFINE_PROPERTY(PROP_RELAY_PARENT_JOINTS, RelayParentJoints, relayParentJoints, bool, ENTITY_ITEM_DEFAULT_RELAY_PARENT_JOINTS);
     DEFINE_PROPERTY_REF(PROP_GROUP_CULLED, GroupCulled, groupCulled, bool, false);
+    DEFINE_PROPERTY_REF(PROP_BLENDSHAPE_COEFFICIENTS, BlendshapeCoefficients, blendshapeCoefficients, QString, "");
     DEFINE_PROPERTY_GROUP(Animation, animation, AnimationPropertyGroup);
 
     // Light

libraries/entities/src/EntityPropertyFlags.h

@@ -216,16 +216,17 @@ enum EntityPropertyList {
     PROP_JOINT_TRANSLATIONS = PROP_DERIVED_5,
     PROP_RELAY_PARENT_JOINTS = PROP_DERIVED_6,
     PROP_GROUP_CULLED = PROP_DERIVED_7,
+    PROP_BLENDSHAPE_COEFFICIENTS = PROP_DERIVED_8,
     // Animation
-    PROP_ANIMATION_URL = PROP_DERIVED_8,
+    PROP_ANIMATION_URL = PROP_DERIVED_9,
-    PROP_ANIMATION_ALLOW_TRANSLATION = PROP_DERIVED_9,
+    PROP_ANIMATION_ALLOW_TRANSLATION = PROP_DERIVED_10,
-    PROP_ANIMATION_FPS = PROP_DERIVED_10,
+    PROP_ANIMATION_FPS = PROP_DERIVED_11,
-    PROP_ANIMATION_FRAME_INDEX = PROP_DERIVED_11,
+    PROP_ANIMATION_FRAME_INDEX = PROP_DERIVED_12,
-    PROP_ANIMATION_PLAYING = PROP_DERIVED_12,
+    PROP_ANIMATION_PLAYING = PROP_DERIVED_13,
-    PROP_ANIMATION_LOOP = PROP_DERIVED_13,
+    PROP_ANIMATION_LOOP = PROP_DERIVED_14,
-    PROP_ANIMATION_FIRST_FRAME = PROP_DERIVED_14,
+    PROP_ANIMATION_FIRST_FRAME = PROP_DERIVED_15,
-    PROP_ANIMATION_LAST_FRAME = PROP_DERIVED_15,
+    PROP_ANIMATION_LAST_FRAME = PROP_DERIVED_16,
-    PROP_ANIMATION_HOLD = PROP_DERIVED_16,
+    PROP_ANIMATION_HOLD = PROP_DERIVED_17,
 
     // Light
     PROP_IS_SPOTLIGHT = PROP_DERIVED_0,

libraries/entities/src/ModelEntityItem.cpp

@@ -33,7 +33,8 @@ EntityItemPointer ModelEntityItem::factory(const EntityItemID& entityID, const E
     return entity;
 }
 
-ModelEntityItem::ModelEntityItem(const EntityItemID& entityItemID) : EntityItem(entityItemID)
+ModelEntityItem::ModelEntityItem(const EntityItemID& entityItemID) : EntityItem(entityItemID),
+    _blendshapeCoefficientsVector((int)Blendshapes::BlendshapeCount, 0.0f)
 {
     _lastAnimated = usecTimestampNow();
     // set the last animated when interface (re)starts
@@ -71,6 +72,7 @@ EntityItemProperties ModelEntityItem::getProperties(const EntityPropertyFlags& d
     COPY_ENTITY_PROPERTY_TO_PROPERTIES(jointTranslations, getJointTranslations);
     COPY_ENTITY_PROPERTY_TO_PROPERTIES(relayParentJoints, getRelayParentJoints);
     COPY_ENTITY_PROPERTY_TO_PROPERTIES(groupCulled, getGroupCulled);
+    COPY_ENTITY_PROPERTY_TO_PROPERTIES(blendshapeCoefficients, getBlendshapeCoefficients);
     withReadLock([&] {
         _animationProperties.getProperties(properties);
     });
@@ -94,6 +96,7 @@ bool ModelEntityItem::setProperties(const EntityItemProperties& properties) {
     SET_ENTITY_PROPERTY_FROM_PROPERTIES(jointTranslations, setJointTranslations);
     SET_ENTITY_PROPERTY_FROM_PROPERTIES(relayParentJoints, setRelayParentJoints);
     SET_ENTITY_PROPERTY_FROM_PROPERTIES(groupCulled, setGroupCulled);
+    SET_ENTITY_PROPERTY_FROM_PROPERTIES(blendshapeCoefficients, setBlendshapeCoefficients);
 
     withWriteLock([&] {
         AnimationPropertyGroup animationProperties = _animationProperties;
@@ -138,6 +141,7 @@ int ModelEntityItem::readEntitySubclassDataFromBuffer(const unsigned char* data,
     READ_ENTITY_PROPERTY(PROP_JOINT_TRANSLATIONS, QVector<glm::vec3>, setJointTranslations);
     READ_ENTITY_PROPERTY(PROP_RELAY_PARENT_JOINTS, bool, setRelayParentJoints);
     READ_ENTITY_PROPERTY(PROP_GROUP_CULLED, bool, setGroupCulled);
+    READ_ENTITY_PROPERTY(PROP_BLENDSHAPE_COEFFICIENTS, QString, setBlendshapeCoefficients);
 
     // grab a local copy of _animationProperties to avoid multiple locks
     int bytesFromAnimation;
@@ -176,6 +180,7 @@ EntityPropertyFlags ModelEntityItem::getEntityProperties(EncodeBitstreamParams&
     requestedProperties += PROP_JOINT_TRANSLATIONS;
     requestedProperties += PROP_RELAY_PARENT_JOINTS;
     requestedProperties += PROP_GROUP_CULLED;
+    requestedProperties += PROP_BLENDSHAPE_COEFFICIENTS;
     requestedProperties += _animationProperties.getEntityProperties(params);
 
     return requestedProperties;
@@ -204,6 +209,7 @@ void ModelEntityItem::appendSubclassData(OctreePacketData* packetData, EncodeBit
     APPEND_ENTITY_PROPERTY(PROP_JOINT_TRANSLATIONS, getJointTranslations());
     APPEND_ENTITY_PROPERTY(PROP_RELAY_PARENT_JOINTS, getRelayParentJoints());
     APPEND_ENTITY_PROPERTY(PROP_GROUP_CULLED, getGroupCulled());
+    APPEND_ENTITY_PROPERTY(PROP_BLENDSHAPE_COEFFICIENTS, getBlendshapeCoefficients());
 
     withReadLock([&] {
         _animationProperties.appendSubclassData(packetData, params, entityTreeElementExtraEncodeData, requestedProperties,
@@ -256,6 +262,7 @@ void ModelEntityItem::debugDump() const {
     qCDebug(entities) << "    dimensions:" << getScaledDimensions();
     qCDebug(entities) << "    model URL:" << getModelURL();
     qCDebug(entities) << "    compound shape URL:" << getCompoundShapeURL();
+    qCDebug(entities) << "    blendshapeCoefficients:" << getBlendshapeCoefficients();
 }
 
 void ModelEntityItem::setShapeType(ShapeType type) {
@@ -743,3 +750,39 @@ void ModelEntityItem::setModelScale(const glm::vec3& modelScale) {
         _modelScale = modelScale;
     });
 }
+
+QString ModelEntityItem::getBlendshapeCoefficients() const {
+    return resultWithReadLock<QString>([&] {
+        return QJsonDocument::fromVariant(_blendshapeCoefficientsMap).toJson();
+    });
+}
+
+void ModelEntityItem::setBlendshapeCoefficients(const QString& blendshapeCoefficients) {
+    QJsonParseError error;
+    QJsonDocument newCoefficientsJSON = QJsonDocument::fromJson(blendshapeCoefficients.toUtf8(), &error);
+    if (error.error != QJsonParseError::NoError) {
+        qWarning() << "Could not evaluate blendshapeCoefficients property value:" << newCoefficientsJSON;
+        return;
+    }
+
+    QVariantMap newCoefficientsMap = newCoefficientsJSON.toVariant().toMap();
+    withWriteLock([&] {
+        for (auto& blendshape : newCoefficientsMap.keys()) {
+            auto newCoefficient = newCoefficientsMap[blendshape];
+            auto blendshapeIter = BLENDSHAPE_LOOKUP_MAP.find(blendshape);
+            if (newCoefficient.canConvert<float>() && blendshapeIter != BLENDSHAPE_LOOKUP_MAP.end()) {
+                float newCoefficientValue = newCoefficient.toFloat();
+                _blendshapeCoefficientsVector[blendshapeIter.value()] = newCoefficientValue;
+                _blendshapeCoefficientsMap[blendshape] = newCoefficientValue;
+                _blendshapesChanged = true;
+            }
+        }
+    });
+}
+
+QVector<float> ModelEntityItem::getBlendshapeCoefficientVector() {
+    return resultWithReadLock<QVector<float>>([&] {
+        _blendshapesChanged = false; // ok to change this within read lock here
+        return _blendshapeCoefficientsVector;
+    });
+}

libraries/entities/src/ModelEntityItem.h

@@ -17,6 +17,7 @@
 #include <ThreadSafeValueCache.h>
 #include "AnimationPropertyGroup.h"
 
+#include "BlendshapeConstants.h"
 
 class ModelEntityItem : public EntityItem {
 public:
@@ -133,6 +134,11 @@ public:
     glm::vec3 getModelScale() const;
     void setModelScale(const glm::vec3& modelScale);
 
+    QString getBlendshapeCoefficients() const;
+    void setBlendshapeCoefficients(const QString& blendshapeCoefficients);
+    bool blendshapesChanged() const { return _blendshapesChanged; }
+    QVector<float> getBlendshapeCoefficientVector();
+
 private:
     void setAnimationSettings(const QString& value); // only called for old bitstream format
     bool applyNewAnimationProperties(AnimationPropertyGroup newProperties);
@@ -166,6 +172,7 @@ protected:
     QString _modelURL;
     bool _relayParentJoints;
     bool _groupCulled { false };
+    QVariantMap _blendshapeCoefficientsMap;
 
     ThreadSafeValueCache<QString> _compoundShapeURL;
 
@@ -178,6 +185,9 @@ protected:
 private:
     uint64_t _lastAnimated{ 0 };
     float _currentFrame{ -1.0f };
+
+    QVector<float> _blendshapeCoefficientsVector;
+    bool _blendshapesChanged { false };
 };
 
 #endif // hifi_ModelEntityItem_h

libraries/networking/src/udt/PacketHeaders.h

@@ -280,6 +280,7 @@ enum class EntityVersion : PacketVersion {
     ScriptServerKinematicMotion,
     ScreenshareZone,
     ZoneOcclusion,
+    ModelBlendshapes,
 
     // Add new versions above here
     NUM_PACKET_TYPE,

libraries/render-utils/src/CauterizedModel.cpp

@@ -150,12 +150,7 @@ void CauterizedModel::updateClusterMatrices() {
         }
     }
 
-    // post the blender if we're not currently waiting for one to finish
+    updateBlendshapes();
-    auto modelBlender = DependencyManager::get<ModelBlender>();
-    if (modelBlender->shouldComputeBlendshapes() && getHFMModel().hasBlendedMeshes() && _blendshapeCoefficients != _blendedBlendshapeCoefficients) {
-        _blendedBlendshapeCoefficients = _blendshapeCoefficients;
-        modelBlender->noteRequiresBlend(getThisPointer());
-    }
 }
 
 void CauterizedModel::updateRenderItems() {

libraries/render-utils/src/MeshPartPayload.cpp

@@ -461,6 +461,7 @@ void ModelMeshPartPayload::setShapeKey(bool invalidateShapeKey, PrimitiveMode pr
         }
     }
 
+    _prevUseDualQuaternionSkinning = useDualQuaternionSkinning;
     _shapeKey = builder.build();
 }
 
@@ -538,6 +539,14 @@ void ModelMeshPartPayload::setBlendshapeBuffer(const std::unordered_map<int, gpu
         auto blendshapeBuffer = blendshapeBuffers.find(_meshIndex);
         if (blendshapeBuffer != blendshapeBuffers.end()) {
             _meshBlendshapeBuffer = blendshapeBuffer->second;
+            if (_isSkinned || (_isBlendShaped && _meshBlendshapeBuffer)) {
+                ShapeKey::Builder builder(_shapeKey);
+                builder.withDeformed();
+                if (_prevUseDualQuaternionSkinning) {
+                    builder.withDualQuatSkinned();
+                }
+                _shapeKey = builder.build();
+            }
         }
     }
 }

libraries/render-utils/src/MeshPartPayload.h

@@ -143,6 +143,7 @@ private:
     gpu::BufferPointer _meshBlendshapeBuffer;
     int _meshNumVertices;
     render::ShapeKey _shapeKey { render::ShapeKey::Builder::invalid() };
+    bool _prevUseDualQuaternionSkinning { false };
     bool _cauterized { false };
 
 };

libraries/render-utils/src/Model.cpp

@@ -1449,6 +1449,10 @@ void Model::updateClusterMatrices() {
         }
     }
 
+    updateBlendshapes();
+}
+
+void Model::updateBlendshapes() {
     // post the blender if we're not currently waiting for one to finish
     auto modelBlender = DependencyManager::get<ModelBlender>();
     if (modelBlender->shouldComputeBlendshapes() && getHFMModel().hasBlendedMeshes() && _blendshapeCoefficients != _blendedBlendshapeCoefficients) {

libraries/render-utils/src/Model.h

@@ -76,17 +76,6 @@ struct SortedTriangleSet {
     int subMeshIndex;
 };
 
-struct BlendshapeOffsetPacked {
-    glm::uvec4 packedPosNorTan;
-};
-
-struct BlendshapeOffsetUnpacked {
-    glm::vec3 positionOffset;
-    glm::vec3 normalOffset;
-    glm::vec3 tangentOffset;
-};
-
-using BlendshapeOffset = BlendshapeOffsetPacked;
 using BlendShapeOperator = std::function<void(int, const QVector<BlendshapeOffset>&, const QVector<int>&, const render::ItemIDs&)>;
 
 /// A generic 3D model displaying geometry loaded from a URL.
@@ -180,6 +169,7 @@ public:
 
     virtual void simulate(float deltaTime, bool fullUpdate = true);
     virtual void updateClusterMatrices();
+    virtual void updateBlendshapes();
 
     /// Returns a reference to the shared geometry.
     const NetworkModel::Pointer& getNetworkModel() const { return _renderGeometry; }
@@ -378,6 +368,8 @@ public:
     void addMaterial(graphics::MaterialLayer material, const std::string& parentMaterialName);
     void removeMaterial(graphics::MaterialPointer material, const std::string& parentMaterialName);
 
+    void setBlendshapeCoefficients(const QVector<float>& coefficients) { _blendshapeCoefficients = coefficients; }
+
 public slots:
     void loadURLFinished(bool success);
 
@@ -395,7 +387,6 @@ protected:
     std::mutex _materialMappingMutex;
     void applyMaterialMapping();
 
-    void setBlendshapeCoefficients(const QVector<float>& coefficients) { _blendshapeCoefficients = coefficients; }
     const QVector<float>& getBlendshapeCoefficients() const { return _blendshapeCoefficients; }
 
     /// Clear the joint states

libraries/render-utils/src/SoftAttachmentModel.cpp

@@ -68,10 +68,5 @@ void SoftAttachmentModel::updateClusterMatrices() {
         }
     }
 
-    // post the blender if we're not currently waiting for one to finish
+    updateBlendshapes();
-    auto modelBlender = DependencyManager::get<ModelBlender>();
-    if (modelBlender->shouldComputeBlendshapes() && getHFMModel().hasBlendedMeshes() && _blendshapeCoefficients != _blendedBlendshapeCoefficients) {
-        _blendedBlendshapeCoefficients = _blendshapeCoefficients;
-        modelBlender->noteRequiresBlend(getThisPointer());
-    }
 }

libraries/render/src/render/Item.h

@@ -28,6 +28,8 @@
 #include <graphics/Material.h>
 #include "ShapePipeline.h"
 
+#include "BlendshapeConstants.h"
+
 namespace render {
 
 typedef int32_t Index;
@@ -626,6 +628,10 @@ public:
     virtual void render(RenderArgs* args) = 0;
     virtual uint32_t metaFetchMetaSubItems(ItemIDs& subItems) const = 0;
     virtual bool passesZoneOcclusionTest(const std::unordered_set<QUuid>& containingZones) const = 0;
+
+    // FIXME: this isn't the best place for this since it's only used for ModelEntities, but currently all Entities use PayloadProxyInterface
+    virtual void handleBlendedVertices(int blendshapeNumber, const QVector<BlendshapeOffset>& blendshapeOffsets,
+                                       const QVector<int>& blendedMeshSizes, const render::ItemIDs& subItemIDs) {};
 };
 
 template <> const ItemKey payloadGetKey(const PayloadProxyInterface::Pointer& payload);

libraries/shared/src/BlendshapeConstants.cpp

@@ -76,3 +76,11 @@ const char* FACESHIFT_BLENDSHAPES[] = {
     "UserBlendshape9",
     ""
 };
+
+const QMap<QString, int> BLENDSHAPE_LOOKUP_MAP = [] {
+    QMap<QString, int> toReturn;
+    for (int i = 0; i < (int)Blendshapes::BlendshapeCount; i++) {
+        toReturn[FACESHIFT_BLENDSHAPES[i]] = i;
+    }
+    return toReturn;
+}();

libraries/shared/src/BlendshapeConstants.h

@@ -12,8 +12,14 @@
 #ifndef hifi_BlendshapeConstants_h
 #define hifi_BlendshapeConstants_h
 
+#include <QMap>
+#include <QString>
+
+#include <glm/glm.hpp>
+
 /// The names of the blendshapes expected by Faceshift, terminated with an empty string.
 extern const char* FACESHIFT_BLENDSHAPES[];
+extern const QMap<QString, int> BLENDSHAPE_LOOKUP_MAP;
 
 enum class Blendshapes : int {
     EyeBlink_L = 0,
@@ -115,5 +121,16 @@ enum class LegacyBlendshpaes : int {
 // * LipsUpperOpen (not in ARKit)
 // * LipsLowerOpen (not in ARKit)
 
+struct BlendshapeOffsetPacked {
+    glm::uvec4 packedPosNorTan;
+};
+
+struct BlendshapeOffsetUnpacked {
+    glm::vec3 positionOffset;
+    glm::vec3 normalOffset;
+    glm::vec3 tangentOffset;
+};
+
+using BlendshapeOffset = BlendshapeOffsetPacked;
 
 #endif // hifi_BlendshapeConstants_h

plugins/KasenAPIExample/src/KasenAPIExample.cpp

@@ -113,7 +113,7 @@ private:
     }
     struct _HeadHelper : public HeadData {
         QMap<QString,int> getBlendshapeMap() const {
-            return _blendshapeLookupMap;
+            return BLENDSHAPE_LOOKUP_MAP;
         }
         struct States { QVector<float> base, summed, transient; };
         States getBlendshapeStates() const {