diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 5e8d5c457f..f19ee3de86 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -34,7 +34,6 @@ HeadData::HeadData(AvatarData* owningAvatar) : { _userProceduralAnimationFlags.assign((size_t)ProceduralAnimaitonTypeCount, true); _suppressProceduralAnimationFlags.assign((size_t)ProceduralAnimaitonTypeCount, false); - computeBlendshapesLookupMap(); } glm::quat HeadData::getRawOrientation() const { @@ -72,12 +71,6 @@ void HeadData::setOrientation(const glm::quat& orientation) { setHeadOrientation(orientation); } -void HeadData::computeBlendshapesLookupMap(){ - for (int i = 0; i < (int)Blendshapes::BlendshapeCount; i++) { - _blendshapeLookupMap[FACESHIFT_BLENDSHAPES[i]] = i; - } -} - int HeadData::getNumSummedBlendshapeCoefficients() const { int maxSize = std::max(_blendshapeCoefficients.size(), _transientBlendshapeCoefficients.size()); return maxSize; @@ -109,8 +102,8 @@ const QVector& HeadData::getSummedBlendshapeCoefficients() { void HeadData::setBlendshape(QString name, float val) { // Check to see if the named blendshape exists, and then set its value if it does - auto it = _blendshapeLookupMap.find(name); - if (it != _blendshapeLookupMap.end()) { + auto it = blendshapeLookupMap.find(name); + if (it != blendshapeLookupMap.end()) { if (_blendshapeCoefficients.size() <= it.value()) { _blendshapeCoefficients.resize(it.value() + 1); } @@ -135,8 +128,8 @@ void HeadData::setBlendshape(QString name, float val) { } int HeadData::getBlendshapeIndex(const QString& name) { - auto it = _blendshapeLookupMap.find(name); - int index = it != _blendshapeLookupMap.end() ? it.value() : -1; + auto it = blendshapeLookupMap.find(name); + int index = it != blendshapeLookupMap.end() ? it.value() : -1; return index; } @@ -155,8 +148,8 @@ static const QString JSON_AVATAR_HEAD_LOOKAT = QStringLiteral("lookAt"); QJsonObject HeadData::toJson() const { QJsonObject headJson; QJsonObject blendshapesJson; - for (auto name : _blendshapeLookupMap.keys()) { - auto index = _blendshapeLookupMap[name]; + for (auto name : blendshapeLookupMap.keys()) { + auto index = blendshapeLookupMap[name]; float value = 0.0f; if (index < _blendshapeCoefficients.size()) { value += _blendshapeCoefficients[index]; diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index 2fa91c0bed..af71fd883d 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -125,7 +125,6 @@ protected: QVector _blendshapeCoefficients; QVector _transientBlendshapeCoefficients; QVector _summedBlendshapeCoefficients; - QMap _blendshapeLookupMap; AvatarData* _owningAvatar; private: @@ -134,7 +133,6 @@ private: HeadData& operator= (const HeadData&); void setHeadOrientation(const glm::quat& orientation); - void computeBlendshapesLookupMap(); }; #endif // hifi_HeadData_h diff --git a/libraries/entities-renderer/src/RenderableModelEntityItem.cpp b/libraries/entities-renderer/src/RenderableModelEntityItem.cpp index 5fbbdfa0b8..40182a6965 100644 --- a/libraries/entities-renderer/src/RenderableModelEntityItem.cpp +++ b/libraries/entities-renderer/src/RenderableModelEntityItem.cpp @@ -1083,6 +1083,11 @@ uint32_t ModelEntityRenderer::metaFetchMetaSubItems(ItemIDs& subItems) const { return 0; } +void ModelEntityRenderer::handleBlendedVertices(int blendshapeNumber, const QVector& blendshapeOffsets, + const QVector& blendedMeshSizes, const render::ItemIDs& subItemIDs) { + setBlendedVertices(blendshapeNumber, blendshapeOffsets, blendedMeshSizes, subItemIDs); +} + void ModelEntityRenderer::removeFromScene(const ScenePointer& scene, Transaction& transaction) { if (_model) { _model->removeFromScene(scene, transaction); @@ -1251,7 +1256,11 @@ bool ModelEntityRenderer::needsRenderUpdateFromTypedEntity(const TypedEntityPoin if (model && model->isLoaded()) { if (!entity->_dimensionsInitialized || entity->_needsInitialSimulation || !entity->_originalTexturesRead) { return true; - } + } + + if (entity->blendshapesChanged()) { + return true; + } // Check to see if we need to update the model bounds if (entity->needsUpdateModelBounds()) { @@ -1407,6 +1416,11 @@ void ModelEntityRenderer::doRenderUpdateSynchronousTyped(const ScenePointer& sce model->setTagMask(tagMask, scene); } + if (entity->blendshapesChanged()) { + model->setBlendshapeCoefficients(entity->getBlendshapeCoefficientVector()); + model->updateBlendshapes(); + } + // TODO? early exit here when not visible? if (model->canCastShadow() != _canCastShadow) { @@ -1427,7 +1441,8 @@ void ModelEntityRenderer::doRenderUpdateSynchronousTyped(const ScenePointer& sce model->removeFromScene(scene, transaction); render::Item::Status::Getters statusGetters; makeStatusGetters(entity, statusGetters); - model->addToScene(scene, transaction, statusGetters); + using namespace std::placeholders; + model->addToScene(scene, transaction, statusGetters, std::bind(&ModelEntityRenderer::metaBlendshapeOperator, _renderItemID, _1, _2, _3, _4)); entity->bumpAncestorChainRenderableVersion(); processMaterials(); } @@ -1579,3 +1594,12 @@ void ModelEntityRenderer::processMaterials() { } } } + +void ModelEntityRenderer::metaBlendshapeOperator(render::ItemID renderItemID, int blendshapeNumber, const QVector& blendshapeOffsets, + const QVector& blendedMeshSizes, const render::ItemIDs& subItemIDs) { + render::Transaction transaction; + transaction.updateItem(renderItemID, [blendshapeNumber, blendshapeOffsets, blendedMeshSizes, subItemIDs](PayloadProxyInterface& self) { + self.handleBlendedVertices(blendshapeNumber, blendshapeOffsets, blendedMeshSizes, subItemIDs); + }); + AbstractViewStateInterface::instance()->getMain3DScene()->enqueueTransaction(transaction); +} \ No newline at end of file diff --git a/libraries/entities-renderer/src/RenderableModelEntityItem.h b/libraries/entities-renderer/src/RenderableModelEntityItem.h index 0119c7bc26..dfff088c24 100644 --- a/libraries/entities-renderer/src/RenderableModelEntityItem.h +++ b/libraries/entities-renderer/src/RenderableModelEntityItem.h @@ -21,6 +21,7 @@ #include #include #include +#include #include "RenderableEntityItem.h" @@ -131,7 +132,7 @@ private: namespace render { namespace entities { -class ModelEntityRenderer : public TypedEntityRenderer { +class ModelEntityRenderer : public TypedEntityRenderer, public MetaModelPayload { using Parent = TypedEntityRenderer; friend class EntityRenderer; Q_OBJECT @@ -155,6 +156,8 @@ protected: void setKey(bool didVisualGeometryRequestSucceed); virtual ItemKey getKey() override; virtual uint32_t metaFetchMetaSubItems(ItemIDs& subItems) const override; + virtual void handleBlendedVertices(int blendshapeNumber, const QVector& blendshapeOffsets, + const QVector& blendedMeshSizes, const render::ItemIDs& subItemIDs) override; virtual bool needsRenderUpdateFromTypedEntity(const TypedEntityPointer& entity) const override; virtual bool needsRenderUpdate() const override; @@ -198,6 +201,10 @@ private: bool _prevModelLoaded { false }; void processMaterials(); + + static void metaBlendshapeOperator(render::ItemID renderItemID, int blendshapeNumber, const QVector& blendshapeOffsets, + const QVector& blendedMeshSizes, const render::ItemIDs& subItemIDs); + }; } } // namespace diff --git a/libraries/entities/src/EntityItemProperties.cpp b/libraries/entities/src/EntityItemProperties.cpp index 2b8f2b4c14..bfb259c1ba 100644 --- a/libraries/entities/src/EntityItemProperties.cpp +++ b/libraries/entities/src/EntityItemProperties.cpp @@ -525,6 +525,7 @@ EntityPropertyFlags EntityItemProperties::getChangedProperties() const { CHECK_PROPERTY_CHANGE(PROP_JOINT_TRANSLATIONS, jointTranslations); CHECK_PROPERTY_CHANGE(PROP_RELAY_PARENT_JOINTS, relayParentJoints); CHECK_PROPERTY_CHANGE(PROP_GROUP_CULLED, groupCulled); + CHECK_PROPERTY_CHANGE(PROP_BLENDSHAPE_COEFFICIENTS, blendshapeCoefficients); changedProperties += _animation.getChangedProperties(); // Light @@ -980,6 +981,9 @@ EntityPropertyFlags EntityItemProperties::getChangedProperties() const { * compressed in GZ format, in which case the URL ends in ".gz". * @property {Vec3} modelScale - The scale factor applied to the model's dimensions. *

Deprecated: This property is deprecated and will be removed.

+ * @property {string} blendshapeCoefficients - A JSON string of a map of blendshape names to values. Only stores set values. + * When editing this property, only coefficients that you are editing will change; it will not explicitly reset other + * coefficients. * @property {string} textures="" - A JSON string of texture name, URL pairs used when rendering the model in place of the * model's original textures. Use a texture name from the originalTextures property to override that texture. * Only the texture names and URLs to be overridden need be specified; original textures are used where there are no @@ -1704,6 +1708,7 @@ QScriptValue EntityItemProperties::copyToScriptValue(QScriptEngine* engine, bool COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_JOINT_TRANSLATIONS, jointTranslations); COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_RELAY_PARENT_JOINTS, relayParentJoints); COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_GROUP_CULLED, groupCulled); + COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_BLENDSHAPE_COEFFICIENTS, blendshapeCoefficients); if (!psuedoPropertyFlagsButDesiredEmpty) { _animation.copyToScriptValue(_desiredProperties, properties, engine, skipDefaults, defaultEntityProperties); } @@ -2109,6 +2114,7 @@ void EntityItemProperties::copyFromScriptValue(const QScriptValue& object, bool COPY_PROPERTY_FROM_QSCRIPTVALUE(jointTranslations, qVectorVec3, setJointTranslations); COPY_PROPERTY_FROM_QSCRIPTVALUE(relayParentJoints, bool, setRelayParentJoints); COPY_PROPERTY_FROM_QSCRIPTVALUE(groupCulled, bool, setGroupCulled); + COPY_PROPERTY_FROM_QSCRIPTVALUE(blendshapeCoefficients, QString, setBlendshapeCoefficients); _animation.copyFromScriptValue(object, _defaultSettings); // Light @@ -2397,6 +2403,7 @@ void EntityItemProperties::merge(const EntityItemProperties& other) { COPY_PROPERTY_IF_CHANGED(jointTranslations); COPY_PROPERTY_IF_CHANGED(relayParentJoints); COPY_PROPERTY_IF_CHANGED(groupCulled); + COPY_PROPERTY_IF_CHANGED(blendshapeCoefficients); _animation.merge(other._animation); // Light @@ -2749,6 +2756,7 @@ bool EntityItemProperties::getPropertyInfo(const QString& propertyName, EntityPr ADD_PROPERTY_TO_MAP(PROP_JOINT_TRANSLATIONS, JointTranslations, jointTranslations, QVector); ADD_PROPERTY_TO_MAP(PROP_RELAY_PARENT_JOINTS, RelayParentJoints, relayParentJoints, bool); ADD_PROPERTY_TO_MAP(PROP_GROUP_CULLED, GroupCulled, groupCulled, bool); + ADD_PROPERTY_TO_MAP(PROP_BLENDSHAPE_COEFFICIENTS, BlendshapeCoefficients, blendshapeCoefficients, QString); { // Animation ADD_GROUP_PROPERTY_TO_MAP(PROP_ANIMATION_URL, Animation, animation, URL, url); ADD_GROUP_PROPERTY_TO_MAP(PROP_ANIMATION_ALLOW_TRANSLATION, Animation, animation, AllowTranslation, allowTranslation); @@ -3186,6 +3194,7 @@ OctreeElement::AppendState EntityItemProperties::encodeEntityEditPacket(PacketTy APPEND_ENTITY_PROPERTY(PROP_JOINT_TRANSLATIONS, properties.getJointTranslations()); APPEND_ENTITY_PROPERTY(PROP_RELAY_PARENT_JOINTS, properties.getRelayParentJoints()); APPEND_ENTITY_PROPERTY(PROP_GROUP_CULLED, properties.getGroupCulled()); + APPEND_ENTITY_PROPERTY(PROP_BLENDSHAPE_COEFFICIENTS, properties.getBlendshapeCoefficients()); _staticAnimation.setProperties(properties); _staticAnimation.appendToEditPacket(packetData, requestedProperties, propertyFlags, propertiesDidntFit, propertyCount, appendState); @@ -3671,6 +3680,7 @@ bool EntityItemProperties::decodeEntityEditPacket(const unsigned char* data, int READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_JOINT_TRANSLATIONS, QVector, setJointTranslations); READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_RELAY_PARENT_JOINTS, bool, setRelayParentJoints); READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_GROUP_CULLED, bool, setGroupCulled); + READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_BLENDSHAPE_COEFFICIENTS, QString, setBlendshapeCoefficients); properties.getAnimation().decodeFromEditPacket(propertyFlags, dataAt, processedBytes); } @@ -4076,6 +4086,7 @@ void EntityItemProperties::markAllChanged() { _jointTranslationsChanged = true; _relayParentJointsChanged = true; _groupCulledChanged = true; + _blendshapeCoefficientsChanged = true; _animation.markAllChanged(); // Light @@ -4640,6 +4651,9 @@ QList EntityItemProperties::listChangedProperties() { if (groupCulledChanged()) { out += "groupCulled"; } + if (blendshapeCoefficientsChanged()) { + out += "blendshapeCoefficients"; + } getAnimation().listChangedProperties(out); // Light diff --git a/libraries/entities/src/EntityItemProperties.h b/libraries/entities/src/EntityItemProperties.h index 63d8183899..e498f6f22a 100644 --- a/libraries/entities/src/EntityItemProperties.h +++ b/libraries/entities/src/EntityItemProperties.h @@ -296,6 +296,7 @@ public: DEFINE_PROPERTY_REF(PROP_JOINT_TRANSLATIONS, JointTranslations, jointTranslations, QVector, ENTITY_ITEM_DEFAULT_EMPTY_VEC3_QVEC); DEFINE_PROPERTY(PROP_RELAY_PARENT_JOINTS, RelayParentJoints, relayParentJoints, bool, ENTITY_ITEM_DEFAULT_RELAY_PARENT_JOINTS); DEFINE_PROPERTY_REF(PROP_GROUP_CULLED, GroupCulled, groupCulled, bool, false); + DEFINE_PROPERTY_REF(PROP_BLENDSHAPE_COEFFICIENTS, BlendshapeCoefficients, blendshapeCoefficients, QString, ""); DEFINE_PROPERTY_GROUP(Animation, animation, AnimationPropertyGroup); // Light diff --git a/libraries/entities/src/EntityPropertyFlags.h b/libraries/entities/src/EntityPropertyFlags.h index d5af337a7d..b355952656 100644 --- a/libraries/entities/src/EntityPropertyFlags.h +++ b/libraries/entities/src/EntityPropertyFlags.h @@ -214,16 +214,17 @@ enum EntityPropertyList { PROP_JOINT_TRANSLATIONS = PROP_DERIVED_5, PROP_RELAY_PARENT_JOINTS = PROP_DERIVED_6, PROP_GROUP_CULLED = PROP_DERIVED_7, + PROP_BLENDSHAPE_COEFFICIENTS = PROP_DERIVED_8, // Animation - PROP_ANIMATION_URL = PROP_DERIVED_8, - PROP_ANIMATION_ALLOW_TRANSLATION = PROP_DERIVED_9, - PROP_ANIMATION_FPS = PROP_DERIVED_10, - PROP_ANIMATION_FRAME_INDEX = PROP_DERIVED_11, - PROP_ANIMATION_PLAYING = PROP_DERIVED_12, - PROP_ANIMATION_LOOP = PROP_DERIVED_13, - PROP_ANIMATION_FIRST_FRAME = PROP_DERIVED_14, - PROP_ANIMATION_LAST_FRAME = PROP_DERIVED_15, - PROP_ANIMATION_HOLD = PROP_DERIVED_16, + PROP_ANIMATION_URL = PROP_DERIVED_9, + PROP_ANIMATION_ALLOW_TRANSLATION = PROP_DERIVED_10, + PROP_ANIMATION_FPS = PROP_DERIVED_11, + PROP_ANIMATION_FRAME_INDEX = PROP_DERIVED_12, + PROP_ANIMATION_PLAYING = PROP_DERIVED_13, + PROP_ANIMATION_LOOP = PROP_DERIVED_14, + PROP_ANIMATION_FIRST_FRAME = PROP_DERIVED_15, + PROP_ANIMATION_LAST_FRAME = PROP_DERIVED_16, + PROP_ANIMATION_HOLD = PROP_DERIVED_17, // Light PROP_IS_SPOTLIGHT = PROP_DERIVED_0, diff --git a/libraries/entities/src/ModelEntityItem.cpp b/libraries/entities/src/ModelEntityItem.cpp index 0b6e62e89e..dd9841b519 100644 --- a/libraries/entities/src/ModelEntityItem.cpp +++ b/libraries/entities/src/ModelEntityItem.cpp @@ -33,7 +33,8 @@ EntityItemPointer ModelEntityItem::factory(const EntityItemID& entityID, const E return entity; } -ModelEntityItem::ModelEntityItem(const EntityItemID& entityItemID) : EntityItem(entityItemID) +ModelEntityItem::ModelEntityItem(const EntityItemID& entityItemID) : EntityItem(entityItemID), + _blendshapeCoefficientsVector((int)Blendshapes::BlendshapeCount, 0.0f) { _lastAnimated = usecTimestampNow(); // set the last animated when interface (re)starts @@ -71,6 +72,7 @@ EntityItemProperties ModelEntityItem::getProperties(const EntityPropertyFlags& d COPY_ENTITY_PROPERTY_TO_PROPERTIES(jointTranslations, getJointTranslations); COPY_ENTITY_PROPERTY_TO_PROPERTIES(relayParentJoints, getRelayParentJoints); COPY_ENTITY_PROPERTY_TO_PROPERTIES(groupCulled, getGroupCulled); + COPY_ENTITY_PROPERTY_TO_PROPERTIES(blendshapeCoefficients, getBlendshapeCoefficients); withReadLock([&] { _animationProperties.getProperties(properties); }); @@ -94,6 +96,7 @@ bool ModelEntityItem::setProperties(const EntityItemProperties& properties) { SET_ENTITY_PROPERTY_FROM_PROPERTIES(jointTranslations, setJointTranslations); SET_ENTITY_PROPERTY_FROM_PROPERTIES(relayParentJoints, setRelayParentJoints); SET_ENTITY_PROPERTY_FROM_PROPERTIES(groupCulled, setGroupCulled); + SET_ENTITY_PROPERTY_FROM_PROPERTIES(blendshapeCoefficients, setBlendshapeCoefficients); withWriteLock([&] { AnimationPropertyGroup animationProperties = _animationProperties; @@ -138,6 +141,7 @@ int ModelEntityItem::readEntitySubclassDataFromBuffer(const unsigned char* data, READ_ENTITY_PROPERTY(PROP_JOINT_TRANSLATIONS, QVector, setJointTranslations); READ_ENTITY_PROPERTY(PROP_RELAY_PARENT_JOINTS, bool, setRelayParentJoints); READ_ENTITY_PROPERTY(PROP_GROUP_CULLED, bool, setGroupCulled); + READ_ENTITY_PROPERTY(PROP_BLENDSHAPE_COEFFICIENTS, QString, setBlendshapeCoefficients); // grab a local copy of _animationProperties to avoid multiple locks int bytesFromAnimation; @@ -176,6 +180,7 @@ EntityPropertyFlags ModelEntityItem::getEntityProperties(EncodeBitstreamParams& requestedProperties += PROP_JOINT_TRANSLATIONS; requestedProperties += PROP_RELAY_PARENT_JOINTS; requestedProperties += PROP_GROUP_CULLED; + requestedProperties += PROP_BLENDSHAPE_COEFFICIENTS; requestedProperties += _animationProperties.getEntityProperties(params); return requestedProperties; @@ -204,6 +209,7 @@ void ModelEntityItem::appendSubclassData(OctreePacketData* packetData, EncodeBit APPEND_ENTITY_PROPERTY(PROP_JOINT_TRANSLATIONS, getJointTranslations()); APPEND_ENTITY_PROPERTY(PROP_RELAY_PARENT_JOINTS, getRelayParentJoints()); APPEND_ENTITY_PROPERTY(PROP_GROUP_CULLED, getGroupCulled()); + APPEND_ENTITY_PROPERTY(PROP_BLENDSHAPE_COEFFICIENTS, getBlendshapeCoefficients()); withReadLock([&] { _animationProperties.appendSubclassData(packetData, params, entityTreeElementExtraEncodeData, requestedProperties, @@ -256,6 +262,7 @@ void ModelEntityItem::debugDump() const { qCDebug(entities) << " dimensions:" << getScaledDimensions(); qCDebug(entities) << " model URL:" << getModelURL(); qCDebug(entities) << " compound shape URL:" << getCompoundShapeURL(); + qCDebug(entities) << " blendshapeCoefficients:" << getBlendshapeCoefficients(); } void ModelEntityItem::setShapeType(ShapeType type) { @@ -743,3 +750,39 @@ void ModelEntityItem::setModelScale(const glm::vec3& modelScale) { _modelScale = modelScale; }); } + +QString ModelEntityItem::getBlendshapeCoefficients() const { + return resultWithReadLock([&] { + return QJsonDocument::fromVariant(_blendshapeCoefficientsMap).toJson(); + }); +} + +void ModelEntityItem::setBlendshapeCoefficients(const QString& blendshapeCoefficients) { + QJsonParseError error; + QJsonDocument newCoefficientsJSON = QJsonDocument::fromJson(blendshapeCoefficients.toUtf8(), &error); + if (error.error != QJsonParseError::NoError) { + qWarning() << "Could not evaluate blendshapeCoefficients property value:" << newCoefficientsJSON; + return; + } + + QVariantMap newCoefficientsMap = newCoefficientsJSON.toVariant().toMap(); + withWriteLock([&] { + for (auto& blendshape : newCoefficientsMap.keys()) { + auto newCoefficient = newCoefficientsMap[blendshape]; + auto blendshapeIter = blendshapeLookupMap.find(blendshape); + if (newCoefficient.canConvert() && blendshapeIter != blendshapeLookupMap.end()) { + float newCoefficientValue = newCoefficient.toFloat(); + _blendshapeCoefficientsVector[blendshapeIter.value()] = newCoefficientValue; + _blendshapeCoefficientsMap[blendshape] = newCoefficientValue; + _blendshapesChanged = true; + } + } + }); +} + +QVector ModelEntityItem::getBlendshapeCoefficientVector() { + return resultWithReadLock>([&] { + _blendshapesChanged = false; // ok to change this within read lock here + return _blendshapeCoefficientsVector; + }); +} \ No newline at end of file diff --git a/libraries/entities/src/ModelEntityItem.h b/libraries/entities/src/ModelEntityItem.h index 75a695f1c0..c331a94adf 100644 --- a/libraries/entities/src/ModelEntityItem.h +++ b/libraries/entities/src/ModelEntityItem.h @@ -17,6 +17,7 @@ #include #include "AnimationPropertyGroup.h" +#include "BlendshapeConstants.h" class ModelEntityItem : public EntityItem { public: @@ -133,6 +134,11 @@ public: glm::vec3 getModelScale() const; void setModelScale(const glm::vec3& modelScale); + QString getBlendshapeCoefficients() const; + void setBlendshapeCoefficients(const QString& blendshapeCoefficients); + bool blendshapesChanged() const { return _blendshapesChanged; } + QVector getBlendshapeCoefficientVector(); + private: void setAnimationSettings(const QString& value); // only called for old bitstream format bool applyNewAnimationProperties(AnimationPropertyGroup newProperties); @@ -166,6 +172,7 @@ protected: QString _modelURL; bool _relayParentJoints; bool _groupCulled { false }; + QVariantMap _blendshapeCoefficientsMap; ThreadSafeValueCache _compoundShapeURL; @@ -178,6 +185,9 @@ protected: private: uint64_t _lastAnimated{ 0 }; float _currentFrame{ -1.0f }; + + QVector _blendshapeCoefficientsVector; + bool _blendshapesChanged { false }; }; #endif // hifi_ModelEntityItem_h diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index fbf575065e..a139f0236c 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -275,6 +275,7 @@ enum class EntityVersion : PacketVersion { ShadowBiasAndDistance, TextEntityFonts, ScriptServerKinematicMotion, + ModelBlendshapes, // Add new versions above here NUM_PACKET_TYPE, diff --git a/libraries/render-utils/src/CauterizedModel.cpp b/libraries/render-utils/src/CauterizedModel.cpp index 7f42b05fa4..94b7661b2f 100644 --- a/libraries/render-utils/src/CauterizedModel.cpp +++ b/libraries/render-utils/src/CauterizedModel.cpp @@ -175,12 +175,7 @@ void CauterizedModel::updateClusterMatrices() { } } - // post the blender if we're not currently waiting for one to finish - auto modelBlender = DependencyManager::get(); - if (modelBlender->shouldComputeBlendshapes() && hfmModel.hasBlendedMeshes() && _blendshapeCoefficients != _blendedBlendshapeCoefficients) { - _blendedBlendshapeCoefficients = _blendshapeCoefficients; - modelBlender->noteRequiresBlend(getThisPointer()); - } + updateBlendshapes(); } void CauterizedModel::updateRenderItems() { diff --git a/libraries/render-utils/src/MeshPartPayload.cpp b/libraries/render-utils/src/MeshPartPayload.cpp index e82af5395f..318d2f298f 100644 --- a/libraries/render-utils/src/MeshPartPayload.cpp +++ b/libraries/render-utils/src/MeshPartPayload.cpp @@ -450,6 +450,7 @@ void ModelMeshPartPayload::setShapeKey(bool invalidateShapeKey, PrimitiveMode pr } } + _prevUseDualQuaternionSkinning = useDualQuaternionSkinning; _shapeKey = builder.build(); } @@ -559,6 +560,14 @@ void ModelMeshPartPayload::setBlendshapeBuffer(const std::unordered_mapsecond; + if (_isSkinned || (_isBlendShaped && _meshBlendshapeBuffer)) { + ShapeKey::Builder builder(_shapeKey); + builder.withDeformed(); + if (_prevUseDualQuaternionSkinning) { + builder.withDualQuatSkinned(); + } + _shapeKey = builder.build(); + } } } } diff --git a/libraries/render-utils/src/MeshPartPayload.h b/libraries/render-utils/src/MeshPartPayload.h index ee205bd778..dc0fe9614d 100644 --- a/libraries/render-utils/src/MeshPartPayload.h +++ b/libraries/render-utils/src/MeshPartPayload.h @@ -145,6 +145,7 @@ private: gpu::BufferPointer _meshBlendshapeBuffer; int _meshNumVertices; render::ShapeKey _shapeKey { render::ShapeKey::Builder::invalid() }; + bool _prevUseDualQuaternionSkinning { false }; bool _cauterized { false }; }; diff --git a/libraries/render-utils/src/Model.cpp b/libraries/render-utils/src/Model.cpp index d8172112ff..c83e0e726f 100644 --- a/libraries/render-utils/src/Model.cpp +++ b/libraries/render-utils/src/Model.cpp @@ -1427,9 +1427,13 @@ void Model::updateClusterMatrices() { } } + updateBlendshapes(); +} + +void Model::updateBlendshapes() { // post the blender if we're not currently waiting for one to finish auto modelBlender = DependencyManager::get(); - if (modelBlender->shouldComputeBlendshapes() && hfmModel.hasBlendedMeshes() && _blendshapeCoefficients != _blendedBlendshapeCoefficients) { + if (modelBlender->shouldComputeBlendshapes() && getHFMModel().hasBlendedMeshes() && _blendshapeCoefficients != _blendedBlendshapeCoefficients) { _blendedBlendshapeCoefficients = _blendshapeCoefficients; modelBlender->noteRequiresBlend(getThisPointer()); } diff --git a/libraries/render-utils/src/Model.h b/libraries/render-utils/src/Model.h index b677861e9f..ac91a310df 100644 --- a/libraries/render-utils/src/Model.h +++ b/libraries/render-utils/src/Model.h @@ -76,17 +76,6 @@ struct SortedTriangleSet { int subMeshIndex; }; -struct BlendshapeOffsetPacked { - glm::uvec4 packedPosNorTan; -}; - -struct BlendshapeOffsetUnpacked { - glm::vec3 positionOffset; - glm::vec3 normalOffset; - glm::vec3 tangentOffset; -}; - -using BlendshapeOffset = BlendshapeOffsetPacked; using BlendShapeOperator = std::function&, const QVector&, const render::ItemIDs&)>; /// A generic 3D model displaying geometry loaded from a URL. @@ -178,6 +167,7 @@ public: virtual void simulate(float deltaTime, bool fullUpdate = true); virtual void updateClusterMatrices(); + virtual void updateBlendshapes(); /// Returns a reference to the shared geometry. const Geometry::Pointer& getGeometry() const { return _renderGeometry; } @@ -365,6 +355,8 @@ public: void addMaterial(graphics::MaterialLayer material, const std::string& parentMaterialName); void removeMaterial(graphics::MaterialPointer material, const std::string& parentMaterialName); + void setBlendshapeCoefficients(const QVector& coefficients) { _blendshapeCoefficients = coefficients; } + public slots: void loadURLFinished(bool success); @@ -382,7 +374,6 @@ protected: std::mutex _materialMappingMutex; void applyMaterialMapping(); - void setBlendshapeCoefficients(const QVector& coefficients) { _blendshapeCoefficients = coefficients; } const QVector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } /// Clear the joint states diff --git a/libraries/render-utils/src/SoftAttachmentModel.cpp b/libraries/render-utils/src/SoftAttachmentModel.cpp index 186f9e682a..24d6081743 100644 --- a/libraries/render-utils/src/SoftAttachmentModel.cpp +++ b/libraries/render-utils/src/SoftAttachmentModel.cpp @@ -69,10 +69,5 @@ void SoftAttachmentModel::updateClusterMatrices() { } } - // post the blender if we're not currently waiting for one to finish - auto modelBlender = DependencyManager::get(); - if (modelBlender->shouldComputeBlendshapes() && hfmModel.hasBlendedMeshes() && _blendshapeCoefficients != _blendedBlendshapeCoefficients) { - _blendedBlendshapeCoefficients = _blendshapeCoefficients; - modelBlender->noteRequiresBlend(getThisPointer()); - } + updateBlendshapes(); } diff --git a/libraries/render/src/render/Item.h b/libraries/render/src/render/Item.h index 3383101b5b..f59ef6cc1b 100644 --- a/libraries/render/src/render/Item.h +++ b/libraries/render/src/render/Item.h @@ -28,6 +28,8 @@ #include #include "ShapePipeline.h" +#include "BlendshapeConstants.h" + namespace render { typedef int32_t Index; @@ -615,6 +617,10 @@ public: virtual Item::Bound getBound() = 0; virtual void render(RenderArgs* args) = 0; virtual uint32_t metaFetchMetaSubItems(ItemIDs& subItems) const = 0; + + // FIXME: this isn't the best place for this since it's only used for ModelEntities, but currently all Entities use PayloadProxyInterface + virtual void handleBlendedVertices(int blendshapeNumber, const QVector& blendshapeOffsets, + const QVector& blendedMeshSizes, const render::ItemIDs& subItemIDs) {}; }; template <> const ItemKey payloadGetKey(const PayloadProxyInterface::Pointer& payload); diff --git a/libraries/shared/src/BlendshapeConstants.cpp b/libraries/shared/src/BlendshapeConstants.cpp index 172df461fd..8486477c78 100644 --- a/libraries/shared/src/BlendshapeConstants.cpp +++ b/libraries/shared/src/BlendshapeConstants.cpp @@ -76,3 +76,11 @@ const char* FACESHIFT_BLENDSHAPES[] = { "UserBlendshape9", "" }; + +QMap blendshapeLookupMap = [] { + QMap toReturn; + for (int i = 0; i < (int)Blendshapes::BlendshapeCount; i++) { + toReturn[FACESHIFT_BLENDSHAPES[i]] = i; + } + return toReturn; +}(); \ No newline at end of file diff --git a/libraries/shared/src/BlendshapeConstants.h b/libraries/shared/src/BlendshapeConstants.h index e74146eb56..358efb6597 100644 --- a/libraries/shared/src/BlendshapeConstants.h +++ b/libraries/shared/src/BlendshapeConstants.h @@ -12,8 +12,14 @@ #ifndef hifi_BlendshapeConstants_h #define hifi_BlendshapeConstants_h +#include +#include + +#include + /// The names of the blendshapes expected by Faceshift, terminated with an empty string. extern const char* FACESHIFT_BLENDSHAPES[]; +extern QMap blendshapeLookupMap; enum class Blendshapes : int { EyeBlink_L = 0, @@ -115,5 +121,16 @@ enum class LegacyBlendshpaes : int { // * LipsUpperOpen (not in ARKit) // * LipsLowerOpen (not in ARKit) +struct BlendshapeOffsetPacked { + glm::uvec4 packedPosNorTan; +}; + +struct BlendshapeOffsetUnpacked { + glm::vec3 positionOffset; + glm::vec3 normalOffset; + glm::vec3 tangentOffset; +}; + +using BlendshapeOffset = BlendshapeOffsetPacked; #endif // hifi_BlendshapeConstants_h diff --git a/plugins/KasenAPIExample/src/KasenAPIExample.cpp b/plugins/KasenAPIExample/src/KasenAPIExample.cpp index d566d11376..b0a9c29519 100644 --- a/plugins/KasenAPIExample/src/KasenAPIExample.cpp +++ b/plugins/KasenAPIExample/src/KasenAPIExample.cpp @@ -113,7 +113,7 @@ private: } struct _HeadHelper : public HeadData { QMap getBlendshapeMap() const { - return _blendshapeLookupMap; + return blendshapeLookupMap; } struct States { QVector base, summed, transient; }; States getBlendshapeStates() const {