Merge pull request #6732 from sethalves/set-model-entity-joints

hook up a way for scripts to set model-entity joint state

examples/example/avatarcontrol/doppelganger.js

@@ -0,0 +1,121 @@
+//
+//  doppelganger.js
+//
+//  Created by James B. Pollack @imgntn on 12/28/2015
+//  Copyright 2015 High Fidelity, Inc.
+//
+//  This script shows how to hook up a model entity to your avatar to act as a doppelganger.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+//  To-Do:  mirror joints, rotate avatar fully, automatically get avatar fbx, make sure dimensions for avatar are right when u bring it in
+
+var TEST_MODEL_URL = 'https://s3.amazonaws.com/hifi-public/ozan/avatars/albert/albert/albert.fbx';
+
+var doppelgangers = [];
+
+function Doppelganger(avatar) {
+    this.initialProperties = {
+        name: 'Hifi-Doppelganger',
+        type: 'Model',
+        modelURL: TEST_MODEL_URL,
+        // dimensions: getAvatarDimensions(avatar),
+        position: putDoppelgangerAcrossFromAvatar(this, avatar),
+        rotation: rotateDoppelgangerTowardAvatar(this, avatar),
+    };
+
+    this.id = createDoppelgangerEntity(this);
+    this.avatar = avatar;
+    return this;
+}
+
+function getJointData(avatar) {
+    var allJointData = [];
+    var jointNames = MyAvatar.jointNames;
+    jointNames.forEach(function(joint, index) {
+        var translation = MyAvatar.getJointTranslation(index);
+        var rotation = MyAvatar.getJointRotation(index)
+        allJointData.push({
+            joint: joint,
+            index: index,
+            translation: translation,
+            rotation: rotation
+        })
+    });
+
+    return allJointData;
+}
+
+function setJointData(doppelganger, allJointData) {
+    var jointRotations = [];
+    allJointData.forEach(function(jointData, index) {
+        jointRotations.push(jointData.rotation);
+    });
+    Entities.setAbsoluteJointRotationsInObjectFrame(doppelganger.id, jointRotations);
+
+    return true;
+}
+
+function mirrorJointData() {
+    return mirroredJointData;
+}
+
+function createDoppelganger(avatar) {
+    return new Doppelganger(avatar);
+}
+
+function createDoppelgangerEntity(doppelganger) {
+    return Entities.addEntity(doppelganger.initialProperties);
+}
+
+function putDoppelgangerAcrossFromAvatar(doppelganger, avatar) {
+    var avatarRot = Quat.fromPitchYawRollDegrees(0, avatar.bodyYaw, 0.0);
+    var basePosition = Vec3.sum(avatar.position, Vec3.multiply(1.5, Quat.getFront(avatarRot)));
+    return basePosition;
+}
+
+function getAvatarDimensions(avatar) {
+    return dimensions;
+}
+
+function rotateDoppelgangerTowardAvatar(doppelganger, avatar) {
+    var avatarRot = Quat.fromPitchYawRollDegrees(0, avatar.bodyYaw, 0.0);
+    avatarRot = Vec3.multiply(-1, avatarRot);
+    return avatarRot;
+}
+
+function connectDoppelgangerUpdates() {
+    // Script.update.connect(updateDoppelganger);
+    Script.setInterval(updateDoppelganger, 100);
+}
+
+function disconnectDoppelgangerUpdates() {
+    Script.update.disconnect(updateDoppelganger);
+}
+
+function updateDoppelganger() {
+    doppelgangers.forEach(function(doppelganger) {
+        var joints = getJointData(MyAvatar);
+        //var mirroredJoints = mirrorJointData(joints);
+        setJointData(doppelganger, joints);
+    });
+}
+
+function makeDoppelgangerForMyAvatar() {
+    var doppelganger = createDoppelganger(MyAvatar);
+    doppelgangers.push(doppelganger);
+    connectDoppelgangerUpdates();
+}
+
+makeDoppelgangerForMyAvatar();
+
+function cleanup() {
+    disconnectDoppelgangerUpdates();
+
+    doppelgangers.forEach(function(doppelganger) {
+        Entities.deleteEntity(doppelganger.id);
+    });
+}
+
+Script.scriptEnding.connect(cleanup);

interface/src/avatar/Avatar.h

@@ -113,6 +113,8 @@ public:
 
     virtual glm::quat getAbsoluteJointRotationInObjectFrame(int index) const override;
     virtual glm::vec3 getAbsoluteJointTranslationInObjectFrame(int index) const override;
+    virtual bool setAbsoluteJointRotationInObjectFrame(int index, const glm::quat& rotation) override { return false; }
+    virtual bool setAbsoluteJointTranslationInObjectFrame(int index, const glm::vec3& translation) override { return false; }
 
     virtual void setFaceModelURL(const QUrl& faceModelURL) override;
     virtual void setSkeletonModelURL(const QUrl& skeletonModelURL) override;

libraries/avatars/src/AvatarData.h

@@ -358,6 +358,8 @@ public slots:
 
     virtual glm::quat getAbsoluteJointRotationInObjectFrame(int index) const override;
     virtual glm::vec3 getAbsoluteJointTranslationInObjectFrame(int index) const override;
+    virtual bool setAbsoluteJointRotationInObjectFrame(int index, const glm::quat& rotation) override { return false; }
+    virtual bool setAbsoluteJointTranslationInObjectFrame(int index, const glm::vec3& translation) override { return false; }
 
 protected:
     glm::vec3 _handPosition;

libraries/entities-renderer/src/RenderableModelEntityItem.cpp

@@ -13,6 +13,7 @@
 
 #include <QJsonDocument>
 #include <QtCore/QThread>
+#include <glm/gtx/transform.hpp>
 
 #include <AbstractViewStateInterface.h>
 #include <DeferredLightingEffect.h>
@@ -235,8 +236,8 @@ bool RenderableModelEntityItem::addToScene(EntityItemPointer self, std::shared_p
 
     return true;
 }
-    
+
-void RenderableModelEntityItem::removeFromScene(EntityItemPointer self, std::shared_ptr<render::Scene> scene, 
+void RenderableModelEntityItem::removeFromScene(EntityItemPointer self, std::shared_ptr<render::Scene> scene,
                                                 render::PendingChanges& pendingChanges) {
     pendingChanges.removeItem(_myMetaItem);
     if (_model) {
@@ -244,6 +245,84 @@ void RenderableModelEntityItem::removeFromScene(EntityItemPointer self, std::sha
     }
 }
 
+void RenderableModelEntityItem::resizeJointArrays(int newSize) {
+    if (newSize < 0) {
+        if (_model && _model->isActive() && _model->isLoaded() && !_needsInitialSimulation) {
+            newSize = _model->getJointStateCount();
+        }
+    }
+    ModelEntityItem::resizeJointArrays(newSize);
+}
+
+bool RenderableModelEntityItem::getAnimationFrame() {
+    bool newFrame = false;
+
+    if (!_model || !_model->isActive() || !_model->isLoaded() || _needsInitialSimulation) {
+        return false;
+    }
+
+    if (!hasAnimation() || !_jointMappingCompleted) {
+        return false;
+    }
+    AnimationPointer myAnimation = getAnimation(_animationProperties.getURL()); // FIXME: this could be optimized
+    if (myAnimation && myAnimation->isLoaded()) {
+
+        const QVector<FBXAnimationFrame>&  frames = myAnimation->getFramesReference(); // NOTE: getFrames() is too heavy
+        auto& fbxJoints = myAnimation->getGeometry().joints;
+
+        int frameCount = frames.size();
+        if (frameCount > 0) {
+            int animationCurrentFrame = (int)(glm::floor(getAnimationCurrentFrame())) % frameCount;
+            if (animationCurrentFrame < 0 || animationCurrentFrame > frameCount) {
+                animationCurrentFrame = 0;
+            }
+
+            if (animationCurrentFrame != _lastKnownCurrentFrame) {
+                _lastKnownCurrentFrame = animationCurrentFrame;
+                newFrame = true;
+
+                resizeJointArrays();
+                if (_jointMapping.size() != _model->getJointStateCount()) {
+                    qDebug() << "RenderableModelEntityItem::getAnimationFrame -- joint count mismatch"
+                             << _jointMapping.size() << _model->getJointStateCount();
+                    assert(false);
+                    return false;
+                }
+
+                const QVector<glm::quat>& rotations = frames[animationCurrentFrame].rotations;
+                const QVector<glm::vec3>& translations = frames[animationCurrentFrame].translations;
+
+                for (int j = 0; j < _jointMapping.size(); j++) {
+                    int index = _jointMapping[j];
+                    if (index >= 0) {
+                        glm::mat4 translationMat;
+                        if (index < translations.size()) {
+                            translationMat = glm::translate(translations[index]);
+                        }
+                        glm::mat4 rotationMat(glm::mat4::_null);
+                        if (index < rotations.size()) {
+                            rotationMat = glm::mat4_cast(fbxJoints[index].preRotation * rotations[index] * fbxJoints[index].postRotation);
+                        } else {
+                            rotationMat = glm::mat4_cast(fbxJoints[index].preRotation * fbxJoints[index].postRotation);
+                        }
+                        glm::mat4 finalMat = (translationMat * fbxJoints[index].preTransform *
+                                              rotationMat * fbxJoints[index].postTransform);
+                        _absoluteJointTranslationsInObjectFrame[j] = extractTranslation(finalMat);
+                        _absoluteJointTranslationsInObjectFrameSet[j] = true;
+                        _absoluteJointTranslationsInObjectFrameDirty[j] = true;
+
+                        _absoluteJointRotationsInObjectFrame[j] = glmExtractRotation(finalMat);
+
+                        _absoluteJointRotationsInObjectFrameSet[j] = true;
+                        _absoluteJointRotationsInObjectFrameDirty[j] = true;
+                    }
+                }
+            }
+        }
+    }
+
+    return newFrame;
+}
 
 // NOTE: this only renders the "meta" portion of the Model, namely it renders debugging items, and it handles
 // the per frame simulation/update that might be required if the models properties changed.
@@ -297,26 +376,32 @@ void RenderableModelEntityItem::render(RenderArgs* args) {
             }
 
             if (_model) {
-                // handle animations..
                 if (hasAnimation()) {
                     if (!jointsMapped()) {
                         QStringList modelJointNames = _model->getJointNames();
                         mapJoints(modelJointNames);
                     }
+                }
 
-                    if (jointsMapped()) {
+                _jointDataLock.withWriteLock([&] {
-                        bool newFrame;
+                    getAnimationFrame();
-                        QVector<glm::quat> frameDataRotations;
+
-                        QVector<glm::vec3> frameDataTranslations;
+                    // relay any inbound joint changes from scripts/animation/network to the model/rig
-                        getAnimationFrame(newFrame, frameDataRotations, frameDataTranslations);
+                    for (int index = 0; index < _absoluteJointRotationsInObjectFrame.size(); index++) {
-                        assert(frameDataRotations.size() == frameDataTranslations.size());
+                        if (_absoluteJointRotationsInObjectFrameDirty[index]) {
-                        if (newFrame) {
+                            glm::quat rotation = _absoluteJointRotationsInObjectFrame[index];
-                            for (int i = 0; i < frameDataRotations.size(); i++) {
+                            _model->setJointRotation(index, true, rotation, 1.0f);
-                                _model->setJointState(i, true, frameDataRotations[i], frameDataTranslations[i], 1.0f);
+                            _absoluteJointRotationsInObjectFrameDirty[index] = false;
-                            }
                         }
                     }
-                }
+                    for (int index = 0; index < _absoluteJointTranslationsInObjectFrame.size(); index++) {
+                        if (_absoluteJointTranslationsInObjectFrameDirty[index]) {
+                            glm::vec3 translation = _absoluteJointTranslationsInObjectFrame[index];
+                            _model->setJointTranslation(index, true, translation, 1.0f);
+                            _absoluteJointTranslationsInObjectFrameDirty[index] = false;
+                        }
+                    }
+                });
 
                 bool movingOrAnimating = isMoving() || isAnimatingSomething();
                 if ((movingOrAnimating ||
@@ -608,7 +693,7 @@ bool RenderableModelEntityItem::contains(const glm::vec3& point) const {
         const FBXGeometry& collisionGeometry = collisionNetworkGeometry->getFBXGeometry();
         return collisionGeometry.convexHullContains(worldToEntity(point));
     }
-    
+
     return false;
 }
 
@@ -632,6 +717,36 @@ glm::vec3 RenderableModelEntityItem::getAbsoluteJointTranslationInObjectFrame(in
     return glm::vec3(0.0f);
 }
 
+bool RenderableModelEntityItem::setAbsoluteJointRotationInObjectFrame(int index, const glm::quat& rotation) {
+    bool result = false;
+    _jointDataLock.withWriteLock([&] {
+        resizeJointArrays();
+        if (index >= 0 && index < _absoluteJointRotationsInObjectFrame.size() &&
+            _absoluteJointRotationsInObjectFrame[index] != rotation) {
+            _absoluteJointRotationsInObjectFrame[index] = rotation;
+            _absoluteJointRotationsInObjectFrameSet[index] = true;
+            _absoluteJointRotationsInObjectFrameDirty[index] = true;
+            result = true;
+        }
+    });
+    return result;
+}
+
+bool RenderableModelEntityItem::setAbsoluteJointTranslationInObjectFrame(int index, const glm::vec3& translation) {
+    bool result = false;
+    _jointDataLock.withWriteLock([&] {
+        resizeJointArrays();
+        if (index >= 0 && index < _absoluteJointTranslationsInObjectFrame.size() &&
+            _absoluteJointTranslationsInObjectFrame[index] != translation) {
+            _absoluteJointTranslationsInObjectFrame[index] = translation;
+            _absoluteJointTranslationsInObjectFrameSet[index] = true;
+            _absoluteJointTranslationsInObjectFrameDirty[index] = true;
+            result = true;
+        }
+    });
+    return result;
+}
+
 void RenderableModelEntityItem::locationChanged() {
     EntityItem::locationChanged();
     if (_model && _model->isActive()) {

libraries/entities-renderer/src/RenderableModelEntityItem.h

@@ -33,15 +33,15 @@ public:
 
     virtual EntityItemProperties getProperties(EntityPropertyFlags desiredProperties = EntityPropertyFlags()) const override;
     virtual bool setProperties(const EntityItemProperties& properties) override;
-    virtual int readEntitySubclassDataFromBuffer(const unsigned char* data, int bytesLeftToRead, 
+    virtual int readEntitySubclassDataFromBuffer(const unsigned char* data, int bytesLeftToRead,
                                                 ReadBitstreamToTreeParams& args,
                                                 EntityPropertyFlags& propertyFlags, bool overwriteLocalData,
                                                 bool& somethingChanged) override;
-                                                
+
     virtual void somethingChangedNotification() override {
         // FIX ME: this is overly aggressive. We only really need to simulate() if something about
         // the world space transform has changed and/or if some animation is occurring.
-        _needsInitialSimulation = true;  
+        _needsInitialSimulation = true;
     }
 
     virtual bool readyToAddToScene(RenderArgs* renderArgs = nullptr);
@@ -52,12 +52,12 @@ public:
     virtual void render(RenderArgs* args) override;
     virtual bool supportsDetailedRayIntersection() const override { return true; }
     virtual bool findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
-                        bool& keepSearching, OctreeElementPointer& element, float& distance, 
+                        bool& keepSearching, OctreeElementPointer& element, float& distance,
                         BoxFace& face, glm::vec3& surfaceNormal,
                         void** intersectedObject, bool precisionPicking) const override;
 
     Model* getModel(EntityTreeRenderer* renderer);
-    
+
     virtual bool needsToCallUpdate() const override;
     virtual void update(const quint64& now) override;
 
@@ -65,19 +65,23 @@ public:
 
     virtual bool isReadyToComputeShape() override;
     virtual void computeShapeInfo(ShapeInfo& info) override;
-    
+
     virtual bool contains(const glm::vec3& point) const override;
 
     // these are in the frame of this object (model space)
     virtual glm::quat getAbsoluteJointRotationInObjectFrame(int index) const override;
     virtual glm::vec3 getAbsoluteJointTranslationInObjectFrame(int index) const override;
+    virtual bool setAbsoluteJointRotationInObjectFrame(int index, const glm::quat& rotation) override;
+    virtual bool setAbsoluteJointTranslationInObjectFrame(int index, const glm::vec3& translation) override;
 
     virtual void loader() override;
     virtual void locationChanged() override;
 
+    virtual void resizeJointArrays(int newSize = -1) override;
+
 private:
     void remapTextures();
-    
+
     Model* _model = nullptr;
     bool _needsInitialSimulation = true;
     bool _needsModelReload = true;
@@ -87,10 +91,12 @@ private:
     bool _originalTexturesRead = false;
     QVector<QVector<glm::vec3>> _points;
     bool _dimensionsInitialized = true;
-    
+
     render::ItemID _myMetaItem;
 
     bool _showCollisionHull = false;
+
+    bool getAnimationFrame();
 };
 
 #endif // hifi_RenderableModelEntityItem_h

libraries/entities/src/EntityItem.cpp

@@ -1853,14 +1853,6 @@ QList<EntityActionPointer> EntityItem::getActionsOfType(EntityActionType typeToG
     return result;
 }
 
-glm::quat EntityItem::getAbsoluteJointRotationInObjectFrame(int index) const {
-    return glm::quat();
-}
-
-glm::vec3 EntityItem::getAbsoluteJointTranslationInObjectFrame(int index) const {
-    return glm::vec3(0.0f);
-}
-
 void EntityItem::locationChanged() {
     requiresRecalcBoxes();
     SpatiallyNestable::locationChanged(); // tell all the children, also

libraries/entities/src/EntityItem.h

@@ -381,8 +381,10 @@ public:
     QList<EntityActionPointer> getActionsOfType(EntityActionType typeToGet);
 
     // these are in the frame of this object
-    virtual glm::quat getAbsoluteJointRotationInObjectFrame(int index) const override;
+    virtual glm::quat getAbsoluteJointRotationInObjectFrame(int index) const override { return glm::quat(); }
-    virtual glm::vec3 getAbsoluteJointTranslationInObjectFrame(int index) const override;
+    virtual glm::vec3 getAbsoluteJointTranslationInObjectFrame(int index) const override { return glm::vec3(0.0f); }
+    virtual bool setAbsoluteJointRotationInObjectFrame(int index, const glm::quat& rotation) override { return false; }
+    virtual bool setAbsoluteJointTranslationInObjectFrame(int index, const glm::vec3& translation) override { return false; }
 
     virtual void loader() {} // called indirectly when urls for geometry are updated
 

libraries/entities/src/EntityItemProperties.cpp

@@ -262,6 +262,10 @@ EntityPropertyFlags EntityItemProperties::getChangedProperties() const {
     CHECK_PROPERTY_CHANGE(PROP_Z_P_NEIGHBOR_ID, zPNeighborID);
     CHECK_PROPERTY_CHANGE(PROP_PARENT_ID, parentID);
     CHECK_PROPERTY_CHANGE(PROP_PARENT_JOINT_INDEX, parentJointIndex);
+    CHECK_PROPERTY_CHANGE(PROP_JOINT_ROTATIONS_SET, jointRotationsSet);
+    CHECK_PROPERTY_CHANGE(PROP_JOINT_ROTATIONS, jointRotations);
+    CHECK_PROPERTY_CHANGE(PROP_JOINT_TRANSLATIONS_SET, jointTranslationsSet);
+    CHECK_PROPERTY_CHANGE(PROP_JOINT_TRANSLATIONS, jointTranslations);
     CHECK_PROPERTY_CHANGE(PROP_QUERY_AA_CUBE, queryAACube);
 
     changedProperties += _animation.getChangedProperties();
@@ -364,6 +368,10 @@ QScriptValue EntityItemProperties::copyToScriptValue(QScriptEngine* engine, bool
     if (_type == EntityTypes::Model) {
         COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_MODEL_URL, modelURL);
         _animation.copyToScriptValue(_desiredProperties, properties, engine, skipDefaults, defaultEntityProperties);
+        COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_JOINT_ROTATIONS_SET, jointRotationsSet);
+        COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_JOINT_ROTATIONS, jointRotations);
+        COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_JOINT_TRANSLATIONS_SET, jointTranslationsSet);
+        COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_JOINT_TRANSLATIONS, jointTranslations);
     }
 
     if (_type == EntityTypes::Model || _type == EntityTypes::Zone || _type == EntityTypes::ParticleEffect) {
@@ -479,6 +487,11 @@ QScriptValue EntityItemProperties::copyToScriptValue(QScriptEngine* engine, bool
     COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_LOCAL_POSITION, localPosition);
     COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_LOCAL_ROTATION, localRotation);
 
+    COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_JOINT_ROTATIONS_SET, jointRotationsSet);
+    COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_JOINT_ROTATIONS, jointRotations);
+    COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_JOINT_TRANSLATIONS_SET, jointTranslationsSet);
+    COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_JOINT_TRANSLATIONS, jointTranslations);
+
     COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_QUERY_AA_CUBE, queryAACube);
 
     // FIXME - I don't think these properties are supported any more
@@ -610,6 +623,11 @@ void EntityItemProperties::copyFromScriptValue(const QScriptValue& object, bool
     COPY_PROPERTY_FROM_QSCRIPTVALUE(localPosition, glmVec3, setLocalPosition);
     COPY_PROPERTY_FROM_QSCRIPTVALUE(localRotation, glmQuat, setLocalRotation);
 
+    COPY_PROPERTY_FROM_QSCRIPTVALUE(jointRotationsSet, qVectorBool, setJointRotationsSet);
+    COPY_PROPERTY_FROM_QSCRIPTVALUE(jointRotations, qVectorQuat, setJointRotations);
+    COPY_PROPERTY_FROM_QSCRIPTVALUE(jointTranslationsSet, qVectorBool, setJointTranslationsSet);
+    COPY_PROPERTY_FROM_QSCRIPTVALUE(jointTranslations, qVectorVec3, setJointTranslations);
+
     _lastEdited = usecTimestampNow();
 }
 
@@ -744,6 +762,14 @@ void EntityItemProperties::entityPropertyFlagsFromScriptValue(const QScriptValue
         ADD_PROPERTY_TO_MAP(PROP_Y_P_NEIGHBOR_ID, YPNeighborID, yPNeighborID, EntityItemID);
         ADD_PROPERTY_TO_MAP(PROP_Z_P_NEIGHBOR_ID, ZPNeighborID, zPNeighborID, EntityItemID);
 
+        ADD_PROPERTY_TO_MAP(PROP_PARENT_ID, ParentID, parentID, QUuid);
+        ADD_PROPERTY_TO_MAP(PROP_PARENT_JOINT_INDEX, ParentJointIndex, parentJointIndex, uint16_t);
+
+        ADD_PROPERTY_TO_MAP(PROP_JOINT_ROTATIONS_SET, JointRotationsSet, jointRotationsSet, QVector<bool>);
+        ADD_PROPERTY_TO_MAP(PROP_JOINT_ROTATIONS, JointRotations, jointRotations, QVector<glm::quat>);
+        ADD_PROPERTY_TO_MAP(PROP_JOINT_TRANSLATIONS_SET, JointTranslationsSet, jointTranslationsSet, QVector<bool>);
+        ADD_PROPERTY_TO_MAP(PROP_JOINT_TRANSLATIONS, JointTranslations, jointTranslations, QVector<glm::vec3>);
+
         ADD_GROUP_PROPERTY_TO_MAP(PROP_ANIMATION_URL, Animation, animation, URL, url);
         ADD_GROUP_PROPERTY_TO_MAP(PROP_ANIMATION_FPS, Animation, animation, FPS, fps);
         ADD_GROUP_PROPERTY_TO_MAP(PROP_ANIMATION_FRAME_INDEX, Animation, animation, CurrentFrame, currentFrame);
@@ -949,6 +975,11 @@ bool EntityItemProperties::encodeEntityEditPacket(PacketType command, EntityItem
 
                 _staticAnimation.setProperties(properties);
                 _staticAnimation.appendToEditPacket(packetData, requestedProperties, propertyFlags, propertiesDidntFit, propertyCount, appendState);
+
+                APPEND_ENTITY_PROPERTY(PROP_JOINT_ROTATIONS_SET, properties.getJointRotationsSet());
+                APPEND_ENTITY_PROPERTY(PROP_JOINT_ROTATIONS, properties.getJointRotations());
+                APPEND_ENTITY_PROPERTY(PROP_JOINT_TRANSLATIONS_SET, properties.getJointTranslationsSet());
+                APPEND_ENTITY_PROPERTY(PROP_JOINT_TRANSLATIONS, properties.getJointTranslations());
             }
 
             if (properties.getType() == EntityTypes::Light) {
@@ -1235,6 +1266,11 @@ bool EntityItemProperties::decodeEntityEditPacket(const unsigned char* data, int
         READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_SHAPE_TYPE, ShapeType, setShapeType);
 
         properties.getAnimation().decodeFromEditPacket(propertyFlags, dataAt, processedBytes);
+
+        READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_JOINT_ROTATIONS_SET, QVector<bool>, setJointRotationsSet);
+        READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_JOINT_ROTATIONS, QVector<glm::quat>, setJointRotations);
+        READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_JOINT_TRANSLATIONS_SET, QVector<bool>, setJointTranslationsSet);
+        READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_JOINT_TRANSLATIONS, QVector<glm::vec3>, setJointTranslations);
     }
 
     if (properties.getType() == EntityTypes::Light) {
@@ -1468,6 +1504,11 @@ void EntityItemProperties::markAllChanged() {
     _parentIDChanged = true;
     _parentJointIndexChanged = true;
 
+    _jointRotationsSetChanged = true;
+    _jointRotationsChanged = true;
+    _jointTranslationsSetChanged = true;
+    _jointTranslationsChanged = true;
+
     _queryAACubeChanged = true;
 }
 
@@ -1768,6 +1809,18 @@ QList<QString> EntityItemProperties::listChangedProperties() {
     if (parentJointIndexChanged()) {
         out += "parentJointIndex";
     }
+    if (jointRotationsSetChanged()) {
+        out += "jointRotationsSet";
+    }
+    if (jointRotationsChanged()) {
+        out += "jointRotations";
+    }
+    if (jointTranslationsSetChanged()) {
+        out += "jointTranslationsSet";
+    }
+    if (jointTranslationsChanged()) {
+        out += "jointTranslations";
+    }
     if (queryAACubeChanged()) {
         out += "queryAACube";
     }

libraries/entities/src/EntityItemProperties.h

@@ -199,6 +199,11 @@ public:
     DEFINE_PROPERTY_REF(PROP_LOCAL_POSITION, LocalPosition, localPosition, glmVec3, ENTITY_ITEM_ZERO_VEC3);
     DEFINE_PROPERTY_REF(PROP_LOCAL_ROTATION, LocalRotation, localRotation, glmQuat, ENTITY_ITEM_DEFAULT_ROTATION);
 
+    DEFINE_PROPERTY_REF(PROP_JOINT_ROTATIONS_SET, JointRotationsSet, jointRotationsSet, QVector<bool>, QVector<bool>());
+    DEFINE_PROPERTY_REF(PROP_JOINT_ROTATIONS, JointRotations, jointRotations, QVector<glm::quat>, QVector<glm::quat>());
+    DEFINE_PROPERTY_REF(PROP_JOINT_TRANSLATIONS_SET, JointTranslationsSet, jointTranslationsSet, QVector<bool>, QVector<bool>());
+    DEFINE_PROPERTY_REF(PROP_JOINT_TRANSLATIONS, JointTranslations, jointTranslations, QVector<glm::vec3>, QVector<glm::vec3>());
+
     static QString getBackgroundModeString(BackgroundMode mode);
 
 
@@ -261,10 +266,13 @@ public:
     void setActionDataDirty() { _actionDataChanged = true; }
 
     QList<QString> listChangedProperties();
-    
+
     bool getDimensionsInitialized() const { return _dimensionsInitialized; }
     void setDimensionsInitialized(bool dimensionsInitialized) { _dimensionsInitialized = dimensionsInitialized; }
-    
+
+    void setJointRotationsDirty() { _jointRotationsSetChanged = true; _jointRotationsChanged = true; }
+    void setJointTranslationsDirty() { _jointTranslationsSetChanged = true; _jointTranslationsChanged = true; }
+
 private:
     QUuid _id;
     bool _idSet;
@@ -400,6 +408,11 @@ inline QDebug operator<<(QDebug debug, const EntityItemProperties& properties) {
     DEBUG_PROPERTY_IF_CHANGED(debug, properties, ParentJointIndex, parentJointIndex, "");
     DEBUG_PROPERTY_IF_CHANGED(debug, properties, QueryAACube, queryAACube, "");
 
+    DEBUG_PROPERTY_IF_CHANGED(debug, properties, JointRotationsSet, jointRotationsSet, "");
+    DEBUG_PROPERTY_IF_CHANGED(debug, properties, JointRotations, jointRotations, "");
+    DEBUG_PROPERTY_IF_CHANGED(debug, properties, JointTranslationsSet, jointTranslationsSet, "");
+    DEBUG_PROPERTY_IF_CHANGED(debug, properties, JointTranslations, jointTranslations, "");
+
     properties.getAnimation().debugDump();
     properties.getAtmosphere().debugDump();
     properties.getSkybox().debugDump();

libraries/entities/src/EntityItemPropertiesMacros.h

@@ -113,6 +113,8 @@ inline QScriptValue convertScriptValue(QScriptEngine* e, const xColor& v) { retu
 inline QScriptValue convertScriptValue(QScriptEngine* e, const glm::quat& v) { return quatToScriptValue(e, v); }
 inline QScriptValue convertScriptValue(QScriptEngine* e, const QScriptValue& v) { return v; }
 inline QScriptValue convertScriptValue(QScriptEngine* e, const QVector<glm::vec3>& v) {return qVectorVec3ToScriptValue(e, v); }
+inline QScriptValue convertScriptValue(QScriptEngine* e, const QVector<glm::quat>& v) {return qVectorQuatToScriptValue(e, v); }
+inline QScriptValue convertScriptValue(QScriptEngine* e, const QVector<bool>& v) {return qVectorBoolToScriptValue(e, v); }
 inline QScriptValue convertScriptValue(QScriptEngine* e, const QVector<float>& v) { return qVectorFloatToScriptValue(e, v); }
 
 inline QScriptValue convertScriptValue(QScriptEngine* e, const QByteArray& v) {
@@ -176,6 +178,8 @@ inline QScriptValue convertScriptValue(QScriptEngine* e, const AACube& v) { retu
 typedef glm::vec3 glmVec3;
 typedef glm::quat glmQuat;
 typedef QVector<glm::vec3> qVectorVec3;
+typedef QVector<glm::quat> qVectorQuat;
+typedef QVector<bool> qVectorBool;
 typedef QVector<float> qVectorFloat;
 inline float float_convertFromScriptValue(const QScriptValue& v, bool& isValid) { return v.toVariant().toFloat(&isValid); }
 inline quint64 quint64_convertFromScriptValue(const QScriptValue& v, bool& isValid) { return v.toVariant().toULongLong(&isValid); }
@@ -245,6 +249,16 @@ inline qVectorVec3 qVectorVec3_convertFromScriptValue(const QScriptValue& v, boo
     return qVectorVec3FromScriptValue(v);
 }
 
+inline qVectorQuat qVectorQuat_convertFromScriptValue(const QScriptValue& v, bool& isValid) {
+    isValid = true;
+    return qVectorQuatFromScriptValue(v);
+}
+
+inline qVectorBool qVectorBool_convertFromScriptValue(const QScriptValue& v, bool& isValid) {
+    isValid = true;
+    return qVectorBoolFromScriptValue(v);
+}
+
 inline glmQuat glmQuat_convertFromScriptValue(const QScriptValue& v, bool& isValid) { 
     isValid = false; /// assume it can't be converted
     QScriptValue x = v.property("x");

libraries/entities/src/EntityPropertyFlags.h

@@ -159,6 +159,12 @@ enum EntityPropertyList {
 
     PROP_QUERY_AA_CUBE, // how the EntityTree considers the size and position on an entity
 
+    // ModelEntity joint state
+    PROP_JOINT_ROTATIONS_SET,
+    PROP_JOINT_ROTATIONS,
+    PROP_JOINT_TRANSLATIONS_SET,
+    PROP_JOINT_TRANSLATIONS,
+
     ////////////////////////////////////////////////////////////////////////////////////////////////////
     // ATTENTION: add new properties to end of list just ABOVE this line
     PROP_AFTER_LAST_ITEM,

libraries/entities/src/EntityScriptingInterface.cpp

@@ -854,3 +854,113 @@ glm::quat EntityScriptingInterface::getAbsoluteJointRotationInObjectFrame(const
         return glm::quat();
     }
 }
+
+bool EntityScriptingInterface::setAbsoluteJointTranslationInObjectFrame(const QUuid& entityID,
+                                                                        int jointIndex, glm::vec3 translation) {
+    if (auto entity = checkForTreeEntityAndTypeMatch(entityID, EntityTypes::Model)) {
+        auto now = usecTimestampNow();
+        auto modelEntity = std::dynamic_pointer_cast<ModelEntityItem>(entity);
+        bool result = modelEntity->setAbsoluteJointTranslationInObjectFrame(jointIndex, translation);
+        if (result) {
+            EntityItemProperties properties;
+            _entityTree->withWriteLock([&] {
+                properties = entity->getProperties();
+                entity->setLastBroadcast(now);
+            });
+
+            properties.setJointTranslationsDirty();
+            properties.setLastEdited(now);
+            queueEntityMessage(PacketType::EntityEdit, entityID, properties);
+            return true;
+        }
+    }
+    return false;
+}
+
+bool EntityScriptingInterface::setAbsoluteJointRotationInObjectFrame(const QUuid& entityID,
+                                                                     int jointIndex, glm::quat rotation) {
+    if (auto entity = checkForTreeEntityAndTypeMatch(entityID, EntityTypes::Model)) {
+        auto now = usecTimestampNow();
+        auto modelEntity = std::dynamic_pointer_cast<ModelEntityItem>(entity);
+        bool result = modelEntity->setAbsoluteJointRotationInObjectFrame(jointIndex, rotation);
+        if (result) {
+            EntityItemProperties properties;
+            _entityTree->withWriteLock([&] {
+                properties = entity->getProperties();
+                entity->setLastBroadcast(now);
+            });
+
+            properties.setJointRotationsDirty();
+            properties.setLastEdited(now);
+            queueEntityMessage(PacketType::EntityEdit, entityID, properties);
+            return true;
+        }
+    }
+    return false;
+}
+
+
+
+bool EntityScriptingInterface::setAbsoluteJointRotationsInObjectFrame(const QUuid& entityID,
+                                                                      const QVector<glm::quat>& rotations) {
+    if (auto entity = checkForTreeEntityAndTypeMatch(entityID, EntityTypes::Model)) {
+        auto now = usecTimestampNow();
+        auto modelEntity = std::dynamic_pointer_cast<ModelEntityItem>(entity);
+
+        bool result = false;
+        for (int index = 0; index < rotations.size(); index++) {
+            result |= modelEntity->setAbsoluteJointRotationInObjectFrame(index, rotations[index]);
+        }
+        if (result) {
+            EntityItemProperties properties;
+            _entityTree->withWriteLock([&] {
+                entity->setLastEdited(now);
+                entity->setLastBroadcast(now);
+                properties = entity->getProperties();
+            });
+
+            properties.setJointRotationsDirty();
+            properties.setLastEdited(now);
+            queueEntityMessage(PacketType::EntityEdit, entityID, properties);
+            return true;
+        }
+    }
+    return false;
+}
+
+
+bool EntityScriptingInterface::setAbsoluteJointTranslationsInObjectFrame(const QUuid& entityID,
+                                                                         const QVector<glm::vec3>& translations) {
+    if (auto entity = checkForTreeEntityAndTypeMatch(entityID, EntityTypes::Model)) {
+        auto now = usecTimestampNow();
+        auto modelEntity = std::dynamic_pointer_cast<ModelEntityItem>(entity);
+
+        bool result = false;
+        for (int index = 0; index < translations.size(); index++) {
+            result |= modelEntity->setAbsoluteJointTranslationInObjectFrame(index, translations[index]);
+        }
+        if (result) {
+            EntityItemProperties properties;
+            _entityTree->withWriteLock([&] {
+                entity->setLastEdited(now);
+                entity->setLastBroadcast(now);
+                properties = entity->getProperties();
+            });
+
+            properties.setJointTranslationsDirty();
+            properties.setLastEdited(now);
+            queueEntityMessage(PacketType::EntityEdit, entityID, properties);
+            return true;
+        }
+    }
+    return false;
+}
+
+
+bool EntityScriptingInterface::setAbsoluteJointsDataInObjectFrame(const QUuid& entityID,
+                                                                  const QVector<glm::quat>& rotations,
+                                                                  const QVector<glm::vec3>& translations) {
+    // for a model with 80 joints, sending both these in one edit packet causes the packet to be too large.
+    return setAbsoluteJointRotationsInObjectFrame(entityID, rotations) ||
+        setAbsoluteJointTranslationsInObjectFrame(entityID, translations);
+}

libraries/entities/src/EntityScriptingInterface.h

@@ -151,6 +151,15 @@ public slots:
 
     Q_INVOKABLE glm::vec3 getAbsoluteJointTranslationInObjectFrame(const QUuid& entityID, int jointIndex);
     Q_INVOKABLE glm::quat getAbsoluteJointRotationInObjectFrame(const QUuid& entityID, int jointIndex);
+    Q_INVOKABLE bool setAbsoluteJointTranslationInObjectFrame(const QUuid& entityID, int jointIndex, glm::vec3 translation);
+    Q_INVOKABLE bool setAbsoluteJointRotationInObjectFrame(const QUuid& entityID, int jointIndex, glm::quat rotation);
+    Q_INVOKABLE bool setAbsoluteJointRotationsInObjectFrame(const QUuid& entityID,
+                                                            const QVector<glm::quat>& rotations);
+    Q_INVOKABLE bool setAbsoluteJointTranslationsInObjectFrame(const QUuid& entityID,
+                                                               const QVector<glm::vec3>& translations);
+    Q_INVOKABLE bool setAbsoluteJointsDataInObjectFrame(const QUuid& entityID,
+                                                        const QVector<glm::quat>& rotations,
+                                                        const QVector<glm::vec3>& translations);
 
 signals:
     void collisionWithEntity(const EntityItemID& idA, const EntityItemID& idB, const Collision& collision);

libraries/entities/src/EntityTree.cpp

@@ -773,7 +773,6 @@ void EntityTree::fixupTerseEditLogging(EntityItemProperties& properties, QList<Q
             changedProperties[index] = QString("parentJointIndex:") + QString::number((int)value);
         }
     }
-
     if (properties.parentIDChanged()) {
         int index = changedProperties.indexOf("parentID");
         if (index >= 0) {
@@ -781,6 +780,35 @@ void EntityTree::fixupTerseEditLogging(EntityItemProperties& properties, QList<Q
             changedProperties[index] = QString("parentID:") + value.toString();
         }
     }
+
+    if (properties.jointRotationsSetChanged()) {
+        int index = changedProperties.indexOf("jointRotationsSet");
+        if (index >= 0) {
+            auto value = properties.getJointRotationsSet().size();
+            changedProperties[index] = QString("jointRotationsSet:") + QString::number((int)value);
+        }
+    }
+    if (properties.jointRotationsChanged()) {
+        int index = changedProperties.indexOf("jointRotations");
+        if (index >= 0) {
+            auto value = properties.getJointRotations().size();
+            changedProperties[index] = QString("jointRotations:") + QString::number((int)value);
+        }
+    }
+    if (properties.jointTranslationsSetChanged()) {
+        int index = changedProperties.indexOf("jointTranslationsSet");
+        if (index >= 0) {
+            auto value = properties.getJointTranslationsSet().size();
+            changedProperties[index] = QString("jointTranslationsSet:") + QString::number((int)value);
+        }
+    }
+    if (properties.jointTranslationsChanged()) {
+        int index = changedProperties.indexOf("jointTranslations");
+        if (index >= 0) {
+            auto value = properties.getJointTranslations().size();
+            changedProperties[index] = QString("jointTranslations:") + QString::number((int)value);
+        }
+    }
 }
 
 int EntityTree::processEditPacketData(ReceivedMessage& message, const unsigned char* editData, int maxLength,

libraries/entities/src/ModelEntityItem.cpp

@@ -50,6 +50,11 @@ EntityItemProperties ModelEntityItem::getProperties(EntityPropertyFlags desiredP
     COPY_ENTITY_PROPERTY_TO_PROPERTIES(glowLevel, getGlowLevel);
     COPY_ENTITY_PROPERTY_TO_PROPERTIES(textures, getTextures);
     COPY_ENTITY_PROPERTY_TO_PROPERTIES(shapeType, getShapeType);
+    COPY_ENTITY_PROPERTY_TO_PROPERTIES(jointRotationsSet, getJointRotationsSet);
+    COPY_ENTITY_PROPERTY_TO_PROPERTIES(jointRotations, getJointRotations);
+    COPY_ENTITY_PROPERTY_TO_PROPERTIES(jointTranslationsSet, getJointTranslationsSet);
+    COPY_ENTITY_PROPERTY_TO_PROPERTIES(jointTranslations, getJointTranslations);
+
     _animationProperties.getProperties(properties);
     return properties;
 }
@@ -63,6 +68,10 @@ bool ModelEntityItem::setProperties(const EntityItemProperties& properties) {
     SET_ENTITY_PROPERTY_FROM_PROPERTIES(compoundShapeURL, setCompoundShapeURL);
     SET_ENTITY_PROPERTY_FROM_PROPERTIES(textures, setTextures);
     SET_ENTITY_PROPERTY_FROM_PROPERTIES(shapeType, updateShapeType);
+    SET_ENTITY_PROPERTY_FROM_PROPERTIES(jointRotationsSet, setJointRotationsSet);
+    SET_ENTITY_PROPERTY_FROM_PROPERTIES(jointRotations, setJointRotations);
+    SET_ENTITY_PROPERTY_FROM_PROPERTIES(jointTranslationsSet, setJointTranslationsSet);
+    SET_ENTITY_PROPERTY_FROM_PROPERTIES(jointTranslations, setJointTranslations);
 
     bool somethingChangedInAnimations = _animationProperties.setProperties(properties);
 
@@ -133,6 +142,11 @@ int ModelEntityItem::readEntitySubclassDataFromBuffer(const unsigned char* data,
         somethingChanged = true;
     }
 
+    READ_ENTITY_PROPERTY(PROP_JOINT_ROTATIONS_SET, QVector<bool>, setJointRotationsSet);
+    READ_ENTITY_PROPERTY(PROP_JOINT_ROTATIONS, QVector<glm::quat>, setJointRotations);
+    READ_ENTITY_PROPERTY(PROP_JOINT_TRANSLATIONS_SET, QVector<bool>, setJointTranslationsSet);
+    READ_ENTITY_PROPERTY(PROP_JOINT_TRANSLATIONS, QVector<glm::vec3>, setJointTranslations);
+
     return bytesRead;
 }
 
@@ -145,6 +159,10 @@ EntityPropertyFlags ModelEntityItem::getEntityProperties(EncodeBitstreamParams&
     requestedProperties += PROP_TEXTURES;
     requestedProperties += PROP_SHAPE_TYPE;
     requestedProperties += _animationProperties.getEntityProperties(params);
+    requestedProperties += PROP_JOINT_ROTATIONS_SET;
+    requestedProperties += PROP_JOINT_ROTATIONS;
+    requestedProperties += PROP_JOINT_TRANSLATIONS_SET;
+    requestedProperties += PROP_JOINT_TRANSLATIONS;
 
     return requestedProperties;
 }
@@ -168,6 +186,11 @@ void ModelEntityItem::appendSubclassData(OctreePacketData* packetData, EncodeBit
         propertyFlags, propertiesDidntFit, propertyCount, appendState);
 
     APPEND_ENTITY_PROPERTY(PROP_SHAPE_TYPE, (uint32_t)getShapeType());
+
+    APPEND_ENTITY_PROPERTY(PROP_JOINT_ROTATIONS_SET, getJointRotationsSet());
+    APPEND_ENTITY_PROPERTY(PROP_JOINT_ROTATIONS, getJointRotations());
+    APPEND_ENTITY_PROPERTY(PROP_JOINT_TRANSLATIONS_SET, getJointTranslationsSet());
+    APPEND_ENTITY_PROPERTY(PROP_JOINT_TRANSLATIONS, getJointTranslations());
 }
 
 
@@ -214,64 +237,6 @@ void ModelEntityItem::mapJoints(const QStringList& modelJointNames) {
     }
 }
 
-void ModelEntityItem::getAnimationFrame(bool& newFrame,
-                                        QVector<glm::quat>& rotationsResult, QVector<glm::vec3>& translationsResult) {
-    newFrame = false;
-
-    if (!hasAnimation() || !_jointMappingCompleted) {
-        rotationsResult = _lastKnownFrameDataRotations;
-        translationsResult = _lastKnownFrameDataTranslations;
-    }
-    AnimationPointer myAnimation = getAnimation(_animationProperties.getURL()); // FIXME: this could be optimized
-    if (myAnimation && myAnimation->isLoaded()) {
-
-        const QVector<FBXAnimationFrame>&  frames = myAnimation->getFramesReference(); // NOTE: getFrames() is too heavy
-        auto& fbxJoints = myAnimation->getGeometry().joints;
-
-        int frameCount = frames.size();
-        if (frameCount > 0) {
-            int animationCurrentFrame = (int)(glm::floor(getAnimationCurrentFrame())) % frameCount;
-            if (animationCurrentFrame < 0 || animationCurrentFrame > frameCount) {
-                animationCurrentFrame = 0;
-            }
-
-            if (animationCurrentFrame != _lastKnownCurrentFrame) {
-                _lastKnownCurrentFrame = animationCurrentFrame;
-                newFrame = true;
-
-                const QVector<glm::quat>& rotations = frames[animationCurrentFrame].rotations;
-                const QVector<glm::vec3>& translations = frames[animationCurrentFrame].translations;
-
-                _lastKnownFrameDataRotations.resize(_jointMapping.size());
-                _lastKnownFrameDataTranslations.resize(_jointMapping.size());
-
-                for (int j = 0; j < _jointMapping.size(); j++) {
-                    int index = _jointMapping[j];
-                    if (index >= 0) {
-                        glm::mat4 translationMat;
-                        if (index < translations.size()) {
-                            translationMat = glm::translate(translations[index]);
-                        }
-                        glm::mat4 rotationMat(glm::mat4::_null);
-                        if (index < rotations.size()) {
-                            rotationMat = glm::mat4_cast(fbxJoints[index].preRotation * rotations[index] * fbxJoints[index].postRotation);
-                        } else {
-                            rotationMat = glm::mat4_cast(fbxJoints[index].preRotation * fbxJoints[index].postRotation);
-                        }
-                        glm::mat4 finalMat = (translationMat * fbxJoints[index].preTransform *
-                                              rotationMat * fbxJoints[index].postTransform);
-                        _lastKnownFrameDataTranslations[j] = extractTranslation(finalMat);
-                        _lastKnownFrameDataRotations[j] = glmExtractRotation(finalMat);
-                    }
-                }
-            }
-        }
-    }
-
-    rotationsResult = _lastKnownFrameDataRotations;
-    translationsResult = _lastKnownFrameDataTranslations;
-}
-
 bool ModelEntityItem::isAnimatingSomething() const {
     return getAnimationIsPlaying() &&
             getAnimationFPS() != 0.0f &&
@@ -414,3 +379,88 @@ void ModelEntityItem::setAnimationFPS(float value) {
 bool ModelEntityItem::shouldBePhysical() const {
     return getShapeType() != SHAPE_TYPE_NONE;
 }
+
+void ModelEntityItem::resizeJointArrays(int newSize) {
+    if (newSize >= 0 && newSize > _absoluteJointRotationsInObjectFrame.size()) {
+        _absoluteJointRotationsInObjectFrame.resize(newSize);
+        _absoluteJointRotationsInObjectFrameSet.resize(newSize);
+        _absoluteJointRotationsInObjectFrameDirty.resize(newSize);
+        _absoluteJointTranslationsInObjectFrame.resize(newSize);
+        _absoluteJointTranslationsInObjectFrameSet.resize(newSize);
+        _absoluteJointTranslationsInObjectFrameDirty.resize(newSize);
+    }
+}
+
+void ModelEntityItem::setJointRotations(const QVector<glm::quat>& rotations) {
+    _jointDataLock.withWriteLock([&] {
+        resizeJointArrays(rotations.size());
+        for (int index = 0; index < rotations.size(); index++) {
+            if (_absoluteJointRotationsInObjectFrameSet[index]) {
+                _absoluteJointRotationsInObjectFrame[index] = rotations[index];
+                _absoluteJointRotationsInObjectFrameDirty[index] = true;
+            }
+        }
+    });
+}
+
+void ModelEntityItem::setJointRotationsSet(const QVector<bool>& rotationsSet) {
+    _jointDataLock.withWriteLock([&] {
+        resizeJointArrays(rotationsSet.size());
+        for (int index = 0; index < rotationsSet.size(); index++) {
+            _absoluteJointRotationsInObjectFrameSet[index] = rotationsSet[index];
+        }
+    });
+}
+
+void ModelEntityItem::setJointTranslations(const QVector<glm::vec3>& translations) {
+    _jointDataLock.withWriteLock([&] {
+        resizeJointArrays(translations.size());
+        for (int index = 0; index < translations.size(); index++) {
+            if (_absoluteJointTranslationsInObjectFrameSet[index]) {
+                _absoluteJointTranslationsInObjectFrame[index] = translations[index];
+                _absoluteJointTranslationsInObjectFrameSet[index] = true;
+            }
+        }
+    });
+}
+
+void ModelEntityItem::setJointTranslationsSet(const QVector<bool>& translationsSet) {
+    _jointDataLock.withWriteLock([&] {
+        resizeJointArrays(translationsSet.size());
+        for (int index = 0; index < translationsSet.size(); index++) {
+            _absoluteJointTranslationsInObjectFrameSet[index] = translationsSet[index];
+        }
+    });
+}
+
+QVector<glm::quat> ModelEntityItem::getJointRotations() const {
+    QVector<glm::quat> result;
+    _jointDataLock.withReadLock([&] {
+        result = _absoluteJointRotationsInObjectFrame;
+    });
+    return result;
+}
+
+QVector<bool> ModelEntityItem::getJointRotationsSet() const {
+    QVector<bool> result;
+    _jointDataLock.withReadLock([&] {
+        result = _absoluteJointRotationsInObjectFrameSet;
+    });
+    return result;
+}
+
+QVector<glm::vec3> ModelEntityItem::getJointTranslations() const {
+    QVector<glm::vec3> result;
+    _jointDataLock.withReadLock([&] {
+        result = _absoluteJointTranslationsInObjectFrame;
+    });
+    return result;
+}
+
+QVector<bool> ModelEntityItem::getJointTranslationsSet() const {
+    QVector<bool> result;
+    _jointDataLock.withReadLock([&] {
+        result = _absoluteJointTranslationsInObjectFrameSet;
+    });
+    return result;
+}

libraries/entities/src/ModelEntityItem.h

@@ -103,7 +103,6 @@ public:
     float getAnimationLastFrame() const { return _animationLoop.getLastFrame(); }
 
     void mapJoints(const QStringList& modelJointNames);
-    void getAnimationFrame(bool& newFrame, QVector<glm::quat>& rotationsResult, QVector<glm::vec3>& translationsResult);
     bool jointsMapped() const { return _jointMappingURL == getAnimationURL() && _jointMappingCompleted; }
 
     bool getAnimationIsPlaying() const { return _animationLoop.getRunning(); }
@@ -121,14 +120,34 @@ public:
     virtual glm::vec3 getJointPosition(int jointIndex) const { return glm::vec3(); }
     virtual glm::quat getJointRotation(int jointIndex) const { return glm::quat(); }
 
+    void setJointRotations(const QVector<glm::quat>& rotations);
+    void setJointRotationsSet(const QVector<bool>& rotationsSet);
+    void setJointTranslations(const QVector<glm::vec3>& translations);
+    void setJointTranslationsSet(const QVector<bool>& translationsSet);
+    QVector<glm::quat> getJointRotations() const;
+    QVector<bool> getJointRotationsSet() const;
+    QVector<glm::vec3> getJointTranslations() const;
+    QVector<bool> getJointTranslationsSet() const;
+
 private:
     void setAnimationSettings(const QString& value); // only called for old bitstream format
 
 protected:
-    QVector<glm::quat> _lastKnownFrameDataRotations;
+    // these are used:
-    QVector<glm::vec3> _lastKnownFrameDataTranslations;
+    // - to bounce joint data from an animation into the model/rig.
+    // - to relay changes from scripts to model/rig.
+    // - to relay between network and model/rig
+    // they aren't currently updated from data in the model/rig, and they don't have a direct effect
+    // on what's rendered.
+    ReadWriteLockable _jointDataLock;
+    QVector<glm::quat> _absoluteJointRotationsInObjectFrame;
+    QVector<bool> _absoluteJointRotationsInObjectFrameSet; // ever set?
+    QVector<bool> _absoluteJointRotationsInObjectFrameDirty; // needs a relay to model/rig?
+    QVector<glm::vec3> _absoluteJointTranslationsInObjectFrame;
+    QVector<bool> _absoluteJointTranslationsInObjectFrameSet; // ever set?
+    QVector<bool> _absoluteJointTranslationsInObjectFrameDirty; // needs a relay to model/rig?
     int _lastKnownCurrentFrame;
-
+    virtual void resizeJointArrays(int newSize = -1);
 
     bool isAnimatingSomething() const;
 
@@ -145,7 +164,7 @@ protected:
 
     // used on client side
     bool _jointMappingCompleted;
-    QVector<int> _jointMapping;
+    QVector<int> _jointMapping; // domain is index into model-joints, range is index into animation-joints
     QString _jointMappingURL;
 
     static AnimationPointer getAnimation(const QString& url);

libraries/networking/src/udt/PacketHeaders.cpp

@@ -41,7 +41,7 @@ PacketVersion versionForPacketType(PacketType packetType) {
         case PacketType::EntityAdd:
         case PacketType::EntityEdit:
         case PacketType::EntityData:
-            return VERSION_ENTITITES_HAVE_QUERY_BOX;
+            return VERSION_MODEL_ENTITIES_JOINTS_ON_WIRE;
         case PacketType::AvatarData:
         case PacketType::BulkAvatarData:
             return static_cast<PacketVersion>(AvatarMixerPacketVersion::SoftAttachmentSupport);

libraries/networking/src/udt/PacketHeaders.h

@@ -161,8 +161,9 @@ const PacketVersion VERSION_ENTITIES_KEYLIGHT_PROPERTIES_GROUP_BIS = 48;
 const PacketVersion VERSION_ENTITIES_PARTICLES_ADDITIVE_BLENDING = 49;
 const PacketVersion VERSION_ENTITIES_POLYLINE_TEXTURE = 50;
 const PacketVersion VERSION_ENTITIES_HAVE_PARENTS = 51;
-const PacketVersion VERSION_ENTITITES_REMOVED_START_AUTOMATICALLY_FROM_ANIMATION_PROPERTY_GROUP = 52;
+const PacketVersion VERSION_ENTITIES_REMOVED_START_AUTOMATICALLY_FROM_ANIMATION_PROPERTY_GROUP = 52;
-const PacketVersion VERSION_ENTITITES_HAVE_QUERY_BOX = 53;
+const PacketVersion VERSION_MODEL_ENTITIES_JOINTS_ON_WIRE = 53;
+const PacketVersion VERSION_ENTITITES_HAVE_QUERY_BOX = 54;
 
 enum class AvatarMixerPacketVersion : PacketVersion {
     TranslationSupport = 17,

libraries/octree/src/OctreePacketData.cpp

@@ -14,6 +14,7 @@
 
 #include "OctreeLogging.h"
 #include "OctreePacketData.h"
+#include "NumericalConstants.h"
 
 bool OctreePacketData::_debug = false;
 AtomicUIntStat OctreePacketData::_totalBytesOfOctalCodes { 0 };
@@ -397,6 +398,28 @@ bool OctreePacketData::appendValue(const QVector<glm::vec3>& value) {
     return success;
 }
 
+bool OctreePacketData::appendValue(const QVector<glm::quat>& value) {
+    uint16_t qVecSize = value.size();
+    bool success = appendValue(qVecSize);
+
+    if (success) {
+        QByteArray dataByteArray(udt::MAX_PACKET_SIZE, 0);
+        unsigned char* start = reinterpret_cast<unsigned char*>(dataByteArray.data());
+        unsigned char* destinationBuffer = start;
+        for (int index = 0; index < value.size(); index++) {
+            destinationBuffer += packOrientationQuatToBytes(destinationBuffer, value[index]);
+        }
+        int quatsSize = destinationBuffer - start;
+        success = append(start, quatsSize);
+        if (success) {
+            _bytesOfValues += quatsSize;
+            _totalBytesOfValues += quatsSize;
+        }
+    }
+
+    return success;
+}
+
 bool OctreePacketData::appendValue(const QVector<float>& value) {
     uint16_t qVecSize = value.size();
     bool success = appendValue(qVecSize);
@@ -410,6 +433,34 @@ bool OctreePacketData::appendValue(const QVector<float>& value) {
     return success;
 }
 
+bool OctreePacketData::appendValue(const QVector<bool>& value) {
+    uint16_t qVecSize = value.size();
+    bool success = appendValue(qVecSize);
+
+    if (success) {
+        QByteArray dataByteArray(udt::MAX_PACKET_SIZE, 0);
+        unsigned char* start = reinterpret_cast<unsigned char*>(dataByteArray.data());
+        unsigned char* destinationBuffer = start;
+        int bit = 0;
+        for (int index = 0; index < value.size(); index++) {
+            if (value[index]) {
+                (*destinationBuffer) |= (1 << bit);
+            }
+            if (++bit == BITS_IN_BYTE) {
+                destinationBuffer++;
+                bit = 0;
+            }
+        }
+        int boolsSize = destinationBuffer - start;
+        success = append(start, boolsSize);
+        if (success) {
+            _bytesOfValues += boolsSize;
+            _totalBytesOfValues += boolsSize;
+        }
+    }
+    return success;
+}
+
 bool OctreePacketData::appendValue(const glm::quat& value) {
     const size_t VALUES_PER_QUAT = 4;
     const size_t PACKED_QUAT_SIZE = sizeof(uint16_t) * VALUES_PER_QUAT;
@@ -637,6 +688,20 @@ int OctreePacketData::unpackDataFromBytes(const unsigned char *dataBytes, QVecto
     return sizeof(uint16_t) + length * sizeof(glm::vec3);
 }
 
+int OctreePacketData::unpackDataFromBytes(const unsigned char *dataBytes, QVector<glm::quat>& result) {
+    uint16_t length;
+    memcpy(&length, dataBytes, sizeof(uint16_t));
+    dataBytes += sizeof(length);
+    result.resize(length);
+
+    const unsigned char *start = dataBytes;
+    for (int i = 0; i < length; i++) {
+        dataBytes += unpackOrientationQuatFromBytes(dataBytes, result[i]);
+    }
+
+    return (dataBytes - start) + sizeof(uint16_t);
+}
+
 int OctreePacketData::unpackDataFromBytes(const unsigned char* dataBytes, QVector<float>& result) {
     uint16_t length;
     memcpy(&length, dataBytes, sizeof(uint16_t));
@@ -646,7 +711,27 @@ int OctreePacketData::unpackDataFromBytes(const unsigned char* dataBytes, QVecto
     return sizeof(uint16_t) + length * sizeof(float);
 }
 
-int OctreePacketData::unpackDataFromBytes(const unsigned char* dataBytes, QByteArray& result) { 
+int OctreePacketData::unpackDataFromBytes(const unsigned char* dataBytes, QVector<bool>& result) {
+    uint16_t length;
+    memcpy(&length, dataBytes, sizeof(uint16_t));
+    dataBytes += sizeof(length);
+    result.resize(length);
+
+    int bit = 0;
+    unsigned char current = 0;
+    const unsigned char *start = dataBytes;
+    for (int i = 0; i < length; i ++) {
+        if (bit == 0) {
+            current = *dataBytes++;
+        }
+        result[i] = (bool)(current & (1 << bit));
+        bit = (bit + 1) % BITS_IN_BYTE;
+    }
+
+    return (dataBytes - start) + sizeof(uint16_t);
+}
+
+int OctreePacketData::unpackDataFromBytes(const unsigned char* dataBytes, QByteArray& result) {
     uint16_t length;
     memcpy(&length, dataBytes, sizeof(length));
     dataBytes += sizeof(length);

libraries/octree/src/OctreePacketData.h

@@ -165,19 +165,25 @@ public:
 
     /// appends a non-position vector to the end of the stream, may fail if new data stream is too long to fit in packet
     bool appendValue(const glm::vec3& value);
-    
+
-    //appends a QVector of vec3's to the end of the stream, may fail if new data stream is too long to fit in packet
+    /// appends a QVector of vec3s to the end of the stream, may fail if new data stream is too long to fit in packet
     bool appendValue(const QVector<glm::vec3>& value);
-    
+
-    //appends a QVector of floats to the end of the stream, may fail if new data stream is too long to fit in packet
+    /// appends a QVector of quats to the end of the stream, may fail if new data stream is too long to fit in packet
+    bool appendValue(const QVector<glm::quat>& value);
+
+    /// appends a QVector of floats to the end of the stream, may fail if new data stream is too long to fit in packet
     bool appendValue(const QVector<float>& value);
 
+    /// appends a QVector of bools to the end of the stream, may fail if new data stream is too long to fit in packet
+    bool appendValue(const QVector<bool>& value);
+
     /// appends a packed quat to the end of the stream, may fail if new data stream is too long to fit in packet
     bool appendValue(const glm::quat& value);
 
     /// appends a bool value to the end of the stream, may fail if new data stream is too long to fit in packet
     bool appendValue(bool value);
-    
+
     /// appends a string value to the end of the stream, may fail if new data stream is too long to fit in packet
     bool appendValue(const QString& string);
 
@@ -254,7 +260,9 @@ public:
     static int unpackDataFromBytes(const unsigned char* dataBytes, QUuid& result);
     static int unpackDataFromBytes(const unsigned char* dataBytes, xColor& result);
     static int unpackDataFromBytes(const unsigned char* dataBytes, QVector<glm::vec3>& result);
+    static int unpackDataFromBytes(const unsigned char* dataBytes, QVector<glm::quat>& result);
     static int unpackDataFromBytes(const unsigned char* dataBytes, QVector<float>& result);
+    static int unpackDataFromBytes(const unsigned char* dataBytes, QVector<bool>& result);
     static int unpackDataFromBytes(const unsigned char* dataBytes, QByteArray& result);
     static int unpackDataFromBytes(const unsigned char* dataBytes, AACube& result);
 

libraries/shared/src/RegisteredMetaTypes.cpp

@@ -20,6 +20,8 @@
 static int vec4MetaTypeId = qRegisterMetaType<glm::vec4>();
 static int vec3MetaTypeId = qRegisterMetaType<glm::vec3>();
 static int qVectorVec3MetaTypeId = qRegisterMetaType<QVector<glm::vec3>>();
+static int qVectorQuatMetaTypeId = qRegisterMetaType<QVector<glm::quat>>();
+static int qVectorBoolMetaTypeId = qRegisterMetaType<QVector<bool>>();
 static int vec2MetaTypeId = qRegisterMetaType<glm::vec2>();
 static int quatMetaTypeId = qRegisterMetaType<glm::quat>();
 static int xColorMetaTypeId = qRegisterMetaType<xColor>();
@@ -31,6 +33,8 @@ void registerMetaTypes(QScriptEngine* engine) {
     qScriptRegisterMetaType(engine, vec4toScriptValue, vec4FromScriptValue);
     qScriptRegisterMetaType(engine, vec3toScriptValue, vec3FromScriptValue);
     qScriptRegisterMetaType(engine, qVectorVec3ToScriptValue, qVectorVec3FromScriptValue);
+    qScriptRegisterMetaType(engine, qVectorQuatToScriptValue, qVectorQuatFromScriptValue);
+    qScriptRegisterMetaType(engine, qVectorBoolToScriptValue, qVectorBoolFromScriptValue);
     qScriptRegisterMetaType(engine, qVectorFloatToScriptValue, qVectorFloatFromScriptValue);
     qScriptRegisterMetaType(engine, vec2toScriptValue, vec2FromScriptValue);
     qScriptRegisterMetaType(engine, quatToScriptValue, quatFromScriptValue);
@@ -87,6 +91,42 @@ QScriptValue qVectorVec3ToScriptValue(QScriptEngine* engine, const QVector<glm::
     return array;
 }
 
+QScriptValue quatToScriptValue(QScriptEngine* engine, const glm::quat &quat) {
+    QScriptValue obj = engine->newObject();
+    if (quat.x != quat.x || quat.y != quat.y || quat.z != quat.z || quat.w != quat.w) {
+        // if quat contains a NaN don't try to convert it
+        return obj;
+    }
+    obj.setProperty("x", quat.x);
+    obj.setProperty("y", quat.y);
+    obj.setProperty("z", quat.z);
+    obj.setProperty("w", quat.w);
+    return obj;
+}
+
+void quatFromScriptValue(const QScriptValue &object, glm::quat &quat) {
+    quat.x = object.property("x").toVariant().toFloat();
+    quat.y = object.property("y").toVariant().toFloat();
+    quat.z = object.property("z").toVariant().toFloat();
+    quat.w = object.property("w").toVariant().toFloat();
+}
+
+QScriptValue qVectorQuatToScriptValue(QScriptEngine* engine, const QVector<glm::quat>& vector) {
+    QScriptValue array = engine->newArray();
+    for (int i = 0; i < vector.size(); i++) {
+        array.setProperty(i, quatToScriptValue(engine, vector.at(i)));
+    }
+    return array;
+}
+
+QScriptValue qVectorBoolToScriptValue(QScriptEngine* engine, const QVector<bool>& vector) {
+    QScriptValue array = engine->newArray();
+    for (int i = 0; i < vector.size(); i++) {
+        array.setProperty(i, vector.at(i));
+    }
+    return array;
+}
+
 QVector<float> qVectorFloatFromScriptValue(const QScriptValue& array) {
     if(!array.isArray()) {
         return QVector<float>();
@@ -149,7 +189,7 @@ QVector<glm::vec3> qVectorVec3FromScriptValue(const QScriptValue& array){
 
 void qVectorVec3FromScriptValue(const QScriptValue& array, QVector<glm::vec3>& vector ) {
     int length = array.property("length").toInteger();
-    
+
     for (int i = 0; i < length; i++) {
         glm::vec3 newVec3 = glm::vec3();
         vec3FromScriptValue(array.property(i), newVec3);
@@ -157,6 +197,46 @@ void qVectorVec3FromScriptValue(const QScriptValue& array, QVector<glm::vec3>& v
     }
 }
 
+QVector<glm::quat> qVectorQuatFromScriptValue(const QScriptValue& array){
+    QVector<glm::quat> newVector;
+    int length = array.property("length").toInteger();
+
+    for (int i = 0; i < length; i++) {
+        glm::quat newQuat = glm::quat();
+        quatFromScriptValue(array.property(i), newQuat);
+        newVector << newQuat;
+    }
+    return newVector;
+}
+
+void qVectorQuatFromScriptValue(const QScriptValue& array, QVector<glm::quat>& vector ) {
+    int length = array.property("length").toInteger();
+
+    for (int i = 0; i < length; i++) {
+        glm::quat newQuat = glm::quat();
+        quatFromScriptValue(array.property(i), newQuat);
+        vector << newQuat;
+    }
+}
+
+QVector<bool> qVectorBoolFromScriptValue(const QScriptValue& array){
+    QVector<bool> newVector;
+    int length = array.property("length").toInteger();
+
+    for (int i = 0; i < length; i++) {
+        newVector << array.property(i).toBool();
+    }
+    return newVector;
+}
+
+void qVectorBoolFromScriptValue(const QScriptValue& array, QVector<bool>& vector ) {
+    int length = array.property("length").toInteger();
+
+    for (int i = 0; i < length; i++) {
+        vector << array.property(i).toBool();
+    }
+}
+
 QScriptValue vec2toScriptValue(QScriptEngine* engine, const glm::vec2 &vec2) {
     QScriptValue obj = engine->newObject();
     obj.setProperty("x", vec2.x);
@@ -169,22 +249,6 @@ void vec2FromScriptValue(const QScriptValue &object, glm::vec2 &vec2) {
     vec2.y = object.property("y").toVariant().toFloat();
 }
 
-QScriptValue quatToScriptValue(QScriptEngine* engine, const glm::quat& quat) {
-    QScriptValue obj = engine->newObject();
-    obj.setProperty("x", quat.x);
-    obj.setProperty("y", quat.y);
-    obj.setProperty("z", quat.z);
-    obj.setProperty("w", quat.w);
-    return obj;
-}
-
-void quatFromScriptValue(const QScriptValue &object, glm::quat& quat) {
-    quat.x = object.property("x").toVariant().toFloat();
-    quat.y = object.property("y").toVariant().toFloat();
-    quat.z = object.property("z").toVariant().toFloat();
-    quat.w = object.property("w").toVariant().toFloat();
-}
-
 QScriptValue qRectToScriptValue(QScriptEngine* engine, const QRect& rect) {
     QScriptValue obj = engine->newObject();
     obj.setProperty("x", rect.x());

libraries/shared/src/RegisteredMetaTypes.h

@@ -63,6 +63,14 @@ QScriptValue qVectorVec3ToScriptValue(QScriptEngine* engine, const QVector<glm::
 void qVectorVec3FromScriptValue(const QScriptValue& array, QVector<glm::vec3>& vector);
 QVector<glm::vec3> qVectorVec3FromScriptValue(const QScriptValue& array);
 
+QScriptValue qVectorQuatToScriptValue(QScriptEngine* engine, const QVector<glm::quat>& vector);
+void qVectorQuatFromScriptValue(const QScriptValue& array, QVector<glm::quat>& vector);
+QVector<glm::quat> qVectorQuatFromScriptValue(const QScriptValue& array);
+
+QScriptValue qVectorBoolToScriptValue(QScriptEngine* engine, const QVector<bool>& vector);
+void qVectorBoolFromScriptValue(const QScriptValue& array, QVector<bool>& vector);
+QVector<bool> qVectorBoolFromScriptValue(const QScriptValue& array);
+
 QScriptValue qVectorFloatToScriptValue(QScriptEngine* engine, const QVector<float>& vector);
 void qVectorFloatFromScriptValue(const QScriptValue& array, QVector<float>& vector);
 QVector<float> qVectorFloatFromScriptValue(const QScriptValue& array);

libraries/shared/src/SpatiallyNestable.h

@@ -104,6 +104,8 @@ public:
     virtual const Transform getAbsoluteJointTransformInObjectFrame(int jointIndex) const;
     virtual glm::quat getAbsoluteJointRotationInObjectFrame(int index) const = 0;
     virtual glm::vec3 getAbsoluteJointTranslationInObjectFrame(int index) const = 0;
+    virtual bool setAbsoluteJointRotationInObjectFrame(int index, const glm::quat& rotation) = 0;
+    virtual bool setAbsoluteJointTranslationInObjectFrame(int index, const glm::vec3& translation) = 0;
 
     SpatiallyNestablePointer getThisPointer() const;