Merge pull request #9257 from ZappoMan/tuneAvatarInfo

avatar mixer bandwidth optimization

assignment-client/src/Agent.cpp

@@ -502,8 +502,8 @@ void Agent::processAgentAvatar() {
     if (!_scriptEngine->isFinished() && _isAvatar) {
         auto scriptedAvatar = DependencyManager::get<ScriptableAvatar>();
 
-        QByteArray avatarByteArray = scriptedAvatar->toByteArray((randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO) 
+        AvatarData::AvatarDataDetail dataDetail = (randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO) ? AvatarData::SendAllData : AvatarData::CullSmallData;
-                                                                    ? AvatarData::SendAllData : AvatarData::CullSmallData);
+        QByteArray avatarByteArray = scriptedAvatar->toByteArray(dataDetail, 0, scriptedAvatar->getLastSentJointData());
         scriptedAvatar->doneEncoding(true);
 
         static AvatarDataSequenceNumber sequenceNumber = 0;

assignment-client/src/assets/AssetServer.cpp

@@ -35,7 +35,9 @@
 static const uint8_t MIN_CORES_FOR_MULTICORE = 4;
 static const uint8_t CPU_AFFINITY_COUNT_HIGH = 2;
 static const uint8_t CPU_AFFINITY_COUNT_LOW = 1;
+#ifdef Q_OS_WIN
 static const int INTERFACE_RUNNING_CHECK_FREQUENCY_MS = 1000;
+#endif
 
 const QString ASSET_SERVER_LOGGING_TARGET_NAME = "asset-server";
 

assignment-client/src/avatars/AvatarMixer.cpp

@@ -423,12 +423,17 @@ void AvatarMixer::broadcastAvatarData() {
                         nodeData->incrementAvatarOutOfView();
                     } else {
                         detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO
-                                        ? AvatarData::SendAllData : AvatarData::IncludeSmallData;
+                                        ? AvatarData::SendAllData : AvatarData::CullSmallData;
                         nodeData->incrementAvatarInView();
                     }
 
                     numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122());
-                    numAvatarDataBytes += avatarPacketList->write(otherAvatar.toByteArray(detail));
+                    auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID());
+                    QVector<JointData>& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID());
+                    bool distanceAdjust = true;
+                    glm::vec3 viewerPosition = nodeData->getPosition();
+                    auto bytes = otherAvatar.toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, distanceAdjust, viewerPosition, &lastSentJointsForOther);
+                    numAvatarDataBytes += avatarPacketList->write(bytes);
 
                     avatarPacketList->endSegment();
             });

assignment-client/src/avatars/AvatarMixerClientData.h

@@ -104,6 +104,22 @@ public:
     bool getRequestsDomainListData() { return _requestsDomainListData; }
     void setRequestsDomainListData(bool requesting) { _requestsDomainListData = requesting; }
 
+    quint64 getLastOtherAvatarEncodeTime(QUuid otherAvatar) {
+        quint64 result = 0;
+        if (_lastOtherAvatarEncodeTime.find(otherAvatar) != _lastOtherAvatarEncodeTime.end()) {
+            result = _lastOtherAvatarEncodeTime[otherAvatar];
+        }
+        _lastOtherAvatarEncodeTime[otherAvatar] = usecTimestampNow();
+        return result;
+    }
+
+    QVector<JointData>& getLastOtherAvatarSentJoints(QUuid otherAvatar) {
+        _lastOtherAvatarSentJoints[otherAvatar].resize(_avatar->getJointCount());
+        return _lastOtherAvatarSentJoints[otherAvatar];
+    }
+
+    
+
 private:
     AvatarSharedPointer _avatar { new AvatarData() };
 
@@ -111,6 +127,11 @@ private:
     std::unordered_map<QUuid, uint16_t> _lastBroadcastSequenceNumbers;
     std::unordered_set<QUuid> _hasReceivedFirstPacketsFrom;
 
+    // this is a map of the last time we encoded an "other" avatar for
+    // sending to "this" node
+    std::unordered_map<QUuid, quint64> _lastOtherAvatarEncodeTime;
+    std::unordered_map<QUuid, QVector<JointData>> _lastOtherAvatarSentJoints;
+
     HRCTime _identityChangeTimestamp;
     bool _avatarSessionDisplayNameMustChange{ false };
 

assignment-client/src/avatars/ScriptableAvatar.cpp

@@ -14,6 +14,13 @@
 #include <GLMHelpers.h>
 #include "ScriptableAvatar.h"
 
+QByteArray ScriptableAvatar::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
+                                    bool distanceAdjust, glm::vec3 viewerPosition, QVector<JointData>* sentJointDataOut) {
+    _globalPosition = getPosition();
+    return AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut);
+}
+
+
 // hold and priority unused but kept so that client side JS can run.
 void ScriptableAvatar::startAnimation(const QString& url, float fps, float priority,
                               bool loop, bool hold, float firstFrame, float lastFrame, const QStringList& maskedJoints) {

assignment-client/src/avatars/ScriptableAvatar.h

@@ -27,6 +27,10 @@ public:
     Q_INVOKABLE void stopAnimation();
     Q_INVOKABLE AnimationDetails getAnimationDetails();
     virtual void setSkeletonModelURL(const QUrl& skeletonModelURL) override;
+
+    virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
+                        bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector<JointData>* sentJointDataOut = nullptr) override;
+
     
 private slots:
     void update(float deltatime);

interface/src/Application.cpp

@@ -5173,6 +5173,7 @@ void Application::nodeAdded(SharedNodePointer node) const {
     if (node->getType() == NodeType::AvatarMixer) {
         // new avatar mixer, send off our identity packet right away
         getMyAvatar()->sendIdentityPacket();
+        getMyAvatar()->resetLastSent();
     }
 }
 

interface/src/avatar/AvatarManager.cpp

@@ -132,6 +132,11 @@ void AvatarManager::updateMyAvatar(float deltaTime) {
 
 Q_LOGGING_CATEGORY(trace_simulation_avatar, "trace.simulation.avatar");
 
+float AvatarManager::getAvatarDataRate(const QUuid& sessionID, const QString& rateName) {
+    auto avatar = getAvatarBySessionID(sessionID);
+    return avatar->getDataRate(rateName);
+}
+
 class AvatarPriority {
 public:
     AvatarPriority(AvatarSharedPointer a, float p) : avatar(a), priority(p) {}

interface/src/avatar/AvatarManager.h

@@ -69,6 +69,7 @@ public:
     void handleOutgoingChanges(const VectorOfMotionStates& motionStates);
     void handleCollisionEvents(const CollisionEvents& collisionEvents);
 
+    Q_INVOKABLE float getAvatarDataRate(const QUuid& sessionID, const QString& rateName = QString(""));
     Q_INVOKABLE RayToAvatarIntersectionResult findRayIntersection(const PickRay& ray,
                                                                   const QScriptValue& avatarIdsToInclude = QScriptValue(),
                                                                   const QScriptValue& avatarIdsToDiscard = QScriptValue());

interface/src/avatar/MyAvatar.cpp

@@ -226,23 +226,24 @@ void MyAvatar::simulateAttachments(float deltaTime) {
     // don't update attachments here, do it in harvestResultsFromPhysicsSimulation()
 }
 
-QByteArray MyAvatar::toByteArray(AvatarDataDetail dataDetail) {
+QByteArray MyAvatar::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
+                        bool distanceAdjust, glm::vec3 viewerPosition, QVector<JointData>* sentJointDataOut) {
     CameraMode mode = qApp->getCamera()->getMode();
     _globalPosition = getPosition();
-    _globalBoundingBoxCorner.x = _characterController.getCapsuleRadius();
+    _globalBoundingBoxDimensions.x = _characterController.getCapsuleRadius();
-    _globalBoundingBoxCorner.y = _characterController.getCapsuleHalfHeight();
+    _globalBoundingBoxDimensions.y = _characterController.getCapsuleHalfHeight();
-    _globalBoundingBoxCorner.z = _characterController.getCapsuleRadius();
+    _globalBoundingBoxDimensions.z = _characterController.getCapsuleRadius();
-    _globalBoundingBoxCorner += _characterController.getCapsuleLocalOffset();
+    _globalBoundingBoxOffset = _characterController.getCapsuleLocalOffset();
     if (mode == CAMERA_MODE_THIRD_PERSON || mode == CAMERA_MODE_INDEPENDENT) {
         // fake the avatar position that is sent up to the AvatarMixer
         glm::vec3 oldPosition = getPosition();
         setPosition(getSkeletonPosition());
-        QByteArray array = AvatarData::toByteArray(dataDetail);
+        QByteArray array = AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut);
         // copy the correct position back
         setPosition(oldPosition);
         return array;
     }
-    return AvatarData::toByteArray(dataDetail);
+    return AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut);
 }
 
 void MyAvatar::centerBody() {

interface/src/avatar/MyAvatar.h

@@ -333,7 +333,11 @@ private:
 
     glm::vec3 getWorldBodyPosition() const;
     glm::quat getWorldBodyOrientation() const;
-    QByteArray toByteArray(AvatarDataDetail dataDetail) override;
+
+
+    virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
+                            bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector<JointData>* sentJointDataOut = nullptr) override;
+
     void simulate(float deltaTime);
     void updateFromTrackers(float deltaTime);
     virtual void render(RenderArgs* renderArgs, const glm::vec3& cameraPositio) override;

libraries/avatars/src/AvatarData.h

@@ -56,6 +56,7 @@ typedef unsigned long long quint64;
 #include <Packed.h>
 #include <ThreadSafeValueCache.h>
 #include <SharedUtil.h>
+#include <shared/RateCounter.h>
 
 #include "AABox.h"
 #include "HeadData.h"
@@ -99,6 +100,7 @@ const int IS_EYE_TRACKER_CONNECTED = 5; // 6th bit (was CHAT_CIRCLING)
 const int HAS_REFERENTIAL = 6; // 7th bit
 const int HAND_STATE_FINGER_POINTING_BIT = 7; // 8th bit
 
+
 const char HAND_STATE_NULL = 0;
 const char LEFT_HAND_POINTING_FLAG = 1;
 const char RIGHT_HAND_POINTING_FLAG = 2;
@@ -108,6 +110,131 @@ const char IS_FINGER_POINTING_FLAG = 4;
 // before the "header" structure
 const char AVATARDATA_FLAGS_MINIMUM = 0;
 
+using SmallFloat = uint16_t; // a compressed float with less precision, user defined radix
+
+namespace AvatarDataPacket {
+
+    // NOTE: every time AvatarData is sent from mixer to client, it also includes the GUIID for the session
+    // this is 16bytes of data at 45hz that's 5.76kbps
+    // it might be nice to use a dictionary to compress that
+
+    // Packet State Flags - we store the details about the existence of other records in this bitset:
+    // AvatarGlobalPosition, Avatar Faceshift, eye tracking, and existence of
+    using HasFlags = uint16_t;
+    const HasFlags PACKET_HAS_AVATAR_GLOBAL_POSITION = 1U << 0;
+    const HasFlags PACKET_HAS_AVATAR_BOUNDING_BOX    = 1U << 1;
+    const HasFlags PACKET_HAS_AVATAR_ORIENTATION     = 1U << 2;
+    const HasFlags PACKET_HAS_AVATAR_SCALE           = 1U << 3;
+    const HasFlags PACKET_HAS_LOOK_AT_POSITION       = 1U << 4;
+    const HasFlags PACKET_HAS_AUDIO_LOUDNESS         = 1U << 5;
+    const HasFlags PACKET_HAS_SENSOR_TO_WORLD_MATRIX = 1U << 6;
+    const HasFlags PACKET_HAS_ADDITIONAL_FLAGS       = 1U << 7;
+    const HasFlags PACKET_HAS_PARENT_INFO            = 1U << 8;
+    const HasFlags PACKET_HAS_AVATAR_LOCAL_POSITION  = 1U << 9;
+    const HasFlags PACKET_HAS_FACE_TRACKER_INFO      = 1U << 10;
+    const HasFlags PACKET_HAS_JOINT_DATA             = 1U << 11;
+
+    // NOTE: AvatarDataPackets start with a uint16_t sequence number that is not reflected in the Header structure.
+
+    PACKED_BEGIN struct Header {
+        HasFlags packetHasFlags;        // state flags, indicated which additional records are included in the packet
+    } PACKED_END;
+    const size_t HEADER_SIZE = 2;
+
+    PACKED_BEGIN struct AvatarGlobalPosition {
+        float globalPosition[3];          // avatar's position
+    } PACKED_END;
+    const size_t AVATAR_GLOBAL_POSITION_SIZE = 12;
+
+    PACKED_BEGIN struct AvatarBoundingBox {
+        float avatarDimensions[3];        // avatar's bounding box in world space units, but relative to the position.
+        float boundOriginOffset[3];       // offset from the position of the avatar to the origin of the bounding box
+    } PACKED_END;
+    const size_t AVATAR_BOUNDING_BOX_SIZE = 24;
+
+
+    using SixByteQuat = uint8_t[6];
+    PACKED_BEGIN struct AvatarOrientation {
+        SixByteQuat avatarOrientation;      // encodeded and compressed by packOrientationQuatToSixBytes()
+    } PACKED_END;
+    const size_t AVATAR_ORIENTATION_SIZE = 6;
+
+    PACKED_BEGIN struct AvatarScale {
+        SmallFloat scale;                 // avatar's scale, compressed by packFloatRatioToTwoByte() 
+    } PACKED_END;
+    const size_t AVATAR_SCALE_SIZE = 2;
+
+    PACKED_BEGIN struct LookAtPosition {
+        float lookAtPosition[3];          // world space position that eyes are focusing on.
+                                          // FIXME - unless the person has an eye tracker, this is simulated... 
+                                          //    a) maybe we can just have the client calculate this
+                                          //    b) at distance this will be hard to discern and can likely be 
+                                          //       descimated or dropped completely
+                                          //
+                                          // POTENTIAL SAVINGS - 12 bytes
+    } PACKED_END;
+    const size_t LOOK_AT_POSITION_SIZE = 12;
+
+    PACKED_BEGIN struct AudioLoudness {
+        uint8_t audioLoudness;            // current loudness of microphone compressed with packFloatGainToByte()
+    } PACKED_END;
+    const size_t AUDIO_LOUDNESS_SIZE = 1;
+
+    PACKED_BEGIN struct SensorToWorldMatrix {
+        // FIXME - these 20 bytes are only used by viewers if my avatar has "attachments"
+        // we could save these bytes if no attachments are active.
+        //
+        // POTENTIAL SAVINGS - 20 bytes
+
+        SixByteQuat sensorToWorldQuat;     // 6 byte compressed quaternion part of sensor to world matrix
+        uint16_t sensorToWorldScale;      // uniform scale of sensor to world matrix
+        float sensorToWorldTrans[3];      // fourth column of sensor to world matrix
+                                          // FIXME - sensorToWorldTrans might be able to be better compressed if it was
+                                          // relative to the avatar position.
+    } PACKED_END;
+    const size_t SENSOR_TO_WORLD_SIZE = 20;
+
+    PACKED_BEGIN struct AdditionalFlags {
+        uint8_t flags;                    // additional flags: hand state, key state, eye tracking
+    } PACKED_END;
+    const size_t ADDITIONAL_FLAGS_SIZE = 1;
+
+    // only present if HAS_REFERENTIAL flag is set in AvatarInfo.flags
+    PACKED_BEGIN struct ParentInfo {
+        uint8_t parentUUID[16];       // rfc 4122 encoded
+        uint16_t parentJointIndex;
+    } PACKED_END;
+    const size_t PARENT_INFO_SIZE = 18;
+
+    // will only ever be included if the avatar has a parent but can change independent of changes to parent info
+    // and so we keep it a separate record
+    PACKED_BEGIN struct AvatarLocalPosition {
+        float localPosition[3];           // parent frame translation of the avatar
+    } PACKED_END;
+    const size_t AVATAR_LOCAL_POSITION_SIZE = 12;
+
+    // only present if IS_FACESHIFT_CONNECTED flag is set in AvatarInfo.flags
+    PACKED_BEGIN struct FaceTrackerInfo {
+        float leftEyeBlink;
+        float rightEyeBlink;
+        float averageLoudness;
+        float browAudioLift;
+        uint8_t numBlendshapeCoefficients;
+        // float blendshapeCoefficients[numBlendshapeCoefficients];
+    } PACKED_END;
+    const size_t FACE_TRACKER_INFO_SIZE = 17;
+
+    // variable length structure follows
+    /*
+    struct JointData {
+        uint8_t numJoints;
+        uint8_t rotationValidityBits[ceil(numJoints / 8)];     // one bit per joint, if true then a compressed rotation follows.
+        SixByteQuat rotation[numValidRotations];               // encodeded and compressed by packOrientationQuatToSixBytes()
+        uint8_t translationValidityBits[ceil(numJoints / 8)];  // one bit per joint, if true then a compressed translation follows.
+        SixByteTrans translation[numValidTranslations];        // encodeded and compressed by packFloatVec3ToSignedTwoByteFixed()
+    };
+    */
+}
 
 static const float MAX_AVATAR_SCALE = 1000.0f;
 static const float MIN_AVATAR_SCALE = .005f;
@@ -125,6 +252,16 @@ const float AVATAR_SEND_FULL_UPDATE_RATIO = 0.02f;
 const float AVATAR_MIN_ROTATION_DOT = 0.9999999f;
 const float AVATAR_MIN_TRANSLATION = 0.0001f;
 
+const float ROTATION_CHANGE_15D = 0.9914449f;
+const float ROTATION_CHANGE_45D = 0.9238795f;
+const float ROTATION_CHANGE_90D = 0.7071068f;
+const float ROTATION_CHANGE_179D = 0.0087266f;
+
+const float AVATAR_DISTANCE_LEVEL_1 = 10.0f;
+const float AVATAR_DISTANCE_LEVEL_2 = 100.0f;
+const float AVATAR_DISTANCE_LEVEL_3 = 1000.0f;
+const float AVATAR_DISTANCE_LEVEL_4 = 10000.0f;
+
 
 // Where one's own Avatar begins in the world (will be overwritten if avatar data file is found).
 // This is the start location in the Sandbox (xyz: 6270, 211, 6000).
@@ -214,7 +351,9 @@ public:
         SendAllData
     } AvatarDataDetail;
 
-    virtual QByteArray toByteArray(AvatarDataDetail dataDetail);
+    virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
+                        bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector<JointData>* sentJointDataOut = nullptr);
+
     virtual void doneEncoding(bool cullSmallChanges);
 
     /// \return true if an error should be logged
@@ -265,10 +404,11 @@ public:
     virtual void setTargetScale(float targetScale);
 
     float getDomainLimitedScale() const { return glm::clamp(_targetScale, _domainMinimumScale, _domainMaximumScale); }
+
     void setDomainMinimumScale(float domainMinimumScale)
-        { _domainMinimumScale = glm::clamp(domainMinimumScale, MIN_AVATAR_SCALE, MAX_AVATAR_SCALE); }
+        { _domainMinimumScale = glm::clamp(domainMinimumScale, MIN_AVATAR_SCALE, MAX_AVATAR_SCALE); _scaleChanged = usecTimestampNow(); }
-    void setDomainMaximumScale(float domainMaximumScale)
+    void setDomainMaximumScale(float domainMaximumScale) 
-        { _domainMaximumScale = glm::clamp(domainMaximumScale, MIN_AVATAR_SCALE, MAX_AVATAR_SCALE); }
+        { _domainMaximumScale = glm::clamp(domainMaximumScale, MIN_AVATAR_SCALE, MAX_AVATAR_SCALE); _scaleChanged = usecTimestampNow(); }
 
     //  Hand State
     Q_INVOKABLE void setHandState(char s) { _handState = s; }
@@ -375,7 +515,7 @@ public:
     void fromJson(const QJsonObject& json);
 
     glm::vec3 getClientGlobalPosition() { return _globalPosition; }
-    glm::vec3 getGlobalBoundingBoxCorner() { return _globalBoundingBoxCorner; }
+    glm::vec3 getGlobalBoundingBoxCorner() { return _globalPosition + _globalBoundingBoxOffset - _globalBoundingBoxDimensions; }
 
     Q_INVOKABLE AvatarEntityMap getAvatarEntityData() const;
     Q_INVOKABLE void setAvatarEntityData(const AvatarEntityMap& avatarEntityData);
@@ -387,6 +527,17 @@ public:
     Q_INVOKABLE glm::mat4 getControllerLeftHandMatrix() const;
     Q_INVOKABLE glm::mat4 getControllerRightHandMatrix() const;
 
+    float getDataRate(const QString& rateName = QString(""));
+
+    int getJointCount() { return _jointData.size(); }
+
+    QVector<JointData> getLastSentJointData() { 
+        QReadLocker readLock(&_jointDataLock);
+        _lastSentJointData.resize(_jointData.size());
+        return _lastSentJointData;
+    }
+
+
 public slots:
     void sendAvatarDataPacket();
     void sendIdentityPacket();
@@ -401,7 +552,27 @@ public slots:
 
     float getTargetScale() { return _targetScale; }
 
+    void resetLastSent() { _lastToByteArray = 0; }
+
 protected:
+    void lazyInitHeadData();
+
+    float getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition);
+    float getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition);
+
+    bool avatarBoundingBoxChangedSince(quint64 time);
+    bool avatarScaleChangedSince(quint64 time);
+    bool lookAtPositionChangedSince(quint64 time);
+    bool audioLoudnessChangedSince(quint64 time);
+    bool sensorToWorldMatrixChangedSince(quint64 time);
+    bool additionalFlagsChangedSince(quint64 time);
+
+    bool hasParent() { return !getParentID().isNull(); }
+    bool parentInfoChangedSince(quint64 time);
+
+    bool hasFaceTracker() { return _headData ? _headData->_isFaceTrackerConnected : false; }
+    bool faceTrackerInfoChangedSince(quint64 time);
+
     glm::vec3 _handPosition;
     virtual const QString& getSessionDisplayNameForTransport() const { return _sessionDisplayName; }
     virtual void maybeUpdateSessionDisplayNameFromTransport(const QString& sessionDisplayName) { } // No-op in AvatarMixer
@@ -460,8 +631,35 @@ protected:
     // _globalPosition is sent along with localPosition + parent because the avatar-mixer doesn't know
     // where Entities are located.  This is currently only used by the mixer to decide how often to send
     // updates about one avatar to another.
-    glm::vec3 _globalPosition;
+    glm::vec3 _globalPosition { 0, 0, 0 };
-    glm::vec3 _globalBoundingBoxCorner;
+
+
+    quint64 _globalPositionChanged { 0 };
+    quint64 _avatarBoundingBoxChanged { 0 };
+    quint64 _avatarScaleChanged { 0 };
+    quint64 _sensorToWorldMatrixChanged { 0 };
+    quint64 _additionalFlagsChanged { 0 };
+    quint64 _parentChanged { 0 };
+
+    quint64  _lastToByteArray { 0 }; // tracks the last time we did a toByteArray
+
+    // Some rate data for incoming data
+    RateCounter<> _parseBufferRate;
+    RateCounter<> _globalPositionRate;
+    RateCounter<> _localPositionRate;
+    RateCounter<> _avatarBoundingBoxRate;
+    RateCounter<> _avatarOrientationRate;
+    RateCounter<> _avatarScaleRate;
+    RateCounter<> _lookAtPositionRate;
+    RateCounter<> _audioLoudnessRate;
+    RateCounter<> _sensorToWorldRate;
+    RateCounter<> _additionalFlagsRate;
+    RateCounter<> _parentInfoRate;
+    RateCounter<> _faceTrackerRate;
+    RateCounter<> _jointDataRate;
+
+    glm::vec3 _globalBoundingBoxDimensions;
+    glm::vec3 _globalBoundingBoxOffset;
 
     mutable ReadWriteLockable _avatarEntitiesLock;
     AvatarEntityIDs _avatarEntityDetached; // recently detached from this avatar

libraries/avatars/src/HeadData.h

@@ -19,6 +19,8 @@
 #include <glm/glm.hpp>
 #include <glm/gtc/quaternion.hpp>
 
+#include <SharedUtil.h>
+
 // degrees
 const float MIN_HEAD_YAW = -180.0f;
 const float MAX_HEAD_YAW = 180.0f;
@@ -56,7 +58,13 @@ public:
     void setOrientation(const glm::quat& orientation);
 
     float getAudioLoudness() const { return _audioLoudness; }
-    void setAudioLoudness(float audioLoudness) { _audioLoudness = audioLoudness; }
+    void setAudioLoudness(float audioLoudness) { 
+        if (audioLoudness != _audioLoudness) {
+            _audioLoudnessChanged = usecTimestampNow();
+        }
+        _audioLoudness = audioLoudness; 
+    }
+    bool audioLoudnessChangedSince(quint64 time) { return _audioLoudnessChanged >= time; }
 
     float getAudioAverageLoudness() const { return _audioAverageLoudness; }
     void setAudioAverageLoudness(float audioAverageLoudness) { _audioAverageLoudness = audioAverageLoudness; }
@@ -66,7 +74,13 @@ public:
     void setBlendshapeCoefficients(const QVector<float>& blendshapeCoefficients) { _blendshapeCoefficients = blendshapeCoefficients; }
 
     const glm::vec3& getLookAtPosition() const { return _lookAtPosition; }
-    void setLookAtPosition(const glm::vec3& lookAtPosition) { _lookAtPosition = lookAtPosition; }
+    void setLookAtPosition(const glm::vec3& lookAtPosition) { 
+        if (_lookAtPosition != lookAtPosition) {
+            _lookAtPositionChanged = usecTimestampNow();
+        }
+        _lookAtPosition = lookAtPosition;
+    }
+    bool lookAtPositionChangedSince(quint64 time) { return _lookAtPositionChanged >= time; }
 
     friend class AvatarData;
 
@@ -80,7 +94,11 @@ protected:
     float _baseRoll;
 
     glm::vec3 _lookAtPosition;
+    quint64 _lookAtPositionChanged { 0 };
+
     float _audioLoudness;
+    quint64 _audioLoudnessChanged { 0 };
+
     bool _isFaceTrackerConnected;
     bool _isEyeTrackerConnected;
     float _leftEyeBlink;

libraries/networking/src/udt/PacketHeaders.cpp

@@ -55,7 +55,7 @@ PacketVersion versionForPacketType(PacketType packetType) {
         case PacketType::AvatarData:
         case PacketType::BulkAvatarData:
         case PacketType::KillAvatar:
-            return static_cast<PacketVersion>(AvatarMixerPacketVersion::Unignore);
+            return static_cast<PacketVersion>(AvatarMixerPacketVersion::VariableAvatarData);
         case PacketType::ICEServerHeartbeat:
             return 18; // ICE Server Heartbeat signing
         case PacketType::AssetGetInfo:

libraries/networking/src/udt/PacketHeaders.h

@@ -220,7 +220,8 @@ enum class AvatarMixerPacketVersion : PacketVersion {
     HasKillAvatarReason,
     SessionDisplayName,
     Unignore,
-    ImmediateSessionDisplayNameUpdates
+    ImmediateSessionDisplayNameUpdates,
+    VariableAvatarData
 };
 
 enum class DomainConnectRequestVersion : PacketVersion {

libraries/shared/src/SpatiallyNestable.cpp

@@ -26,6 +26,9 @@ SpatiallyNestable::SpatiallyNestable(NestableType nestableType, QUuid id) :
     // set flags in _transform
     _transform.setTranslation(glm::vec3(0.0f));
     _transform.setRotation(glm::quat());
+    _scaleChanged = usecTimestampNow();
+    _translationChanged = usecTimestampNow();
+    _rotationChanged = usecTimestampNow();
 }
 
 SpatiallyNestable::~SpatiallyNestable() {
@@ -399,6 +402,7 @@ void SpatiallyNestable::setPosition(const glm::vec3& position, bool& success, bo
             changed = true;
             myWorldTransform.setTranslation(position);
             Transform::inverseMult(_transform, parentTransform, myWorldTransform);
+            _translationChanged = usecTimestampNow();
         }
     });
     if (success && changed) {
@@ -451,6 +455,7 @@ void SpatiallyNestable::setOrientation(const glm::quat& orientation, bool& succe
             changed = true;
             myWorldTransform.setRotation(orientation);
             Transform::inverseMult(_transform, parentTransform, myWorldTransform);
+            _rotationChanged = usecTimestampNow();
         }
     });
     if (success && changed) {
@@ -649,6 +654,8 @@ void SpatiallyNestable::setTransform(const Transform& transform, bool& success)
         Transform::inverseMult(_transform, parentTransform, transform);
         if (_transform != beforeTransform) {
             changed = true;
+            _translationChanged = usecTimestampNow();
+            _rotationChanged = usecTimestampNow();
         }
     });
     if (success && changed) {
@@ -689,6 +696,7 @@ void SpatiallyNestable::setScale(const glm::vec3& scale) {
         if (_transform.getScale() != scale) {
             _transform.setScale(scale);
             changed = true;
+            _scaleChanged = usecTimestampNow();
         }
     });
     if (changed) {
@@ -710,6 +718,7 @@ void SpatiallyNestable::setScale(float value) {
         _transform.setScale(value);
         if (_transform.getScale() != beforeScale) {
             changed = true;
+            _scaleChanged = usecTimestampNow();
         }
     });
 
@@ -738,6 +747,9 @@ void SpatiallyNestable::setLocalTransform(const Transform& transform) {
         if (_transform != transform) {
             _transform = transform;
             changed = true;
+            _scaleChanged = usecTimestampNow();
+            _translationChanged = usecTimestampNow();
+            _rotationChanged = usecTimestampNow();
         }
     });
 
@@ -765,6 +777,7 @@ void SpatiallyNestable::setLocalPosition(const glm::vec3& position, bool tellPhy
         if (_transform.getTranslation() != position) {
             _transform.setTranslation(position);
             changed = true;
+            _translationChanged = usecTimestampNow();
         }
     });
     if (changed) {
@@ -791,6 +804,7 @@ void SpatiallyNestable::setLocalOrientation(const glm::quat& orientation) {
         if (_transform.getRotation() != orientation) {
             _transform.setRotation(orientation);
             changed = true;
+            _rotationChanged = usecTimestampNow();
         }
     });
     if (changed) {
@@ -848,9 +862,12 @@ void SpatiallyNestable::setLocalScale(const glm::vec3& scale) {
         if (_transform.getScale() != scale) {
             _transform.setScale(scale);
             changed = true;
+            _scaleChanged = usecTimestampNow();
         }
     });
-    dimensionsChanged();
+    if (changed) {
+        dimensionsChanged();
+    }
 }
 
 QList<SpatiallyNestablePointer> SpatiallyNestable::getChildren() const {
@@ -1059,6 +1076,9 @@ void SpatiallyNestable::setLocalTransformAndVelocities(
         if (_transform != localTransform) {
             _transform = localTransform;
             changed = true;
+            _scaleChanged = usecTimestampNow();
+            _translationChanged = usecTimestampNow();
+            _rotationChanged = usecTimestampNow();
         }
     });
     // linear velocity

libraries/shared/src/SpatiallyNestable.h

@@ -178,6 +178,10 @@ public:
             const glm::vec3& localVelocity,
             const glm::vec3& localAngularVelocity);
 
+    bool scaleChangedSince(quint64 time) { return _scaleChanged > time; }
+    bool tranlationChangedSince(quint64 time) { return _translationChanged > time; }
+    bool rotationChangedSince(quint64 time) { return _rotationChanged > time; }
+
 protected:
     const NestableType _nestableType; // EntityItem or an AvatarData
     QUuid _id;
@@ -201,6 +205,9 @@ protected:
     mutable bool _queryAACubeSet { false };
 
     bool _missingAncestor { false };
+    quint64 _scaleChanged { 0 };
+    quint64 _translationChanged { 0 };
+    quint64 _rotationChanged { 0 };
 
 private:
     mutable ReadWriteLockable _transformLock;

script-archive/acScripts/simpleBot.js

@@ -0,0 +1,63 @@
+//
+//  simpleBot.js
+//  examples
+//
+//  Created by Brad Hefta-Gaub on 12/23/16.
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+HIFI_PUBLIC_BUCKET = "http://s3.amazonaws.com/hifi-public/";
+
+function getRandomFloat(min, max) {
+    return Math.random() * (max - min) + min;
+}
+
+function getRandomInt (min, max) {
+    return Math.floor(Math.random() * (max - min + 1)) + min;
+}
+
+function printVector(string, vector) {
+    print(string + " " + vector.x + ", " + vector.y + ", " + vector.z);
+}
+
+var timePassed = 0.0;
+var updateSpeed = 3.0;
+
+var X_MIN = 5.0;
+var X_MAX = 15.0;
+var Z_MIN = 5.0;
+var Z_MAX = 15.0;
+var Y_PELVIS = 1.0;
+
+Agent.isAvatar = true;
+
+// change the avatar's position to the random one
+Avatar.position = {x:0,y:1.1,z:0}; // { x: getRandomFloat(X_MIN, X_MAX), y: Y_PELVIS, z: getRandomFloat(Z_MIN, Z_MAX) };;  
+printVector("New bot, position = ", Avatar.position);
+
+var animationData = {url: "file:///D:/Development/HiFi/hifi/interface/resources/avatar/animations/walk_fwd.fbx", lastFrame: 35};
+//Avatar.startAnimation(animationData.url, animationData.fps || 30, 1, true, false, animationData.firstFrame || 0, animationData.lastFrame);
+//Avatar.skeletonModelURL = "file:///D:/Development/HiFi/hifi/interface/resources/meshes/being_of_light/being_of_light.fbx";
+
+var millisecondsToWaitBeforeStarting = 4 * 1000;
+Script.setTimeout(function () {
+    print("Starting at", JSON.stringify(Avatar.position));
+    Avatar.startAnimation(animationData.url, animationData.fps || 30, 1, true, false, animationData.firstFrame || 0, animationData.lastFrame);
+}, millisecondsToWaitBeforeStarting);
+
+
+
+function update(deltaTime) {
+    timePassed += deltaTime;
+    if (timePassed > updateSpeed) {
+        timePassed = 0;
+        var newPosition = Vec3.sum(Avatar.position, { x: getRandomFloat(-0.1, 0.1), y: 0, z: getRandomFloat(-0.1, 0.1) });
+        Avatar.position = newPosition;
+        Vec3.print("new:", newPosition);
+    }
+}
+
+Script.update.connect(update);

scripts/developer/debugging/debugAvatarMixer.js

@@ -0,0 +1,130 @@
+"use strict";
+
+//
+//  debugAvatarMixer.js
+//  scripts/developer/debugging
+//
+//  Created by Brad Hefta-Gaub on 01/09/2017
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+/* global Toolbars, Script, Users, Overlays, AvatarList, Controller, Camera, getControllerWorldLocation */
+
+
+(function() { // BEGIN LOCAL_SCOPE
+
+Script.include("/~/system/libraries/controllers.js");
+
+var isShowingOverlays = true;
+var debugOverlays = {};
+
+function removeOverlays() {
+    // enumerate the overlays and remove them
+    var overlayKeys = Object.keys(debugOverlays);
+
+    for (var i = 0; i < overlayKeys.length; ++i) {
+        var avatarID = overlayKeys[i];
+        for (var j = 0; j < debugOverlays[avatarID].length; ++j) {
+            Overlays.deleteOverlay(debugOverlays[avatarID][j]);
+        }
+    }
+
+    debugOverlays = {};
+}
+
+function updateOverlays() {
+    if (isShowingOverlays) {
+
+        var identifiers = AvatarList.getAvatarIdentifiers();
+
+        for (var i = 0; i < identifiers.length; ++i) {
+            var avatarID = identifiers[i];
+
+            if (avatarID === null) {
+                // this is our avatar, skip it
+                continue;
+            }
+
+            // get the position for this avatar
+            var avatar = AvatarList.getAvatar(avatarID);
+            var avatarPosition = avatar && avatar.position;
+
+            if (!avatarPosition) {
+                // we don't have a valid position for this avatar, skip it
+                continue;
+            }
+
+            // setup a position for the overlay that is just above this avatar's head
+            var overlayPosition = avatar.getJointPosition("Head");
+            overlayPosition.y += 1.05;
+
+            var text = " All: " + AvatarManager.getAvatarDataRate(avatarID).toFixed(2) + "\n"
+                       +" GP: " + AvatarManager.getAvatarDataRate(avatarID,"globalPosition").toFixed(2) + "\n"
+                       +" LP: " + AvatarManager.getAvatarDataRate(avatarID,"localPosition").toFixed(2) + "\n"
+                       +" BB: " + AvatarManager.getAvatarDataRate(avatarID,"avatarBoundingBox").toFixed(2) + "\n"
+                       +" AO: " + AvatarManager.getAvatarDataRate(avatarID,"avatarOrientation").toFixed(2) + "\n"
+                       +" AS: " + AvatarManager.getAvatarDataRate(avatarID,"avatarScale").toFixed(2) + "\n"
+                       +" LA: "  + AvatarManager.getAvatarDataRate(avatarID,"lookAtPosition").toFixed(2) + "\n"
+                       +" AL: " + AvatarManager.getAvatarDataRate(avatarID,"audioLoudness").toFixed(2) + "\n"
+                       +" SW: " + AvatarManager.getAvatarDataRate(avatarID,"sensorToWorkMatrix").toFixed(2) + "\n"
+                       +" AF: " + AvatarManager.getAvatarDataRate(avatarID,"additionalFlags").toFixed(2) + "\n"
+                       +" PI: " + AvatarManager.getAvatarDataRate(avatarID,"parentInfo").toFixed(2) + "\n"
+                       +" FT: " + AvatarManager.getAvatarDataRate(avatarID,"faceTracker").toFixed(2) + "\n"
+                       +" JD: " + AvatarManager.getAvatarDataRate(avatarID,"jointData").toFixed(2);
+
+            if (avatarID in debugOverlays) {
+                // keep the overlay above the current position of this avatar
+                Overlays.editOverlay(debugOverlays[avatarID][0], {
+                    position: overlayPosition,
+                    text: text
+                });
+            } else {
+                // add the overlay above this avatar
+                var newOverlay = Overlays.addOverlay("text3d", {
+                    position: overlayPosition,
+                    dimensions: {
+                        x: 1,
+                        y: 13 * 0.13
+                    },
+                    lineHeight: 0.1,
+                    font:{size:0.1},
+                    text: text,
+                    size: 1,
+                    scale: 0.4,
+                    color: { red: 255, green: 255, blue: 255},
+                    alpha: 1,
+                    solid: true,
+                    isFacingAvatar: true,
+                    drawInFront: true
+                });
+
+                debugOverlays[avatarID]=[newOverlay];
+            }
+        }
+    }
+}
+
+Script.update.connect(updateOverlays);
+
+AvatarList.avatarRemovedEvent.connect(function(avatarID){
+    if (isShowingOverlays) {
+        // we are currently showing overlays and an avatar just went away
+
+        // first remove the rendered overlays
+        for (var j = 0; j < debugOverlays[avatarID].length; ++j) {
+            Overlays.deleteOverlay(debugOverlays[avatarID][j]);
+        }
+        
+        // delete the saved ID of the overlay from our mod overlays object
+        delete debugOverlays[avatarID];
+    }
+});
+
+// cleanup the toolbar button and overlays when script is stopped
+Script.scriptEnding.connect(function() {
+    removeOverlays();
+});
+
+}()); // END LOCAL_SCOPE