Merge branch 'master' of https://github.com/highfidelity/hifi into dk/palVolumeSorting

assignment-client/src/Agent.cpp

@@ -434,8 +434,16 @@ void Agent::executeScript() {
     connect(&_avatarAudioTimerThread, &QThread::finished, audioTimerWorker, &QObject::deleteLater); 
     _avatarAudioTimerThread.start();
     
-    // 60Hz timer for avatar
-    QObject::connect(_scriptEngine.get(), &ScriptEngine::update, this, &Agent::processAgentAvatar);
+    // Agents should run at 45hz
+    static const int AVATAR_DATA_HZ = 45;
+    static const int AVATAR_DATA_IN_MSECS = MSECS_PER_SECOND / AVATAR_DATA_HZ;
+    QTimer* avatarDataTimer = new QTimer(this);
+    connect(avatarDataTimer, &QTimer::timeout, this, &Agent::processAgentAvatar);
+    avatarDataTimer->setSingleShot(false);
+    avatarDataTimer->setInterval(AVATAR_DATA_IN_MSECS);
+    avatarDataTimer->setTimerType(Qt::PreciseTimer);
+    avatarDataTimer->start();
+
     _scriptEngine->run();
 
     Frame::clearFrameHandler(AUDIO_FRAME_TYPE);
@@ -538,7 +546,7 @@ void Agent::processAgentAvatar() {
         auto scriptedAvatar = DependencyManager::get<ScriptableAvatar>();
 
         AvatarData::AvatarDataDetail dataDetail = (randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO) ? AvatarData::SendAllData : AvatarData::CullSmallData;
-        QByteArray avatarByteArray = scriptedAvatar->toByteArray(dataDetail, 0, scriptedAvatar->getLastSentJointData());
+        QByteArray avatarByteArray = scriptedAvatar->toByteArrayStateful(dataDetail);
         scriptedAvatar->doneEncoding(true);
 
         static AvatarDataSequenceNumber sequenceNumber = 0;

assignment-client/src/avatars/AvatarMixer.h

@@ -21,6 +21,8 @@
 #include <ThreadedAssignment.h>
 #include "AvatarMixerClientData.h"
 
+#include "AvatarMixerSlavePool.h"
+
 /// Handles assignments of type AvatarMixer - distribution of avatar data to various clients
 class AvatarMixer : public ThreadedAssignment {
     Q_OBJECT
@@ -36,8 +38,9 @@ public slots:
     void sendStatsPacket() override;
 
 private slots:
+    void queueIncomingPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer node);
+    void handleAdjustAvatarSorting(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleViewFrustumPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
-    void handleAvatarDataPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleAvatarIdentityPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleKillAvatarPacket(QSharedPointer<ReceivedMessage> message);
     void handleNodeIgnoreRequestPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
@@ -45,22 +48,29 @@ private slots:
     void handleRequestsDomainListDataPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void domainSettingsRequestComplete();
     void handlePacketVersionMismatch(PacketType type, const HifiSockAddr& senderSockAddr, const QUuid& senderUUID);
+    void start();
 
 
 private:
-    void broadcastAvatarData();
+    AvatarMixerClientData* getOrCreateClientData(SharedNodePointer node);
+    std::chrono::microseconds timeFrame(p_high_resolution_clock::time_point& timestamp);
+    void throttle(std::chrono::microseconds duration, int frame);
+
     void parseDomainServerSettings(const QJsonObject& domainSettings);
     void sendIdentityPacket(AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode);
 
-    QThread _broadcastThread;
+    void manageDisplayName(const SharedNodePointer& node);
 
     p_high_resolution_clock::time_point _lastFrameTimestamp;
 
-    float _trailingSleepRatio { 1.0f };
-    float _performanceThrottlingRatio { 0.0f };
+    // FIXME - new throttling - use these values somehow
+    float _trailingMixRatio { 0.0f };
+    float _throttlingRatio { 0.0f };
+
 
     int _sumListeners { 0 };
     int _numStatFrames { 0 };
+    int _numTightLoopFrames { 0 };
     int _sumIdentityPackets { 0 };
 
     float _maxKbpsPerNode = 0.0f;
@@ -68,11 +78,41 @@ private:
     float _domainMinimumScale { MIN_AVATAR_SCALE };
     float _domainMaximumScale { MAX_AVATAR_SCALE };
 
-    QTimer* _broadcastTimer = nullptr;
-
     RateCounter<> _broadcastRate;
     p_high_resolution_clock::time_point _lastDebugMessage;
     QHash<QString, QPair<int, int>> _sessionDisplayNames;
+
+    quint64 _displayNameManagementElapsedTime { 0 }; // total time spent in broadcastAvatarData/display name management... since last stats window
+    quint64 _ignoreCalculationElapsedTime { 0 };
+    quint64 _avatarDataPackingElapsedTime { 0 };
+    quint64 _packetSendingElapsedTime { 0 };
+
+    quint64 _broadcastAvatarDataElapsedTime { 0 }; // total time spent in broadcastAvatarData since last stats window
+    quint64 _broadcastAvatarDataInner { 0 };
+    quint64 _broadcastAvatarDataLockWait { 0 };
+    quint64 _broadcastAvatarDataNodeTransform { 0 };
+    quint64 _broadcastAvatarDataNodeFunctor { 0 };
+
+    quint64 _handleAdjustAvatarSortingElapsedTime { 0 };
+    quint64 _handleViewFrustumPacketElapsedTime { 0 };
+    quint64 _handleAvatarIdentityPacketElapsedTime { 0 };
+    quint64 _handleKillAvatarPacketElapsedTime { 0 };
+    quint64 _handleNodeIgnoreRequestPacketElapsedTime { 0 };
+    quint64 _handleRadiusIgnoreRequestPacketElapsedTime { 0 };
+    quint64 _handleRequestsDomainListDataPacketElapsedTime { 0 };
+    quint64 _processQueuedAvatarDataPacketsElapsedTime { 0 };
+    quint64 _processQueuedAvatarDataPacketsLockWaitElapsedTime { 0 };
+
+    quint64 _processEventsElapsedTime { 0 };
+    quint64 _sendStatsElapsedTime { 0 };
+    quint64 _queueIncomingPacketElapsedTime { 0 };
+    quint64 _lastStatsTime { usecTimestampNow() };
+
+    RateCounter<> _loopRate; // this is the rate that the main thread tight loop runs
+
+
+    AvatarMixerSlavePool _slavePool;
+
 };
 
 #endif // hifi_AvatarMixer_h

assignment-client/src/avatars/AvatarMixerClientData.cpp

@@ -16,10 +16,52 @@
 
 #include "AvatarMixerClientData.h"
 
+
+
+void AvatarMixerClientData::queuePacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer node) {
+    if (!_packetQueue.node) {
+        _packetQueue.node = node;
+    }
+    _packetQueue.push(message);
+}
+
+int AvatarMixerClientData::processPackets() {
+    int packetsProcessed = 0;
+    SharedNodePointer node = _packetQueue.node;
+    assert(_packetQueue.empty() || node);
+    _packetQueue.node.clear();
+
+    while (!_packetQueue.empty()) {
+        auto& packet = _packetQueue.front();
+
+        packetsProcessed++;
+
+        switch (packet->getType()) {
+            case PacketType::AvatarData:
+                parseData(*packet);
+                break;
+            default:
+                Q_UNREACHABLE();
+        }
+        _packetQueue.pop();
+    }
+    assert(_packetQueue.empty());
+
+    return packetsProcessed;
+}
+
 int AvatarMixerClientData::parseData(ReceivedMessage& message) {
+
     // pull the sequence number from the data first
-    message.readPrimitive(&_lastReceivedSequenceNumber);
+    uint16_t sequenceNumber;
+
+    message.readPrimitive(&sequenceNumber);
     
+    if (sequenceNumber < _lastReceivedSequenceNumber && _lastReceivedSequenceNumber != UINT16_MAX) {
+        incrementNumOutOfOrderSends();
+    }
+    _lastReceivedSequenceNumber = sequenceNumber;
+
     // compute the offset to the data payload
     return _avatar->parseDataFromBuffer(message.readWithoutCopy(message.getBytesLeftToRead()));
 }
@@ -32,14 +74,22 @@ bool AvatarMixerClientData::checkAndSetHasReceivedFirstPacketsFrom(const QUuid&
     return true;
 }
 
+uint64_t AvatarMixerClientData::getLastBroadcastTime(const QUuid& nodeUUID) const {
+    // return the matching PacketSequenceNumber, or the default if we don't have it
+    auto nodeMatch = _lastBroadcastTimes.find(nodeUUID);
+    if (nodeMatch != _lastBroadcastTimes.end()) {
+        return nodeMatch->second;
+    }
+    return 0;
+}
+
 uint16_t AvatarMixerClientData::getLastBroadcastSequenceNumber(const QUuid& nodeUUID) const {
     // return the matching PacketSequenceNumber, or the default if we don't have it
     auto nodeMatch = _lastBroadcastSequenceNumbers.find(nodeUUID);
     if (nodeMatch != _lastBroadcastSequenceNumbers.end()) {
         return nodeMatch->second;
-    } else {
-        return 0;
     }
+    return 0;
 }
 
 void AvatarMixerClientData::ignoreOther(SharedNodePointer self, SharedNodePointer other) {
@@ -76,8 +126,6 @@ bool AvatarMixerClientData::otherAvatarInView(const AABox& otherAvatarBox) {
 
 void AvatarMixerClientData::loadJSONStats(QJsonObject& jsonObject) const {
     jsonObject["display_name"] = _avatar->getDisplayName();
-    jsonObject["full_rate_distance"] = _fullRateDistance;
-    jsonObject["max_av_distance"] = _maxAvatarDistance;
     jsonObject["num_avs_sent_last_frame"] = _numAvatarsSentLastFrame;
     jsonObject["avg_other_av_starves_per_second"] = getAvgNumOtherAvatarStarvesPerSecond();
     jsonObject["avg_other_av_skips_per_second"] = getAvgNumOtherAvatarSkipsPerSecond();

assignment-client/src/avatars/AvatarMixerClientData.h

@@ -16,6 +16,7 @@
 #include <cfloat>
 #include <unordered_map>
 #include <unordered_set>
+#include <queue>
 
 #include <QtCore/QJsonObject>
 #include <QtCore/QUrl>
@@ -41,6 +42,8 @@ public:
 
     int parseData(ReceivedMessage& message) override;
     AvatarData& getAvatar() { return *_avatar; }
+    const AvatarData* getConstAvatarData() const { return _avatar.get(); }
+    AvatarSharedPointer getAvatarSharedPointer() const { return _avatar; }
 
     bool checkAndSetHasReceivedFirstPacketsFrom(const QUuid& uuid);
 
@@ -49,6 +52,15 @@ public:
         { _lastBroadcastSequenceNumbers[nodeUUID] = sequenceNumber; }
     Q_INVOKABLE void removeLastBroadcastSequenceNumber(const QUuid& nodeUUID) { _lastBroadcastSequenceNumbers.erase(nodeUUID); }
 
+    uint64_t getLastBroadcastTime(const QUuid& nodeUUID) const;
+    void setLastBroadcastTime(const QUuid& nodeUUID, uint64_t broadcastTime) { _lastBroadcastTimes[nodeUUID] = broadcastTime; }
+    Q_INVOKABLE void removeLastBroadcastTime(const QUuid& nodeUUID) { _lastBroadcastTimes.erase(nodeUUID); }
+
+    Q_INVOKABLE void cleanupKilledNode(const QUuid& nodeUUID) {
+        removeLastBroadcastSequenceNumber(nodeUUID);
+        removeLastBroadcastTime(nodeUUID);
+    }
+
     uint16_t getLastReceivedSequenceNumber() const { return _lastReceivedSequenceNumber; }
 
     HRCTime getIdentityChangeTimestamp() const { return _identityChangeTimestamp; }
@@ -56,12 +68,6 @@ public:
     bool getAvatarSessionDisplayNameMustChange() const { return _avatarSessionDisplayNameMustChange; }
     void setAvatarSessionDisplayNameMustChange(bool set = true) { _avatarSessionDisplayNameMustChange = set; }
 
-    void setFullRateDistance(float fullRateDistance) { _fullRateDistance = fullRateDistance; }
-    float getFullRateDistance() const { return _fullRateDistance; }
-
-    void setMaxAvatarDistance(float maxAvatarDistance) { _maxAvatarDistance = maxAvatarDistance; }
-    float getMaxAvatarDistance() const { return _maxAvatarDistance; }
-
     void resetNumAvatarsSentLastFrame() { _numAvatarsSentLastFrame = 0; }
     void incrementNumAvatarsSentLastFrame() { ++_numAvatarsSentLastFrame; }
     int getNumAvatarsSentLastFrame() const { return _numAvatarsSentLastFrame; }
@@ -85,9 +91,9 @@ public:
 
     void loadJSONStats(QJsonObject& jsonObject) const;
 
-    glm::vec3 getPosition() { return _avatar ? _avatar->getPosition() : glm::vec3(0); }
-    glm::vec3 getGlobalBoundingBoxCorner() { return _avatar ? _avatar->getGlobalBoundingBoxCorner() : glm::vec3(0); }
-    bool isRadiusIgnoring(const QUuid& other) { return _radiusIgnoredOthers.find(other) != _radiusIgnoredOthers.end(); }
+    glm::vec3 getPosition() const { return _avatar ? _avatar->getPosition() : glm::vec3(0); }
+    glm::vec3 getGlobalBoundingBoxCorner() const { return _avatar ? _avatar->getGlobalBoundingBoxCorner() : glm::vec3(0); }
+    bool isRadiusIgnoring(const QUuid& other) const { return _radiusIgnoredOthers.find(other) != _radiusIgnoredOthers.end(); }
     void addToRadiusIgnoringSet(const QUuid& other) { _radiusIgnoredOthers.insert(other); }
     void removeFromRadiusIgnoringSet(SharedNodePointer self, const QUuid& other);
     void ignoreOther(SharedNodePointer self, SharedNodePointer other);
@@ -104,6 +110,8 @@ public:
     bool getRequestsDomainListData() { return _requestsDomainListData; }
     void setRequestsDomainListData(bool requesting) { _requestsDomainListData = requesting; }
 
+    ViewFrustum getViewFrustom() const { return _currentViewFrustum; }
+
     quint64 getLastOtherAvatarEncodeTime(QUuid otherAvatar) {
         quint64 result = 0;
         if (_lastOtherAvatarEncodeTime.find(otherAvatar) != _lastOtherAvatarEncodeTime.end()) {
@@ -118,14 +126,21 @@ public:
         return _lastOtherAvatarSentJoints[otherAvatar];
     }
 
-    
+    void queuePacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer node);
+    int processPackets(); // returns number of packets processed
 
 private:
+    struct PacketQueue : public std::queue<QSharedPointer<ReceivedMessage>> {
+        QWeakPointer<Node> node;
+    };
+    PacketQueue _packetQueue;
+
     AvatarSharedPointer _avatar { new AvatarData() };
 
     uint16_t _lastReceivedSequenceNumber { 0 };
     std::unordered_map<QUuid, uint16_t> _lastBroadcastSequenceNumbers;
     std::unordered_set<QUuid> _hasReceivedFirstPacketsFrom;
+    std::unordered_map<QUuid, uint64_t> _lastBroadcastTimes;
 
     // this is a map of the last time we encoded an "other" avatar for
     // sending to "this" node
@@ -135,9 +150,6 @@ private:
     HRCTime _identityChangeTimestamp;
     bool _avatarSessionDisplayNameMustChange{ false };
 
-    float _fullRateDistance = FLT_MAX;
-    float _maxAvatarDistance = FLT_MAX;
-
     int _numAvatarsSentLastFrame = 0;
     int _numFramesSinceAdjustment = 0;
 

assignment-client/src/avatars/AvatarMixerSlave.cpp

@@ -0,0 +1,432 @@
+//
+//  AvatarMixerSlave.cpp
+//  assignment-client/src/avatar
+//
+//  Created by Brad Hefta-Gaub on 2/14/2017.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <algorithm>
+#include <random>
+
+#include <glm/glm.hpp>
+#include <glm/gtx/norm.hpp>
+#include <glm/gtx/vector_angle.hpp>
+
+#include <AvatarLogging.h>
+#include <LogHandler.h>
+#include <NetworkAccessManager.h>
+#include <NodeList.h>
+#include <Node.h>
+#include <OctreeConstants.h>
+#include <udt/PacketHeaders.h>
+#include <SharedUtil.h>
+#include <StDev.h>
+#include <UUID.h>
+
+
+#include "AvatarMixer.h"
+#include "AvatarMixerClientData.h"
+#include "AvatarMixerSlave.h"
+
+
+void AvatarMixerSlave::configure(ConstIter begin, ConstIter end) {
+    _begin = begin;
+    _end = end;
+}
+
+void AvatarMixerSlave::configureBroadcast(ConstIter begin, ConstIter end, 
+                                p_high_resolution_clock::time_point lastFrameTimestamp,
+                                float maxKbpsPerNode, float throttlingRatio) {
+    _begin = begin;
+    _end = end;
+    _lastFrameTimestamp = lastFrameTimestamp;
+    _maxKbpsPerNode = maxKbpsPerNode;
+    _throttlingRatio = throttlingRatio;
+}
+
+void AvatarMixerSlave::harvestStats(AvatarMixerSlaveStats& stats) {
+    stats = _stats;
+    _stats.reset();
+}
+
+
+void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) {
+    auto start = usecTimestampNow();
+    auto nodeData = dynamic_cast<AvatarMixerClientData*>(node->getLinkedData());
+    if (nodeData) {
+        _stats.nodesProcessed++;
+        _stats.packetsProcessed += nodeData->processPackets();
+    }
+    auto end = usecTimestampNow();
+    _stats.processIncomingPacketsElapsedTime += (end - start);
+}
+
+
+int AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) {
+    int bytesSent = 0;
+    QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray();
+    auto identityPacket = NLPacket::create(PacketType::AvatarIdentity, individualData.size());
+    individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious
+    bytesSent += individualData.size();
+    identityPacket->write(individualData);
+    DependencyManager::get<NodeList>()->sendPacket(std::move(identityPacket), *destinationNode);
+    _stats.numIdentityPackets++;
+    return bytesSent;
+}
+
+static const int AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND = 45;
+
+// FIXME - There is some old logic (unchanged as of 2/17/17) that randomly decides to send an identity
+// packet. That logic had the following comment about the constants it uses...
+//
+//         An 80% chance of sending a identity packet within a 5 second interval.
+//         assuming 60 htz update rate.
+//
+// Assuming the calculation of the constant is in fact correct for 80% and 60hz and 5 seconds (an assumption
+// that I have not verified) then the constant is definitely wrong now, since we send at 45hz.
+const float IDENTITY_SEND_PROBABILITY = 1.0f / 187.0f;
+
+void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) {
+    quint64 start = usecTimestampNow();
+
+    auto nodeList = DependencyManager::get<NodeList>();
+
+    // setup for distributed random floating point values
+    std::random_device randomDevice;
+    std::mt19937 generator(randomDevice());
+    std::uniform_real_distribution<float> distribution;
+
+    if (node->getLinkedData() && (node->getType() == NodeType::Agent) && node->getActiveSocket()) {
+        _stats.nodesBroadcastedTo++;
+
+        AvatarMixerClientData* nodeData = reinterpret_cast<AvatarMixerClientData*>(node->getLinkedData());
+
+        nodeData->resetInViewStats();
+
+        const AvatarData& avatar = nodeData->getAvatar();
+        glm::vec3 myPosition = avatar.getClientGlobalPosition();
+
+        // reset the internal state for correct random number distribution
+        distribution.reset();
+
+        // reset the number of sent avatars
+        nodeData->resetNumAvatarsSentLastFrame();
+
+        // keep a counter of the number of considered avatars
+        int numOtherAvatars = 0;
+
+        // keep track of outbound data rate specifically for avatar data
+        int numAvatarDataBytes = 0;
+        int identityBytesSent = 0;
+
+        // max number of avatarBytes per frame
+        auto maxAvatarBytesPerFrame = (_maxKbpsPerNode * BYTES_PER_KILOBIT) / AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND;
+
+        // FIXME - find a way to not send the sessionID for every avatar
+        int minimumBytesPerAvatar = AvatarDataPacket::AVATAR_HAS_FLAGS_SIZE + NUM_BYTES_RFC4122_UUID;
+
+        int overBudgetAvatars = 0;
+
+        // keep track of the number of other avatars held back in this frame
+        int numAvatarsHeldBack = 0;
+
+        // keep track of the number of other avatar frames skipped
+        int numAvatarsWithSkippedFrames = 0;
+
+        // When this is true, the AvatarMixer will send Avatar data to a client about avatars that are not in the view frustrum
+        bool getsOutOfView = nodeData->getRequestsDomainListData();
+
+        // When this is true, the AvatarMixer will send Avatar data to a client about avatars that they've ignored
+        bool getsIgnoredByMe = getsOutOfView;
+
+        // When this is true, the AvatarMixer will send Avatar data to a client about avatars that have ignored them
+        bool getsAnyIgnored = getsIgnoredByMe && node->getCanKick();
+
+        // setup a PacketList for the avatarPackets
+        auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData);
+
+        // Define the minimum bubble size
+        static const glm::vec3 minBubbleSize = glm::vec3(0.3f, 1.3f, 0.3f);
+        // Define the scale of the box for the current node
+        glm::vec3 nodeBoxScale = (nodeData->getPosition() - nodeData->getGlobalBoundingBoxCorner()) * 2.0f;
+        // Set up the bounding box for the current node
+        AABox nodeBox(nodeData->getGlobalBoundingBoxCorner(), nodeBoxScale);
+        // Clamp the size of the bounding box to a minimum scale
+        if (glm::any(glm::lessThan(nodeBoxScale, minBubbleSize))) {
+            nodeBox.setScaleStayCentered(minBubbleSize);
+        }
+        // Quadruple the scale of both bounding boxes
+        nodeBox.embiggen(4.0f);
+
+
+        // setup list of AvatarData as well as maps to map betweeen the AvatarData and the original nodes
+        // for calling the AvatarData::sortAvatars() function and getting our sorted list of client nodes
+        QList<AvatarSharedPointer> avatarList;
+        std::unordered_map<AvatarSharedPointer, SharedNodePointer> avatarDataToNodes;
+
+        int listItem = 0;
+        std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) {
+            const AvatarMixerClientData* otherNodeData = reinterpret_cast<const AvatarMixerClientData*>(otherNode->getLinkedData());
+
+            // theoretically it's possible for a Node to be in the NodeList (and therefore end up here),
+            // but not have yet sent data that's linked to the node. Check for that case and don't
+            // consider those nodes.
+            if (otherNodeData) {
+                listItem++;
+                AvatarSharedPointer otherAvatar = otherNodeData->getAvatarSharedPointer();
+                avatarList << otherAvatar;
+                avatarDataToNodes[otherAvatar] = otherNode;
+            }
+        });
+
+        AvatarSharedPointer thisAvatar = nodeData->getAvatarSharedPointer();
+        ViewFrustum cameraView = nodeData->getViewFrustom();
+        std::priority_queue<AvatarPriority> sortedAvatars = AvatarData::sortAvatars(
+                avatarList, cameraView,
+
+                [&](AvatarSharedPointer avatar)->uint64_t{
+                    auto avatarNode = avatarDataToNodes[avatar];
+                    assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map
+                    return nodeData->getLastBroadcastTime(avatarNode->getUUID());
+                },
+
+                [&](AvatarSharedPointer avatar)->float{
+                    glm::vec3 nodeBoxHalfScale = (avatar->getPosition() - avatar->getGlobalBoundingBoxCorner());
+                    return glm::max(nodeBoxHalfScale.x, glm::max(nodeBoxHalfScale.y, nodeBoxHalfScale.z));
+                },
+
+                [&](AvatarSharedPointer avatar)->bool{
+                    if (avatar == thisAvatar) {
+                        return true; // ignore ourselves...
+                    }
+
+                    bool shouldIgnore = false;
+
+                    // We will also ignore other nodes for a couple of different reasons:
+                    //   1) ignore bubbles and ignore specific node
+                    //   2) the node hasn't really updated it's frame data recently, this can
+                    //      happen if for example the avatar is connected on a desktop and sending
+                    //      updates at ~30hz. So every 3 frames we skip a frame.
+                    auto avatarNode = avatarDataToNodes[avatar];
+
+                    assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map
+
+                    const AvatarMixerClientData* avatarNodeData = reinterpret_cast<const AvatarMixerClientData*>(avatarNode->getLinkedData());
+                    assert(avatarNodeData); // we can't have gotten here without avatarNode having valid data
+                    quint64 startIgnoreCalculation = usecTimestampNow();
+
+                    // make sure we have data for this avatar, that it isn't the same node,
+                    // and isn't an avatar that the viewing node has ignored
+                    // or that has ignored the viewing node
+                    if (!avatarNode->getLinkedData()
+                        || avatarNode->getUUID() == node->getUUID()
+                        || (node->isIgnoringNodeWithID(avatarNode->getUUID()) && !getsIgnoredByMe)
+                        || (avatarNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) {
+                        shouldIgnore = true;
+                    } else {
+
+                        // Check to see if the space bubble is enabled
+                        if (node->isIgnoreRadiusEnabled() || avatarNode->isIgnoreRadiusEnabled()) {
+
+                            // Define the scale of the box for the current other node
+                            glm::vec3 otherNodeBoxScale = (avatarNodeData->getPosition() - avatarNodeData->getGlobalBoundingBoxCorner()) * 2.0f;
+                            // Set up the bounding box for the current other node
+                            AABox otherNodeBox(avatarNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale);
+                            // Clamp the size of the bounding box to a minimum scale
+                            if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) {
+                                otherNodeBox.setScaleStayCentered(minBubbleSize);
+                            }
+                            // Quadruple the scale of both bounding boxes
+                            otherNodeBox.embiggen(4.0f);
+
+                            // Perform the collision check between the two bounding boxes
+                            if (nodeBox.touches(otherNodeBox)) {
+                                nodeData->ignoreOther(node, avatarNode);
+                                shouldIgnore = !getsAnyIgnored;
+                            }
+                        }
+                        // Not close enough to ignore
+                        if (!shouldIgnore) {
+                            nodeData->removeFromRadiusIgnoringSet(node, avatarNode->getUUID());
+                        }
+                    }
+                    quint64 endIgnoreCalculation = usecTimestampNow();
+                    _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation);
+
+                    if (!shouldIgnore) {
+                        AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getUUID());
+                        AvatarDataSequenceNumber lastSeqFromSender = avatarNodeData->getLastReceivedSequenceNumber();
+
+                        // FIXME - This code does appear to be working. But it seems brittle.
+                        //         It supports determining if the frame of data for this "other"
+                        //         avatar has already been sent to the reciever. This has been
+                        //         verified to work on a desktop display that renders at 60hz and
+                        //         therefore sends to mixer at 30hz. Each second you'd expect to
+                        //         have 15 (45hz-30hz) duplicate frames. In this case, the stat
+                        //         avg_other_av_skips_per_second does report 15.
+                        //
+                        // make sure we haven't already sent this data from this sender to this receiver
+                        // or that somehow we haven't sent
+                        if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) {
+                            ++numAvatarsHeldBack;
+                            shouldIgnore = true;
+                        } else if (lastSeqFromSender - lastSeqToReceiver > 1) {
+                            // this is a skip - we still send the packet but capture the presence of the skip so we see it happening
+                            ++numAvatarsWithSkippedFrames;
+                        }
+                    }
+                    return shouldIgnore;
+                });
+
+        // loop through our sorted avatars and allocate our bandwidth to them accordingly
+        int avatarRank = 0;
+
+        // this is overly conservative, because it includes some avatars we might not consider
+        int remainingAvatars = (int)sortedAvatars.size(); 
+
+        while (!sortedAvatars.empty()) {
+            AvatarPriority sortData = sortedAvatars.top();
+            sortedAvatars.pop();
+            const auto& avatarData = sortData.avatar;
+            avatarRank++;
+            remainingAvatars--;
+
+            auto otherNode = avatarDataToNodes[avatarData];
+            assert(otherNode); // we can't have gotten here without the avatarData being a valid key in the map
+
+            // NOTE: Here's where we determine if we are over budget and drop to bare minimum data
+            int minimRemainingAvatarBytes = minimumBytesPerAvatar * remainingAvatars;
+            bool overBudget = (identityBytesSent + numAvatarDataBytes + minimRemainingAvatarBytes) > maxAvatarBytesPerFrame;
+
+            quint64 startAvatarDataPacking = usecTimestampNow();
+
+            ++numOtherAvatars;
+
+            const AvatarMixerClientData* otherNodeData = reinterpret_cast<const AvatarMixerClientData*>(otherNode->getLinkedData());
+
+            // make sure we send out identity packets to and from new arrivals.
+            bool forceSend = !nodeData->checkAndSetHasReceivedFirstPacketsFrom(otherNode->getUUID());
+
+            // FIXME - this clause seems suspicious "... || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp ..."
+            if (!overBudget
+                && otherNodeData->getIdentityChangeTimestamp().time_since_epoch().count() > 0
+                && (forceSend
+                || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp
+                || distribution(generator) < IDENTITY_SEND_PROBABILITY)) {
+
+                identityBytesSent += sendIdentityPacket(otherNodeData, node);
+            }
+
+            const AvatarData* otherAvatar = otherNodeData->getConstAvatarData();
+            glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition();
+
+            // determine if avatar is in view, to determine how much data to include...
+            glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f;
+            AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale);
+            bool isInView = nodeData->otherAvatarInView(otherNodeBox);
+
+            // start a new segment in the PacketList for this avatar
+            avatarPacketList->startSegment();
+
+            AvatarData::AvatarDataDetail detail;
+
+            if (overBudget) {
+                overBudgetAvatars++;
+                _stats.overBudgetAvatars++;
+                detail = AvatarData::NoData;
+            } else  if (!isInView && !getsOutOfView) {
+                detail = AvatarData::NoData;
+                nodeData->incrementAvatarOutOfView();
+            } else {
+                detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO
+                    ? AvatarData::SendAllData : AvatarData::CullSmallData;
+                nodeData->incrementAvatarInView();
+            }
+
+            bool includeThisAvatar = true;
+            auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID());
+            QVector<JointData>& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID());
+            bool distanceAdjust = true;
+            glm::vec3 viewerPosition = myPosition;
+            AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray
+            bool dropFaceTracking = false;
+
+            quint64 start = usecTimestampNow();
+            QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, 
+                                            hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther);
+            quint64 end = usecTimestampNow();
+            _stats.toByteArrayElapsedTime += (end - start);
+
+            static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID);
+            if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) {
+                qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data";
+
+                dropFaceTracking = true; // first try dropping the facial data
+                bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther,
+                    hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther);
+
+                if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) {
+                    qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData";
+                    bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther,
+                        hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther);
+                }
+
+                if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) {
+                    qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!";
+                    includeThisAvatar = false;
+                }
+            }
+
+            if (includeThisAvatar) {
+                numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122());
+                numAvatarDataBytes += avatarPacketList->write(bytes);
+                _stats.numOthersIncluded++;
+
+                // increment the number of avatars sent to this reciever
+                nodeData->incrementNumAvatarsSentLastFrame();
+
+                // set the last sent sequence number for this sender on the receiver
+                nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(),
+                    otherNodeData->getLastReceivedSequenceNumber());
+
+                // remember the last time we sent details about this other node to the receiver
+                nodeData->setLastBroadcastTime(otherNode->getUUID(), start);
+
+            }
+
+            avatarPacketList->endSegment();
+
+            quint64 endAvatarDataPacking = usecTimestampNow();
+            _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking);
+        };
+
+        quint64 startPacketSending = usecTimestampNow();
+
+        // close the current packet so that we're always sending something
+        avatarPacketList->closeCurrentPacket(true);
+
+        _stats.numPacketsSent += (int)avatarPacketList->getNumPackets();
+        _stats.numBytesSent += numAvatarDataBytes;
+
+        // send the avatar data PacketList
+        nodeList->sendPacketList(std::move(avatarPacketList), *node);
+
+        // record the bytes sent for other avatar data in the AvatarMixerClientData
+        nodeData->recordSentAvatarData(numAvatarDataBytes);
+
+        // record the number of avatars held back this frame
+        nodeData->recordNumOtherAvatarStarves(numAvatarsHeldBack);
+        nodeData->recordNumOtherAvatarSkips(numAvatarsWithSkippedFrames);
+
+        quint64 endPacketSending = usecTimestampNow();
+        _stats.packetSendingElapsedTime += (endPacketSending - startPacketSending);
+    }
+
+    quint64 end = usecTimestampNow();
+    _stats.jobElapsedTime += (end - start);
+}
+

assignment-client/src/avatars/AvatarMixerSlave.h

@@ -0,0 +1,107 @@
+//
+//  AvatarMixerSlave.h
+//  assignment-client/src/avatar
+//
+//  Created by Brad Hefta-Gaub on 2/14/2017.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_AvatarMixerSlave_h
+#define hifi_AvatarMixerSlave_h
+
+class AvatarMixerClientData;
+
+class AvatarMixerSlaveStats {
+public:
+    int nodesProcessed { 0 };
+    int packetsProcessed { 0 };
+    quint64 processIncomingPacketsElapsedTime { 0 };
+
+    int nodesBroadcastedTo { 0 };
+    int numPacketsSent { 0 };
+    int numBytesSent { 0 };
+    int numIdentityPackets { 0 };
+    int numOthersIncluded { 0 };
+    int overBudgetAvatars { 0 };
+
+    quint64 ignoreCalculationElapsedTime { 0 };
+    quint64 avatarDataPackingElapsedTime { 0 };
+    quint64 packetSendingElapsedTime { 0 };
+    quint64 toByteArrayElapsedTime { 0 };
+    quint64 jobElapsedTime { 0 };
+
+    void reset() {
+        // receiving job stats
+        nodesProcessed = 0;
+        packetsProcessed = 0;
+        processIncomingPacketsElapsedTime = 0;
+
+        // sending job stats
+        nodesBroadcastedTo = 0;
+        numPacketsSent = 0;
+        numBytesSent = 0;
+        numIdentityPackets = 0;
+        numOthersIncluded = 0;
+        overBudgetAvatars = 0;
+
+        ignoreCalculationElapsedTime = 0;
+        avatarDataPackingElapsedTime = 0;
+        packetSendingElapsedTime = 0;
+        toByteArrayElapsedTime = 0;
+        jobElapsedTime = 0;
+    }
+
+    AvatarMixerSlaveStats& operator+=(const AvatarMixerSlaveStats& rhs) {
+        nodesProcessed += rhs.nodesProcessed;
+        packetsProcessed += rhs.packetsProcessed;
+        processIncomingPacketsElapsedTime += rhs.processIncomingPacketsElapsedTime;
+
+        nodesBroadcastedTo += rhs.nodesBroadcastedTo;
+        numPacketsSent += rhs.numPacketsSent;
+        numBytesSent += rhs.numBytesSent;
+        numIdentityPackets += rhs.numIdentityPackets;
+        numOthersIncluded += rhs.numOthersIncluded;
+        overBudgetAvatars += rhs.overBudgetAvatars;
+
+        ignoreCalculationElapsedTime += rhs.ignoreCalculationElapsedTime;
+        avatarDataPackingElapsedTime += rhs.avatarDataPackingElapsedTime;
+        packetSendingElapsedTime += rhs.packetSendingElapsedTime;
+        toByteArrayElapsedTime += rhs.toByteArrayElapsedTime;
+        jobElapsedTime += rhs.jobElapsedTime;
+        return *this;
+    }
+
+};
+
+class AvatarMixerSlave {
+public:
+    using ConstIter = NodeList::const_iterator;
+
+    void configure(ConstIter begin, ConstIter end);
+    void configureBroadcast(ConstIter begin, ConstIter end, 
+                    p_high_resolution_clock::time_point lastFrameTimestamp, 
+                    float maxKbpsPerNode, float throttlingRatio);
+
+    void processIncomingPackets(const SharedNodePointer& node);
+    void broadcastAvatarData(const SharedNodePointer& node);
+
+    void harvestStats(AvatarMixerSlaveStats& stats);
+
+private:
+    int sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode);
+
+    // frame state
+    ConstIter _begin;
+    ConstIter _end;
+
+    p_high_resolution_clock::time_point _lastFrameTimestamp;
+    float _maxKbpsPerNode { 0.0f };
+    float _throttlingRatio { 0.0f };
+
+    AvatarMixerSlaveStats _stats;
+};
+
+#endif // hifi_AvatarMixerSlave_h

assignment-client/src/avatars/AvatarMixerSlavePool.cpp

@@ -0,0 +1,208 @@
+//
+//  AvatarMixerSlavePool.cpp
+//  assignment-client/src/avatar
+//
+//  Created by Brad Hefta-Gaub on 2/14/2017.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <assert.h>
+#include <algorithm>
+
+#include "AvatarMixerSlavePool.h"
+
+void AvatarMixerSlaveThread::run() {
+    while (true) {
+        wait();
+
+        // iterate over all available nodes
+        SharedNodePointer node;
+        while (try_pop(node)) {
+            (this->*_function)(node);
+        }
+
+        bool stopping = _stop;
+        notify(stopping);
+        if (stopping) {
+            return;
+        }
+    }
+}
+
+void AvatarMixerSlaveThread::wait() {
+    {
+        Lock lock(_pool._mutex);
+        _pool._slaveCondition.wait(lock, [&] {
+            assert(_pool._numStarted <= _pool._numThreads);
+            return _pool._numStarted != _pool._numThreads;
+        });
+        ++_pool._numStarted;
+    }
+    if (_pool._configure) {
+        _pool._configure(*this);
+    }
+    _function = _pool._function;
+}
+
+void AvatarMixerSlaveThread::notify(bool stopping) {
+    {
+        Lock lock(_pool._mutex);
+        assert(_pool._numFinished < _pool._numThreads);
+        ++_pool._numFinished;
+        if (stopping) {
+            ++_pool._numStopped;
+        }
+    }
+    _pool._poolCondition.notify_one();
+}
+
+bool AvatarMixerSlaveThread::try_pop(SharedNodePointer& node) {
+    return _pool._queue.try_pop(node);
+}
+
+#ifdef AVATAR_SINGLE_THREADED
+static AvatarMixerSlave slave;
+#endif
+
+void AvatarMixerSlavePool::processIncomingPackets(ConstIter begin, ConstIter end) {
+    _function = &AvatarMixerSlave::processIncomingPackets;
+    _configure = [&](AvatarMixerSlave& slave) { 
+        slave.configure(begin, end);
+    };
+    run(begin, end);
+}
+
+void AvatarMixerSlavePool::broadcastAvatarData(ConstIter begin, ConstIter end, 
+                                     p_high_resolution_clock::time_point lastFrameTimestamp, 
+                                     float maxKbpsPerNode, float throttlingRatio) {
+    _function = &AvatarMixerSlave::broadcastAvatarData;
+    _configure = [&](AvatarMixerSlave& slave) { 
+        slave.configureBroadcast(begin, end, lastFrameTimestamp, maxKbpsPerNode, throttlingRatio);
+   };
+    run(begin, end);
+}
+
+void AvatarMixerSlavePool::run(ConstIter begin, ConstIter end) {
+    _begin = begin;
+    _end = end;
+
+#ifdef AUDIO_SINGLE_THREADED
+    _configure(slave);
+    std::for_each(begin, end, [&](const SharedNodePointer& node) {
+        _function(slave, node);
+});
+#else
+    // fill the queue
+    std::for_each(_begin, _end, [&](const SharedNodePointer& node) {
+        _queue.emplace(node);
+    });
+
+    {
+        Lock lock(_mutex);
+
+        // run
+        _numStarted = _numFinished = 0;
+        _slaveCondition.notify_all();
+
+        // wait
+        _poolCondition.wait(lock, [&] {
+            assert(_numFinished <= _numThreads);
+            return _numFinished == _numThreads;
+        });
+
+        assert(_numStarted == _numThreads);
+    }
+
+    assert(_queue.empty());
+#endif
+}
+
+
+void AvatarMixerSlavePool::each(std::function<void(AvatarMixerSlave& slave)> functor) {
+#ifdef AVATAR_SINGLE_THREADED
+    functor(slave);
+#else
+    for (auto& slave : _slaves) {
+        functor(*slave.get());
+    }
+#endif
+}
+
+void AvatarMixerSlavePool::setNumThreads(int numThreads) {
+    // clamp to allowed size
+    {
+        int maxThreads = QThread::idealThreadCount();
+        if (maxThreads == -1) {
+            // idealThreadCount returns -1 if cores cannot be detected
+            static const int MAX_THREADS_IF_UNKNOWN = 4;
+            maxThreads = MAX_THREADS_IF_UNKNOWN;
+        }
+
+        int clampedThreads = std::min(std::max(1, numThreads), maxThreads);
+        if (clampedThreads != numThreads) {
+            qWarning("%s: clamped to %d (was %d)", __FUNCTION__, clampedThreads, numThreads);
+            numThreads = clampedThreads;
+        }
+    }
+
+    resize(numThreads);
+}
+
+void AvatarMixerSlavePool::resize(int numThreads) {
+    assert(_numThreads == (int)_slaves.size());
+
+#ifdef AVATAR_SINGLE_THREADED
+    qDebug("%s: running single threaded", __FUNCTION__, numThreads);
+#else
+    qDebug("%s: set %d threads (was %d)", __FUNCTION__, numThreads, _numThreads);
+
+    Lock lock(_mutex);
+
+    if (numThreads > _numThreads) {
+        // start new slaves
+        for (int i = 0; i < numThreads - _numThreads; ++i) {
+            auto slave = new AvatarMixerSlaveThread(*this);
+            slave->start();
+            _slaves.emplace_back(slave);
+        }
+    } else if (numThreads < _numThreads) {
+        auto extraBegin = _slaves.begin() + numThreads;
+
+        // mark slaves to stop...
+        auto slave = extraBegin;
+        while (slave != _slaves.end()) {
+            (*slave)->_stop = true;
+            ++slave;
+        }
+
+        // ...cycle them until they do stop...
+        _numStopped = 0;
+        while (_numStopped != (_numThreads - numThreads)) {
+            _numStarted = _numFinished = _numStopped;
+            _slaveCondition.notify_all();
+            _poolCondition.wait(lock, [&] {
+                assert(_numFinished <= _numThreads);
+                return _numFinished == _numThreads;
+            });
+        }
+
+        // ...wait for threads to finish...
+        slave = extraBegin;
+        while (slave != _slaves.end()) {
+            QThread* thread = reinterpret_cast<QThread*>(slave->get());
+            static const int MAX_THREAD_WAIT_TIME = 10;
+            thread->wait(MAX_THREAD_WAIT_TIME);
+            ++slave;
+        }
+
+        // ...and erase them
+        _slaves.erase(extraBegin, _slaves.end());
+    }
+
+    _numThreads = _numStarted = _numFinished = numThreads;
+    assert(_numThreads == (int)_slaves.size());
+#endif
+}

assignment-client/src/avatars/AvatarMixerSlavePool.h

@@ -0,0 +1,104 @@
+//
+//  AvatarMixerSlavePool.h
+//  assignment-client/src/avatar
+//
+//  Created by Brad Hefta-Gaub on 2/14/2017.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_AvatarMixerSlavePool_h
+#define hifi_AvatarMixerSlavePool_h
+
+#include <condition_variable>
+#include <mutex>
+#include <vector>
+
+#include <tbb/concurrent_queue.h>
+
+#include <QThread>
+
+#include <NodeList.h>
+
+#include "AvatarMixerSlave.h"
+
+class AvatarMixerSlavePool;
+
+class AvatarMixerSlaveThread : public QThread, public AvatarMixerSlave {
+    Q_OBJECT
+    using ConstIter = NodeList::const_iterator;
+    using Mutex = std::mutex;
+    using Lock = std::unique_lock<Mutex>;
+
+public:
+    AvatarMixerSlaveThread(AvatarMixerSlavePool& pool) : _pool(pool) {}
+
+    void run() override final;
+
+private:
+    friend class AvatarMixerSlavePool;
+
+    void wait();
+    void notify(bool stopping);
+    bool try_pop(SharedNodePointer& node);
+
+    AvatarMixerSlavePool& _pool;
+    void (AvatarMixerSlave::*_function)(const SharedNodePointer& node) { nullptr };
+    bool _stop { false };
+};
+
+// Slave pool for audio mixers
+//   AvatarMixerSlavePool is not thread-safe! It should be instantiated and used from a single thread.
+class AvatarMixerSlavePool {
+    using Queue = tbb::concurrent_queue<SharedNodePointer>;
+    using Mutex = std::mutex;
+    using Lock = std::unique_lock<Mutex>;
+    using ConditionVariable = std::condition_variable;
+
+public:
+    using ConstIter = NodeList::const_iterator;
+
+    AvatarMixerSlavePool(int numThreads = QThread::idealThreadCount()) { setNumThreads(numThreads); }
+    ~AvatarMixerSlavePool() { resize(0); }
+
+    // Jobs the slave pool can do...
+    void processIncomingPackets(ConstIter begin, ConstIter end);
+    void broadcastAvatarData(ConstIter begin, ConstIter end, 
+                    p_high_resolution_clock::time_point lastFrameTimestamp, float maxKbpsPerNode, float throttlingRatio);
+
+    // iterate over all slaves
+    void each(std::function<void(AvatarMixerSlave& slave)> functor);
+
+    void setNumThreads(int numThreads);
+    int numThreads() { return _numThreads; }
+
+private:
+    void run(ConstIter begin, ConstIter end);
+    void resize(int numThreads);
+
+    std::vector<std::unique_ptr<AvatarMixerSlaveThread>> _slaves;
+
+    friend void AvatarMixerSlaveThread::wait();
+    friend void AvatarMixerSlaveThread::notify(bool stopping);
+    friend bool AvatarMixerSlaveThread::try_pop(SharedNodePointer& node);
+
+    // synchronization state
+    Mutex _mutex;
+    ConditionVariable _slaveCondition;
+    ConditionVariable _poolCondition;
+    void (AvatarMixerSlave::*_function)(const SharedNodePointer& node);
+    std::function<void(AvatarMixerSlave&)> _configure;
+    int _numThreads { 0 };
+    int _numStarted { 0 }; // guarded by _mutex
+    int _numFinished { 0 }; // guarded by _mutex
+    int _numStopped { 0 }; // guarded by _mutex
+
+    // frame state
+    Queue _queue;
+    ConstIter _begin;
+    ConstIter _end;
+};
+
+#endif // hifi_AvatarMixerSlavePool_h

assignment-client/src/avatars/ScriptableAvatar.cpp

@@ -14,10 +14,9 @@
 #include <GLMHelpers.h>
 #include "ScriptableAvatar.h"
 
-QByteArray ScriptableAvatar::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
-                                    bool distanceAdjust, glm::vec3 viewerPosition, QVector<JointData>* sentJointDataOut) {
+QByteArray ScriptableAvatar::toByteArrayStateful(AvatarDataDetail dataDetail) {
     _globalPosition = getPosition();
-    return AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut);
+    return AvatarData::toByteArrayStateful(dataDetail);
 }
 
 

assignment-client/src/avatars/ScriptableAvatar.h

@@ -28,8 +28,7 @@ public:
     Q_INVOKABLE AnimationDetails getAnimationDetails();
     virtual void setSkeletonModelURL(const QUrl& skeletonModelURL) override;
 
-    virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
-                        bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector<JointData>* sentJointDataOut = nullptr) override;
+    virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail) override;
 
     
 private slots:

assignment-client/src/entities/EntityServer.cpp

@@ -17,10 +17,11 @@
 #include <ScriptCache.h>
 #include <EntityEditFilters.h>
 
+#include "AssignmentParentFinder.h"
+#include "EntityNodeData.h"
 #include "EntityServer.h"
 #include "EntityServerConsts.h"
-#include "EntityNodeData.h"
-#include "AssignmentParentFinder.h"
+#include "EntityTreeSendThread.h"
 
 const char* MODEL_SERVER_NAME = "Entity";
 const char* MODEL_SERVER_LOGGING_TARGET_NAME = "entity-server";
@@ -77,6 +78,10 @@ OctreePointer EntityServer::createTree() {
     return tree;
 }
 
+OctreeServer::UniqueSendThread EntityServer::newSendThread(const SharedNodePointer& node) {
+    return std::unique_ptr<EntityTreeSendThread>(new EntityTreeSendThread(this, node));
+}
+
 void EntityServer::beforeRun() {
     _pruneDeletedEntitiesTimer = new QTimer();
     connect(_pruneDeletedEntitiesTimer, SIGNAL(timeout()), this, SLOT(pruneDeletedEntities()));

assignment-client/src/entities/EntityServer.h

@@ -67,6 +67,7 @@ public slots:
 
 protected:
     virtual OctreePointer createTree() override;
+    virtual UniqueSendThread newSendThread(const SharedNodePointer& node) override;
 
 private slots:
     void handleEntityPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);

assignment-client/src/entities/EntityTreeSendThread.cpp

@@ -0,0 +1,132 @@
+//
+//  EntityTreeSendThread.cpp
+//  assignment-client/src/entities
+//
+//  Created by Stephen Birarda on 2/15/17.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "EntityTreeSendThread.h"
+
+#include <EntityNodeData.h>
+#include <EntityTypes.h>
+
+#include "EntityServer.h"
+
+void EntityTreeSendThread::preDistributionProcessing() {
+    auto node = _node.toStrongRef();
+    auto nodeData = static_cast<EntityNodeData*>(node->getLinkedData());
+
+    if (nodeData) {
+        auto jsonQuery = nodeData->getJSONParameters();
+
+        // check if we have a JSON query with flags
+        auto flags = jsonQuery[EntityJSONQueryProperties::FLAGS_PROPERTY].toObject();
+        if (!flags.isEmpty()) {
+            // check the flags object for specific flags that require special pre-processing
+
+            bool includeAncestors = flags[EntityJSONQueryProperties::INCLUDE_ANCESTORS_PROPERTY].toBool();
+            bool includeDescendants = flags[EntityJSONQueryProperties::INCLUDE_DESCENDANTS_PROPERTY].toBool();
+
+            if (includeAncestors || includeDescendants) {
+                // we need to either include the ancestors, descendants, or both for entities matching the filter
+                // included in the JSON query
+
+                // first reset our flagged extra entities so we start with an empty set
+                nodeData->resetFlaggedExtraEntities();
+
+                auto entityTree = std::static_pointer_cast<EntityTree>(_myServer->getOctree());
+
+                bool requiresFullScene = false;
+
+                // enumerate the set of entity IDs we know currently match the filter
+                foreach(const QUuid& entityID, nodeData->getSentFilteredEntities()) {
+                    if (includeAncestors) {
+                        // we need to include ancestors - recurse up to reach them all and add their IDs
+                        // to the set of extra entities to include for this node
+                        entityTree->withReadLock([&]{
+                            auto filteredEntity = entityTree->findEntityByID(entityID);
+                            if (filteredEntity) {
+                                requiresFullScene |= addAncestorsToExtraFlaggedEntities(entityID, *filteredEntity, *nodeData);
+                            }
+                        });
+                    }
+
+                    if (includeDescendants) {
+                        // we need to include descendants - recurse down to reach them all and add their IDs
+                        // to the set of extra entities to include for this node
+                        entityTree->withReadLock([&]{
+                            auto filteredEntity = entityTree->findEntityByID(entityID);
+                            if (filteredEntity) {
+                                requiresFullScene |= addDescendantsToExtraFlaggedEntities(entityID, *filteredEntity, *nodeData);
+                            }
+                        });
+                    }
+                }
+
+                if (requiresFullScene) {
+                    // for one or more of the entities matching our filter we found new extra entities to include
+
+                    // because it is possible that one of these entities hasn't changed since our last send
+                    // and therefore would not be recursed to, we need to force a full traversal for this pass
+                    // of the tree to allow it to grab all of the extra entities we're asking it to include
+                    nodeData->setShouldForceFullScene(requiresFullScene);
+                }
+            }
+        }
+    }
+}
+
+bool EntityTreeSendThread::addAncestorsToExtraFlaggedEntities(const QUuid& filteredEntityID,
+                                                              EntityItem& entityItem, EntityNodeData& nodeData) {
+    // check if this entity has a parent that is also an entity
+    bool success = false;
+    auto entityParent = entityItem.getParentPointer(success);
+
+    if (success && entityParent && entityParent->getNestableType() == NestableType::Entity) {
+        // we found a parent that is an entity item
+
+        // first add it to the extra list of things we need to send
+        bool parentWasNew = nodeData.insertFlaggedExtraEntity(filteredEntityID, entityParent->getID());
+
+        // now recursively call ourselves to get its ancestors added too
+        auto parentEntityItem = std::static_pointer_cast<EntityItem>(entityParent);
+        bool ancestorsWereNew = addAncestorsToExtraFlaggedEntities(filteredEntityID, *parentEntityItem, nodeData);
+
+        // return boolean if our parent or any of our ancestors were new additions (via insertFlaggedExtraEntity)
+        return parentWasNew || ancestorsWereNew;
+    }
+
+    // since we didn't have a parent niether of our parents or ancestors could be new additions
+    return false;
+}
+
+bool EntityTreeSendThread::addDescendantsToExtraFlaggedEntities(const QUuid& filteredEntityID,
+                                                                EntityItem& entityItem, EntityNodeData& nodeData) {
+    bool hasNewChild = false;
+    bool hasNewDescendants = false;
+
+    // enumerate the immediate children of this entity
+    foreach (SpatiallyNestablePointer child, entityItem.getChildren()) {
+
+        if (child && child->getNestableType() == NestableType::Entity) {
+            // this is a child that is an entity
+
+            // first add it to the extra list of things we need to send
+            hasNewChild |= nodeData.insertFlaggedExtraEntity(filteredEntityID, child->getID());
+
+            // now recursively call ourselves to get its descendants added too
+            auto childEntityItem = std::static_pointer_cast<EntityItem>(child);
+            hasNewDescendants |= addDescendantsToExtraFlaggedEntities(filteredEntityID, *childEntityItem, nodeData);
+        }
+    }
+
+    // return our boolean indicating if we added new children or descendants as extra entities to send
+    // (via insertFlaggedExtraEntity)
+    return hasNewChild || hasNewDescendants;
+}
+
+

assignment-client/src/entities/EntityTreeSendThread.h

@@ -0,0 +1,35 @@
+//
+//  EntityTreeSendThread.h
+//  assignment-client/src/entities
+//
+//  Created by Stephen Birarda on 2/15/17.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_EntityTreeSendThread_h
+#define hifi_EntityTreeSendThread_h
+
+#include "../octree/OctreeSendThread.h"
+
+class EntityNodeData;
+class EntityItem;
+
+class EntityTreeSendThread : public OctreeSendThread {
+
+public:
+    EntityTreeSendThread(OctreeServer* myServer, const SharedNodePointer& node) : OctreeSendThread(myServer, node) {};
+
+protected:
+    virtual void preDistributionProcessing() override;
+
+private:
+    // the following two methods return booleans to indicate if any extra flagged entities were new additions to set
+    bool addAncestorsToExtraFlaggedEntities(const QUuid& filteredEntityID, EntityItem& entityItem, EntityNodeData& nodeData);
+    bool addDescendantsToExtraFlaggedEntities(const QUuid& filteredEntityID, EntityItem& entityItem, EntityNodeData& nodeData);
+
+};
+
+#endif // hifi_EntityTreeSendThread_h

assignment-client/src/octree/OctreeSendThread.cpp

@@ -309,6 +309,12 @@ int OctreeSendThread::packetDistributor(SharedNodePointer node, OctreeQueryNode*
         return 0;
     }
 
+    if (nodeData->elementBag.isEmpty()) {
+        // if we're about to do a fresh pass,
+        // give our pre-distribution processing a chance to do what it needs
+        preDistributionProcessing();
+    }
+
     // calculate max number of packets that can be sent during this interval
     int clientMaxPacketsPerInterval = std::max(1, (nodeData->getMaxQueryPacketsPerSecond() / INTERVALS_PER_SECOND));
     int maxPacketsPerInterval = std::min(clientMaxPacketsPerInterval, _myServer->getPacketsPerClientPerInterval());
@@ -316,9 +322,17 @@ int OctreeSendThread::packetDistributor(SharedNodePointer node, OctreeQueryNode*
     int truePacketsSent = 0;
     int trueBytesSent = 0;
     int packetsSentThisInterval = 0;
-    bool isFullScene = nodeData->haveJSONParametersChanged() ||
-        (nodeData->getUsesFrustum()
-         && ((!viewFrustumChanged && nodeData->getViewFrustumJustStoppedChanging()) || nodeData->hasLodChanged()));
+
+    bool isFullScene = nodeData->shouldForceFullScene();
+    if (isFullScene) {
+        // we're forcing a full scene, clear the force in OctreeQueryNode so we don't force it next time again
+        nodeData->setShouldForceFullScene(false);
+    } else {
+        // we aren't forcing a full scene, check if something else suggests we should
+        isFullScene = nodeData->haveJSONParametersChanged() ||
+            (nodeData->getUsesFrustum()
+             && ((!viewFrustumChanged && nodeData->getViewFrustumJustStoppedChanging()) || nodeData->hasLodChanged()));
+    }
 
     bool somethingToSend = true; // assume we have something
 

assignment-client/src/octree/OctreeSendThread.h

@@ -17,6 +17,8 @@
 #include <atomic>
 
 #include <GenericThread.h>
+#include <Node.h>
+#include <OctreePacketData.h>
 
 class OctreeQueryNode;
 class OctreeServer;
@@ -49,13 +51,17 @@ protected:
     /// Implements generic processing behavior for this thread.
     virtual bool process() override;
 
+    /// Called before a packetDistributor pass to allow for pre-distribution processing
+    virtual void preDistributionProcessing() {};
+
+    OctreeServer* _myServer { nullptr };
+    QWeakPointer<Node> _node;
+
 private:
     int handlePacketSend(SharedNodePointer node, OctreeQueryNode* nodeData, int& trueBytesSent, int& truePacketsSent, bool dontSuppressDuplicate = false);
     int packetDistributor(SharedNodePointer node, OctreeQueryNode* nodeData, bool viewFrustumChanged);
     
-    
-    OctreeServer* _myServer { nullptr };
-    QWeakPointer<Node> _node;
+
     QUuid _nodeUuid;
 
     OctreePacketData _packetData;

assignment-client/src/octree/OctreeServer.cpp

@@ -872,8 +872,12 @@ void OctreeServer::parsePayload() {
     }
 }
 
+OctreeServer::UniqueSendThread OctreeServer::newSendThread(const SharedNodePointer& node) {
+    return std::unique_ptr<OctreeSendThread>(new OctreeSendThread(this, node));
+}
+
 OctreeServer::UniqueSendThread OctreeServer::createSendThread(const SharedNodePointer& node) {
-    auto sendThread = std::unique_ptr<OctreeSendThread>(new OctreeSendThread(this, node));
+    auto sendThread = newSendThread(node);
     
     // we want to be notified when the thread finishes
     connect(sendThread.get(), &GenericThread::finished, this, &OctreeServer::removeSendThread);

assignment-client/src/octree/OctreeServer.h

@@ -158,6 +158,7 @@ protected:
     QString getStatusLink();
     
     UniqueSendThread createSendThread(const SharedNodePointer& node);
+    virtual UniqueSendThread newSendThread(const SharedNodePointer& node);
 
     int _argc;
     const char** _argv;

assignment-client/src/scripts/EntityScriptServer.cpp

@@ -16,6 +16,7 @@
 #include <AudioConstants.h>
 #include <AudioInjectorManager.h>
 #include <ClientServerUtils.h>
+#include <EntityNodeData.h>
 #include <EntityScriptingInterface.h>
 #include <LogHandler.h>
 #include <MessagesClient.h>
@@ -264,8 +265,14 @@ void EntityScriptServer::run() {
     
     // setup the JSON filter that asks for entities with a non-default serverScripts property
     QJsonObject queryJSONParameters;
-    static const QString SERVER_SCRIPTS_PROPERTY = "serverScripts";
-    queryJSONParameters[SERVER_SCRIPTS_PROPERTY] = EntityQueryFilterSymbol::NonDefault;
+    queryJSONParameters[EntityJSONQueryProperties::SERVER_SCRIPTS_PROPERTY] = EntityQueryFilterSymbol::NonDefault;
+
+    QJsonObject queryFlags;
+
+    queryFlags[EntityJSONQueryProperties::INCLUDE_ANCESTORS_PROPERTY] = true;
+    queryFlags[EntityJSONQueryProperties::INCLUDE_DESCENDANTS_PROPERTY] = true;
+
+    queryJSONParameters[EntityJSONQueryProperties::FLAGS_PROPERTY] = queryFlags;
     
     // setup the JSON parameters so that OctreeQuery does not use a frustum and uses our JSON filter
     _entityViewer.getOctreeQuery().setUsesFrustum(false);
@@ -412,6 +419,7 @@ void EntityScriptServer::resetEntitiesScriptEngine() {
 
     connect(newEngine.data(), &ScriptEngine::update, this, [this] {
         _entityViewer.queryOctree();
+        _entityViewer.getTree()->update();
     });
 
 

domain-server/resources/describe-settings.json

@@ -1299,6 +1299,22 @@
           "placeholder": 5.0,
           "default": 5.0,
           "advanced": true
+        },
+        {
+          "name": "auto_threads",
+          "label": "Automatically determine thread count",
+          "type": "checkbox",
+          "help": "Allow system to determine number of threads (recommended)",
+          "default": false,
+          "advanced": true
+        },
+        {
+          "name": "num_threads",
+          "label": "Number of Threads",
+          "help": "Threads to spin up for avatar mixing (if not automatically set)",
+          "placeholder": "1",
+          "default": "1",
+          "advanced": true
         }
       ]
     }

interface/resources/qml/Stats.qml

@@ -107,11 +107,11 @@ Item {
                     }
                     StatText {
                         visible: root.expanded
-                        text: "Fully Simulated Avatars: " + root.fullySimulatedAvatarCount
+                        text: "Avatars Updated: " + root.updatedAvatarCount
                     }
                     StatText {
                         visible: root.expanded
-                        text: "Partially Simulated Avatars: " + root.partiallySimulatedAvatarCount
+                        text: "Avatars NOT Updated: " + root.notUpdatedAvatarCount
                     }
                 }
             }

interface/resources/qml/hifi/toolbars/Toolbar.qml

@@ -120,6 +120,8 @@ Window {
     function addButton(properties) {
         properties = properties || {}
 
+        unpinnedAlpha = 1;
+
         // If a name is specified, then check if there's an existing button with that name
         // and return it if so.  This will allow multiple clients to listen to a single button,
         // and allow scripts to be idempotent so they don't duplicate buttons if they're reloaded
@@ -154,7 +156,7 @@ Window {
         updatePinned();
 
         if (buttons.length === 0) {
-            visible = false;
+            unpinnedAlpha = 0;
         }
     }
 

interface/src/Application.cpp

@@ -170,6 +170,7 @@
 #include "ui/StandAloneJSConsole.h"
 #include "ui/Stats.h"
 #include "ui/UpdateDialog.h"
+#include "ui/overlays/Overlays.h"
 #include "Util.h"
 #include "InterfaceParentFinder.h"
 
@@ -528,7 +529,7 @@ bool setupEssentials(int& argc, char** argv) {
 // to take care of highlighting keyboard focused items, rather than
 // continuing to overburden Application.cpp
 std::shared_ptr<Cube3DOverlay> _keyboardFocusHighlight{ nullptr };
-int _keyboardFocusHighlightID{ -1 };
+OverlayID _keyboardFocusHighlightID{ UNKNOWN_OVERLAY_ID };
 
 
 // FIXME hack access to the internal share context for the Chromium helper
@@ -1232,12 +1233,12 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
     // Keyboard focus handling for Web overlays.
     auto overlays = &(qApp->getOverlays());
 
-    connect(overlays, &Overlays::mousePressOnOverlay, [=](unsigned int overlayID, const PointerEvent& event) {
+    connect(overlays, &Overlays::mousePressOnOverlay, [=](OverlayID overlayID, const PointerEvent& event) {
         setKeyboardFocusEntity(UNKNOWN_ENTITY_ID);
         setKeyboardFocusOverlay(overlayID);
     });
 
-    connect(overlays, &Overlays::overlayDeleted, [=](unsigned int overlayID) {
+    connect(overlays, &Overlays::overlayDeleted, [=](OverlayID overlayID) {
         if (overlayID == _keyboardFocusedOverlay.get()) {
             setKeyboardFocusOverlay(UNKNOWN_OVERLAY_ID);
         }
@@ -1689,9 +1690,9 @@ void Application::cleanupBeforeQuit() {
     _applicationStateDevice.reset();
 
     {
-        if (_keyboardFocusHighlightID > 0) {
+        if (_keyboardFocusHighlightID != UNKNOWN_OVERLAY_ID) {
             getOverlays().deleteOverlay(_keyboardFocusHighlightID);
-            _keyboardFocusHighlightID = -1;
+            _keyboardFocusHighlightID = UNKNOWN_OVERLAY_ID;
         }
         _keyboardFocusHighlight = nullptr;
     }
@@ -3088,7 +3089,7 @@ void Application::mouseMoveEvent(QMouseEvent* event) {
         buttons, event->modifiers());
 
     if (compositor.getReticleVisible() || !isHMDMode() || !compositor.getReticleOverDesktop() ||
-        getOverlays().getOverlayAtPoint(glm::vec2(transformedPos.x(), transformedPos.y()))) {
+        getOverlays().getOverlayAtPoint(glm::vec2(transformedPos.x(), transformedPos.y())) != UNKNOWN_OVERLAY_ID) {
         getOverlays().mouseMoveEvent(&mappedEvent);
         getEntities()->mouseMoveEvent(&mappedEvent);
     }
@@ -4107,7 +4108,7 @@ void Application::rotationModeChanged() const {
 
 void Application::setKeyboardFocusHighlight(const glm::vec3& position, const glm::quat& rotation, const glm::vec3& dimensions) {
     // Create focus
-    if (_keyboardFocusHighlightID < 0 || !getOverlays().isAddedOverlay(_keyboardFocusHighlightID)) {
+    if (_keyboardFocusHighlightID == UNKNOWN_OVERLAY_ID || !getOverlays().isAddedOverlay(_keyboardFocusHighlightID)) {
         _keyboardFocusHighlight = std::make_shared<Cube3DOverlay>();
         _keyboardFocusHighlight->setAlpha(1.0f);
         _keyboardFocusHighlight->setBorderSize(1.0f);
@@ -4169,11 +4170,11 @@ void Application::setKeyboardFocusEntity(EntityItemID entityItemID) {
     }
 }
 
-unsigned int Application::getKeyboardFocusOverlay() {
+OverlayID Application::getKeyboardFocusOverlay() {
     return _keyboardFocusedOverlay.get();
 }
 
-void Application::setKeyboardFocusOverlay(unsigned int overlayID) {
+void Application::setKeyboardFocusOverlay(OverlayID overlayID) {
     if (overlayID != _keyboardFocusedOverlay.get()) {
         _keyboardFocusedOverlay.set(overlayID);
 
@@ -5546,6 +5547,8 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEngine* scri
     auto entityScriptServerLog = DependencyManager::get<EntityScriptServerLogClient>();
     scriptEngine->registerGlobalObject("EntityScriptServerLog", entityScriptServerLog.data());
 
+    qScriptRegisterMetaType(scriptEngine, OverlayIDtoScriptValue, OverlayIDfromScriptValue);
+
     // connect this script engines printedMessage signal to the global ScriptEngines these various messages
     connect(scriptEngine, &ScriptEngine::printedMessage, DependencyManager::get<ScriptEngines>().data(), &ScriptEngines::onPrintedMessage);
     connect(scriptEngine, &ScriptEngine::errorMessage, DependencyManager::get<ScriptEngines>().data(), &ScriptEngines::onErrorMessage);
@@ -6891,3 +6894,13 @@ void Application::toggleMuteAudio() {
     auto menu = Menu::getInstance();
     menu->setIsOptionChecked(MenuOption::MuteAudio, !menu->isOptionChecked(MenuOption::MuteAudio));
 }
+
+OverlayID Application::getTabletScreenID() const {
+    auto HMD = DependencyManager::get<HMDScriptingInterface>();
+    return HMD->getCurrentTabletScreenID();
+}
+
+OverlayID Application::getTabletHomeButtonID() const {
+    auto HMD = DependencyManager::get<HMDScriptingInterface>();
+    return HMD->getCurrentHomeButtonUUID();
+}

interface/src/Application.h

@@ -298,6 +298,9 @@ public:
     Q_INVOKABLE void sendHoverOverEntity(QUuid id, PointerEvent event);
     Q_INVOKABLE void sendHoverLeaveEntity(QUuid id, PointerEvent event);
 
+    OverlayID getTabletScreenID() const;
+    OverlayID getTabletHomeButtonID() const;
+
 signals:
     void svoImportRequested(const QString& url);
 
@@ -387,8 +390,8 @@ public slots:
     void setKeyboardFocusEntity(QUuid id);
     void setKeyboardFocusEntity(EntityItemID entityItemID);
 
-    unsigned int getKeyboardFocusOverlay();
-    void setKeyboardFocusOverlay(unsigned int overlayID);
+    OverlayID getKeyboardFocusOverlay();
+    void setKeyboardFocusOverlay(OverlayID overlayID);
 
     void addAssetToWorldMessageClose();
 
@@ -618,7 +621,7 @@ private:
     DialogsManagerScriptingInterface* _dialogsManagerScriptingInterface = new DialogsManagerScriptingInterface();
 
     ThreadSafeValueCache<EntityItemID> _keyboardFocusedEntity;
-    ThreadSafeValueCache<unsigned int> _keyboardFocusedOverlay;
+    ThreadSafeValueCache<OverlayID> _keyboardFocusedOverlay;
     quint64 _lastAcceptedKeyPress = 0;
     bool _isForeground = true; // starts out assumed to be in foreground
     bool _inPaint = false;

interface/src/InterfaceParentFinder.cpp

@@ -45,12 +45,20 @@ SpatiallyNestableWeakPointer InterfaceParentFinder::find(QUuid parentID, bool& s
         success = true;
         return parent;
     }
-
     if (parentID == AVATAR_SELF_ID) {
         success = true;
         return avatarManager->getMyAvatar();
     }
 
+    // search overlays
+    auto& overlays = qApp->getOverlays();
+    auto overlay = overlays.getOverlay(parentID);
+    parent = std::dynamic_pointer_cast<SpatiallyNestable>(overlay); // this will return nullptr for non-3d overlays
+    if (!parent.expired()) {
+        success = true;
+        return parent;
+    }
+
     success = false;
     return parent;
 }

interface/src/avatar/Avatar.cpp

@@ -334,11 +334,6 @@ void Avatar::updateAvatarEntities() {
     setAvatarEntityDataChanged(false);
 }
 
-bool Avatar::shouldDie() const {
-    const qint64 AVATAR_SILENCE_THRESHOLD_USECS = 5 * USECS_PER_SECOND;
-    return _owningAvatarMixer.isNull() || getUsecsSinceLastUpdate() > AVATAR_SILENCE_THRESHOLD_USECS;
-}
-
 void Avatar::simulate(float deltaTime, bool inView) {
     PROFILE_RANGE(simulation, "simulate");
 

interface/src/avatar/Avatar.h

@@ -178,12 +178,13 @@ public:
     uint64_t getLastRenderUpdateTime() const { return _lastRenderUpdateTime; }
     void setLastRenderUpdateTime(uint64_t time) { _lastRenderUpdateTime = time; }
 
-    bool shouldDie() const;
     void animateScaleChanges(float deltaTime);
     void setTargetScale(float targetScale) override;
 
     Q_INVOKABLE float getSimulationRate(const QString& rateName = QString("")) const;
 
+    bool hasNewJointData() const { return _hasNewJointData; }
+
 public slots:
 
     // FIXME - these should be migrated to use Pose data instead

interface/src/avatar/AvatarManager.cpp

@@ -148,16 +148,6 @@ float AvatarManager::getAvatarSimulationRate(const QUuid& sessionID, const QStri
 }
 
 
-
-class AvatarPriority {
-public:
-    AvatarPriority(AvatarSharedPointer a, float p) : avatar(a), priority(p) {}
-    AvatarSharedPointer avatar;
-    float priority;
-    // NOTE: we invert the less-than operator to sort high priorities to front
-    bool operator<(const AvatarPriority& other) const { return priority > other.priority; }
-};
-
 void AvatarManager::updateOtherAvatars(float deltaTime) {
     // lock the hash for read to check the size
     QReadLocker lock(&_hashLock);
@@ -173,57 +163,35 @@ void AvatarManager::updateOtherAvatars(float deltaTime) {
     QList<AvatarSharedPointer> avatarList = avatarMap.values();
     ViewFrustum cameraView;
     qApp->copyDisplayViewFrustum(cameraView);
-    glm::vec3 frustumCenter = cameraView.getPosition();
 
-    const float OUT_OF_VIEW_PENALTY = -10.0;
+    std::priority_queue<AvatarPriority> sortedAvatars = AvatarData::sortAvatars(
+        avatarList, cameraView,
 
-    std::priority_queue<AvatarPriority> sortedAvatars;
-    {
-        PROFILE_RANGE(simulation, "sort");
-        for (int32_t i = 0; i < avatarList.size(); ++i) {
-            const auto& avatar = std::static_pointer_cast<Avatar>(avatarList.at(i));
-            if (avatar == _myAvatar || !avatar->isInitialized()) {
+        [](AvatarSharedPointer avatar)->uint64_t{
+            return std::static_pointer_cast<Avatar>(avatar)->getLastRenderUpdateTime();
+        },
+
+        [](AvatarSharedPointer avatar)->float{
+            return std::static_pointer_cast<Avatar>(avatar)->getBoundingRadius();
+        },
+
+        [this](AvatarSharedPointer avatar)->bool{
+            const auto& castedAvatar = std::static_pointer_cast<Avatar>(avatar);
+            if (castedAvatar == _myAvatar || !castedAvatar->isInitialized()) {
                 // DO NOT update _myAvatar!  Its update has already been done earlier in the main loop.
                 // DO NOT update or fade out uninitialized Avatars
-                continue;
+                return true; // ignore it
             }
             if (avatar->shouldDie()) {
                 removeAvatar(avatar->getID());
-                continue;
+                return true; // ignore it
             }
             if (avatar->isDead()) {
-                continue;
+                return true; // ignore it
             }
 
-            // priority = weighted linear combination of:
-            //   (a) apparentSize
-            //   (b) proximity to center of view
-            //   (c) time since last update
-            //   (d) TIME_PENALTY to help recently updated entries sort toward back
-            glm::vec3 avatarPosition = avatar->getPosition();
-            glm::vec3 offset = avatarPosition - frustumCenter;
-            float distance = glm::length(offset) + 0.001f; // add 1mm to avoid divide by zero
-            float radius = avatar->getBoundingRadius();
-            const glm::vec3& forward = cameraView.getDirection();
-            float apparentSize = radius / distance;
-            float cosineAngle = glm::length(offset - glm::dot(offset, forward) * forward) / distance;
-            const float TIME_PENALTY = 0.080f; // seconds
-            float age = (float)(startTime - avatar->getLastRenderUpdateTime()) / (float)(USECS_PER_SECOND) - TIME_PENALTY;
-            // NOTE: we are adding values of different units to get a single measure of "priority".
-            // Thus we multiply each component by a conversion "weight" that scales its units
-            // relative to the others.  These weights are pure magic tuning and are hard coded in the
-            // relation below: (hint: unitary weights are not explicityly shown)
-            float priority = apparentSize + 0.25f * cosineAngle + age;
-
-            // decrement priority of avatars outside keyhole
-            if (distance > cameraView.getCenterRadius()) {
-                if (!cameraView.sphereIntersectsFrustum(avatarPosition, radius)) {
-                    priority += OUT_OF_VIEW_PENALTY;
-                }
-            }
-            sortedAvatars.push(AvatarPriority(avatar, priority));
-        }
-    }
+            return false;
+        });
 
     render::PendingChanges pendingChanges;
     const uint64_t RENDER_UPDATE_BUDGET = 1500; // usec
@@ -231,8 +199,8 @@ void AvatarManager::updateOtherAvatars(float deltaTime) {
     uint64_t renderExpiry = startTime + RENDER_UPDATE_BUDGET;
     uint64_t maxExpiry = startTime + MAX_UPDATE_BUDGET;
 
-    int fullySimulatedAvatars = 0;
-    int partiallySimulatedAvatars = 0;
+    int numAvatarsUpdated = 0;
+    int numAVatarsNotUpdated = 0;
     while (!sortedAvatars.empty()) {
         const AvatarPriority& sortData = sortedAvatars.top();
         const auto& avatar = std::static_pointer_cast<Avatar>(sortData.avatar);
@@ -253,33 +221,57 @@ void AvatarManager::updateOtherAvatars(float deltaTime) {
         }
         avatar->animateScaleChanges(deltaTime);
 
+        const float OUT_OF_VIEW_THRESHOLD = 0.5f * AvatarData::OUT_OF_VIEW_PENALTY;
         uint64_t now = usecTimestampNow();
         if (now < renderExpiry) {
             // we're within budget
-            const float OUT_OF_VIEW_THRESHOLD = 0.5f * OUT_OF_VIEW_PENALTY;
             bool inView = sortData.priority > OUT_OF_VIEW_THRESHOLD;
+            if (inView && avatar->hasNewJointData()) {
+                numAvatarsUpdated++;
+            }
             avatar->simulate(deltaTime, inView);
             avatar->updateRenderItem(pendingChanges);
             avatar->setLastRenderUpdateTime(startTime);
-            fullySimulatedAvatars++;
         } else if (now < maxExpiry) {
             // we've spent most of our time budget, but we still simulate() the avatar as it if were out of view
             // --> some avatars may freeze until their priority trickles up
-            const bool inView = false;
-            avatar->simulate(deltaTime, inView);
-            partiallySimulatedAvatars++;
+            bool inView = sortData.priority > OUT_OF_VIEW_THRESHOLD;
+            if (inView && avatar->hasNewJointData()) {
+                numAVatarsNotUpdated++;
+            }
+            avatar->simulate(deltaTime, false);
         } else {
             // we've spent ALL of our time budget --> bail on the rest of the avatar updates
+            // --> more avatars may freeze until their priority trickles up
             // --> some scale or fade animations may glitch
             // --> some avatar velocity measurements may be a little off
+
+            // HACK: no time simulate, but we will take the time to count how many were tragically missed
+            bool inView = sortData.priority > OUT_OF_VIEW_THRESHOLD;
+            if (!inView) {
+                break;
+            }
+            if (inView && avatar->hasNewJointData()) {
+                numAVatarsNotUpdated++;
+            }
+            sortedAvatars.pop();
+            while (inView && !sortedAvatars.empty()) {
+                const AvatarPriority& newSortData = sortedAvatars.top();
+                const auto& newAvatar = std::static_pointer_cast<Avatar>(newSortData.avatar);
+                inView = newSortData.priority > OUT_OF_VIEW_THRESHOLD;
+                if (inView && newAvatar->hasNewJointData()) {
+                    numAVatarsNotUpdated++;
+                }
+                sortedAvatars.pop();
+            }
             break;
         }
         sortedAvatars.pop();
     }
 
     _avatarSimulationTime = (float)(usecTimestampNow() - startTime) / (float)USECS_PER_MSEC;
-    _fullySimulatedAvatars = fullySimulatedAvatars;
-    _partiallySimulatedAvatars = partiallySimulatedAvatars;
+    _numAvatarsUpdated = numAvatarsUpdated;
+    _numAvatarsNotUpdated = numAVatarsNotUpdated;
     qApp->getMain3DScene()->enqueuePendingChanges(pendingChanges);
 
     simulateAvatarFades(deltaTime);
@@ -593,3 +585,44 @@ RayToAvatarIntersectionResult AvatarManager::findRayIntersection(const PickRay&
 
     return result;
 }
+
+// HACK
+float AvatarManager::getAvatarSortCoefficient(const QString& name) {
+    if (name == "size") {
+        return AvatarData::_avatarSortCoefficientSize;
+    } else if (name == "center") {
+        return AvatarData::_avatarSortCoefficientCenter;
+    } else if (name == "age") {
+        return AvatarData::_avatarSortCoefficientAge;
+    }
+    return 0.0f;
+}
+
+// HACK
+void AvatarManager::setAvatarSortCoefficient(const QString& name, const QScriptValue& value) {
+    bool somethingChanged = false;
+    if (value.isNumber()) {
+        float numericalValue = (float)value.toNumber();
+        if (name == "size") {
+            AvatarData::_avatarSortCoefficientSize = numericalValue;
+            somethingChanged = true;
+        } else if (name == "center") {
+            AvatarData::_avatarSortCoefficientCenter = numericalValue;
+            somethingChanged = true;
+        } else if (name == "age") {
+            AvatarData::_avatarSortCoefficientAge = numericalValue;
+            somethingChanged = true;
+        }
+    }
+    if (somethingChanged) {
+        size_t packetSize = sizeof(AvatarData::_avatarSortCoefficientSize) + 
+                            sizeof(AvatarData::_avatarSortCoefficientCenter) +
+                            sizeof(AvatarData::_avatarSortCoefficientAge);
+
+        auto packet = NLPacket::create(PacketType::AdjustAvatarSorting, packetSize);
+        packet->writePrimitive(AvatarData::_avatarSortCoefficientSize);
+        packet->writePrimitive(AvatarData::_avatarSortCoefficientCenter);
+        packet->writePrimitive(AvatarData::_avatarSortCoefficientAge);
+        DependencyManager::get<NodeList>()->broadcastToNodes(std::move(packet), NodeSet() << NodeType::AvatarMixer);
+    }
+}

interface/src/avatar/AvatarManager.h

@@ -43,8 +43,8 @@ public:
     std::shared_ptr<MyAvatar> getMyAvatar() { return _myAvatar; }
     AvatarSharedPointer getAvatarBySessionID(const QUuid& sessionID) const override;
 
-    int getFullySimulatedAvatars() const { return _fullySimulatedAvatars; }
-    int getPartiallySimulatedAvatars() const { return _partiallySimulatedAvatars; }
+    int getNumAvatarsUpdated() const { return _numAvatarsUpdated; }
+    int getNumAvatarsNotUpdated() const { return _numAvatarsNotUpdated; }
     float getAvatarSimulationTime() const { return _avatarSimulationTime; }
 
     void updateMyAvatar(float deltaTime);
@@ -81,6 +81,10 @@ public:
                                                                   const QScriptValue& avatarIdsToInclude = QScriptValue(),
                                                                   const QScriptValue& avatarIdsToDiscard = QScriptValue());
 
+    // TODO: remove this HACK once we settle on optimal default sort coefficients
+    Q_INVOKABLE float getAvatarSortCoefficient(const QString& name);
+    Q_INVOKABLE void setAvatarSortCoefficient(const QString& name, const QScriptValue& value);
+
     float getMyAvatarSendRate() const { return _myAvatarSendRate.rate(); }
 
 public slots:
@@ -116,10 +120,9 @@ private:
     VectorOfMotionStates _motionStatesToRemoveFromPhysics;
 
     RateCounter<> _myAvatarSendRate;
-    int _fullySimulatedAvatars { 0 };
-    int _partiallySimulatedAvatars { 0 };
+    int _numAvatarsUpdated { 0 };
+    int _numAvatarsNotUpdated { 0 };
     float _avatarSimulationTime { 0.0f };
-
 };
 
 Q_DECLARE_METATYPE(AvatarManager::LocalLight)

interface/src/avatar/MyAvatar.cpp

@@ -148,13 +148,22 @@ MyAvatar::MyAvatar(RigPointer rig) :
     auto player = DependencyManager::get<Deck>();
     auto recorder = DependencyManager::get<Recorder>();
     connect(player.data(), &Deck::playbackStateChanged, [=] {
-        if (player->isPlaying()) {
+        bool isPlaying = player->isPlaying();
+        if (isPlaying) {
             auto recordingInterface = DependencyManager::get<RecordingScriptingInterface>();
             if (recordingInterface->getPlayFromCurrentLocation()) {
                 setRecordingBasis();
             }
         } else {
             clearRecordingBasis();
+            useFullAvatarURL(_fullAvatarURLFromPreferences, _fullAvatarModelName);
+        }
+
+        auto audioIO = DependencyManager::get<AudioClient>();
+        audioIO->setIsPlayingBackRecording(isPlaying);
+
+        if (_rig) {
+            _rig->setEnableAnimations(!isPlaying);
         }
     });
 
@@ -180,8 +189,8 @@ MyAvatar::MyAvatar(RigPointer rig) :
 
         if (recordingInterface->getPlayerUseSkeletonModel() && dummyAvatar.getSkeletonModelURL().isValid() &&
             (dummyAvatar.getSkeletonModelURL() != getSkeletonModelURL())) {
-            // FIXME
-            //myAvatar->useFullAvatarURL()
+
+            setSkeletonModelURL(dummyAvatar.getSkeletonModelURL());
         }
 
         if (recordingInterface->getPlayerUseDisplayName() && dummyAvatar.getDisplayName() != getDisplayName()) {
@@ -204,6 +213,11 @@ MyAvatar::MyAvatar(RigPointer rig) :
             // head orientation
             _headData->setLookAtPosition(headData->getLookAtPosition());
         }
+
+        auto jointData = dummyAvatar.getRawJointData();
+        if (jointData.length() > 0 && _rig) {
+            _rig->copyJointsFromJointData(jointData);
+        }
     });
 
     connect(rig.get(), SIGNAL(onLoadComplete()), this, SIGNAL(onLoadComplete()));
@@ -227,8 +241,7 @@ void MyAvatar::simulateAttachments(float deltaTime) {
     // don't update attachments here, do it in harvestResultsFromPhysicsSimulation()
 }
 
-QByteArray MyAvatar::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
-                        bool distanceAdjust, glm::vec3 viewerPosition, QVector<JointData>* sentJointDataOut) {
+QByteArray MyAvatar::toByteArrayStateful(AvatarDataDetail dataDetail) {
     CameraMode mode = qApp->getCamera()->getMode();
     _globalPosition = getPosition();
     _globalBoundingBoxDimensions.x = _characterController.getCapsuleRadius();
@@ -239,12 +252,12 @@ QByteArray MyAvatar::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTi
         // fake the avatar position that is sent up to the AvatarMixer
         glm::vec3 oldPosition = getPosition();
         setPosition(getSkeletonPosition());
-        QByteArray array = AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut);
+        QByteArray array = AvatarData::toByteArrayStateful(dataDetail);
         // copy the correct position back
         setPosition(oldPosition);
         return array;
     }
-    return AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut);
+    return AvatarData::toByteArrayStateful(dataDetail);
 }
 
 void MyAvatar::centerBody() {
@@ -472,7 +485,9 @@ void MyAvatar::simulate(float deltaTime) {
     {
         PerformanceTimer perfTimer("joints");
         // copy out the skeleton joints from the model
-        _rig->copyJointsIntoJointData(_jointData);
+        if (_rigEnabled) {
+            _rig->copyJointsIntoJointData(_jointData);
+        }
     }
 
     {
@@ -808,7 +823,7 @@ void MyAvatar::saveData() {
     auto hmdInterface = DependencyManager::get<HMDScriptingInterface>();
     _avatarEntitiesLock.withReadLock([&] {
         for (auto entityID : _avatarEntityData.keys()) {
-            if (hmdInterface->getCurrentTableUIID() == entityID) {
+            if (hmdInterface->getCurrentTabletUIID() == entityID) {
                 // don't persist the tablet between domains / sessions
                 continue;
             }

interface/src/avatar/MyAvatar.h

@@ -338,8 +338,7 @@ private:
     glm::quat getWorldBodyOrientation() const;
 
 
-    virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
-                            bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector<JointData>* sentJointDataOut = nullptr) override;
+    virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail) override;
 
     void simulate(float deltaTime);
     void updateFromTrackers(float deltaTime);
@@ -486,6 +485,7 @@ private:
     std::unordered_set<int> _headBoneSet;
     RigPointer _rig;
     bool _prevShouldDrawHead;
+    bool _rigEnabled { true };
 
     bool _enableDebugDrawDefaultPose { false };
     bool _enableDebugDrawAnimPose { false };

interface/src/devices/DdeFaceTracker.cpp

@@ -193,7 +193,7 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui
     _calibrationCount(0),
     _calibrationValues(),
     _calibrationBillboard(NULL),
-    _calibrationBillboardID(0),
+    _calibrationBillboardID(UNKNOWN_OVERLAY_ID),
     _calibrationMessage(QString()),
     _isCalibrated(false)
 {

interface/src/devices/DdeFaceTracker.h

@@ -149,7 +149,7 @@ private:
     int _calibrationCount;
     QVector<float> _calibrationValues;
     TextOverlay* _calibrationBillboard;
-    int _calibrationBillboardID;
+    OverlayID _calibrationBillboardID;
     QString _calibrationMessage;
     bool _isCalibrated;
     void addCalibrationDatum();

interface/src/scripting/HMDScriptingInterface.h

@@ -29,9 +29,9 @@ class HMDScriptingInterface : public AbstractHMDScriptingInterface, public Depen
     Q_PROPERTY(glm::quat orientation READ getOrientation)
     Q_PROPERTY(bool mounted READ isMounted)
     Q_PROPERTY(bool showTablet READ getShouldShowTablet)
-    Q_PROPERTY(QUuid tabletID READ getCurrentTableUIID WRITE setCurrentTabletUIID)
-    Q_PROPERTY(unsigned int homeButtonID READ getCurrentHomeButtonUUID WRITE setCurrentHomeButtonUUID)
-    
+    Q_PROPERTY(QUuid tabletID READ getCurrentTabletUIID WRITE setCurrentTabletUIID)
+    Q_PROPERTY(QUuid homeButtonID READ getCurrentHomeButtonUUID WRITE setCurrentHomeButtonUUID)
+    Q_PROPERTY(QUuid tabletScreenID READ getCurrentTabletScreenID WRITE setCurrentTabletScreenID)
 
 public:
     Q_INVOKABLE glm::vec3 calculateRayUICollisionPoint(const glm::vec3& position, const glm::vec3& direction) const;
@@ -91,15 +91,19 @@ public:
     bool getShouldShowTablet() const { return _showTablet; }
 
     void setCurrentTabletUIID(QUuid tabletID) { _tabletUIID = tabletID; }
-    QUuid getCurrentTableUIID() const { return _tabletUIID; }
+    QUuid getCurrentTabletUIID() const { return _tabletUIID; }
 
-    void setCurrentHomeButtonUUID(unsigned int homeButtonID) { _homeButtonID = homeButtonID; }
-    unsigned int getCurrentHomeButtonUUID() const { return _homeButtonID; }
+    void setCurrentHomeButtonUUID(QUuid homeButtonID) { _homeButtonID = homeButtonID; }
+    QUuid getCurrentHomeButtonUUID() const { return _homeButtonID; }
+
+    void setCurrentTabletScreenID(QUuid tabletID) { _tabletScreenID = tabletID; }
+    QUuid getCurrentTabletScreenID() const { return _tabletScreenID; }
 
 private:
     bool _showTablet { false };
-    QUuid _tabletUIID; // this is the entityID of the WebEntity which is part of (a child of) the tablet-ui.
-    unsigned int _homeButtonID;
+    QUuid _tabletUIID; // this is the entityID of the tablet frame
+    QUuid _tabletScreenID; // this is the overlayID which is part of (a child of) the tablet-ui.
+    QUuid _homeButtonID;
     QUuid _tabletEntityID;
 
     // Get the position of the HMD

interface/src/ui/Stats.cpp

@@ -121,8 +121,8 @@ void Stats::updateStats(bool force) {
     auto avatarManager = DependencyManager::get<AvatarManager>();
     // we need to take one avatar out so we don't include ourselves
     STAT_UPDATE(avatarCount, avatarManager->size() - 1);
-    STAT_UPDATE(fullySimulatedAvatarCount, avatarManager->getFullySimulatedAvatars());
-    STAT_UPDATE(partiallySimulatedAvatarCount, avatarManager->getPartiallySimulatedAvatars());
+    STAT_UPDATE(updatedAvatarCount, avatarManager->getNumAvatarsUpdated());
+    STAT_UPDATE(notUpdatedAvatarCount, avatarManager->getNumAvatarsNotUpdated());
     STAT_UPDATE(serverCount, (int)nodeList->size());
     STAT_UPDATE(framerate, qApp->getFps());
     if (qApp->getActiveDisplayPlugin()) {

interface/src/ui/Stats.h

@@ -49,8 +49,8 @@ class Stats : public QQuickItem {
     STATS_PROPERTY(int, simrate, 0)
     STATS_PROPERTY(int, avatarSimrate, 0)
     STATS_PROPERTY(int, avatarCount, 0)
-    STATS_PROPERTY(int, fullySimulatedAvatarCount, 0)
-    STATS_PROPERTY(int, partiallySimulatedAvatarCount, 0)
+    STATS_PROPERTY(int, updatedAvatarCount, 0)
+    STATS_PROPERTY(int, notUpdatedAvatarCount, 0)
     STATS_PROPERTY(int, packetInCount, 0)
     STATS_PROPERTY(int, packetOutCount, 0)
     STATS_PROPERTY(float, mbpsIn, 0)
@@ -159,8 +159,8 @@ signals:
     void simrateChanged();
     void avatarSimrateChanged();
     void avatarCountChanged();
-    void fullySimulatedAvatarCountChanged();
-    void partiallySimulatedAvatarCountChanged();
+    void updatedAvatarCountChanged();
+    void notUpdatedAvatarCountChanged();
     void packetInCountChanged();
     void packetOutCountChanged();
     void mbpsInChanged();

interface/src/ui/overlays/Base3DOverlay.h

@@ -23,6 +23,9 @@ public:
     Base3DOverlay();
     Base3DOverlay(const Base3DOverlay* base3DOverlay);
 
+    virtual OverlayID getOverlayID() const override { return OverlayID(getID().toString()); }
+    void setOverlayID(OverlayID overlayID) override { setID(overlayID); }
+
     // getters
     virtual bool is3D() const override { return true; }
 

interface/src/ui/overlays/Overlay.cpp

@@ -189,7 +189,7 @@ float Overlay::updatePulse() {
         _pulseDirection *= -1.0f;
     }
     _pulse += pulseDelta;
-    
+
     return _pulse;
 }
 
@@ -204,3 +204,23 @@ void Overlay::removeFromScene(Overlay::Pointer overlay, std::shared_ptr<render::
     render::Item::clearID(_renderItemID);
 }
 
+QScriptValue OverlayIDtoScriptValue(QScriptEngine* engine, const OverlayID& id) {
+    return quuidToScriptValue(engine, id);
+}
+
+void OverlayIDfromScriptValue(const QScriptValue &object, OverlayID& id) {
+    quuidFromScriptValue(object, id);
+}
+
+QVector<OverlayID> qVectorOverlayIDFromScriptValue(const QScriptValue& array) {
+    if (!array.isArray()) {
+        return QVector<OverlayID>();
+    }
+    QVector<OverlayID> newVector;
+    int length = array.property("length").toInteger();
+    newVector.reserve(length);
+    for (int i = 0; i < length; i++) {
+        newVector << OverlayID(array.property(i).toString());
+    }
+    return newVector;
+}

interface/src/ui/overlays/Overlay.h

@@ -15,6 +15,13 @@
 #include <SharedUtil.h> // for xColor
 #include <render/Scene.h>
 
+class OverlayID : public QUuid {
+public:
+    OverlayID() : QUuid() {}
+    OverlayID(QString v) : QUuid(v) {}
+    OverlayID(QUuid v) : QUuid(v) {}
+};
+
 class Overlay : public QObject {
     Q_OBJECT
 
@@ -32,8 +39,8 @@ public:
     Overlay(const Overlay* overlay);
     ~Overlay();
 
-    unsigned int getOverlayID() { return _overlayID; }
-    void setOverlayID(unsigned int overlayID) { _overlayID = overlayID; }
+    virtual OverlayID getOverlayID() const { return _overlayID; }
+    virtual void setOverlayID(OverlayID overlayID) { _overlayID = overlayID; }
 
     virtual void update(float deltatime) {}
     virtual void render(RenderArgs* args) = 0;
@@ -84,13 +91,14 @@ public:
     render::ItemID getRenderItemID() const { return _renderItemID; }
     void setRenderItemID(render::ItemID renderItemID) { _renderItemID = renderItemID; }
 
+    unsigned int getStackOrder() const { return _stackOrder; }
+    void setStackOrder(unsigned int stackOrder) { _stackOrder = stackOrder; }
+
 protected:
     float updatePulse();
 
     render::ItemID _renderItemID{ render::Item::INVALID_ITEM_ID };
 
-    unsigned int _overlayID; // what Overlays.cpp knows this instance as
-
     bool _isLoaded;
     float _alpha;
 
@@ -107,15 +115,25 @@ protected:
     xColor _color;
     bool _visible; // should the overlay be drawn at all
     Anchor _anchor;
+
+    unsigned int _stackOrder { 0 };
+
+private:
+    OverlayID _overlayID; // only used for non-3d overlays
 };
 
 namespace render {
-   template <> const ItemKey payloadGetKey(const Overlay::Pointer& overlay); 
+   template <> const ItemKey payloadGetKey(const Overlay::Pointer& overlay);
    template <> const Item::Bound payloadGetBound(const Overlay::Pointer& overlay);
    template <> int payloadGetLayer(const Overlay::Pointer& overlay);
    template <> void payloadRender(const Overlay::Pointer& overlay, RenderArgs* args);
    template <> const ShapeKey shapeGetShapeKey(const Overlay::Pointer& overlay);
 }
 
- 
+Q_DECLARE_METATYPE(OverlayID);
+Q_DECLARE_METATYPE(QVector<OverlayID>);
+QScriptValue OverlayIDtoScriptValue(QScriptEngine* engine, const OverlayID& id);
+void OverlayIDfromScriptValue(const QScriptValue& object, OverlayID& id);
+QVector<OverlayID> qVectorOverlayIDFromScriptValue(const QScriptValue& array);
+
 #endif // hifi_Overlay_h

interface/src/ui/overlays/OverlayPanel.cpp

@@ -51,13 +51,13 @@ void propertyBindingFromVariant(const QVariant& objectVar, PropertyBinding& valu
 }
 
 
-void OverlayPanel::addChild(unsigned int childId) {
+void OverlayPanel::addChild(OverlayID childId) {
     if (!_children.contains(childId)) {
         _children.append(childId);
     }
 }
 
-void OverlayPanel::removeChild(unsigned int childId) {
+void OverlayPanel::removeChild(OverlayID childId) {
     if (_children.contains(childId)) {
         _children.removeOne(childId);
     }
@@ -89,7 +89,7 @@ QVariant OverlayPanel::getProperty(const QString &property) {
     if (property == "children") {
         QVariantList array;
         for (int i = 0; i < _children.length(); i++) {
-            array.append(_children[i]);
+            array.append(OverlayIDtoScriptValue(nullptr, _children[i]).toVariant());
         }
         return array;
     }

interface/src/ui/overlays/OverlayPanel.h

@@ -20,6 +20,7 @@
 
 #include "PanelAttachable.h"
 #include "Billboardable.h"
+#include "Overlay.h"
 
 class PropertyBinding {
 public:
@@ -54,10 +55,10 @@ public:
     void setAnchorScale(const glm::vec3& scale) { _anchorTransform.setScale(scale); }
     void setVisible(bool visible) { _visible = visible; }
 
-    const QList<unsigned int>& getChildren() { return _children; }
-    void addChild(unsigned int childId);
-    void removeChild(unsigned int childId);
-    unsigned int popLastChild() { return _children.takeLast(); }
+    const QList<OverlayID>& getChildren() { return _children; }
+    void addChild(OverlayID childId);
+    void removeChild(OverlayID childId);
+    OverlayID popLastChild() { return _children.takeLast(); }
 
     void setProperties(const QVariantMap& properties);
     QVariant getProperty(const QString& property);
@@ -74,7 +75,7 @@ private:
     QUuid _anchorRotationBindEntity;
 
     bool _visible = true;
-    QList<unsigned int> _children;
+    QList<OverlayID> _children;
 
     QScriptEngine* _scriptEngine;
 };

interface/src/ui/overlays/Overlays.cpp

@@ -39,9 +39,6 @@
 
 Q_LOGGING_CATEGORY(trace_render_overlays, "trace.render.overlays")
 
-Overlays::Overlays() :
-    _nextOverlayID(1) {}
-
 void Overlays::cleanupAllOverlays() {
     {
         QWriteLocker lock(&_lock);
@@ -139,7 +136,7 @@ void Overlays::enable() {
     _enabled = true;
 }
 
-Overlay::Pointer Overlays::getOverlay(unsigned int id) const {
+Overlay::Pointer Overlays::getOverlay(OverlayID id) const {
     if (_overlaysHUD.contains(id)) {
         return _overlaysHUD[id];
     }
@@ -149,7 +146,7 @@ Overlay::Pointer Overlays::getOverlay(unsigned int id) const {
     return nullptr;
 }
 
-unsigned int Overlays::addOverlay(const QString& type, const QVariant& properties) {
+OverlayID Overlays::addOverlay(const QString& type, const QVariant& properties) {
     Overlay::Pointer thisOverlay = nullptr;
 
     if (type == ImageOverlay::TYPE) {
@@ -188,14 +185,14 @@ unsigned int Overlays::addOverlay(const QString& type, const QVariant& propertie
         thisOverlay->setProperties(properties.toMap());
         return addOverlay(thisOverlay);
     }
-    return 0;
+    return UNKNOWN_OVERLAY_ID;
 }
 
-unsigned int Overlays::addOverlay(Overlay::Pointer overlay) {
+OverlayID Overlays::addOverlay(Overlay::Pointer overlay) {
     QWriteLocker lock(&_lock);
-    unsigned int thisID = _nextOverlayID;
+    OverlayID thisID = OverlayID(QUuid::createUuid());
     overlay->setOverlayID(thisID);
-    _nextOverlayID++;
+    overlay->setStackOrder(_stackOrder++);
     if (overlay->is3D()) {
         _overlaysWorld[thisID] = overlay;
 
@@ -210,22 +207,22 @@ unsigned int Overlays::addOverlay(Overlay::Pointer overlay) {
     return thisID;
 }
 
-unsigned int Overlays::cloneOverlay(unsigned int id) {
+OverlayID Overlays::cloneOverlay(OverlayID id) {
     Overlay::Pointer thisOverlay = getOverlay(id);
 
     if (thisOverlay) {
-        unsigned int cloneId = addOverlay(Overlay::Pointer(thisOverlay->createClone()));
+        OverlayID cloneId = addOverlay(Overlay::Pointer(thisOverlay->createClone()));
         auto attachable = std::dynamic_pointer_cast<PanelAttachable>(thisOverlay);
         if (attachable && attachable->getParentPanel()) {
             attachable->getParentPanel()->addChild(cloneId);
         }
         return cloneId;
-    } 
-    
-    return 0;  // Not found
+    }
+
+    return UNKNOWN_OVERLAY_ID;  // Not found
 }
 
-bool Overlays::editOverlay(unsigned int id, const QVariant& properties) {
+bool Overlays::editOverlay(OverlayID id, const QVariant& properties) {
     QWriteLocker lock(&_lock);
 
     Overlay::Pointer thisOverlay = getOverlay(id);
@@ -242,13 +239,7 @@ bool Overlays::editOverlays(const QVariant& propertiesById) {
     bool success = true;
     QWriteLocker lock(&_lock);
     for (const auto& key : map.keys()) {
-        bool convertSuccess;
-        unsigned int id = key.toUInt(&convertSuccess);
-        if (!convertSuccess) {
-            success = false;
-            continue;
-        }
-
+        OverlayID id = OverlayID(key);
         Overlay::Pointer thisOverlay = getOverlay(id);
         if (!thisOverlay) {
             success = false;
@@ -260,7 +251,7 @@ bool Overlays::editOverlays(const QVariant& propertiesById) {
     return success;
 }
 
-void Overlays::deleteOverlay(unsigned int id) {
+void Overlays::deleteOverlay(OverlayID id) {
     Overlay::Pointer overlayToDelete;
 
     {
@@ -286,7 +277,7 @@ void Overlays::deleteOverlay(unsigned int id) {
     emit overlayDeleted(id);
 }
 
-QString Overlays::getOverlayType(unsigned int overlayId) const {
+QString Overlays::getOverlayType(OverlayID overlayId) const {
     Overlay::Pointer overlay = getOverlay(overlayId);
     if (overlay) {
         return overlay->getType();
@@ -294,7 +285,7 @@ QString Overlays::getOverlayType(unsigned int overlayId) const {
     return "";
 }
 
-QObject* Overlays::getOverlayObject(unsigned int id) {
+QObject* Overlays::getOverlayObject(OverlayID id) {
     Overlay::Pointer thisOverlay = getOverlay(id);
     if (thisOverlay) {
         return qobject_cast<QObject*>(&(*thisOverlay));
@@ -302,7 +293,7 @@ QObject* Overlays::getOverlayObject(unsigned int id) {
     return nullptr;
 }
 
-unsigned int Overlays::getParentPanel(unsigned int childId) const {
+OverlayID Overlays::getParentPanel(OverlayID childId) const {
     Overlay::Pointer overlay = getOverlay(childId);
     auto attachable = std::dynamic_pointer_cast<PanelAttachable>(overlay);
     if (attachable) {
@@ -310,10 +301,10 @@ unsigned int Overlays::getParentPanel(unsigned int childId) const {
     } else if (_panels.contains(childId)) {
         return _panels.key(getPanel(childId)->getParentPanel());
     }
-    return 0;
+    return UNKNOWN_OVERLAY_ID;
 }
 
-void Overlays::setParentPanel(unsigned int childId, unsigned int panelId) {
+void Overlays::setParentPanel(OverlayID childId, OverlayID panelId) {
     auto attachable = std::dynamic_pointer_cast<PanelAttachable>(getOverlay(childId));
     if (attachable) {
         if (_panels.contains(panelId)) {
@@ -343,13 +334,13 @@ void Overlays::setParentPanel(unsigned int childId, unsigned int panelId) {
     }
 }
 
-unsigned int Overlays::getOverlayAtPoint(const glm::vec2& point) {
+OverlayID Overlays::getOverlayAtPoint(const glm::vec2& point) {
     glm::vec2 pointCopy = point;
     QReadLocker lock(&_lock);
     if (!_enabled) {
-        return 0;
+        return UNKNOWN_OVERLAY_ID;
     }
-    QMapIterator<unsigned int, Overlay::Pointer> i(_overlaysHUD);
+    QMapIterator<OverlayID, Overlay::Pointer> i(_overlaysHUD);
     i.toBack();
 
     const float LARGE_NEGATIVE_FLOAT = -9999999;
@@ -358,10 +349,12 @@ unsigned int Overlays::getOverlayAtPoint(const glm::vec2& point) {
     BoxFace thisFace;
     glm::vec3 thisSurfaceNormal;
     float distance;
+    unsigned int bestStackOrder = 0;
+    OverlayID bestOverlayID = UNKNOWN_OVERLAY_ID;
 
     while (i.hasPrevious()) {
         i.previous();
-        unsigned int thisID = i.key();
+        OverlayID thisID = i.key();
         if (i.value()->is3D()) {
             auto thisOverlay = std::dynamic_pointer_cast<Base3DOverlay>(i.value());
             if (thisOverlay && !thisOverlay->getIgnoreRayIntersection()) {
@@ -373,15 +366,18 @@ unsigned int Overlays::getOverlayAtPoint(const glm::vec2& point) {
             auto thisOverlay = std::dynamic_pointer_cast<Overlay2D>(i.value());
             if (thisOverlay && thisOverlay->getVisible() && thisOverlay->isLoaded() &&
                 thisOverlay->getBoundingRect().contains(pointCopy.x, pointCopy.y, false)) {
-                return thisID;
+                if (thisOverlay->getStackOrder() > bestStackOrder) {
+                    bestOverlayID = thisID;
+                    bestStackOrder = thisOverlay->getStackOrder();
+                }
             }
         }
     }
 
-    return 0; // not found
+    return bestOverlayID;
 }
 
-OverlayPropertyResult Overlays::getProperty(unsigned int id, const QString& property) {
+OverlayPropertyResult Overlays::getProperty(OverlayID id, const QString& property) {
     OverlayPropertyResult result;
     Overlay::Pointer thisOverlay = getOverlay(id);
     QReadLocker lock(&_lock);
@@ -406,23 +402,35 @@ void OverlayPropertyResultFromScriptValue(const QScriptValue& object, OverlayPro
 }
 
 
-RayToOverlayIntersectionResult Overlays::findRayIntersection(const PickRay& ray) {
+RayToOverlayIntersectionResult Overlays::findRayIntersection(const PickRay& ray, bool precisionPicking,
+                                                             const QScriptValue& overlayIDsToInclude,
+                                                             const QScriptValue& overlayIDsToDiscard,
+                                                             bool visibleOnly, bool collidableOnly) {
     float bestDistance = std::numeric_limits<float>::max();
     bool bestIsFront = false;
+    const QVector<OverlayID> overlaysToInclude = qVectorOverlayIDFromScriptValue(overlayIDsToInclude);
+    const QVector<OverlayID> overlaysToDiscard = qVectorOverlayIDFromScriptValue(overlayIDsToDiscard);
+
     RayToOverlayIntersectionResult result;
-    QMapIterator<unsigned int, Overlay::Pointer> i(_overlaysWorld);
+    QMapIterator<OverlayID, Overlay::Pointer> i(_overlaysWorld);
     i.toBack();
     while (i.hasPrevious()) {
         i.previous();
-        unsigned int thisID = i.key();
+        OverlayID thisID = i.key();
         auto thisOverlay = std::dynamic_pointer_cast<Base3DOverlay>(i.value());
+
+        if ((overlaysToDiscard.size() > 0 && overlaysToDiscard.contains(thisID)) ||
+            (overlaysToInclude.size() > 0 && !overlaysToInclude.contains(thisID))) {
+            continue;
+        }
+
         if (thisOverlay && thisOverlay->getVisible() && !thisOverlay->getIgnoreRayIntersection() && thisOverlay->isLoaded()) {
             float thisDistance;
             BoxFace thisFace;
             glm::vec3 thisSurfaceNormal;
             QString thisExtraInfo;
-            if (thisOverlay->findRayIntersectionExtraInfo(ray.origin, ray.direction, thisDistance, 
-                                                            thisFace, thisSurfaceNormal, thisExtraInfo)) {
+            if (thisOverlay->findRayIntersectionExtraInfo(ray.origin, ray.direction, thisDistance,
+                                                          thisFace, thisSurfaceNormal, thisExtraInfo)) {
                 bool isDrawInFront = thisOverlay->getDrawInFront();
                 if (thisDistance < bestDistance && (!bestIsFront || isDrawInFront)) {
                     bestIsFront = isDrawInFront;
@@ -441,23 +449,23 @@ RayToOverlayIntersectionResult Overlays::findRayIntersection(const PickRay& ray)
     return result;
 }
 
-RayToOverlayIntersectionResult::RayToOverlayIntersectionResult() : 
-    intersects(false), 
-    overlayID(-1),
+RayToOverlayIntersectionResult::RayToOverlayIntersectionResult() :
+    intersects(false),
+    overlayID(UNKNOWN_OVERLAY_ID),
     distance(0),
     face(),
     intersection(),
     extraInfo()
-{ 
+{
 }
 
 QScriptValue RayToOverlayIntersectionResultToScriptValue(QScriptEngine* engine, const RayToOverlayIntersectionResult& value) {
     auto obj = engine->newObject();
     obj.setProperty("intersects", value.intersects);
-    obj.setProperty("overlayID", value.overlayID);
+    obj.setProperty("overlayID", OverlayIDtoScriptValue(engine, value.overlayID));
     obj.setProperty("distance", value.distance);
 
-    QString faceName = "";    
+    QString faceName = "";
     // handle BoxFace
     switch (value.face) {
         case MIN_X_FACE:
@@ -493,7 +501,7 @@ QScriptValue RayToOverlayIntersectionResultToScriptValue(QScriptEngine* engine,
 void RayToOverlayIntersectionResultFromScriptValue(const QScriptValue& objectVar, RayToOverlayIntersectionResult& value) {
     QVariantMap object = objectVar.toVariant().toMap();
     value.intersects = object["intersects"].toBool();
-    value.overlayID = object["overlayID"].toInt();
+    value.overlayID = OverlayID(QUuid(object["overlayID"].toString()));
     value.distance = object["distance"].toFloat();
 
     QString faceName = object["face"].toString();
@@ -523,7 +531,7 @@ void RayToOverlayIntersectionResultFromScriptValue(const QScriptValue& objectVar
     value.extraInfo = object["extraInfo"].toString();
 }
 
-bool Overlays::isLoaded(unsigned int id) {
+bool Overlays::isLoaded(OverlayID id) {
     QReadLocker lock(&_lock);
     Overlay::Pointer thisOverlay = getOverlay(id);
     if (!thisOverlay) {
@@ -532,7 +540,7 @@ bool Overlays::isLoaded(unsigned int id) {
     return thisOverlay->isLoaded();
 }
 
-QSizeF Overlays::textSize(unsigned int id, const QString& text) const {
+QSizeF Overlays::textSize(OverlayID id, const QString& text) const {
     Overlay::Pointer thisOverlay = _overlaysHUD[id];
     if (thisOverlay) {
         if (auto textOverlay = std::dynamic_pointer_cast<TextOverlay>(thisOverlay)) {
@@ -547,30 +555,29 @@ QSizeF Overlays::textSize(unsigned int id, const QString& text) const {
     return QSizeF(0.0f, 0.0f);
 }
 
-unsigned int Overlays::addPanel(OverlayPanel::Pointer panel) {
+OverlayID Overlays::addPanel(OverlayPanel::Pointer panel) {
     QWriteLocker lock(&_lock);
 
-    unsigned int thisID = _nextOverlayID;
-    _nextOverlayID++;
+    OverlayID thisID = QUuid::createUuid();
     _panels[thisID] = panel;
 
     return thisID;
 }
 
-unsigned int Overlays::addPanel(const QVariant& properties) {
+OverlayID Overlays::addPanel(const QVariant& properties) {
     OverlayPanel::Pointer panel = std::make_shared<OverlayPanel>();
     panel->init(_scriptEngine);
     panel->setProperties(properties.toMap());
     return addPanel(panel);
 }
 
-void Overlays::editPanel(unsigned int panelId, const QVariant& properties) {
+void Overlays::editPanel(OverlayID panelId, const QVariant& properties) {
     if (_panels.contains(panelId)) {
         _panels[panelId]->setProperties(properties.toMap());
     }
 }
 
-OverlayPropertyResult Overlays::getPanelProperty(unsigned int panelId, const QString& property) {
+OverlayPropertyResult Overlays::getPanelProperty(OverlayID panelId, const QString& property) {
     OverlayPropertyResult result;
     if (_panels.contains(panelId)) {
         OverlayPanel::Pointer thisPanel = getPanel(panelId);
@@ -581,7 +588,7 @@ OverlayPropertyResult Overlays::getPanelProperty(unsigned int panelId, const QSt
 }
 
 
-void Overlays::deletePanel(unsigned int panelId) {
+void Overlays::deletePanel(OverlayID panelId) {
     OverlayPanel::Pointer panelToDelete;
 
     {
@@ -594,7 +601,7 @@ void Overlays::deletePanel(unsigned int panelId) {
     }
 
     while (!panelToDelete->getChildren().isEmpty()) {
-        unsigned int childId = panelToDelete->popLastChild();
+        OverlayID childId = panelToDelete->popLastChild();
         deleteOverlay(childId);
         deletePanel(childId);
     }
@@ -602,39 +609,39 @@ void Overlays::deletePanel(unsigned int panelId) {
     emit panelDeleted(panelId);
 }
 
-bool Overlays::isAddedOverlay(unsigned int id) {
+bool Overlays::isAddedOverlay(OverlayID id) {
     return _overlaysHUD.contains(id) || _overlaysWorld.contains(id);
 }
 
-void Overlays::sendMousePressOnOverlay(unsigned int overlayID, const PointerEvent& event) {
+void Overlays::sendMousePressOnOverlay(OverlayID overlayID, const PointerEvent& event) {
     emit mousePressOnOverlay(overlayID, event);
 }
 
-void Overlays::sendMouseReleaseOnOverlay(unsigned int overlayID, const PointerEvent& event) {
+void Overlays::sendMouseReleaseOnOverlay(OverlayID overlayID, const PointerEvent& event) {
     emit mouseReleaseOnOverlay(overlayID, event);
 }
 
-void Overlays::sendMouseMoveOnOverlay(unsigned int overlayID, const PointerEvent& event) {
+void Overlays::sendMouseMoveOnOverlay(OverlayID overlayID, const PointerEvent& event) {
     emit mouseMoveOnOverlay(overlayID, event);
 }
 
-void Overlays::sendHoverEnterOverlay(unsigned int id, PointerEvent event) {
+void Overlays::sendHoverEnterOverlay(OverlayID id, PointerEvent event) {
     emit hoverEnterOverlay(id, event);
 }
 
-void Overlays::sendHoverOverOverlay(unsigned int  id, PointerEvent event) {
+void Overlays::sendHoverOverOverlay(OverlayID  id, PointerEvent event) {
     emit hoverOverOverlay(id, event);
 }
 
-void Overlays::sendHoverLeaveOverlay(unsigned int  id, PointerEvent event) {
+void Overlays::sendHoverLeaveOverlay(OverlayID  id, PointerEvent event) {
     emit hoverLeaveOverlay(id, event);
 }
 
-unsigned int Overlays::getKeyboardFocusOverlay() const {
+OverlayID Overlays::getKeyboardFocusOverlay() const {
     return qApp->getKeyboardFocusOverlay();
 }
 
-void Overlays::setKeyboardFocusOverlay(unsigned int id) {
+void Overlays::setKeyboardFocusOverlay(OverlayID id) {
     qApp->setKeyboardFocusOverlay(id);
 }
 

interface/src/ui/overlays/Overlays.h

@@ -53,7 +53,7 @@ class RayToOverlayIntersectionResult {
 public:
     RayToOverlayIntersectionResult();
     bool intersects;
-    unsigned int overlayID;
+    OverlayID overlayID;
     float distance;
     BoxFace face;
     glm::vec3 surfaceNormal;
@@ -77,15 +77,15 @@ void RayToOverlayIntersectionResultFromScriptValue(const QScriptValue& object, R
  * @namespace Overlays
  */
 
-const unsigned int UNKNOWN_OVERLAY_ID = 0;
+const OverlayID UNKNOWN_OVERLAY_ID = OverlayID();
 
 class Overlays : public QObject {
     Q_OBJECT
 
-    Q_PROPERTY(unsigned int keyboardFocusOverlay READ getKeyboardFocusOverlay WRITE setKeyboardFocusOverlay)
+    Q_PROPERTY(OverlayID keyboardFocusOverlay READ getKeyboardFocusOverlay WRITE setKeyboardFocusOverlay)
 
 public:
-    Overlays();
+    Overlays() {};
 
     void init();
     void update(float deltatime);
@@ -93,12 +93,12 @@ public:
     void disable();
     void enable();
 
-    Overlay::Pointer getOverlay(unsigned int id) const;
-    OverlayPanel::Pointer getPanel(unsigned int id) const { return _panels[id]; }
+    Overlay::Pointer getOverlay(OverlayID id) const;
+    OverlayPanel::Pointer getPanel(OverlayID id) const { return _panels[id]; }
 
     /// adds an overlay that's already been created
-    unsigned int addOverlay(Overlay* overlay) { return addOverlay(Overlay::Pointer(overlay)); }
-    unsigned int addOverlay(Overlay::Pointer overlay);
+    OverlayID addOverlay(Overlay* overlay) { return addOverlay(Overlay::Pointer(overlay)); }
+    OverlayID addOverlay(Overlay::Pointer overlay);
 
     void mousePressEvent(QMouseEvent* event);
     void mouseReleaseEvent(QMouseEvent* event);
@@ -116,7 +116,7 @@ public slots:
      * @param {Overlays.OverlayProperties} The properties of the overlay that you want to add.
      * @return {Overlays.OverlayID} The ID of the newly created overlay.
      */
-    unsigned int addOverlay(const QString& type, const QVariant& properties);
+    OverlayID addOverlay(const QString& type, const QVariant& properties);
 
     /**jsdoc
      * Create a clone of an existing overlay.
@@ -125,7 +125,7 @@ public slots:
      * @param {Overlays.OverlayID} overlayID The ID of the overlay to clone.
      * @return {Overlays.OverlayID} The ID of the new overlay.
      */
-    unsigned int cloneOverlay(unsigned int id);
+    OverlayID cloneOverlay(OverlayID id);
 
     /**jsdoc
      * Edit an overlay's properties. 
@@ -134,7 +134,7 @@ public slots:
      * @param {Overlays.OverlayID} overlayID The ID of the overlay to edit.
      * @return {bool} `true` if the overlay was found and edited, otherwise false.
      */
-    bool editOverlay(unsigned int id, const QVariant& properties);
+    bool editOverlay(OverlayID id, const QVariant& properties);
 
     /// edits an overlay updating only the included properties, will return the identified OverlayID in case of
     /// successful edit, if the input id is for an unknown overlay this function will have no effect
@@ -146,7 +146,7 @@ public slots:
      * @function Overlays.deleteOverlay
      * @param {Overlays.OverlayID} overlayID The ID of the overlay to delete.
      */
-    void deleteOverlay(unsigned int id);
+    void deleteOverlay(OverlayID id);
 
     /**jsdoc
      * Get the type of an overlay.
@@ -155,7 +155,7 @@ public slots:
      * @param {Overlays.OverlayID} overlayID The ID of the overlay to get the type of.
      * @return {string} The type of the overlay if found, otherwise the empty string.
      */
-    QString getOverlayType(unsigned int overlayId) const;
+    QString getOverlayType(OverlayID overlayId) const;
 
     /**jsdoc
     * Get the overlay Script object.
@@ -164,7 +164,7 @@ public slots:
     * @param {Overlays.OverlayID} overlayID The ID of the overlay to get the script object of.
     * @return {Object} The script object for the overlay if found.
     */
-    QObject* getOverlayObject(unsigned int id);
+    QObject* getOverlayObject(OverlayID id);
 
     /**jsdoc
      * Get the ID of the overlay at a particular point on the HUD/screen.
@@ -174,7 +174,7 @@ public slots:
      * @return {Overlays.OverlayID} The ID of the overlay at the point specified.
      *     If no overlay is found, `0` will be returned.
      */
-    unsigned int getOverlayAtPoint(const glm::vec2& point);
+    OverlayID getOverlayAtPoint(const glm::vec2& point);
 
     /**jsdoc
      * Get the value of a an overlay's property.
@@ -185,16 +185,26 @@ public slots:
      * @return {Object} The value of the property. If the overlay or the property could
      *     not be found, `undefined` will be returned.
      */
-    OverlayPropertyResult getProperty(unsigned int id, const QString& property);
+    OverlayPropertyResult getProperty(OverlayID id, const QString& property);
 
     /*jsdoc
      * Find the closest 3D overlay hit by a pick ray.
      *
      * @function Overlays.findRayIntersection
      * @param {PickRay} The PickRay to use for finding overlays.
+     * @param {bool} Unused; Exists to match Entity interface
+     * @param {List of Overlays.OverlayID} Whitelist for intersection test.
+     * @param {List of Overlays.OverlayID} Blacklist for intersection test.
+     * @param {bool} Unused; Exists to match Entity interface
+     * @param {bool} Unused; Exists to match Entity interface
      * @return {Overlays.RayToOverlayIntersectionResult} The result of the ray cast.
      */
-    RayToOverlayIntersectionResult findRayIntersection(const PickRay& ray);
+    RayToOverlayIntersectionResult findRayIntersection(const PickRay& ray,
+                                                       bool precisionPicking = false,
+                                                       const QScriptValue& overlayIDsToInclude = QScriptValue(),
+                                                       const QScriptValue& overlayIDsToDiscard = QScriptValue(),
+                                                       bool visibleOnly = false,
+                                                       bool collidableOnly = false);
 
     /**jsdoc
      * Check whether an overlay's assets have been loaded. For example, if the
@@ -204,7 +214,7 @@ public slots:
      * @param {Overlays.OverlayID} The ID of the overlay to check.
      * @return {bool} `true` if the overlay's assets have been loaded, otherwise `false`.
      */
-    bool isLoaded(unsigned int id);
+    bool isLoaded(OverlayID id);
 
     /**jsdoc
      * Calculates the size of the given text in the specified overlay if it is a text overlay.
@@ -216,7 +226,7 @@ public slots:
      * @param {string} The string to measure.
      * @return {Vec2} The size of the text.
      */
-    QSizeF textSize(unsigned int id, const QString& text) const;
+    QSizeF textSize(OverlayID id, const QString& text) const;
 
     /**jsdoc
      * Get the width of the virtual 2D HUD.
@@ -235,39 +245,39 @@ public slots:
     float height() const;
 
     /// return true if there is an overlay with that id else false
-    bool isAddedOverlay(unsigned int id);
+    bool isAddedOverlay(OverlayID id);
 
-    unsigned int getParentPanel(unsigned int childId) const;
-    void setParentPanel(unsigned int childId, unsigned int panelId);
+    OverlayID getParentPanel(OverlayID childId) const;
+    void setParentPanel(OverlayID childId, OverlayID panelId);
 
     /// adds a panel that has already been created
-    unsigned int addPanel(OverlayPanel::Pointer panel);
+    OverlayID addPanel(OverlayPanel::Pointer panel);
 
     /// creates and adds a panel based on a set of properties
-    unsigned int addPanel(const QVariant& properties);
+    OverlayID addPanel(const QVariant& properties);
 
     /// edit the properties of a panel
-    void editPanel(unsigned int panelId, const QVariant& properties);
+    void editPanel(OverlayID panelId, const QVariant& properties);
 
     /// get a property of a panel
-    OverlayPropertyResult getPanelProperty(unsigned int panelId, const QString& property);
+    OverlayPropertyResult getPanelProperty(OverlayID panelId, const QString& property);
 
     /// deletes a panel and all child overlays
-    void deletePanel(unsigned int panelId);
+    void deletePanel(OverlayID panelId);
 
     /// return true if there is a panel with that id else false
-    bool isAddedPanel(unsigned int id) { return _panels.contains(id); }
+    bool isAddedPanel(OverlayID id) { return _panels.contains(id); }
 
-    void sendMousePressOnOverlay(unsigned int overlayID, const PointerEvent& event);
-    void sendMouseReleaseOnOverlay(unsigned int overlayID, const PointerEvent& event);
-    void sendMouseMoveOnOverlay(unsigned int overlayID, const PointerEvent& event);
+    void sendMousePressOnOverlay(OverlayID overlayID, const PointerEvent& event);
+    void sendMouseReleaseOnOverlay(OverlayID overlayID, const PointerEvent& event);
+    void sendMouseMoveOnOverlay(OverlayID overlayID, const PointerEvent& event);
 
-    void sendHoverEnterOverlay(unsigned int id, PointerEvent event);
-    void sendHoverOverOverlay(unsigned int id, PointerEvent event);
-    void sendHoverLeaveOverlay(unsigned int id, PointerEvent event);
+    void sendHoverEnterOverlay(OverlayID id, PointerEvent event);
+    void sendHoverOverOverlay(OverlayID id, PointerEvent event);
+    void sendHoverLeaveOverlay(OverlayID id, PointerEvent event);
 
-    unsigned int getKeyboardFocusOverlay() const;
-    void setKeyboardFocusOverlay(unsigned int id);
+    OverlayID getKeyboardFocusOverlay() const;
+    void setKeyboardFocusOverlay(OverlayID id);
 
 signals:
     /**jsdoc
@@ -276,26 +286,26 @@ signals:
      * @function Overlays.overlayDeleted
      * @param {OverlayID} The ID of the overlay that was deleted.
      */
-    void overlayDeleted(unsigned int id);
-    void panelDeleted(unsigned int id);
+    void overlayDeleted(OverlayID id);
+    void panelDeleted(OverlayID id);
 
-    void mousePressOnOverlay(unsigned int overlayID, const PointerEvent& event);
-    void mouseReleaseOnOverlay(unsigned int overlayID, const PointerEvent& event);
-    void mouseMoveOnOverlay(unsigned int overlayID, const PointerEvent& event);
+    void mousePressOnOverlay(OverlayID overlayID, const PointerEvent& event);
+    void mouseReleaseOnOverlay(OverlayID overlayID, const PointerEvent& event);
+    void mouseMoveOnOverlay(OverlayID overlayID, const PointerEvent& event);
     void mousePressOffOverlay();
 
-    void hoverEnterOverlay(unsigned int overlayID, const PointerEvent& event);
-    void hoverOverOverlay(unsigned int overlayID, const PointerEvent& event);
-    void hoverLeaveOverlay(unsigned int overlayID, const PointerEvent& event);
+    void hoverEnterOverlay(OverlayID overlayID, const PointerEvent& event);
+    void hoverOverOverlay(OverlayID overlayID, const PointerEvent& event);
+    void hoverLeaveOverlay(OverlayID overlayID, const PointerEvent& event);
 
 private:
     void cleanupOverlaysToDelete();
 
-    QMap<unsigned int, Overlay::Pointer> _overlaysHUD;
-    QMap<unsigned int, Overlay::Pointer> _overlaysWorld;
-    QMap<unsigned int, OverlayPanel::Pointer> _panels;
+    QMap<OverlayID, Overlay::Pointer> _overlaysHUD;
+    QMap<OverlayID, Overlay::Pointer> _overlaysWorld;
+    QMap<OverlayID, OverlayPanel::Pointer> _panels;
     QList<Overlay::Pointer> _overlaysToDelete;
-    unsigned int _nextOverlayID;
+    unsigned int _stackOrder { 1 };
 
     QReadWriteLock _lock;
     QReadWriteLock _deleteLock;
@@ -305,8 +315,8 @@ private:
     PointerEvent calculatePointerEvent(Overlay::Pointer overlay, PickRay ray, RayToOverlayIntersectionResult rayPickResult,
         QMouseEvent* event, PointerEvent::EventType eventType);
 
-    unsigned int _currentClickingOnOverlayID { UNKNOWN_OVERLAY_ID };
-    unsigned int _currentHoverOverOverlayID { UNKNOWN_OVERLAY_ID };
+    OverlayID _currentClickingOnOverlayID { UNKNOWN_OVERLAY_ID };
+    OverlayID _currentHoverOverOverlayID { UNKNOWN_OVERLAY_ID };
 };
 
 #endif // hifi_Overlays_h

interface/src/ui/overlays/Web3DOverlay.cpp

@@ -198,7 +198,7 @@ void Web3DOverlay::render(RenderArgs* args) {
         _webSurface->getRootItem()->setProperty("scriptURL", _scriptURL);
         currentContext->makeCurrent(currentSurface);
 
-        auto forwardPointerEvent = [=](unsigned int overlayID, const PointerEvent& event) {
+        auto forwardPointerEvent = [=](OverlayID overlayID, const PointerEvent& event) {
             if (overlayID == getOverlayID()) {
                 handlePointerEvent(event);
             }
@@ -208,7 +208,7 @@ void Web3DOverlay::render(RenderArgs* args) {
         _mouseReleaseConnection = connect(&(qApp->getOverlays()), &Overlays::mouseReleaseOnOverlay, forwardPointerEvent);
         _mouseMoveConnection = connect(&(qApp->getOverlays()), &Overlays::mouseMoveOnOverlay, forwardPointerEvent);
         _hoverLeaveConnection = connect(&(qApp->getOverlays()), &Overlays::hoverLeaveOverlay,
-            [=](unsigned int overlayID, const PointerEvent& event) {
+            [=](OverlayID overlayID, const PointerEvent& event) {
             if (this->_pressed && this->getOverlayID() == overlayID) {
                 // If the user mouses off the overlay while the button is down, simulate a touch end.
                 QTouchEvent::TouchPoint point;

libraries/animation/src/Rig.cpp

@@ -483,6 +483,10 @@ void Rig::setEnableInverseKinematics(bool enable) {
     _enableInverseKinematics = enable;
 }
 
+void Rig::setEnableAnimations(bool enable) {
+    _enabledAnimations = enable;
+}
+
 AnimPose Rig::getAbsoluteDefaultPose(int index) const {
     if (_animSkeleton && index >= 0 && index < _animSkeleton->getNumJoints()) {
         return _absoluteDefaultPoses[index];
@@ -907,7 +911,7 @@ void Rig::updateAnimations(float deltaTime, glm::mat4 rootTransform) {
 
     setModelOffset(rootTransform);
 
-    if (_animNode) {
+    if (_animNode && _enabledAnimations) {
         PerformanceTimer perfTimer("handleTriggers");
 
         updateAnimationStateHandlers();

libraries/animation/src/Rig.h

@@ -210,6 +210,7 @@ public:
     void computeAvatarBoundingCapsule(const FBXGeometry& geometry, float& radiusOut, float& heightOut, glm::vec3& offsetOut) const;
 
     void setEnableInverseKinematics(bool enable);
+    void setEnableAnimations(bool enable);
 
     const glm::mat4& getGeometryToRigTransform() const { return _geometryToRigTransform; }
 
@@ -314,6 +315,7 @@ protected:
     int32_t _numOverrides { 0 };
     bool _lastEnableInverseKinematics { true };
     bool _enableInverseKinematics { true };
+    bool _enabledAnimations { true };
 
     mutable uint32_t _jointNameWarningCount { 0 };
 

libraries/audio-client/src/AudioClient.cpp

@@ -39,13 +39,10 @@
 #include <plugins/CodecPlugin.h>
 #include <plugins/PluginManager.h>
 #include <udt/PacketHeaders.h>
-#include <PositionalAudioStream.h>
 #include <SettingHandle.h>
 #include <SharedUtil.h>
-#include <UUID.h>
 #include <Transform.h>
 
-#include "PositionalAudioStream.h"
 #include "AudioClientLogging.h"
 #include "AudioLogging.h"
 
@@ -294,12 +291,12 @@ QString friendlyNameForAudioDevice(IMMDevice* pEndpoint) {
     IPropertyStore* pPropertyStore;
     pEndpoint->OpenPropertyStore(STGM_READ, &pPropertyStore);
     pEndpoint->Release();
-    pEndpoint = NULL;
+    pEndpoint = nullptr;
     PROPVARIANT pv;
     PropVariantInit(&pv);
     HRESULT hr = pPropertyStore->GetValue(PKEY_Device_FriendlyName, &pv);
     pPropertyStore->Release();
-    pPropertyStore = NULL;
+    pPropertyStore = nullptr;
     deviceName = QString::fromWCharArray((wchar_t*)pv.pwszVal);
     if (!IsWindows8OrGreater()) {
         // Windows 7 provides only the 31 first characters of the device name.
@@ -313,9 +310,9 @@ QString friendlyNameForAudioDevice(IMMDevice* pEndpoint) {
 QString AudioClient::friendlyNameForAudioDevice(wchar_t* guid) {
     QString deviceName;
     HRESULT hr = S_OK;
-    CoInitialize(NULL);
-    IMMDeviceEnumerator* pMMDeviceEnumerator = NULL;
-    CoCreateInstance(__uuidof(MMDeviceEnumerator), NULL, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), (void**)&pMMDeviceEnumerator);
+    CoInitialize(nullptr);
+    IMMDeviceEnumerator* pMMDeviceEnumerator = nullptr;
+    CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), (void**)&pMMDeviceEnumerator);
     IMMDevice* pEndpoint;
     hr = pMMDeviceEnumerator->GetDevice(guid, &pEndpoint);
     if (hr == E_NOTFOUND) {
@@ -325,7 +322,7 @@ QString AudioClient::friendlyNameForAudioDevice(wchar_t* guid) {
         deviceName = ::friendlyNameForAudioDevice(pEndpoint);
     }
     pMMDeviceEnumerator->Release();
-    pMMDeviceEnumerator = NULL;
+    pMMDeviceEnumerator = nullptr;
     CoUninitialize();
     return deviceName;
 }
@@ -968,8 +965,7 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) {
 }
 
 void AudioClient::handleAudioInput() {
-
-    if (!_inputDevice) {
+    if (!_inputDevice || _isPlayingBackRecording) {
         return;
     }
 
@@ -1120,7 +1116,7 @@ void AudioClient::prepareLocalAudioInjectors() {
     while (samplesNeeded > 0) {
         // lock for every write to avoid locking out the device callback
         // this lock is intentional - the buffer is only lock-free in its use in the device callback
-        Lock lock(_localAudioMutex);
+        RecursiveLock lock(_localAudioMutex);
 
         samplesNeeded = bufferCapacity - _localSamplesAvailable.load(std::memory_order_relaxed);
         if (samplesNeeded <= 0) {
@@ -1457,7 +1453,7 @@ void AudioClient::outputNotify() {
 bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo) {
     bool supportedFormat = false;
 
-    Lock lock(_localAudioMutex);
+    RecursiveLock lock(_localAudioMutex);
     _localSamplesAvailable.exchange(0, std::memory_order_release);
 
     // cleanup any previously initialized device
@@ -1671,7 +1667,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
 
     int injectorSamplesPopped = 0;
     {
-        Lock lock(_audio->_localAudioMutex);
+        RecursiveLock lock(_audio->_localAudioMutex);
         bool append = networkSamplesPopped > 0;
         samplesRequested = std::min(samplesRequested, _audio->_localSamplesAvailable.load(std::memory_order_acquire));
         if ((injectorSamplesPopped = _localInjectorsStream.appendSamples(mixBuffer, samplesRequested, append)) > 0) {

libraries/audio-client/src/AudioClient.h

@@ -94,6 +94,8 @@ public:
     using AudioPositionGetter = std::function<glm::vec3()>;
     using AudioOrientationGetter = std::function<glm::quat()>;
 
+    using RecursiveMutex = std::recursive_mutex;
+    using RecursiveLock = std::unique_lock<RecursiveMutex>;
     using Mutex = std::mutex;
     using Lock = std::unique_lock<Mutex>;
 
@@ -145,6 +147,8 @@ public:
     void setPositionGetter(AudioPositionGetter positionGetter) { _positionGetter = positionGetter; }
     void setOrientationGetter(AudioOrientationGetter orientationGetter) { _orientationGetter = orientationGetter; }
 
+    void setIsPlayingBackRecording(bool isPlayingBackRecording) { _isPlayingBackRecording = isPlayingBackRecording; }
+
     Q_INVOKABLE void setAvatarBoundingBoxParameters(glm::vec3 corner, glm::vec3 scale);
 
     void checkDevices();
@@ -328,7 +332,7 @@ private:
     int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC];
     float* _localOutputMixBuffer { NULL };
     AudioInjectorsThread _localAudioThread;
-    Mutex _localAudioMutex;
+    RecursiveMutex _localAudioMutex;
 
     // for output audio (used by this thread)
     int _outputPeriod { 0 };
@@ -367,10 +371,12 @@ private:
     QVector<QString> _inputDevices;
     QVector<QString> _outputDevices;
 
-    bool _hasReceivedFirstPacket = false;
+    bool _hasReceivedFirstPacket { false };
 
     QVector<AudioInjector*> _activeLocalAudioInjectors;
 
+    bool _isPlayingBackRecording { false };
+
     CodecPluginPointer _codec;
     QString _selectedCodecName;
     Encoder* _encoder { nullptr }; // for outbound mic stream

libraries/audio/src/AudioInjectorOptions.cpp

@@ -91,4 +91,4 @@ void injectorOptionsFromScriptValue(const QScriptValue& object, AudioInjectorOpt
             qCWarning(audio) << "Unknown audio injector option:" << it.name();
         }
     }
- }
+}

libraries/avatars/src/AvatarData.cpp

@@ -36,6 +36,8 @@
 #include <shared/JSONHelpers.h>
 #include <ShapeInfo.h>
 #include <AudioHelpers.h>
+#include <Profile.h>
+#include <VariantMapToScriptValue.h>
 
 #include "AvatarLogging.h"
 
@@ -68,8 +70,7 @@ AvatarData::AvatarData() :
     _displayNameAlpha(1.0f),
     _errorLogExpiry(0),
     _owningAvatarMixer(),
-    _targetVelocity(0.0f),
-    _localAABox(DEFAULT_LOCAL_AABOX_CORNER, DEFAULT_LOCAL_AABOX_SCALE)
+    _targetVelocity(0.0f)
 {
     setBodyPitch(0.0f);
     setBodyYaw(-90.0f);
@@ -120,10 +121,6 @@ void AvatarData::nextAttitude(glm::vec3 position, glm::quat orientation) {
     updateAttitude();
 }
 
-float AvatarData::getTargetScale() const {
-    return _targetScale;
-}
-
 void AvatarData::setTargetScale(float targetScale) {
     auto newValue = glm::clamp(targetScale, MIN_AVATAR_SCALE, MAX_AVATAR_SCALE);
     if (_targetScale != newValue) {
@@ -141,10 +138,10 @@ void AvatarData::setHandPosition(const glm::vec3& handPosition) {
     _handPosition = glm::inverse(getOrientation()) * (handPosition - getPosition());
 }
 
-void AvatarData::lazyInitHeadData() {
+void AvatarData::lazyInitHeadData() const {
     // lazily allocate memory for HeadData in case we're not an Avatar instance
     if (!_headData) {
-        _headData = new HeadData(this);
+        _headData = new HeadData(const_cast<AvatarData*>(this));
     }
     if (_forceFaceTrackerConnected) {
         _headData->_isFaceTrackerConnected = true;
@@ -152,39 +149,7 @@ void AvatarData::lazyInitHeadData() {
 }
 
 
-bool AvatarData::avatarBoundingBoxChangedSince(quint64 time) {
-    return _avatarBoundingBoxChanged >= time;
-}
-
-bool AvatarData::avatarScaleChangedSince(quint64 time) {
-    return _avatarScaleChanged >= time;
-}
-
-bool AvatarData::lookAtPositionChangedSince(quint64 time) {
-    return _headData->lookAtPositionChangedSince(time);
-}
-
-bool AvatarData::audioLoudnessChangedSince(quint64 time) {
-    return _headData->audioLoudnessChangedSince(time);
-}
-
-bool AvatarData::sensorToWorldMatrixChangedSince(quint64 time) {
-    return _sensorToWorldMatrixChanged >= time;
-}
-
-bool AvatarData::additionalFlagsChangedSince(quint64 time) {
-    return _additionalFlagsChanged >= time;
-}
-
-bool AvatarData::parentInfoChangedSince(quint64 time) {
-    return _parentChanged >= time;
-}
-
-bool AvatarData::faceTrackerInfoChangedSince(quint64 time) {
-    return true; // FIXME!
-}
-
-float AvatarData::getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) {
+float AvatarData::getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) const {
     auto distance = glm::distance(_globalPosition, viewerPosition);
     float result = ROTATION_CHANGE_179D; // assume worst
     if (distance < AVATAR_DISTANCE_LEVEL_1) {
@@ -199,20 +164,24 @@ float AvatarData::getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) {
     return result;
 }
 
-float AvatarData::getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition) {
+float AvatarData::getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition) const {
     return AVATAR_MIN_TRANSLATION; // Eventually make this distance sensitive as well
 }
 
 
-QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
-                        bool distanceAdjust, glm::vec3 viewerPosition, QVector<JointData>* sentJointDataOut) {
+// we want to track outbound data in this case...
+QByteArray AvatarData::toByteArrayStateful(AvatarDataDetail dataDetail) {
+    AvatarDataPacket::HasFlags hasFlagsOut;
+    auto lastSentTime = _lastToByteArray;
+    _lastToByteArray = usecTimestampNow();
+    return AvatarData::toByteArray(dataDetail, lastSentTime, getLastSentJointData(), 
+                        hasFlagsOut, false, false, glm::vec3(0), nullptr,
+                        &_outboundDataRate);
+}
 
-    // if no timestamp was included, then assume the avatarData is single instance 
-    // and is tracking its own last encoding time.
-    if (lastSentTime == 0) {
-        lastSentTime = _lastToByteArray;
-        _lastToByteArray = usecTimestampNow();
-    }
+QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
+    AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, 
+    glm::vec3 viewerPosition, QVector<JointData>* sentJointDataOut, AvatarDataRate* outboundDataRateOut) const {
 
     bool cullSmallChanges = (dataDetail == CullSmallData);
     bool sendAll = (dataDetail == SendAllData);
@@ -224,6 +193,13 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
     unsigned char* destinationBuffer = reinterpret_cast<unsigned char*>(avatarDataByteArray.data());
     unsigned char* startPosition = destinationBuffer;
 
+    // special case, if we were asked for no data, then just include the flags all set to nothing
+    if (dataDetail == NoData) {
+        AvatarDataPacket::HasFlags packetStateFlags = 0;
+        memcpy(destinationBuffer, &packetStateFlags, sizeof(packetStateFlags));
+        return avatarDataByteArray.left(sizeof(packetStateFlags));
+    }
+
     // FIXME -
     //
     //    BUG -- if you enter a space bubble, and then back away, the avatar has wrong orientation until "send all" happens... 
@@ -259,26 +235,26 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
     // separately
     bool hasParentInfo = sendAll || parentInfoChangedSince(lastSentTime);
     bool hasAvatarLocalPosition = hasParent() && (sendAll ||
-                                                  tranlationChangedSince(lastSentTime) ||
-                                                  parentInfoChangedSince(lastSentTime));
+        tranlationChangedSince(lastSentTime) ||
+        parentInfoChangedSince(lastSentTime));
 
-    bool hasFaceTrackerInfo = hasFaceTracker() && (sendAll || faceTrackerInfoChangedSince(lastSentTime));
+    bool hasFaceTrackerInfo = !dropFaceTracking && hasFaceTracker() && (sendAll || faceTrackerInfoChangedSince(lastSentTime));
     bool hasJointData = sendAll || !sendMinimum;
 
     // Leading flags, to indicate how much data is actually included in the packet...
     AvatarDataPacket::HasFlags packetStateFlags =
-            (hasAvatarGlobalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_GLOBAL_POSITION : 0)
-          | (hasAvatarBoundingBox    ? AvatarDataPacket::PACKET_HAS_AVATAR_BOUNDING_BOX : 0)
-          | (hasAvatarOrientation    ? AvatarDataPacket::PACKET_HAS_AVATAR_ORIENTATION : 0)
-          | (hasAvatarScale          ? AvatarDataPacket::PACKET_HAS_AVATAR_SCALE : 0)
-          | (hasLookAtPosition       ? AvatarDataPacket::PACKET_HAS_LOOK_AT_POSITION : 0)
-          | (hasAudioLoudness        ? AvatarDataPacket::PACKET_HAS_AUDIO_LOUDNESS : 0)
-          | (hasSensorToWorldMatrix  ? AvatarDataPacket::PACKET_HAS_SENSOR_TO_WORLD_MATRIX : 0)
-          | (hasAdditionalFlags      ? AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS : 0)
-          | (hasParentInfo           ? AvatarDataPacket::PACKET_HAS_PARENT_INFO : 0)
-          | (hasAvatarLocalPosition  ? AvatarDataPacket::PACKET_HAS_AVATAR_LOCAL_POSITION : 0)
-          | (hasFaceTrackerInfo      ? AvatarDataPacket::PACKET_HAS_FACE_TRACKER_INFO : 0)
-          | (hasJointData            ? AvatarDataPacket::PACKET_HAS_JOINT_DATA : 0);
+        (hasAvatarGlobalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_GLOBAL_POSITION : 0)
+        | (hasAvatarBoundingBox ? AvatarDataPacket::PACKET_HAS_AVATAR_BOUNDING_BOX : 0)
+        | (hasAvatarOrientation ? AvatarDataPacket::PACKET_HAS_AVATAR_ORIENTATION : 0)
+        | (hasAvatarScale ? AvatarDataPacket::PACKET_HAS_AVATAR_SCALE : 0)
+        | (hasLookAtPosition ? AvatarDataPacket::PACKET_HAS_LOOK_AT_POSITION : 0)
+        | (hasAudioLoudness ? AvatarDataPacket::PACKET_HAS_AUDIO_LOUDNESS : 0)
+        | (hasSensorToWorldMatrix ? AvatarDataPacket::PACKET_HAS_SENSOR_TO_WORLD_MATRIX : 0)
+        | (hasAdditionalFlags ? AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS : 0)
+        | (hasParentInfo ? AvatarDataPacket::PACKET_HAS_PARENT_INFO : 0)
+        | (hasAvatarLocalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_LOCAL_POSITION : 0)
+        | (hasFaceTrackerInfo ? AvatarDataPacket::PACKET_HAS_FACE_TRACKER_INFO : 0)
+        | (hasJointData ? AvatarDataPacket::PACKET_HAS_JOINT_DATA : 0);
 
     memcpy(destinationBuffer, &packetStateFlags, sizeof(packetStateFlags));
     destinationBuffer += sizeof(packetStateFlags);
@@ -293,7 +269,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
 
         int numBytes = destinationBuffer - startSection;
 
-        _globalPositionRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->globalPositionRate.increment(numBytes);
+        }
     }
 
     if (hasAvatarBoundingBox) {
@@ -311,7 +289,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::AvatarBoundingBox);
 
         int numBytes = destinationBuffer - startSection;
-        _avatarBoundingBoxRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->avatarBoundingBoxRate.increment(numBytes);
+        }
     }
 
     if (hasAvatarOrientation) {
@@ -320,7 +300,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, localOrientation);
 
         int numBytes = destinationBuffer - startSection;
-        _avatarOrientationRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->avatarOrientationRate.increment(numBytes);
+        }
     }
 
     if (hasAvatarScale) {
@@ -331,7 +313,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::AvatarScale);
 
         int numBytes = destinationBuffer - startSection;
-        _avatarScaleRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->avatarScaleRate.increment(numBytes);
+        }
     }
 
     if (hasLookAtPosition) {
@@ -344,7 +328,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::LookAtPosition);
 
         int numBytes = destinationBuffer - startSection;
-        _lookAtPositionRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->lookAtPositionRate.increment(numBytes);
+        }
     }
 
     if (hasAudioLoudness) {
@@ -354,7 +340,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::AudioLoudness);
 
         int numBytes = destinationBuffer - startSection;
-        _audioLoudnessRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->audioLoudnessRate.increment(numBytes);
+        }
     }
 
     if (hasSensorToWorldMatrix) {
@@ -370,7 +358,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::SensorToWorldMatrix);
 
         int numBytes = destinationBuffer - startSection;
-        _sensorToWorldRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->sensorToWorldRate.increment(numBytes);
+        }
     }
 
     if (hasAdditionalFlags) {
@@ -403,7 +393,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::AdditionalFlags);
 
         int numBytes = destinationBuffer - startSection;
-        _additionalFlagsRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->additionalFlagsRate.increment(numBytes);
+        }
     }
 
     if (hasParentInfo) {
@@ -415,7 +407,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::ParentInfo);
 
         int numBytes = destinationBuffer - startSection;
-        _parentInfoRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->parentInfoRate.increment(numBytes);
+        }
     }
 
     if (hasAvatarLocalPosition) {
@@ -428,7 +422,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::AvatarLocalPosition);
 
         int numBytes = destinationBuffer - startSection;
-        _localPositionRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->localPositionRate.increment(numBytes);
+        }
     }
 
     // If it is connected, pack up the data
@@ -448,7 +444,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float);
 
         int numBytes = destinationBuffer - startSection;
-        _faceTrackerRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->faceTrackerRate.increment(numBytes);
+        }
     }
 
     // If it is connected, pack up the data
@@ -540,9 +538,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
                     glm::distance(data.translation, lastSentJointData[i].translation) > minTranslation) {
                     if (data.translationSet) {
                         validity |= (1 << validityBit);
-                        #ifdef WANT_DEBUG
+#ifdef WANT_DEBUG
                         translationSentCount++;
-                        #endif
+#endif
                         maxTranslationDimension = glm::max(fabsf(data.translation.x), maxTranslationDimension);
                         maxTranslationDimension = glm::max(fabsf(data.translation.y), maxTranslationDimension);
                         maxTranslationDimension = glm::max(fabsf(data.translation.z), maxTranslationDimension);
@@ -592,24 +590,25 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
 #ifdef WANT_DEBUG
         if (sendAll) {
             qCDebug(avatars) << "AvatarData::toByteArray" << cullSmallChanges << sendAll
-                    << "rotations:" << rotationSentCount << "translations:" << translationSentCount
-                    << "largest:" << maxTranslationDimension
-                    << "size:"
-                    << (beforeRotations - startPosition) << "+"
-                    << (beforeTranslations - beforeRotations) << "+"
-                    << (destinationBuffer - beforeTranslations) << "="
-                    << (destinationBuffer - startPosition);
+                << "rotations:" << rotationSentCount << "translations:" << translationSentCount
+                << "largest:" << maxTranslationDimension
+                << "size:"
+                << (beforeRotations - startPosition) << "+"
+                << (beforeTranslations - beforeRotations) << "+"
+                << (destinationBuffer - beforeTranslations) << "="
+                << (destinationBuffer - startPosition);
         }
 #endif
 
         int numBytes = destinationBuffer - startSection;
-        _jointDataRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->jointDataRate.increment(numBytes);
+        }
     }
 
     int avatarDataSize = destinationBuffer - startPosition;
     return avatarDataByteArray.left(avatarDataSize);
 }
-
 // NOTE: This is never used in a "distanceAdjust" mode, so it's ok that it doesn't use a variable minimum rotation/translation
 void AvatarData::doneEncoding(bool cullSmallChanges) {
     // The server has finished sending this version of the joint-data to other nodes.  Update _lastSentJointData.
@@ -1089,29 +1088,29 @@ float AvatarData::getDataRate(const QString& rateName) const {
     } else if (rateName == "jointData") {
         return _jointDataRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "globalPositionOutbound") {
-        return _globalPositionRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.globalPositionRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "localPositionOutbound") {
-        return _localPositionRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.localPositionRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "avatarBoundingBoxOutbound") {
-        return _avatarBoundingBoxRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.avatarBoundingBoxRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "avatarOrientationOutbound") {
-        return _avatarOrientationRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.avatarOrientationRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "avatarScaleOutbound") {
-        return _avatarScaleRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.avatarScaleRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "lookAtPositionOutbound") {
-        return _lookAtPositionRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.lookAtPositionRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "audioLoudnessOutbound") {
-        return _audioLoudnessRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.audioLoudnessRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "sensorToWorkMatrixOutbound") {
-        return _sensorToWorldRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.sensorToWorldRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "additionalFlagsOutbound") {
-        return _additionalFlagsRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.additionalFlagsRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "parentInfoOutbound") {
-        return _parentInfoRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.parentInfoRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "faceTrackerOutbound") {
-        return _faceTrackerRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.faceTrackerRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "jointDataOutbound") {
-        return _jointDataRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.jointDataRate.rate() / BYTES_PER_KILOBIT;
     }
     return 0.0f;
 }
@@ -1445,7 +1444,7 @@ void AvatarData::parseAvatarIdentityPacket(const QByteArray& data, Identity& ide
 }
 
 static const QUrl emptyURL("");
-const QUrl& AvatarData::cannonicalSkeletonModelURL(const QUrl& emptyURL) {
+QUrl AvatarData::cannonicalSkeletonModelURL(const QUrl& emptyURL) const {
     // We don't put file urls on the wire, but instead convert to empty.
     return _skeletonModelURL.scheme() == "file" ? emptyURL : _skeletonModelURL;
 }
@@ -1483,7 +1482,7 @@ void AvatarData::processAvatarIdentity(const Identity& identity, bool& identityC
     }
 }
 
-QByteArray AvatarData::identityByteArray() {
+QByteArray AvatarData::identityByteArray() const {
     QByteArray identityData;
     QDataStream identityStream(&identityData, QIODevice::Append);
     const QUrl& urlToSend = cannonicalSkeletonModelURL(emptyURL);
@@ -1646,13 +1645,7 @@ void AvatarData::sendAvatarDataPacket() {
 
     bool cullSmallData = (randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO);
     auto dataDetail = cullSmallData ? SendAllData : CullSmallData;
-    QVector<JointData> lastSentJointData;
-    {
-        QReadLocker readLock(&_jointDataLock);
-        _lastSentJointData.resize(_jointData.size());
-        lastSentJointData = _lastSentJointData;
-    } 
-    QByteArray avatarByteArray = toByteArray(dataDetail, 0, lastSentJointData);
+    QByteArray avatarByteArray = toByteArrayStateful(dataDetail);
     doneEncoding(cullSmallData);
 
     static AvatarDataSequenceNumber sequenceNumber = 0;
@@ -2324,3 +2317,100 @@ void RayToAvatarIntersectionResultFromScriptValue(const QScriptValue& object, Ra
         vec3FromScriptValue(intersection, value.intersection);
     }
 }
+
+const float AvatarData::OUT_OF_VIEW_PENALTY = -10.0f;
+
+float AvatarData::_avatarSortCoefficientSize { 0.5f };
+float AvatarData::_avatarSortCoefficientCenter { 0.25 };
+float AvatarData::_avatarSortCoefficientAge { 1.0f };
+
+std::priority_queue<AvatarPriority> AvatarData::sortAvatars(
+    QList<AvatarSharedPointer> avatarList,
+    const ViewFrustum& cameraView,
+    std::function<uint64_t(AvatarSharedPointer)> getLastUpdated,
+    std::function<float(AvatarSharedPointer)> getBoundingRadius,
+    std::function<bool(AvatarSharedPointer)> shouldIgnore) {
+
+    uint64_t startTime = usecTimestampNow();
+
+    glm::vec3 frustumCenter = cameraView.getPosition();
+
+    std::priority_queue<AvatarPriority> sortedAvatars;
+    {
+        PROFILE_RANGE(simulation, "sort");
+        for (int32_t i = 0; i < avatarList.size(); ++i) {
+            const auto& avatar = avatarList.at(i);
+
+            if (shouldIgnore(avatar)) {
+                continue;
+            }
+
+            // priority = weighted linear combination of:
+            //   (a) apparentSize
+            //   (b) proximity to center of view
+            //   (c) time since last update
+            glm::vec3 avatarPosition = avatar->getPosition();
+            glm::vec3 offset = avatarPosition - frustumCenter;
+            float distance = glm::length(offset) + 0.001f; // add 1mm to avoid divide by zero
+
+            // FIXME - AvatarData has something equivolent to this
+            float radius = getBoundingRadius(avatar);
+
+            const glm::vec3& forward = cameraView.getDirection();
+            float apparentSize = 2.0f * radius / distance;
+            float cosineAngle = glm::length(glm::dot(offset, forward) * forward) / distance;
+            float age = (float)(startTime - getLastUpdated(avatar)) / (float)(USECS_PER_SECOND);
+
+            // NOTE: we are adding values of different units to get a single measure of "priority".
+            // Thus we multiply each component by a conversion "weight" that scales its units relative to the others.
+            // These weights are pure magic tuning and should be hard coded in the relation below,
+            // but are currently exposed for anyone who would like to explore fine tuning:
+            float priority = _avatarSortCoefficientSize * apparentSize
+                + _avatarSortCoefficientCenter * cosineAngle
+                + _avatarSortCoefficientAge * age;
+
+            // decrement priority of avatars outside keyhole
+            if (distance > cameraView.getCenterRadius()) {
+                if (!cameraView.sphereIntersectsFrustum(avatarPosition, radius)) {
+                    priority += OUT_OF_VIEW_PENALTY;
+                }
+            }
+            sortedAvatars.push(AvatarPriority(avatar, priority));
+        }
+    }
+    return sortedAvatars;
+}
+
+QScriptValue AvatarEntityMapToScriptValue(QScriptEngine* engine, const AvatarEntityMap& value) {
+    QScriptValue obj = engine->newObject();
+    for (auto entityID : value.keys()) {
+        QByteArray entityProperties = value.value(entityID);
+        QJsonDocument jsonEntityProperties = QJsonDocument::fromBinaryData(entityProperties);
+        if (!jsonEntityProperties.isObject()) {
+            qCDebug(avatars) << "bad AvatarEntityData in AvatarEntityMap" << QString(entityProperties.toHex());
+        }
+        
+        QVariant variantEntityProperties = jsonEntityProperties.toVariant();
+        QVariantMap entityPropertiesMap = variantEntityProperties.toMap();
+        QScriptValue scriptEntityProperties = variantMapToScriptValue(entityPropertiesMap, *engine);
+        
+        QString key = entityID.toString();
+        obj.setProperty(key, scriptEntityProperties);
+    }
+    return obj;
+}
+
+void AvatarEntityMapFromScriptValue(const QScriptValue& object, AvatarEntityMap& value) {
+    QScriptValueIterator itr(object);
+    while (itr.hasNext()) {
+        itr.next();
+        QUuid EntityID = QUuid(itr.name());
+        
+        QScriptValue scriptEntityProperties = itr.value();
+        QVariant variantEntityProperties = scriptEntityProperties.toVariant();
+        QJsonDocument jsonEntityProperties = QJsonDocument::fromVariant(variantEntityProperties);
+        QByteArray binaryEntityProperties = jsonEntityProperties.toBinaryData();
+        
+        value[EntityID] = binaryEntityProperties;
+    }
+}

libraries/avatars/src/AvatarData.h

@@ -14,6 +14,8 @@
 
 #include <string>
 #include <memory>
+#include <queue>
+
 /* VS2010 defines stdint.h, but not inttypes.h */
 #if defined(_MSC_VER)
 typedef signed char  int8_t;
@@ -44,6 +46,7 @@ typedef unsigned long long quint64;
 #include <QVariantMap>
 #include <QVector>
 #include <QtScript/QScriptable>
+#include <QtScript/QScriptValueIterator>
 #include <QReadWriteLock>
 
 #include <JointData.h>
@@ -57,6 +60,7 @@ typedef unsigned long long quint64;
 #include <ThreadSafeValueCache.h>
 #include <SharedUtil.h>
 #include <shared/RateCounter.h>
+#include <ViewFrustum.h>
 
 #include "AABox.h"
 #include "HeadData.h"
@@ -133,6 +137,7 @@ namespace AvatarDataPacket {
     const HasFlags PACKET_HAS_AVATAR_LOCAL_POSITION  = 1U << 9;
     const HasFlags PACKET_HAS_FACE_TRACKER_INFO      = 1U << 10;
     const HasFlags PACKET_HAS_JOINT_DATA             = 1U << 11;
+    const size_t AVATAR_HAS_FLAGS_SIZE = 2;
 
     // NOTE: AvatarDataPackets start with a uint16_t sequence number that is not reflected in the Header structure.
 
@@ -288,6 +293,31 @@ class AttachmentData;
 class Transform;
 using TransformPointer = std::shared_ptr<Transform>;
 
+class AvatarDataRate {
+public:
+    RateCounter<> globalPositionRate;
+    RateCounter<> localPositionRate;
+    RateCounter<> avatarBoundingBoxRate;
+    RateCounter<> avatarOrientationRate;
+    RateCounter<> avatarScaleRate;
+    RateCounter<> lookAtPositionRate;
+    RateCounter<> audioLoudnessRate;
+    RateCounter<> sensorToWorldRate;
+    RateCounter<> additionalFlagsRate;
+    RateCounter<> parentInfoRate;
+    RateCounter<> faceTrackerRate;
+    RateCounter<> jointDataRate;
+};
+
+class AvatarPriority {
+public:
+    AvatarPriority(AvatarSharedPointer a, float p) : avatar(a), priority(p) {}
+    AvatarSharedPointer avatar;
+    float priority;
+    // NOTE: we invert the less-than operator to sort high priorities to front
+    bool operator<(const AvatarPriority& other) const { return priority < other.priority; }
+};
+
 class AvatarData : public QObject, public SpatiallyNestable {
     Q_OBJECT
 
@@ -345,14 +375,18 @@ public:
     void setHandPosition(const glm::vec3& handPosition);
 
     typedef enum { 
+        NoData,
         MinimumData, 
         CullSmallData,
         IncludeSmallData,
         SendAllData
     } AvatarDataDetail;
 
+    virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail);
+
     virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
-                        bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector<JointData>* sentJointDataOut = nullptr);
+        AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, 
+        QVector<JointData>* sentJointDataOut, AvatarDataRate* outboundDataRateOut = nullptr) const;
 
     virtual void doneEncoding(bool cullSmallChanges);
 
@@ -380,7 +414,7 @@ public:
     void nextAttitude(glm::vec3 position, glm::quat orientation); // Can be safely called at any time.
     virtual void updateAttitude() {} // Tell skeleton mesh about changes
 
-    glm::quat getHeadOrientation() { 
+    glm::quat getHeadOrientation() const { 
         lazyInitHeadData();
         return _headData->getOrientation(); 
     }
@@ -419,7 +453,6 @@ public:
     void setAudioAverageLoudness(float value) { _headData->setAudioAverageLoudness(value); }
 
     //  Scale
-    float getTargetScale() const;
     virtual void setTargetScale(float targetScale);
 
     float getDomainLimitedScale() const { return glm::clamp(_targetScale, _domainMinimumScale, _domainMaximumScale); }
@@ -494,7 +527,7 @@ public:
     // displayNameChanged returns true if displayName has changed, false otherwise.
     void processAvatarIdentity(const Identity& identity, bool& identityChanged, bool& displayNameChanged);
 
-    QByteArray identityByteArray();
+    QByteArray identityByteArray() const;
 
     const QUrl& getSkeletonModelURL() const { return _skeletonModelURL; }
     const QString& getDisplayName() const { return _displayName; }
@@ -520,8 +553,6 @@ public:
 
     void setOwningAvatarMixer(const QWeakPointer<Node>& owningAvatarMixer) { _owningAvatarMixer = owningAvatarMixer; }
 
-    const AABox& getLocalAABox() const { return _localAABox; }
-
     int getUsecsSinceLastUpdate() const { return _averageBytesReceived.getUsecsSinceLastEvent(); }
     int getAverageBytesReceivedPerSecond() const;
     int getReceiveRate() const;
@@ -534,8 +565,8 @@ public:
     QJsonObject toJson() const;
     void fromJson(const QJsonObject& json, bool useFrameSkeleton = true);
 
-    glm::vec3 getClientGlobalPosition() { return _globalPosition; }
-    glm::vec3 getGlobalBoundingBoxCorner() { return _globalPosition + _globalBoundingBoxOffset - _globalBoundingBoxDimensions; }
+    glm::vec3 getClientGlobalPosition() const { return _globalPosition; }
+    glm::vec3 getGlobalBoundingBoxCorner() const { return _globalPosition + _globalBoundingBoxOffset - _globalBoundingBoxDimensions; }
 
     Q_INVOKABLE AvatarEntityMap getAvatarEntityData() const;
     Q_INVOKABLE void setAvatarEntityData(const AvatarEntityMap& avatarEntityData);
@@ -550,7 +581,7 @@ public:
     Q_INVOKABLE float getDataRate(const QString& rateName = QString("")) const;
     Q_INVOKABLE float getUpdateRate(const QString& rateName = QString("")) const;
 
-    int getJointCount() { return _jointData.size(); }
+    int getJointCount() const { return _jointData.size(); }
 
     QVector<JointData> getLastSentJointData() {
         QReadLocker readLock(&_jointDataLock);
@@ -559,6 +590,28 @@ public:
     }
 
 
+    bool shouldDie() const {
+        const qint64 AVATAR_SILENCE_THRESHOLD_USECS = 5 * USECS_PER_SECOND;
+        return _owningAvatarMixer.isNull() || getUsecsSinceLastUpdate() > AVATAR_SILENCE_THRESHOLD_USECS;
+    }
+
+    static const float OUT_OF_VIEW_PENALTY;
+
+    static std::priority_queue<AvatarPriority> sortAvatars(
+        QList<AvatarSharedPointer> avatarList,
+        const ViewFrustum& cameraView,
+        std::function<uint64_t(AvatarSharedPointer)> getLastUpdated,
+        std::function<float(AvatarSharedPointer)> getBoundingRadius,
+        std::function<bool(AvatarSharedPointer)> shouldIgnore);
+
+    // TODO: remove this HACK once we settle on optimal sort coefficients
+    // These coefficients exposed for fine tuning the sort priority for transfering new _jointData to the render pipeline.
+    static float _avatarSortCoefficientSize;
+    static float _avatarSortCoefficientCenter;
+    static float _avatarSortCoefficientAge;
+
+
+
 public slots:
     void sendAvatarDataPacket();
     void sendIdentityPacket();
@@ -571,28 +624,27 @@ public slots:
     virtual bool setAbsoluteJointRotationInObjectFrame(int index, const glm::quat& rotation) override { return false; }
     virtual bool setAbsoluteJointTranslationInObjectFrame(int index, const glm::vec3& translation) override { return false; }
 
-    float getTargetScale() { return _targetScale; }
+    float getTargetScale() const { return _targetScale; } // why is this a slot?
 
     void resetLastSent() { _lastToByteArray = 0; }
 
 protected:
-    void lazyInitHeadData();
+    void lazyInitHeadData() const;
 
-    float getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition);
-    float getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition);
+    float getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) const;
+    float getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition) const;
 
-    bool avatarBoundingBoxChangedSince(quint64 time);
-    bool avatarScaleChangedSince(quint64 time);
-    bool lookAtPositionChangedSince(quint64 time);
-    bool audioLoudnessChangedSince(quint64 time);
-    bool sensorToWorldMatrixChangedSince(quint64 time);
-    bool additionalFlagsChangedSince(quint64 time);
+    bool avatarBoundingBoxChangedSince(quint64 time) const { return _avatarBoundingBoxChanged >= time; }
+    bool avatarScaleChangedSince(quint64 time) const { return _avatarScaleChanged >= time; }
+    bool lookAtPositionChangedSince(quint64 time) const { return _headData->lookAtPositionChangedSince(time); }
+    bool audioLoudnessChangedSince(quint64 time) const { return _headData->audioLoudnessChangedSince(time); }
+    bool sensorToWorldMatrixChangedSince(quint64 time) const { return _sensorToWorldMatrixChanged >= time; }
+    bool additionalFlagsChangedSince(quint64 time) const { return _additionalFlagsChanged >= time; }
+    bool parentInfoChangedSince(quint64 time) const { return _parentChanged >= time; }
+    bool faceTrackerInfoChangedSince(quint64 time) const { return true; } // FIXME
 
-    bool hasParent() { return !getParentID().isNull(); }
-    bool parentInfoChangedSince(quint64 time);
-
-    bool hasFaceTracker() { return _headData ? _headData->_isFaceTrackerConnected : false; }
-    bool faceTrackerInfoChangedSince(quint64 time);
+    bool hasParent() const { return !getParentID().isNull(); }
+    bool hasFaceTracker() const { return _headData ? _headData->_isFaceTrackerConnected : false; }
 
     glm::vec3 _handPosition;
     virtual const QString& getSessionDisplayNameForTransport() const { return _sessionDisplayName; }
@@ -616,7 +668,7 @@ protected:
     bool _forceFaceTrackerConnected;
     bool _hasNewJointData { true }; // set in AvatarData, cleared in Avatar
 
-    HeadData* _headData { nullptr };
+    mutable HeadData* _headData { nullptr };
 
     QUrl _skeletonModelURL;
     bool _firstSkeletonCheck { true };
@@ -624,7 +676,7 @@ protected:
     QVector<AttachmentData> _attachmentData;
     QString _displayName;
     QString _sessionDisplayName { };
-    const QUrl& cannonicalSkeletonModelURL(const QUrl& empty);
+    QUrl cannonicalSkeletonModelURL(const QUrl& empty) const;
 
     float _displayNameTargetAlpha;
     float _displayNameAlpha;
@@ -641,8 +693,6 @@ protected:
 
     glm::vec3 _targetVelocity;
 
-    AABox _localAABox;
-
     SimpleMovingAverage _averageBytesReceived;
 
     // During recording, this holds the starting position, orientation & scale of the recorded avatar
@@ -695,18 +745,7 @@ protected:
     RateCounter<> _jointDataUpdateRate;
 
     // Some rate data for outgoing data
-    RateCounter<> _globalPositionRateOutbound;
-    RateCounter<> _localPositionRateOutbound;
-    RateCounter<> _avatarBoundingBoxRateOutbound;
-    RateCounter<> _avatarOrientationRateOutbound;
-    RateCounter<> _avatarScaleRateOutbound;
-    RateCounter<> _lookAtPositionRateOutbound;
-    RateCounter<> _audioLoudnessRateOutbound;
-    RateCounter<> _sensorToWorldRateOutbound;
-    RateCounter<> _additionalFlagsRateOutbound;
-    RateCounter<> _parentInfoRateOutbound;
-    RateCounter<> _faceTrackerRateOutbound;
-    RateCounter<> _jointDataRateOutbound;
+    AvatarDataRate _outboundDataRate;
 
     glm::vec3 _globalBoundingBoxDimensions;
     glm::vec3 _globalBoundingBoxOffset;
@@ -809,6 +848,11 @@ Q_DECLARE_METATYPE(RayToAvatarIntersectionResult)
 QScriptValue RayToAvatarIntersectionResultToScriptValue(QScriptEngine* engine, const RayToAvatarIntersectionResult& results);
 void RayToAvatarIntersectionResultFromScriptValue(const QScriptValue& object, RayToAvatarIntersectionResult& results);
 
+Q_DECLARE_METATYPE(AvatarEntityMap)
+
+QScriptValue AvatarEntityMapToScriptValue(QScriptEngine* engine, const AvatarEntityMap& value);
+void AvatarEntityMapFromScriptValue(const QScriptValue& object, AvatarEntityMap& value);
+
 // faux joint indexes (-1 means invalid)
 const int SENSOR_TO_WORLD_MATRIX_INDEX = 65534; // -2
 const int CONTROLLER_RIGHTHAND_INDEX = 65533; // -3

libraries/avatars/src/AvatarHashMap.cpp

@@ -190,3 +190,4 @@ void AvatarHashMap::sessionUUIDChanged(const QUuid& sessionUUID, const QUuid& ol
     _lastOwnerSessionUUID = oldUUID;
     emit avatarSessionChangedEvent(sessionUUID, oldUUID);
 }
+

libraries/avatars/src/AvatarHashMap.h

@@ -27,7 +27,6 @@
 
 #include "AvatarData.h"
 
-
 class AvatarHashMap : public QObject, public Dependency {
     Q_OBJECT
     SINGLETON_DEPENDENCY

libraries/entities/src/EntityNodeData.cpp

@@ -0,0 +1,30 @@
+//
+//  EntityNodeData.cpp
+//  libraries/entities/src
+//
+//  Created by Stephen Birarda on 2/15/17
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "EntityNodeData.h"
+
+bool EntityNodeData::insertFlaggedExtraEntity(const QUuid& filteredEntityID, const QUuid& extraEntityID) {
+    _flaggedExtraEntities[filteredEntityID].insert(extraEntityID);
+    return !_previousFlaggedExtraEntities[filteredEntityID].contains(extraEntityID);
+}
+
+bool EntityNodeData::isEntityFlaggedAsExtra(const QUuid& entityID) const {
+
+    // enumerate each of the sets for the entities that matched our filter
+    // and immediately return true if any of them contain this entity ID
+    foreach(QSet<QUuid> entitySet, _flaggedExtraEntities) {
+        if (entitySet.contains(entityID)) {
+            return true;
+        }
+    }
+
+    return false;
+}

libraries/entities/src/EntityNodeData.h

@@ -1,6 +1,6 @@
 //
 //  EntityNodeData.h
-//  assignment-client/src/entities
+//  libraries/entities/src
 //
 //  Created by Brad Hefta-Gaub on 4/29/14
 //  Copyright 2014 High Fidelity, Inc.
@@ -16,6 +16,13 @@
 
 #include <OctreeQueryNode.h>
 
+namespace EntityJSONQueryProperties {
+    static const QString SERVER_SCRIPTS_PROPERTY = "serverScripts";
+    static const QString FLAGS_PROPERTY = "flags";
+    static const QString INCLUDE_ANCESTORS_PROPERTY = "includeAncestors";
+    static const QString INCLUDE_DESCENDANTS_PROPERTY = "includeDescendants";
+}
+
 class EntityNodeData : public OctreeQueryNode {
 public:
     virtual PacketType getMyPacketType() const override { return PacketType::EntityData; }
@@ -24,13 +31,24 @@ public:
     void setLastDeletedEntitiesSentAt(quint64 sentAt) { _lastDeletedEntitiesSentAt = sentAt; }
     
     // these can only be called from the OctreeSendThread for the given Node
-    void insertEntitySentLastFrame(const QUuid& entityID) { _entitiesSentLastFrame.insert(entityID); }
-    void removeEntitySentLastFrame(const QUuid& entityID) { _entitiesSentLastFrame.remove(entityID); }
-    bool sentEntityLastFrame(const QUuid& entityID) { return _entitiesSentLastFrame.contains(entityID); }
+    void insertSentFilteredEntity(const QUuid& entityID) { _sentFilteredEntities.insert(entityID); }
+    void removeSentFilteredEntity(const QUuid& entityID) { _sentFilteredEntities.remove(entityID); }
+    bool sentFilteredEntity(const QUuid& entityID) { return _sentFilteredEntities.contains(entityID); }
+    QSet<QUuid> getSentFilteredEntities() { return _sentFilteredEntities; }
+
+    // the following flagged extra entity methods can only be called from the OctreeSendThread for the given Node
+
+    // inserts the extra entity and returns a boolean indicating wether the extraEntityID was a new addition
+    bool insertFlaggedExtraEntity(const QUuid& filteredEntityID, const QUuid& extraEntityID);
+    
+    bool isEntityFlaggedAsExtra(const QUuid& entityID) const;
+    void resetFlaggedExtraEntities() { _previousFlaggedExtraEntities = _flaggedExtraEntities; _flaggedExtraEntities.clear(); }
 
 private:
     quint64 _lastDeletedEntitiesSentAt { usecTimestampNow() };
-    QSet<QUuid> _entitiesSentLastFrame;
+    QSet<QUuid> _sentFilteredEntities;
+    QHash<QUuid, QSet<QUuid>> _flaggedExtraEntities;
+    QHash<QUuid, QSet<QUuid>> _previousFlaggedExtraEntities;
 };
 
 #endif // hifi_EntityNodeData_h

libraries/entities/src/EntityTreeElement.cpp

@@ -310,16 +310,17 @@ OctreeElement::AppendState EntityTreeElement::appendElementData(OctreePacketData
 
                     if (entityMatchesFilters) {
                         // make sure this entity is in the set of entities sent last frame
-                        entityNodeData->insertEntitySentLastFrame(entity->getID());
-
-                    } else {
-                        // we might include this entity if it matched in the previous frame
-                        if (entityNodeData->sentEntityLastFrame(entity->getID())) {
-
-                            entityNodeData->removeEntitySentLastFrame(entity->getID());
-                        } else {
-                            includeThisEntity = false;
-                        }
+                        entityNodeData->insertSentFilteredEntity(entity->getID());
+                    } else if (entityNodeData->sentFilteredEntity(entity->getID())) {
+                        // this entity matched in the previous frame - we send it still so the client realizes it just
+                        // fell outside of their filter
+                        entityNodeData->removeSentFilteredEntity(entity->getID());
+                    } else if (!entityNodeData->isEntityFlaggedAsExtra(entity->getID())) {
+                        // we don't send this entity because
+                        // (1) it didn't match our filter
+                        // (2) it didn't match our filter last frame
+                        // (3) it isn't one the JSON query flags told us we should still include
+                        includeThisEntity = false;
                     }
                 }
 

libraries/networking/src/LimitedNodeList.h

@@ -34,6 +34,7 @@
 #include <tbb/concurrent_unordered_map.h>
 
 #include <DependencyManager.h>
+#include <SharedUtil.h>
 
 #include "DomainHandler.h"
 #include "Node.h"
@@ -182,15 +183,33 @@ public:
     //   This allows multiple threads (i.e. a thread pool) to share a lock
     //   without deadlocking when a dying node attempts to acquire a write lock
     template<typename NestedNodeLambda>
-    void nestedEach(NestedNodeLambda functor) {
-        QReadLocker readLock(&_nodeMutex);
+    void nestedEach(NestedNodeLambda functor, 
+                    int* lockWaitOut = nullptr, 
+                    int* nodeTransformOut = nullptr, 
+                    int* functorOut = nullptr) {
+        auto start = usecTimestampNow();
+        {
+            QReadLocker readLock(&_nodeMutex);
+            auto endLock = usecTimestampNow();
+            if (lockWaitOut) {
+                *lockWaitOut = (endLock - start);
+            }
 
-        std::vector<SharedNodePointer> nodes(_nodeHash.size());
-        std::transform(_nodeHash.cbegin(), _nodeHash.cend(), nodes.begin(), [](const NodeHash::value_type& it) {
-            return it.second;
-        });
+            std::vector<SharedNodePointer> nodes(_nodeHash.size());
+            std::transform(_nodeHash.cbegin(), _nodeHash.cend(), nodes.begin(), [](const NodeHash::value_type& it) {
+                return it.second;
+            });
+            auto endTransform = usecTimestampNow();
+            if (nodeTransformOut) {
+                *nodeTransformOut = (endTransform - endLock);
+            }
 
-        functor(nodes.cbegin(), nodes.cend());
+            functor(nodes.cbegin(), nodes.cend());
+            auto endFunctor = usecTimestampNow();
+            if (functorOut) {
+                *functorOut = (endFunctor - endTransform);
+            }
+        }
     }
 
     template<typename NodeLambda>

libraries/networking/src/udt/PacketHeaders.cpp

@@ -51,7 +51,7 @@ PacketVersion versionForPacketType(PacketType packetType) {
         case PacketType::EntityPhysics:
             return VERSION_ENTITIES_ZONE_FILTERS;
         case PacketType::EntityQuery:
-            return static_cast<PacketVersion>(EntityQueryPacketVersion::JsonFilter);
+            return static_cast<PacketVersion>(EntityQueryPacketVersion::JSONFilterWithFamilyTree);
         case PacketType::AvatarIdentity:
         case PacketType::AvatarData:
         case PacketType::BulkAvatarData:

libraries/networking/src/udt/PacketHeaders.h

@@ -112,7 +112,8 @@ public:
         ReloadEntityServerScript,
         EntityPhysics,
         EntityServerScriptLog,
-        LAST_PACKET_TYPE = EntityServerScriptLog
+        AdjustAvatarSorting,
+        LAST_PACKET_TYPE = AdjustAvatarSorting
     };
 };
 
@@ -207,7 +208,8 @@ const PacketVersion VERSION_ENTITIES_PHYSICS_PACKET = 67;
 const PacketVersion VERSION_ENTITIES_ZONE_FILTERS = 68;
 
 enum class EntityQueryPacketVersion: PacketVersion {
-    JsonFilter = 18
+    JSONFilter = 18,
+    JSONFilterWithFamilyTree = 19
 };
 
 enum class AssetServerPacketVersion: PacketVersion {

libraries/octree/src/OctreeQueryNode.h

@@ -103,6 +103,9 @@ public:
     // call only from OctreeSendThread for the given node
     bool haveJSONParametersChanged();
 
+    bool shouldForceFullScene() const { return _shouldForceFullScene; }
+    void setShouldForceFullScene(bool shouldForceFullScene) { _shouldForceFullScene = shouldForceFullScene; }
+
 private:
     OctreeQueryNode(const OctreeQueryNode &);
     OctreeQueryNode& operator= (const OctreeQueryNode&);
@@ -148,6 +151,8 @@ private:
     std::array<char, udt::MAX_PACKET_SIZE> _lastOctreePayload;
 
     QJsonObject _lastCheckJSONParameters;
+
+    bool _shouldForceFullScene { false };
 };
 
 #endif // hifi_OctreeQueryNode_h

libraries/script-engine/src/RecordingScriptingInterface.cpp

@@ -56,6 +56,11 @@ bool RecordingScriptingInterface::loadRecording(const QString& url) {
     using namespace recording;
 
     auto loader = ClipCache::instance().getClipLoader(url);
+    if (!loader) {
+        qWarning() << "Clip failed to load from " << url;
+        return false;
+    }
+
     if (!loader->isLoaded()) {
         QEventLoop loop;
         QObject::connect(loader.data(), &Resource::loaded, &loop, &QEventLoop::quit);

libraries/script-engine/src/ScriptEngine.cpp

@@ -576,6 +576,7 @@ void ScriptEngine::init() {
     qScriptRegisterMetaType(this, EntityItemIDtoScriptValue, EntityItemIDfromScriptValue);
     qScriptRegisterMetaType(this, RayToEntityIntersectionResultToScriptValue, RayToEntityIntersectionResultFromScriptValue);
     qScriptRegisterMetaType(this, RayToAvatarIntersectionResultToScriptValue, RayToAvatarIntersectionResultFromScriptValue);
+    qScriptRegisterMetaType(this, AvatarEntityMapToScriptValue, AvatarEntityMapFromScriptValue);
     qScriptRegisterSequenceMetaType<QVector<QUuid>>(this);
     qScriptRegisterSequenceMetaType<QVector<EntityItemID>>(this);
 

libraries/script-engine/src/TabletScriptingInterface.cpp

@@ -172,7 +172,7 @@ static const char* WEB_VIEW_SOURCE_URL = "TabletWebView.qml";
 static const char* VRMENU_SOURCE_URL = "TabletMenu.qml";
 
 class TabletRootWindow : public QmlWindowClass {
-    virtual QString qmlSource() const { return "hifi/tablet/WindowRoot.qml"; }
+    virtual QString qmlSource() const override { return "hifi/tablet/WindowRoot.qml"; }
 };
 
 TabletProxy::TabletProxy(QString name) : _name(name) {
@@ -366,6 +366,7 @@ void TabletProxy::gotoWebScreen(const QString& url, const QString& injectedJavaS
     }
 
     if (root) {
+        removeButtonsFromHomeScreen();
         QMetaObject::invokeMethod(root, "loadSource", Q_ARG(const QVariant&, QVariant(WEB_VIEW_SOURCE_URL)));
         QMetaObject::invokeMethod(root, "setShown", Q_ARG(const QVariant&, QVariant(true)));
         QMetaObject::invokeMethod(root, "loadWebUrl", Q_ARG(const QVariant&, QVariant(url)), Q_ARG(const QVariant&, QVariant(injectedJavaScriptUrl)));

libraries/shared/src/SpatiallyNestable.cpp

@@ -70,6 +70,9 @@ void SpatiallyNestable::setParentID(const QUuid& parentID) {
             _parentKnowsMe = false;
         }
     });
+
+    bool success = false;
+    getParentPointer(success);
 }
 
 Transform SpatiallyNestable::getParentTransform(bool& success, int depth) const {

libraries/shared/src/SpatiallyNestable.h

@@ -179,9 +179,9 @@ public:
             const glm::vec3& localVelocity,
             const glm::vec3& localAngularVelocity);
 
-    bool scaleChangedSince(quint64 time) { return _scaleChanged > time; }
-    bool tranlationChangedSince(quint64 time) { return _translationChanged > time; }
-    bool rotationChangedSince(quint64 time) { return _rotationChanged > time; }
+    bool scaleChangedSince(quint64 time) const { return _scaleChanged > time; }
+    bool tranlationChangedSince(quint64 time) const { return _translationChanged > time; }
+    bool rotationChangedSince(quint64 time) const { return _rotationChanged > time; }
 
 protected:
     const NestableType _nestableType; // EntityItem or an AvatarData

scripts/developer/debugging/debugAvatarMixer.js

@@ -58,11 +58,14 @@ function updateOverlays() {
 
             // setup a position for the overlay that is just above this avatar's head
             var overlayPosition = avatar.getJointPosition("Head");
-            overlayPosition.y += 1.05;
+            overlayPosition.y += 1.15;
+
+            var rows = 8;
 
             var text = avatarID + "\n"
                        +"--- Data from Mixer ---\n"
                        +"All: " + AvatarManager.getAvatarDataRate(avatarID).toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID).toFixed(2) + "hz)" + "\n"
+                       /*
                        +" GP: " + AvatarManager.getAvatarDataRate(avatarID,"globalPosition").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"globalPosition").toFixed(2) + "hz)" + "\n"
                        +" LP: " + AvatarManager.getAvatarDataRate(avatarID,"localPosition").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"localPosition").toFixed(2) + "hz)" + "\n"
                        +" BB: " + AvatarManager.getAvatarDataRate(avatarID,"avatarBoundingBox").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"avatarBoundingBox").toFixed(2) + "hz)" + "\n"
@@ -74,11 +77,12 @@ function updateOverlays() {
                        +" AF: " + AvatarManager.getAvatarDataRate(avatarID,"additionalFlags").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"additionalFlags").toFixed(2) + "hz)" + "\n"
                        +" PI: " + AvatarManager.getAvatarDataRate(avatarID,"parentInfo").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"parentInfo").toFixed(2) + "hz)" + "\n"
                        +" FT: " + AvatarManager.getAvatarDataRate(avatarID,"faceTracker").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"faceTracker").toFixed(2) + "hz)" + "\n"
+                       */
                        +" JD: " + AvatarManager.getAvatarDataRate(avatarID,"jointData").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"jointData").toFixed(2) + "hz)" + "\n"
                        +"--- Simulation ---\n"
                        +"All: " + AvatarManager.getAvatarSimulationRate(avatarID,"avatar").toFixed(2) + "hz \n"
                        +" inView: " + AvatarManager.getAvatarSimulationRate(avatarID,"avatarInView").toFixed(2) + "hz \n"
-                       +" SM: " + AvatarManager.getAvatarSimulationRate(avatarID,"skeletonModel").toFixed(2) + "hz \n"
+                       //+" SM: " + AvatarManager.getAvatarSimulationRate(avatarID,"skeletonModel").toFixed(2) + "hz \n"
                        +" JD: " + AvatarManager.getAvatarSimulationRate(avatarID,"jointData").toFixed(2) + "hz \n"
 
             if (avatarID in debugOverlays) {
@@ -93,7 +97,7 @@ function updateOverlays() {
                     position: overlayPosition,
                     dimensions: {
                         x: 1.25,
-                        y: 19 * 0.13
+                        y: rows * 0.13
                     },
                     lineHeight: 0.1,
                     font:{size:0.1},

scripts/developer/utilities/record/recorder.js

@@ -12,14 +12,14 @@
 HIFI_PUBLIC_BUCKET = "http://s3.amazonaws.com/hifi-public/";
 Script.include("/~/system/libraries/toolBars.js");
 
-var recordingFile = "recording.rec";
+var recordingFile = "recording.hfr";
 
 function setPlayerOptions() {
     Recording.setPlayFromCurrentLocation(true);
     Recording.setPlayerUseDisplayName(false);
     Recording.setPlayerUseAttachments(false);
     Recording.setPlayerUseHeadModel(false);
-    Recording.setPlayerUseSkeletonModel(false);
+    Recording.setPlayerUseSkeletonModel(true);
 }
 
 var windowDimensions = Controller.getViewportDimensions();
@@ -142,7 +142,6 @@ function setupTimer() {
         backgroundAlpha: 1.0,
         visible: true
     });
-
 }
 
 function updateTimer() {
@@ -272,7 +271,7 @@ function mousePressEvent(event) {
         }
     } else if (loadIcon === toolBar.clicked(clickedOverlay)) {
         if (!Recording.isRecording() && !Recording.isPlaying()) {
-            recordingFile = Window.browse("Load recorcding from file", ".", "Recordings (*.hfr *.rec *.HFR *.REC)");
+            recordingFile = Window.browse("Load recording from file", ".", "Recordings (*.hfr *.rec *.HFR *.REC)");
             if (!(recordingFile === "null" || recordingFile === null || recordingFile === "")) {
                 Recording.loadRecording(recordingFile);
             }
@@ -345,5 +344,3 @@ Script.scriptEnding.connect(scriptEnding);
 
 // Should be called last to put everything into position
 moveUI();
-
-

scripts/system/audio.js

@@ -16,19 +16,51 @@
 var TABLET_BUTTON_NAME = "AUDIO";
 var HOME_BUTTON_TEXTURE = "http://hifi-content.s3.amazonaws.com/alan/dev/tablet-with-home-button.fbx/tablet-with-home-button.fbm/button-root.png";
 
+var MUTE_ICONS = {
+    icon: "icons/tablet-icons/mic-mute-i.svg",
+    activeIcon: "icons/tablet-icons/mic-mute-a.svg"
+};
+
+var UNMUTE_ICONS = {
+    icon: "icons/tablet-icons/mic-unmute-i.svg",
+    activeIcon: "icons/tablet-icons/mic-unmute-a.svg"
+};
+
 function onMuteToggled() {
-    button.editProperties({ isActive: AudioDevice.getMuted() });
+    if (AudioDevice.getMuted()) {
+        button.editProperties(MUTE_ICONS);
+    } else {
+        button.editProperties(UNMUTE_ICONS);
+    }
 }
-function onClicked(){
-    var entity = HMD.tabletID;
-    Entities.editEntity(entity, { textures: JSON.stringify({ "tex.close": HOME_BUTTON_TEXTURE }) });
-    tablet.gotoMenuScreen("Audio");
+
+var shouldActivateButton = false;
+var onAudioScreen = false;
+
+function onClicked() {
+    if (onAudioScreen) {
+        // for toolbar-mode: go back to home screen, this will close the window.
+        tablet.gotoHomeScreen();
+    } else {
+        var entity = HMD.tabletID;
+        Entities.editEntity(entity, { textures: JSON.stringify({ "tex.close": HOME_BUTTON_TEXTURE }) });
+        shouldActivateButton = true;
+        tablet.gotoMenuScreen("Audio");
+        onAudioScreen = true;
+    }
+}
+
+function onScreenChanged(type, url) {
+    // for toolbar mode: change button to active when window is first openend, false otherwise.
+    button.editProperties({isActive: shouldActivateButton});
+    shouldActivateButton = false;
+    onAudioScreen = false;
 }
 
 var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
 var button = tablet.addButton({
-    icon: "icons/tablet-icons/mic-unmute-i.svg",
-    activeIcon: "icons/tablet-icons/mic-mute-a.svg",
+    icon: AudioDevice.getMuted() ? MUTE_ICONS.icon : UNMUTE_ICONS.icon,
+    activeIcon: AudioDevice.getMuted() ? MUTE_ICONS.activeIcon : UNMUTE_ICONS.activeIcon,
     text: TABLET_BUTTON_NAME,
     sortOrder: 1
 });
@@ -36,10 +68,12 @@ var button = tablet.addButton({
 onMuteToggled();
 
 button.clicked.connect(onClicked);
+tablet.screenChanged.connect(onScreenChanged);
 AudioDevice.muteToggled.connect(onMuteToggled);
 
 Script.scriptEnding.connect(function () {
     button.clicked.disconnect(onClicked);
+    tablet.screenChanged.disconnect(onScreenChanged);
     AudioDevice.muteToggled.disconnect(onMuteToggled);
     tablet.removeButton(button);
 });

scripts/system/controllers/handControllerGrab.js

@@ -1233,7 +1233,13 @@ function MyController(hand) {
         });
         if (grabbableEntities.length > 0) {
             if (!this.grabPointIntersectsEntity) {
-                Controller.triggerHapticPulse(1, 20, this.hand);
+                // don't do haptic pulse for tablet
+                var nonTabletEntities = grabbableEntities.filter(function(entityID) {
+                    return entityID != HMD.tabletID && entityID != HMD.homeButtonID;
+                });
+                if (nonTabletEntities.length > 0) {
+                    Controller.triggerHapticPulse(1, 20, this.hand);
+                }
                 this.grabPointIntersectsEntity = true;
                 this.grabPointSphereOn();
             }

scripts/system/libraries/WebTablet.js

@@ -49,24 +49,30 @@ function calcSpawnInfo(hand, height) {
         var handController = getControllerWorldLocation(hand, true);
         var controllerPosition = handController.position;
 
-        // compute the angle of the chord with length (height / 2)
-        var theta = Math.asin(height / (2 * Vec3.distance(headPos, controllerPosition)));
+        // base of the tablet is slightly above controller position
+        var TABLET_BASE_DISPLACEMENT = {x: 0, y: 0.1, z: 0};
+        var tabletBase = Vec3.sum(controllerPosition, TABLET_BASE_DISPLACEMENT);
 
-        // then we can use this angle to rotate the vector between the HMD position and the center of the tablet.
-        // this vector, u, will become our new look at direction.
-        var d = Vec3.normalize(Vec3.subtract(headPos, controllerPosition));
+        var d = Vec3.subtract(headPos, tabletBase);
+        var theta = Math.acos(d.y / Vec3.length(d));
+        d.y = 0;
+        if (Vec3.length(d) < 0.0001) {
+            d = {x: 1, y: 0, z: 0};
+        } else {
+            d = Vec3.normalize(d);
+        }
         var w = Vec3.normalize(Vec3.cross(Y_AXIS, d));
-        var q = Quat.angleAxis(theta * (180 / Math.PI), w);
+        var ANGLE_OFFSET = 25;
+        var q = Quat.angleAxis(theta * (180 / Math.PI) - (90 - ANGLE_OFFSET), w);
         var u = Vec3.multiplyQbyV(q, d);
 
         // use u to compute a full lookAt quaternion.
-        var lookAtRot = Quat.lookAt(controllerPosition, Vec3.sum(controllerPosition, u), Y_AXIS);
-
-        // adjust the tablet position by a small amount.
-        var yDisplacement = (height / 2) + 0.1;
+        var lookAtRot = Quat.lookAt(tabletBase, Vec3.sum(tabletBase, u), Y_AXIS);
+        var yDisplacement = (height / 2);
         var zDisplacement = 0.05;
         var tabletOffset = Vec3.multiplyQbyV(lookAtRot, {x: 0, y: yDisplacement, z: zDisplacement});
-        finalPosition = Vec3.sum(controllerPosition, tabletOffset);
+        finalPosition = Vec3.sum(tabletBase, tabletOffset);
+
         return {
             position: finalPosition,
             rotation: lookAtRot
@@ -159,7 +165,7 @@ WebTablet = function (url, width, dpi, hand, clientOnly) {
     });
 
     this.receive = function (channel, senderID, senderUUID, localOnly) {
-        if (_this.homeButtonEntity === parseInt(senderID)) {
+        if (_this.homeButtonEntity == senderID) {
             var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
             var onHomeScreen = tablet.onHomeScreen();
             if (onHomeScreen) {
@@ -219,7 +225,6 @@ WebTablet = function (url, width, dpi, hand, clientOnly) {
 };
 
 WebTablet.prototype.setHomeButtonTexture = function() {
-    print(this.homeButtonEntity);
     Entities.editEntity(this.tabletEntityID, {textures: JSON.stringify({"tex.close": HOME_BUTTON_TEXTURE})});
 };
 
@@ -385,14 +390,10 @@ WebTablet.prototype.register = function() {
 WebTablet.prototype.cleanUpOldTabletsOnJoint = function(jointIndex) {
     var children = Entities.getChildrenIDsOfJoint(MyAvatar.sessionUUID, jointIndex);
     children = children.concat(Entities.getChildrenIDsOfJoint(AVATAR_SELF_ID, jointIndex));
-    print("cleanup " + children);
     children.forEach(function(childID) {
         var props = Entities.getEntityProperties(childID, ["name"]);
         if (props.name === "WebTablet Tablet") {
-            print("cleaning up " + props.name);
             Entities.deleteEntity(childID);
-        } else {
-            print("not cleaning up " + props.name);
         }
     });
 };

scripts/system/marketplaces/marketplaces.js

@@ -52,10 +52,16 @@ function onMessageBoxClosed(id, button) {
 
 Window.messageBoxClosed.connect(onMessageBoxClosed);
 
+var shouldActivateButton = false;
+var onMarketplaceScreen = false;
+
 function showMarketplace() {
     UserActivityLogger.openedMarketplace();
 
+    shouldActivateButton = true;
     tablet.gotoWebScreen(MARKETPLACE_URL_INITIAL, MARKETPLACES_INJECT_SCRIPT_URL);
+    onMarketplaceScreen = true;
+
     tablet.webEventReceived.connect(function (message) {
 
         if (message === GOTO_DIRECTORY) {
@@ -98,15 +104,10 @@ function showMarketplace() {
     });
 }
 
-function toggleMarketplace() {
-    var entity = HMD.tabletID;
-    Entities.editEntity(entity, {textures: JSON.stringify({"tex.close": HOME_BUTTON_TEXTURE})});
-    showMarketplace();
-}
-
 var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
 var marketplaceButton = tablet.addButton({
     icon: "icons/tablet-icons/market-i.svg",
+    activeIcon: "icons/tablet-icons/market-a.svg",
     text: "MARKET",
     sortOrder: 9
 });
@@ -117,16 +118,30 @@ function onCanWriteAssetsChanged() {
 }
 
 function onClick() {
-    toggleMarketplace();
+    if (onMarketplaceScreen) {
+        // for toolbar-mode: go back to home screen, this will close the window.
+        tablet.gotoHomeScreen();
+    } else {
+        var entity = HMD.tabletID;
+        Entities.editEntity(entity, {textures: JSON.stringify({"tex.close": HOME_BUTTON_TEXTURE})});
+        showMarketplace();
+    }
+}
+
+function onScreenChanged(type, url) {
+    // for toolbar mode: change button to active when window is first openend, false otherwise.
+    marketplaceButton.editProperties({isActive: shouldActivateButton});
+    shouldActivateButton = false;
+    onMarketplaceScreen = false;
 }
 
 marketplaceButton.clicked.connect(onClick);
+tablet.screenChanged.connect(onScreenChanged);
 Entities.canWriteAssetsChanged.connect(onCanWriteAssetsChanged);
 
 Script.scriptEnding.connect(function () {
-    if (tablet) {
-        tablet.removeButton(marketplaceButton);
-    }
+    tablet.removeButton(marketplaceButton);
+    tablet.screenChanged.disconnect(onScreenChanged);
     Entities.canWriteAssetsChanged.disconnect(onCanWriteAssetsChanged);
 });
 

scripts/system/menu.js

@@ -10,26 +10,46 @@
 //
 
 var HOME_BUTTON_TEXTURE = "http://hifi-content.s3.amazonaws.com/alan/dev/tablet-with-home-button.fbx/tablet-with-home-button.fbm/button-root.png";
-//var HOME_BUTTON_TEXTURE = Script.resourcesPath() + "meshes/tablet-with-home-button.fbx/tablet-with-home-button.fbm/button-root.png";
+// var HOME_BUTTON_TEXTURE = Script.resourcesPath() + "meshes/tablet-with-home-button.fbx/tablet-with-home-button.fbm/button-root.png";
 
 (function() {
     var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
     var button = tablet.addButton({
         icon: "icons/tablet-icons/menu-i.svg",
+        activeIcon: "icons/tablet-icons/menu-a.svg",
         text: "MENU",
         sortOrder: 3
     });
 
+    var shouldActivateButton = false;
+    var onMenuScreen = false;
+
     function onClicked() {
-        var entity = HMD.tabletID;
-        Entities.editEntity(entity, {textures: JSON.stringify({"tex.close": HOME_BUTTON_TEXTURE})});
-        tablet.gotoMenuScreen();
+        if (onMenuScreen) {
+            // for toolbar-mode: go back to home screen, this will close the window.
+            tablet.gotoHomeScreen();
+        } else {
+            var entity = HMD.tabletID;
+            Entities.editEntity(entity, {textures: JSON.stringify({"tex.close": HOME_BUTTON_TEXTURE})});
+            shouldActivateButton = true;
+            tablet.gotoMenuScreen();
+            onMenuScreen = true;
+        }
+    }
+
+    function onScreenChanged(type, url) {
+        // for toolbar mode: change button to active when window is first openend, false otherwise.
+        button.editProperties({isActive: shouldActivateButton});
+        shouldActivateButton = false;
+        onMenuScreen = false;
     }
 
     button.clicked.connect(onClicked);
+    tablet.screenChanged.connect(onScreenChanged);
 
     Script.scriptEnding.connect(function () {
         button.clicked.disconnect(onClicked);
         tablet.removeButton(button);
+        tablet.screenChanged.disconnect(onScreenChanged);
     });
 }());

scripts/system/pal.js

@@ -491,23 +491,17 @@ var button;
 var buttonName = "PEOPLE";
 var tablet = null;
 
-function onTabletScreenChanged(type, url) {
-    if (type !== "QML" || url !== "../Pal.qml") {
-        off();
-    }
-}
-
 function startup() {
     tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
     button = tablet.addButton({
         text: buttonName,
         icon: "icons/tablet-icons/people-i.svg",
+        activeIcon: "icons/tablet-icons/people-a.svg",
         sortOrder: 7
     });
     tablet.fromQml.connect(fromQml);
     button.clicked.connect(onTabletButtonClicked);
     tablet.screenChanged.connect(onTabletScreenChanged);
-
     Users.usernameFromIDReply.connect(usernameFromIDReply);
     Window.domainChanged.connect(clearLocalQMLDataAndClosePAL);
     Window.domainConnectionRefused.connect(clearLocalQMLDataAndClosePAL);
@@ -538,17 +532,39 @@ function off() {
     Users.requestsDomainListData = false;
 }
 
+var onPalScreen = false;
+var shouldActivateButton = false;
+
 function onTabletButtonClicked() {
-    tablet.loadQMLSource("../Pal.qml");
-    Users.requestsDomainListData = true;
-    populateUserList();
-    isWired = true;
-    Script.update.connect(updateOverlays);
-    Controller.mousePressEvent.connect(handleMouseEvent);
-    Controller.mouseMoveEvent.connect(handleMouseMoveEvent);
-    triggerMapping.enable();
-    triggerPressMapping.enable();
-    audioTimer = createAudioInterval(conserveResources ? AUDIO_LEVEL_CONSERVED_UPDATE_INTERVAL_MS : AUDIO_LEVEL_UPDATE_INTERVAL_MS);
+    if (onPalScreen) {
+        // for toolbar-mode: go back to home screen, this will close the window.
+        tablet.gotoHomeScreen();
+    } else {
+        shouldActivateButton = true;
+        tablet.loadQMLSource("../Pal.qml");
+        onPalScreen = true;
+        Users.requestsDomainListData = true;
+        populateUserList();
+        isWired = true;
+        Script.update.connect(updateOverlays);
+        Controller.mousePressEvent.connect(handleMouseEvent);
+        Controller.mouseMoveEvent.connect(handleMouseMoveEvent);
+        triggerMapping.enable();
+        triggerPressMapping.enable();
+        audioTimer = createAudioInterval(conserveResources ? AUDIO_LEVEL_CONSERVED_UPDATE_INTERVAL_MS : AUDIO_LEVEL_UPDATE_INTERVAL_MS);
+    }
+}
+
+function onTabletScreenChanged(type, url) {
+    // for toolbar mode: change button to active when window is first openend, false otherwise.
+    button.editProperties({isActive: shouldActivateButton});
+    shouldActivateButton = false;
+    onPalScreen = false;
+
+    // disable sphere overlays when not on pal screen.
+    if (type !== "QML" || url !== "../Pal.qml") {
+        off();
+    }
 }
 
 //
@@ -647,14 +663,12 @@ function shutdown() {
     button.clicked.disconnect(onTabletButtonClicked);
     tablet.removeButton(button);
     tablet.screenChanged.disconnect(onTabletScreenChanged);
-
     Users.usernameFromIDReply.disconnect(usernameFromIDReply);
     Window.domainChanged.disconnect(clearLocalQMLDataAndClosePAL);
     Window.domainConnectionRefused.disconnect(clearLocalQMLDataAndClosePAL);
     Messages.subscribe(CHANNEL);
     Messages.messageReceived.disconnect(receiveMessage);
     Users.avatarDisconnected.disconnect(avatarDisconnected);
-
     off();
 }
 

scripts/system/tablet-goto.js

@@ -14,10 +14,27 @@
 (function() { // BEGIN LOCAL_SCOPE
     var gotoQmlSource = "TabletAddressDialog.qml";
     var buttonName = "GOTO";
+    var onGotoScreen = false;
+    var shouldActivateButton = false;
 
-    function onClicked(){
-        tablet.loadQMLSource(gotoQmlSource);
+    function onClicked() {
+        if (onGotoScreen) {
+            // for toolbar-mode: go back to home screen, this will close the window.
+            tablet.gotoHomeScreen();
+        } else {
+            shouldActivateButton = true;
+            tablet.loadQMLSource(gotoQmlSource);
+            onGotoScreen = true;
+        }
     }
+
+    function onScreenChanged(type, url) {
+        // for toolbar mode: change button to active when window is first openend, false otherwise.
+        button.editProperties({isActive: shouldActivateButton});
+        shouldActivateButton = false;
+        onGotoScreen = false;
+    }
+
     var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
     var button = tablet.addButton({
         icon: "icons/tablet-icons/goto-i.svg",
@@ -27,12 +44,12 @@
     });
 
     button.clicked.connect(onClicked);
+    tablet.screenChanged.connect(onScreenChanged);
 
     Script.scriptEnding.connect(function () {
         button.clicked.disconnect(onClicked);
-        if (tablet) {
-            tablet.removeButton(button);
-        }
+        tablet.removeButton(button);
+        tablet.screenChanged.disconnect(onScreenChanged);
     });
 
 }()); // END LOCAL_SCOPE

scripts/system/tablet-users.js

@@ -1,7 +1,7 @@
 "use strict";
 
 //
-//  users.js
+//  tablet-users.js
 //
 //  Created by Faye Li on 18 Jan 2017.
 //  Copyright 2017 High Fidelity, Inc.
@@ -36,16 +36,34 @@
     var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
     var button = tablet.addButton({
         icon: "icons/tablet-icons/users-i.svg",
+        activeIcon: "icons/tablet-icons/users-a.svg",
         text: "USERS",
         sortOrder: 11
     });
 
+    var onUsersScreen = false;
+    var shouldActivateButton = false;
+
     function onClicked() {
-        var tabletEntity = HMD.tabletID;
-        if (tabletEntity) {
-            Entities.editEntity(tabletEntity, {textures: JSON.stringify({"tex.close" : HOME_BUTTON_TEXTURE})});
+        if (onUsersScreen) {
+            // for toolbar-mode: go back to home screen, this will close the window.
+            tablet.gotoHomeScreen();
+        } else {
+            var tabletEntity = HMD.tabletID;
+            if (tabletEntity) {
+                Entities.editEntity(tabletEntity, {textures: JSON.stringify({"tex.close" : HOME_BUTTON_TEXTURE})});
+            }
+            shouldActivateButton = true;
+            tablet.gotoWebScreen(USERS_URL);
+            onUsersScreen = true;
         }
-        tablet.gotoWebScreen(USERS_URL);
+    }
+
+    function onScreenChanged() {
+        // for toolbar mode: change button to active when window is first openend, false otherwise.
+        button.editProperties({isActive: shouldActivateButton});
+        shouldActivateButton = false;
+        onUsersScreen = false;
     }
 
     function onWebEventReceived(event) {
@@ -88,12 +106,13 @@
                 // update your visibility (all, friends, or none)
                 myVisibility = event.data.visibility;
                 GlobalServices.findableBy = myVisibility;
-            } 
+            }
         }
     }
 
     button.clicked.connect(onClicked);
     tablet.webEventReceived.connect(onWebEventReceived);
+    tablet.screenChanged.connect(onScreenChanged);
 
     function cleanup() {
         button.clicked.disconnect(onClicked);

scripts/tutorials/createDice.js

@@ -33,7 +33,7 @@ var BUTTON_SIZE = 32;
 var PADDING = 3;
 var BOTTOM_PADDING = 50;
 //a helper library for creating toolbars
-Script.include("http://hifi-production.s3.amazonaws.com/tutorials/dice/toolBars.js");
+Script.include("/~/system/libraries/toolBars.js");
 
 var toolBar = new ToolBar(0, 0, ToolBar.HORIZONTAL, "highfidelity.toolbars-dice", function(screenSize) {
   return {
@@ -139,4 +139,4 @@ function scriptEnding() {
 }
 
 Controller.mousePressEvent.connect(mousePressEvent);
-Script.scriptEnding.connect(scriptEnding);
+Script.scriptEnding.connect(scriptEnding);