Merge pull request #9728 from samcake/texmex

Updating with upstream master and fixing bad srgb format assignment

assignment-client/src/Agent.cpp

@@ -434,8 +434,16 @@ void Agent::executeScript() {
     connect(&_avatarAudioTimerThread, &QThread::finished, audioTimerWorker, &QObject::deleteLater); 
     _avatarAudioTimerThread.start();
     
-    // 60Hz timer for avatar
+    // Agents should run at 45hz
-    QObject::connect(_scriptEngine.get(), &ScriptEngine::update, this, &Agent::processAgentAvatar);
+    static const int AVATAR_DATA_HZ = 45;
+    static const int AVATAR_DATA_IN_MSECS = MSECS_PER_SECOND / AVATAR_DATA_HZ;
+    QTimer* avatarDataTimer = new QTimer(this);
+    connect(avatarDataTimer, &QTimer::timeout, this, &Agent::processAgentAvatar);
+    avatarDataTimer->setSingleShot(false);
+    avatarDataTimer->setInterval(AVATAR_DATA_IN_MSECS);
+    avatarDataTimer->setTimerType(Qt::PreciseTimer);
+    avatarDataTimer->start();
+
     _scriptEngine->run();
 
     Frame::clearFrameHandler(AUDIO_FRAME_TYPE);
@@ -538,7 +546,7 @@ void Agent::processAgentAvatar() {
         auto scriptedAvatar = DependencyManager::get<ScriptableAvatar>();
 
         AvatarData::AvatarDataDetail dataDetail = (randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO) ? AvatarData::SendAllData : AvatarData::CullSmallData;
-        QByteArray avatarByteArray = scriptedAvatar->toByteArray(dataDetail, 0, scriptedAvatar->getLastSentJointData());
+        QByteArray avatarByteArray = scriptedAvatar->toByteArrayStateful(dataDetail);
         scriptedAvatar->doneEncoding(true);
 
         static AvatarDataSequenceNumber sequenceNumber = 0;

assignment-client/src/avatars/AvatarMixer.h

@@ -21,6 +21,8 @@
 #include <ThreadedAssignment.h>
 #include "AvatarMixerClientData.h"
 
+#include "AvatarMixerSlavePool.h"
+
 /// Handles assignments of type AvatarMixer - distribution of avatar data to various clients
 class AvatarMixer : public ThreadedAssignment {
     Q_OBJECT
@@ -36,8 +38,8 @@ public slots:
     void sendStatsPacket() override;
 
 private slots:
+    void queueIncomingPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer node);
     void handleViewFrustumPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
-    void handleAvatarDataPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleAvatarIdentityPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleKillAvatarPacket(QSharedPointer<ReceivedMessage> message);
     void handleNodeIgnoreRequestPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
@@ -45,22 +47,29 @@ private slots:
     void handleRequestsDomainListDataPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void domainSettingsRequestComplete();
     void handlePacketVersionMismatch(PacketType type, const HifiSockAddr& senderSockAddr, const QUuid& senderUUID);
+    void start();
 
 
 private:
-    void broadcastAvatarData();
+    AvatarMixerClientData* getOrCreateClientData(SharedNodePointer node);
+    std::chrono::microseconds timeFrame(p_high_resolution_clock::time_point& timestamp);
+    void throttle(std::chrono::microseconds duration, int frame);
+
     void parseDomainServerSettings(const QJsonObject& domainSettings);
     void sendIdentityPacket(AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode);
 
-    QThread _broadcastThread;
+    void manageDisplayName(const SharedNodePointer& node);
 
     p_high_resolution_clock::time_point _lastFrameTimestamp;
 
-    float _trailingSleepRatio { 1.0f };
+    // FIXME - new throttling - use these values somehow
-    float _performanceThrottlingRatio { 0.0f };
+    float _trailingMixRatio { 0.0f };
+    float _throttlingRatio { 0.0f };
+
 
     int _sumListeners { 0 };
     int _numStatFrames { 0 };
+    int _numTightLoopFrames { 0 };
     int _sumIdentityPackets { 0 };
 
     float _maxKbpsPerNode = 0.0f;
@@ -68,11 +77,40 @@ private:
     float _domainMinimumScale { MIN_AVATAR_SCALE };
     float _domainMaximumScale { MAX_AVATAR_SCALE };
 
-    QTimer* _broadcastTimer = nullptr;
-
     RateCounter<> _broadcastRate;
     p_high_resolution_clock::time_point _lastDebugMessage;
     QHash<QString, QPair<int, int>> _sessionDisplayNames;
+
+    quint64 _displayNameManagementElapsedTime { 0 }; // total time spent in broadcastAvatarData/display name management... since last stats window
+    quint64 _ignoreCalculationElapsedTime { 0 };
+    quint64 _avatarDataPackingElapsedTime { 0 };
+    quint64 _packetSendingElapsedTime { 0 };
+
+    quint64 _broadcastAvatarDataElapsedTime { 0 }; // total time spent in broadcastAvatarData since last stats window
+    quint64 _broadcastAvatarDataInner { 0 };
+    quint64 _broadcastAvatarDataLockWait { 0 };
+    quint64 _broadcastAvatarDataNodeTransform { 0 };
+    quint64 _broadcastAvatarDataNodeFunctor { 0 };
+
+    quint64 _handleViewFrustumPacketElapsedTime { 0 };
+    quint64 _handleAvatarIdentityPacketElapsedTime { 0 };
+    quint64 _handleKillAvatarPacketElapsedTime { 0 };
+    quint64 _handleNodeIgnoreRequestPacketElapsedTime { 0 };
+    quint64 _handleRadiusIgnoreRequestPacketElapsedTime { 0 };
+    quint64 _handleRequestsDomainListDataPacketElapsedTime { 0 };
+    quint64 _processQueuedAvatarDataPacketsElapsedTime { 0 };
+    quint64 _processQueuedAvatarDataPacketsLockWaitElapsedTime { 0 };
+
+    quint64 _processEventsElapsedTime { 0 };
+    quint64 _sendStatsElapsedTime { 0 };
+    quint64 _queueIncomingPacketElapsedTime { 0 };
+    quint64 _lastStatsTime { usecTimestampNow() };
+
+    RateCounter<> _loopRate; // this is the rate that the main thread tight loop runs
+
+
+    AvatarMixerSlavePool _slavePool;
+
 };
 
 #endif // hifi_AvatarMixer_h

assignment-client/src/avatars/AvatarMixerClientData.cpp

@@ -16,10 +16,52 @@
 
 #include "AvatarMixerClientData.h"
 
+
+
+void AvatarMixerClientData::queuePacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer node) {
+    if (!_packetQueue.node) {
+        _packetQueue.node = node;
+    }
+    _packetQueue.push(message);
+}
+
+int AvatarMixerClientData::processPackets() {
+    int packetsProcessed = 0;
+    SharedNodePointer node = _packetQueue.node;
+    assert(_packetQueue.empty() || node);
+    _packetQueue.node.clear();
+
+    while (!_packetQueue.empty()) {
+        auto& packet = _packetQueue.front();
+
+        packetsProcessed++;
+
+        switch (packet->getType()) {
+            case PacketType::AvatarData:
+                parseData(*packet);
+                break;
+            default:
+                Q_UNREACHABLE();
+        }
+        _packetQueue.pop();
+    }
+    assert(_packetQueue.empty());
+
+    return packetsProcessed;
+}
+
 int AvatarMixerClientData::parseData(ReceivedMessage& message) {
+
     // pull the sequence number from the data first
-    message.readPrimitive(&_lastReceivedSequenceNumber);
+    uint16_t sequenceNumber;
+
+    message.readPrimitive(&sequenceNumber);
     
+    if (sequenceNumber < _lastReceivedSequenceNumber && _lastReceivedSequenceNumber != UINT16_MAX) {
+        incrementNumOutOfOrderSends();
+    }
+    _lastReceivedSequenceNumber = sequenceNumber;
+
     // compute the offset to the data payload
     return _avatar->parseDataFromBuffer(message.readWithoutCopy(message.getBytesLeftToRead()));
 }

assignment-client/src/avatars/AvatarMixerClientData.h

@@ -16,6 +16,7 @@
 #include <cfloat>
 #include <unordered_map>
 #include <unordered_set>
+#include <queue>
 
 #include <QtCore/QJsonObject>
 #include <QtCore/QUrl>
@@ -41,6 +42,7 @@ public:
 
     int parseData(ReceivedMessage& message) override;
     AvatarData& getAvatar() { return *_avatar; }
+    const AvatarData* getConstAvatarData() const { return _avatar.get(); }
 
     bool checkAndSetHasReceivedFirstPacketsFrom(const QUuid& uuid);
 
@@ -85,9 +87,9 @@ public:
 
     void loadJSONStats(QJsonObject& jsonObject) const;
 
-    glm::vec3 getPosition() { return _avatar ? _avatar->getPosition() : glm::vec3(0); }
+    glm::vec3 getPosition() const { return _avatar ? _avatar->getPosition() : glm::vec3(0); }
-    glm::vec3 getGlobalBoundingBoxCorner() { return _avatar ? _avatar->getGlobalBoundingBoxCorner() : glm::vec3(0); }
+    glm::vec3 getGlobalBoundingBoxCorner() const { return _avatar ? _avatar->getGlobalBoundingBoxCorner() : glm::vec3(0); }
-    bool isRadiusIgnoring(const QUuid& other) { return _radiusIgnoredOthers.find(other) != _radiusIgnoredOthers.end(); }
+    bool isRadiusIgnoring(const QUuid& other) const { return _radiusIgnoredOthers.find(other) != _radiusIgnoredOthers.end(); }
     void addToRadiusIgnoringSet(const QUuid& other) { _radiusIgnoredOthers.insert(other); }
     void removeFromRadiusIgnoringSet(SharedNodePointer self, const QUuid& other);
     void ignoreOther(SharedNodePointer self, SharedNodePointer other);
@@ -118,9 +120,15 @@ public:
         return _lastOtherAvatarSentJoints[otherAvatar];
     }
 
-    
+    void queuePacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer node);
+    int processPackets(); // returns number of packets processed
 
 private:
+    struct PacketQueue : public std::queue<QSharedPointer<ReceivedMessage>> {
+        QWeakPointer<Node> node;
+    };
+    PacketQueue _packetQueue;
+
     AvatarSharedPointer _avatar { new AvatarData() };
 
     uint16_t _lastReceivedSequenceNumber { 0 };

assignment-client/src/avatars/AvatarMixerSlave.cpp

@@ -0,0 +1,443 @@
+//
+//  AvatarMixerSlave.cpp
+//  assignment-client/src/avatar
+//
+//  Created by Brad Hefta-Gaub on 2/14/2017.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <algorithm>
+#include <random>
+
+#include <glm/glm.hpp>
+#include <glm/gtx/norm.hpp>
+#include <glm/gtx/vector_angle.hpp>
+
+#include <AvatarLogging.h>
+#include <LogHandler.h>
+#include <NetworkAccessManager.h>
+#include <NodeList.h>
+#include <Node.h>
+#include <OctreeConstants.h>
+#include <udt/PacketHeaders.h>
+#include <SharedUtil.h>
+#include <StDev.h>
+#include <UUID.h>
+
+
+#include "AvatarMixer.h"
+#include "AvatarMixerClientData.h"
+#include "AvatarMixerSlave.h"
+
+
+void AvatarMixerSlave::configure(ConstIter begin, ConstIter end) {
+    _begin = begin;
+    _end = end;
+}
+
+void AvatarMixerSlave::configureBroadcast(ConstIter begin, ConstIter end, 
+                                p_high_resolution_clock::time_point lastFrameTimestamp,
+                                float maxKbpsPerNode, float throttlingRatio) {
+    _begin = begin;
+    _end = end;
+    _lastFrameTimestamp = lastFrameTimestamp;
+    _maxKbpsPerNode = maxKbpsPerNode;
+    _throttlingRatio = throttlingRatio;
+}
+
+void AvatarMixerSlave::harvestStats(AvatarMixerSlaveStats& stats) {
+    stats = _stats;
+    _stats.reset();
+}
+
+
+void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) {
+    auto start = usecTimestampNow();
+    auto nodeData = dynamic_cast<AvatarMixerClientData*>(node->getLinkedData());
+    if (nodeData) {
+        _stats.nodesProcessed++;
+        _stats.packetsProcessed += nodeData->processPackets();
+    }
+    auto end = usecTimestampNow();
+    _stats.processIncomingPacketsElapsedTime += (end - start);
+}
+
+
+void AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) {
+    QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray();
+    auto identityPacket = NLPacket::create(PacketType::AvatarIdentity, individualData.size());
+    individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122());
+    identityPacket->write(individualData);
+    DependencyManager::get<NodeList>()->sendPacket(std::move(identityPacket), *destinationNode);
+    _stats.numIdentityPackets++;
+}
+
+static const int AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND = 45;
+
+// only send extra avatar data (avatars out of view, ignored) every Nth AvatarData frame
+// Extra avatar data will be sent (AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND/EXTRA_AVATAR_DATA_FRAME_RATIO) times
+// per second.
+// This value should be a power of two for performance purposes, as the mixer performs a modulo operation every frame
+// to determine whether the extra data should be sent.
+static const int EXTRA_AVATAR_DATA_FRAME_RATIO = 16;
+
+// FIXME - There is some old logic (unchanged as of 2/17/17) that randomly decides to send an identity
+// packet. That logic had the following comment about the constants it uses...
+//
+//         An 80% chance of sending a identity packet within a 5 second interval.
+//         assuming 60 htz update rate.
+//
+// Assuming the calculation of the constant is in fact correct for 80% and 60hz and 5 seconds (an assumption
+// that I have not verified) then the constant is definitely wrong now, since we send at 45hz.
+const float IDENTITY_SEND_PROBABILITY = 1.0f / 187.0f;
+
+void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) {
+    quint64 start = usecTimestampNow();
+
+    auto nodeList = DependencyManager::get<NodeList>();
+
+    // setup for distributed random floating point values
+    std::random_device randomDevice;
+    std::mt19937 generator(randomDevice());
+    std::uniform_real_distribution<float> distribution;
+
+    if (node->getLinkedData() && (node->getType() == NodeType::Agent) && node->getActiveSocket()) {
+        _stats.nodesBroadcastedTo++;
+
+        AvatarMixerClientData* nodeData = reinterpret_cast<AvatarMixerClientData*>(node->getLinkedData());
+
+        nodeData->resetInViewStats();
+
+        const AvatarData& avatar = nodeData->getAvatar();
+        glm::vec3 myPosition = avatar.getClientGlobalPosition();
+
+        // reset the internal state for correct random number distribution
+        distribution.reset();
+
+        // reset the max distance for this frame
+        float maxAvatarDistanceThisFrame = 0.0f;
+
+        // reset the number of sent avatars
+        nodeData->resetNumAvatarsSentLastFrame();
+
+        // keep a counter of the number of considered avatars
+        int numOtherAvatars = 0;
+
+        // keep track of outbound data rate specifically for avatar data
+        int numAvatarDataBytes = 0;
+
+        // keep track of the number of other avatars held back in this frame
+        int numAvatarsHeldBack = 0;
+
+        // keep track of the number of other avatar frames skipped
+        int numAvatarsWithSkippedFrames = 0;
+
+        // use the data rate specifically for avatar data for FRD adjustment checks
+        float avatarDataRateLastSecond = nodeData->getOutboundAvatarDataKbps();
+
+        // When this is true, the AvatarMixer will send Avatar data to a client about avatars that are not in the view frustrum
+        bool getsOutOfView = nodeData->getRequestsDomainListData();
+
+        // When this is true, the AvatarMixer will send Avatar data to a client about avatars that they've ignored
+        bool getsIgnoredByMe = getsOutOfView;
+
+        // When this is true, the AvatarMixer will send Avatar data to a client about avatars that have ignored them
+        bool getsAnyIgnored = getsIgnoredByMe && node->getCanKick();
+
+        // Check if it is time to adjust what we send this client based on the observed
+        // bandwidth to this node. We do this once a second, which is also the window for
+        // the bandwidth reported by node->getOutboundBandwidth();
+        if (nodeData->getNumFramesSinceFRDAdjustment() > AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND) {
+
+            const float FRD_ADJUSTMENT_ACCEPTABLE_RATIO = 0.8f;
+            const float HYSTERISIS_GAP = (1 - FRD_ADJUSTMENT_ACCEPTABLE_RATIO);
+            const float HYSTERISIS_MIDDLE_PERCENTAGE = (1 - (HYSTERISIS_GAP * 0.5f));
+
+            // get the current full rate distance so we can work with it
+            float currentFullRateDistance = nodeData->getFullRateDistance();
+
+            if (avatarDataRateLastSecond > _maxKbpsPerNode) {
+
+                // is the FRD greater than the farthest avatar?
+                // if so, before we calculate anything, set it to that distance
+                currentFullRateDistance = std::min(currentFullRateDistance, nodeData->getMaxAvatarDistance());
+
+                // we're adjusting the full rate distance to target a bandwidth in the middle
+                // of the hysterisis gap
+                currentFullRateDistance *= (_maxKbpsPerNode * HYSTERISIS_MIDDLE_PERCENTAGE) / avatarDataRateLastSecond;
+
+                nodeData->setFullRateDistance(currentFullRateDistance);
+                nodeData->resetNumFramesSinceFRDAdjustment();
+            } else if (currentFullRateDistance < nodeData->getMaxAvatarDistance()
+                && avatarDataRateLastSecond < _maxKbpsPerNode * FRD_ADJUSTMENT_ACCEPTABLE_RATIO) {
+                // we are constrained AND we've recovered to below the acceptable ratio
+                // lets adjust the full rate distance to target a bandwidth in the middle of the hyterisis gap
+                currentFullRateDistance *= (_maxKbpsPerNode * HYSTERISIS_MIDDLE_PERCENTAGE) / avatarDataRateLastSecond;
+
+                nodeData->setFullRateDistance(currentFullRateDistance);
+                nodeData->resetNumFramesSinceFRDAdjustment();
+            }
+        } else {
+            nodeData->incrementNumFramesSinceFRDAdjustment();
+        }
+
+        // setup a PacketList for the avatarPackets
+        auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData);
+
+        // this is an AGENT we have received head data from
+        // send back a packet with other active node data to this node
+        std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) {
+
+            bool shouldConsider = false;
+            quint64 startIgnoreCalculation = usecTimestampNow();
+
+            // make sure we have data for this avatar, that it isn't the same node,
+            // and isn't an avatar that the viewing node has ignored
+            // or that has ignored the viewing node
+            if (!otherNode->getLinkedData()
+                || otherNode->getUUID() == node->getUUID()
+                || (node->isIgnoringNodeWithID(otherNode->getUUID()) && !getsIgnoredByMe)
+                || (otherNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) {
+
+                shouldConsider = false;
+
+            } else {
+                const AvatarMixerClientData* otherData = reinterpret_cast<AvatarMixerClientData*>(otherNode->getLinkedData());
+
+                shouldConsider = true; // assume we will consider...
+
+                // Check to see if the space bubble is enabled
+                if (node->isIgnoreRadiusEnabled() || otherNode->isIgnoreRadiusEnabled()) {
+                    // Define the minimum bubble size
+                    static const glm::vec3 minBubbleSize = glm::vec3(0.3f, 1.3f, 0.3f);
+                    // Define the scale of the box for the current node
+                    glm::vec3 nodeBoxScale = (nodeData->getPosition() - nodeData->getGlobalBoundingBoxCorner()) * 2.0f;
+                    // Define the scale of the box for the current other node
+                    glm::vec3 otherNodeBoxScale = (otherData->getPosition() - otherData->getGlobalBoundingBoxCorner()) * 2.0f;
+
+                    // Set up the bounding box for the current node
+                    AABox nodeBox(nodeData->getGlobalBoundingBoxCorner(), nodeBoxScale);
+                    // Clamp the size of the bounding box to a minimum scale
+                    if (glm::any(glm::lessThan(nodeBoxScale, minBubbleSize))) {
+                        nodeBox.setScaleStayCentered(minBubbleSize);
+                    }
+                    // Set up the bounding box for the current other node
+                    AABox otherNodeBox(otherData->getGlobalBoundingBoxCorner(), otherNodeBoxScale);
+                    // Clamp the size of the bounding box to a minimum scale
+                    if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) {
+                        otherNodeBox.setScaleStayCentered(minBubbleSize);
+                    }
+                    // Quadruple the scale of both bounding boxes
+                    nodeBox.embiggen(4.0f);
+                    otherNodeBox.embiggen(4.0f);
+
+                    // Perform the collision check between the two bounding boxes
+                    if (nodeBox.touches(otherNodeBox)) {
+                        nodeData->ignoreOther(node, otherNode);
+                        shouldConsider = getsAnyIgnored;
+                    }
+                }
+                // Not close enough to ignore
+                if (shouldConsider) {
+                    nodeData->removeFromRadiusIgnoringSet(node, otherNode->getUUID());
+                }
+
+                quint64 endIgnoreCalculation = usecTimestampNow();
+                _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation);
+            }
+
+            if (shouldConsider) {
+                quint64 startAvatarDataPacking = usecTimestampNow();
+
+                ++numOtherAvatars;
+
+                const AvatarMixerClientData* otherNodeData = reinterpret_cast<const AvatarMixerClientData*>(otherNode->getLinkedData());
+
+                // make sure we send out identity packets to and from new arrivals.
+                bool forceSend = !nodeData->checkAndSetHasReceivedFirstPacketsFrom(otherNode->getUUID());
+
+                // FIXME - this clause seems suspicious "... || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp ..."
+                if (otherNodeData->getIdentityChangeTimestamp().time_since_epoch().count() > 0
+                    && (forceSend
+                    || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp
+                    || distribution(generator) < IDENTITY_SEND_PROBABILITY)) {
+
+                    sendIdentityPacket(otherNodeData, node);
+                }
+
+                const AvatarData* otherAvatar = otherNodeData->getConstAvatarData();
+                //  Decide whether to send this avatar's data based on it's distance from us
+
+                //  The full rate distance is the distance at which EVERY update will be sent for this avatar
+                //  at twice the full rate distance, there will be a 50% chance of sending this avatar's update
+                glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition();
+                float distanceToAvatar = glm::length(myPosition - otherPosition);
+
+                // potentially update the max full rate distance for this frame
+                maxAvatarDistanceThisFrame = std::max(maxAvatarDistanceThisFrame, distanceToAvatar);
+
+                // This code handles the random dropping of avatar data based on the ratio of
+                // "getFullRateDistance" to actual distance.
+                //
+                // NOTE: If the recieving node is in "PAL mode" then it's asked to get things even that
+                //       are out of view, this also appears to disable this random distribution.
+                if (distanceToAvatar != 0.0f
+                    && !getsOutOfView
+                    && distribution(generator) > (nodeData->getFullRateDistance() / distanceToAvatar)) {
+
+                    quint64 endAvatarDataPacking = usecTimestampNow();
+                    _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking);
+                    shouldConsider = false;
+                }
+
+                if (shouldConsider) {
+                    AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(otherNode->getUUID());
+                    AvatarDataSequenceNumber lastSeqFromSender = otherNodeData->getLastReceivedSequenceNumber();
+
+                    // FIXME - This code does appear to be working. But it seems brittle.
+                    //         It supports determining if the frame of data for this "other"
+                    //         avatar has already been sent to the reciever. This has been
+                    //         verified to work on a desktop display that renders at 60hz and
+                    //         therefore sends to mixer at 30hz. Each second you'd expect to
+                    //         have 15 (45hz-30hz) duplicate frames. In this case, the stat
+                    //         avg_other_av_skips_per_second does report 15.
+                    //
+                    // make sure we haven't already sent this data from this sender to this receiver
+                    // or that somehow we haven't sent
+                    if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) {
+                        ++numAvatarsHeldBack;
+
+                        quint64 endAvatarDataPacking = usecTimestampNow();
+                        _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking);
+                        shouldConsider = false;
+                    } else if (lastSeqFromSender - lastSeqToReceiver > 1) {
+                        // this is a skip - we still send the packet but capture the presence of the skip so we see it happening
+                        ++numAvatarsWithSkippedFrames;
+                    }
+
+                    // we're going to send this avatar
+                    if (shouldConsider) {
+
+                        // determine if avatar is in view, to determine how much data to include...
+                        glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f;
+                        AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale);
+                        bool isInView = nodeData->otherAvatarInView(otherNodeBox);
+
+                        // this throttles the extra data to only be sent every Nth message
+                        if (!isInView && !getsOutOfView && (lastSeqToReceiver % EXTRA_AVATAR_DATA_FRAME_RATIO > 0)) {
+                            quint64 endAvatarDataPacking = usecTimestampNow();
+
+                            _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking);
+                            shouldConsider = false;
+                        }
+
+                        if (shouldConsider) {
+                            // start a new segment in the PacketList for this avatar
+                            avatarPacketList->startSegment();
+
+                            AvatarData::AvatarDataDetail detail;
+                            if (!isInView && !getsOutOfView) {
+                                detail = AvatarData::MinimumData;
+                                nodeData->incrementAvatarOutOfView();
+                            } else {
+                                detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO
+                                    ? AvatarData::SendAllData : AvatarData::CullSmallData;
+                                nodeData->incrementAvatarInView();
+                            }
+
+                            {
+                                bool includeThisAvatar = true;
+                                auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID());
+                                QVector<JointData>& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID());
+                                bool distanceAdjust = true;
+                                glm::vec3 viewerPosition = myPosition;
+                                AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray
+                                bool dropFaceTracking = false;
+
+                                quint64 start = usecTimestampNow();
+                                QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, 
+                                                                hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther);
+                                quint64 end = usecTimestampNow();
+                                _stats.toByteArrayElapsedTime += (end - start);
+
+                                static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID);
+                                if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) {
+                                    qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data";
+
+                                    dropFaceTracking = true; // first try dropping the facial data
+                                    bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther,
+                                        hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther);
+
+                                    if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) {
+                                        qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData";
+                                        bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther,
+                                            hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther);
+                                    }
+
+                                    if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) {
+                                        qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!";
+                                        includeThisAvatar = false;
+                                    }
+                                }
+
+                                if (includeThisAvatar) {
+                                    numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122());
+                                    numAvatarDataBytes += avatarPacketList->write(bytes);
+                                    _stats.numOthersIncluded++;
+
+                                    // increment the number of avatars sent to this reciever
+                                    nodeData->incrementNumAvatarsSentLastFrame();
+
+                                    // set the last sent sequence number for this sender on the receiver
+                                    nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(),
+                                        otherNodeData->getLastReceivedSequenceNumber());
+
+                                }
+                            }
+
+                            avatarPacketList->endSegment();
+
+                            quint64 endAvatarDataPacking = usecTimestampNow();
+                            _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking);
+                        }
+                    }
+                }
+            }
+        });
+
+        quint64 startPacketSending = usecTimestampNow();
+
+        // close the current packet so that we're always sending something
+        avatarPacketList->closeCurrentPacket(true);
+
+        _stats.numPacketsSent += (int)avatarPacketList->getNumPackets();
+        _stats.numBytesSent += numAvatarDataBytes;
+
+        // send the avatar data PacketList
+        nodeList->sendPacketList(std::move(avatarPacketList), *node);
+
+        // record the bytes sent for other avatar data in the AvatarMixerClientData
+        nodeData->recordSentAvatarData(numAvatarDataBytes);
+
+        // record the number of avatars held back this frame
+        nodeData->recordNumOtherAvatarStarves(numAvatarsHeldBack);
+        nodeData->recordNumOtherAvatarSkips(numAvatarsWithSkippedFrames);
+
+        if (numOtherAvatars == 0) {
+            // update the full rate distance to FLOAT_MAX since we didn't have any other avatars to send
+            nodeData->setMaxAvatarDistance(FLT_MAX);
+        } else {
+            nodeData->setMaxAvatarDistance(maxAvatarDistanceThisFrame);
+        }
+
+        quint64 endPacketSending = usecTimestampNow();
+        _stats.packetSendingElapsedTime += (endPacketSending - startPacketSending);
+    }
+
+    quint64 end = usecTimestampNow();
+    _stats.jobElapsedTime += (end - start);
+}
+

assignment-client/src/avatars/AvatarMixerSlave.h

@@ -0,0 +1,101 @@
+//
+//  AvatarMixerSlave.h
+//  assignment-client/src/avatar
+//
+//  Created by Brad Hefta-Gaub on 2/14/2017.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_AvatarMixerSlave_h
+#define hifi_AvatarMixerSlave_h
+
+class AvatarMixerClientData;
+
+class AvatarMixerSlaveStats {
+public:
+    int nodesProcessed { 0 };
+    int packetsProcessed { 0 };
+    quint64 processIncomingPacketsElapsedTime { 0 };
+
+    int nodesBroadcastedTo { 0 };
+    int numPacketsSent { 0 };
+    int numBytesSent { 0 };
+    int numIdentityPackets { 0 };
+    int numOthersIncluded { 0 };
+    quint64 ignoreCalculationElapsedTime { 0 };
+    quint64 avatarDataPackingElapsedTime { 0 };
+    quint64 packetSendingElapsedTime { 0 };
+    quint64 toByteArrayElapsedTime { 0 };
+    quint64 jobElapsedTime { 0 };
+
+    void reset() {
+        // receiving job stats
+        nodesProcessed = 0;
+        packetsProcessed = 0;
+        processIncomingPacketsElapsedTime = 0;
+
+        // sending job stats
+        nodesBroadcastedTo = 0;
+        numPacketsSent = 0;
+        numBytesSent = 0;
+        numIdentityPackets = 0;
+        numOthersIncluded = 0;
+        ignoreCalculationElapsedTime = 0;
+        avatarDataPackingElapsedTime = 0;
+        packetSendingElapsedTime = 0;
+        toByteArrayElapsedTime = 0;
+        jobElapsedTime = 0;
+    }
+
+    AvatarMixerSlaveStats& operator+=(const AvatarMixerSlaveStats& rhs) {
+        nodesProcessed += rhs.nodesProcessed;
+        packetsProcessed += rhs.packetsProcessed;
+        processIncomingPacketsElapsedTime += rhs.processIncomingPacketsElapsedTime;
+
+        nodesBroadcastedTo += rhs.nodesBroadcastedTo;
+        numPacketsSent += rhs.numPacketsSent;
+        numBytesSent += rhs.numBytesSent;
+        numIdentityPackets += rhs.numIdentityPackets;
+        numOthersIncluded += rhs.numOthersIncluded;
+        ignoreCalculationElapsedTime += rhs.ignoreCalculationElapsedTime;
+        avatarDataPackingElapsedTime += rhs.avatarDataPackingElapsedTime;
+        packetSendingElapsedTime += rhs.packetSendingElapsedTime;
+        toByteArrayElapsedTime += rhs.toByteArrayElapsedTime;
+        jobElapsedTime += rhs.jobElapsedTime;
+        return *this;
+    }
+
+};
+
+class AvatarMixerSlave {
+public:
+    using ConstIter = NodeList::const_iterator;
+
+    void configure(ConstIter begin, ConstIter end);
+    void configureBroadcast(ConstIter begin, ConstIter end, 
+                    p_high_resolution_clock::time_point lastFrameTimestamp, 
+                    float maxKbpsPerNode, float throttlingRatio);
+
+    void processIncomingPackets(const SharedNodePointer& node);
+    void broadcastAvatarData(const SharedNodePointer& node);
+
+    void harvestStats(AvatarMixerSlaveStats& stats);
+
+private:
+    void sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode);
+
+    // frame state
+    ConstIter _begin;
+    ConstIter _end;
+
+    p_high_resolution_clock::time_point _lastFrameTimestamp;
+    float _maxKbpsPerNode { 0.0f };
+    float _throttlingRatio { 0.0f };
+
+    AvatarMixerSlaveStats _stats;
+};
+
+#endif // hifi_AvatarMixerSlave_h

assignment-client/src/avatars/AvatarMixerSlavePool.cpp

@@ -0,0 +1,208 @@
+//
+//  AvatarMixerSlavePool.cpp
+//  assignment-client/src/avatar
+//
+//  Created by Brad Hefta-Gaub on 2/14/2017.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <assert.h>
+#include <algorithm>
+
+#include "AvatarMixerSlavePool.h"
+
+void AvatarMixerSlaveThread::run() {
+    while (true) {
+        wait();
+
+        // iterate over all available nodes
+        SharedNodePointer node;
+        while (try_pop(node)) {
+            (this->*_function)(node);
+        }
+
+        bool stopping = _stop;
+        notify(stopping);
+        if (stopping) {
+            return;
+        }
+    }
+}
+
+void AvatarMixerSlaveThread::wait() {
+    {
+        Lock lock(_pool._mutex);
+        _pool._slaveCondition.wait(lock, [&] {
+            assert(_pool._numStarted <= _pool._numThreads);
+            return _pool._numStarted != _pool._numThreads;
+        });
+        ++_pool._numStarted;
+    }
+    if (_pool._configure) {
+        _pool._configure(*this);
+    }
+    _function = _pool._function;
+}
+
+void AvatarMixerSlaveThread::notify(bool stopping) {
+    {
+        Lock lock(_pool._mutex);
+        assert(_pool._numFinished < _pool._numThreads);
+        ++_pool._numFinished;
+        if (stopping) {
+            ++_pool._numStopped;
+        }
+    }
+    _pool._poolCondition.notify_one();
+}
+
+bool AvatarMixerSlaveThread::try_pop(SharedNodePointer& node) {
+    return _pool._queue.try_pop(node);
+}
+
+#ifdef AVATAR_SINGLE_THREADED
+static AvatarMixerSlave slave;
+#endif
+
+void AvatarMixerSlavePool::processIncomingPackets(ConstIter begin, ConstIter end) {
+    _function = &AvatarMixerSlave::processIncomingPackets;
+    _configure = [&](AvatarMixerSlave& slave) { 
+        slave.configure(begin, end);
+    };
+    run(begin, end);
+}
+
+void AvatarMixerSlavePool::broadcastAvatarData(ConstIter begin, ConstIter end, 
+                                     p_high_resolution_clock::time_point lastFrameTimestamp, 
+                                     float maxKbpsPerNode, float throttlingRatio) {
+    _function = &AvatarMixerSlave::broadcastAvatarData;
+    _configure = [&](AvatarMixerSlave& slave) { 
+        slave.configureBroadcast(begin, end, lastFrameTimestamp, maxKbpsPerNode, throttlingRatio);
+   };
+    run(begin, end);
+}
+
+void AvatarMixerSlavePool::run(ConstIter begin, ConstIter end) {
+    _begin = begin;
+    _end = end;
+
+#ifdef AUDIO_SINGLE_THREADED
+    _configure(slave);
+    std::for_each(begin, end, [&](const SharedNodePointer& node) {
+        _function(slave, node);
+});
+#else
+    // fill the queue
+    std::for_each(_begin, _end, [&](const SharedNodePointer& node) {
+        _queue.emplace(node);
+    });
+
+    {
+        Lock lock(_mutex);
+
+        // run
+        _numStarted = _numFinished = 0;
+        _slaveCondition.notify_all();
+
+        // wait
+        _poolCondition.wait(lock, [&] {
+            assert(_numFinished <= _numThreads);
+            return _numFinished == _numThreads;
+        });
+
+        assert(_numStarted == _numThreads);
+    }
+
+    assert(_queue.empty());
+#endif
+}
+
+
+void AvatarMixerSlavePool::each(std::function<void(AvatarMixerSlave& slave)> functor) {
+#ifdef AVATAR_SINGLE_THREADED
+    functor(slave);
+#else
+    for (auto& slave : _slaves) {
+        functor(*slave.get());
+    }
+#endif
+}
+
+void AvatarMixerSlavePool::setNumThreads(int numThreads) {
+    // clamp to allowed size
+    {
+        int maxThreads = QThread::idealThreadCount();
+        if (maxThreads == -1) {
+            // idealThreadCount returns -1 if cores cannot be detected
+            static const int MAX_THREADS_IF_UNKNOWN = 4;
+            maxThreads = MAX_THREADS_IF_UNKNOWN;
+        }
+
+        int clampedThreads = std::min(std::max(1, numThreads), maxThreads);
+        if (clampedThreads != numThreads) {
+            qWarning("%s: clamped to %d (was %d)", __FUNCTION__, clampedThreads, numThreads);
+            numThreads = clampedThreads;
+        }
+    }
+
+    resize(numThreads);
+}
+
+void AvatarMixerSlavePool::resize(int numThreads) {
+    assert(_numThreads == (int)_slaves.size());
+
+#ifdef AVATAR_SINGLE_THREADED
+    qDebug("%s: running single threaded", __FUNCTION__, numThreads);
+#else
+    qDebug("%s: set %d threads (was %d)", __FUNCTION__, numThreads, _numThreads);
+
+    Lock lock(_mutex);
+
+    if (numThreads > _numThreads) {
+        // start new slaves
+        for (int i = 0; i < numThreads - _numThreads; ++i) {
+            auto slave = new AvatarMixerSlaveThread(*this);
+            slave->start();
+            _slaves.emplace_back(slave);
+        }
+    } else if (numThreads < _numThreads) {
+        auto extraBegin = _slaves.begin() + numThreads;
+
+        // mark slaves to stop...
+        auto slave = extraBegin;
+        while (slave != _slaves.end()) {
+            (*slave)->_stop = true;
+            ++slave;
+        }
+
+        // ...cycle them until they do stop...
+        _numStopped = 0;
+        while (_numStopped != (_numThreads - numThreads)) {
+            _numStarted = _numFinished = _numStopped;
+            _slaveCondition.notify_all();
+            _poolCondition.wait(lock, [&] {
+                assert(_numFinished <= _numThreads);
+                return _numFinished == _numThreads;
+            });
+        }
+
+        // ...wait for threads to finish...
+        slave = extraBegin;
+        while (slave != _slaves.end()) {
+            QThread* thread = reinterpret_cast<QThread*>(slave->get());
+            static const int MAX_THREAD_WAIT_TIME = 10;
+            thread->wait(MAX_THREAD_WAIT_TIME);
+            ++slave;
+        }
+
+        // ...and erase them
+        _slaves.erase(extraBegin, _slaves.end());
+    }
+
+    _numThreads = _numStarted = _numFinished = numThreads;
+    assert(_numThreads == (int)_slaves.size());
+#endif
+}

assignment-client/src/avatars/AvatarMixerSlavePool.h

@@ -0,0 +1,104 @@
+//
+//  AvatarMixerSlavePool.h
+//  assignment-client/src/avatar
+//
+//  Created by Brad Hefta-Gaub on 2/14/2017.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_AvatarMixerSlavePool_h
+#define hifi_AvatarMixerSlavePool_h
+
+#include <condition_variable>
+#include <mutex>
+#include <vector>
+
+#include <tbb/concurrent_queue.h>
+
+#include <QThread>
+
+#include <NodeList.h>
+
+#include "AvatarMixerSlave.h"
+
+class AvatarMixerSlavePool;
+
+class AvatarMixerSlaveThread : public QThread, public AvatarMixerSlave {
+    Q_OBJECT
+    using ConstIter = NodeList::const_iterator;
+    using Mutex = std::mutex;
+    using Lock = std::unique_lock<Mutex>;
+
+public:
+    AvatarMixerSlaveThread(AvatarMixerSlavePool& pool) : _pool(pool) {}
+
+    void run() override final;
+
+private:
+    friend class AvatarMixerSlavePool;
+
+    void wait();
+    void notify(bool stopping);
+    bool try_pop(SharedNodePointer& node);
+
+    AvatarMixerSlavePool& _pool;
+    void (AvatarMixerSlave::*_function)(const SharedNodePointer& node) { nullptr };
+    bool _stop { false };
+};
+
+// Slave pool for audio mixers
+//   AvatarMixerSlavePool is not thread-safe! It should be instantiated and used from a single thread.
+class AvatarMixerSlavePool {
+    using Queue = tbb::concurrent_queue<SharedNodePointer>;
+    using Mutex = std::mutex;
+    using Lock = std::unique_lock<Mutex>;
+    using ConditionVariable = std::condition_variable;
+
+public:
+    using ConstIter = NodeList::const_iterator;
+
+    AvatarMixerSlavePool(int numThreads = QThread::idealThreadCount()) { setNumThreads(numThreads); }
+    ~AvatarMixerSlavePool() { resize(0); }
+
+    // Jobs the slave pool can do...
+    void processIncomingPackets(ConstIter begin, ConstIter end);
+    void broadcastAvatarData(ConstIter begin, ConstIter end, 
+                    p_high_resolution_clock::time_point lastFrameTimestamp, float maxKbpsPerNode, float throttlingRatio);
+
+    // iterate over all slaves
+    void each(std::function<void(AvatarMixerSlave& slave)> functor);
+
+    void setNumThreads(int numThreads);
+    int numThreads() { return _numThreads; }
+
+private:
+    void run(ConstIter begin, ConstIter end);
+    void resize(int numThreads);
+
+    std::vector<std::unique_ptr<AvatarMixerSlaveThread>> _slaves;
+
+    friend void AvatarMixerSlaveThread::wait();
+    friend void AvatarMixerSlaveThread::notify(bool stopping);
+    friend bool AvatarMixerSlaveThread::try_pop(SharedNodePointer& node);
+
+    // synchronization state
+    Mutex _mutex;
+    ConditionVariable _slaveCondition;
+    ConditionVariable _poolCondition;
+    void (AvatarMixerSlave::*_function)(const SharedNodePointer& node);
+    std::function<void(AvatarMixerSlave&)> _configure;
+    int _numThreads { 0 };
+    int _numStarted { 0 }; // guarded by _mutex
+    int _numFinished { 0 }; // guarded by _mutex
+    int _numStopped { 0 }; // guarded by _mutex
+
+    // frame state
+    Queue _queue;
+    ConstIter _begin;
+    ConstIter _end;
+};
+
+#endif // hifi_AvatarMixerSlavePool_h

assignment-client/src/avatars/ScriptableAvatar.cpp

@@ -14,10 +14,9 @@
 #include <GLMHelpers.h>
 #include "ScriptableAvatar.h"
 
-QByteArray ScriptableAvatar::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
+QByteArray ScriptableAvatar::toByteArrayStateful(AvatarDataDetail dataDetail) {
-                                    bool distanceAdjust, glm::vec3 viewerPosition, QVector<JointData>* sentJointDataOut) {
     _globalPosition = getPosition();
-    return AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut);
+    return AvatarData::toByteArrayStateful(dataDetail);
 }
 
 

assignment-client/src/avatars/ScriptableAvatar.h

@@ -28,8 +28,7 @@ public:
     Q_INVOKABLE AnimationDetails getAnimationDetails();
     virtual void setSkeletonModelURL(const QUrl& skeletonModelURL) override;
 
-    virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
+    virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail) override;
-                        bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector<JointData>* sentJointDataOut = nullptr) override;
 
     
 private slots:

assignment-client/src/entities/EntityServer.cpp

@@ -17,10 +17,11 @@
 #include <ScriptCache.h>
 #include <EntityEditFilters.h>
 
+#include "AssignmentParentFinder.h"
+#include "EntityNodeData.h"
 #include "EntityServer.h"
 #include "EntityServerConsts.h"
-#include "EntityNodeData.h"
+#include "EntityTreeSendThread.h"
-#include "AssignmentParentFinder.h"
 
 const char* MODEL_SERVER_NAME = "Entity";
 const char* MODEL_SERVER_LOGGING_TARGET_NAME = "entity-server";
@@ -77,6 +78,10 @@ OctreePointer EntityServer::createTree() {
     return tree;
 }
 
+OctreeServer::UniqueSendThread EntityServer::newSendThread(const SharedNodePointer& node) {
+    return std::unique_ptr<EntityTreeSendThread>(new EntityTreeSendThread(this, node));
+}
+
 void EntityServer::beforeRun() {
     _pruneDeletedEntitiesTimer = new QTimer();
     connect(_pruneDeletedEntitiesTimer, SIGNAL(timeout()), this, SLOT(pruneDeletedEntities()));

assignment-client/src/entities/EntityServer.h

@@ -67,6 +67,7 @@ public slots:
 
 protected:
     virtual OctreePointer createTree() override;
+    virtual UniqueSendThread newSendThread(const SharedNodePointer& node) override;
 
 private slots:
     void handleEntityPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);

assignment-client/src/entities/EntityTreeSendThread.cpp

@@ -0,0 +1,132 @@
+//
+//  EntityTreeSendThread.cpp
+//  assignment-client/src/entities
+//
+//  Created by Stephen Birarda on 2/15/17.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "EntityTreeSendThread.h"
+
+#include <EntityNodeData.h>
+#include <EntityTypes.h>
+
+#include "EntityServer.h"
+
+void EntityTreeSendThread::preDistributionProcessing() {
+    auto node = _node.toStrongRef();
+    auto nodeData = static_cast<EntityNodeData*>(node->getLinkedData());
+
+    if (nodeData) {
+        auto jsonQuery = nodeData->getJSONParameters();
+
+        // check if we have a JSON query with flags
+        auto flags = jsonQuery[EntityJSONQueryProperties::FLAGS_PROPERTY].toObject();
+        if (!flags.isEmpty()) {
+            // check the flags object for specific flags that require special pre-processing
+
+            bool includeAncestors = flags[EntityJSONQueryProperties::INCLUDE_ANCESTORS_PROPERTY].toBool();
+            bool includeDescendants = flags[EntityJSONQueryProperties::INCLUDE_DESCENDANTS_PROPERTY].toBool();
+
+            if (includeAncestors || includeDescendants) {
+                // we need to either include the ancestors, descendants, or both for entities matching the filter
+                // included in the JSON query
+
+                // first reset our flagged extra entities so we start with an empty set
+                nodeData->resetFlaggedExtraEntities();
+
+                auto entityTree = std::static_pointer_cast<EntityTree>(_myServer->getOctree());
+
+                bool requiresFullScene = false;
+
+                // enumerate the set of entity IDs we know currently match the filter
+                foreach(const QUuid& entityID, nodeData->getSentFilteredEntities()) {
+                    if (includeAncestors) {
+                        // we need to include ancestors - recurse up to reach them all and add their IDs
+                        // to the set of extra entities to include for this node
+                        entityTree->withReadLock([&]{
+                            auto filteredEntity = entityTree->findEntityByID(entityID);
+                            if (filteredEntity) {
+                                requiresFullScene |= addAncestorsToExtraFlaggedEntities(entityID, *filteredEntity, *nodeData);
+                            }
+                        });
+                    }
+
+                    if (includeDescendants) {
+                        // we need to include descendants - recurse down to reach them all and add their IDs
+                        // to the set of extra entities to include for this node
+                        entityTree->withReadLock([&]{
+                            auto filteredEntity = entityTree->findEntityByID(entityID);
+                            if (filteredEntity) {
+                                requiresFullScene |= addDescendantsToExtraFlaggedEntities(entityID, *filteredEntity, *nodeData);
+                            }
+                        });
+                    }
+                }
+
+                if (requiresFullScene) {
+                    // for one or more of the entities matching our filter we found new extra entities to include
+
+                    // because it is possible that one of these entities hasn't changed since our last send
+                    // and therefore would not be recursed to, we need to force a full traversal for this pass
+                    // of the tree to allow it to grab all of the extra entities we're asking it to include
+                    nodeData->setShouldForceFullScene(requiresFullScene);
+                }
+            }
+        }
+    }
+}
+
+bool EntityTreeSendThread::addAncestorsToExtraFlaggedEntities(const QUuid& filteredEntityID,
+                                                              EntityItem& entityItem, EntityNodeData& nodeData) {
+    // check if this entity has a parent that is also an entity
+    bool success = false;
+    auto entityParent = entityItem.getParentPointer(success);
+
+    if (success && entityParent && entityParent->getNestableType() == NestableType::Entity) {
+        // we found a parent that is an entity item
+
+        // first add it to the extra list of things we need to send
+        bool parentWasNew = nodeData.insertFlaggedExtraEntity(filteredEntityID, entityParent->getID());
+
+        // now recursively call ourselves to get its ancestors added too
+        auto parentEntityItem = std::static_pointer_cast<EntityItem>(entityParent);
+        bool ancestorsWereNew = addAncestorsToExtraFlaggedEntities(filteredEntityID, *parentEntityItem, nodeData);
+
+        // return boolean if our parent or any of our ancestors were new additions (via insertFlaggedExtraEntity)
+        return parentWasNew || ancestorsWereNew;
+    }
+
+    // since we didn't have a parent niether of our parents or ancestors could be new additions
+    return false;
+}
+
+bool EntityTreeSendThread::addDescendantsToExtraFlaggedEntities(const QUuid& filteredEntityID,
+                                                                EntityItem& entityItem, EntityNodeData& nodeData) {
+    bool hasNewChild = false;
+    bool hasNewDescendants = false;
+
+    // enumerate the immediate children of this entity
+    foreach (SpatiallyNestablePointer child, entityItem.getChildren()) {
+
+        if (child && child->getNestableType() == NestableType::Entity) {
+            // this is a child that is an entity
+
+            // first add it to the extra list of things we need to send
+            hasNewChild |= nodeData.insertFlaggedExtraEntity(filteredEntityID, child->getID());
+
+            // now recursively call ourselves to get its descendants added too
+            auto childEntityItem = std::static_pointer_cast<EntityItem>(child);
+            hasNewDescendants |= addDescendantsToExtraFlaggedEntities(filteredEntityID, *childEntityItem, nodeData);
+        }
+    }
+
+    // return our boolean indicating if we added new children or descendants as extra entities to send
+    // (via insertFlaggedExtraEntity)
+    return hasNewChild || hasNewDescendants;
+}
+
+

assignment-client/src/entities/EntityTreeSendThread.h

@@ -0,0 +1,35 @@
+//
+//  EntityTreeSendThread.h
+//  assignment-client/src/entities
+//
+//  Created by Stephen Birarda on 2/15/17.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_EntityTreeSendThread_h
+#define hifi_EntityTreeSendThread_h
+
+#include "../octree/OctreeSendThread.h"
+
+class EntityNodeData;
+class EntityItem;
+
+class EntityTreeSendThread : public OctreeSendThread {
+
+public:
+    EntityTreeSendThread(OctreeServer* myServer, const SharedNodePointer& node) : OctreeSendThread(myServer, node) {};
+
+protected:
+    virtual void preDistributionProcessing() override;
+
+private:
+    // the following two methods return booleans to indicate if any extra flagged entities were new additions to set
+    bool addAncestorsToExtraFlaggedEntities(const QUuid& filteredEntityID, EntityItem& entityItem, EntityNodeData& nodeData);
+    bool addDescendantsToExtraFlaggedEntities(const QUuid& filteredEntityID, EntityItem& entityItem, EntityNodeData& nodeData);
+
+};
+
+#endif // hifi_EntityTreeSendThread_h

assignment-client/src/octree/OctreeSendThread.cpp

@@ -309,6 +309,12 @@ int OctreeSendThread::packetDistributor(SharedNodePointer node, OctreeQueryNode*
         return 0;
     }
 
+    if (nodeData->elementBag.isEmpty()) {
+        // if we're about to do a fresh pass,
+        // give our pre-distribution processing a chance to do what it needs
+        preDistributionProcessing();
+    }
+
     // calculate max number of packets that can be sent during this interval
     int clientMaxPacketsPerInterval = std::max(1, (nodeData->getMaxQueryPacketsPerSecond() / INTERVALS_PER_SECOND));
     int maxPacketsPerInterval = std::min(clientMaxPacketsPerInterval, _myServer->getPacketsPerClientPerInterval());
@@ -316,9 +322,17 @@ int OctreeSendThread::packetDistributor(SharedNodePointer node, OctreeQueryNode*
     int truePacketsSent = 0;
     int trueBytesSent = 0;
     int packetsSentThisInterval = 0;
-    bool isFullScene = nodeData->haveJSONParametersChanged() ||
+
-        (nodeData->getUsesFrustum()
+    bool isFullScene = nodeData->shouldForceFullScene();
-         && ((!viewFrustumChanged && nodeData->getViewFrustumJustStoppedChanging()) || nodeData->hasLodChanged()));
+    if (isFullScene) {
+        // we're forcing a full scene, clear the force in OctreeQueryNode so we don't force it next time again
+        nodeData->setShouldForceFullScene(false);
+    } else {
+        // we aren't forcing a full scene, check if something else suggests we should
+        isFullScene = nodeData->haveJSONParametersChanged() ||
+            (nodeData->getUsesFrustum()
+             && ((!viewFrustumChanged && nodeData->getViewFrustumJustStoppedChanging()) || nodeData->hasLodChanged()));
+    }
 
     bool somethingToSend = true; // assume we have something
 

assignment-client/src/octree/OctreeSendThread.h

@@ -17,6 +17,8 @@
 #include <atomic>
 
 #include <GenericThread.h>
+#include <Node.h>
+#include <OctreePacketData.h>
 
 class OctreeQueryNode;
 class OctreeServer;
@@ -49,13 +51,17 @@ protected:
     /// Implements generic processing behavior for this thread.
     virtual bool process() override;
 
+    /// Called before a packetDistributor pass to allow for pre-distribution processing
+    virtual void preDistributionProcessing() {};
+
+    OctreeServer* _myServer { nullptr };
+    QWeakPointer<Node> _node;
+
 private:
     int handlePacketSend(SharedNodePointer node, OctreeQueryNode* nodeData, int& trueBytesSent, int& truePacketsSent, bool dontSuppressDuplicate = false);
     int packetDistributor(SharedNodePointer node, OctreeQueryNode* nodeData, bool viewFrustumChanged);
     
-    
+
-    OctreeServer* _myServer { nullptr };
-    QWeakPointer<Node> _node;
     QUuid _nodeUuid;
 
     OctreePacketData _packetData;

assignment-client/src/octree/OctreeServer.cpp

@@ -872,8 +872,12 @@ void OctreeServer::parsePayload() {
     }
 }
 
+OctreeServer::UniqueSendThread OctreeServer::newSendThread(const SharedNodePointer& node) {
+    return std::unique_ptr<OctreeSendThread>(new OctreeSendThread(this, node));
+}
+
 OctreeServer::UniqueSendThread OctreeServer::createSendThread(const SharedNodePointer& node) {
-    auto sendThread = std::unique_ptr<OctreeSendThread>(new OctreeSendThread(this, node));
+    auto sendThread = newSendThread(node);
     
     // we want to be notified when the thread finishes
     connect(sendThread.get(), &GenericThread::finished, this, &OctreeServer::removeSendThread);

assignment-client/src/octree/OctreeServer.h

@@ -158,6 +158,7 @@ protected:
     QString getStatusLink();
     
     UniqueSendThread createSendThread(const SharedNodePointer& node);
+    virtual UniqueSendThread newSendThread(const SharedNodePointer& node);
 
     int _argc;
     const char** _argv;

assignment-client/src/scripts/EntityScriptServer.cpp

@@ -16,6 +16,7 @@
 #include <AudioConstants.h>
 #include <AudioInjectorManager.h>
 #include <ClientServerUtils.h>
+#include <EntityNodeData.h>
 #include <EntityScriptingInterface.h>
 #include <LogHandler.h>
 #include <MessagesClient.h>
@@ -264,8 +265,14 @@ void EntityScriptServer::run() {
     
     // setup the JSON filter that asks for entities with a non-default serverScripts property
     QJsonObject queryJSONParameters;
-    static const QString SERVER_SCRIPTS_PROPERTY = "serverScripts";
+    queryJSONParameters[EntityJSONQueryProperties::SERVER_SCRIPTS_PROPERTY] = EntityQueryFilterSymbol::NonDefault;
-    queryJSONParameters[SERVER_SCRIPTS_PROPERTY] = EntityQueryFilterSymbol::NonDefault;
+
+    QJsonObject queryFlags;
+
+    queryFlags[EntityJSONQueryProperties::INCLUDE_ANCESTORS_PROPERTY] = true;
+    queryFlags[EntityJSONQueryProperties::INCLUDE_DESCENDANTS_PROPERTY] = true;
+
+    queryJSONParameters[EntityJSONQueryProperties::FLAGS_PROPERTY] = queryFlags;
     
     // setup the JSON parameters so that OctreeQuery does not use a frustum and uses our JSON filter
     _entityViewer.getOctreeQuery().setUsesFrustum(false);
@@ -412,6 +419,7 @@ void EntityScriptServer::resetEntitiesScriptEngine() {
 
     connect(newEngine.data(), &ScriptEngine::update, this, [this] {
         _entityViewer.queryOctree();
+        _entityViewer.getTree()->update();
     });
 
 

domain-server/resources/describe-settings.json

@@ -1299,6 +1299,22 @@
           "placeholder": 5.0,
           "default": 5.0,
           "advanced": true
+        },
+        {
+          "name": "auto_threads",
+          "label": "Automatically determine thread count",
+          "type": "checkbox",
+          "help": "Allow system to determine number of threads (recommended)",
+          "default": false,
+          "advanced": true
+        },
+        {
+          "name": "num_threads",
+          "label": "Number of Threads",
+          "help": "Threads to spin up for avatar mixing (if not automatically set)",
+          "placeholder": "1",
+          "default": "1",
+          "advanced": true
         }
       ]
     }

interface/resources/qml/hifi/toolbars/Toolbar.qml

@@ -120,6 +120,8 @@ Window {
     function addButton(properties) {
         properties = properties || {}
 
+        unpinnedAlpha = 1;
+
         // If a name is specified, then check if there's an existing button with that name
         // and return it if so.  This will allow multiple clients to listen to a single button,
         // and allow scripts to be idempotent so they don't duplicate buttons if they're reloaded
@@ -154,7 +156,7 @@ Window {
         updatePinned();
 
         if (buttons.length === 0) {
-            visible = false;
+            unpinnedAlpha = 0;
         }
     }
 

interface/src/Application.cpp

@@ -170,6 +170,7 @@
 #include "ui/StandAloneJSConsole.h"
 #include "ui/Stats.h"
 #include "ui/UpdateDialog.h"
+#include "ui/overlays/Overlays.h"
 #include "Util.h"
 #include "InterfaceParentFinder.h"
 
@@ -528,7 +529,7 @@ bool setupEssentials(int& argc, char** argv) {
 // to take care of highlighting keyboard focused items, rather than
 // continuing to overburden Application.cpp
 std::shared_ptr<Cube3DOverlay> _keyboardFocusHighlight{ nullptr };
-int _keyboardFocusHighlightID{ -1 };
+OverlayID _keyboardFocusHighlightID{ UNKNOWN_OVERLAY_ID };
 
 
 // FIXME hack access to the internal share context for the Chromium helper
@@ -1232,12 +1233,12 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
     // Keyboard focus handling for Web overlays.
     auto overlays = &(qApp->getOverlays());
 
-    connect(overlays, &Overlays::mousePressOnOverlay, [=](unsigned int overlayID, const PointerEvent& event) {
+    connect(overlays, &Overlays::mousePressOnOverlay, [=](OverlayID overlayID, const PointerEvent& event) {
         setKeyboardFocusEntity(UNKNOWN_ENTITY_ID);
         setKeyboardFocusOverlay(overlayID);
     });
 
-    connect(overlays, &Overlays::overlayDeleted, [=](unsigned int overlayID) {
+    connect(overlays, &Overlays::overlayDeleted, [=](OverlayID overlayID) {
         if (overlayID == _keyboardFocusedOverlay.get()) {
             setKeyboardFocusOverlay(UNKNOWN_OVERLAY_ID);
         }
@@ -1689,9 +1690,9 @@ void Application::cleanupBeforeQuit() {
     _applicationStateDevice.reset();
 
     {
-        if (_keyboardFocusHighlightID > 0) {
+        if (_keyboardFocusHighlightID != UNKNOWN_OVERLAY_ID) {
             getOverlays().deleteOverlay(_keyboardFocusHighlightID);
-            _keyboardFocusHighlightID = -1;
+            _keyboardFocusHighlightID = UNKNOWN_OVERLAY_ID;
         }
         _keyboardFocusHighlight = nullptr;
     }
@@ -3088,7 +3089,7 @@ void Application::mouseMoveEvent(QMouseEvent* event) {
         buttons, event->modifiers());
 
     if (compositor.getReticleVisible() || !isHMDMode() || !compositor.getReticleOverDesktop() ||
-        getOverlays().getOverlayAtPoint(glm::vec2(transformedPos.x(), transformedPos.y()))) {
+        getOverlays().getOverlayAtPoint(glm::vec2(transformedPos.x(), transformedPos.y())) != UNKNOWN_OVERLAY_ID) {
         getOverlays().mouseMoveEvent(&mappedEvent);
         getEntities()->mouseMoveEvent(&mappedEvent);
     }
@@ -4107,7 +4108,7 @@ void Application::rotationModeChanged() const {
 
 void Application::setKeyboardFocusHighlight(const glm::vec3& position, const glm::quat& rotation, const glm::vec3& dimensions) {
     // Create focus
-    if (_keyboardFocusHighlightID < 0 || !getOverlays().isAddedOverlay(_keyboardFocusHighlightID)) {
+    if (_keyboardFocusHighlightID == UNKNOWN_OVERLAY_ID || !getOverlays().isAddedOverlay(_keyboardFocusHighlightID)) {
         _keyboardFocusHighlight = std::make_shared<Cube3DOverlay>();
         _keyboardFocusHighlight->setAlpha(1.0f);
         _keyboardFocusHighlight->setBorderSize(1.0f);
@@ -4169,11 +4170,11 @@ void Application::setKeyboardFocusEntity(EntityItemID entityItemID) {
     }
 }
 
-unsigned int Application::getKeyboardFocusOverlay() {
+OverlayID Application::getKeyboardFocusOverlay() {
     return _keyboardFocusedOverlay.get();
 }
 
-void Application::setKeyboardFocusOverlay(unsigned int overlayID) {
+void Application::setKeyboardFocusOverlay(OverlayID overlayID) {
     if (overlayID != _keyboardFocusedOverlay.get()) {
         _keyboardFocusedOverlay.set(overlayID);
 
@@ -5546,6 +5547,8 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEngine* scri
     auto entityScriptServerLog = DependencyManager::get<EntityScriptServerLogClient>();
     scriptEngine->registerGlobalObject("EntityScriptServerLog", entityScriptServerLog.data());
 
+    qScriptRegisterMetaType(scriptEngine, OverlayIDtoScriptValue, OverlayIDfromScriptValue);
+
     // connect this script engines printedMessage signal to the global ScriptEngines these various messages
     connect(scriptEngine, &ScriptEngine::printedMessage, DependencyManager::get<ScriptEngines>().data(), &ScriptEngines::onPrintedMessage);
     connect(scriptEngine, &ScriptEngine::errorMessage, DependencyManager::get<ScriptEngines>().data(), &ScriptEngines::onErrorMessage);
@@ -6891,3 +6894,13 @@ void Application::toggleMuteAudio() {
     auto menu = Menu::getInstance();
     menu->setIsOptionChecked(MenuOption::MuteAudio, !menu->isOptionChecked(MenuOption::MuteAudio));
 }
+
+OverlayID Application::getTabletScreenID() const {
+    auto HMD = DependencyManager::get<HMDScriptingInterface>();
+    return HMD->getCurrentTabletScreenID();
+}
+
+OverlayID Application::getTabletHomeButtonID() const {
+    auto HMD = DependencyManager::get<HMDScriptingInterface>();
+    return HMD->getCurrentHomeButtonUUID();
+}

interface/src/Application.h

@@ -298,6 +298,9 @@ public:
     Q_INVOKABLE void sendHoverOverEntity(QUuid id, PointerEvent event);
     Q_INVOKABLE void sendHoverLeaveEntity(QUuid id, PointerEvent event);
 
+    OverlayID getTabletScreenID() const;
+    OverlayID getTabletHomeButtonID() const;
+
 signals:
     void svoImportRequested(const QString& url);
 
@@ -387,8 +390,8 @@ public slots:
     void setKeyboardFocusEntity(QUuid id);
     void setKeyboardFocusEntity(EntityItemID entityItemID);
 
-    unsigned int getKeyboardFocusOverlay();
+    OverlayID getKeyboardFocusOverlay();
-    void setKeyboardFocusOverlay(unsigned int overlayID);
+    void setKeyboardFocusOverlay(OverlayID overlayID);
 
     void addAssetToWorldMessageClose();
 
@@ -618,7 +621,7 @@ private:
     DialogsManagerScriptingInterface* _dialogsManagerScriptingInterface = new DialogsManagerScriptingInterface();
 
     ThreadSafeValueCache<EntityItemID> _keyboardFocusedEntity;
-    ThreadSafeValueCache<unsigned int> _keyboardFocusedOverlay;
+    ThreadSafeValueCache<OverlayID> _keyboardFocusedOverlay;
     quint64 _lastAcceptedKeyPress = 0;
     bool _isForeground = true; // starts out assumed to be in foreground
     bool _inPaint = false;

interface/src/InterfaceParentFinder.cpp

@@ -45,12 +45,20 @@ SpatiallyNestableWeakPointer InterfaceParentFinder::find(QUuid parentID, bool& s
         success = true;
         return parent;
     }
-
     if (parentID == AVATAR_SELF_ID) {
         success = true;
         return avatarManager->getMyAvatar();
     }
 
+    // search overlays
+    auto& overlays = qApp->getOverlays();
+    auto overlay = overlays.getOverlay(parentID);
+    parent = std::dynamic_pointer_cast<SpatiallyNestable>(overlay); // this will return nullptr for non-3d overlays
+    if (!parent.expired()) {
+        success = true;
+        return parent;
+    }
+
     success = false;
     return parent;
 }

interface/src/avatar/AvatarManager.cpp

@@ -154,8 +154,7 @@ public:
     AvatarPriority(AvatarSharedPointer a, float p) : avatar(a), priority(p) {}
     AvatarSharedPointer avatar;
     float priority;
-    // NOTE: we invert the less-than operator to sort high priorities to front
+    bool operator<(const AvatarPriority& other) const { return priority < other.priority; }
-    bool operator<(const AvatarPriority& other) const { return priority > other.priority; }
 };
 
 void AvatarManager::updateOtherAvatars(float deltaTime) {
@@ -205,15 +204,17 @@ void AvatarManager::updateOtherAvatars(float deltaTime) {
             float distance = glm::length(offset) + 0.001f; // add 1mm to avoid divide by zero
             float radius = avatar->getBoundingRadius();
             const glm::vec3& forward = cameraView.getDirection();
-            float apparentSize = radius / distance;
+            float apparentSize = 2.0f * radius / distance;
-            float cosineAngle = glm::length(offset - glm::dot(offset, forward) * forward) / distance;
+            float cosineAngle = glm::length(glm::dot(offset, forward) * forward) / distance;
-            const float TIME_PENALTY = 0.080f; // seconds
+            float age = (float)(startTime - avatar->getLastRenderUpdateTime()) / (float)(USECS_PER_SECOND);
-            float age = (float)(startTime - avatar->getLastRenderUpdateTime()) / (float)(USECS_PER_SECOND) - TIME_PENALTY;
+
             // NOTE: we are adding values of different units to get a single measure of "priority".
-            // Thus we multiply each component by a conversion "weight" that scales its units
+            // Thus we multiply each component by a conversion "weight" that scales its units relative to the others.
-            // relative to the others.  These weights are pure magic tuning and are hard coded in the
+            // These weights are pure magic tuning and should be hard coded in the relation below,
-            // relation below: (hint: unitary weights are not explicityly shown)
+            // but are currently exposed for anyone who would like to explore fine tuning:
-            float priority = apparentSize + 0.25f * cosineAngle + age;
+            float priority = _avatarSortCoefficientSize * apparentSize
+                    + _avatarSortCoefficientCenter * cosineAngle
+                    + _avatarSortCoefficientAge * age;
 
             // decrement priority of avatars outside keyhole
             if (distance > cameraView.getCenterRadius()) {
@@ -593,3 +594,29 @@ RayToAvatarIntersectionResult AvatarManager::findRayIntersection(const PickRay&
 
     return result;
 }
+
+// HACK
+float AvatarManager::getAvatarSortCoefficient(const QString& name) {
+    if (name == "size") {
+        return _avatarSortCoefficientSize;
+    } else if (name == "center") {
+        return _avatarSortCoefficientCenter;
+    } else if (name == "age") {
+        return _avatarSortCoefficientAge;
+    }
+    return 0.0f;
+}
+
+// HACK
+void AvatarManager::setAvatarSortCoefficient(const QString& name, const QScriptValue& value) {
+    if (value.isNumber()) {
+        float numericalValue = (float)value.toNumber();
+        if (name == "size") {
+            _avatarSortCoefficientSize = numericalValue;
+        } else if (name == "center") {
+            _avatarSortCoefficientCenter = numericalValue;
+        } else if (name == "age") {
+            _avatarSortCoefficientAge = numericalValue;
+        }
+    }
+}

interface/src/avatar/AvatarManager.h

@@ -81,6 +81,10 @@ public:
                                                                   const QScriptValue& avatarIdsToInclude = QScriptValue(),
                                                                   const QScriptValue& avatarIdsToDiscard = QScriptValue());
 
+    // TODO: remove this HACK once we settle on optimal default sort coefficients
+    Q_INVOKABLE float getAvatarSortCoefficient(const QString& name);
+    Q_INVOKABLE void setAvatarSortCoefficient(const QString& name, const QScriptValue& value);
+
     float getMyAvatarSendRate() const { return _myAvatarSendRate.rate(); }
 
 public slots:
@@ -120,6 +124,11 @@ private:
     int _partiallySimulatedAvatars { 0 };
     float _avatarSimulationTime { 0.0f };
 
+    // TODO: remove this HACK once we settle on optimal sort coefficients
+    // These coefficients exposed for fine tuning the sort priority for transfering new _jointData to the render pipeline.
+    float _avatarSortCoefficientSize { 0.5f };
+    float _avatarSortCoefficientCenter { 0.25 };
+    float _avatarSortCoefficientAge { 1.0f };
 };
 
 Q_DECLARE_METATYPE(AvatarManager::LocalLight)

interface/src/avatar/MyAvatar.cpp

@@ -227,8 +227,7 @@ void MyAvatar::simulateAttachments(float deltaTime) {
     // don't update attachments here, do it in harvestResultsFromPhysicsSimulation()
 }
 
-QByteArray MyAvatar::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
+QByteArray MyAvatar::toByteArrayStateful(AvatarDataDetail dataDetail) {
-                        bool distanceAdjust, glm::vec3 viewerPosition, QVector<JointData>* sentJointDataOut) {
     CameraMode mode = qApp->getCamera()->getMode();
     _globalPosition = getPosition();
     _globalBoundingBoxDimensions.x = _characterController.getCapsuleRadius();
@@ -239,12 +238,12 @@ QByteArray MyAvatar::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTi
         // fake the avatar position that is sent up to the AvatarMixer
         glm::vec3 oldPosition = getPosition();
         setPosition(getSkeletonPosition());
-        QByteArray array = AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut);
+        QByteArray array = AvatarData::toByteArrayStateful(dataDetail);
         // copy the correct position back
         setPosition(oldPosition);
         return array;
     }
-    return AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut);
+    return AvatarData::toByteArrayStateful(dataDetail);
 }
 
 void MyAvatar::centerBody() {
@@ -808,7 +807,7 @@ void MyAvatar::saveData() {
     auto hmdInterface = DependencyManager::get<HMDScriptingInterface>();
     _avatarEntitiesLock.withReadLock([&] {
         for (auto entityID : _avatarEntityData.keys()) {
-            if (hmdInterface->getCurrentTableUIID() == entityID) {
+            if (hmdInterface->getCurrentTabletUIID() == entityID) {
                 // don't persist the tablet between domains / sessions
                 continue;
             }

interface/src/avatar/MyAvatar.h

@@ -338,8 +338,7 @@ private:
     glm::quat getWorldBodyOrientation() const;
 
 
-    virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
+    virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail) override;
-                            bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector<JointData>* sentJointDataOut = nullptr) override;
 
     void simulate(float deltaTime);
     void updateFromTrackers(float deltaTime);

interface/src/devices/DdeFaceTracker.cpp

@@ -193,7 +193,7 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui
     _calibrationCount(0),
     _calibrationValues(),
     _calibrationBillboard(NULL),
-    _calibrationBillboardID(0),
+    _calibrationBillboardID(UNKNOWN_OVERLAY_ID),
     _calibrationMessage(QString()),
     _isCalibrated(false)
 {

interface/src/devices/DdeFaceTracker.h

@@ -149,7 +149,7 @@ private:
     int _calibrationCount;
     QVector<float> _calibrationValues;
     TextOverlay* _calibrationBillboard;
-    int _calibrationBillboardID;
+    OverlayID _calibrationBillboardID;
     QString _calibrationMessage;
     bool _isCalibrated;
     void addCalibrationDatum();

interface/src/scripting/HMDScriptingInterface.h

@@ -29,9 +29,9 @@ class HMDScriptingInterface : public AbstractHMDScriptingInterface, public Depen
     Q_PROPERTY(glm::quat orientation READ getOrientation)
     Q_PROPERTY(bool mounted READ isMounted)
     Q_PROPERTY(bool showTablet READ getShouldShowTablet)
-    Q_PROPERTY(QUuid tabletID READ getCurrentTableUIID WRITE setCurrentTabletUIID)
+    Q_PROPERTY(QUuid tabletID READ getCurrentTabletUIID WRITE setCurrentTabletUIID)
-    Q_PROPERTY(unsigned int homeButtonID READ getCurrentHomeButtonUUID WRITE setCurrentHomeButtonUUID)
+    Q_PROPERTY(QUuid homeButtonID READ getCurrentHomeButtonUUID WRITE setCurrentHomeButtonUUID)
-    
+    Q_PROPERTY(QUuid tabletScreenID READ getCurrentTabletScreenID WRITE setCurrentTabletScreenID)
 
 public:
     Q_INVOKABLE glm::vec3 calculateRayUICollisionPoint(const glm::vec3& position, const glm::vec3& direction) const;
@@ -91,15 +91,19 @@ public:
     bool getShouldShowTablet() const { return _showTablet; }
 
     void setCurrentTabletUIID(QUuid tabletID) { _tabletUIID = tabletID; }
-    QUuid getCurrentTableUIID() const { return _tabletUIID; }
+    QUuid getCurrentTabletUIID() const { return _tabletUIID; }
 
-    void setCurrentHomeButtonUUID(unsigned int homeButtonID) { _homeButtonID = homeButtonID; }
+    void setCurrentHomeButtonUUID(QUuid homeButtonID) { _homeButtonID = homeButtonID; }
-    unsigned int getCurrentHomeButtonUUID() const { return _homeButtonID; }
+    QUuid getCurrentHomeButtonUUID() const { return _homeButtonID; }
+
+    void setCurrentTabletScreenID(QUuid tabletID) { _tabletScreenID = tabletID; }
+    QUuid getCurrentTabletScreenID() const { return _tabletScreenID; }
 
 private:
     bool _showTablet { false };
-    QUuid _tabletUIID; // this is the entityID of the WebEntity which is part of (a child of) the tablet-ui.
+    QUuid _tabletUIID; // this is the entityID of the tablet frame
-    unsigned int _homeButtonID;
+    QUuid _tabletScreenID; // this is the overlayID which is part of (a child of) the tablet-ui.
+    QUuid _homeButtonID;
     QUuid _tabletEntityID;
 
     // Get the position of the HMD

interface/src/ui/overlays/Base3DOverlay.h

@@ -23,6 +23,9 @@ public:
     Base3DOverlay();
     Base3DOverlay(const Base3DOverlay* base3DOverlay);
 
+    virtual OverlayID getOverlayID() const override { return OverlayID(getID().toString()); }
+    void setOverlayID(OverlayID overlayID) override { setID(overlayID); }
+
     // getters
     virtual bool is3D() const override { return true; }
 

interface/src/ui/overlays/Overlay.cpp

@@ -189,7 +189,7 @@ float Overlay::updatePulse() {
         _pulseDirection *= -1.0f;
     }
     _pulse += pulseDelta;
-    
+
     return _pulse;
 }
 
@@ -204,3 +204,23 @@ void Overlay::removeFromScene(Overlay::Pointer overlay, std::shared_ptr<render::
     render::Item::clearID(_renderItemID);
 }
 
+QScriptValue OverlayIDtoScriptValue(QScriptEngine* engine, const OverlayID& id) {
+    return quuidToScriptValue(engine, id);
+}
+
+void OverlayIDfromScriptValue(const QScriptValue &object, OverlayID& id) {
+    quuidFromScriptValue(object, id);
+}
+
+QVector<OverlayID> qVectorOverlayIDFromScriptValue(const QScriptValue& array) {
+    if (!array.isArray()) {
+        return QVector<OverlayID>();
+    }
+    QVector<OverlayID> newVector;
+    int length = array.property("length").toInteger();
+    newVector.reserve(length);
+    for (int i = 0; i < length; i++) {
+        newVector << OverlayID(array.property(i).toString());
+    }
+    return newVector;
+}

interface/src/ui/overlays/Overlay.h

@@ -15,6 +15,13 @@
 #include <SharedUtil.h> // for xColor
 #include <render/Scene.h>
 
+class OverlayID : public QUuid {
+public:
+    OverlayID() : QUuid() {}
+    OverlayID(QString v) : QUuid(v) {}
+    OverlayID(QUuid v) : QUuid(v) {}
+};
+
 class Overlay : public QObject {
     Q_OBJECT
 
@@ -32,8 +39,8 @@ public:
     Overlay(const Overlay* overlay);
     ~Overlay();
 
-    unsigned int getOverlayID() { return _overlayID; }
+    virtual OverlayID getOverlayID() const { return _overlayID; }
-    void setOverlayID(unsigned int overlayID) { _overlayID = overlayID; }
+    virtual void setOverlayID(OverlayID overlayID) { _overlayID = overlayID; }
 
     virtual void update(float deltatime) {}
     virtual void render(RenderArgs* args) = 0;
@@ -84,13 +91,14 @@ public:
     render::ItemID getRenderItemID() const { return _renderItemID; }
     void setRenderItemID(render::ItemID renderItemID) { _renderItemID = renderItemID; }
 
+    unsigned int getStackOrder() const { return _stackOrder; }
+    void setStackOrder(unsigned int stackOrder) { _stackOrder = stackOrder; }
+
 protected:
     float updatePulse();
 
     render::ItemID _renderItemID{ render::Item::INVALID_ITEM_ID };
 
-    unsigned int _overlayID; // what Overlays.cpp knows this instance as
-
     bool _isLoaded;
     float _alpha;
 
@@ -107,15 +115,25 @@ protected:
     xColor _color;
     bool _visible; // should the overlay be drawn at all
     Anchor _anchor;
+
+    unsigned int _stackOrder { 0 };
+
+private:
+    OverlayID _overlayID; // only used for non-3d overlays
 };
 
 namespace render {
-   template <> const ItemKey payloadGetKey(const Overlay::Pointer& overlay); 
+   template <> const ItemKey payloadGetKey(const Overlay::Pointer& overlay);
    template <> const Item::Bound payloadGetBound(const Overlay::Pointer& overlay);
    template <> int payloadGetLayer(const Overlay::Pointer& overlay);
    template <> void payloadRender(const Overlay::Pointer& overlay, RenderArgs* args);
    template <> const ShapeKey shapeGetShapeKey(const Overlay::Pointer& overlay);
 }
 
- 
+Q_DECLARE_METATYPE(OverlayID);
+Q_DECLARE_METATYPE(QVector<OverlayID>);
+QScriptValue OverlayIDtoScriptValue(QScriptEngine* engine, const OverlayID& id);
+void OverlayIDfromScriptValue(const QScriptValue& object, OverlayID& id);
+QVector<OverlayID> qVectorOverlayIDFromScriptValue(const QScriptValue& array);
+
 #endif // hifi_Overlay_h

interface/src/ui/overlays/OverlayPanel.cpp

@@ -51,13 +51,13 @@ void propertyBindingFromVariant(const QVariant& objectVar, PropertyBinding& valu
 }
 
 
-void OverlayPanel::addChild(unsigned int childId) {
+void OverlayPanel::addChild(OverlayID childId) {
     if (!_children.contains(childId)) {
         _children.append(childId);
     }
 }
 
-void OverlayPanel::removeChild(unsigned int childId) {
+void OverlayPanel::removeChild(OverlayID childId) {
     if (_children.contains(childId)) {
         _children.removeOne(childId);
     }
@@ -89,7 +89,7 @@ QVariant OverlayPanel::getProperty(const QString &property) {
     if (property == "children") {
         QVariantList array;
         for (int i = 0; i < _children.length(); i++) {
-            array.append(_children[i]);
+            array.append(OverlayIDtoScriptValue(nullptr, _children[i]).toVariant());
         }
         return array;
     }

interface/src/ui/overlays/OverlayPanel.h

@@ -20,6 +20,7 @@
 
 #include "PanelAttachable.h"
 #include "Billboardable.h"
+#include "Overlay.h"
 
 class PropertyBinding {
 public:
@@ -54,10 +55,10 @@ public:
     void setAnchorScale(const glm::vec3& scale) { _anchorTransform.setScale(scale); }
     void setVisible(bool visible) { _visible = visible; }
 
-    const QList<unsigned int>& getChildren() { return _children; }
+    const QList<OverlayID>& getChildren() { return _children; }
-    void addChild(unsigned int childId);
+    void addChild(OverlayID childId);
-    void removeChild(unsigned int childId);
+    void removeChild(OverlayID childId);
-    unsigned int popLastChild() { return _children.takeLast(); }
+    OverlayID popLastChild() { return _children.takeLast(); }
 
     void setProperties(const QVariantMap& properties);
     QVariant getProperty(const QString& property);
@@ -74,7 +75,7 @@ private:
     QUuid _anchorRotationBindEntity;
 
     bool _visible = true;
-    QList<unsigned int> _children;
+    QList<OverlayID> _children;
 
     QScriptEngine* _scriptEngine;
 };

interface/src/ui/overlays/Overlays.cpp

@@ -39,9 +39,6 @@
 
 Q_LOGGING_CATEGORY(trace_render_overlays, "trace.render.overlays")
 
-Overlays::Overlays() :
-    _nextOverlayID(1) {}
-
 void Overlays::cleanupAllOverlays() {
     {
         QWriteLocker lock(&_lock);
@@ -139,7 +136,7 @@ void Overlays::enable() {
     _enabled = true;
 }
 
-Overlay::Pointer Overlays::getOverlay(unsigned int id) const {
+Overlay::Pointer Overlays::getOverlay(OverlayID id) const {
     if (_overlaysHUD.contains(id)) {
         return _overlaysHUD[id];
     }
@@ -149,7 +146,7 @@ Overlay::Pointer Overlays::getOverlay(unsigned int id) const {
     return nullptr;
 }
 
-unsigned int Overlays::addOverlay(const QString& type, const QVariant& properties) {
+OverlayID Overlays::addOverlay(const QString& type, const QVariant& properties) {
     Overlay::Pointer thisOverlay = nullptr;
 
     if (type == ImageOverlay::TYPE) {
@@ -188,14 +185,14 @@ unsigned int Overlays::addOverlay(const QString& type, const QVariant& propertie
         thisOverlay->setProperties(properties.toMap());
         return addOverlay(thisOverlay);
     }
-    return 0;
+    return UNKNOWN_OVERLAY_ID;
 }
 
-unsigned int Overlays::addOverlay(Overlay::Pointer overlay) {
+OverlayID Overlays::addOverlay(Overlay::Pointer overlay) {
     QWriteLocker lock(&_lock);
-    unsigned int thisID = _nextOverlayID;
+    OverlayID thisID = OverlayID(QUuid::createUuid());
     overlay->setOverlayID(thisID);
-    _nextOverlayID++;
+    overlay->setStackOrder(_stackOrder++);
     if (overlay->is3D()) {
         _overlaysWorld[thisID] = overlay;
 
@@ -210,22 +207,22 @@ unsigned int Overlays::addOverlay(Overlay::Pointer overlay) {
     return thisID;
 }
 
-unsigned int Overlays::cloneOverlay(unsigned int id) {
+OverlayID Overlays::cloneOverlay(OverlayID id) {
     Overlay::Pointer thisOverlay = getOverlay(id);
 
     if (thisOverlay) {
-        unsigned int cloneId = addOverlay(Overlay::Pointer(thisOverlay->createClone()));
+        OverlayID cloneId = addOverlay(Overlay::Pointer(thisOverlay->createClone()));
         auto attachable = std::dynamic_pointer_cast<PanelAttachable>(thisOverlay);
         if (attachable && attachable->getParentPanel()) {
             attachable->getParentPanel()->addChild(cloneId);
         }
         return cloneId;
-    } 
+    }
-    
+
-    return 0;  // Not found
+    return UNKNOWN_OVERLAY_ID;  // Not found
 }
 
-bool Overlays::editOverlay(unsigned int id, const QVariant& properties) {
+bool Overlays::editOverlay(OverlayID id, const QVariant& properties) {
     QWriteLocker lock(&_lock);
 
     Overlay::Pointer thisOverlay = getOverlay(id);
@@ -242,13 +239,7 @@ bool Overlays::editOverlays(const QVariant& propertiesById) {
     bool success = true;
     QWriteLocker lock(&_lock);
     for (const auto& key : map.keys()) {
-        bool convertSuccess;
+        OverlayID id = OverlayID(key);
-        unsigned int id = key.toUInt(&convertSuccess);
-        if (!convertSuccess) {
-            success = false;
-            continue;
-        }
-
         Overlay::Pointer thisOverlay = getOverlay(id);
         if (!thisOverlay) {
             success = false;
@@ -260,7 +251,7 @@ bool Overlays::editOverlays(const QVariant& propertiesById) {
     return success;
 }
 
-void Overlays::deleteOverlay(unsigned int id) {
+void Overlays::deleteOverlay(OverlayID id) {
     Overlay::Pointer overlayToDelete;
 
     {
@@ -286,7 +277,7 @@ void Overlays::deleteOverlay(unsigned int id) {
     emit overlayDeleted(id);
 }
 
-QString Overlays::getOverlayType(unsigned int overlayId) const {
+QString Overlays::getOverlayType(OverlayID overlayId) const {
     Overlay::Pointer overlay = getOverlay(overlayId);
     if (overlay) {
         return overlay->getType();
@@ -294,7 +285,7 @@ QString Overlays::getOverlayType(unsigned int overlayId) const {
     return "";
 }
 
-QObject* Overlays::getOverlayObject(unsigned int id) {
+QObject* Overlays::getOverlayObject(OverlayID id) {
     Overlay::Pointer thisOverlay = getOverlay(id);
     if (thisOverlay) {
         return qobject_cast<QObject*>(&(*thisOverlay));
@@ -302,7 +293,7 @@ QObject* Overlays::getOverlayObject(unsigned int id) {
     return nullptr;
 }
 
-unsigned int Overlays::getParentPanel(unsigned int childId) const {
+OverlayID Overlays::getParentPanel(OverlayID childId) const {
     Overlay::Pointer overlay = getOverlay(childId);
     auto attachable = std::dynamic_pointer_cast<PanelAttachable>(overlay);
     if (attachable) {
@@ -310,10 +301,10 @@ unsigned int Overlays::getParentPanel(unsigned int childId) const {
     } else if (_panels.contains(childId)) {
         return _panels.key(getPanel(childId)->getParentPanel());
     }
-    return 0;
+    return UNKNOWN_OVERLAY_ID;
 }
 
-void Overlays::setParentPanel(unsigned int childId, unsigned int panelId) {
+void Overlays::setParentPanel(OverlayID childId, OverlayID panelId) {
     auto attachable = std::dynamic_pointer_cast<PanelAttachable>(getOverlay(childId));
     if (attachable) {
         if (_panels.contains(panelId)) {
@@ -343,13 +334,13 @@ void Overlays::setParentPanel(unsigned int childId, unsigned int panelId) {
     }
 }
 
-unsigned int Overlays::getOverlayAtPoint(const glm::vec2& point) {
+OverlayID Overlays::getOverlayAtPoint(const glm::vec2& point) {
     glm::vec2 pointCopy = point;
     QReadLocker lock(&_lock);
     if (!_enabled) {
-        return 0;
+        return UNKNOWN_OVERLAY_ID;
     }
-    QMapIterator<unsigned int, Overlay::Pointer> i(_overlaysHUD);
+    QMapIterator<OverlayID, Overlay::Pointer> i(_overlaysHUD);
     i.toBack();
 
     const float LARGE_NEGATIVE_FLOAT = -9999999;
@@ -358,10 +349,12 @@ unsigned int Overlays::getOverlayAtPoint(const glm::vec2& point) {
     BoxFace thisFace;
     glm::vec3 thisSurfaceNormal;
     float distance;
+    unsigned int bestStackOrder = 0;
+    OverlayID bestOverlayID = UNKNOWN_OVERLAY_ID;
 
     while (i.hasPrevious()) {
         i.previous();
-        unsigned int thisID = i.key();
+        OverlayID thisID = i.key();
         if (i.value()->is3D()) {
             auto thisOverlay = std::dynamic_pointer_cast<Base3DOverlay>(i.value());
             if (thisOverlay && !thisOverlay->getIgnoreRayIntersection()) {
@@ -373,15 +366,18 @@ unsigned int Overlays::getOverlayAtPoint(const glm::vec2& point) {
             auto thisOverlay = std::dynamic_pointer_cast<Overlay2D>(i.value());
             if (thisOverlay && thisOverlay->getVisible() && thisOverlay->isLoaded() &&
                 thisOverlay->getBoundingRect().contains(pointCopy.x, pointCopy.y, false)) {
-                return thisID;
+                if (thisOverlay->getStackOrder() > bestStackOrder) {
+                    bestOverlayID = thisID;
+                    bestStackOrder = thisOverlay->getStackOrder();
+                }
             }
         }
     }
 
-    return 0; // not found
+    return bestOverlayID;
 }
 
-OverlayPropertyResult Overlays::getProperty(unsigned int id, const QString& property) {
+OverlayPropertyResult Overlays::getProperty(OverlayID id, const QString& property) {
     OverlayPropertyResult result;
     Overlay::Pointer thisOverlay = getOverlay(id);
     QReadLocker lock(&_lock);
@@ -406,23 +402,35 @@ void OverlayPropertyResultFromScriptValue(const QScriptValue& object, OverlayPro
 }
 
 
-RayToOverlayIntersectionResult Overlays::findRayIntersection(const PickRay& ray) {
+RayToOverlayIntersectionResult Overlays::findRayIntersection(const PickRay& ray, bool precisionPicking,
+                                                             const QScriptValue& overlayIDsToInclude,
+                                                             const QScriptValue& overlayIDsToDiscard,
+                                                             bool visibleOnly, bool collidableOnly) {
     float bestDistance = std::numeric_limits<float>::max();
     bool bestIsFront = false;
+    const QVector<OverlayID> overlaysToInclude = qVectorOverlayIDFromScriptValue(overlayIDsToInclude);
+    const QVector<OverlayID> overlaysToDiscard = qVectorOverlayIDFromScriptValue(overlayIDsToDiscard);
+
     RayToOverlayIntersectionResult result;
-    QMapIterator<unsigned int, Overlay::Pointer> i(_overlaysWorld);
+    QMapIterator<OverlayID, Overlay::Pointer> i(_overlaysWorld);
     i.toBack();
     while (i.hasPrevious()) {
         i.previous();
-        unsigned int thisID = i.key();
+        OverlayID thisID = i.key();
         auto thisOverlay = std::dynamic_pointer_cast<Base3DOverlay>(i.value());
+
+        if ((overlaysToDiscard.size() > 0 && overlaysToDiscard.contains(thisID)) ||
+            (overlaysToInclude.size() > 0 && !overlaysToInclude.contains(thisID))) {
+            continue;
+        }
+
         if (thisOverlay && thisOverlay->getVisible() && !thisOverlay->getIgnoreRayIntersection() && thisOverlay->isLoaded()) {
             float thisDistance;
             BoxFace thisFace;
             glm::vec3 thisSurfaceNormal;
             QString thisExtraInfo;
-            if (thisOverlay->findRayIntersectionExtraInfo(ray.origin, ray.direction, thisDistance, 
+            if (thisOverlay->findRayIntersectionExtraInfo(ray.origin, ray.direction, thisDistance,
-                                                            thisFace, thisSurfaceNormal, thisExtraInfo)) {
+                                                          thisFace, thisSurfaceNormal, thisExtraInfo)) {
                 bool isDrawInFront = thisOverlay->getDrawInFront();
                 if (thisDistance < bestDistance && (!bestIsFront || isDrawInFront)) {
                     bestIsFront = isDrawInFront;
@@ -441,23 +449,23 @@ RayToOverlayIntersectionResult Overlays::findRayIntersection(const PickRay& ray)
     return result;
 }
 
-RayToOverlayIntersectionResult::RayToOverlayIntersectionResult() : 
+RayToOverlayIntersectionResult::RayToOverlayIntersectionResult() :
-    intersects(false), 
+    intersects(false),
-    overlayID(-1),
+    overlayID(UNKNOWN_OVERLAY_ID),
     distance(0),
     face(),
     intersection(),
     extraInfo()
-{ 
+{
 }
 
 QScriptValue RayToOverlayIntersectionResultToScriptValue(QScriptEngine* engine, const RayToOverlayIntersectionResult& value) {
     auto obj = engine->newObject();
     obj.setProperty("intersects", value.intersects);
-    obj.setProperty("overlayID", value.overlayID);
+    obj.setProperty("overlayID", OverlayIDtoScriptValue(engine, value.overlayID));
     obj.setProperty("distance", value.distance);
 
-    QString faceName = "";    
+    QString faceName = "";
     // handle BoxFace
     switch (value.face) {
         case MIN_X_FACE:
@@ -493,7 +501,7 @@ QScriptValue RayToOverlayIntersectionResultToScriptValue(QScriptEngine* engine,
 void RayToOverlayIntersectionResultFromScriptValue(const QScriptValue& objectVar, RayToOverlayIntersectionResult& value) {
     QVariantMap object = objectVar.toVariant().toMap();
     value.intersects = object["intersects"].toBool();
-    value.overlayID = object["overlayID"].toInt();
+    value.overlayID = OverlayID(QUuid(object["overlayID"].toString()));
     value.distance = object["distance"].toFloat();
 
     QString faceName = object["face"].toString();
@@ -523,7 +531,7 @@ void RayToOverlayIntersectionResultFromScriptValue(const QScriptValue& objectVar
     value.extraInfo = object["extraInfo"].toString();
 }
 
-bool Overlays::isLoaded(unsigned int id) {
+bool Overlays::isLoaded(OverlayID id) {
     QReadLocker lock(&_lock);
     Overlay::Pointer thisOverlay = getOverlay(id);
     if (!thisOverlay) {
@@ -532,7 +540,7 @@ bool Overlays::isLoaded(unsigned int id) {
     return thisOverlay->isLoaded();
 }
 
-QSizeF Overlays::textSize(unsigned int id, const QString& text) const {
+QSizeF Overlays::textSize(OverlayID id, const QString& text) const {
     Overlay::Pointer thisOverlay = _overlaysHUD[id];
     if (thisOverlay) {
         if (auto textOverlay = std::dynamic_pointer_cast<TextOverlay>(thisOverlay)) {
@@ -547,30 +555,29 @@ QSizeF Overlays::textSize(unsigned int id, const QString& text) const {
     return QSizeF(0.0f, 0.0f);
 }
 
-unsigned int Overlays::addPanel(OverlayPanel::Pointer panel) {
+OverlayID Overlays::addPanel(OverlayPanel::Pointer panel) {
     QWriteLocker lock(&_lock);
 
-    unsigned int thisID = _nextOverlayID;
+    OverlayID thisID = QUuid::createUuid();
-    _nextOverlayID++;
     _panels[thisID] = panel;
 
     return thisID;
 }
 
-unsigned int Overlays::addPanel(const QVariant& properties) {
+OverlayID Overlays::addPanel(const QVariant& properties) {
     OverlayPanel::Pointer panel = std::make_shared<OverlayPanel>();
     panel->init(_scriptEngine);
     panel->setProperties(properties.toMap());
     return addPanel(panel);
 }
 
-void Overlays::editPanel(unsigned int panelId, const QVariant& properties) {
+void Overlays::editPanel(OverlayID panelId, const QVariant& properties) {
     if (_panels.contains(panelId)) {
         _panels[panelId]->setProperties(properties.toMap());
     }
 }
 
-OverlayPropertyResult Overlays::getPanelProperty(unsigned int panelId, const QString& property) {
+OverlayPropertyResult Overlays::getPanelProperty(OverlayID panelId, const QString& property) {
     OverlayPropertyResult result;
     if (_panels.contains(panelId)) {
         OverlayPanel::Pointer thisPanel = getPanel(panelId);
@@ -581,7 +588,7 @@ OverlayPropertyResult Overlays::getPanelProperty(unsigned int panelId, const QSt
 }
 
 
-void Overlays::deletePanel(unsigned int panelId) {
+void Overlays::deletePanel(OverlayID panelId) {
     OverlayPanel::Pointer panelToDelete;
 
     {
@@ -594,7 +601,7 @@ void Overlays::deletePanel(unsigned int panelId) {
     }
 
     while (!panelToDelete->getChildren().isEmpty()) {
-        unsigned int childId = panelToDelete->popLastChild();
+        OverlayID childId = panelToDelete->popLastChild();
         deleteOverlay(childId);
         deletePanel(childId);
     }
@@ -602,39 +609,39 @@ void Overlays::deletePanel(unsigned int panelId) {
     emit panelDeleted(panelId);
 }
 
-bool Overlays::isAddedOverlay(unsigned int id) {
+bool Overlays::isAddedOverlay(OverlayID id) {
     return _overlaysHUD.contains(id) || _overlaysWorld.contains(id);
 }
 
-void Overlays::sendMousePressOnOverlay(unsigned int overlayID, const PointerEvent& event) {
+void Overlays::sendMousePressOnOverlay(OverlayID overlayID, const PointerEvent& event) {
     emit mousePressOnOverlay(overlayID, event);
 }
 
-void Overlays::sendMouseReleaseOnOverlay(unsigned int overlayID, const PointerEvent& event) {
+void Overlays::sendMouseReleaseOnOverlay(OverlayID overlayID, const PointerEvent& event) {
     emit mouseReleaseOnOverlay(overlayID, event);
 }
 
-void Overlays::sendMouseMoveOnOverlay(unsigned int overlayID, const PointerEvent& event) {
+void Overlays::sendMouseMoveOnOverlay(OverlayID overlayID, const PointerEvent& event) {
     emit mouseMoveOnOverlay(overlayID, event);
 }
 
-void Overlays::sendHoverEnterOverlay(unsigned int id, PointerEvent event) {
+void Overlays::sendHoverEnterOverlay(OverlayID id, PointerEvent event) {
     emit hoverEnterOverlay(id, event);
 }
 
-void Overlays::sendHoverOverOverlay(unsigned int  id, PointerEvent event) {
+void Overlays::sendHoverOverOverlay(OverlayID  id, PointerEvent event) {
     emit hoverOverOverlay(id, event);
 }
 
-void Overlays::sendHoverLeaveOverlay(unsigned int  id, PointerEvent event) {
+void Overlays::sendHoverLeaveOverlay(OverlayID  id, PointerEvent event) {
     emit hoverLeaveOverlay(id, event);
 }
 
-unsigned int Overlays::getKeyboardFocusOverlay() const {
+OverlayID Overlays::getKeyboardFocusOverlay() const {
     return qApp->getKeyboardFocusOverlay();
 }
 
-void Overlays::setKeyboardFocusOverlay(unsigned int id) {
+void Overlays::setKeyboardFocusOverlay(OverlayID id) {
     qApp->setKeyboardFocusOverlay(id);
 }
 

interface/src/ui/overlays/Overlays.h

@@ -53,7 +53,7 @@ class RayToOverlayIntersectionResult {
 public:
     RayToOverlayIntersectionResult();
     bool intersects;
-    unsigned int overlayID;
+    OverlayID overlayID;
     float distance;
     BoxFace face;
     glm::vec3 surfaceNormal;
@@ -77,15 +77,15 @@ void RayToOverlayIntersectionResultFromScriptValue(const QScriptValue& object, R
  * @namespace Overlays
  */
 
-const unsigned int UNKNOWN_OVERLAY_ID = 0;
+const OverlayID UNKNOWN_OVERLAY_ID = OverlayID();
 
 class Overlays : public QObject {
     Q_OBJECT
 
-    Q_PROPERTY(unsigned int keyboardFocusOverlay READ getKeyboardFocusOverlay WRITE setKeyboardFocusOverlay)
+    Q_PROPERTY(OverlayID keyboardFocusOverlay READ getKeyboardFocusOverlay WRITE setKeyboardFocusOverlay)
 
 public:
-    Overlays();
+    Overlays() {};
 
     void init();
     void update(float deltatime);
@@ -93,12 +93,12 @@ public:
     void disable();
     void enable();
 
-    Overlay::Pointer getOverlay(unsigned int id) const;
+    Overlay::Pointer getOverlay(OverlayID id) const;
-    OverlayPanel::Pointer getPanel(unsigned int id) const { return _panels[id]; }
+    OverlayPanel::Pointer getPanel(OverlayID id) const { return _panels[id]; }
 
     /// adds an overlay that's already been created
-    unsigned int addOverlay(Overlay* overlay) { return addOverlay(Overlay::Pointer(overlay)); }
+    OverlayID addOverlay(Overlay* overlay) { return addOverlay(Overlay::Pointer(overlay)); }
-    unsigned int addOverlay(Overlay::Pointer overlay);
+    OverlayID addOverlay(Overlay::Pointer overlay);
 
     void mousePressEvent(QMouseEvent* event);
     void mouseReleaseEvent(QMouseEvent* event);
@@ -116,7 +116,7 @@ public slots:
      * @param {Overlays.OverlayProperties} The properties of the overlay that you want to add.
      * @return {Overlays.OverlayID} The ID of the newly created overlay.
      */
-    unsigned int addOverlay(const QString& type, const QVariant& properties);
+    OverlayID addOverlay(const QString& type, const QVariant& properties);
 
     /**jsdoc
      * Create a clone of an existing overlay.
@@ -125,7 +125,7 @@ public slots:
      * @param {Overlays.OverlayID} overlayID The ID of the overlay to clone.
      * @return {Overlays.OverlayID} The ID of the new overlay.
      */
-    unsigned int cloneOverlay(unsigned int id);
+    OverlayID cloneOverlay(OverlayID id);
 
     /**jsdoc
      * Edit an overlay's properties. 
@@ -134,7 +134,7 @@ public slots:
      * @param {Overlays.OverlayID} overlayID The ID of the overlay to edit.
      * @return {bool} `true` if the overlay was found and edited, otherwise false.
      */
-    bool editOverlay(unsigned int id, const QVariant& properties);
+    bool editOverlay(OverlayID id, const QVariant& properties);
 
     /// edits an overlay updating only the included properties, will return the identified OverlayID in case of
     /// successful edit, if the input id is for an unknown overlay this function will have no effect
@@ -146,7 +146,7 @@ public slots:
      * @function Overlays.deleteOverlay
      * @param {Overlays.OverlayID} overlayID The ID of the overlay to delete.
      */
-    void deleteOverlay(unsigned int id);
+    void deleteOverlay(OverlayID id);
 
     /**jsdoc
      * Get the type of an overlay.
@@ -155,7 +155,7 @@ public slots:
      * @param {Overlays.OverlayID} overlayID The ID of the overlay to get the type of.
      * @return {string} The type of the overlay if found, otherwise the empty string.
      */
-    QString getOverlayType(unsigned int overlayId) const;
+    QString getOverlayType(OverlayID overlayId) const;
 
     /**jsdoc
     * Get the overlay Script object.
@@ -164,7 +164,7 @@ public slots:
     * @param {Overlays.OverlayID} overlayID The ID of the overlay to get the script object of.
     * @return {Object} The script object for the overlay if found.
     */
-    QObject* getOverlayObject(unsigned int id);
+    QObject* getOverlayObject(OverlayID id);
 
     /**jsdoc
      * Get the ID of the overlay at a particular point on the HUD/screen.
@@ -174,7 +174,7 @@ public slots:
      * @return {Overlays.OverlayID} The ID of the overlay at the point specified.
      *     If no overlay is found, `0` will be returned.
      */
-    unsigned int getOverlayAtPoint(const glm::vec2& point);
+    OverlayID getOverlayAtPoint(const glm::vec2& point);
 
     /**jsdoc
      * Get the value of a an overlay's property.
@@ -185,16 +185,26 @@ public slots:
      * @return {Object} The value of the property. If the overlay or the property could
      *     not be found, `undefined` will be returned.
      */
-    OverlayPropertyResult getProperty(unsigned int id, const QString& property);
+    OverlayPropertyResult getProperty(OverlayID id, const QString& property);
 
     /*jsdoc
      * Find the closest 3D overlay hit by a pick ray.
      *
      * @function Overlays.findRayIntersection
      * @param {PickRay} The PickRay to use for finding overlays.
+     * @param {bool} Unused; Exists to match Entity interface
+     * @param {List of Overlays.OverlayID} Whitelist for intersection test.
+     * @param {List of Overlays.OverlayID} Blacklist for intersection test.
+     * @param {bool} Unused; Exists to match Entity interface
+     * @param {bool} Unused; Exists to match Entity interface
      * @return {Overlays.RayToOverlayIntersectionResult} The result of the ray cast.
      */
-    RayToOverlayIntersectionResult findRayIntersection(const PickRay& ray);
+    RayToOverlayIntersectionResult findRayIntersection(const PickRay& ray,
+                                                       bool precisionPicking = false,
+                                                       const QScriptValue& overlayIDsToInclude = QScriptValue(),
+                                                       const QScriptValue& overlayIDsToDiscard = QScriptValue(),
+                                                       bool visibleOnly = false,
+                                                       bool collidableOnly = false);
 
     /**jsdoc
      * Check whether an overlay's assets have been loaded. For example, if the
@@ -204,7 +214,7 @@ public slots:
      * @param {Overlays.OverlayID} The ID of the overlay to check.
      * @return {bool} `true` if the overlay's assets have been loaded, otherwise `false`.
      */
-    bool isLoaded(unsigned int id);
+    bool isLoaded(OverlayID id);
 
     /**jsdoc
      * Calculates the size of the given text in the specified overlay if it is a text overlay.
@@ -216,7 +226,7 @@ public slots:
      * @param {string} The string to measure.
      * @return {Vec2} The size of the text.
      */
-    QSizeF textSize(unsigned int id, const QString& text) const;
+    QSizeF textSize(OverlayID id, const QString& text) const;
 
     /**jsdoc
      * Get the width of the virtual 2D HUD.
@@ -235,39 +245,39 @@ public slots:
     float height() const;
 
     /// return true if there is an overlay with that id else false
-    bool isAddedOverlay(unsigned int id);
+    bool isAddedOverlay(OverlayID id);
 
-    unsigned int getParentPanel(unsigned int childId) const;
+    OverlayID getParentPanel(OverlayID childId) const;
-    void setParentPanel(unsigned int childId, unsigned int panelId);
+    void setParentPanel(OverlayID childId, OverlayID panelId);
 
     /// adds a panel that has already been created
-    unsigned int addPanel(OverlayPanel::Pointer panel);
+    OverlayID addPanel(OverlayPanel::Pointer panel);
 
     /// creates and adds a panel based on a set of properties
-    unsigned int addPanel(const QVariant& properties);
+    OverlayID addPanel(const QVariant& properties);
 
     /// edit the properties of a panel
-    void editPanel(unsigned int panelId, const QVariant& properties);
+    void editPanel(OverlayID panelId, const QVariant& properties);
 
     /// get a property of a panel
-    OverlayPropertyResult getPanelProperty(unsigned int panelId, const QString& property);
+    OverlayPropertyResult getPanelProperty(OverlayID panelId, const QString& property);
 
     /// deletes a panel and all child overlays
-    void deletePanel(unsigned int panelId);
+    void deletePanel(OverlayID panelId);
 
     /// return true if there is a panel with that id else false
-    bool isAddedPanel(unsigned int id) { return _panels.contains(id); }
+    bool isAddedPanel(OverlayID id) { return _panels.contains(id); }
 
-    void sendMousePressOnOverlay(unsigned int overlayID, const PointerEvent& event);
+    void sendMousePressOnOverlay(OverlayID overlayID, const PointerEvent& event);
-    void sendMouseReleaseOnOverlay(unsigned int overlayID, const PointerEvent& event);
+    void sendMouseReleaseOnOverlay(OverlayID overlayID, const PointerEvent& event);
-    void sendMouseMoveOnOverlay(unsigned int overlayID, const PointerEvent& event);
+    void sendMouseMoveOnOverlay(OverlayID overlayID, const PointerEvent& event);
 
-    void sendHoverEnterOverlay(unsigned int id, PointerEvent event);
+    void sendHoverEnterOverlay(OverlayID id, PointerEvent event);
-    void sendHoverOverOverlay(unsigned int id, PointerEvent event);
+    void sendHoverOverOverlay(OverlayID id, PointerEvent event);
-    void sendHoverLeaveOverlay(unsigned int id, PointerEvent event);
+    void sendHoverLeaveOverlay(OverlayID id, PointerEvent event);
 
-    unsigned int getKeyboardFocusOverlay() const;
+    OverlayID getKeyboardFocusOverlay() const;
-    void setKeyboardFocusOverlay(unsigned int id);
+    void setKeyboardFocusOverlay(OverlayID id);
 
 signals:
     /**jsdoc
@@ -276,26 +286,26 @@ signals:
      * @function Overlays.overlayDeleted
      * @param {OverlayID} The ID of the overlay that was deleted.
      */
-    void overlayDeleted(unsigned int id);
+    void overlayDeleted(OverlayID id);
-    void panelDeleted(unsigned int id);
+    void panelDeleted(OverlayID id);
 
-    void mousePressOnOverlay(unsigned int overlayID, const PointerEvent& event);
+    void mousePressOnOverlay(OverlayID overlayID, const PointerEvent& event);
-    void mouseReleaseOnOverlay(unsigned int overlayID, const PointerEvent& event);
+    void mouseReleaseOnOverlay(OverlayID overlayID, const PointerEvent& event);
-    void mouseMoveOnOverlay(unsigned int overlayID, const PointerEvent& event);
+    void mouseMoveOnOverlay(OverlayID overlayID, const PointerEvent& event);
     void mousePressOffOverlay();
 
-    void hoverEnterOverlay(unsigned int overlayID, const PointerEvent& event);
+    void hoverEnterOverlay(OverlayID overlayID, const PointerEvent& event);
-    void hoverOverOverlay(unsigned int overlayID, const PointerEvent& event);
+    void hoverOverOverlay(OverlayID overlayID, const PointerEvent& event);
-    void hoverLeaveOverlay(unsigned int overlayID, const PointerEvent& event);
+    void hoverLeaveOverlay(OverlayID overlayID, const PointerEvent& event);
 
 private:
     void cleanupOverlaysToDelete();
 
-    QMap<unsigned int, Overlay::Pointer> _overlaysHUD;
+    QMap<OverlayID, Overlay::Pointer> _overlaysHUD;
-    QMap<unsigned int, Overlay::Pointer> _overlaysWorld;
+    QMap<OverlayID, Overlay::Pointer> _overlaysWorld;
-    QMap<unsigned int, OverlayPanel::Pointer> _panels;
+    QMap<OverlayID, OverlayPanel::Pointer> _panels;
     QList<Overlay::Pointer> _overlaysToDelete;
-    unsigned int _nextOverlayID;
+    unsigned int _stackOrder { 1 };
 
     QReadWriteLock _lock;
     QReadWriteLock _deleteLock;
@@ -305,8 +315,8 @@ private:
     PointerEvent calculatePointerEvent(Overlay::Pointer overlay, PickRay ray, RayToOverlayIntersectionResult rayPickResult,
         QMouseEvent* event, PointerEvent::EventType eventType);
 
-    unsigned int _currentClickingOnOverlayID { UNKNOWN_OVERLAY_ID };
+    OverlayID _currentClickingOnOverlayID { UNKNOWN_OVERLAY_ID };
-    unsigned int _currentHoverOverOverlayID { UNKNOWN_OVERLAY_ID };
+    OverlayID _currentHoverOverOverlayID { UNKNOWN_OVERLAY_ID };
 };
 
 #endif // hifi_Overlays_h

interface/src/ui/overlays/Web3DOverlay.cpp

@@ -198,7 +198,7 @@ void Web3DOverlay::render(RenderArgs* args) {
         _webSurface->getRootItem()->setProperty("scriptURL", _scriptURL);
         currentContext->makeCurrent(currentSurface);
 
-        auto forwardPointerEvent = [=](unsigned int overlayID, const PointerEvent& event) {
+        auto forwardPointerEvent = [=](OverlayID overlayID, const PointerEvent& event) {
             if (overlayID == getOverlayID()) {
                 handlePointerEvent(event);
             }
@@ -208,7 +208,7 @@ void Web3DOverlay::render(RenderArgs* args) {
         _mouseReleaseConnection = connect(&(qApp->getOverlays()), &Overlays::mouseReleaseOnOverlay, forwardPointerEvent);
         _mouseMoveConnection = connect(&(qApp->getOverlays()), &Overlays::mouseMoveOnOverlay, forwardPointerEvent);
         _hoverLeaveConnection = connect(&(qApp->getOverlays()), &Overlays::hoverLeaveOverlay,
-            [=](unsigned int overlayID, const PointerEvent& event) {
+            [=](OverlayID overlayID, const PointerEvent& event) {
             if (this->_pressed && this->getOverlayID() == overlayID) {
                 // If the user mouses off the overlay while the button is down, simulate a touch end.
                 QTouchEvent::TouchPoint point;

libraries/audio-client/src/AudioClient.cpp

@@ -1120,7 +1120,7 @@ void AudioClient::prepareLocalAudioInjectors() {
     while (samplesNeeded > 0) {
         // lock for every write to avoid locking out the device callback
         // this lock is intentional - the buffer is only lock-free in its use in the device callback
-        Lock lock(_localAudioMutex);
+        RecursiveLock lock(_localAudioMutex);
 
         samplesNeeded = bufferCapacity - _localSamplesAvailable.load(std::memory_order_relaxed);
         if (samplesNeeded <= 0) {
@@ -1457,7 +1457,7 @@ void AudioClient::outputNotify() {
 bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo) {
     bool supportedFormat = false;
 
-    Lock lock(_localAudioMutex);
+    RecursiveLock lock(_localAudioMutex);
     _localSamplesAvailable.exchange(0, std::memory_order_release);
 
     // cleanup any previously initialized device
@@ -1671,7 +1671,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
 
     int injectorSamplesPopped = 0;
     {
-        Lock lock(_audio->_localAudioMutex);
+        RecursiveLock lock(_audio->_localAudioMutex);
         bool append = networkSamplesPopped > 0;
         samplesRequested = std::min(samplesRequested, _audio->_localSamplesAvailable.load(std::memory_order_acquire));
         if ((injectorSamplesPopped = _localInjectorsStream.appendSamples(mixBuffer, samplesRequested, append)) > 0) {

libraries/audio-client/src/AudioClient.h

@@ -94,6 +94,8 @@ public:
     using AudioPositionGetter = std::function<glm::vec3()>;
     using AudioOrientationGetter = std::function<glm::quat()>;
 
+    using RecursiveMutex = std::recursive_mutex;
+    using RecursiveLock = std::unique_lock<RecursiveMutex>;
     using Mutex = std::mutex;
     using Lock = std::unique_lock<Mutex>;
 
@@ -328,7 +330,7 @@ private:
     int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC];
     float* _localOutputMixBuffer { NULL };
     AudioInjectorsThread _localAudioThread;
-    Mutex _localAudioMutex;
+    RecursiveMutex _localAudioMutex;
 
     // for output audio (used by this thread)
     int _outputPeriod { 0 };

libraries/avatars/src/AvatarData.cpp

@@ -120,10 +120,6 @@ void AvatarData::nextAttitude(glm::vec3 position, glm::quat orientation) {
     updateAttitude();
 }
 
-float AvatarData::getTargetScale() const {
-    return _targetScale;
-}
-
 void AvatarData::setTargetScale(float targetScale) {
     auto newValue = glm::clamp(targetScale, MIN_AVATAR_SCALE, MAX_AVATAR_SCALE);
     if (_targetScale != newValue) {
@@ -141,10 +137,10 @@ void AvatarData::setHandPosition(const glm::vec3& handPosition) {
     _handPosition = glm::inverse(getOrientation()) * (handPosition - getPosition());
 }
 
-void AvatarData::lazyInitHeadData() {
+void AvatarData::lazyInitHeadData() const {
     // lazily allocate memory for HeadData in case we're not an Avatar instance
     if (!_headData) {
-        _headData = new HeadData(this);
+        _headData = new HeadData(const_cast<AvatarData*>(this));
     }
     if (_forceFaceTrackerConnected) {
         _headData->_isFaceTrackerConnected = true;
@@ -152,39 +148,7 @@ void AvatarData::lazyInitHeadData() {
 }
 
 
-bool AvatarData::avatarBoundingBoxChangedSince(quint64 time) {
+float AvatarData::getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) const {
-    return _avatarBoundingBoxChanged >= time;
-}
-
-bool AvatarData::avatarScaleChangedSince(quint64 time) {
-    return _avatarScaleChanged >= time;
-}
-
-bool AvatarData::lookAtPositionChangedSince(quint64 time) {
-    return _headData->lookAtPositionChangedSince(time);
-}
-
-bool AvatarData::audioLoudnessChangedSince(quint64 time) {
-    return _headData->audioLoudnessChangedSince(time);
-}
-
-bool AvatarData::sensorToWorldMatrixChangedSince(quint64 time) {
-    return _sensorToWorldMatrixChanged >= time;
-}
-
-bool AvatarData::additionalFlagsChangedSince(quint64 time) {
-    return _additionalFlagsChanged >= time;
-}
-
-bool AvatarData::parentInfoChangedSince(quint64 time) {
-    return _parentChanged >= time;
-}
-
-bool AvatarData::faceTrackerInfoChangedSince(quint64 time) {
-    return true; // FIXME!
-}
-
-float AvatarData::getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) {
     auto distance = glm::distance(_globalPosition, viewerPosition);
     float result = ROTATION_CHANGE_179D; // assume worst
     if (distance < AVATAR_DISTANCE_LEVEL_1) {
@@ -199,20 +163,24 @@ float AvatarData::getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) {
     return result;
 }
 
-float AvatarData::getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition) {
+float AvatarData::getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition) const {
     return AVATAR_MIN_TRANSLATION; // Eventually make this distance sensitive as well
 }
 
 
-QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
+// we want to track outbound data in this case...
-                        bool distanceAdjust, glm::vec3 viewerPosition, QVector<JointData>* sentJointDataOut) {
+QByteArray AvatarData::toByteArrayStateful(AvatarDataDetail dataDetail) {
+    AvatarDataPacket::HasFlags hasFlagsOut;
+    auto lastSentTime = _lastToByteArray;
+    _lastToByteArray = usecTimestampNow();
+    return AvatarData::toByteArray(dataDetail, lastSentTime, getLastSentJointData(), 
+                        hasFlagsOut, false, false, glm::vec3(0), nullptr,
+                        &_outboundDataRate);
+}
 
-    // if no timestamp was included, then assume the avatarData is single instance 
+QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
-    // and is tracking its own last encoding time.
+    AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, 
-    if (lastSentTime == 0) {
+    glm::vec3 viewerPosition, QVector<JointData>* sentJointDataOut, AvatarDataRate* outboundDataRateOut) const {
-        lastSentTime = _lastToByteArray;
-        _lastToByteArray = usecTimestampNow();
-    }
 
     bool cullSmallChanges = (dataDetail == CullSmallData);
     bool sendAll = (dataDetail == SendAllData);
@@ -259,26 +227,26 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
     // separately
     bool hasParentInfo = sendAll || parentInfoChangedSince(lastSentTime);
     bool hasAvatarLocalPosition = hasParent() && (sendAll ||
-                                                  tranlationChangedSince(lastSentTime) ||
+        tranlationChangedSince(lastSentTime) ||
-                                                  parentInfoChangedSince(lastSentTime));
+        parentInfoChangedSince(lastSentTime));
 
-    bool hasFaceTrackerInfo = hasFaceTracker() && (sendAll || faceTrackerInfoChangedSince(lastSentTime));
+    bool hasFaceTrackerInfo = !dropFaceTracking && hasFaceTracker() && (sendAll || faceTrackerInfoChangedSince(lastSentTime));
     bool hasJointData = sendAll || !sendMinimum;
 
     // Leading flags, to indicate how much data is actually included in the packet...
     AvatarDataPacket::HasFlags packetStateFlags =
-            (hasAvatarGlobalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_GLOBAL_POSITION : 0)
+        (hasAvatarGlobalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_GLOBAL_POSITION : 0)
-          | (hasAvatarBoundingBox    ? AvatarDataPacket::PACKET_HAS_AVATAR_BOUNDING_BOX : 0)
+        | (hasAvatarBoundingBox ? AvatarDataPacket::PACKET_HAS_AVATAR_BOUNDING_BOX : 0)
-          | (hasAvatarOrientation    ? AvatarDataPacket::PACKET_HAS_AVATAR_ORIENTATION : 0)
+        | (hasAvatarOrientation ? AvatarDataPacket::PACKET_HAS_AVATAR_ORIENTATION : 0)
-          | (hasAvatarScale          ? AvatarDataPacket::PACKET_HAS_AVATAR_SCALE : 0)
+        | (hasAvatarScale ? AvatarDataPacket::PACKET_HAS_AVATAR_SCALE : 0)
-          | (hasLookAtPosition       ? AvatarDataPacket::PACKET_HAS_LOOK_AT_POSITION : 0)
+        | (hasLookAtPosition ? AvatarDataPacket::PACKET_HAS_LOOK_AT_POSITION : 0)
-          | (hasAudioLoudness        ? AvatarDataPacket::PACKET_HAS_AUDIO_LOUDNESS : 0)
+        | (hasAudioLoudness ? AvatarDataPacket::PACKET_HAS_AUDIO_LOUDNESS : 0)
-          | (hasSensorToWorldMatrix  ? AvatarDataPacket::PACKET_HAS_SENSOR_TO_WORLD_MATRIX : 0)
+        | (hasSensorToWorldMatrix ? AvatarDataPacket::PACKET_HAS_SENSOR_TO_WORLD_MATRIX : 0)
-          | (hasAdditionalFlags      ? AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS : 0)
+        | (hasAdditionalFlags ? AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS : 0)
-          | (hasParentInfo           ? AvatarDataPacket::PACKET_HAS_PARENT_INFO : 0)
+        | (hasParentInfo ? AvatarDataPacket::PACKET_HAS_PARENT_INFO : 0)
-          | (hasAvatarLocalPosition  ? AvatarDataPacket::PACKET_HAS_AVATAR_LOCAL_POSITION : 0)
+        | (hasAvatarLocalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_LOCAL_POSITION : 0)
-          | (hasFaceTrackerInfo      ? AvatarDataPacket::PACKET_HAS_FACE_TRACKER_INFO : 0)
+        | (hasFaceTrackerInfo ? AvatarDataPacket::PACKET_HAS_FACE_TRACKER_INFO : 0)
-          | (hasJointData            ? AvatarDataPacket::PACKET_HAS_JOINT_DATA : 0);
+        | (hasJointData ? AvatarDataPacket::PACKET_HAS_JOINT_DATA : 0);
 
     memcpy(destinationBuffer, &packetStateFlags, sizeof(packetStateFlags));
     destinationBuffer += sizeof(packetStateFlags);
@@ -293,7 +261,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
 
         int numBytes = destinationBuffer - startSection;
 
-        _globalPositionRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->globalPositionRate.increment(numBytes);
+        }
     }
 
     if (hasAvatarBoundingBox) {
@@ -311,7 +281,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::AvatarBoundingBox);
 
         int numBytes = destinationBuffer - startSection;
-        _avatarBoundingBoxRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->avatarBoundingBoxRate.increment(numBytes);
+        }
     }
 
     if (hasAvatarOrientation) {
@@ -320,7 +292,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, localOrientation);
 
         int numBytes = destinationBuffer - startSection;
-        _avatarOrientationRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->avatarOrientationRate.increment(numBytes);
+        }
     }
 
     if (hasAvatarScale) {
@@ -331,7 +305,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::AvatarScale);
 
         int numBytes = destinationBuffer - startSection;
-        _avatarScaleRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->avatarScaleRate.increment(numBytes);
+        }
     }
 
     if (hasLookAtPosition) {
@@ -344,7 +320,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::LookAtPosition);
 
         int numBytes = destinationBuffer - startSection;
-        _lookAtPositionRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->lookAtPositionRate.increment(numBytes);
+        }
     }
 
     if (hasAudioLoudness) {
@@ -354,7 +332,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::AudioLoudness);
 
         int numBytes = destinationBuffer - startSection;
-        _audioLoudnessRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->audioLoudnessRate.increment(numBytes);
+        }
     }
 
     if (hasSensorToWorldMatrix) {
@@ -370,7 +350,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::SensorToWorldMatrix);
 
         int numBytes = destinationBuffer - startSection;
-        _sensorToWorldRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->sensorToWorldRate.increment(numBytes);
+        }
     }
 
     if (hasAdditionalFlags) {
@@ -403,7 +385,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::AdditionalFlags);
 
         int numBytes = destinationBuffer - startSection;
-        _additionalFlagsRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->additionalFlagsRate.increment(numBytes);
+        }
     }
 
     if (hasParentInfo) {
@@ -415,7 +399,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::ParentInfo);
 
         int numBytes = destinationBuffer - startSection;
-        _parentInfoRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->parentInfoRate.increment(numBytes);
+        }
     }
 
     if (hasAvatarLocalPosition) {
@@ -428,7 +414,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::AvatarLocalPosition);
 
         int numBytes = destinationBuffer - startSection;
-        _localPositionRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->localPositionRate.increment(numBytes);
+        }
     }
 
     // If it is connected, pack up the data
@@ -448,7 +436,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float);
 
         int numBytes = destinationBuffer - startSection;
-        _faceTrackerRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->faceTrackerRate.increment(numBytes);
+        }
     }
 
     // If it is connected, pack up the data
@@ -540,9 +530,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
                     glm::distance(data.translation, lastSentJointData[i].translation) > minTranslation) {
                     if (data.translationSet) {
                         validity |= (1 << validityBit);
-                        #ifdef WANT_DEBUG
+#ifdef WANT_DEBUG
                         translationSentCount++;
-                        #endif
+#endif
                         maxTranslationDimension = glm::max(fabsf(data.translation.x), maxTranslationDimension);
                         maxTranslationDimension = glm::max(fabsf(data.translation.y), maxTranslationDimension);
                         maxTranslationDimension = glm::max(fabsf(data.translation.z), maxTranslationDimension);
@@ -592,24 +582,25 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
 #ifdef WANT_DEBUG
         if (sendAll) {
             qCDebug(avatars) << "AvatarData::toByteArray" << cullSmallChanges << sendAll
-                    << "rotations:" << rotationSentCount << "translations:" << translationSentCount
+                << "rotations:" << rotationSentCount << "translations:" << translationSentCount
-                    << "largest:" << maxTranslationDimension
+                << "largest:" << maxTranslationDimension
-                    << "size:"
+                << "size:"
-                    << (beforeRotations - startPosition) << "+"
+                << (beforeRotations - startPosition) << "+"
-                    << (beforeTranslations - beforeRotations) << "+"
+                << (beforeTranslations - beforeRotations) << "+"
-                    << (destinationBuffer - beforeTranslations) << "="
+                << (destinationBuffer - beforeTranslations) << "="
-                    << (destinationBuffer - startPosition);
+                << (destinationBuffer - startPosition);
         }
 #endif
 
         int numBytes = destinationBuffer - startSection;
-        _jointDataRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->jointDataRate.increment(numBytes);
+        }
     }
 
     int avatarDataSize = destinationBuffer - startPosition;
     return avatarDataByteArray.left(avatarDataSize);
 }
-
 // NOTE: This is never used in a "distanceAdjust" mode, so it's ok that it doesn't use a variable minimum rotation/translation
 void AvatarData::doneEncoding(bool cullSmallChanges) {
     // The server has finished sending this version of the joint-data to other nodes.  Update _lastSentJointData.
@@ -1089,29 +1080,29 @@ float AvatarData::getDataRate(const QString& rateName) const {
     } else if (rateName == "jointData") {
         return _jointDataRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "globalPositionOutbound") {
-        return _globalPositionRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.globalPositionRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "localPositionOutbound") {
-        return _localPositionRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.localPositionRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "avatarBoundingBoxOutbound") {
-        return _avatarBoundingBoxRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.avatarBoundingBoxRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "avatarOrientationOutbound") {
-        return _avatarOrientationRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.avatarOrientationRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "avatarScaleOutbound") {
-        return _avatarScaleRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.avatarScaleRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "lookAtPositionOutbound") {
-        return _lookAtPositionRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.lookAtPositionRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "audioLoudnessOutbound") {
-        return _audioLoudnessRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.audioLoudnessRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "sensorToWorkMatrixOutbound") {
-        return _sensorToWorldRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.sensorToWorldRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "additionalFlagsOutbound") {
-        return _additionalFlagsRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.additionalFlagsRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "parentInfoOutbound") {
-        return _parentInfoRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.parentInfoRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "faceTrackerOutbound") {
-        return _faceTrackerRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.faceTrackerRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "jointDataOutbound") {
-        return _jointDataRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.jointDataRate.rate() / BYTES_PER_KILOBIT;
     }
     return 0.0f;
 }
@@ -1445,7 +1436,7 @@ void AvatarData::parseAvatarIdentityPacket(const QByteArray& data, Identity& ide
 }
 
 static const QUrl emptyURL("");
-const QUrl& AvatarData::cannonicalSkeletonModelURL(const QUrl& emptyURL) {
+QUrl AvatarData::cannonicalSkeletonModelURL(const QUrl& emptyURL) const {
     // We don't put file urls on the wire, but instead convert to empty.
     return _skeletonModelURL.scheme() == "file" ? emptyURL : _skeletonModelURL;
 }
@@ -1483,7 +1474,7 @@ void AvatarData::processAvatarIdentity(const Identity& identity, bool& identityC
     }
 }
 
-QByteArray AvatarData::identityByteArray() {
+QByteArray AvatarData::identityByteArray() const {
     QByteArray identityData;
     QDataStream identityStream(&identityData, QIODevice::Append);
     const QUrl& urlToSend = cannonicalSkeletonModelURL(emptyURL);
@@ -1646,13 +1637,7 @@ void AvatarData::sendAvatarDataPacket() {
 
     bool cullSmallData = (randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO);
     auto dataDetail = cullSmallData ? SendAllData : CullSmallData;
-    QVector<JointData> lastSentJointData;
+    QByteArray avatarByteArray = toByteArrayStateful(dataDetail);
-    {
-        QReadLocker readLock(&_jointDataLock);
-        _lastSentJointData.resize(_jointData.size());
-        lastSentJointData = _lastSentJointData;
-    } 
-    QByteArray avatarByteArray = toByteArray(dataDetail, 0, lastSentJointData);
     doneEncoding(cullSmallData);
 
     static AvatarDataSequenceNumber sequenceNumber = 0;

libraries/avatars/src/AvatarData.h

@@ -288,6 +288,23 @@ class AttachmentData;
 class Transform;
 using TransformPointer = std::shared_ptr<Transform>;
 
+class AvatarDataRate {
+public:
+    RateCounter<> globalPositionRate;
+    RateCounter<> localPositionRate;
+    RateCounter<> avatarBoundingBoxRate;
+    RateCounter<> avatarOrientationRate;
+    RateCounter<> avatarScaleRate;
+    RateCounter<> lookAtPositionRate;
+    RateCounter<> audioLoudnessRate;
+    RateCounter<> sensorToWorldRate;
+    RateCounter<> additionalFlagsRate;
+    RateCounter<> parentInfoRate;
+    RateCounter<> faceTrackerRate;
+    RateCounter<> jointDataRate;
+};
+
+
 class AvatarData : public QObject, public SpatiallyNestable {
     Q_OBJECT
 
@@ -351,8 +368,11 @@ public:
         SendAllData
     } AvatarDataDetail;
 
+    virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail);
+
     virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
-                        bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector<JointData>* sentJointDataOut = nullptr);
+        AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, 
+        QVector<JointData>* sentJointDataOut, AvatarDataRate* outboundDataRateOut = nullptr) const;
 
     virtual void doneEncoding(bool cullSmallChanges);
 
@@ -380,7 +400,7 @@ public:
     void nextAttitude(glm::vec3 position, glm::quat orientation); // Can be safely called at any time.
     virtual void updateAttitude() {} // Tell skeleton mesh about changes
 
-    glm::quat getHeadOrientation() { 
+    glm::quat getHeadOrientation() const { 
         lazyInitHeadData();
         return _headData->getOrientation(); 
     }
@@ -419,7 +439,6 @@ public:
     void setAudioAverageLoudness(float value) { _headData->setAudioAverageLoudness(value); }
 
     //  Scale
-    float getTargetScale() const;
     virtual void setTargetScale(float targetScale);
 
     float getDomainLimitedScale() const { return glm::clamp(_targetScale, _domainMinimumScale, _domainMaximumScale); }
@@ -494,7 +513,7 @@ public:
     // displayNameChanged returns true if displayName has changed, false otherwise.
     void processAvatarIdentity(const Identity& identity, bool& identityChanged, bool& displayNameChanged);
 
-    QByteArray identityByteArray();
+    QByteArray identityByteArray() const;
 
     const QUrl& getSkeletonModelURL() const { return _skeletonModelURL; }
     const QString& getDisplayName() const { return _displayName; }
@@ -534,8 +553,8 @@ public:
     QJsonObject toJson() const;
     void fromJson(const QJsonObject& json, bool useFrameSkeleton = true);
 
-    glm::vec3 getClientGlobalPosition() { return _globalPosition; }
+    glm::vec3 getClientGlobalPosition() const { return _globalPosition; }
-    glm::vec3 getGlobalBoundingBoxCorner() { return _globalPosition + _globalBoundingBoxOffset - _globalBoundingBoxDimensions; }
+    glm::vec3 getGlobalBoundingBoxCorner() const { return _globalPosition + _globalBoundingBoxOffset - _globalBoundingBoxDimensions; }
 
     Q_INVOKABLE AvatarEntityMap getAvatarEntityData() const;
     Q_INVOKABLE void setAvatarEntityData(const AvatarEntityMap& avatarEntityData);
@@ -550,7 +569,7 @@ public:
     Q_INVOKABLE float getDataRate(const QString& rateName = QString("")) const;
     Q_INVOKABLE float getUpdateRate(const QString& rateName = QString("")) const;
 
-    int getJointCount() { return _jointData.size(); }
+    int getJointCount() const { return _jointData.size(); }
 
     QVector<JointData> getLastSentJointData() {
         QReadLocker readLock(&_jointDataLock);
@@ -571,28 +590,27 @@ public slots:
     virtual bool setAbsoluteJointRotationInObjectFrame(int index, const glm::quat& rotation) override { return false; }
     virtual bool setAbsoluteJointTranslationInObjectFrame(int index, const glm::vec3& translation) override { return false; }
 
-    float getTargetScale() { return _targetScale; }
+    float getTargetScale() const { return _targetScale; } // why is this a slot?
 
     void resetLastSent() { _lastToByteArray = 0; }
 
 protected:
-    void lazyInitHeadData();
+    void lazyInitHeadData() const;
 
-    float getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition);
+    float getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) const;
-    float getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition);
+    float getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition) const;
 
-    bool avatarBoundingBoxChangedSince(quint64 time);
+    bool avatarBoundingBoxChangedSince(quint64 time) const { return _avatarBoundingBoxChanged >= time; }
-    bool avatarScaleChangedSince(quint64 time);
+    bool avatarScaleChangedSince(quint64 time) const { return _avatarScaleChanged >= time; }
-    bool lookAtPositionChangedSince(quint64 time);
+    bool lookAtPositionChangedSince(quint64 time) const { return _headData->lookAtPositionChangedSince(time); }
-    bool audioLoudnessChangedSince(quint64 time);
+    bool audioLoudnessChangedSince(quint64 time) const { return _headData->audioLoudnessChangedSince(time); }
-    bool sensorToWorldMatrixChangedSince(quint64 time);
+    bool sensorToWorldMatrixChangedSince(quint64 time) const { return _sensorToWorldMatrixChanged >= time; }
-    bool additionalFlagsChangedSince(quint64 time);
+    bool additionalFlagsChangedSince(quint64 time) const { return _additionalFlagsChanged >= time; }
+    bool parentInfoChangedSince(quint64 time) const { return _parentChanged >= time; }
+    bool faceTrackerInfoChangedSince(quint64 time) const { return true; } // FIXME
 
-    bool hasParent() { return !getParentID().isNull(); }
+    bool hasParent() const { return !getParentID().isNull(); }
-    bool parentInfoChangedSince(quint64 time);
+    bool hasFaceTracker() const { return _headData ? _headData->_isFaceTrackerConnected : false; }
-
-    bool hasFaceTracker() { return _headData ? _headData->_isFaceTrackerConnected : false; }
-    bool faceTrackerInfoChangedSince(quint64 time);
 
     glm::vec3 _handPosition;
     virtual const QString& getSessionDisplayNameForTransport() const { return _sessionDisplayName; }
@@ -616,7 +634,7 @@ protected:
     bool _forceFaceTrackerConnected;
     bool _hasNewJointData { true }; // set in AvatarData, cleared in Avatar
 
-    HeadData* _headData { nullptr };
+    mutable HeadData* _headData { nullptr };
 
     QUrl _skeletonModelURL;
     bool _firstSkeletonCheck { true };
@@ -624,7 +642,7 @@ protected:
     QVector<AttachmentData> _attachmentData;
     QString _displayName;
     QString _sessionDisplayName { };
-    const QUrl& cannonicalSkeletonModelURL(const QUrl& empty);
+    QUrl cannonicalSkeletonModelURL(const QUrl& empty) const;
 
     float _displayNameTargetAlpha;
     float _displayNameAlpha;
@@ -695,18 +713,7 @@ protected:
     RateCounter<> _jointDataUpdateRate;
 
     // Some rate data for outgoing data
-    RateCounter<> _globalPositionRateOutbound;
+    AvatarDataRate _outboundDataRate;
-    RateCounter<> _localPositionRateOutbound;
-    RateCounter<> _avatarBoundingBoxRateOutbound;
-    RateCounter<> _avatarOrientationRateOutbound;
-    RateCounter<> _avatarScaleRateOutbound;
-    RateCounter<> _lookAtPositionRateOutbound;
-    RateCounter<> _audioLoudnessRateOutbound;
-    RateCounter<> _sensorToWorldRateOutbound;
-    RateCounter<> _additionalFlagsRateOutbound;
-    RateCounter<> _parentInfoRateOutbound;
-    RateCounter<> _faceTrackerRateOutbound;
-    RateCounter<> _jointDataRateOutbound;
 
     glm::vec3 _globalBoundingBoxDimensions;
     glm::vec3 _globalBoundingBoxOffset;

libraries/entities/src/EntityNodeData.cpp

@@ -0,0 +1,30 @@
+//
+//  EntityNodeData.cpp
+//  libraries/entities/src
+//
+//  Created by Stephen Birarda on 2/15/17
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "EntityNodeData.h"
+
+bool EntityNodeData::insertFlaggedExtraEntity(const QUuid& filteredEntityID, const QUuid& extraEntityID) {
+    _flaggedExtraEntities[filteredEntityID].insert(extraEntityID);
+    return !_previousFlaggedExtraEntities[filteredEntityID].contains(extraEntityID);
+}
+
+bool EntityNodeData::isEntityFlaggedAsExtra(const QUuid& entityID) const {
+
+    // enumerate each of the sets for the entities that matched our filter
+    // and immediately return true if any of them contain this entity ID
+    foreach(QSet<QUuid> entitySet, _flaggedExtraEntities) {
+        if (entitySet.contains(entityID)) {
+            return true;
+        }
+    }
+
+    return false;
+}

libraries/entities/src/EntityNodeData.h

@@ -1,6 +1,6 @@
 //
 //  EntityNodeData.h
-//  assignment-client/src/entities
+//  libraries/entities/src
 //
 //  Created by Brad Hefta-Gaub on 4/29/14
 //  Copyright 2014 High Fidelity, Inc.
@@ -16,6 +16,13 @@
 
 #include <OctreeQueryNode.h>
 
+namespace EntityJSONQueryProperties {
+    static const QString SERVER_SCRIPTS_PROPERTY = "serverScripts";
+    static const QString FLAGS_PROPERTY = "flags";
+    static const QString INCLUDE_ANCESTORS_PROPERTY = "includeAncestors";
+    static const QString INCLUDE_DESCENDANTS_PROPERTY = "includeDescendants";
+}
+
 class EntityNodeData : public OctreeQueryNode {
 public:
     virtual PacketType getMyPacketType() const override { return PacketType::EntityData; }
@@ -24,13 +31,24 @@ public:
     void setLastDeletedEntitiesSentAt(quint64 sentAt) { _lastDeletedEntitiesSentAt = sentAt; }
     
     // these can only be called from the OctreeSendThread for the given Node
-    void insertEntitySentLastFrame(const QUuid& entityID) { _entitiesSentLastFrame.insert(entityID); }
+    void insertSentFilteredEntity(const QUuid& entityID) { _sentFilteredEntities.insert(entityID); }
-    void removeEntitySentLastFrame(const QUuid& entityID) { _entitiesSentLastFrame.remove(entityID); }
+    void removeSentFilteredEntity(const QUuid& entityID) { _sentFilteredEntities.remove(entityID); }
-    bool sentEntityLastFrame(const QUuid& entityID) { return _entitiesSentLastFrame.contains(entityID); }
+    bool sentFilteredEntity(const QUuid& entityID) { return _sentFilteredEntities.contains(entityID); }
+    QSet<QUuid> getSentFilteredEntities() { return _sentFilteredEntities; }
+
+    // the following flagged extra entity methods can only be called from the OctreeSendThread for the given Node
+
+    // inserts the extra entity and returns a boolean indicating wether the extraEntityID was a new addition
+    bool insertFlaggedExtraEntity(const QUuid& filteredEntityID, const QUuid& extraEntityID);
+    
+    bool isEntityFlaggedAsExtra(const QUuid& entityID) const;
+    void resetFlaggedExtraEntities() { _previousFlaggedExtraEntities = _flaggedExtraEntities; _flaggedExtraEntities.clear(); }
 
 private:
     quint64 _lastDeletedEntitiesSentAt { usecTimestampNow() };
-    QSet<QUuid> _entitiesSentLastFrame;
+    QSet<QUuid> _sentFilteredEntities;
+    QHash<QUuid, QSet<QUuid>> _flaggedExtraEntities;
+    QHash<QUuid, QSet<QUuid>> _previousFlaggedExtraEntities;
 };
 
 #endif // hifi_EntityNodeData_h

libraries/entities/src/EntityTreeElement.cpp

@@ -310,16 +310,17 @@ OctreeElement::AppendState EntityTreeElement::appendElementData(OctreePacketData
 
                     if (entityMatchesFilters) {
                         // make sure this entity is in the set of entities sent last frame
-                        entityNodeData->insertEntitySentLastFrame(entity->getID());
+                        entityNodeData->insertSentFilteredEntity(entity->getID());
-
+                    } else if (entityNodeData->sentFilteredEntity(entity->getID())) {
-                    } else {
+                        // this entity matched in the previous frame - we send it still so the client realizes it just
-                        // we might include this entity if it matched in the previous frame
+                        // fell outside of their filter
-                        if (entityNodeData->sentEntityLastFrame(entity->getID())) {
+                        entityNodeData->removeSentFilteredEntity(entity->getID());
-
+                    } else if (!entityNodeData->isEntityFlaggedAsExtra(entity->getID())) {
-                            entityNodeData->removeEntitySentLastFrame(entity->getID());
+                        // we don't send this entity because
-                        } else {
+                        // (1) it didn't match our filter
-                            includeThisEntity = false;
+                        // (2) it didn't match our filter last frame
-                        }
+                        // (3) it isn't one the JSON query flags told us we should still include
+                        includeThisEntity = false;
                     }
                 }
 

libraries/gpu-gl/src/gpu/gl45/GL45BackendTexture.cpp

@@ -142,8 +142,12 @@ void GL45Texture::copyMipFaceFromTexture(uint16_t sourceMip, uint16_t targetMip,
     }
     auto size = _gpuObject.evalMipDimensions(sourceMip);
     auto mipData = _gpuObject.accessStoredMipFace(sourceMip, face);
-    GLTexelFormat texelFormat = GLTexelFormat::evalGLTexelFormat(_gpuObject.getTexelFormat(), _gpuObject.getStoredMipFormat());
+    if (mipData) {
-    copyMipFaceLinesFromTexture(targetMip, face, size, 0, texelFormat.format, texelFormat.type, mipData->readData());
+        GLTexelFormat texelFormat = GLTexelFormat::evalGLTexelFormat(_gpuObject.getTexelFormat(), _gpuObject.getStoredMipFormat());
+        copyMipFaceLinesFromTexture(targetMip, face, size, 0, texelFormat.format, texelFormat.type, mipData->readData());
+    } else {
+        qCDebug(gpugllogging) << "Missing mipData level=" << sourceMip << " face=" << (int)face << " for texture " << _gpuObject.source().c_str();
+    }
 }
 
 void GL45Texture::syncSampler() const {

libraries/gpu/src/gpu/Texture_ktx.cpp

@@ -24,8 +24,12 @@ ktx::KTXUniquePointer Texture::serialize(const Texture& texture) {
 
     if (texelFormat == Format::COLOR_RGBA_32 && mipFormat == Format::COLOR_BGRA_32) {
         header.setUncompressed(ktx::GLType::UNSIGNED_BYTE, 4, ktx::GLFormat::BGRA, ktx::GLInternalFormat_Uncompressed::RGBA8, ktx::GLBaseInternalFormat::RGBA);
-    } else if (texelFormat == Format::COLOR_SRGBA_32 && mipFormat == Format::COLOR_BGRA_32) {
+    } else if (texelFormat == Format::COLOR_RGBA_32 && mipFormat == Format::COLOR_RGBA_32) {
+        header.setUncompressed(ktx::GLType::UNSIGNED_BYTE, 4, ktx::GLFormat::RGBA, ktx::GLInternalFormat_Uncompressed::RGBA8, ktx::GLBaseInternalFormat::RGBA);
+    } else if (texelFormat == Format::COLOR_SRGBA_32 && mipFormat == Format::COLOR_SBGRA_32) {
         header.setUncompressed(ktx::GLType::UNSIGNED_BYTE, 4, ktx::GLFormat::BGRA, ktx::GLInternalFormat_Uncompressed::SRGB8_ALPHA8, ktx::GLBaseInternalFormat::RGBA);
+    } else if (texelFormat == Format::COLOR_SRGBA_32 && mipFormat == Format::COLOR_SRGBA_32) {
+        header.setUncompressed(ktx::GLType::UNSIGNED_BYTE, 4, ktx::GLFormat::RGBA, ktx::GLInternalFormat_Uncompressed::SRGB8_ALPHA8, ktx::GLBaseInternalFormat::RGBA);
     } else if (texelFormat == Format::COLOR_R_8 && mipFormat == Format::COLOR_R_8) {
         header.setUncompressed(ktx::GLType::UNSIGNED_BYTE, 1, ktx::GLFormat::RED, ktx::GLInternalFormat_Uncompressed::R8, ktx::GLBaseInternalFormat::RED);
     } else {
@@ -127,10 +131,21 @@ Texture* Texture::unserialize(Usage usage, TextureUsageType usageType, const ktx
     Format texelFormat = Format::COLOR_SRGBA_32;
 
     if (header.getGLFormat() == ktx::GLFormat::BGRA && header.getGLType() == ktx::GLType::UNSIGNED_BYTE && header.getTypeSize() == 4) {
-        mipFormat = Format::COLOR_BGRA_32;
         if (header.getGLInternaFormat_Uncompressed() == ktx::GLInternalFormat_Uncompressed::RGBA8) {
+            mipFormat = Format::COLOR_BGRA_32;
             texelFormat = Format::COLOR_RGBA_32;
         } else if (header.getGLInternaFormat_Uncompressed() == ktx::GLInternalFormat_Uncompressed::SRGB8_ALPHA8) {
+            mipFormat = Format::COLOR_SBGRA_32;
+            texelFormat = Format::COLOR_SRGBA_32;
+        } else {
+            return nullptr;
+        }
+    } else if (header.getGLFormat() == ktx::GLFormat::RGBA && header.getGLType() == ktx::GLType::UNSIGNED_BYTE && header.getTypeSize() == 4) {
+        if (header.getGLInternaFormat_Uncompressed() == ktx::GLInternalFormat_Uncompressed::RGBA8) {
+            mipFormat = Format::COLOR_RGBA_32;
+            texelFormat = Format::COLOR_RGBA_32;
+        } else if (header.getGLInternaFormat_Uncompressed() == ktx::GLInternalFormat_Uncompressed::SRGB8_ALPHA8) {
+            mipFormat = Format::COLOR_SRGBA_32;
             texelFormat = Format::COLOR_SRGBA_32;
         } else {
             return nullptr;

libraries/ktx/src/ktx/KTX.cpp

@@ -34,10 +34,6 @@ uint32_t Header::evalMaxDimension() const {
     return std::max(getPixelWidth(), std::max(getPixelHeight(), getPixelDepth()));
 }
 
-uint32_t Header::evalMaxLevel() const {
-    return 1 + log2(evalMaxDimension());
-}
-
 uint32_t Header::evalPixelWidth(uint32_t level) const {
     return std::max(getPixelWidth() >> level, 1U);
 }

libraries/ktx/src/ktx/KTX.h

@@ -294,7 +294,7 @@ namespace ktx {
     };
 
     using Storage = storage::Storage;
-    using StoragePointer = std::unique_ptr<Storage>;
+    using StoragePointer = std::shared_ptr<Storage>;
 
     // Header
     struct Header {
@@ -334,7 +334,6 @@ namespace ktx {
         uint32_t getNumberOfLevels() const { return (numberOfMipmapLevels ? numberOfMipmapLevels : 1); }
 
         uint32_t evalMaxDimension() const;
-        uint32_t evalMaxLevel() const;
         uint32_t evalPixelWidth(uint32_t level) const;
         uint32_t evalPixelHeight(uint32_t level) const;
         uint32_t evalPixelDepth(uint32_t level) const;

libraries/ktx/src/ktx/Reader.cpp

@@ -149,10 +149,10 @@ namespace ktx {
                         faces[face] = currentPtr;
                         currentPtr += faceSize;
                     }
-                    images.emplace_back(Image(faceSize, padding, faces));
+                    images.emplace_back(Image((uint32_t) faceSize, padding, faces));
                     currentPtr += padding;
                 } else {
-                    images.emplace_back(Image(imageSize, padding, currentPtr));
+                    images.emplace_back(Image((uint32_t) imageSize, padding, currentPtr));
                     currentPtr += imageSize + padding;
                 }
             } else {

libraries/ktx/src/ktx/Writer.cpp

@@ -95,7 +95,7 @@ namespace ktx {
         for (uint32_t l = 0; l < srcImages.size(); l++) {
             if (currentDataSize + sizeof(uint32_t) < allocatedImagesDataSize) {
                 size_t imageSize = srcImages[l]._imageSize;
-                *(reinterpret_cast<uint32_t*> (currentPtr)) = imageSize;
+                *(reinterpret_cast<uint32_t*> (currentPtr)) = (uint32_t) imageSize;
                 currentPtr += sizeof(uint32_t);
                 currentDataSize += sizeof(uint32_t);
 
@@ -106,7 +106,7 @@ namespace ktx {
                     // Single face vs cubes
                     if (srcImages[l]._numFaces == 1) {
                         auto copied = memcpy(currentPtr, srcImages[l]._faceBytes[0], imageSize);
-                        destImages.emplace_back(Image(imageSize, padding, currentPtr));
+                        destImages.emplace_back(Image((uint32_t) imageSize, padding, currentPtr));
                         currentPtr += imageSize;
                     } else {
                         Image::FaceBytes faceBytes(6);

libraries/model/src/model/TextureMap.cpp

@@ -85,7 +85,7 @@ QImage processSourceImage(const QImage& srcImage, bool cubemap) {
     return srcImage;
 }
 
-gpu::Texture* cacheTexture(const std::string& name, gpu::Texture* srcTexture, bool write = true, bool read = true) {
+gpu::Texture* cacheTexture(const std::string& name, gpu::Texture* srcTexture, bool write = true, bool read = false) { // FIXME: set read to false for a working state
     if (!srcTexture) {
         return nullptr;
     }
@@ -103,10 +103,10 @@ gpu::Texture* cacheTexture(const std::string& name, gpu::Texture* srcTexture, bo
         }
     });
 
-
     std::string cleanedName = name;
-    cleanedName = cleanedName.substr(cleanedName.find_last_of('//') + 1);
+    cleanedName = cleanedName.substr(cleanedName.find_last_of((char) '//') + 1);
     std::string cacheFilename(ktxCacheFolder.toStdString());
+    cacheFilename += "/";
     cacheFilename += cleanedName;
     cacheFilename += ".ktx";
 
@@ -368,8 +368,8 @@ gpu::Texture* TextureUsage::createNormalTextureFromNormalImage(const QImage& src
     gpu::Texture* theTexture = nullptr;
     if ((image.width() > 0) && (image.height() > 0)) {
 
-        gpu::Element formatGPU = gpu::Element(gpu::VEC4, gpu::NUINT8, gpu::RGBA);
+        gpu::Element formatMip = gpu::Element::COLOR_RGBA_32;
-        gpu::Element formatMip = gpu::Element(gpu::VEC4, gpu::NUINT8, gpu::RGBA);
+        gpu::Element formatGPU = gpu::Element::COLOR_RGBA_32;
 
         theTexture = (gpu::Texture::create2D(formatGPU, image.width(), image.height(), gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_MIP_LINEAR)));
         theTexture->setSource(srcImageName);
@@ -378,7 +378,7 @@ gpu::Texture* TextureUsage::createNormalTextureFromNormalImage(const QImage& src
         generateMips(theTexture, image, true);
 
         theTexture->setSource(srcImageName);
-        theTexture = cacheTexture(theTexture->source(), theTexture);
+        theTexture = cacheTexture(theTexture->source(), theTexture, true, false);
     }
 
     return theTexture;
@@ -458,8 +458,9 @@ gpu::Texture* TextureUsage::createNormalTextureFromBumpImage(const QImage& srcIm
     gpu::Texture* theTexture = nullptr;
     if ((image.width() > 0) && (image.height() > 0)) {
 
-        gpu::Element formatGPU = gpu::Element(gpu::VEC4, gpu::NUINT8, gpu::RGBA);
+
-        gpu::Element formatMip = gpu::Element(gpu::VEC4, gpu::NUINT8, gpu::RGBA);
+        gpu::Element formatMip = gpu::Element::COLOR_RGBA_32;
+        gpu::Element formatGPU = gpu::Element::COLOR_RGBA_32;
 
         theTexture = (gpu::Texture::create2D(formatGPU, image.width(), image.height(), gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_MIP_LINEAR)));
         theTexture->setSource(srcImageName);
@@ -468,7 +469,7 @@ gpu::Texture* TextureUsage::createNormalTextureFromBumpImage(const QImage& srcIm
         generateMips(theTexture, image, true);
 
         theTexture->setSource(srcImageName);
-        theTexture = cacheTexture(theTexture->source(), theTexture);
+        theTexture = cacheTexture(theTexture->source(), theTexture, true, false);
     }
 
     return theTexture;

libraries/networking/src/LimitedNodeList.h

@@ -34,6 +34,7 @@
 #include <tbb/concurrent_unordered_map.h>
 
 #include <DependencyManager.h>
+#include <SharedUtil.h>
 
 #include "DomainHandler.h"
 #include "Node.h"
@@ -182,15 +183,33 @@ public:
     //   This allows multiple threads (i.e. a thread pool) to share a lock
     //   without deadlocking when a dying node attempts to acquire a write lock
     template<typename NestedNodeLambda>
-    void nestedEach(NestedNodeLambda functor) {
+    void nestedEach(NestedNodeLambda functor, 
-        QReadLocker readLock(&_nodeMutex);
+                    int* lockWaitOut = nullptr, 
+                    int* nodeTransformOut = nullptr, 
+                    int* functorOut = nullptr) {
+        auto start = usecTimestampNow();
+        {
+            QReadLocker readLock(&_nodeMutex);
+            auto endLock = usecTimestampNow();
+            if (lockWaitOut) {
+                *lockWaitOut = (endLock - start);
+            }
 
-        std::vector<SharedNodePointer> nodes(_nodeHash.size());
+            std::vector<SharedNodePointer> nodes(_nodeHash.size());
-        std::transform(_nodeHash.cbegin(), _nodeHash.cend(), nodes.begin(), [](const NodeHash::value_type& it) {
+            std::transform(_nodeHash.cbegin(), _nodeHash.cend(), nodes.begin(), [](const NodeHash::value_type& it) {
-            return it.second;
+                return it.second;
-        });
+            });
+            auto endTransform = usecTimestampNow();
+            if (nodeTransformOut) {
+                *nodeTransformOut = (endTransform - endLock);
+            }
 
-        functor(nodes.cbegin(), nodes.cend());
+            functor(nodes.cbegin(), nodes.cend());
+            auto endFunctor = usecTimestampNow();
+            if (functorOut) {
+                *functorOut = (endFunctor - endTransform);
+            }
+        }
     }
 
     template<typename NodeLambda>

libraries/networking/src/udt/PacketHeaders.cpp

@@ -51,7 +51,7 @@ PacketVersion versionForPacketType(PacketType packetType) {
         case PacketType::EntityPhysics:
             return VERSION_ENTITIES_ZONE_FILTERS;
         case PacketType::EntityQuery:
-            return static_cast<PacketVersion>(EntityQueryPacketVersion::JsonFilter);
+            return static_cast<PacketVersion>(EntityQueryPacketVersion::JSONFilterWithFamilyTree);
         case PacketType::AvatarIdentity:
         case PacketType::AvatarData:
         case PacketType::BulkAvatarData:

libraries/networking/src/udt/PacketHeaders.h

@@ -207,7 +207,8 @@ const PacketVersion VERSION_ENTITIES_PHYSICS_PACKET = 67;
 const PacketVersion VERSION_ENTITIES_ZONE_FILTERS = 68;
 
 enum class EntityQueryPacketVersion: PacketVersion {
-    JsonFilter = 18
+    JSONFilter = 18,
+    JSONFilterWithFamilyTree = 19
 };
 
 enum class AssetServerPacketVersion: PacketVersion {

libraries/octree/src/OctreeQueryNode.h

@@ -103,6 +103,9 @@ public:
     // call only from OctreeSendThread for the given node
     bool haveJSONParametersChanged();
 
+    bool shouldForceFullScene() const { return _shouldForceFullScene; }
+    void setShouldForceFullScene(bool shouldForceFullScene) { _shouldForceFullScene = shouldForceFullScene; }
+
 private:
     OctreeQueryNode(const OctreeQueryNode &);
     OctreeQueryNode& operator= (const OctreeQueryNode&);
@@ -148,6 +151,8 @@ private:
     std::array<char, udt::MAX_PACKET_SIZE> _lastOctreePayload;
 
     QJsonObject _lastCheckJSONParameters;
+
+    bool _shouldForceFullScene { false };
 };
 
 #endif // hifi_OctreeQueryNode_h

libraries/render-utils/src/MaterialTextures.slh

@@ -64,7 +64,7 @@ float fetchRoughnessMap(vec2 uv) {
 uniform sampler2D normalMap;
 vec3 fetchNormalMap(vec2 uv) {
     // unpack normal, swizzle to get into hifi tangent space with Y axis pointing out
-    return normalize(texture(normalMap, uv).xzy -vec3(0.5, 0.5, 0.5));
+    return normalize(texture(normalMap, uv).rbg -vec3(0.5, 0.5, 0.5));
 }
 <@endif@>
 

libraries/script-engine/src/TabletScriptingInterface.cpp

@@ -172,7 +172,7 @@ static const char* WEB_VIEW_SOURCE_URL = "TabletWebView.qml";
 static const char* VRMENU_SOURCE_URL = "TabletMenu.qml";
 
 class TabletRootWindow : public QmlWindowClass {
-    virtual QString qmlSource() const { return "hifi/tablet/WindowRoot.qml"; }
+    virtual QString qmlSource() const override { return "hifi/tablet/WindowRoot.qml"; }
 };
 
 TabletProxy::TabletProxy(QString name) : _name(name) {

libraries/shared/src/SpatiallyNestable.cpp

@@ -70,6 +70,9 @@ void SpatiallyNestable::setParentID(const QUuid& parentID) {
             _parentKnowsMe = false;
         }
     });
+
+    bool success = false;
+    getParentPointer(success);
 }
 
 Transform SpatiallyNestable::getParentTransform(bool& success, int depth) const {

libraries/shared/src/SpatiallyNestable.h

@@ -179,9 +179,9 @@ public:
             const glm::vec3& localVelocity,
             const glm::vec3& localAngularVelocity);
 
-    bool scaleChangedSince(quint64 time) { return _scaleChanged > time; }
+    bool scaleChangedSince(quint64 time) const { return _scaleChanged > time; }
-    bool tranlationChangedSince(quint64 time) { return _translationChanged > time; }
+    bool tranlationChangedSince(quint64 time) const { return _translationChanged > time; }
-    bool rotationChangedSince(quint64 time) { return _rotationChanged > time; }
+    bool rotationChangedSince(quint64 time) const { return _rotationChanged > time; }
 
 protected:
     const NestableType _nestableType; // EntityItem or an AvatarData

libraries/shared/src/shared/Storage.cpp

@@ -57,7 +57,8 @@ FileStoragePointer FileStorage::create(const QString& filename, size_t size, con
         }
     }
     file.close();
-    return FileStoragePointer(new FileStorage(filename));
+    //return FileStoragePointer(new FileStorage(filename));
+    return std::make_shared<FileStorage>(filename);
 }
 
 FileStorage::FileStorage(const QString& filename) : _file(filename) {

scripts/system/controllers/handControllerGrab.js

@@ -1680,6 +1680,7 @@ function MyController(hand) {
             } else if (this.entityIsDistanceGrabbable(rayPickInfo.entityID, handPosition)) {
                 if (this.triggerSmoothedGrab() && !isEditing() && farGrabEnabled && farSearching) {
                     this.grabbedEntity = entity;
+                    this.grabbedDistance = rayPickInfo.distance;
                     this.setState(STATE_DISTANCE_HOLDING, "distance hold '" + name + "'");
                     return;
                 } else {
@@ -2006,7 +2007,7 @@ function MyController(hand) {
         this.currentObjectTime = now;
         this.currentCameraOrientation = Camera.orientation;
 
-        this.grabRadius = Vec3.distance(this.currentObjectPosition, worldControllerPosition);
+        this.grabRadius = this.grabbedDistance;
         this.grabRadialVelocity = 0.0;
 
         // offset between controller vector at the grab radius and the entity position
@@ -2160,7 +2161,7 @@ function MyController(hand) {
 
         var rayPickInfo = this.calcRayPickInfo(this.hand);
 
-        this.overlayLineOn(rayPickInfo.searchRay.origin, grabbedProperties.position, COLORS_GRAB_DISTANCE_HOLD);
+        this.overlayLineOn(rayPickInfo.searchRay.origin, Vec3.subtract(grabbedProperties.position, this.offsetPosition), COLORS_GRAB_DISTANCE_HOLD);
 
         var distanceToObject = Vec3.length(Vec3.subtract(MyAvatar.position, this.currentObjectPosition));
         var success = Entities.updateAction(this.grabbedEntity, this.actionID, {

scripts/system/libraries/WebTablet.js

@@ -159,7 +159,7 @@ WebTablet = function (url, width, dpi, hand, clientOnly) {
     });
 
     this.receive = function (channel, senderID, senderUUID, localOnly) {
-        if (_this.homeButtonEntity === parseInt(senderID)) {
+        if (_this.homeButtonEntity == senderID) {
             var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
             var onHomeScreen = tablet.onHomeScreen();
             if (onHomeScreen) {
@@ -219,7 +219,6 @@ WebTablet = function (url, width, dpi, hand, clientOnly) {
 };
 
 WebTablet.prototype.setHomeButtonTexture = function() {
-    print(this.homeButtonEntity);
     Entities.editEntity(this.tabletEntityID, {textures: JSON.stringify({"tex.close": HOME_BUTTON_TEXTURE})});
 };
 
@@ -385,14 +384,10 @@ WebTablet.prototype.register = function() {
 WebTablet.prototype.cleanUpOldTabletsOnJoint = function(jointIndex) {
     var children = Entities.getChildrenIDsOfJoint(MyAvatar.sessionUUID, jointIndex);
     children = children.concat(Entities.getChildrenIDsOfJoint(AVATAR_SELF_ID, jointIndex));
-    print("cleanup " + children);
     children.forEach(function(childID) {
         var props = Entities.getEntityProperties(childID, ["name"]);
         if (props.name === "WebTablet Tablet") {
-            print("cleaning up " + props.name);
             Entities.deleteEntity(childID);
-        } else {
-            print("not cleaning up " + props.name);
         }
     });
 };

scripts/tutorials/createDice.js

@@ -33,7 +33,7 @@ var BUTTON_SIZE = 32;
 var PADDING = 3;
 var BOTTOM_PADDING = 50;
 //a helper library for creating toolbars
-Script.include("http://hifi-production.s3.amazonaws.com/tutorials/dice/toolBars.js");
+Script.include("/~/system/libraries/toolBars.js");
 
 var toolBar = new ToolBar(0, 0, ToolBar.HORIZONTAL, "highfidelity.toolbars-dice", function(screenSize) {
   return {
@@ -139,4 +139,4 @@ function scriptEnding() {
 }
 
 Controller.mousePressEvent.connect(mousePressEvent);
-Script.scriptEnding.connect(scriptEnding);
+Script.scriptEnding.connect(scriptEnding);