Merge branch 'master' of https://github.com/highfidelity/hifi into dk/agentAvatarDataUpdates

assignment-client/src/Agent.cpp

@@ -434,8 +434,16 @@ void Agent::executeScript() {
     connect(&_avatarAudioTimerThread, &QThread::finished, audioTimerWorker, &QObject::deleteLater);
     _avatarAudioTimerThread.start();
 
-    // 60Hz timer for avatar
-    QObject::connect(_scriptEngine.get(), &ScriptEngine::update, this, &Agent::processAgentAvatar);
+    // Agents should run at 45hz
+    static const int AVATAR_DATA_HZ = 45;
+    static const int AVATAR_DATA_IN_MSECS = MSECS_PER_SECOND / AVATAR_DATA_HZ;
+    QTimer* avatarDataTimer = new QTimer(this);
+    connect(avatarDataTimer, &QTimer::timeout, this, &Agent::processAgentAvatar);
+    avatarDataTimer->setSingleShot(false);
+    avatarDataTimer->setInterval(AVATAR_DATA_IN_MSECS);
+    avatarDataTimer->setTimerType(Qt::PreciseTimer);
+    avatarDataTimer->start();
+
     _scriptEngine->run();
 
     Frame::clearFrameHandler(AUDIO_FRAME_TYPE);
@@ -538,7 +546,7 @@ void Agent::processAgentAvatar() {
         auto scriptedAvatar = DependencyManager::get<ScriptableAvatar>();
 
         AvatarData::AvatarDataDetail dataDetail = (randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO) ? AvatarData::SendAllData : AvatarData::CullSmallData;
-        QByteArray avatarByteArray = scriptedAvatar->toByteArray(dataDetail, 0, scriptedAvatar->getLastSentJointData());
+        QByteArray avatarByteArray = scriptedAvatar->toByteArrayStateful(dataDetail);
         scriptedAvatar->doneEncoding(true);
 
         static AvatarDataSequenceNumber sequenceNumber = 0;

assignment-client/src/avatars/AvatarMixer.h

@@ -21,6 +21,8 @@
 #include <ThreadedAssignment.h>
 #include "AvatarMixerClientData.h"
 
+#include "AvatarMixerSlavePool.h"
+
 /// Handles assignments of type AvatarMixer - distribution of avatar data to various clients
 class AvatarMixer : public ThreadedAssignment {
     Q_OBJECT
@@ -36,8 +38,9 @@ public slots:
     void sendStatsPacket() override;
 
 private slots:
+    void queueIncomingPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer node);
+    void handleAdjustAvatarSorting(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleViewFrustumPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
-    void handleAvatarDataPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleAvatarIdentityPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleKillAvatarPacket(QSharedPointer<ReceivedMessage> message);
     void handleNodeIgnoreRequestPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
@@ -45,22 +48,29 @@ private slots:
     void handleRequestsDomainListDataPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void domainSettingsRequestComplete();
     void handlePacketVersionMismatch(PacketType type, const HifiSockAddr& senderSockAddr, const QUuid& senderUUID);
+    void start();
 
 
 private:
-    void broadcastAvatarData();
+    AvatarMixerClientData* getOrCreateClientData(SharedNodePointer node);
+    std::chrono::microseconds timeFrame(p_high_resolution_clock::time_point& timestamp);
+    void throttle(std::chrono::microseconds duration, int frame);
+
     void parseDomainServerSettings(const QJsonObject& domainSettings);
     void sendIdentityPacket(AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode);
 
-    QThread _broadcastThread;
+    void manageDisplayName(const SharedNodePointer& node);
 
     p_high_resolution_clock::time_point _lastFrameTimestamp;
 
-    float _trailingSleepRatio { 1.0f };
-    float _performanceThrottlingRatio { 0.0f };
+    // FIXME - new throttling - use these values somehow
+    float _trailingMixRatio { 0.0f };
+    float _throttlingRatio { 0.0f };
+
 
     int _sumListeners { 0 };
     int _numStatFrames { 0 };
+    int _numTightLoopFrames { 0 };
     int _sumIdentityPackets { 0 };
 
     float _maxKbpsPerNode = 0.0f;
@@ -68,11 +78,41 @@ private:
     float _domainMinimumScale { MIN_AVATAR_SCALE };
     float _domainMaximumScale { MAX_AVATAR_SCALE };
 
-    QTimer* _broadcastTimer = nullptr;
-
     RateCounter<> _broadcastRate;
     p_high_resolution_clock::time_point _lastDebugMessage;
     QHash<QString, QPair<int, int>> _sessionDisplayNames;
+
+    quint64 _displayNameManagementElapsedTime { 0 }; // total time spent in broadcastAvatarData/display name management... since last stats window
+    quint64 _ignoreCalculationElapsedTime { 0 };
+    quint64 _avatarDataPackingElapsedTime { 0 };
+    quint64 _packetSendingElapsedTime { 0 };
+
+    quint64 _broadcastAvatarDataElapsedTime { 0 }; // total time spent in broadcastAvatarData since last stats window
+    quint64 _broadcastAvatarDataInner { 0 };
+    quint64 _broadcastAvatarDataLockWait { 0 };
+    quint64 _broadcastAvatarDataNodeTransform { 0 };
+    quint64 _broadcastAvatarDataNodeFunctor { 0 };
+
+    quint64 _handleAdjustAvatarSortingElapsedTime { 0 };
+    quint64 _handleViewFrustumPacketElapsedTime { 0 };
+    quint64 _handleAvatarIdentityPacketElapsedTime { 0 };
+    quint64 _handleKillAvatarPacketElapsedTime { 0 };
+    quint64 _handleNodeIgnoreRequestPacketElapsedTime { 0 };
+    quint64 _handleRadiusIgnoreRequestPacketElapsedTime { 0 };
+    quint64 _handleRequestsDomainListDataPacketElapsedTime { 0 };
+    quint64 _processQueuedAvatarDataPacketsElapsedTime { 0 };
+    quint64 _processQueuedAvatarDataPacketsLockWaitElapsedTime { 0 };
+
+    quint64 _processEventsElapsedTime { 0 };
+    quint64 _sendStatsElapsedTime { 0 };
+    quint64 _queueIncomingPacketElapsedTime { 0 };
+    quint64 _lastStatsTime { usecTimestampNow() };
+
+    RateCounter<> _loopRate; // this is the rate that the main thread tight loop runs
+
+
+    AvatarMixerSlavePool _slavePool;
+
 };
 
 #endif // hifi_AvatarMixer_h

assignment-client/src/avatars/AvatarMixerClientData.cpp

@@ -16,10 +16,52 @@
 
 #include "AvatarMixerClientData.h"
 
+
+
+void AvatarMixerClientData::queuePacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer node) {
+    if (!_packetQueue.node) {
+        _packetQueue.node = node;
+    }
+    _packetQueue.push(message);
+}
+
+int AvatarMixerClientData::processPackets() {
+    int packetsProcessed = 0;
+    SharedNodePointer node = _packetQueue.node;
+    assert(_packetQueue.empty() || node);
+    _packetQueue.node.clear();
+
+    while (!_packetQueue.empty()) {
+        auto& packet = _packetQueue.front();
+
+        packetsProcessed++;
+
+        switch (packet->getType()) {
+            case PacketType::AvatarData:
+                parseData(*packet);
+                break;
+            default:
+                Q_UNREACHABLE();
+        }
+        _packetQueue.pop();
+    }
+    assert(_packetQueue.empty());
+
+    return packetsProcessed;
+}
+
 int AvatarMixerClientData::parseData(ReceivedMessage& message) {
+
     // pull the sequence number from the data first
-    message.readPrimitive(&_lastReceivedSequenceNumber);
+    uint16_t sequenceNumber;
+
+    message.readPrimitive(&sequenceNumber);
     
+    if (sequenceNumber < _lastReceivedSequenceNumber && _lastReceivedSequenceNumber != UINT16_MAX) {
+        incrementNumOutOfOrderSends();
+    }
+    _lastReceivedSequenceNumber = sequenceNumber;
+
     // compute the offset to the data payload
     return _avatar->parseDataFromBuffer(message.readWithoutCopy(message.getBytesLeftToRead()));
 }
@@ -32,14 +74,22 @@ bool AvatarMixerClientData::checkAndSetHasReceivedFirstPacketsFrom(const QUuid&
     return true;
 }
 
+uint64_t AvatarMixerClientData::getLastBroadcastTime(const QUuid& nodeUUID) const {
+    // return the matching PacketSequenceNumber, or the default if we don't have it
+    auto nodeMatch = _lastBroadcastTimes.find(nodeUUID);
+    if (nodeMatch != _lastBroadcastTimes.end()) {
+        return nodeMatch->second;
+    }
+    return 0;
+}
+
 uint16_t AvatarMixerClientData::getLastBroadcastSequenceNumber(const QUuid& nodeUUID) const {
     // return the matching PacketSequenceNumber, or the default if we don't have it
     auto nodeMatch = _lastBroadcastSequenceNumbers.find(nodeUUID);
     if (nodeMatch != _lastBroadcastSequenceNumbers.end()) {
         return nodeMatch->second;
-    } else {
-        return 0;
     }
+    return 0;
 }
 
 void AvatarMixerClientData::ignoreOther(SharedNodePointer self, SharedNodePointer other) {
@@ -76,8 +126,6 @@ bool AvatarMixerClientData::otherAvatarInView(const AABox& otherAvatarBox) {
 
 void AvatarMixerClientData::loadJSONStats(QJsonObject& jsonObject) const {
     jsonObject["display_name"] = _avatar->getDisplayName();
-    jsonObject["full_rate_distance"] = _fullRateDistance;
-    jsonObject["max_av_distance"] = _maxAvatarDistance;
     jsonObject["num_avs_sent_last_frame"] = _numAvatarsSentLastFrame;
     jsonObject["avg_other_av_starves_per_second"] = getAvgNumOtherAvatarStarvesPerSecond();
     jsonObject["avg_other_av_skips_per_second"] = getAvgNumOtherAvatarSkipsPerSecond();

assignment-client/src/avatars/AvatarMixerClientData.h

@@ -16,6 +16,7 @@
 #include <cfloat>
 #include <unordered_map>
 #include <unordered_set>
+#include <queue>
 
 #include <QtCore/QJsonObject>
 #include <QtCore/QUrl>
@@ -41,6 +42,8 @@ public:
 
     int parseData(ReceivedMessage& message) override;
     AvatarData& getAvatar() { return *_avatar; }
+    const AvatarData* getConstAvatarData() const { return _avatar.get(); }
+    AvatarSharedPointer getAvatarSharedPointer() const { return _avatar; }
 
     bool checkAndSetHasReceivedFirstPacketsFrom(const QUuid& uuid);
 
@@ -49,6 +52,15 @@ public:
         { _lastBroadcastSequenceNumbers[nodeUUID] = sequenceNumber; }
     Q_INVOKABLE void removeLastBroadcastSequenceNumber(const QUuid& nodeUUID) { _lastBroadcastSequenceNumbers.erase(nodeUUID); }
 
+    uint64_t getLastBroadcastTime(const QUuid& nodeUUID) const;
+    void setLastBroadcastTime(const QUuid& nodeUUID, uint64_t broadcastTime) { _lastBroadcastTimes[nodeUUID] = broadcastTime; }
+    Q_INVOKABLE void removeLastBroadcastTime(const QUuid& nodeUUID) { _lastBroadcastTimes.erase(nodeUUID); }
+
+    Q_INVOKABLE void cleanupKilledNode(const QUuid& nodeUUID) {
+        removeLastBroadcastSequenceNumber(nodeUUID);
+        removeLastBroadcastTime(nodeUUID);
+    }
+
     uint16_t getLastReceivedSequenceNumber() const { return _lastReceivedSequenceNumber; }
 
     HRCTime getIdentityChangeTimestamp() const { return _identityChangeTimestamp; }
@@ -56,12 +68,6 @@ public:
     bool getAvatarSessionDisplayNameMustChange() const { return _avatarSessionDisplayNameMustChange; }
     void setAvatarSessionDisplayNameMustChange(bool set = true) { _avatarSessionDisplayNameMustChange = set; }
 
-    void setFullRateDistance(float fullRateDistance) { _fullRateDistance = fullRateDistance; }
-    float getFullRateDistance() const { return _fullRateDistance; }
-
-    void setMaxAvatarDistance(float maxAvatarDistance) { _maxAvatarDistance = maxAvatarDistance; }
-    float getMaxAvatarDistance() const { return _maxAvatarDistance; }
-
     void resetNumAvatarsSentLastFrame() { _numAvatarsSentLastFrame = 0; }
     void incrementNumAvatarsSentLastFrame() { ++_numAvatarsSentLastFrame; }
     int getNumAvatarsSentLastFrame() const { return _numAvatarsSentLastFrame; }
@@ -85,9 +91,9 @@ public:
 
     void loadJSONStats(QJsonObject& jsonObject) const;
 
-    glm::vec3 getPosition() { return _avatar ? _avatar->getPosition() : glm::vec3(0); }
-    glm::vec3 getGlobalBoundingBoxCorner() { return _avatar ? _avatar->getGlobalBoundingBoxCorner() : glm::vec3(0); }
-    bool isRadiusIgnoring(const QUuid& other) { return _radiusIgnoredOthers.find(other) != _radiusIgnoredOthers.end(); }
+    glm::vec3 getPosition() const { return _avatar ? _avatar->getPosition() : glm::vec3(0); }
+    glm::vec3 getGlobalBoundingBoxCorner() const { return _avatar ? _avatar->getGlobalBoundingBoxCorner() : glm::vec3(0); }
+    bool isRadiusIgnoring(const QUuid& other) const { return _radiusIgnoredOthers.find(other) != _radiusIgnoredOthers.end(); }
     void addToRadiusIgnoringSet(const QUuid& other) { _radiusIgnoredOthers.insert(other); }
     void removeFromRadiusIgnoringSet(SharedNodePointer self, const QUuid& other);
     void ignoreOther(SharedNodePointer self, SharedNodePointer other);
@@ -104,6 +110,8 @@ public:
     bool getRequestsDomainListData() { return _requestsDomainListData; }
     void setRequestsDomainListData(bool requesting) { _requestsDomainListData = requesting; }
 
+    ViewFrustum getViewFrustom() const { return _currentViewFrustum; }
+
     quint64 getLastOtherAvatarEncodeTime(QUuid otherAvatar) {
         quint64 result = 0;
         if (_lastOtherAvatarEncodeTime.find(otherAvatar) != _lastOtherAvatarEncodeTime.end()) {
@@ -118,14 +126,21 @@ public:
         return _lastOtherAvatarSentJoints[otherAvatar];
     }
 
-    
+    void queuePacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer node);
+    int processPackets(); // returns number of packets processed
 
 private:
+    struct PacketQueue : public std::queue<QSharedPointer<ReceivedMessage>> {
+        QWeakPointer<Node> node;
+    };
+    PacketQueue _packetQueue;
+
     AvatarSharedPointer _avatar { new AvatarData() };
 
     uint16_t _lastReceivedSequenceNumber { 0 };
     std::unordered_map<QUuid, uint16_t> _lastBroadcastSequenceNumbers;
     std::unordered_set<QUuid> _hasReceivedFirstPacketsFrom;
+    std::unordered_map<QUuid, uint64_t> _lastBroadcastTimes;
 
     // this is a map of the last time we encoded an "other" avatar for
     // sending to "this" node
@@ -135,9 +150,6 @@ private:
     HRCTime _identityChangeTimestamp;
     bool _avatarSessionDisplayNameMustChange{ false };
 
-    float _fullRateDistance = FLT_MAX;
-    float _maxAvatarDistance = FLT_MAX;
-
     int _numAvatarsSentLastFrame = 0;
     int _numFramesSinceAdjustment = 0;
 

assignment-client/src/avatars/AvatarMixerSlave.cpp

@@ -0,0 +1,434 @@
+//
+//  AvatarMixerSlave.cpp
+//  assignment-client/src/avatar
+//
+//  Created by Brad Hefta-Gaub on 2/14/2017.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <algorithm>
+#include <random>
+
+#include <glm/glm.hpp>
+#include <glm/gtx/norm.hpp>
+#include <glm/gtx/vector_angle.hpp>
+
+#include <AvatarLogging.h>
+#include <LogHandler.h>
+#include <NetworkAccessManager.h>
+#include <NodeList.h>
+#include <Node.h>
+#include <OctreeConstants.h>
+#include <udt/PacketHeaders.h>
+#include <SharedUtil.h>
+#include <StDev.h>
+#include <UUID.h>
+
+
+#include "AvatarMixer.h"
+#include "AvatarMixerClientData.h"
+#include "AvatarMixerSlave.h"
+
+
+void AvatarMixerSlave::configure(ConstIter begin, ConstIter end) {
+    _begin = begin;
+    _end = end;
+}
+
+void AvatarMixerSlave::configureBroadcast(ConstIter begin, ConstIter end, 
+                                p_high_resolution_clock::time_point lastFrameTimestamp,
+                                float maxKbpsPerNode, float throttlingRatio) {
+    _begin = begin;
+    _end = end;
+    _lastFrameTimestamp = lastFrameTimestamp;
+    _maxKbpsPerNode = maxKbpsPerNode;
+    _throttlingRatio = throttlingRatio;
+}
+
+void AvatarMixerSlave::harvestStats(AvatarMixerSlaveStats& stats) {
+    stats = _stats;
+    _stats.reset();
+}
+
+
+void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) {
+    auto start = usecTimestampNow();
+    auto nodeData = dynamic_cast<AvatarMixerClientData*>(node->getLinkedData());
+    if (nodeData) {
+        _stats.nodesProcessed++;
+        _stats.packetsProcessed += nodeData->processPackets();
+    }
+    auto end = usecTimestampNow();
+    _stats.processIncomingPacketsElapsedTime += (end - start);
+}
+
+
+int AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) {
+    int bytesSent = 0;
+    QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray();
+    auto identityPacket = NLPacket::create(PacketType::AvatarIdentity, individualData.size());
+    individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious
+    bytesSent += individualData.size();
+    identityPacket->write(individualData);
+    DependencyManager::get<NodeList>()->sendPacket(std::move(identityPacket), *destinationNode);
+    _stats.numIdentityPackets++;
+    return bytesSent;
+}
+
+static const int AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND = 45;
+
+// FIXME - There is some old logic (unchanged as of 2/17/17) that randomly decides to send an identity
+// packet. That logic had the following comment about the constants it uses...
+//
+//         An 80% chance of sending a identity packet within a 5 second interval.
+//         assuming 60 htz update rate.
+//
+// Assuming the calculation of the constant is in fact correct for 80% and 60hz and 5 seconds (an assumption
+// that I have not verified) then the constant is definitely wrong now, since we send at 45hz.
+const float IDENTITY_SEND_PROBABILITY = 1.0f / 187.0f;
+
+void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) {
+    quint64 start = usecTimestampNow();
+
+    auto nodeList = DependencyManager::get<NodeList>();
+
+    // setup for distributed random floating point values
+    std::random_device randomDevice;
+    std::mt19937 generator(randomDevice());
+    std::uniform_real_distribution<float> distribution;
+
+    if (node->getLinkedData() && (node->getType() == NodeType::Agent) && node->getActiveSocket()) {
+        _stats.nodesBroadcastedTo++;
+
+        AvatarMixerClientData* nodeData = reinterpret_cast<AvatarMixerClientData*>(node->getLinkedData());
+
+        nodeData->resetInViewStats();
+
+        const AvatarData& avatar = nodeData->getAvatar();
+        glm::vec3 myPosition = avatar.getClientGlobalPosition();
+
+        // reset the internal state for correct random number distribution
+        distribution.reset();
+
+        // reset the number of sent avatars
+        nodeData->resetNumAvatarsSentLastFrame();
+
+        // keep a counter of the number of considered avatars
+        int numOtherAvatars = 0;
+
+        // keep track of outbound data rate specifically for avatar data
+        int numAvatarDataBytes = 0;
+        int identityBytesSent = 0;
+
+        // max number of avatarBytes per frame
+        auto maxAvatarBytesPerFrame = (_maxKbpsPerNode * BYTES_PER_KILOBIT) / AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND;
+
+        // FIXME - find a way to not send the sessionID for every avatar
+        int minimumBytesPerAvatar = AvatarDataPacket::AVATAR_HAS_FLAGS_SIZE + NUM_BYTES_RFC4122_UUID;
+
+        int overBudgetAvatars = 0;
+
+        // keep track of the number of other avatars held back in this frame
+        int numAvatarsHeldBack = 0;
+
+        // keep track of the number of other avatar frames skipped
+        int numAvatarsWithSkippedFrames = 0;
+
+        // When this is true, the AvatarMixer will send Avatar data to a client about avatars that are not in the view frustrum
+        bool getsOutOfView = nodeData->getRequestsDomainListData();
+
+        // When this is true, the AvatarMixer will send Avatar data to a client about avatars that they've ignored
+        bool getsIgnoredByMe = getsOutOfView;
+
+        // When this is true, the AvatarMixer will send Avatar data to a client about avatars that have ignored them
+        bool getsAnyIgnored = getsIgnoredByMe && node->getCanKick();
+
+        // setup a PacketList for the avatarPackets
+        auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData);
+
+        // Define the minimum bubble size
+        static const glm::vec3 minBubbleSize = glm::vec3(0.3f, 1.3f, 0.3f);
+        // Define the scale of the box for the current node
+        glm::vec3 nodeBoxScale = (nodeData->getPosition() - nodeData->getGlobalBoundingBoxCorner()) * 2.0f;
+        // Set up the bounding box for the current node
+        AABox nodeBox(nodeData->getGlobalBoundingBoxCorner(), nodeBoxScale);
+        // Clamp the size of the bounding box to a minimum scale
+        if (glm::any(glm::lessThan(nodeBoxScale, minBubbleSize))) {
+            nodeBox.setScaleStayCentered(minBubbleSize);
+        }
+        // Quadruple the scale of both bounding boxes
+        nodeBox.embiggen(4.0f);
+
+
+        // setup list of AvatarData as well as maps to map betweeen the AvatarData and the original nodes
+        // for calling the AvatarData::sortAvatars() function and getting our sorted list of client nodes
+        QList<AvatarSharedPointer> avatarList;
+        std::unordered_map<AvatarSharedPointer, SharedNodePointer> avatarDataToNodes;
+
+        int listItem = 0;
+        std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) {
+            const AvatarMixerClientData* otherNodeData = reinterpret_cast<const AvatarMixerClientData*>(otherNode->getLinkedData());
+
+            // theoretically it's possible for a Node to be in the NodeList (and therefore end up here),
+            // but not have yet sent data that's linked to the node. Check for that case and don't
+            // consider those nodes.
+            if (otherNodeData) {
+                listItem++;
+                AvatarSharedPointer otherAvatar = otherNodeData->getAvatarSharedPointer();
+                avatarList << otherAvatar;
+                avatarDataToNodes[otherAvatar] = otherNode;
+            }
+        });
+
+        AvatarSharedPointer thisAvatar = nodeData->getAvatarSharedPointer();
+        ViewFrustum cameraView = nodeData->getViewFrustom();
+        std::priority_queue<AvatarPriority> sortedAvatars = AvatarData::sortAvatars(
+                avatarList, cameraView,
+
+                [&](AvatarSharedPointer avatar)->uint64_t{
+                    auto avatarNode = avatarDataToNodes[avatar];
+                    assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map
+                    return nodeData->getLastBroadcastTime(avatarNode->getUUID());
+                },
+
+                [&](AvatarSharedPointer avatar)->float{
+                    glm::vec3 nodeBoxHalfScale = (avatar->getPosition() - avatar->getGlobalBoundingBoxCorner());
+                    return glm::max(nodeBoxHalfScale.x, glm::max(nodeBoxHalfScale.y, nodeBoxHalfScale.z));
+                },
+
+                [&](AvatarSharedPointer avatar)->bool{
+                    if (avatar == thisAvatar) {
+                        return true; // ignore ourselves...
+                    }
+
+                    bool shouldIgnore = false;
+
+                    // We will also ignore other nodes for a couple of different reasons:
+                    //   1) ignore bubbles and ignore specific node
+                    //   2) the node hasn't really updated it's frame data recently, this can
+                    //      happen if for example the avatar is connected on a desktop and sending
+                    //      updates at ~30hz. So every 3 frames we skip a frame.
+                    auto avatarNode = avatarDataToNodes[avatar];
+
+                    assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map
+
+                    const AvatarMixerClientData* avatarNodeData = reinterpret_cast<const AvatarMixerClientData*>(avatarNode->getLinkedData());
+                    assert(avatarNodeData); // we can't have gotten here without avatarNode having valid data
+                    quint64 startIgnoreCalculation = usecTimestampNow();
+
+                    // make sure we have data for this avatar, that it isn't the same node,
+                    // and isn't an avatar that the viewing node has ignored
+                    // or that has ignored the viewing node
+                    if (!avatarNode->getLinkedData()
+                        || avatarNode->getUUID() == node->getUUID()
+                        || (node->isIgnoringNodeWithID(avatarNode->getUUID()) && !getsIgnoredByMe)
+                        || (avatarNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) {
+                        shouldIgnore = true;
+                    } else {
+
+                        // Check to see if the space bubble is enabled
+                        if (node->isIgnoreRadiusEnabled() || avatarNode->isIgnoreRadiusEnabled()) {
+
+                            // Define the scale of the box for the current other node
+                            glm::vec3 otherNodeBoxScale = (avatarNodeData->getPosition() - avatarNodeData->getGlobalBoundingBoxCorner()) * 2.0f;
+                            // Set up the bounding box for the current other node
+                            AABox otherNodeBox(avatarNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale);
+                            // Clamp the size of the bounding box to a minimum scale
+                            if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) {
+                                otherNodeBox.setScaleStayCentered(minBubbleSize);
+                            }
+                            // Quadruple the scale of both bounding boxes
+                            otherNodeBox.embiggen(4.0f);
+
+                            // Perform the collision check between the two bounding boxes
+                            if (nodeBox.touches(otherNodeBox)) {
+                                nodeData->ignoreOther(node, avatarNode);
+                                shouldIgnore = !getsAnyIgnored;
+                            }
+                        }
+                        // Not close enough to ignore
+                        if (!shouldIgnore) {
+                            nodeData->removeFromRadiusIgnoringSet(node, avatarNode->getUUID());
+                        }
+                    }
+                    quint64 endIgnoreCalculation = usecTimestampNow();
+                    _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation);
+
+                    if (!shouldIgnore) {
+                        AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getUUID());
+                        AvatarDataSequenceNumber lastSeqFromSender = avatarNodeData->getLastReceivedSequenceNumber();
+
+                        // FIXME - This code does appear to be working. But it seems brittle.
+                        //         It supports determining if the frame of data for this "other"
+                        //         avatar has already been sent to the reciever. This has been
+                        //         verified to work on a desktop display that renders at 60hz and
+                        //         therefore sends to mixer at 30hz. Each second you'd expect to
+                        //         have 15 (45hz-30hz) duplicate frames. In this case, the stat
+                        //         avg_other_av_skips_per_second does report 15.
+                        //
+                        // make sure we haven't already sent this data from this sender to this receiver
+                        // or that somehow we haven't sent
+                        if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) {
+                            ++numAvatarsHeldBack;
+                            shouldIgnore = true;
+                        } else if (lastSeqFromSender - lastSeqToReceiver > 1) {
+                            // this is a skip - we still send the packet but capture the presence of the skip so we see it happening
+                            ++numAvatarsWithSkippedFrames;
+                        }
+                    }
+                    return shouldIgnore;
+                });
+
+        // loop through our sorted avatars and allocate our bandwidth to them accordingly
+        int avatarRank = 0;
+
+        // this is overly conservative, because it includes some avatars we might not consider
+        int remainingAvatars = (int)sortedAvatars.size(); 
+
+        while (!sortedAvatars.empty()) {
+            AvatarPriority sortData = sortedAvatars.top();
+            sortedAvatars.pop();
+            const auto& avatarData = sortData.avatar;
+            avatarRank++;
+            remainingAvatars--;
+
+            auto otherNode = avatarDataToNodes[avatarData];
+            assert(otherNode); // we can't have gotten here without the avatarData being a valid key in the map
+
+            // NOTE: Here's where we determine if we are over budget and drop to bare minimum data
+            int minimRemainingAvatarBytes = minimumBytesPerAvatar * remainingAvatars;
+            bool overBudget = (identityBytesSent + numAvatarDataBytes + minimRemainingAvatarBytes) > maxAvatarBytesPerFrame;
+
+            quint64 startAvatarDataPacking = usecTimestampNow();
+
+            ++numOtherAvatars;
+
+            const AvatarMixerClientData* otherNodeData = reinterpret_cast<const AvatarMixerClientData*>(otherNode->getLinkedData());
+
+            // make sure we send out identity packets to and from new arrivals.
+            bool forceSend = !nodeData->checkAndSetHasReceivedFirstPacketsFrom(otherNode->getUUID());
+
+            // FIXME - this clause seems suspicious "... || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp ..."
+            if (!overBudget
+                && otherNodeData->getIdentityChangeTimestamp().time_since_epoch().count() > 0
+                && (forceSend
+                || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp
+                || distribution(generator) < IDENTITY_SEND_PROBABILITY)) {
+
+                identityBytesSent += sendIdentityPacket(otherNodeData, node);
+            }
+
+            const AvatarData* otherAvatar = otherNodeData->getConstAvatarData();
+            glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition();
+
+            // determine if avatar is in view, to determine how much data to include...
+            glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f;
+            AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale);
+            bool isInView = nodeData->otherAvatarInView(otherNodeBox);
+
+            // start a new segment in the PacketList for this avatar
+            avatarPacketList->startSegment();
+
+            AvatarData::AvatarDataDetail detail;
+
+            if (overBudget) {
+                overBudgetAvatars++;
+                _stats.overBudgetAvatars++;
+                detail = AvatarData::NoData;
+            } else  if (!isInView && !getsOutOfView) {
+                detail = AvatarData::NoData;
+                nodeData->incrementAvatarOutOfView();
+            } else {
+                detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO
+                    ? AvatarData::SendAllData : AvatarData::CullSmallData;
+                nodeData->incrementAvatarInView();
+            }
+
+            bool includeThisAvatar = true;
+            auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID());
+            QVector<JointData>& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID());
+            bool distanceAdjust = true;
+            glm::vec3 viewerPosition = myPosition;
+            AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray
+            bool dropFaceTracking = false;
+
+            quint64 start = usecTimestampNow();
+            QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, 
+                                            hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther);
+            quint64 end = usecTimestampNow();
+            _stats.toByteArrayElapsedTime += (end - start);
+
+            static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID);
+            if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) {
+                qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data";
+
+                dropFaceTracking = true; // first try dropping the facial data
+                bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther,
+                    hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther);
+
+                if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) {
+                    qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData";
+                    bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther,
+                        hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther);
+                }
+
+                if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) {
+                    qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!";
+                    includeThisAvatar = false;
+                }
+            }
+
+            if (includeThisAvatar) {
+                numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122());
+                numAvatarDataBytes += avatarPacketList->write(bytes);
+
+                if (detail != AvatarData::NoData) {
+                    _stats.numOthersIncluded++;
+
+                    // increment the number of avatars sent to this reciever
+                    nodeData->incrementNumAvatarsSentLastFrame();
+
+                    // set the last sent sequence number for this sender on the receiver
+                    nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), 
+                                    otherNodeData->getLastReceivedSequenceNumber());
+
+                    // remember the last time we sent details about this other node to the receiver
+                    nodeData->setLastBroadcastTime(otherNode->getUUID(), start);
+                }
+            }
+
+            avatarPacketList->endSegment();
+
+            quint64 endAvatarDataPacking = usecTimestampNow();
+            _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking);
+        };
+
+        quint64 startPacketSending = usecTimestampNow();
+
+        // close the current packet so that we're always sending something
+        avatarPacketList->closeCurrentPacket(true);
+
+        _stats.numPacketsSent += (int)avatarPacketList->getNumPackets();
+        _stats.numBytesSent += numAvatarDataBytes;
+
+        // send the avatar data PacketList
+        nodeList->sendPacketList(std::move(avatarPacketList), *node);
+
+        // record the bytes sent for other avatar data in the AvatarMixerClientData
+        nodeData->recordSentAvatarData(numAvatarDataBytes);
+
+        // record the number of avatars held back this frame
+        nodeData->recordNumOtherAvatarStarves(numAvatarsHeldBack);
+        nodeData->recordNumOtherAvatarSkips(numAvatarsWithSkippedFrames);
+
+        quint64 endPacketSending = usecTimestampNow();
+        _stats.packetSendingElapsedTime += (endPacketSending - startPacketSending);
+    }
+
+    quint64 end = usecTimestampNow();
+    _stats.jobElapsedTime += (end - start);
+}
+

assignment-client/src/avatars/AvatarMixerSlave.h

@@ -0,0 +1,107 @@
+//
+//  AvatarMixerSlave.h
+//  assignment-client/src/avatar
+//
+//  Created by Brad Hefta-Gaub on 2/14/2017.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_AvatarMixerSlave_h
+#define hifi_AvatarMixerSlave_h
+
+class AvatarMixerClientData;
+
+class AvatarMixerSlaveStats {
+public:
+    int nodesProcessed { 0 };
+    int packetsProcessed { 0 };
+    quint64 processIncomingPacketsElapsedTime { 0 };
+
+    int nodesBroadcastedTo { 0 };
+    int numPacketsSent { 0 };
+    int numBytesSent { 0 };
+    int numIdentityPackets { 0 };
+    int numOthersIncluded { 0 };
+    int overBudgetAvatars { 0 };
+
+    quint64 ignoreCalculationElapsedTime { 0 };
+    quint64 avatarDataPackingElapsedTime { 0 };
+    quint64 packetSendingElapsedTime { 0 };
+    quint64 toByteArrayElapsedTime { 0 };
+    quint64 jobElapsedTime { 0 };
+
+    void reset() {
+        // receiving job stats
+        nodesProcessed = 0;
+        packetsProcessed = 0;
+        processIncomingPacketsElapsedTime = 0;
+
+        // sending job stats
+        nodesBroadcastedTo = 0;
+        numPacketsSent = 0;
+        numBytesSent = 0;
+        numIdentityPackets = 0;
+        numOthersIncluded = 0;
+        overBudgetAvatars = 0;
+
+        ignoreCalculationElapsedTime = 0;
+        avatarDataPackingElapsedTime = 0;
+        packetSendingElapsedTime = 0;
+        toByteArrayElapsedTime = 0;
+        jobElapsedTime = 0;
+    }
+
+    AvatarMixerSlaveStats& operator+=(const AvatarMixerSlaveStats& rhs) {
+        nodesProcessed += rhs.nodesProcessed;
+        packetsProcessed += rhs.packetsProcessed;
+        processIncomingPacketsElapsedTime += rhs.processIncomingPacketsElapsedTime;
+
+        nodesBroadcastedTo += rhs.nodesBroadcastedTo;
+        numPacketsSent += rhs.numPacketsSent;
+        numBytesSent += rhs.numBytesSent;
+        numIdentityPackets += rhs.numIdentityPackets;
+        numOthersIncluded += rhs.numOthersIncluded;
+        overBudgetAvatars += rhs.overBudgetAvatars;
+
+        ignoreCalculationElapsedTime += rhs.ignoreCalculationElapsedTime;
+        avatarDataPackingElapsedTime += rhs.avatarDataPackingElapsedTime;
+        packetSendingElapsedTime += rhs.packetSendingElapsedTime;
+        toByteArrayElapsedTime += rhs.toByteArrayElapsedTime;
+        jobElapsedTime += rhs.jobElapsedTime;
+        return *this;
+    }
+
+};
+
+class AvatarMixerSlave {
+public:
+    using ConstIter = NodeList::const_iterator;
+
+    void configure(ConstIter begin, ConstIter end);
+    void configureBroadcast(ConstIter begin, ConstIter end, 
+                    p_high_resolution_clock::time_point lastFrameTimestamp, 
+                    float maxKbpsPerNode, float throttlingRatio);
+
+    void processIncomingPackets(const SharedNodePointer& node);
+    void broadcastAvatarData(const SharedNodePointer& node);
+
+    void harvestStats(AvatarMixerSlaveStats& stats);
+
+private:
+    int sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode);
+
+    // frame state
+    ConstIter _begin;
+    ConstIter _end;
+
+    p_high_resolution_clock::time_point _lastFrameTimestamp;
+    float _maxKbpsPerNode { 0.0f };
+    float _throttlingRatio { 0.0f };
+
+    AvatarMixerSlaveStats _stats;
+};
+
+#endif // hifi_AvatarMixerSlave_h

assignment-client/src/avatars/AvatarMixerSlavePool.cpp

@@ -0,0 +1,208 @@
+//
+//  AvatarMixerSlavePool.cpp
+//  assignment-client/src/avatar
+//
+//  Created by Brad Hefta-Gaub on 2/14/2017.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <assert.h>
+#include <algorithm>
+
+#include "AvatarMixerSlavePool.h"
+
+void AvatarMixerSlaveThread::run() {
+    while (true) {
+        wait();
+
+        // iterate over all available nodes
+        SharedNodePointer node;
+        while (try_pop(node)) {
+            (this->*_function)(node);
+        }
+
+        bool stopping = _stop;
+        notify(stopping);
+        if (stopping) {
+            return;
+        }
+    }
+}
+
+void AvatarMixerSlaveThread::wait() {
+    {
+        Lock lock(_pool._mutex);
+        _pool._slaveCondition.wait(lock, [&] {
+            assert(_pool._numStarted <= _pool._numThreads);
+            return _pool._numStarted != _pool._numThreads;
+        });
+        ++_pool._numStarted;
+    }
+    if (_pool._configure) {
+        _pool._configure(*this);
+    }
+    _function = _pool._function;
+}
+
+void AvatarMixerSlaveThread::notify(bool stopping) {
+    {
+        Lock lock(_pool._mutex);
+        assert(_pool._numFinished < _pool._numThreads);
+        ++_pool._numFinished;
+        if (stopping) {
+            ++_pool._numStopped;
+        }
+    }
+    _pool._poolCondition.notify_one();
+}
+
+bool AvatarMixerSlaveThread::try_pop(SharedNodePointer& node) {
+    return _pool._queue.try_pop(node);
+}
+
+#ifdef AVATAR_SINGLE_THREADED
+static AvatarMixerSlave slave;
+#endif
+
+void AvatarMixerSlavePool::processIncomingPackets(ConstIter begin, ConstIter end) {
+    _function = &AvatarMixerSlave::processIncomingPackets;
+    _configure = [&](AvatarMixerSlave& slave) { 
+        slave.configure(begin, end);
+    };
+    run(begin, end);
+}
+
+void AvatarMixerSlavePool::broadcastAvatarData(ConstIter begin, ConstIter end, 
+                                     p_high_resolution_clock::time_point lastFrameTimestamp, 
+                                     float maxKbpsPerNode, float throttlingRatio) {
+    _function = &AvatarMixerSlave::broadcastAvatarData;
+    _configure = [&](AvatarMixerSlave& slave) { 
+        slave.configureBroadcast(begin, end, lastFrameTimestamp, maxKbpsPerNode, throttlingRatio);
+   };
+    run(begin, end);
+}
+
+void AvatarMixerSlavePool::run(ConstIter begin, ConstIter end) {
+    _begin = begin;
+    _end = end;
+
+#ifdef AUDIO_SINGLE_THREADED
+    _configure(slave);
+    std::for_each(begin, end, [&](const SharedNodePointer& node) {
+        _function(slave, node);
+});
+#else
+    // fill the queue
+    std::for_each(_begin, _end, [&](const SharedNodePointer& node) {
+        _queue.emplace(node);
+    });
+
+    {
+        Lock lock(_mutex);
+
+        // run
+        _numStarted = _numFinished = 0;
+        _slaveCondition.notify_all();
+
+        // wait
+        _poolCondition.wait(lock, [&] {
+            assert(_numFinished <= _numThreads);
+            return _numFinished == _numThreads;
+        });
+
+        assert(_numStarted == _numThreads);
+    }
+
+    assert(_queue.empty());
+#endif
+}
+
+
+void AvatarMixerSlavePool::each(std::function<void(AvatarMixerSlave& slave)> functor) {
+#ifdef AVATAR_SINGLE_THREADED
+    functor(slave);
+#else
+    for (auto& slave : _slaves) {
+        functor(*slave.get());
+    }
+#endif
+}
+
+void AvatarMixerSlavePool::setNumThreads(int numThreads) {
+    // clamp to allowed size
+    {
+        int maxThreads = QThread::idealThreadCount();
+        if (maxThreads == -1) {
+            // idealThreadCount returns -1 if cores cannot be detected
+            static const int MAX_THREADS_IF_UNKNOWN = 4;
+            maxThreads = MAX_THREADS_IF_UNKNOWN;
+        }
+
+        int clampedThreads = std::min(std::max(1, numThreads), maxThreads);
+        if (clampedThreads != numThreads) {
+            qWarning("%s: clamped to %d (was %d)", __FUNCTION__, clampedThreads, numThreads);
+            numThreads = clampedThreads;
+        }
+    }
+
+    resize(numThreads);
+}
+
+void AvatarMixerSlavePool::resize(int numThreads) {
+    assert(_numThreads == (int)_slaves.size());
+
+#ifdef AVATAR_SINGLE_THREADED
+    qDebug("%s: running single threaded", __FUNCTION__, numThreads);
+#else
+    qDebug("%s: set %d threads (was %d)", __FUNCTION__, numThreads, _numThreads);
+
+    Lock lock(_mutex);
+
+    if (numThreads > _numThreads) {
+        // start new slaves
+        for (int i = 0; i < numThreads - _numThreads; ++i) {
+            auto slave = new AvatarMixerSlaveThread(*this);
+            slave->start();
+            _slaves.emplace_back(slave);
+        }
+    } else if (numThreads < _numThreads) {
+        auto extraBegin = _slaves.begin() + numThreads;
+
+        // mark slaves to stop...
+        auto slave = extraBegin;
+        while (slave != _slaves.end()) {
+            (*slave)->_stop = true;
+            ++slave;
+        }
+
+        // ...cycle them until they do stop...
+        _numStopped = 0;
+        while (_numStopped != (_numThreads - numThreads)) {
+            _numStarted = _numFinished = _numStopped;
+            _slaveCondition.notify_all();
+            _poolCondition.wait(lock, [&] {
+                assert(_numFinished <= _numThreads);
+                return _numFinished == _numThreads;
+            });
+        }
+
+        // ...wait for threads to finish...
+        slave = extraBegin;
+        while (slave != _slaves.end()) {
+            QThread* thread = reinterpret_cast<QThread*>(slave->get());
+            static const int MAX_THREAD_WAIT_TIME = 10;
+            thread->wait(MAX_THREAD_WAIT_TIME);
+            ++slave;
+        }
+
+        // ...and erase them
+        _slaves.erase(extraBegin, _slaves.end());
+    }
+
+    _numThreads = _numStarted = _numFinished = numThreads;
+    assert(_numThreads == (int)_slaves.size());
+#endif
+}

assignment-client/src/avatars/AvatarMixerSlavePool.h

@@ -0,0 +1,104 @@
+//
+//  AvatarMixerSlavePool.h
+//  assignment-client/src/avatar
+//
+//  Created by Brad Hefta-Gaub on 2/14/2017.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_AvatarMixerSlavePool_h
+#define hifi_AvatarMixerSlavePool_h
+
+#include <condition_variable>
+#include <mutex>
+#include <vector>
+
+#include <tbb/concurrent_queue.h>
+
+#include <QThread>
+
+#include <NodeList.h>
+
+#include "AvatarMixerSlave.h"
+
+class AvatarMixerSlavePool;
+
+class AvatarMixerSlaveThread : public QThread, public AvatarMixerSlave {
+    Q_OBJECT
+    using ConstIter = NodeList::const_iterator;
+    using Mutex = std::mutex;
+    using Lock = std::unique_lock<Mutex>;
+
+public:
+    AvatarMixerSlaveThread(AvatarMixerSlavePool& pool) : _pool(pool) {}
+
+    void run() override final;
+
+private:
+    friend class AvatarMixerSlavePool;
+
+    void wait();
+    void notify(bool stopping);
+    bool try_pop(SharedNodePointer& node);
+
+    AvatarMixerSlavePool& _pool;
+    void (AvatarMixerSlave::*_function)(const SharedNodePointer& node) { nullptr };
+    bool _stop { false };
+};
+
+// Slave pool for audio mixers
+//   AvatarMixerSlavePool is not thread-safe! It should be instantiated and used from a single thread.
+class AvatarMixerSlavePool {
+    using Queue = tbb::concurrent_queue<SharedNodePointer>;
+    using Mutex = std::mutex;
+    using Lock = std::unique_lock<Mutex>;
+    using ConditionVariable = std::condition_variable;
+
+public:
+    using ConstIter = NodeList::const_iterator;
+
+    AvatarMixerSlavePool(int numThreads = QThread::idealThreadCount()) { setNumThreads(numThreads); }
+    ~AvatarMixerSlavePool() { resize(0); }
+
+    // Jobs the slave pool can do...
+    void processIncomingPackets(ConstIter begin, ConstIter end);
+    void broadcastAvatarData(ConstIter begin, ConstIter end, 
+                    p_high_resolution_clock::time_point lastFrameTimestamp, float maxKbpsPerNode, float throttlingRatio);
+
+    // iterate over all slaves
+    void each(std::function<void(AvatarMixerSlave& slave)> functor);
+
+    void setNumThreads(int numThreads);
+    int numThreads() { return _numThreads; }
+
+private:
+    void run(ConstIter begin, ConstIter end);
+    void resize(int numThreads);
+
+    std::vector<std::unique_ptr<AvatarMixerSlaveThread>> _slaves;
+
+    friend void AvatarMixerSlaveThread::wait();
+    friend void AvatarMixerSlaveThread::notify(bool stopping);
+    friend bool AvatarMixerSlaveThread::try_pop(SharedNodePointer& node);
+
+    // synchronization state
+    Mutex _mutex;
+    ConditionVariable _slaveCondition;
+    ConditionVariable _poolCondition;
+    void (AvatarMixerSlave::*_function)(const SharedNodePointer& node);
+    std::function<void(AvatarMixerSlave&)> _configure;
+    int _numThreads { 0 };
+    int _numStarted { 0 }; // guarded by _mutex
+    int _numFinished { 0 }; // guarded by _mutex
+    int _numStopped { 0 }; // guarded by _mutex
+
+    // frame state
+    Queue _queue;
+    ConstIter _begin;
+    ConstIter _end;
+};
+
+#endif // hifi_AvatarMixerSlavePool_h

assignment-client/src/avatars/ScriptableAvatar.cpp

@@ -14,10 +14,9 @@
 #include <GLMHelpers.h>
 #include "ScriptableAvatar.h"
 
-QByteArray ScriptableAvatar::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
-                                    bool distanceAdjust, glm::vec3 viewerPosition, QVector<JointData>* sentJointDataOut) {
+QByteArray ScriptableAvatar::toByteArrayStateful(AvatarDataDetail dataDetail) {
     _globalPosition = getPosition();
-    return AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut);
+    return AvatarData::toByteArrayStateful(dataDetail);
 }
 
 

assignment-client/src/avatars/ScriptableAvatar.h

@@ -28,8 +28,7 @@ public:
     Q_INVOKABLE AnimationDetails getAnimationDetails();
     virtual void setSkeletonModelURL(const QUrl& skeletonModelURL) override;
 
-    virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
-                        bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector<JointData>* sentJointDataOut = nullptr) override;
+    virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail) override;
 
     
 private slots:

assignment-client/src/scripts/EntityScriptServer.cpp

@@ -324,16 +324,8 @@ void EntityScriptServer::nodeActivated(SharedNodePointer activatedNode) {
 void EntityScriptServer::nodeKilled(SharedNodePointer killedNode) {
     switch (killedNode->getType()) {
         case NodeType::EntityServer: {
-            if (!_shuttingDown) {
-                if (_entitiesScriptEngine) {
-                    _entitiesScriptEngine->unloadAllEntityScripts();
-                    _entitiesScriptEngine->stop();
-                }
-
-                resetEntitiesScriptEngine();
-
-                _entityViewer.clear();
-            }
+            clear();
+            
             break;
         }
         case NodeType::Agent: {
@@ -440,12 +432,12 @@ void EntityScriptServer::clear() {
         _entitiesScriptEngine->stop();
     }
 
+    _entityViewer.clear();
+
     // reset the engine
     if (!_shuttingDown) {
         resetEntitiesScriptEngine();
     }
-
-    _entityViewer.clear();
 }
 
 void EntityScriptServer::shutdownScriptEngine() {

domain-server/resources/describe-settings.json

@@ -1299,6 +1299,22 @@
           "placeholder": 5.0,
           "default": 5.0,
           "advanced": true
+        },
+        {
+          "name": "auto_threads",
+          "label": "Automatically determine thread count",
+          "type": "checkbox",
+          "help": "Allow system to determine number of threads (recommended)",
+          "default": false,
+          "advanced": true
+        },
+        {
+          "name": "num_threads",
+          "label": "Number of Threads",
+          "help": "Threads to spin up for avatar mixing (if not automatically set)",
+          "placeholder": "1",
+          "default": "1",
+          "advanced": true
         }
       ]
     }

domain-server/resources/web/settings/js/settings.js

@@ -996,6 +996,10 @@ function saveSettings() {
       if (password && password.length > 0) {
         formJSON["security"]["http_password"] = sha256_digest(password);
       }
+      var verify_password = formJSON["security"]["verify_http_password"];
+      if (verify_password && verify_password.length > 0) {
+        formJSON["security"]["verify_http_password"] = sha256_digest(verify_password);
+      }
     }
 
     // verify that the password and confirmation match before saving
@@ -1010,7 +1014,6 @@ function saveSettings() {
           bootbox.alert({"message": "Passwords must match!", "title":"Password Error"});
           canPost = false;
         } else {
-          formJSON["security"]["http_password"] = sha256_digest(password);
           delete formJSON["security"]["verify_http_password"];
         }
       }

interface/resources/avatar/avatar-animation.json

@@ -128,7 +128,41 @@
                                                     "id": "rightHandGrasp",
                                                     "interpTarget": 3,
                                                     "interpDuration": 3,
-                                                    "transitions": []
+                                                    "transitions": [
+                                                        { "var": "isRightIndexPoint", "state": "rightIndexPoint" },
+                                                        { "var": "isRightThumbRaise", "state": "rightThumbRaise" },
+                                                        { "var": "isRightIndexPointAndThumbRaise", "state": "rightIndexPointAndThumbRaise" }
+                                                    ]
+                                                },
+                                                {
+                                                    "id": "rightIndexPoint",
+                                                    "interpTarget": 15,
+                                                    "interpDuration": 3,
+                                                    "transitions": [
+                                                        { "var": "isRightHandGrasp", "state": "rightHandGrasp" },
+                                                        { "var": "isRightThumbRaise", "state": "rightThumbRaise" },
+                                                        { "var": "isRightIndexPointAndThumbRaise", "state": "rightIndexPointAndThumbRaise" }
+                                                    ]
+                                                },
+                                                {
+                                                    "id": "rightThumbRaise",
+                                                    "interpTarget": 15,
+                                                    "interpDuration": 3,
+                                                    "transitions": [
+                                                        { "var": "isRightHandGrasp", "state": "rightHandGrasp" },
+                                                        { "var": "isRightIndexPoint", "state": "rightIndexPoint" },
+                                                        { "var": "isRightIndexPointAndThumbRaise", "state": "rightIndexPointAndThumbRaise" }
+                                                    ]
+                                                },
+                                                {
+                                                    "id": "rightIndexPointAndThumbRaise",
+                                                    "interpTarget": 15,
+                                                    "interpDuration": 3,
+                                                    "transitions": [
+                                                        { "var": "isRightHandGrasp", "state": "rightHandGrasp" },
+                                                        { "var": "isRightIndexPoint", "state": "rightIndexPoint" },
+                                                        { "var": "isRightThumbRaise", "state": "rightThumbRaise" }
+                                                    ]
                                                 }
                                             ]
                                         },
@@ -166,6 +200,108 @@
                                                         "children": []
                                                     }
                                                 ]
+                                            },
+                                            {
+                                                "id": "rightIndexPoint",
+                                                "type": "blendLinear",
+                                                "data":  {
+                                                    "alpha": 0.0,
+                                                    "alphaVar": "rightHandGraspAlpha"
+                                                },
+                                                "children": [
+                                                    {
+                                                        "id": "rightIndexPointOpen",
+                                                        "type": "clip",
+                                                        "data": {
+                                                            "url": "animations/touch_point_open_right.fbx",
+                                                            "startFrame": 15.0,
+                                                            "endFrame": 15.0,
+                                                            "timeScale": 1.0,
+                                                            "loopFlag": true
+                                                        },
+                                                        "children": []
+                                                    },
+                                                    {
+                                                        "id": "rightIndexPointClosed",
+                                                        "type": "clip",
+                                                        "data": {
+                                                            "url": "animations/touch_point_closed_right.fbx",
+                                                            "startFrame": 15.0,
+                                                            "endFrame": 15.0,
+                                                            "timeScale": 1.0,
+                                                            "loopFlag": true
+                                                        },
+                                                        "children": []
+                                                    }
+                                                ]
+                                            },
+                                            {
+                                                "id": "rightThumbRaise",
+                                                "type": "blendLinear",
+                                                "data":  {
+                                                    "alpha": 0.0,
+                                                    "alphaVar": "rightHandGraspAlpha"
+                                                },
+                                                "children": [
+                                                    {
+                                                        "id": "rightThumbRaiseOpen",
+                                                        "type": "clip",
+                                                        "data": {
+                                                            "url": "animations/touch_thumb_open_right.fbx",
+                                                            "startFrame": 15.0,
+                                                            "endFrame": 15.0,
+                                                            "timeScale": 1.0,
+                                                            "loopFlag": true
+                                                        },
+                                                        "children": []
+                                                    },
+                                                    {
+                                                        "id": "rightThumbRaiseClosed",
+                                                        "type": "clip",
+                                                        "data": {
+                                                            "url": "animations/touch_thumb_closed_right.fbx",
+                                                            "startFrame": 15.0,
+                                                            "endFrame": 15.0,
+                                                            "timeScale": 1.0,
+                                                            "loopFlag": true
+                                                        },
+                                                        "children": []
+                                                    }
+                                                ]
+                                            },
+                                            {
+                                                "id": "rightIndexPointAndThumbRaise",
+                                                "type": "blendLinear",
+                                                "data":  {
+                                                    "alpha": 0.0,
+                                                    "alphaVar": "rightHandGraspAlpha"
+                                                },
+                                                "children": [
+                                                    {
+                                                        "id": "rightIndexPointAndThumbRaiseOpen",
+                                                        "type": "clip",
+                                                        "data": {
+                                                            "url": "animations/touch_thumb_point_open_right.fbx",
+                                                            "startFrame": 15.0,
+                                                            "endFrame": 15.0,
+                                                            "timeScale": 1.0,
+                                                            "loopFlag": true
+                                                        },
+                                                        "children": []
+                                                    },
+                                                    {
+                                                        "id": "rightIndexPointAndThumbRaiseClosed",
+                                                        "type": "clip",
+                                                        "data": {
+                                                            "url": "animations/touch_thumb_point_closed_right.fbx",
+                                                            "startFrame": 15.0,
+                                                            "endFrame": 15.0,
+                                                            "timeScale": 1.0,
+                                                            "loopFlag": true
+                                                        },
+                                                        "children": []
+                                                    }
+                                                ]
                                             }
                                         ]
                                     },
@@ -175,7 +311,7 @@
                                         "data": {
                                             "alpha": 0.0,
                                             "boneSet": "leftHand",
-                                            "alphaVar" : "leftHandOverlayAlpha"
+                                            "alphaVar": "leftHandOverlayAlpha"
                                         },
                                         "children": [
                                             {
@@ -188,7 +324,41 @@
                                                             "id": "leftHandGrasp",
                                                             "interpTarget": 3,
                                                             "interpDuration": 3,
-                                                            "transitions": []
+                                                            "transitions": [
+                                                                { "var": "isLeftIndexPoint", "state": "leftIndexPoint" },
+                                                                { "var": "isLeftThumbRaise", "state": "leftThumbRaise" },
+                                                                { "var": "isLeftIndexPointAndThumbRaise", "state": "leftIndexPointAndThumbRaise" }
+                                                            ]
+                                                        },
+                                                        {
+                                                            "id": "leftIndexPoint",
+                                                            "interpTarget": 15,
+                                                            "interpDuration": 3,
+                                                            "transitions": [
+                                                                { "var": "isLeftHandGrasp", "state": "leftHandGrasp" },
+                                                                { "var": "isLeftThumbRaise", "state": "leftThumbRaise" },
+                                                                { "var": "isLeftIndexPointAndThumbRaise", "state": "leftIndexPointAndThumbRaise" }
+                                                            ]
+                                                        },
+                                                        {
+                                                            "id": "leftThumbRaise",
+                                                            "interpTarget": 15,
+                                                            "interpDuration": 3,
+                                                            "transitions": [
+                                                                { "var": "isLeftHandGrasp", "state": "leftHandGrasp" },
+                                                                { "var": "isLeftIndexPoint", "state": "leftIndexPoint" },
+                                                                { "var": "isLeftIndexPointAndThumbRaise", "state": "leftIndexPointAndThumbRaise" }
+                                                            ]
+                                                        },
+                                                        {
+                                                            "id": "leftIndexPointAndThumbRaise",
+                                                            "interpTarget": 15,
+                                                            "interpDuration": 3,
+                                                            "transitions": [
+                                                                { "var": "isLeftHandGrasp", "state": "leftHandGrasp" },
+                                                                { "var": "isLeftIndexPoint", "state": "leftIndexPoint" },
+                                                                { "var": "isLeftThumbRaise", "state": "leftThumbRaise" }
+                                                            ]
                                                         }
                                                     ]
                                                 },
@@ -226,6 +396,108 @@
                                                                 "children": []
                                                             }
                                                         ]
+                                                    },
+                                                    {
+                                                        "id": "leftIndexPoint",
+                                                        "type": "blendLinear",
+                                                        "data":  {
+                                                            "alpha": 0.0,
+                                                            "alphaVar": "leftHandGraspAlpha"
+                                                        },
+                                                        "children": [
+                                                            {
+                                                                "id": "leftIndexPointOpen",
+                                                                "type": "clip",
+                                                                "data": {
+                                                                    "url": "animations/touch_point_open_left.fbx",
+                                                                    "startFrame": 15.0,
+                                                                    "endFrame": 15.0,
+                                                                    "timeScale": 1.0,
+                                                                    "loopFlag": true
+                                                                },
+                                                                "children": []
+                                                            },
+                                                            {
+                                                                "id": "leftIndexPointClosed",
+                                                                "type": "clip",
+                                                                "data": {
+                                                                    "url": "animations/touch_point_closed_left.fbx",
+                                                                    "startFrame": 15.0,
+                                                                    "endFrame": 15.0,
+                                                                    "timeScale": 1.0,
+                                                                    "loopFlag": true
+                                                                },
+                                                                "children": []
+                                                            }
+                                                        ]
+                                                    },
+                                                    {
+                                                        "id": "leftThumbRaise",
+                                                        "type": "blendLinear",
+                                                        "data":  {
+                                                            "alpha": 0.0,
+                                                            "alphaVar": "leftHandGraspAlpha"
+                                                        },
+                                                        "children": [
+                                                            {
+                                                                "id": "leftThumbRaiseOpen",
+                                                                "type": "clip",
+                                                                "data": {
+                                                                    "url": "animations/touch_thumb_open_left.fbx",
+                                                                    "startFrame": 15.0,
+                                                                    "endFrame": 15.0,
+                                                                    "timeScale": 1.0,
+                                                                    "loopFlag": true
+                                                                },
+                                                                "children": []
+                                                            },
+                                                            {
+                                                                "id": "leftThumbRaiseClosed",
+                                                                "type": "clip",
+                                                                "data": {
+                                                                    "url": "animations/touch_thumb_closed_left.fbx",
+                                                                    "startFrame": 15.0,
+                                                                    "endFrame": 15.0,
+                                                                    "timeScale": 1.0,
+                                                                    "loopFlag": true
+                                                                },
+                                                                "children": []
+                                                            }
+                                                        ]
+                                                    },
+                                                    {
+                                                        "id": "leftIndexPointAndThumbRaise",
+                                                        "type": "blendLinear",
+                                                        "data":  {
+                                                            "alpha": 0.0,
+                                                            "alphaVar": "leftHandGraspAlpha"
+                                                        },
+                                                        "children": [
+                                                            {
+                                                                "id": "leftIndexPointAndThumbRaiseOpen",
+                                                                "type": "clip",
+                                                                "data": {
+                                                                    "url": "animations/touch_thumb_point_open_left.fbx",
+                                                                    "startFrame": 15.0,
+                                                                    "endFrame": 15.0,
+                                                                    "timeScale": 1.0,
+                                                                    "loopFlag": true
+                                                                },
+                                                                "children": []
+                                                            },
+                                                            {
+                                                                "id": "leftIndexPointAndThumbRaiseClosed",
+                                                                "type": "clip",
+                                                                "data": {
+                                                                    "url": "animations/touch_thumb_point_closed_left.fbx",
+                                                                    "startFrame": 15.0,
+                                                                    "endFrame": 15.0,
+                                                                    "timeScale": 1.0,
+                                                                    "loopFlag": true
+                                                                },
+                                                                "children": []
+                                                            }
+                                                        ]
                                                     }
                                                 ]
                                             },

interface/resources/qml/Stats.qml

@@ -107,11 +107,11 @@ Item {
                     }
                     StatText {
                         visible: root.expanded
-                        text: "Fully Simulated Avatars: " + root.fullySimulatedAvatarCount
+                        text: "Avatars Updated: " + root.updatedAvatarCount
                     }
                     StatText {
                         visible: root.expanded
-                        text: "Partially Simulated Avatars: " + root.partiallySimulatedAvatarCount
+                        text: "Avatars NOT Updated: " + root.notUpdatedAvatarCount
                     }
                 }
             }

interface/resources/qml/hifi/Pal.qml

@@ -13,6 +13,7 @@
 
 import QtQuick 2.5
 import QtQuick.Controls 1.4
+import Qt.labs.settings 1.0
 import "../styles-uit"
 import "../controls-uit" as HifiControls
 
@@ -29,7 +30,9 @@ Rectangle {
     property int myCardHeight: 90
     property int rowHeight: 70
     property int actionButtonWidth: 55
-    property int nameCardWidth: palContainer.width - actionButtonWidth*(iAmAdmin ? 4 : 2) - 4 - hifi.dimensions.scrollbarBackgroundWidth
+    property int actionButtonAllowance: actionButtonWidth * 2
+    property int minNameCardWidth: palContainer.width - (actionButtonAllowance * 2) - 4 - hifi.dimensions.scrollbarBackgroundWidth
+    property int nameCardWidth: minNameCardWidth + (iAmAdmin ? 0 : actionButtonAllowance)
     property var myData: ({displayName: "", userName: "", audioLevel: 0.0, admin: true}) // valid dummy until set
     property var ignored: ({}); // Keep a local list of ignored avatars & their data. Necessary because HashMap is slow to respond after ignoring.
     property var userModelData: [] // This simple list is essentially a mirror of the userModel listModel without all the extra complexities.
@@ -52,6 +55,16 @@ Rectangle {
         letterboxMessage.visible = true
         letterboxMessage.popupRadius = 0
     }
+    Settings {
+        id: settings
+        category: "pal"
+        property bool filtered: false
+        property int nearDistance: 30
+    }
+    function refreshWithFilter() {
+        // We should just be able to set settings.filtered to filter.checked, but see #3249, so send to .js for saving.
+        pal.sendToScript({method: 'refresh', params: {filter: filter.checked && {distance: settings.nearDistance}}});
+    }
 
     // This is the container for the PAL
     Rectangle {
@@ -88,11 +101,32 @@ Rectangle {
             audioLevel: myData.audioLevel
             isMyCard: true
             // Size
-            width: nameCardWidth
+            width: minNameCardWidth
             height: parent.height
             // Anchors
             anchors.left: parent.left
         }
+        Row {
+            HifiControls.CheckBox {
+                id: filter
+                checked: settings.filtered
+                text: "in view"
+                boxSize: reload.height * 0.70
+                onCheckedChanged: refreshWithFilter()
+            }
+            HifiControls.GlyphButton {
+                id: reload
+                glyph: hifi.glyphs.reload
+                width: reload.height
+                onClicked: refreshWithFilter()
+            }
+            spacing: 50
+            anchors {
+                right: parent.right
+                top: parent.top
+                topMargin: 10
+            }
+        }
     }
     // Rectangles used to cover up rounded edges on bottom of MyInfo Rectangle
     Rectangle {
@@ -306,45 +340,7 @@ Rectangle {
             }
         }
     }
-    // Refresh button
-    Rectangle {
-        // Size
-        width: hifi.dimensions.tableHeaderHeight-1
-        height: hifi.dimensions.tableHeaderHeight-1
-        // Anchors
-        anchors.left: table.left
-        anchors.leftMargin: 4
-        anchors.top: table.top
-        // Style
-        color: hifi.colors.tableBackgroundLight
-        // Actual refresh icon
-        HiFiGlyphs {
-            id: reloadButton
-            text: hifi.glyphs.reloadSmall
-            // Size
-            size: parent.width*1.5
-            // Anchors
-            anchors.fill: parent
-            // Style
-            horizontalAlignment: Text.AlignHCenter
-            color: hifi.colors.darkGray
-        }
-        MouseArea {
-            id: reloadButtonArea
-            // Anchors
-            anchors.fill: parent
-            hoverEnabled: true
-            // Everyone likes a responsive refresh button!
-            // So use onPressed instead of onClicked
-            onPressed: {
-                reloadButton.color = hifi.colors.lightGrayText
-                pal.sendToScript({method: 'refresh'})
-            }
-            onReleased: reloadButton.color = (containsMouse ? hifi.colors.baseGrayHighlight : hifi.colors.darkGray)
-            onEntered: reloadButton.color = hifi.colors.baseGrayHighlight
-            onExited: reloadButton.color = (pressed ?  hifi.colors.lightGrayText: hifi.colors.darkGray)
-        }
-    }
+
     // Separator between user and admin functions
     Rectangle {
         // Size
@@ -501,7 +497,7 @@ Rectangle {
                 if (alreadyRefreshed === true) {
                     letterbox('', '', 'The last editor of this object is either you or not among this list of users.');
                 } else {
-                    pal.sendToScript({method: 'refresh', params: message.params});
+                    pal.sendToScript({method: 'refresh', params: {selected: message.params}});
                 }
             } else {
                 // If we've already refreshed the PAL and found the avatar in the model

interface/resources/qml/hifi/toolbars/Toolbar.qml

@@ -120,6 +120,8 @@ Window {
     function addButton(properties) {
         properties = properties || {}
 
+        unpinnedAlpha = 1;
+
         // If a name is specified, then check if there's an existing button with that name
         // and return it if so.  This will allow multiple clients to listen to a single button,
         // and allow scripts to be idempotent so they don't duplicate buttons if they're reloaded
@@ -154,7 +156,7 @@ Window {
         updatePinned();
 
         if (buttons.length === 0) {
-            visible = false;
+            unpinnedAlpha = 0;
         }
     }
 

interface/src/Application.cpp

@@ -548,6 +548,7 @@ const float DEFAULT_HMD_TABLET_SCALE_PERCENT = 100.0f;
 const float DEFAULT_DESKTOP_TABLET_SCALE_PERCENT = 75.0f;
 const bool DEFAULT_DESKTOP_TABLET_BECOMES_TOOLBAR = true;
 const bool DEFAULT_HMD_TABLET_BECOMES_TOOLBAR = false;
+const bool DEFAULT_TABLET_VISIBLE_TO_OTHERS = false;
 
 Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bool runServer, QString runServerPathOption) :
     QApplication(argc, argv),
@@ -570,6 +571,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
     _desktopTabletScale("desktopTabletScale", DEFAULT_DESKTOP_TABLET_SCALE_PERCENT),
     _desktopTabletBecomesToolbarSetting("desktopTabletBecomesToolbar", DEFAULT_DESKTOP_TABLET_BECOMES_TOOLBAR),
     _hmdTabletBecomesToolbarSetting("hmdTabletBecomesToolbar", DEFAULT_HMD_TABLET_BECOMES_TOOLBAR),
+    _tabletVisibleToOthersSetting("tabletVisibleToOthers", DEFAULT_TABLET_VISIBLE_TO_OTHERS),
     _constrainToolbarPosition("toolbar/constrainToolbarToCenterX", true),
     _scaleMirror(1.0f),
     _rotateMirror(0.0f),
@@ -781,6 +783,11 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
     connect(&domainHandler, SIGNAL(connectedToDomain(const QString&)), SLOT(updateWindowTitle()));
     connect(&domainHandler, SIGNAL(disconnectedFromDomain()), SLOT(updateWindowTitle()));
     connect(&domainHandler, SIGNAL(disconnectedFromDomain()), SLOT(clearDomainOctreeDetails()));
+    connect(&domainHandler, &DomainHandler::disconnectedFromDomain, this, [this]() {
+        getOverlays().deleteOverlay(getTabletScreenID());
+        getOverlays().deleteOverlay(getTabletHomeButtonID());
+        getOverlays().deleteOverlay(getTabletFrameID());
+    });
     connect(&domainHandler, &DomainHandler::domainConnectionRefused, this, &Application::domainConnectionRefused);
 
     // We could clear ATP assets only when changing domains, but it's possible that the domain you are connected
@@ -1216,6 +1223,8 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
         if (entity && entity->wantsKeyboardFocus()) {
             setKeyboardFocusOverlay(UNKNOWN_OVERLAY_ID);
             setKeyboardFocusEntity(entityItemID);
+        } else {
+            setKeyboardFocusEntity(UNKNOWN_ENTITY_ID);
         }
     });
 
@@ -2348,6 +2357,11 @@ void Application::setHmdTabletBecomesToolbarSetting(bool value) {
     updateSystemTabletMode();
 }
 
+void Application::setTabletVisibleToOthersSetting(bool value) {
+    _tabletVisibleToOthersSetting.set(value);
+    updateSystemTabletMode();
+}
+
 void Application::setSettingConstrainToolbarPosition(bool setting) {
     _constrainToolbarPosition.set(setting);
     DependencyManager::get<OffscreenUi>()->setConstrainToolbarToCenterX(setting);
@@ -3093,6 +3107,7 @@ void Application::mouseMoveEvent(QMouseEvent* event) {
         getOverlays().mouseMoveEvent(&mappedEvent);
         getEntities()->mouseMoveEvent(&mappedEvent);
     }
+
     _controllerScriptingInterface->emitMouseMoveEvent(&mappedEvent); // send events to any registered scripts
 
     // if one of our scripts have asked to capture this event, then stop processing it
@@ -3125,7 +3140,6 @@ void Application::mousePressEvent(QMouseEvent* event) {
 
     if (!_aboutToQuit) {
         getOverlays().mousePressEvent(&mappedEvent);
-
         if (!_controllerScriptingInterface->areEntityClicksCaptured()) {
             getEntities()->mousePressEvent(&mappedEvent);
         }
@@ -3429,7 +3443,7 @@ void Application::idle(float nsecsElapsed) {
 #ifdef Q_OS_WIN
     static std::once_flag once;
     std::call_once(once, [] {
-        initCpuUsage(); 
+        initCpuUsage();
     });
 
     vec3 kernelUserAndSystem;
@@ -6902,5 +6916,10 @@ OverlayID Application::getTabletScreenID() const {
 
 OverlayID Application::getTabletHomeButtonID() const {
     auto HMD = DependencyManager::get<HMDScriptingInterface>();
-    return HMD->getCurrentHomeButtonUUID();
+    return HMD->getCurrentHomeButtonID();
+}
+
+QUuid Application::getTabletFrameID() const {
+    auto HMD = DependencyManager::get<HMDScriptingInterface>();
+    return HMD->getCurrentTabletFrameID();
 }

interface/src/Application.h

@@ -218,6 +218,8 @@ public:
     void setDesktopTabletBecomesToolbarSetting(bool value);
     bool getHmdTabletBecomesToolbarSetting() { return _hmdTabletBecomesToolbarSetting.get(); }
     void setHmdTabletBecomesToolbarSetting(bool value);
+    bool getTabletVisibleToOthersSetting() { return _tabletVisibleToOthersSetting.get(); }
+    void setTabletVisibleToOthersSetting(bool value);
 
     float getSettingConstrainToolbarPosition() { return _constrainToolbarPosition.get(); }
     void setSettingConstrainToolbarPosition(bool setting);
@@ -300,6 +302,7 @@ public:
 
     OverlayID getTabletScreenID() const;
     OverlayID getTabletHomeButtonID() const;
+    QUuid getTabletFrameID() const; // may be an entity or an overlay
 
 signals:
     void svoImportRequested(const QString& url);
@@ -561,6 +564,7 @@ private:
     Setting::Handle<float> _desktopTabletScale;
     Setting::Handle<bool> _desktopTabletBecomesToolbarSetting;
     Setting::Handle<bool> _hmdTabletBecomesToolbarSetting;
+    Setting::Handle<bool> _tabletVisibleToOthersSetting;
     Setting::Handle<bool> _constrainToolbarPosition;
 
     float _scaleMirror;

interface/src/Camera.cpp

@@ -192,6 +192,8 @@ QVariantMap Camera::getViewFrustum() {
     result["orientation"].setValue(frustum.getOrientation());
     result["projection"].setValue(frustum.getProjection());
     result["centerRadius"].setValue(frustum.getCenterRadius());
+    result["fieldOfView"].setValue(frustum.getFieldOfView());
+    result["aspectRatio"].setValue(frustum.getAspectRatio());
 
     return result;
 }

interface/src/avatar/Avatar.cpp

@@ -334,11 +334,6 @@ void Avatar::updateAvatarEntities() {
     setAvatarEntityDataChanged(false);
 }
 
-bool Avatar::shouldDie() const {
-    const qint64 AVATAR_SILENCE_THRESHOLD_USECS = 5 * USECS_PER_SECOND;
-    return _owningAvatarMixer.isNull() || getUsecsSinceLastUpdate() > AVATAR_SILENCE_THRESHOLD_USECS;
-}
-
 void Avatar::simulate(float deltaTime, bool inView) {
     PROFILE_RANGE(simulation, "simulate");
 
@@ -938,6 +933,10 @@ glm::vec3 Avatar::getDefaultJointTranslation(int index) const {
 }
 
 glm::quat Avatar::getAbsoluteJointRotationInObjectFrame(int index) const {
+    if (index < 0) {
+        index += numeric_limits<unsigned short>::max() + 1; // 65536
+    }
+
     switch(index) {
         case SENSOR_TO_WORLD_MATRIX_INDEX: {
             glm::mat4 sensorToWorldMatrix = getSensorToWorldMatrix();
@@ -974,6 +973,10 @@ glm::quat Avatar::getAbsoluteJointRotationInObjectFrame(int index) const {
 }
 
 glm::vec3 Avatar::getAbsoluteJointTranslationInObjectFrame(int index) const {
+    if (index < 0) {
+        index += numeric_limits<unsigned short>::max() + 1; // 65536
+    }
+
     switch(index) {
         case SENSOR_TO_WORLD_MATRIX_INDEX: {
             glm::mat4 sensorToWorldMatrix = getSensorToWorldMatrix();

interface/src/avatar/Avatar.h

@@ -178,12 +178,13 @@ public:
     uint64_t getLastRenderUpdateTime() const { return _lastRenderUpdateTime; }
     void setLastRenderUpdateTime(uint64_t time) { _lastRenderUpdateTime = time; }
 
-    bool shouldDie() const;
     void animateScaleChanges(float deltaTime);
     void setTargetScale(float targetScale) override;
 
     Q_INVOKABLE float getSimulationRate(const QString& rateName = QString("")) const;
 
+    bool hasNewJointData() const { return _hasNewJointData; }
+
 public slots:
 
     // FIXME - these should be migrated to use Pose data instead

interface/src/avatar/AvatarManager.cpp

@@ -85,7 +85,7 @@ AvatarManager::AvatarManager(QObject* parent) :
     // immediately remove that avatar instead of waiting for the absence of packets from avatar mixer
     connect(nodeList.data(), &NodeList::ignoredNode, this, [=](const QUuid& nodeID, bool enabled) {
         if (enabled) {
-            removeAvatar(nodeID);
+            removeAvatar(nodeID, KillAvatarReason::AvatarIgnored);
         }
     });
 }
@@ -148,16 +148,6 @@ float AvatarManager::getAvatarSimulationRate(const QUuid& sessionID, const QStri
 }
 
 
-
-class AvatarPriority {
-public:
-    AvatarPriority(AvatarSharedPointer a, float p) : avatar(a), priority(p) {}
-    AvatarSharedPointer avatar;
-    float priority;
-    // NOTE: we invert the less-than operator to sort high priorities to front
-    bool operator<(const AvatarPriority& other) const { return priority > other.priority; }
-};
-
 void AvatarManager::updateOtherAvatars(float deltaTime) {
     // lock the hash for read to check the size
     QReadLocker lock(&_hashLock);
@@ -173,57 +163,35 @@ void AvatarManager::updateOtherAvatars(float deltaTime) {
     QList<AvatarSharedPointer> avatarList = avatarMap.values();
     ViewFrustum cameraView;
     qApp->copyDisplayViewFrustum(cameraView);
-    glm::vec3 frustumCenter = cameraView.getPosition();
 
-    const float OUT_OF_VIEW_PENALTY = -10.0;
+    std::priority_queue<AvatarPriority> sortedAvatars = AvatarData::sortAvatars(
+        avatarList, cameraView,
 
-    std::priority_queue<AvatarPriority> sortedAvatars;
-    {
-        PROFILE_RANGE(simulation, "sort");
-        for (int32_t i = 0; i < avatarList.size(); ++i) {
-            const auto& avatar = std::static_pointer_cast<Avatar>(avatarList.at(i));
-            if (avatar == _myAvatar || !avatar->isInitialized()) {
+        [](AvatarSharedPointer avatar)->uint64_t{
+            return std::static_pointer_cast<Avatar>(avatar)->getLastRenderUpdateTime();
+        },
+
+        [](AvatarSharedPointer avatar)->float{
+            return std::static_pointer_cast<Avatar>(avatar)->getBoundingRadius();
+        },
+
+        [this](AvatarSharedPointer avatar)->bool{
+            const auto& castedAvatar = std::static_pointer_cast<Avatar>(avatar);
+            if (castedAvatar == _myAvatar || !castedAvatar->isInitialized()) {
                 // DO NOT update _myAvatar!  Its update has already been done earlier in the main loop.
                 // DO NOT update or fade out uninitialized Avatars
-                continue;
+                return true; // ignore it
             }
             if (avatar->shouldDie()) {
                 removeAvatar(avatar->getID());
-                continue;
+                return true; // ignore it
             }
             if (avatar->isDead()) {
-                continue;
+                return true; // ignore it
             }
 
-            // priority = weighted linear combination of:
-            //   (a) apparentSize
-            //   (b) proximity to center of view
-            //   (c) time since last update
-            //   (d) TIME_PENALTY to help recently updated entries sort toward back
-            glm::vec3 avatarPosition = avatar->getPosition();
-            glm::vec3 offset = avatarPosition - frustumCenter;
-            float distance = glm::length(offset) + 0.001f; // add 1mm to avoid divide by zero
-            float radius = avatar->getBoundingRadius();
-            const glm::vec3& forward = cameraView.getDirection();
-            float apparentSize = radius / distance;
-            float cosineAngle = glm::length(offset - glm::dot(offset, forward) * forward) / distance;
-            const float TIME_PENALTY = 0.080f; // seconds
-            float age = (float)(startTime - avatar->getLastRenderUpdateTime()) / (float)(USECS_PER_SECOND) - TIME_PENALTY;
-            // NOTE: we are adding values of different units to get a single measure of "priority".
-            // Thus we multiply each component by a conversion "weight" that scales its units
-            // relative to the others.  These weights are pure magic tuning and are hard coded in the
-            // relation below: (hint: unitary weights are not explicityly shown)
-            float priority = apparentSize + 0.25f * cosineAngle + age;
-
-            // decrement priority of avatars outside keyhole
-            if (distance > cameraView.getCenterRadius()) {
-                if (!cameraView.sphereIntersectsFrustum(avatarPosition, radius)) {
-                    priority += OUT_OF_VIEW_PENALTY;
-                }
-            }
-            sortedAvatars.push(AvatarPriority(avatar, priority));
-        }
-    }
+            return false;
+        });
 
     render::PendingChanges pendingChanges;
     const uint64_t RENDER_UPDATE_BUDGET = 1500; // usec
@@ -231,8 +199,8 @@ void AvatarManager::updateOtherAvatars(float deltaTime) {
     uint64_t renderExpiry = startTime + RENDER_UPDATE_BUDGET;
     uint64_t maxExpiry = startTime + MAX_UPDATE_BUDGET;
 
-    int fullySimulatedAvatars = 0;
-    int partiallySimulatedAvatars = 0;
+    int numAvatarsUpdated = 0;
+    int numAVatarsNotUpdated = 0;
     while (!sortedAvatars.empty()) {
         const AvatarPriority& sortData = sortedAvatars.top();
         const auto& avatar = std::static_pointer_cast<Avatar>(sortData.avatar);
@@ -253,33 +221,57 @@ void AvatarManager::updateOtherAvatars(float deltaTime) {
         }
         avatar->animateScaleChanges(deltaTime);
 
+        const float OUT_OF_VIEW_THRESHOLD = 0.5f * AvatarData::OUT_OF_VIEW_PENALTY;
         uint64_t now = usecTimestampNow();
         if (now < renderExpiry) {
             // we're within budget
-            const float OUT_OF_VIEW_THRESHOLD = 0.5f * OUT_OF_VIEW_PENALTY;
             bool inView = sortData.priority > OUT_OF_VIEW_THRESHOLD;
+            if (inView && avatar->hasNewJointData()) {
+                numAvatarsUpdated++;
+            }
             avatar->simulate(deltaTime, inView);
             avatar->updateRenderItem(pendingChanges);
             avatar->setLastRenderUpdateTime(startTime);
-            fullySimulatedAvatars++;
         } else if (now < maxExpiry) {
             // we've spent most of our time budget, but we still simulate() the avatar as it if were out of view
             // --> some avatars may freeze until their priority trickles up
-            const bool inView = false;
-            avatar->simulate(deltaTime, inView);
-            partiallySimulatedAvatars++;
+            bool inView = sortData.priority > OUT_OF_VIEW_THRESHOLD;
+            if (inView && avatar->hasNewJointData()) {
+                numAVatarsNotUpdated++;
+            }
+            avatar->simulate(deltaTime, false);
         } else {
             // we've spent ALL of our time budget --> bail on the rest of the avatar updates
+            // --> more avatars may freeze until their priority trickles up
             // --> some scale or fade animations may glitch
             // --> some avatar velocity measurements may be a little off
+
+            // HACK: no time simulate, but we will take the time to count how many were tragically missed
+            bool inView = sortData.priority > OUT_OF_VIEW_THRESHOLD;
+            if (!inView) {
+                break;
+            }
+            if (inView && avatar->hasNewJointData()) {
+                numAVatarsNotUpdated++;
+            }
+            sortedAvatars.pop();
+            while (inView && !sortedAvatars.empty()) {
+                const AvatarPriority& newSortData = sortedAvatars.top();
+                const auto& newAvatar = std::static_pointer_cast<Avatar>(newSortData.avatar);
+                inView = newSortData.priority > OUT_OF_VIEW_THRESHOLD;
+                if (inView && newAvatar->hasNewJointData()) {
+                    numAVatarsNotUpdated++;
+                }
+                sortedAvatars.pop();
+            }
             break;
         }
         sortedAvatars.pop();
     }
 
     _avatarSimulationTime = (float)(usecTimestampNow() - startTime) / (float)USECS_PER_MSEC;
-    _fullySimulatedAvatars = fullySimulatedAvatars;
-    _partiallySimulatedAvatars = partiallySimulatedAvatars;
+    _numAvatarsUpdated = numAvatarsUpdated;
+    _numAvatarsNotUpdated = numAVatarsNotUpdated;
     qApp->getMain3DScene()->enqueuePendingChanges(pendingChanges);
 
     simulateAvatarFades(deltaTime);
@@ -593,3 +585,44 @@ RayToAvatarIntersectionResult AvatarManager::findRayIntersection(const PickRay&
 
     return result;
 }
+
+// HACK
+float AvatarManager::getAvatarSortCoefficient(const QString& name) {
+    if (name == "size") {
+        return AvatarData::_avatarSortCoefficientSize;
+    } else if (name == "center") {
+        return AvatarData::_avatarSortCoefficientCenter;
+    } else if (name == "age") {
+        return AvatarData::_avatarSortCoefficientAge;
+    }
+    return 0.0f;
+}
+
+// HACK
+void AvatarManager::setAvatarSortCoefficient(const QString& name, const QScriptValue& value) {
+    bool somethingChanged = false;
+    if (value.isNumber()) {
+        float numericalValue = (float)value.toNumber();
+        if (name == "size") {
+            AvatarData::_avatarSortCoefficientSize = numericalValue;
+            somethingChanged = true;
+        } else if (name == "center") {
+            AvatarData::_avatarSortCoefficientCenter = numericalValue;
+            somethingChanged = true;
+        } else if (name == "age") {
+            AvatarData::_avatarSortCoefficientAge = numericalValue;
+            somethingChanged = true;
+        }
+    }
+    if (somethingChanged) {
+        size_t packetSize = sizeof(AvatarData::_avatarSortCoefficientSize) + 
+                            sizeof(AvatarData::_avatarSortCoefficientCenter) +
+                            sizeof(AvatarData::_avatarSortCoefficientAge);
+
+        auto packet = NLPacket::create(PacketType::AdjustAvatarSorting, packetSize);
+        packet->writePrimitive(AvatarData::_avatarSortCoefficientSize);
+        packet->writePrimitive(AvatarData::_avatarSortCoefficientCenter);
+        packet->writePrimitive(AvatarData::_avatarSortCoefficientAge);
+        DependencyManager::get<NodeList>()->broadcastToNodes(std::move(packet), NodeSet() << NodeType::AvatarMixer);
+    }
+}

interface/src/avatar/AvatarManager.h

@@ -43,8 +43,8 @@ public:
     std::shared_ptr<MyAvatar> getMyAvatar() { return _myAvatar; }
     AvatarSharedPointer getAvatarBySessionID(const QUuid& sessionID) const override;
 
-    int getFullySimulatedAvatars() const { return _fullySimulatedAvatars; }
-    int getPartiallySimulatedAvatars() const { return _partiallySimulatedAvatars; }
+    int getNumAvatarsUpdated() const { return _numAvatarsUpdated; }
+    int getNumAvatarsNotUpdated() const { return _numAvatarsNotUpdated; }
     float getAvatarSimulationTime() const { return _avatarSimulationTime; }
 
     void updateMyAvatar(float deltaTime);
@@ -81,6 +81,10 @@ public:
                                                                   const QScriptValue& avatarIdsToInclude = QScriptValue(),
                                                                   const QScriptValue& avatarIdsToDiscard = QScriptValue());
 
+    // TODO: remove this HACK once we settle on optimal default sort coefficients
+    Q_INVOKABLE float getAvatarSortCoefficient(const QString& name);
+    Q_INVOKABLE void setAvatarSortCoefficient(const QString& name, const QScriptValue& value);
+
     float getMyAvatarSendRate() const { return _myAvatarSendRate.rate(); }
 
 public slots:
@@ -116,10 +120,9 @@ private:
     VectorOfMotionStates _motionStatesToRemoveFromPhysics;
 
     RateCounter<> _myAvatarSendRate;
-    int _fullySimulatedAvatars { 0 };
-    int _partiallySimulatedAvatars { 0 };
+    int _numAvatarsUpdated { 0 };
+    int _numAvatarsNotUpdated { 0 };
     float _avatarSimulationTime { 0.0f };
-
 };
 
 Q_DECLARE_METATYPE(AvatarManager::LocalLight)

interface/src/avatar/MyAvatar.cpp

@@ -148,13 +148,22 @@ MyAvatar::MyAvatar(RigPointer rig) :
     auto player = DependencyManager::get<Deck>();
     auto recorder = DependencyManager::get<Recorder>();
     connect(player.data(), &Deck::playbackStateChanged, [=] {
-        if (player->isPlaying()) {
+        bool isPlaying = player->isPlaying();
+        if (isPlaying) {
             auto recordingInterface = DependencyManager::get<RecordingScriptingInterface>();
             if (recordingInterface->getPlayFromCurrentLocation()) {
                 setRecordingBasis();
             }
         } else {
             clearRecordingBasis();
+            useFullAvatarURL(_fullAvatarURLFromPreferences, _fullAvatarModelName);
+        }
+
+        auto audioIO = DependencyManager::get<AudioClient>();
+        audioIO->setIsPlayingBackRecording(isPlaying);
+
+        if (_rig) {
+            _rig->setEnableAnimations(!isPlaying);
         }
     });
 
@@ -180,8 +189,8 @@ MyAvatar::MyAvatar(RigPointer rig) :
 
         if (recordingInterface->getPlayerUseSkeletonModel() && dummyAvatar.getSkeletonModelURL().isValid() &&
             (dummyAvatar.getSkeletonModelURL() != getSkeletonModelURL())) {
-            // FIXME
-            //myAvatar->useFullAvatarURL()
+
+            setSkeletonModelURL(dummyAvatar.getSkeletonModelURL());
         }
 
         if (recordingInterface->getPlayerUseDisplayName() && dummyAvatar.getDisplayName() != getDisplayName()) {
@@ -204,6 +213,11 @@ MyAvatar::MyAvatar(RigPointer rig) :
             // head orientation
             _headData->setLookAtPosition(headData->getLookAtPosition());
         }
+
+        auto jointData = dummyAvatar.getRawJointData();
+        if (jointData.length() > 0 && _rig) {
+            _rig->copyJointsFromJointData(jointData);
+        }
     });
 
     connect(rig.get(), SIGNAL(onLoadComplete()), this, SIGNAL(onLoadComplete()));
@@ -227,8 +241,7 @@ void MyAvatar::simulateAttachments(float deltaTime) {
     // don't update attachments here, do it in harvestResultsFromPhysicsSimulation()
 }
 
-QByteArray MyAvatar::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
-                        bool distanceAdjust, glm::vec3 viewerPosition, QVector<JointData>* sentJointDataOut) {
+QByteArray MyAvatar::toByteArrayStateful(AvatarDataDetail dataDetail) {
     CameraMode mode = qApp->getCamera()->getMode();
     _globalPosition = getPosition();
     _globalBoundingBoxDimensions.x = _characterController.getCapsuleRadius();
@@ -239,12 +252,12 @@ QByteArray MyAvatar::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTi
         // fake the avatar position that is sent up to the AvatarMixer
         glm::vec3 oldPosition = getPosition();
         setPosition(getSkeletonPosition());
-        QByteArray array = AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut);
+        QByteArray array = AvatarData::toByteArrayStateful(dataDetail);
         // copy the correct position back
         setPosition(oldPosition);
         return array;
     }
-    return AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut);
+    return AvatarData::toByteArrayStateful(dataDetail);
 }
 
 void MyAvatar::centerBody() {
@@ -472,7 +485,9 @@ void MyAvatar::simulate(float deltaTime) {
     {
         PerformanceTimer perfTimer("joints");
         // copy out the skeleton joints from the model
-        _rig->copyJointsIntoJointData(_jointData);
+        if (_rigEnabled) {
+            _rig->copyJointsIntoJointData(_jointData);
+        }
     }
 
     {
@@ -808,7 +823,7 @@ void MyAvatar::saveData() {
     auto hmdInterface = DependencyManager::get<HMDScriptingInterface>();
     _avatarEntitiesLock.withReadLock([&] {
         for (auto entityID : _avatarEntityData.keys()) {
-            if (hmdInterface->getCurrentTabletUIID() == entityID) {
+            if (hmdInterface->getCurrentTabletFrameID() == entityID) {
                 // don't persist the tablet between domains / sessions
                 continue;
             }
@@ -2395,6 +2410,10 @@ glm::mat4 MyAvatar::computeCameraRelativeHandControllerMatrix(const glm::mat4& c
 }
 
 glm::quat MyAvatar::getAbsoluteJointRotationInObjectFrame(int index) const {
+    if (index < 0) {
+        index += numeric_limits<unsigned short>::max() + 1; // 65536
+    }
+
     switch (index) {
         case CONTROLLER_LEFTHAND_INDEX: {
             return getLeftHandControllerPoseInAvatarFrame().getRotation();
@@ -2428,6 +2447,10 @@ glm::quat MyAvatar::getAbsoluteJointRotationInObjectFrame(int index) const {
 }
 
 glm::vec3 MyAvatar::getAbsoluteJointTranslationInObjectFrame(int index) const {
+    if (index < 0) {
+        index += numeric_limits<unsigned short>::max() + 1; // 65536
+    }
+
     switch (index) {
         case CONTROLLER_LEFTHAND_INDEX: {
             return getLeftHandControllerPoseInAvatarFrame().getTranslation();

interface/src/avatar/MyAvatar.h

@@ -338,8 +338,7 @@ private:
     glm::quat getWorldBodyOrientation() const;
 
 
-    virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
-                            bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector<JointData>* sentJointDataOut = nullptr) override;
+    virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail) override;
 
     void simulate(float deltaTime);
     void updateFromTrackers(float deltaTime);
@@ -486,6 +485,7 @@ private:
     std::unordered_set<int> _headBoneSet;
     RigPointer _rig;
     bool _prevShouldDrawHead;
+    bool _rigEnabled { true };
 
     bool _enableDebugDrawDefaultPose { false };
     bool _enableDebugDrawAnimPose { false };

interface/src/scripting/HMDScriptingInterface.h

@@ -29,8 +29,8 @@ class HMDScriptingInterface : public AbstractHMDScriptingInterface, public Depen
     Q_PROPERTY(glm::quat orientation READ getOrientation)
     Q_PROPERTY(bool mounted READ isMounted)
     Q_PROPERTY(bool showTablet READ getShouldShowTablet)
-    Q_PROPERTY(QUuid tabletID READ getCurrentTabletUIID WRITE setCurrentTabletUIID)
-    Q_PROPERTY(QUuid homeButtonID READ getCurrentHomeButtonUUID WRITE setCurrentHomeButtonUUID)
+    Q_PROPERTY(QUuid tabletID READ getCurrentTabletFrameID WRITE setCurrentTabletFrameID)
+    Q_PROPERTY(QUuid homeButtonID READ getCurrentHomeButtonID WRITE setCurrentHomeButtonID)
     Q_PROPERTY(QUuid tabletScreenID READ getCurrentTabletScreenID WRITE setCurrentTabletScreenID)
 
 public:
@@ -90,11 +90,11 @@ public:
     void setShouldShowTablet(bool value) { _showTablet = value; }
     bool getShouldShowTablet() const { return _showTablet; }
 
-    void setCurrentTabletUIID(QUuid tabletID) { _tabletUIID = tabletID; }
-    QUuid getCurrentTabletUIID() const { return _tabletUIID; }
+    void setCurrentTabletFrameID(QUuid tabletID) { _tabletUIID = tabletID; }
+    QUuid getCurrentTabletFrameID() const { return _tabletUIID; }
 
-    void setCurrentHomeButtonUUID(QUuid homeButtonID) { _homeButtonID = homeButtonID; }
-    QUuid getCurrentHomeButtonUUID() const { return _homeButtonID; }
+    void setCurrentHomeButtonID(QUuid homeButtonID) { _homeButtonID = homeButtonID; }
+    QUuid getCurrentHomeButtonID() const { return _homeButtonID; }
 
     void setCurrentTabletScreenID(QUuid tabletID) { _tabletScreenID = tabletID; }
     QUuid getCurrentTabletScreenID() const { return _tabletScreenID; }

interface/src/ui/PreferencesDialog.cpp

@@ -102,7 +102,11 @@ void setupPreferences() {
         auto setter = [](bool value) { qApp->setHmdTabletBecomesToolbarSetting(value); };
         preferences->addPreference(new CheckPreference(UI_CATEGORY, "HMD Tablet Becomes Toolbar", getter, setter));
     }
-
+    {
+        auto getter = []()->bool { return qApp->getTabletVisibleToOthersSetting(); };
+        auto setter = [](bool value) { qApp->setTabletVisibleToOthersSetting(value); };
+        preferences->addPreference(new CheckPreference(UI_CATEGORY, "Tablet Is Visible To Others", getter, setter));
+    }
     // Snapshots
     static const QString SNAPSHOTS { "Snapshots" };
     {

interface/src/ui/Stats.cpp

@@ -121,8 +121,8 @@ void Stats::updateStats(bool force) {
     auto avatarManager = DependencyManager::get<AvatarManager>();
     // we need to take one avatar out so we don't include ourselves
     STAT_UPDATE(avatarCount, avatarManager->size() - 1);
-    STAT_UPDATE(fullySimulatedAvatarCount, avatarManager->getFullySimulatedAvatars());
-    STAT_UPDATE(partiallySimulatedAvatarCount, avatarManager->getPartiallySimulatedAvatars());
+    STAT_UPDATE(updatedAvatarCount, avatarManager->getNumAvatarsUpdated());
+    STAT_UPDATE(notUpdatedAvatarCount, avatarManager->getNumAvatarsNotUpdated());
     STAT_UPDATE(serverCount, (int)nodeList->size());
     STAT_UPDATE(framerate, qApp->getFps());
     if (qApp->getActiveDisplayPlugin()) {

interface/src/ui/Stats.h

@@ -49,8 +49,8 @@ class Stats : public QQuickItem {
     STATS_PROPERTY(int, simrate, 0)
     STATS_PROPERTY(int, avatarSimrate, 0)
     STATS_PROPERTY(int, avatarCount, 0)
-    STATS_PROPERTY(int, fullySimulatedAvatarCount, 0)
-    STATS_PROPERTY(int, partiallySimulatedAvatarCount, 0)
+    STATS_PROPERTY(int, updatedAvatarCount, 0)
+    STATS_PROPERTY(int, notUpdatedAvatarCount, 0)
     STATS_PROPERTY(int, packetInCount, 0)
     STATS_PROPERTY(int, packetOutCount, 0)
     STATS_PROPERTY(float, mbpsIn, 0)
@@ -159,8 +159,8 @@ signals:
     void simrateChanged();
     void avatarSimrateChanged();
     void avatarCountChanged();
-    void fullySimulatedAvatarCountChanged();
-    void partiallySimulatedAvatarCountChanged();
+    void updatedAvatarCountChanged();
+    void notUpdatedAvatarCountChanged();
     void packetInCountChanged();
     void packetOutCountChanged();
     void mbpsInChanged();

interface/src/ui/overlays/Base3DOverlay.cpp

@@ -39,7 +39,8 @@ Base3DOverlay::Base3DOverlay(const Base3DOverlay* base3DOverlay) :
     _isDashedLine(base3DOverlay->_isDashedLine),
     _ignoreRayIntersection(base3DOverlay->_ignoreRayIntersection),
     _drawInFront(base3DOverlay->_drawInFront),
-    _isAA(base3DOverlay->_isAA)
+    _isAA(base3DOverlay->_isAA),
+    _isGrabbable(base3DOverlay->_isGrabbable)
 {
     setTransform(base3DOverlay->getTransform());
 }
@@ -59,15 +60,19 @@ QVariantMap convertOverlayLocationFromScriptSemantics(const QVariantMap& propert
     } else if (result["position"].isValid()) {
         glm::vec3 localPosition = SpatiallyNestable::worldToLocal(vec3FromVariant(result["position"]),
                                                                   parentID, parentJointIndex, success);
-        result["position"] = vec3toVariant(localPosition);
+        if (success) {
+            result["position"] = vec3toVariant(localPosition);
+        }
     }
 
     if (result["localOrientation"].isValid()) {
         result["orientation"] = result["localOrientation"];
     } else if (result["orientation"].isValid()) {
         glm::quat localOrientation = SpatiallyNestable::worldToLocal(quatFromVariant(result["orientation"]),
-                                                                  parentID, parentJointIndex, success);
-        result["orientation"] = quatToVariant(localOrientation);
+                                                                     parentID, parentJointIndex, success);
+        if (success) {
+            result["orientation"] = quatToVariant(localOrientation);
+        }
     }
 
     return result;
@@ -125,6 +130,11 @@ void Base3DOverlay::setProperties(const QVariantMap& originalProperties) {
         needRenderItemUpdate = true;
     }
 
+    auto isGrabbable = properties["grabbable"];
+    if (isGrabbable.isValid()) {
+        setIsGrabbable(isGrabbable.toBool());
+    }
+
     if (properties["position"].isValid()) {
         setLocalPosition(vec3FromVariant(properties["position"]));
         needRenderItemUpdate = true;
@@ -227,6 +237,9 @@ QVariant Base3DOverlay::getProperty(const QString& property) {
     if (property == "drawInFront") {
         return _drawInFront;
     }
+    if (property == "grabbable") {
+        return _isGrabbable;
+    }
     if (property == "parentID") {
         return getParentID();
     }
@@ -246,6 +259,8 @@ bool Base3DOverlay::findRayIntersection(const glm::vec3& origin, const glm::vec3
 }
 
 void Base3DOverlay::locationChanged(bool tellPhysics) {
+    SpatiallyNestable::locationChanged(tellPhysics);
+
     auto itemID = getRenderItemID();
     if (render::Item::isValidID(itemID)) {
         render::ScenePointer scene = qApp->getMain3DScene();
@@ -253,8 +268,6 @@ void Base3DOverlay::locationChanged(bool tellPhysics) {
         pendingChanges.updateItem(itemID);
         scene->enqueuePendingChanges(pendingChanges);
     }
-    // Overlays can't currently have children.
-    // SpatiallyNestable::locationChanged(tellPhysics); // tell all the children, also
 }
 
 void Base3DOverlay::parentDeleted() {

interface/src/ui/overlays/Base3DOverlay.h

@@ -38,6 +38,7 @@ public:
     bool getIsSolidLine() const { return !_isDashedLine; }
     bool getIgnoreRayIntersection() const { return _ignoreRayIntersection; }
     bool getDrawInFront() const { return _drawInFront; }
+    bool getIsGrabbable() const { return _isGrabbable; }
 
     virtual bool isAA() const { return _isAA; }
 
@@ -47,6 +48,7 @@ public:
     void setIgnoreRayIntersection(bool value) { _ignoreRayIntersection = value; }
     void setDrawInFront(bool value) { _drawInFront = value; }
     void setIsAA(bool value) { _isAA = value; }
+    void setIsGrabbable(bool value) { _isGrabbable = value; }
 
     virtual AABox getBounds() const override = 0;
 
@@ -71,6 +73,7 @@ protected:
     bool _ignoreRayIntersection;
     bool _drawInFront;
     bool _isAA;
+    bool _isGrabbable { false };
 };
 
 #endif // hifi_Base3DOverlay_h

interface/src/ui/overlays/Overlays.cpp

@@ -341,28 +341,18 @@ OverlayID Overlays::getOverlayAtPoint(const glm::vec2& point) {
         return UNKNOWN_OVERLAY_ID;
     }
     QMapIterator<OverlayID, Overlay::Pointer> i(_overlaysHUD);
-    i.toBack();
 
     const float LARGE_NEGATIVE_FLOAT = -9999999;
     glm::vec3 origin(pointCopy.x, pointCopy.y, LARGE_NEGATIVE_FLOAT);
     glm::vec3 direction(0, 0, 1);
-    BoxFace thisFace;
     glm::vec3 thisSurfaceNormal;
-    float distance;
     unsigned int bestStackOrder = 0;
     OverlayID bestOverlayID = UNKNOWN_OVERLAY_ID;
 
-    while (i.hasPrevious()) {
-        i.previous();
+    while (i.hasNext()) {
+        i.next();
         OverlayID thisID = i.key();
-        if (i.value()->is3D()) {
-            auto thisOverlay = std::dynamic_pointer_cast<Base3DOverlay>(i.value());
-            if (thisOverlay && !thisOverlay->getIgnoreRayIntersection()) {
-                if (thisOverlay->findRayIntersection(origin, direction, distance, thisFace, thisSurfaceNormal)) {
-                    return thisID;
-                }
-            }
-        } else {
+        if (!i.value()->is3D()) {
             auto thisOverlay = std::dynamic_pointer_cast<Overlay2D>(i.value());
             if (thisOverlay && thisOverlay->getVisible() && thisOverlay->isLoaded() &&
                 thisOverlay->getBoundingRect().contains(pointCopy.x, pointCopy.y, false)) {
@@ -406,16 +396,25 @@ RayToOverlayIntersectionResult Overlays::findRayIntersection(const PickRay& ray,
                                                              const QScriptValue& overlayIDsToInclude,
                                                              const QScriptValue& overlayIDsToDiscard,
                                                              bool visibleOnly, bool collidableOnly) {
-    float bestDistance = std::numeric_limits<float>::max();
-    bool bestIsFront = false;
     const QVector<OverlayID> overlaysToInclude = qVectorOverlayIDFromScriptValue(overlayIDsToInclude);
     const QVector<OverlayID> overlaysToDiscard = qVectorOverlayIDFromScriptValue(overlayIDsToDiscard);
 
+    return findRayIntersectionInternal(ray, precisionPicking,
+                                       overlaysToInclude, overlaysToDiscard, visibleOnly, collidableOnly);
+}
+
+
+RayToOverlayIntersectionResult Overlays::findRayIntersectionInternal(const PickRay& ray, bool precisionPicking,
+                                                                     const QVector<OverlayID>& overlaysToInclude,
+                                                                     const QVector<OverlayID>& overlaysToDiscard,
+                                                                     bool visibleOnly, bool collidableOnly) {
+    float bestDistance = std::numeric_limits<float>::max();
+    bool bestIsFront = false;
+
     RayToOverlayIntersectionResult result;
     QMapIterator<OverlayID, Overlay::Pointer> i(_overlaysWorld);
-    i.toBack();
-    while (i.hasPrevious()) {
-        i.previous();
+    while (i.hasNext()) {
+        i.next();
         OverlayID thisID = i.key();
         auto thisOverlay = std::dynamic_pointer_cast<Base3DOverlay>(i.value());
 
@@ -700,8 +699,9 @@ static PointerEvent::Button toPointerButton(const QMouseEvent& event) {
     }
 }
 
-PointerEvent Overlays::calculatePointerEvent(Overlay::Pointer overlay, PickRay ray, 
-    RayToOverlayIntersectionResult rayPickResult, QMouseEvent* event, PointerEvent::EventType eventType) {
+PointerEvent Overlays::calculatePointerEvent(Overlay::Pointer overlay, PickRay ray,
+                                             RayToOverlayIntersectionResult rayPickResult, QMouseEvent* event,
+                                             PointerEvent::EventType eventType) {
 
     auto thisOverlay = std::dynamic_pointer_cast<Web3DOverlay>(overlay);
 
@@ -719,11 +719,41 @@ PointerEvent Overlays::calculatePointerEvent(Overlay::Pointer overlay, PickRay r
     return pointerEvent;
 }
 
-void Overlays::mousePressEvent(QMouseEvent* event) {
+
+RayToOverlayIntersectionResult Overlays::findRayIntersectionForMouseEvent(PickRay ray) {
+    QVector<OverlayID> overlaysToInclude;
+    QVector<OverlayID> overlaysToDiscard;
+    RayToOverlayIntersectionResult rayPickResult;
+
+    // first priority is tablet screen
+    overlaysToInclude << qApp->getTabletScreenID();
+    rayPickResult = findRayIntersectionInternal(ray, true, overlaysToInclude, overlaysToDiscard);
+    if (rayPickResult.intersects) {
+        return rayPickResult;
+    }
+    // then tablet home button
+    overlaysToInclude.clear();
+    overlaysToInclude << qApp->getTabletHomeButtonID();
+    rayPickResult = findRayIntersectionInternal(ray, true, overlaysToInclude, overlaysToDiscard);
+    if (rayPickResult.intersects) {
+        return rayPickResult;
+    }
+    // then tablet frame
+    overlaysToInclude.clear();
+    overlaysToInclude << OverlayID(qApp->getTabletFrameID());
+    rayPickResult = findRayIntersectionInternal(ray, true, overlaysToInclude, overlaysToDiscard);
+    if (rayPickResult.intersects) {
+        return rayPickResult;
+    }
+    // then whatever
+    return findRayIntersection(ray);
+}
+
+bool Overlays::mousePressEvent(QMouseEvent* event) {
     PerformanceTimer perfTimer("Overlays::mousePressEvent");
 
     PickRay ray = qApp->computePickRay(event->x(), event->y());
-    RayToOverlayIntersectionResult rayPickResult = findRayIntersection(ray);
+    RayToOverlayIntersectionResult rayPickResult = findRayIntersectionForMouseEvent(ray);
     if (rayPickResult.intersects) {
         _currentClickingOnOverlayID = rayPickResult.overlayID;
 
@@ -732,19 +762,18 @@ void Overlays::mousePressEvent(QMouseEvent* event) {
         if (thisOverlay) {
             auto pointerEvent = calculatePointerEvent(thisOverlay, ray, rayPickResult, event, PointerEvent::Press);
             emit mousePressOnOverlay(_currentClickingOnOverlayID, pointerEvent);
-        } else {
-            emit mousePressOffOverlay();
+            return true;
         }
-    } else {
-        emit mousePressOffOverlay();
     }
+    emit mousePressOffOverlay();
+    return false;
 }
 
-void Overlays::mouseReleaseEvent(QMouseEvent* event) {
+bool Overlays::mouseReleaseEvent(QMouseEvent* event) {
     PerformanceTimer perfTimer("Overlays::mouseReleaseEvent");
 
     PickRay ray = qApp->computePickRay(event->x(), event->y());
-    RayToOverlayIntersectionResult rayPickResult = findRayIntersection(ray);
+    RayToOverlayIntersectionResult rayPickResult = findRayIntersectionForMouseEvent(ray);
     if (rayPickResult.intersects) {
 
         // Only Web overlays can have focus.
@@ -756,13 +785,14 @@ void Overlays::mouseReleaseEvent(QMouseEvent* event) {
     }
 
     _currentClickingOnOverlayID = UNKNOWN_OVERLAY_ID;
+    return false;
 }
 
-void Overlays::mouseMoveEvent(QMouseEvent* event) {
+bool Overlays::mouseMoveEvent(QMouseEvent* event) {
     PerformanceTimer perfTimer("Overlays::mouseMoveEvent");
 
     PickRay ray = qApp->computePickRay(event->x(), event->y());
-    RayToOverlayIntersectionResult rayPickResult = findRayIntersection(ray);
+    RayToOverlayIntersectionResult rayPickResult = findRayIntersectionForMouseEvent(ray);
     if (rayPickResult.intersects) {
 
         // Only Web overlays can have focus.
@@ -802,4 +832,34 @@ void Overlays::mouseMoveEvent(QMouseEvent* event) {
             _currentHoverOverOverlayID = UNKNOWN_OVERLAY_ID;
         }
     }
+    return false;
+}
+
+QVector<QUuid> Overlays::findOverlays(const glm::vec3& center, float radius) const {
+    QVector<QUuid> result;
+
+    QMapIterator<OverlayID, Overlay::Pointer> i(_overlaysWorld);
+    int checked = 0;
+    while (i.hasNext()) {
+        checked++;
+        i.next();
+        OverlayID thisID = i.key();
+        auto overlay = std::dynamic_pointer_cast<Volume3DOverlay>(i.value());
+        if (overlay && overlay->getVisible() && !overlay->getIgnoreRayIntersection() && overlay->isLoaded()) {
+            // get AABox in frame of overlay
+            glm::vec3 dimensions = overlay->getDimensions();
+            glm::vec3 low = dimensions * -0.5f;
+            AABox overlayFrameBox(low, dimensions);
+
+            Transform overlayToWorldMatrix = overlay->getTransform();
+            glm::mat4 worldToOverlayMatrix = glm::inverse(overlayToWorldMatrix.getMatrix());
+            glm::vec3 overlayFrameSearchPosition = glm::vec3(worldToOverlayMatrix * glm::vec4(center, 1.0f));
+            glm::vec3 penetration;
+            if (overlayFrameBox.findSpherePenetration(overlayFrameSearchPosition, radius, penetration)) {
+                result.append(thisID);
+            }
+        }
+    }
+
+    return result;
 }

interface/src/ui/overlays/Overlays.h

@@ -100,9 +100,9 @@ public:
     OverlayID addOverlay(Overlay* overlay) { return addOverlay(Overlay::Pointer(overlay)); }
     OverlayID addOverlay(Overlay::Pointer overlay);
 
-    void mousePressEvent(QMouseEvent* event);
-    void mouseReleaseEvent(QMouseEvent* event);
-    void mouseMoveEvent(QMouseEvent* event);
+    bool mousePressEvent(QMouseEvent* event);
+    bool mouseReleaseEvent(QMouseEvent* event);
+    bool mouseMoveEvent(QMouseEvent* event);
 
     void cleanupAllOverlays();
 
@@ -206,6 +206,16 @@ public slots:
                                                        bool visibleOnly = false,
                                                        bool collidableOnly = false);
 
+    /**jsdoc
+     * Return a list of 3d overlays with bounding boxes that touch the given sphere
+     *
+     * @function Overlays.findOverlays
+     * @param {Vec3} center the point to search from.
+     * @param {float} radius search radius
+     * @return {List of Overlays.OverlayID} list of overlays withing the radius
+     */
+    QVector<QUuid> findOverlays(const glm::vec3& center, float radius) const;
+
     /**jsdoc
      * Check whether an overlay's assets have been loaded. For example, if the
      * overlay is an "image" overlay, this will indicate whether the its image
@@ -317,6 +327,12 @@ private:
 
     OverlayID _currentClickingOnOverlayID { UNKNOWN_OVERLAY_ID };
     OverlayID _currentHoverOverOverlayID { UNKNOWN_OVERLAY_ID };
+
+    RayToOverlayIntersectionResult findRayIntersectionInternal(const PickRay& ray, bool precisionPicking,
+                                                               const QVector<OverlayID>& overlaysToInclude,
+                                                               const QVector<OverlayID>& overlaysToDiscard,
+                                                               bool visibleOnly = false, bool collidableOnly = false);
+    RayToOverlayIntersectionResult findRayIntersectionForMouseEvent(PickRay ray);
 };
 
 #endif // hifi_Overlays_h

interface/src/ui/overlays/OverlaysPayload.cpp

@@ -53,7 +53,7 @@ namespace render {
         return overlay->getBounds();
     }
     template <> int payloadGetLayer(const Overlay::Pointer& overlay) {
-        // MAgic number while we are defining the layering mechanism:
+        // Magic number while we are defining the layering mechanism:
         const int LAYER_NO_AA = 3;
         const int LAYER_2D = 2;
         const int LAYER_3D_FRONT = 1;

libraries/animation/src/Rig.cpp

@@ -483,6 +483,10 @@ void Rig::setEnableInverseKinematics(bool enable) {
     _enableInverseKinematics = enable;
 }
 
+void Rig::setEnableAnimations(bool enable) {
+    _enabledAnimations = enable;
+}
+
 AnimPose Rig::getAbsoluteDefaultPose(int index) const {
     if (_animSkeleton && index >= 0 && index < _animSkeleton->getNumJoints()) {
         return _absoluteDefaultPoses[index];
@@ -907,7 +911,7 @@ void Rig::updateAnimations(float deltaTime, glm::mat4 rootTransform) {
 
     setModelOffset(rootTransform);
 
-    if (_animNode) {
+    if (_animNode && _enabledAnimations) {
         PerformanceTimer perfTimer("handleTriggers");
 
         updateAnimationStateHandlers();

libraries/animation/src/Rig.h

@@ -210,6 +210,7 @@ public:
     void computeAvatarBoundingCapsule(const FBXGeometry& geometry, float& radiusOut, float& heightOut, glm::vec3& offsetOut) const;
 
     void setEnableInverseKinematics(bool enable);
+    void setEnableAnimations(bool enable);
 
     const glm::mat4& getGeometryToRigTransform() const { return _geometryToRigTransform; }
 
@@ -314,6 +315,7 @@ protected:
     int32_t _numOverrides { 0 };
     bool _lastEnableInverseKinematics { true };
     bool _enableInverseKinematics { true };
+    bool _enabledAnimations { true };
 
     mutable uint32_t _jointNameWarningCount { 0 };
 

libraries/audio-client/src/AudioClient.cpp

@@ -39,13 +39,10 @@
 #include <plugins/CodecPlugin.h>
 #include <plugins/PluginManager.h>
 #include <udt/PacketHeaders.h>
-#include <PositionalAudioStream.h>
 #include <SettingHandle.h>
 #include <SharedUtil.h>
-#include <UUID.h>
 #include <Transform.h>
 
-#include "PositionalAudioStream.h"
 #include "AudioClientLogging.h"
 #include "AudioLogging.h"
 
@@ -294,12 +291,12 @@ QString friendlyNameForAudioDevice(IMMDevice* pEndpoint) {
     IPropertyStore* pPropertyStore;
     pEndpoint->OpenPropertyStore(STGM_READ, &pPropertyStore);
     pEndpoint->Release();
-    pEndpoint = NULL;
+    pEndpoint = nullptr;
     PROPVARIANT pv;
     PropVariantInit(&pv);
     HRESULT hr = pPropertyStore->GetValue(PKEY_Device_FriendlyName, &pv);
     pPropertyStore->Release();
-    pPropertyStore = NULL;
+    pPropertyStore = nullptr;
     deviceName = QString::fromWCharArray((wchar_t*)pv.pwszVal);
     if (!IsWindows8OrGreater()) {
         // Windows 7 provides only the 31 first characters of the device name.
@@ -313,9 +310,9 @@ QString friendlyNameForAudioDevice(IMMDevice* pEndpoint) {
 QString AudioClient::friendlyNameForAudioDevice(wchar_t* guid) {
     QString deviceName;
     HRESULT hr = S_OK;
-    CoInitialize(NULL);
-    IMMDeviceEnumerator* pMMDeviceEnumerator = NULL;
-    CoCreateInstance(__uuidof(MMDeviceEnumerator), NULL, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), (void**)&pMMDeviceEnumerator);
+    CoInitialize(nullptr);
+    IMMDeviceEnumerator* pMMDeviceEnumerator = nullptr;
+    CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), (void**)&pMMDeviceEnumerator);
     IMMDevice* pEndpoint;
     hr = pMMDeviceEnumerator->GetDevice(guid, &pEndpoint);
     if (hr == E_NOTFOUND) {
@@ -325,7 +322,7 @@ QString AudioClient::friendlyNameForAudioDevice(wchar_t* guid) {
         deviceName = ::friendlyNameForAudioDevice(pEndpoint);
     }
     pMMDeviceEnumerator->Release();
-    pMMDeviceEnumerator = NULL;
+    pMMDeviceEnumerator = nullptr;
     CoUninitialize();
     return deviceName;
 }
@@ -968,8 +965,7 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) {
 }
 
 void AudioClient::handleAudioInput() {
-
-    if (!_inputDevice) {
+    if (!_inputDevice || _isPlayingBackRecording) {
         return;
     }
 
@@ -1120,7 +1116,7 @@ void AudioClient::prepareLocalAudioInjectors() {
     while (samplesNeeded > 0) {
         // lock for every write to avoid locking out the device callback
         // this lock is intentional - the buffer is only lock-free in its use in the device callback
-        Lock lock(_localAudioMutex);
+        RecursiveLock lock(_localAudioMutex);
 
         samplesNeeded = bufferCapacity - _localSamplesAvailable.load(std::memory_order_relaxed);
         if (samplesNeeded <= 0) {
@@ -1457,7 +1453,7 @@ void AudioClient::outputNotify() {
 bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo) {
     bool supportedFormat = false;
 
-    Lock lock(_localAudioMutex);
+    RecursiveLock lock(_localAudioMutex);
     _localSamplesAvailable.exchange(0, std::memory_order_release);
 
     // cleanup any previously initialized device
@@ -1671,7 +1667,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
 
     int injectorSamplesPopped = 0;
     {
-        Lock lock(_audio->_localAudioMutex);
+        RecursiveLock lock(_audio->_localAudioMutex);
         bool append = networkSamplesPopped > 0;
         samplesRequested = std::min(samplesRequested, _audio->_localSamplesAvailable.load(std::memory_order_acquire));
         if ((injectorSamplesPopped = _localInjectorsStream.appendSamples(mixBuffer, samplesRequested, append)) > 0) {

libraries/audio-client/src/AudioClient.h

@@ -94,6 +94,8 @@ public:
     using AudioPositionGetter = std::function<glm::vec3()>;
     using AudioOrientationGetter = std::function<glm::quat()>;
 
+    using RecursiveMutex = std::recursive_mutex;
+    using RecursiveLock = std::unique_lock<RecursiveMutex>;
     using Mutex = std::mutex;
     using Lock = std::unique_lock<Mutex>;
 
@@ -145,6 +147,8 @@ public:
     void setPositionGetter(AudioPositionGetter positionGetter) { _positionGetter = positionGetter; }
     void setOrientationGetter(AudioOrientationGetter orientationGetter) { _orientationGetter = orientationGetter; }
 
+    void setIsPlayingBackRecording(bool isPlayingBackRecording) { _isPlayingBackRecording = isPlayingBackRecording; }
+
     Q_INVOKABLE void setAvatarBoundingBoxParameters(glm::vec3 corner, glm::vec3 scale);
 
     void checkDevices();
@@ -328,7 +332,7 @@ private:
     int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC];
     float* _localOutputMixBuffer { NULL };
     AudioInjectorsThread _localAudioThread;
-    Mutex _localAudioMutex;
+    RecursiveMutex _localAudioMutex;
 
     // for output audio (used by this thread)
     int _outputPeriod { 0 };
@@ -367,10 +371,12 @@ private:
     QVector<QString> _inputDevices;
     QVector<QString> _outputDevices;
 
-    bool _hasReceivedFirstPacket = false;
+    bool _hasReceivedFirstPacket { false };
 
     QVector<AudioInjector*> _activeLocalAudioInjectors;
 
+    bool _isPlayingBackRecording { false };
+
     CodecPluginPointer _codec;
     QString _selectedCodecName;
     Encoder* _encoder { nullptr }; // for outbound mic stream

libraries/audio/src/AudioInjectorOptions.cpp

@@ -91,4 +91,4 @@ void injectorOptionsFromScriptValue(const QScriptValue& object, AudioInjectorOpt
             qCWarning(audio) << "Unknown audio injector option:" << it.name();
         }
     }
- }
+}

libraries/avatars/src/AvatarData.cpp

@@ -36,6 +36,8 @@
 #include <shared/JSONHelpers.h>
 #include <ShapeInfo.h>
 #include <AudioHelpers.h>
+#include <Profile.h>
+#include <VariantMapToScriptValue.h>
 
 #include "AvatarLogging.h"
 
@@ -68,8 +70,7 @@ AvatarData::AvatarData() :
     _displayNameAlpha(1.0f),
     _errorLogExpiry(0),
     _owningAvatarMixer(),
-    _targetVelocity(0.0f),
-    _localAABox(DEFAULT_LOCAL_AABOX_CORNER, DEFAULT_LOCAL_AABOX_SCALE)
+    _targetVelocity(0.0f)
 {
     setBodyPitch(0.0f);
     setBodyYaw(-90.0f);
@@ -120,10 +121,6 @@ void AvatarData::nextAttitude(glm::vec3 position, glm::quat orientation) {
     updateAttitude();
 }
 
-float AvatarData::getTargetScale() const {
-    return _targetScale;
-}
-
 void AvatarData::setTargetScale(float targetScale) {
     auto newValue = glm::clamp(targetScale, MIN_AVATAR_SCALE, MAX_AVATAR_SCALE);
     if (_targetScale != newValue) {
@@ -141,10 +138,10 @@ void AvatarData::setHandPosition(const glm::vec3& handPosition) {
     _handPosition = glm::inverse(getOrientation()) * (handPosition - getPosition());
 }
 
-void AvatarData::lazyInitHeadData() {
+void AvatarData::lazyInitHeadData() const {
     // lazily allocate memory for HeadData in case we're not an Avatar instance
     if (!_headData) {
-        _headData = new HeadData(this);
+        _headData = new HeadData(const_cast<AvatarData*>(this));
     }
     if (_forceFaceTrackerConnected) {
         _headData->_isFaceTrackerConnected = true;
@@ -152,39 +149,7 @@ void AvatarData::lazyInitHeadData() {
 }
 
 
-bool AvatarData::avatarBoundingBoxChangedSince(quint64 time) {
-    return _avatarBoundingBoxChanged >= time;
-}
-
-bool AvatarData::avatarScaleChangedSince(quint64 time) {
-    return _avatarScaleChanged >= time;
-}
-
-bool AvatarData::lookAtPositionChangedSince(quint64 time) {
-    return _headData->lookAtPositionChangedSince(time);
-}
-
-bool AvatarData::audioLoudnessChangedSince(quint64 time) {
-    return _headData->audioLoudnessChangedSince(time);
-}
-
-bool AvatarData::sensorToWorldMatrixChangedSince(quint64 time) {
-    return _sensorToWorldMatrixChanged >= time;
-}
-
-bool AvatarData::additionalFlagsChangedSince(quint64 time) {
-    return _additionalFlagsChanged >= time;
-}
-
-bool AvatarData::parentInfoChangedSince(quint64 time) {
-    return _parentChanged >= time;
-}
-
-bool AvatarData::faceTrackerInfoChangedSince(quint64 time) {
-    return true; // FIXME!
-}
-
-float AvatarData::getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) {
+float AvatarData::getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) const {
     auto distance = glm::distance(_globalPosition, viewerPosition);
     float result = ROTATION_CHANGE_179D; // assume worst
     if (distance < AVATAR_DISTANCE_LEVEL_1) {
@@ -199,20 +164,24 @@ float AvatarData::getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) {
     return result;
 }
 
-float AvatarData::getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition) {
+float AvatarData::getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition) const {
     return AVATAR_MIN_TRANSLATION; // Eventually make this distance sensitive as well
 }
 
 
-QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
-                        bool distanceAdjust, glm::vec3 viewerPosition, QVector<JointData>* sentJointDataOut) {
+// we want to track outbound data in this case...
+QByteArray AvatarData::toByteArrayStateful(AvatarDataDetail dataDetail) {
+    AvatarDataPacket::HasFlags hasFlagsOut;
+    auto lastSentTime = _lastToByteArray;
+    _lastToByteArray = usecTimestampNow();
+    return AvatarData::toByteArray(dataDetail, lastSentTime, getLastSentJointData(), 
+                        hasFlagsOut, false, false, glm::vec3(0), nullptr,
+                        &_outboundDataRate);
+}
 
-    // if no timestamp was included, then assume the avatarData is single instance 
-    // and is tracking its own last encoding time.
-    if (lastSentTime == 0) {
-        lastSentTime = _lastToByteArray;
-        _lastToByteArray = usecTimestampNow();
-    }
+QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
+    AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, 
+    glm::vec3 viewerPosition, QVector<JointData>* sentJointDataOut, AvatarDataRate* outboundDataRateOut) const {
 
     bool cullSmallChanges = (dataDetail == CullSmallData);
     bool sendAll = (dataDetail == SendAllData);
@@ -224,6 +193,13 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
     unsigned char* destinationBuffer = reinterpret_cast<unsigned char*>(avatarDataByteArray.data());
     unsigned char* startPosition = destinationBuffer;
 
+    // special case, if we were asked for no data, then just include the flags all set to nothing
+    if (dataDetail == NoData) {
+        AvatarDataPacket::HasFlags packetStateFlags = 0;
+        memcpy(destinationBuffer, &packetStateFlags, sizeof(packetStateFlags));
+        return avatarDataByteArray.left(sizeof(packetStateFlags));
+    }
+
     // FIXME -
     //
     //    BUG -- if you enter a space bubble, and then back away, the avatar has wrong orientation until "send all" happens... 
@@ -259,26 +235,26 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
     // separately
     bool hasParentInfo = sendAll || parentInfoChangedSince(lastSentTime);
     bool hasAvatarLocalPosition = hasParent() && (sendAll ||
-                                                  tranlationChangedSince(lastSentTime) ||
-                                                  parentInfoChangedSince(lastSentTime));
+        tranlationChangedSince(lastSentTime) ||
+        parentInfoChangedSince(lastSentTime));
 
-    bool hasFaceTrackerInfo = hasFaceTracker() && (sendAll || faceTrackerInfoChangedSince(lastSentTime));
+    bool hasFaceTrackerInfo = !dropFaceTracking && hasFaceTracker() && (sendAll || faceTrackerInfoChangedSince(lastSentTime));
     bool hasJointData = sendAll || !sendMinimum;
 
     // Leading flags, to indicate how much data is actually included in the packet...
     AvatarDataPacket::HasFlags packetStateFlags =
-            (hasAvatarGlobalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_GLOBAL_POSITION : 0)
-          | (hasAvatarBoundingBox    ? AvatarDataPacket::PACKET_HAS_AVATAR_BOUNDING_BOX : 0)
-          | (hasAvatarOrientation    ? AvatarDataPacket::PACKET_HAS_AVATAR_ORIENTATION : 0)
-          | (hasAvatarScale          ? AvatarDataPacket::PACKET_HAS_AVATAR_SCALE : 0)
-          | (hasLookAtPosition       ? AvatarDataPacket::PACKET_HAS_LOOK_AT_POSITION : 0)
-          | (hasAudioLoudness        ? AvatarDataPacket::PACKET_HAS_AUDIO_LOUDNESS : 0)
-          | (hasSensorToWorldMatrix  ? AvatarDataPacket::PACKET_HAS_SENSOR_TO_WORLD_MATRIX : 0)
-          | (hasAdditionalFlags      ? AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS : 0)
-          | (hasParentInfo           ? AvatarDataPacket::PACKET_HAS_PARENT_INFO : 0)
-          | (hasAvatarLocalPosition  ? AvatarDataPacket::PACKET_HAS_AVATAR_LOCAL_POSITION : 0)
-          | (hasFaceTrackerInfo      ? AvatarDataPacket::PACKET_HAS_FACE_TRACKER_INFO : 0)
-          | (hasJointData            ? AvatarDataPacket::PACKET_HAS_JOINT_DATA : 0);
+        (hasAvatarGlobalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_GLOBAL_POSITION : 0)
+        | (hasAvatarBoundingBox ? AvatarDataPacket::PACKET_HAS_AVATAR_BOUNDING_BOX : 0)
+        | (hasAvatarOrientation ? AvatarDataPacket::PACKET_HAS_AVATAR_ORIENTATION : 0)
+        | (hasAvatarScale ? AvatarDataPacket::PACKET_HAS_AVATAR_SCALE : 0)
+        | (hasLookAtPosition ? AvatarDataPacket::PACKET_HAS_LOOK_AT_POSITION : 0)
+        | (hasAudioLoudness ? AvatarDataPacket::PACKET_HAS_AUDIO_LOUDNESS : 0)
+        | (hasSensorToWorldMatrix ? AvatarDataPacket::PACKET_HAS_SENSOR_TO_WORLD_MATRIX : 0)
+        | (hasAdditionalFlags ? AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS : 0)
+        | (hasParentInfo ? AvatarDataPacket::PACKET_HAS_PARENT_INFO : 0)
+        | (hasAvatarLocalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_LOCAL_POSITION : 0)
+        | (hasFaceTrackerInfo ? AvatarDataPacket::PACKET_HAS_FACE_TRACKER_INFO : 0)
+        | (hasJointData ? AvatarDataPacket::PACKET_HAS_JOINT_DATA : 0);
 
     memcpy(destinationBuffer, &packetStateFlags, sizeof(packetStateFlags));
     destinationBuffer += sizeof(packetStateFlags);
@@ -293,7 +269,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
 
         int numBytes = destinationBuffer - startSection;
 
-        _globalPositionRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->globalPositionRate.increment(numBytes);
+        }
     }
 
     if (hasAvatarBoundingBox) {
@@ -311,7 +289,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::AvatarBoundingBox);
 
         int numBytes = destinationBuffer - startSection;
-        _avatarBoundingBoxRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->avatarBoundingBoxRate.increment(numBytes);
+        }
     }
 
     if (hasAvatarOrientation) {
@@ -320,7 +300,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, localOrientation);
 
         int numBytes = destinationBuffer - startSection;
-        _avatarOrientationRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->avatarOrientationRate.increment(numBytes);
+        }
     }
 
     if (hasAvatarScale) {
@@ -331,7 +313,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::AvatarScale);
 
         int numBytes = destinationBuffer - startSection;
-        _avatarScaleRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->avatarScaleRate.increment(numBytes);
+        }
     }
 
     if (hasLookAtPosition) {
@@ -344,7 +328,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::LookAtPosition);
 
         int numBytes = destinationBuffer - startSection;
-        _lookAtPositionRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->lookAtPositionRate.increment(numBytes);
+        }
     }
 
     if (hasAudioLoudness) {
@@ -354,7 +340,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::AudioLoudness);
 
         int numBytes = destinationBuffer - startSection;
-        _audioLoudnessRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->audioLoudnessRate.increment(numBytes);
+        }
     }
 
     if (hasSensorToWorldMatrix) {
@@ -370,7 +358,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::SensorToWorldMatrix);
 
         int numBytes = destinationBuffer - startSection;
-        _sensorToWorldRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->sensorToWorldRate.increment(numBytes);
+        }
     }
 
     if (hasAdditionalFlags) {
@@ -403,7 +393,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::AdditionalFlags);
 
         int numBytes = destinationBuffer - startSection;
-        _additionalFlagsRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->additionalFlagsRate.increment(numBytes);
+        }
     }
 
     if (hasParentInfo) {
@@ -415,7 +407,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::ParentInfo);
 
         int numBytes = destinationBuffer - startSection;
-        _parentInfoRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->parentInfoRate.increment(numBytes);
+        }
     }
 
     if (hasAvatarLocalPosition) {
@@ -428,7 +422,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += sizeof(AvatarDataPacket::AvatarLocalPosition);
 
         int numBytes = destinationBuffer - startSection;
-        _localPositionRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->localPositionRate.increment(numBytes);
+        }
     }
 
     // If it is connected, pack up the data
@@ -448,7 +444,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
         destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float);
 
         int numBytes = destinationBuffer - startSection;
-        _faceTrackerRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->faceTrackerRate.increment(numBytes);
+        }
     }
 
     // If it is connected, pack up the data
@@ -540,9 +538,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
                     glm::distance(data.translation, lastSentJointData[i].translation) > minTranslation) {
                     if (data.translationSet) {
                         validity |= (1 << validityBit);
-                        #ifdef WANT_DEBUG
+#ifdef WANT_DEBUG
                         translationSentCount++;
-                        #endif
+#endif
                         maxTranslationDimension = glm::max(fabsf(data.translation.x), maxTranslationDimension);
                         maxTranslationDimension = glm::max(fabsf(data.translation.y), maxTranslationDimension);
                         maxTranslationDimension = glm::max(fabsf(data.translation.z), maxTranslationDimension);
@@ -592,24 +590,25 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
 #ifdef WANT_DEBUG
         if (sendAll) {
             qCDebug(avatars) << "AvatarData::toByteArray" << cullSmallChanges << sendAll
-                    << "rotations:" << rotationSentCount << "translations:" << translationSentCount
-                    << "largest:" << maxTranslationDimension
-                    << "size:"
-                    << (beforeRotations - startPosition) << "+"
-                    << (beforeTranslations - beforeRotations) << "+"
-                    << (destinationBuffer - beforeTranslations) << "="
-                    << (destinationBuffer - startPosition);
+                << "rotations:" << rotationSentCount << "translations:" << translationSentCount
+                << "largest:" << maxTranslationDimension
+                << "size:"
+                << (beforeRotations - startPosition) << "+"
+                << (beforeTranslations - beforeRotations) << "+"
+                << (destinationBuffer - beforeTranslations) << "="
+                << (destinationBuffer - startPosition);
         }
 #endif
 
         int numBytes = destinationBuffer - startSection;
-        _jointDataRateOutbound.increment(numBytes);
+        if (outboundDataRateOut) {
+            outboundDataRateOut->jointDataRate.increment(numBytes);
+        }
     }
 
     int avatarDataSize = destinationBuffer - startPosition;
     return avatarDataByteArray.left(avatarDataSize);
 }
-
 // NOTE: This is never used in a "distanceAdjust" mode, so it's ok that it doesn't use a variable minimum rotation/translation
 void AvatarData::doneEncoding(bool cullSmallChanges) {
     // The server has finished sending this version of the joint-data to other nodes.  Update _lastSentJointData.
@@ -1089,29 +1088,29 @@ float AvatarData::getDataRate(const QString& rateName) const {
     } else if (rateName == "jointData") {
         return _jointDataRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "globalPositionOutbound") {
-        return _globalPositionRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.globalPositionRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "localPositionOutbound") {
-        return _localPositionRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.localPositionRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "avatarBoundingBoxOutbound") {
-        return _avatarBoundingBoxRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.avatarBoundingBoxRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "avatarOrientationOutbound") {
-        return _avatarOrientationRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.avatarOrientationRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "avatarScaleOutbound") {
-        return _avatarScaleRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.avatarScaleRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "lookAtPositionOutbound") {
-        return _lookAtPositionRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.lookAtPositionRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "audioLoudnessOutbound") {
-        return _audioLoudnessRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.audioLoudnessRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "sensorToWorkMatrixOutbound") {
-        return _sensorToWorldRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.sensorToWorldRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "additionalFlagsOutbound") {
-        return _additionalFlagsRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.additionalFlagsRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "parentInfoOutbound") {
-        return _parentInfoRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.parentInfoRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "faceTrackerOutbound") {
-        return _faceTrackerRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.faceTrackerRate.rate() / BYTES_PER_KILOBIT;
     } else if (rateName == "jointDataOutbound") {
-        return _jointDataRateOutbound.rate() / BYTES_PER_KILOBIT;
+        return _outboundDataRate.jointDataRate.rate() / BYTES_PER_KILOBIT;
     }
     return 0.0f;
 }
@@ -1445,7 +1444,7 @@ void AvatarData::parseAvatarIdentityPacket(const QByteArray& data, Identity& ide
 }
 
 static const QUrl emptyURL("");
-const QUrl& AvatarData::cannonicalSkeletonModelURL(const QUrl& emptyURL) {
+QUrl AvatarData::cannonicalSkeletonModelURL(const QUrl& emptyURL) const {
     // We don't put file urls on the wire, but instead convert to empty.
     return _skeletonModelURL.scheme() == "file" ? emptyURL : _skeletonModelURL;
 }
@@ -1483,7 +1482,7 @@ void AvatarData::processAvatarIdentity(const Identity& identity, bool& identityC
     }
 }
 
-QByteArray AvatarData::identityByteArray() {
+QByteArray AvatarData::identityByteArray() const {
     QByteArray identityData;
     QDataStream identityStream(&identityData, QIODevice::Append);
     const QUrl& urlToSend = cannonicalSkeletonModelURL(emptyURL);
@@ -1646,13 +1645,7 @@ void AvatarData::sendAvatarDataPacket() {
 
     bool cullSmallData = (randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO);
     auto dataDetail = cullSmallData ? SendAllData : CullSmallData;
-    QVector<JointData> lastSentJointData;
-    {
-        QReadLocker readLock(&_jointDataLock);
-        _lastSentJointData.resize(_jointData.size());
-        lastSentJointData = _lastSentJointData;
-    } 
-    QByteArray avatarByteArray = toByteArray(dataDetail, 0, lastSentJointData);
+    QByteArray avatarByteArray = toByteArrayStateful(dataDetail);
     doneEncoding(cullSmallData);
 
     static AvatarDataSequenceNumber sequenceNumber = 0;
@@ -2324,3 +2317,100 @@ void RayToAvatarIntersectionResultFromScriptValue(const QScriptValue& object, Ra
         vec3FromScriptValue(intersection, value.intersection);
     }
 }
+
+const float AvatarData::OUT_OF_VIEW_PENALTY = -10.0f;
+
+float AvatarData::_avatarSortCoefficientSize { 0.5f };
+float AvatarData::_avatarSortCoefficientCenter { 0.25 };
+float AvatarData::_avatarSortCoefficientAge { 1.0f };
+
+std::priority_queue<AvatarPriority> AvatarData::sortAvatars(
+    QList<AvatarSharedPointer> avatarList,
+    const ViewFrustum& cameraView,
+    std::function<uint64_t(AvatarSharedPointer)> getLastUpdated,
+    std::function<float(AvatarSharedPointer)> getBoundingRadius,
+    std::function<bool(AvatarSharedPointer)> shouldIgnore) {
+
+    uint64_t startTime = usecTimestampNow();
+
+    glm::vec3 frustumCenter = cameraView.getPosition();
+
+    std::priority_queue<AvatarPriority> sortedAvatars;
+    {
+        PROFILE_RANGE(simulation, "sort");
+        for (int32_t i = 0; i < avatarList.size(); ++i) {
+            const auto& avatar = avatarList.at(i);
+
+            if (shouldIgnore(avatar)) {
+                continue;
+            }
+
+            // priority = weighted linear combination of:
+            //   (a) apparentSize
+            //   (b) proximity to center of view
+            //   (c) time since last update
+            glm::vec3 avatarPosition = avatar->getPosition();
+            glm::vec3 offset = avatarPosition - frustumCenter;
+            float distance = glm::length(offset) + 0.001f; // add 1mm to avoid divide by zero
+
+            // FIXME - AvatarData has something equivolent to this
+            float radius = getBoundingRadius(avatar);
+
+            const glm::vec3& forward = cameraView.getDirection();
+            float apparentSize = 2.0f * radius / distance;
+            float cosineAngle = glm::length(glm::dot(offset, forward) * forward) / distance;
+            float age = (float)(startTime - getLastUpdated(avatar)) / (float)(USECS_PER_SECOND);
+
+            // NOTE: we are adding values of different units to get a single measure of "priority".
+            // Thus we multiply each component by a conversion "weight" that scales its units relative to the others.
+            // These weights are pure magic tuning and should be hard coded in the relation below,
+            // but are currently exposed for anyone who would like to explore fine tuning:
+            float priority = _avatarSortCoefficientSize * apparentSize
+                + _avatarSortCoefficientCenter * cosineAngle
+                + _avatarSortCoefficientAge * age;
+
+            // decrement priority of avatars outside keyhole
+            if (distance > cameraView.getCenterRadius()) {
+                if (!cameraView.sphereIntersectsFrustum(avatarPosition, radius)) {
+                    priority += OUT_OF_VIEW_PENALTY;
+                }
+            }
+            sortedAvatars.push(AvatarPriority(avatar, priority));
+        }
+    }
+    return sortedAvatars;
+}
+
+QScriptValue AvatarEntityMapToScriptValue(QScriptEngine* engine, const AvatarEntityMap& value) {
+    QScriptValue obj = engine->newObject();
+    for (auto entityID : value.keys()) {
+        QByteArray entityProperties = value.value(entityID);
+        QJsonDocument jsonEntityProperties = QJsonDocument::fromBinaryData(entityProperties);
+        if (!jsonEntityProperties.isObject()) {
+            qCDebug(avatars) << "bad AvatarEntityData in AvatarEntityMap" << QString(entityProperties.toHex());
+        }
+        
+        QVariant variantEntityProperties = jsonEntityProperties.toVariant();
+        QVariantMap entityPropertiesMap = variantEntityProperties.toMap();
+        QScriptValue scriptEntityProperties = variantMapToScriptValue(entityPropertiesMap, *engine);
+        
+        QString key = entityID.toString();
+        obj.setProperty(key, scriptEntityProperties);
+    }
+    return obj;
+}
+
+void AvatarEntityMapFromScriptValue(const QScriptValue& object, AvatarEntityMap& value) {
+    QScriptValueIterator itr(object);
+    while (itr.hasNext()) {
+        itr.next();
+        QUuid EntityID = QUuid(itr.name());
+        
+        QScriptValue scriptEntityProperties = itr.value();
+        QVariant variantEntityProperties = scriptEntityProperties.toVariant();
+        QJsonDocument jsonEntityProperties = QJsonDocument::fromVariant(variantEntityProperties);
+        QByteArray binaryEntityProperties = jsonEntityProperties.toBinaryData();
+        
+        value[EntityID] = binaryEntityProperties;
+    }
+}

libraries/avatars/src/AvatarData.h

@@ -14,6 +14,8 @@
 
 #include <string>
 #include <memory>
+#include <queue>
+
 /* VS2010 defines stdint.h, but not inttypes.h */
 #if defined(_MSC_VER)
 typedef signed char  int8_t;
@@ -44,6 +46,7 @@ typedef unsigned long long quint64;
 #include <QVariantMap>
 #include <QVector>
 #include <QtScript/QScriptable>
+#include <QtScript/QScriptValueIterator>
 #include <QReadWriteLock>
 
 #include <JointData.h>
@@ -57,6 +60,7 @@ typedef unsigned long long quint64;
 #include <ThreadSafeValueCache.h>
 #include <SharedUtil.h>
 #include <shared/RateCounter.h>
+#include <ViewFrustum.h>
 
 #include "AABox.h"
 #include "HeadData.h"
@@ -133,6 +137,7 @@ namespace AvatarDataPacket {
     const HasFlags PACKET_HAS_AVATAR_LOCAL_POSITION  = 1U << 9;
     const HasFlags PACKET_HAS_FACE_TRACKER_INFO      = 1U << 10;
     const HasFlags PACKET_HAS_JOINT_DATA             = 1U << 11;
+    const size_t AVATAR_HAS_FLAGS_SIZE = 2;
 
     // NOTE: AvatarDataPackets start with a uint16_t sequence number that is not reflected in the Header structure.
 
@@ -288,6 +293,31 @@ class AttachmentData;
 class Transform;
 using TransformPointer = std::shared_ptr<Transform>;
 
+class AvatarDataRate {
+public:
+    RateCounter<> globalPositionRate;
+    RateCounter<> localPositionRate;
+    RateCounter<> avatarBoundingBoxRate;
+    RateCounter<> avatarOrientationRate;
+    RateCounter<> avatarScaleRate;
+    RateCounter<> lookAtPositionRate;
+    RateCounter<> audioLoudnessRate;
+    RateCounter<> sensorToWorldRate;
+    RateCounter<> additionalFlagsRate;
+    RateCounter<> parentInfoRate;
+    RateCounter<> faceTrackerRate;
+    RateCounter<> jointDataRate;
+};
+
+class AvatarPriority {
+public:
+    AvatarPriority(AvatarSharedPointer a, float p) : avatar(a), priority(p) {}
+    AvatarSharedPointer avatar;
+    float priority;
+    // NOTE: we invert the less-than operator to sort high priorities to front
+    bool operator<(const AvatarPriority& other) const { return priority < other.priority; }
+};
+
 class AvatarData : public QObject, public SpatiallyNestable {
     Q_OBJECT
 
@@ -345,14 +375,18 @@ public:
     void setHandPosition(const glm::vec3& handPosition);
 
     typedef enum { 
+        NoData,
         MinimumData, 
         CullSmallData,
         IncludeSmallData,
         SendAllData
     } AvatarDataDetail;
 
+    virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail);
+
     virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector<JointData>& lastSentJointData,
-                        bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector<JointData>* sentJointDataOut = nullptr);
+        AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, 
+        QVector<JointData>* sentJointDataOut, AvatarDataRate* outboundDataRateOut = nullptr) const;
 
     virtual void doneEncoding(bool cullSmallChanges);
 
@@ -380,7 +414,7 @@ public:
     void nextAttitude(glm::vec3 position, glm::quat orientation); // Can be safely called at any time.
     virtual void updateAttitude() {} // Tell skeleton mesh about changes
 
-    glm::quat getHeadOrientation() { 
+    glm::quat getHeadOrientation() const { 
         lazyInitHeadData();
         return _headData->getOrientation(); 
     }
@@ -419,7 +453,6 @@ public:
     void setAudioAverageLoudness(float value) { _headData->setAudioAverageLoudness(value); }
 
     //  Scale
-    float getTargetScale() const;
     virtual void setTargetScale(float targetScale);
 
     float getDomainLimitedScale() const { return glm::clamp(_targetScale, _domainMinimumScale, _domainMaximumScale); }
@@ -494,7 +527,7 @@ public:
     // displayNameChanged returns true if displayName has changed, false otherwise.
     void processAvatarIdentity(const Identity& identity, bool& identityChanged, bool& displayNameChanged);
 
-    QByteArray identityByteArray();
+    QByteArray identityByteArray() const;
 
     const QUrl& getSkeletonModelURL() const { return _skeletonModelURL; }
     const QString& getDisplayName() const { return _displayName; }
@@ -520,8 +553,6 @@ public:
 
     void setOwningAvatarMixer(const QWeakPointer<Node>& owningAvatarMixer) { _owningAvatarMixer = owningAvatarMixer; }
 
-    const AABox& getLocalAABox() const { return _localAABox; }
-
     int getUsecsSinceLastUpdate() const { return _averageBytesReceived.getUsecsSinceLastEvent(); }
     int getAverageBytesReceivedPerSecond() const;
     int getReceiveRate() const;
@@ -534,8 +565,8 @@ public:
     QJsonObject toJson() const;
     void fromJson(const QJsonObject& json, bool useFrameSkeleton = true);
 
-    glm::vec3 getClientGlobalPosition() { return _globalPosition; }
-    glm::vec3 getGlobalBoundingBoxCorner() { return _globalPosition + _globalBoundingBoxOffset - _globalBoundingBoxDimensions; }
+    glm::vec3 getClientGlobalPosition() const { return _globalPosition; }
+    glm::vec3 getGlobalBoundingBoxCorner() const { return _globalPosition + _globalBoundingBoxOffset - _globalBoundingBoxDimensions; }
 
     Q_INVOKABLE AvatarEntityMap getAvatarEntityData() const;
     Q_INVOKABLE void setAvatarEntityData(const AvatarEntityMap& avatarEntityData);
@@ -550,7 +581,7 @@ public:
     Q_INVOKABLE float getDataRate(const QString& rateName = QString("")) const;
     Q_INVOKABLE float getUpdateRate(const QString& rateName = QString("")) const;
 
-    int getJointCount() { return _jointData.size(); }
+    int getJointCount() const { return _jointData.size(); }
 
     QVector<JointData> getLastSentJointData() {
         QReadLocker readLock(&_jointDataLock);
@@ -559,6 +590,28 @@ public:
     }
 
 
+    bool shouldDie() const {
+        const qint64 AVATAR_SILENCE_THRESHOLD_USECS = 5 * USECS_PER_SECOND;
+        return _owningAvatarMixer.isNull() || getUsecsSinceLastUpdate() > AVATAR_SILENCE_THRESHOLD_USECS;
+    }
+
+    static const float OUT_OF_VIEW_PENALTY;
+
+    static std::priority_queue<AvatarPriority> sortAvatars(
+        QList<AvatarSharedPointer> avatarList,
+        const ViewFrustum& cameraView,
+        std::function<uint64_t(AvatarSharedPointer)> getLastUpdated,
+        std::function<float(AvatarSharedPointer)> getBoundingRadius,
+        std::function<bool(AvatarSharedPointer)> shouldIgnore);
+
+    // TODO: remove this HACK once we settle on optimal sort coefficients
+    // These coefficients exposed for fine tuning the sort priority for transfering new _jointData to the render pipeline.
+    static float _avatarSortCoefficientSize;
+    static float _avatarSortCoefficientCenter;
+    static float _avatarSortCoefficientAge;
+
+
+
 public slots:
     void sendAvatarDataPacket();
     void sendIdentityPacket();
@@ -571,28 +624,27 @@ public slots:
     virtual bool setAbsoluteJointRotationInObjectFrame(int index, const glm::quat& rotation) override { return false; }
     virtual bool setAbsoluteJointTranslationInObjectFrame(int index, const glm::vec3& translation) override { return false; }
 
-    float getTargetScale() { return _targetScale; }
+    float getTargetScale() const { return _targetScale; } // why is this a slot?
 
     void resetLastSent() { _lastToByteArray = 0; }
 
 protected:
-    void lazyInitHeadData();
+    void lazyInitHeadData() const;
 
-    float getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition);
-    float getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition);
+    float getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) const;
+    float getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition) const;
 
-    bool avatarBoundingBoxChangedSince(quint64 time);
-    bool avatarScaleChangedSince(quint64 time);
-    bool lookAtPositionChangedSince(quint64 time);
-    bool audioLoudnessChangedSince(quint64 time);
-    bool sensorToWorldMatrixChangedSince(quint64 time);
-    bool additionalFlagsChangedSince(quint64 time);
+    bool avatarBoundingBoxChangedSince(quint64 time) const { return _avatarBoundingBoxChanged >= time; }
+    bool avatarScaleChangedSince(quint64 time) const { return _avatarScaleChanged >= time; }
+    bool lookAtPositionChangedSince(quint64 time) const { return _headData->lookAtPositionChangedSince(time); }
+    bool audioLoudnessChangedSince(quint64 time) const { return _headData->audioLoudnessChangedSince(time); }
+    bool sensorToWorldMatrixChangedSince(quint64 time) const { return _sensorToWorldMatrixChanged >= time; }
+    bool additionalFlagsChangedSince(quint64 time) const { return _additionalFlagsChanged >= time; }
+    bool parentInfoChangedSince(quint64 time) const { return _parentChanged >= time; }
+    bool faceTrackerInfoChangedSince(quint64 time) const { return true; } // FIXME
 
-    bool hasParent() { return !getParentID().isNull(); }
-    bool parentInfoChangedSince(quint64 time);
-
-    bool hasFaceTracker() { return _headData ? _headData->_isFaceTrackerConnected : false; }
-    bool faceTrackerInfoChangedSince(quint64 time);
+    bool hasParent() const { return !getParentID().isNull(); }
+    bool hasFaceTracker() const { return _headData ? _headData->_isFaceTrackerConnected : false; }
 
     glm::vec3 _handPosition;
     virtual const QString& getSessionDisplayNameForTransport() const { return _sessionDisplayName; }
@@ -616,7 +668,7 @@ protected:
     bool _forceFaceTrackerConnected;
     bool _hasNewJointData { true }; // set in AvatarData, cleared in Avatar
 
-    HeadData* _headData { nullptr };
+    mutable HeadData* _headData { nullptr };
 
     QUrl _skeletonModelURL;
     bool _firstSkeletonCheck { true };
@@ -624,7 +676,7 @@ protected:
     QVector<AttachmentData> _attachmentData;
     QString _displayName;
     QString _sessionDisplayName { };
-    const QUrl& cannonicalSkeletonModelURL(const QUrl& empty);
+    QUrl cannonicalSkeletonModelURL(const QUrl& empty) const;
 
     float _displayNameTargetAlpha;
     float _displayNameAlpha;
@@ -641,8 +693,6 @@ protected:
 
     glm::vec3 _targetVelocity;
 
-    AABox _localAABox;
-
     SimpleMovingAverage _averageBytesReceived;
 
     // During recording, this holds the starting position, orientation & scale of the recorded avatar
@@ -695,18 +745,7 @@ protected:
     RateCounter<> _jointDataUpdateRate;
 
     // Some rate data for outgoing data
-    RateCounter<> _globalPositionRateOutbound;
-    RateCounter<> _localPositionRateOutbound;
-    RateCounter<> _avatarBoundingBoxRateOutbound;
-    RateCounter<> _avatarOrientationRateOutbound;
-    RateCounter<> _avatarScaleRateOutbound;
-    RateCounter<> _lookAtPositionRateOutbound;
-    RateCounter<> _audioLoudnessRateOutbound;
-    RateCounter<> _sensorToWorldRateOutbound;
-    RateCounter<> _additionalFlagsRateOutbound;
-    RateCounter<> _parentInfoRateOutbound;
-    RateCounter<> _faceTrackerRateOutbound;
-    RateCounter<> _jointDataRateOutbound;
+    AvatarDataRate _outboundDataRate;
 
     glm::vec3 _globalBoundingBoxDimensions;
     glm::vec3 _globalBoundingBoxOffset;
@@ -809,6 +848,11 @@ Q_DECLARE_METATYPE(RayToAvatarIntersectionResult)
 QScriptValue RayToAvatarIntersectionResultToScriptValue(QScriptEngine* engine, const RayToAvatarIntersectionResult& results);
 void RayToAvatarIntersectionResultFromScriptValue(const QScriptValue& object, RayToAvatarIntersectionResult& results);
 
+Q_DECLARE_METATYPE(AvatarEntityMap)
+
+QScriptValue AvatarEntityMapToScriptValue(QScriptEngine* engine, const AvatarEntityMap& value);
+void AvatarEntityMapFromScriptValue(const QScriptValue& object, AvatarEntityMap& value);
+
 // faux joint indexes (-1 means invalid)
 const int SENSOR_TO_WORLD_MATRIX_INDEX = 65534; // -2
 const int CONTROLLER_RIGHTHAND_INDEX = 65533; // -3

libraries/avatars/src/AvatarHashMap.cpp

@@ -182,7 +182,7 @@ void AvatarHashMap::removeAvatar(const QUuid& sessionUUID, KillAvatarReason remo
 
 void AvatarHashMap::handleRemovedAvatar(const AvatarSharedPointer& removedAvatar, KillAvatarReason removalReason) {
     qCDebug(avatars) << "Removed avatar with UUID" << uuidStringWithoutCurlyBraces(removedAvatar->getSessionUUID())
-        << "from AvatarHashMap";
+        << "from AvatarHashMap" << removalReason;
     emit avatarRemovedEvent(removedAvatar->getSessionUUID());
 }
 
@@ -190,3 +190,4 @@ void AvatarHashMap::sessionUUIDChanged(const QUuid& sessionUUID, const QUuid& ol
     _lastOwnerSessionUUID = oldUUID;
     emit avatarSessionChangedEvent(sessionUUID, oldUUID);
 }
+

libraries/avatars/src/AvatarHashMap.h

@@ -27,7 +27,6 @@
 
 #include "AvatarData.h"
 
-
 class AvatarHashMap : public QObject, public Dependency {
     Q_OBJECT
     SINGLETON_DEPENDENCY

libraries/entities/src/EntityScriptingInterface.cpp

@@ -1419,8 +1419,7 @@ QVector<QUuid> EntityScriptingInterface::getChildrenIDsOfJoint(const QUuid& pare
             return;
         }
         parent->forEachChild([&](SpatiallyNestablePointer child) {
-            if (child->getParentJointIndex() == jointIndex &&
-                child->getNestableType() != NestableType::Overlay) {
+            if (child->getParentJointIndex() == jointIndex) {
                 result.push_back(child->getID());
             }
         });

libraries/model-networking/src/model-networking/ModelCache.h

@@ -44,6 +44,8 @@ public:
     // Mutable, but must retain structure of vector
     using NetworkMaterials = std::vector<std::shared_ptr<NetworkMaterial>>;
 
+    bool isGeometryLoaded() const { return (bool)_fbxGeometry; }
+
     const FBXGeometry& getFBXGeometry() const { return *_fbxGeometry; }
     const GeometryMeshes& getMeshes() const { return *_meshes; }
     const std::shared_ptr<const NetworkMaterial> getShapeMaterial(int shapeID) const;

libraries/model/src/model/TextureMap.cpp

@@ -336,8 +336,7 @@ gpu::Texture* TextureUsage::createNormalTextureFromBumpImage(const QImage& srcIm
     const double pStrength = 2.0;
     int width = image.width();
     int height = image.height();
-    // THe end result image for normal map is RGBA32 even though the A is not used
-    QImage result(width, height, QImage::Format_RGBA8888);
+    QImage result(width, height, QImage::Format_RGB888);
     
     for (int i = 0; i < width; i++) {
         const int iNextClamped = clampPixelCoordinate(i + 1, width - 1);
@@ -377,21 +376,19 @@ gpu::Texture* TextureUsage::createNormalTextureFromBumpImage(const QImage& srcIm
             glm::normalize(v);
     
             // convert to rgb from the value obtained computing the filter
-            QRgb qRgbValue = qRgb(mapComponent(v.x), mapComponent(v.y), mapComponent(v.z));
+            QRgb qRgbValue = qRgba(mapComponent(v.x), mapComponent(v.y), mapComponent(v.z), 1.0);
             result.setPixel(i, j, qRgbValue);
         }
     }
 
     gpu::Texture* theTexture = nullptr;
     if ((image.width() > 0) && (image.height() > 0)) {
-
-        gpu::Element formatGPU = gpu::Element(gpu::VEC4, gpu::NUINT8, gpu::RGBA);
-        gpu::Element formatMip = gpu::Element(gpu::VEC4, gpu::NUINT8, gpu::RGBA);
+        gpu::Element formatGPU = gpu::Element(gpu::VEC3, gpu::NUINT8, gpu::RGB);
+        gpu::Element formatMip = gpu::Element(gpu::VEC3, gpu::NUINT8, gpu::RGB);
 
         theTexture = (gpu::Texture::create2D(formatGPU, image.width(), image.height(), gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_MIP_LINEAR)));
         theTexture->setSource(srcImageName);
         theTexture->assignStoredMip(0, formatMip, image.byteCount(), image.constBits());
-        generateMips(theTexture, image, formatMip, true);
     }
 
     return theTexture;

libraries/networking/src/LimitedNodeList.h

@@ -34,6 +34,7 @@
 #include <tbb/concurrent_unordered_map.h>
 
 #include <DependencyManager.h>
+#include <SharedUtil.h>
 
 #include "DomainHandler.h"
 #include "Node.h"
@@ -182,15 +183,33 @@ public:
     //   This allows multiple threads (i.e. a thread pool) to share a lock
     //   without deadlocking when a dying node attempts to acquire a write lock
     template<typename NestedNodeLambda>
-    void nestedEach(NestedNodeLambda functor) {
-        QReadLocker readLock(&_nodeMutex);
+    void nestedEach(NestedNodeLambda functor, 
+                    int* lockWaitOut = nullptr, 
+                    int* nodeTransformOut = nullptr, 
+                    int* functorOut = nullptr) {
+        auto start = usecTimestampNow();
+        {
+            QReadLocker readLock(&_nodeMutex);
+            auto endLock = usecTimestampNow();
+            if (lockWaitOut) {
+                *lockWaitOut = (endLock - start);
+            }
 
-        std::vector<SharedNodePointer> nodes(_nodeHash.size());
-        std::transform(_nodeHash.cbegin(), _nodeHash.cend(), nodes.begin(), [](const NodeHash::value_type& it) {
-            return it.second;
-        });
+            std::vector<SharedNodePointer> nodes(_nodeHash.size());
+            std::transform(_nodeHash.cbegin(), _nodeHash.cend(), nodes.begin(), [](const NodeHash::value_type& it) {
+                return it.second;
+            });
+            auto endTransform = usecTimestampNow();
+            if (nodeTransformOut) {
+                *nodeTransformOut = (endTransform - endLock);
+            }
 
-        functor(nodes.cbegin(), nodes.cend());
+            functor(nodes.cbegin(), nodes.cend());
+            auto endFunctor = usecTimestampNow();
+            if (functorOut) {
+                *functorOut = (endFunctor - endTransform);
+            }
+        }
     }
 
     template<typename NodeLambda>

libraries/networking/src/UserActivityLoggerScriptingInterface.cpp

@@ -16,8 +16,8 @@ void UserActivityLoggerScriptingInterface::enabledEdit() {
     logAction("enabled_edit");
 }
 
-void UserActivityLoggerScriptingInterface::openedTablet() {
-    logAction("opened_tablet");
+void UserActivityLoggerScriptingInterface::openedTablet(bool visibleToOthers) {
+    logAction("opened_tablet", { { "visible_to_others", visibleToOthers } });
 }
 
 void UserActivityLoggerScriptingInterface::closedTablet() {

libraries/networking/src/UserActivityLoggerScriptingInterface.h

@@ -21,7 +21,7 @@ class UserActivityLoggerScriptingInterface : public QObject, public Dependency {
     Q_OBJECT
 public:
     Q_INVOKABLE void enabledEdit();
-    Q_INVOKABLE void openedTablet();
+    Q_INVOKABLE void openedTablet(bool visibleToOthers);
     Q_INVOKABLE void closedTablet();
     Q_INVOKABLE void openedMarketplace();
     Q_INVOKABLE void toggledAway(bool isAway);

libraries/networking/src/udt/PacketHeaders.h

@@ -112,7 +112,8 @@ public:
         ReloadEntityServerScript,
         EntityPhysics,
         EntityServerScriptLog,
-        LAST_PACKET_TYPE = EntityServerScriptLog
+        AdjustAvatarSorting,
+        LAST_PACKET_TYPE = AdjustAvatarSorting
     };
 };
 

libraries/render-utils/src/Model.h

@@ -114,7 +114,7 @@ public:
     void setBlendedVertices(int blendNumber, const Geometry::WeakPointer& geometry,
         const QVector<glm::vec3>& vertices, const QVector<glm::vec3>& normals);
 
-    bool isLoaded() const { return (bool)_renderGeometry; }
+    bool isLoaded() const { return (bool)_renderGeometry && _renderGeometry->isGeometryLoaded(); }
 
     void setIsWireframe(bool isWireframe) { _isWireframe = isWireframe; }
     bool isWireframe() const { return _isWireframe; }

libraries/script-engine/src/RecordingScriptingInterface.cpp

@@ -56,6 +56,11 @@ bool RecordingScriptingInterface::loadRecording(const QString& url) {
     using namespace recording;
 
     auto loader = ClipCache::instance().getClipLoader(url);
+    if (!loader) {
+        qWarning() << "Clip failed to load from " << url;
+        return false;
+    }
+
     if (!loader->isLoaded()) {
         QEventLoop loop;
         QObject::connect(loader.data(), &Resource::loaded, &loop, &QEventLoop::quit);

libraries/script-engine/src/ScriptEngine.cpp

@@ -576,6 +576,7 @@ void ScriptEngine::init() {
     qScriptRegisterMetaType(this, EntityItemIDtoScriptValue, EntityItemIDfromScriptValue);
     qScriptRegisterMetaType(this, RayToEntityIntersectionResultToScriptValue, RayToEntityIntersectionResultFromScriptValue);
     qScriptRegisterMetaType(this, RayToAvatarIntersectionResultToScriptValue, RayToAvatarIntersectionResultFromScriptValue);
+    qScriptRegisterMetaType(this, AvatarEntityMapToScriptValue, AvatarEntityMapFromScriptValue);
     qScriptRegisterSequenceMetaType<QVector<QUuid>>(this);
     qScriptRegisterSequenceMetaType<QVector<EntityItemID>>(this);
 

libraries/script-engine/src/TabletScriptingInterface.cpp

@@ -366,6 +366,7 @@ void TabletProxy::gotoWebScreen(const QString& url, const QString& injectedJavaS
     }
 
     if (root) {
+        removeButtonsFromHomeScreen();
         QMetaObject::invokeMethod(root, "loadSource", Q_ARG(const QVariant&, QVariant(WEB_VIEW_SOURCE_URL)));
         QMetaObject::invokeMethod(root, "setShown", Q_ARG(const QVariant&, QVariant(true)));
         QMetaObject::invokeMethod(root, "loadWebUrl", Q_ARG(const QVariant&, QVariant(url)), Q_ARG(const QVariant&, QVariant(injectedJavaScriptUrl)));

libraries/script-engine/src/UsersScriptingInterface.h

@@ -150,7 +150,6 @@ signals:
 private:
     bool getRequestsDomainListData();
     void setRequestsDomainListData(bool requests);
-    bool _requestsDomainListData;
 };
 
 

libraries/shared/src/SpatiallyNestable.h

@@ -179,9 +179,9 @@ public:
             const glm::vec3& localVelocity,
             const glm::vec3& localAngularVelocity);
 
-    bool scaleChangedSince(quint64 time) { return _scaleChanged > time; }
-    bool tranlationChangedSince(quint64 time) { return _translationChanged > time; }
-    bool rotationChangedSince(quint64 time) { return _rotationChanged > time; }
+    bool scaleChangedSince(quint64 time) const { return _scaleChanged > time; }
+    bool tranlationChangedSince(quint64 time) const { return _translationChanged > time; }
+    bool rotationChangedSince(quint64 time) const { return _rotationChanged > time; }
 
 protected:
     const NestableType _nestableType; // EntityItem or an AvatarData

scripts/defaultScripts.js

@@ -20,7 +20,7 @@ var DEFAULT_SCRIPTS = [
     "system/bubble.js",
     "system/snapshot.js",
     "system/help.js",
-    "system/pal.js", //"system/mod.js", // older UX, if you prefer
+    "system/pal.js", // "system/mod.js", // older UX, if you prefer
     "system/goto.js",
     "system/marketplaces/marketplaces.js",
     "system/edit.js",
@@ -54,9 +54,6 @@ if (previousSetting === true || previousSetting === 'true') {
     previousSetting = true;
 }
 
-
-
-
 if (Menu.menuExists(MENU_CATEGORY) && !Menu.menuItemExists(MENU_CATEGORY, MENU_ITEM)) {
     Menu.addMenuItem({
         menuName: MENU_CATEGORY,
@@ -78,11 +75,11 @@ function runDefaultsSeparately() {
         Script.load(DEFAULT_SCRIPTS[i]);
     }
 }
+
 // start all scripts
 if (Menu.isOptionChecked(MENU_ITEM)) {
     // we're debugging individual default scripts
     // so we load each into its own ScriptEngine instance
-    debuggingDefaultScripts = true;
     runDefaultsSeparately();
 } else {
     // include all default scripts into this ScriptEngine
@@ -90,32 +87,14 @@ if (Menu.isOptionChecked(MENU_ITEM)) {
 }
 
 function menuItemEvent(menuItem) {
-    if (menuItem == MENU_ITEM) {
-
-        isChecked = Menu.isOptionChecked(MENU_ITEM);
+    if (menuItem === MENU_ITEM) {
+        var isChecked = Menu.isOptionChecked(MENU_ITEM);
         if (isChecked === true) {
             Settings.setValue(SETTINGS_KEY, true);
         } else if (isChecked === false) {
             Settings.setValue(SETTINGS_KEY, false);
         }
-         Window.alert('You must reload all scripts for this to take effect.')
-    }
-
-
-}
-
-
-
-function stopLoadedScripts() {
-        // remove debug script loads
-    var runningScripts = ScriptDiscoveryService.getRunning();
-    for (var i in runningScripts) {
-        var scriptName = runningScripts[i].name;
-        for (var j in DEFAULT_SCRIPTS) {
-            if (DEFAULT_SCRIPTS[j].slice(-scriptName.length) === scriptName) {
-                ScriptDiscoveryService.stopScript(runningScripts[i].url);
-            }
-        }
+        Menu.triggerOption("Reload All Scripts");
     }
 }
 
@@ -126,7 +105,6 @@ function removeMenuItem() {
 }
 
 Script.scriptEnding.connect(function() {
-    stopLoadedScripts();
     removeMenuItem();
 });
 

scripts/developer/debugging/debugAvatarMixer.js

@@ -58,11 +58,14 @@ function updateOverlays() {
 
             // setup a position for the overlay that is just above this avatar's head
             var overlayPosition = avatar.getJointPosition("Head");
-            overlayPosition.y += 1.05;
+            overlayPosition.y += 1.15;
+
+            var rows = 8;
 
             var text = avatarID + "\n"
                        +"--- Data from Mixer ---\n"
                        +"All: " + AvatarManager.getAvatarDataRate(avatarID).toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID).toFixed(2) + "hz)" + "\n"
+                       /*
                        +" GP: " + AvatarManager.getAvatarDataRate(avatarID,"globalPosition").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"globalPosition").toFixed(2) + "hz)" + "\n"
                        +" LP: " + AvatarManager.getAvatarDataRate(avatarID,"localPosition").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"localPosition").toFixed(2) + "hz)" + "\n"
                        +" BB: " + AvatarManager.getAvatarDataRate(avatarID,"avatarBoundingBox").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"avatarBoundingBox").toFixed(2) + "hz)" + "\n"
@@ -74,11 +77,12 @@ function updateOverlays() {
                        +" AF: " + AvatarManager.getAvatarDataRate(avatarID,"additionalFlags").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"additionalFlags").toFixed(2) + "hz)" + "\n"
                        +" PI: " + AvatarManager.getAvatarDataRate(avatarID,"parentInfo").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"parentInfo").toFixed(2) + "hz)" + "\n"
                        +" FT: " + AvatarManager.getAvatarDataRate(avatarID,"faceTracker").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"faceTracker").toFixed(2) + "hz)" + "\n"
+                       */
                        +" JD: " + AvatarManager.getAvatarDataRate(avatarID,"jointData").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"jointData").toFixed(2) + "hz)" + "\n"
                        +"--- Simulation ---\n"
                        +"All: " + AvatarManager.getAvatarSimulationRate(avatarID,"avatar").toFixed(2) + "hz \n"
                        +" inView: " + AvatarManager.getAvatarSimulationRate(avatarID,"avatarInView").toFixed(2) + "hz \n"
-                       +" SM: " + AvatarManager.getAvatarSimulationRate(avatarID,"skeletonModel").toFixed(2) + "hz \n"
+                       //+" SM: " + AvatarManager.getAvatarSimulationRate(avatarID,"skeletonModel").toFixed(2) + "hz \n"
                        +" JD: " + AvatarManager.getAvatarSimulationRate(avatarID,"jointData").toFixed(2) + "hz \n"
 
             if (avatarID in debugOverlays) {
@@ -93,7 +97,7 @@ function updateOverlays() {
                     position: overlayPosition,
                     dimensions: {
                         x: 1.25,
-                        y: 19 * 0.13
+                        y: rows * 0.13
                     },
                     lineHeight: 0.1,
                     font:{size:0.1},

scripts/developer/utilities/record/recorder.js

@@ -12,14 +12,14 @@
 HIFI_PUBLIC_BUCKET = "http://s3.amazonaws.com/hifi-public/";
 Script.include("/~/system/libraries/toolBars.js");
 
-var recordingFile = "recording.rec";
+var recordingFile = "recording.hfr";
 
 function setPlayerOptions() {
     Recording.setPlayFromCurrentLocation(true);
     Recording.setPlayerUseDisplayName(false);
     Recording.setPlayerUseAttachments(false);
     Recording.setPlayerUseHeadModel(false);
-    Recording.setPlayerUseSkeletonModel(false);
+    Recording.setPlayerUseSkeletonModel(true);
 }
 
 var windowDimensions = Controller.getViewportDimensions();
@@ -142,7 +142,6 @@ function setupTimer() {
         backgroundAlpha: 1.0,
         visible: true
     });
-
 }
 
 function updateTimer() {
@@ -272,7 +271,7 @@ function mousePressEvent(event) {
         }
     } else if (loadIcon === toolBar.clicked(clickedOverlay)) {
         if (!Recording.isRecording() && !Recording.isPlaying()) {
-            recordingFile = Window.browse("Load recorcding from file", ".", "Recordings (*.hfr *.rec *.HFR *.REC)");
+            recordingFile = Window.browse("Load recording from file", ".", "Recordings (*.hfr *.rec *.HFR *.REC)");
             if (!(recordingFile === "null" || recordingFile === null || recordingFile === "")) {
                 Recording.loadRecording(recordingFile);
             }
@@ -345,5 +344,3 @@ Script.scriptEnding.connect(scriptEnding);
 
 // Should be called last to put everything into position
 moveUI();
-
-

scripts/system/audio.js

@@ -16,19 +16,51 @@
 var TABLET_BUTTON_NAME = "AUDIO";
 var HOME_BUTTON_TEXTURE = "http://hifi-content.s3.amazonaws.com/alan/dev/tablet-with-home-button.fbx/tablet-with-home-button.fbm/button-root.png";
 
+var MUTE_ICONS = {
+    icon: "icons/tablet-icons/mic-mute-i.svg",
+    activeIcon: "icons/tablet-icons/mic-mute-a.svg"
+};
+
+var UNMUTE_ICONS = {
+    icon: "icons/tablet-icons/mic-unmute-i.svg",
+    activeIcon: "icons/tablet-icons/mic-unmute-a.svg"
+};
+
 function onMuteToggled() {
-    button.editProperties({ isActive: AudioDevice.getMuted() });
+    if (AudioDevice.getMuted()) {
+        button.editProperties(MUTE_ICONS);
+    } else {
+        button.editProperties(UNMUTE_ICONS);
+    }
 }
-function onClicked(){
-    var entity = HMD.tabletID;
-    Entities.editEntity(entity, { textures: JSON.stringify({ "tex.close": HOME_BUTTON_TEXTURE }) });
-    tablet.gotoMenuScreen("Audio");
+
+var shouldActivateButton = false;
+var onAudioScreen = false;
+
+function onClicked() {
+    if (onAudioScreen) {
+        // for toolbar-mode: go back to home screen, this will close the window.
+        tablet.gotoHomeScreen();
+    } else {
+        var entity = HMD.tabletID;
+        Entities.editEntity(entity, { textures: JSON.stringify({ "tex.close": HOME_BUTTON_TEXTURE }) });
+        shouldActivateButton = true;
+        tablet.gotoMenuScreen("Audio");
+        onAudioScreen = true;
+    }
+}
+
+function onScreenChanged(type, url) {
+    // for toolbar mode: change button to active when window is first openend, false otherwise.
+    button.editProperties({isActive: shouldActivateButton});
+    shouldActivateButton = false;
+    onAudioScreen = false;
 }
 
 var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
 var button = tablet.addButton({
-    icon: "icons/tablet-icons/mic-unmute-i.svg",
-    activeIcon: "icons/tablet-icons/mic-mute-a.svg",
+    icon: AudioDevice.getMuted() ? MUTE_ICONS.icon : UNMUTE_ICONS.icon,
+    activeIcon: AudioDevice.getMuted() ? MUTE_ICONS.activeIcon : UNMUTE_ICONS.activeIcon,
     text: TABLET_BUTTON_NAME,
     sortOrder: 1
 });
@@ -36,10 +68,12 @@ var button = tablet.addButton({
 onMuteToggled();
 
 button.clicked.connect(onClicked);
+tablet.screenChanged.connect(onScreenChanged);
 AudioDevice.muteToggled.connect(onMuteToggled);
 
 Script.scriptEnding.connect(function () {
     button.clicked.disconnect(onClicked);
+    tablet.screenChanged.disconnect(onScreenChanged);
     AudioDevice.muteToggled.disconnect(onMuteToggled);
     tablet.removeButton(button);
 });

scripts/system/controllers/grab.js

@@ -331,6 +331,12 @@ Grabber.prototype.pressEvent = function(event) {
     }
 
     var pickRay = Camera.computePickRay(event.x, event.y);
+
+    var overlayResult = Overlays.findRayIntersection(pickRay, true, [HMD.tabletID, HMD.tabletScreenID, HMD.homeButtonID]);
+    if (overlayResult.intersects) {
+        return;
+    }
+
     var pickResults = Entities.findRayIntersection(pickRay, true); // accurate picking
     if (!pickResults.intersects) {
         // didn't click on anything

scripts/system/controllers/handControllerGrab.js

@@ -14,7 +14,7 @@
 
 /* global getEntityCustomData, flatten, Xform, Script, Quat, Vec3, MyAvatar, Entities, Overlays, Settings,
    Reticle, Controller, Camera, Messages, Mat4, getControllerWorldLocation, getGrabPointSphereOffset, setGrabCommunications,
-   Menu, HMD */
+   Menu, HMD, isInEditMode */
 /* eslint indent: ["error", 4, { "outerIIFEBody": 0 }] */
 
 (function() { // BEGIN LOCAL_SCOPE
@@ -399,7 +399,7 @@ function entityHasActions(entityID) {
 
 function findRayIntersection(pickRay, precise, include, exclude) {
     var entities = Entities.findRayIntersection(pickRay, precise, include, exclude, true);
-    var overlays = Overlays.findRayIntersection(pickRay);
+    var overlays = Overlays.findRayIntersection(pickRay, precise, [], [HMD.tabletID]);
     if (!overlays.intersects || (entities.intersects && (entities.distance <= overlays.distance))) {
         return entities;
     }
@@ -644,6 +644,7 @@ EquipHotspotBuddy.prototype.updateHotspot = function(hotspot, timestamp) {
 
         // override default sphere with a user specified model, if it exists.
         overlayInfoSet.overlays.push(Overlays.addOverlay("model", {
+            name: "hotspot overlay",
             url: hotspot.modelURL ? hotspot.modelURL : DEFAULT_SPHERE_MODEL_URL,
             position: hotspot.worldPosition,
             rotation: {
@@ -776,7 +777,7 @@ function MyController(hand) {
     };
 
     this.actionID = null; // action this script created...
-    this.grabbedEntity = null; // on this entity.
+    this.grabbedThingID = null; // on this entity.
     this.grabbedOverlay = null;
     this.state = STATE_OFF;
     this.pointer = null; // entity-id of line object
@@ -853,14 +854,19 @@ function MyController(hand) {
     };
 
     this.callEntityMethodOnGrabbed = function(entityMethodName) {
+        if (this.grabbedIsOverlay) {
+            return;
+        }
         var args = [this.hand === RIGHT_HAND ? "right" : "left", MyAvatar.sessionUUID];
-        Entities.callEntityMethod(this.grabbedEntity, entityMethodName, args);
+        Entities.callEntityMethod(this.grabbedThingID, entityMethodName, args);
     };
 
     this.setState = function(newState, reason) {
-        if ((isInEditMode() && this.grabbedEntity !== HMD.tabletID )&& (newState !== STATE_OFF &&
-                               newState !== STATE_SEARCHING &&
-                               newState !== STATE_OVERLAY_STYLUS_TOUCHING)) {
+        if ((isInEditMode() && this.grabbedThingID !== HMD.tabletID) &&
+            (newState !== STATE_OFF &&
+             newState !== STATE_SEARCHING &&
+             newState !== STATE_OVERLAY_STYLUS_TOUCHING &&
+             newState !== STATE_OVERLAY_LASER_TOUCHING)) {
             return;
         }
         setGrabCommunications((newState === STATE_DISTANCE_HOLDING) || (newState === STATE_NEAR_GRABBING));
@@ -903,6 +909,7 @@ function MyController(hand) {
 
         if (!this.grabPointSphere) {
             this.grabPointSphere = Overlays.addOverlay("sphere", {
+                name: "grabPointSphere",
                 localPosition: getGrabPointSphereOffset(this.handToController()),
                 localRotation: { x: 0, y: 0, z: 0, w: 1 },
                 dimensions: GRAB_POINT_SPHERE_RADIUS * 2,
@@ -933,6 +940,7 @@ function MyController(hand) {
         var brightColor = colorPow(color, 0.06);
         if (this.searchSphere === null) {
             var sphereProperties = {
+                name: "searchSphere",
                 position: location,
                 rotation: rotation,
                 outerRadius: size * 1.2,
@@ -955,7 +963,8 @@ function MyController(hand) {
                 innerAlpha: 1.0,
                 outerAlpha: 0.0,
                 outerRadius: size * 1.2,
-                visible: true
+                visible: true,
+                ignoreRayIntersection: true
             });
         }
     };
@@ -966,6 +975,7 @@ function MyController(hand) {
         }
 
         var stylusProperties = {
+            name: "stylus",
             url: Script.resourcesPath() + "meshes/tablet-stylus-fat.fbx",
             localPosition: Vec3.sum({ x: 0.0,
                                       y: WEB_TOUCH_Y_OFFSET,
@@ -1000,6 +1010,7 @@ function MyController(hand) {
     this.overlayLineOn = function(closePoint, farPoint, color) {
         if (this.overlayLine === null) {
             var lineProperties = {
+                name: "line",
                 glow: 1.0,
                 start: closePoint,
                 end: farPoint,
@@ -1175,6 +1186,13 @@ function MyController(hand) {
             }
         }
 
+        var candidateOverlays = Overlays.findOverlays(worldHandPosition, WEB_DISPLAY_STYLUS_DISTANCE);
+        for (var j = 0; j < candidateOverlays.length; j++) {
+            if (this.isTablet(candidateOverlays[j])) {
+                nearWeb = true;
+            }
+        }
+
         if (nearWeb) {
             this.showStylus();
             var rayPickInfo = this.calcRayPickInfo(this.hand);
@@ -1233,7 +1251,13 @@ function MyController(hand) {
         });
         if (grabbableEntities.length > 0) {
             if (!this.grabPointIntersectsEntity) {
-                Controller.triggerHapticPulse(1, 20, this.hand);
+                // don't do haptic pulse for tablet
+                var nonTabletEntities = grabbableEntities.filter(function(entityID) {
+                    return entityID != HMD.tabletID && entityID != HMD.homeButtonID;
+                });
+                if (nonTabletEntities.length > 0) {
+                    Controller.triggerHapticPulse(1, 20, this.hand);
+                }
                 this.grabPointIntersectsEntity = true;
                 this.grabPointSphereOn();
             }
@@ -1417,7 +1441,7 @@ function MyController(hand) {
 
         var okToEquipFromOtherHand = ((this.getOtherHandController().state == STATE_NEAR_GRABBING ||
                                        this.getOtherHandController().state == STATE_DISTANCE_HOLDING) &&
-                                      this.getOtherHandController().grabbedEntity == hotspot.entityID);
+                                      this.getOtherHandController().grabbedThingID == hotspot.entityID);
         var hasParent = true;
         if (props.parentID === NULL_UUID) {
             hasParent = false;
@@ -1575,7 +1599,7 @@ function MyController(hand) {
 
         var farSearching =  this.triggerSmoothedSqueezed() && (Date.now() - this.searchStartTime > FAR_SEARCH_DELAY);
 
-        this.grabbedEntity = null;
+        this.grabbedThingID = null;
         this.grabbedOverlay = null;
         this.isInitialGrab = false;
         this.preparingHoldRelease = false;
@@ -1583,7 +1607,7 @@ function MyController(hand) {
         this.checkForUnexpectedChildren();
 
         if ((this.triggerSmoothedReleased() && this.secondaryReleased())) {
-            this.grabbedEntity = null;
+            this.grabbedThingID = null;
             this.setState(STATE_OFF, "trigger released");
             return;
         }
@@ -1604,8 +1628,9 @@ function MyController(hand) {
         if (potentialEquipHotspot) {
             if ((this.triggerSmoothedGrab() || this.secondarySqueezed()) && holdEnabled) {
                 this.grabbedHotspot = potentialEquipHotspot;
-                this.grabbedEntity = potentialEquipHotspot.entityID;
-                this.setState(STATE_HOLD, "equipping '" + entityPropertiesCache.getProps(this.grabbedEntity).name + "'");
+                this.grabbedThingID = potentialEquipHotspot.entityID;
+                this.grabbedIsOverlay = false;
+                this.setState(STATE_HOLD, "equipping '" + entityPropertiesCache.getProps(this.grabbedThingID).name + "'");
 
                 return;
             }
@@ -1616,6 +1641,11 @@ function MyController(hand) {
             return _this.entityIsNearGrabbable(entity, handPosition, NEAR_GRAB_MAX_DISTANCE);
         });
 
+        var candidateOverlays = Overlays.findOverlays(handPosition, NEAR_GRAB_RADIUS);
+        var grabbableOverlays = candidateOverlays.filter(function(overlayID) {
+            return Overlays.getProperty(overlayID, "grabbable");
+        });
+
         if (rayPickInfo.entityID) {
             this.intersectionDistance = rayPickInfo.distance;
             if (this.entityIsGrabbable(rayPickInfo.entityID) && rayPickInfo.distance < NEAR_GRAB_PICK_RADIUS) {
@@ -1627,6 +1657,23 @@ function MyController(hand) {
             this.intersectionDistance = 0;
         }
 
+        if (grabbableOverlays.length > 0) {
+            grabbableOverlays.sort(function(a, b) {
+                var aPosition = Overlays.getProperty(a, "position");
+                var aDistance = Vec3.distance(aPosition, handPosition);
+                var bPosition = Overlays.getProperty(b, "position");
+                var bDistance = Vec3.distance(bPosition, handPosition);
+                return aDistance - bDistance;
+            });
+            this.grabbedThingID = grabbableOverlays[0];
+            this.grabbedIsOverlay = true;
+            if ((this.triggerSmoothedGrab() || this.secondarySqueezed()) && nearGrabEnabled) {
+                this.setState(STATE_NEAR_GRABBING, "near grab overlay '" +
+                              Overlays.getProperty(this.grabbedThingID, "name") + "'");
+                return;
+            }
+        }
+
         var entity;
         if (grabbableEntities.length > 0) {
             // sort by distance
@@ -1637,7 +1684,8 @@ function MyController(hand) {
             });
             entity = grabbableEntities[0];
             name = entityPropertiesCache.getProps(entity).name;
-            this.grabbedEntity = entity;
+            this.grabbedThingID = entity;
+            this.grabbedIsOverlay = false;
             if (this.entityWantsTrigger(entity)) {
                 if (this.triggerSmoothedGrab()) {
                     this.setState(STATE_NEAR_TRIGGER, "near trigger '" + name + "'");
@@ -1648,7 +1696,7 @@ function MyController(hand) {
             } else {
                 //  If near something grabbable, grab it!
                 if ((this.triggerSmoothedGrab() || this.secondarySqueezed()) && nearGrabEnabled) {
-                    this.setState(STATE_NEAR_GRABBING, "near grab '" + name + "'");
+                    this.setState(STATE_NEAR_GRABBING, "near grab entity '" + name + "'");
                     return;
                 } else {
                     // potentialNearGrabEntity = entity;
@@ -1671,7 +1719,8 @@ function MyController(hand) {
             name = entityPropertiesCache.getProps(entity).name;
             if (this.entityWantsTrigger(entity)) {
                 if (this.triggerSmoothedGrab()) {
-                    this.grabbedEntity = entity;
+                    this.grabbedThingID = entity;
+                    this.grabbedIsOverlay = false;
                     this.setState(STATE_FAR_TRIGGER, "far trigger '" + name + "'");
                     return;
                 } else {
@@ -1679,7 +1728,8 @@ function MyController(hand) {
                 }
             } else if (this.entityIsDistanceGrabbable(rayPickInfo.entityID, handPosition)) {
                 if (this.triggerSmoothedGrab() && !isEditing() && farGrabEnabled && farSearching) {
-                    this.grabbedEntity = entity;
+                    this.grabbedThingID = entity;
+                    this.grabbedIsOverlay = false;
                     this.grabbedDistance = rayPickInfo.distance;
                     this.setState(STATE_DISTANCE_HOLDING, "distance hold '" + name + "'");
                     return;
@@ -1759,7 +1809,8 @@ function MyController(hand) {
                 Entities.sendHoverOverEntity(entity, pointerEvent);
             }
 
-            this.grabbedEntity = entity;
+            this.grabbedThingID = entity;
+            this.grabbedIsOverlay = false;
             this.setState(STATE_ENTITY_STYLUS_TOUCHING, "begin touching entity '" + name + "'");
             return true;
 
@@ -1887,7 +1938,8 @@ function MyController(hand) {
             }
 
             if (this.triggerSmoothedGrab() && (!isEditing() || this.isTablet(entity))) {
-                this.grabbedEntity = entity;
+                this.grabbedThingID = entity;
+                this.grabbedIsOverlay = false;
                 this.setState(STATE_ENTITY_LASER_TOUCHING, "begin touching entity '" + name + "'");
                 return true;
             }
@@ -1998,7 +2050,7 @@ function MyController(hand) {
         var worldToSensorMat = Mat4.inverse(MyAvatar.getSensorToWorldMatrix());
         var roomControllerPosition = Mat4.transformPoint(worldToSensorMat, worldControllerPosition);
 
-        var grabbedProperties = Entities.getEntityProperties(this.grabbedEntity, GRABBABLE_PROPERTIES);
+        var grabbedProperties = Entities.getEntityProperties(this.grabbedThingID, GRABBABLE_PROPERTIES);
         var now = Date.now();
 
         // add the action and initialize some variables
@@ -2028,7 +2080,7 @@ function MyController(hand) {
         var timeScale = this.distanceGrabTimescale(this.mass, distanceToObject);
 
         this.actionID = NULL_UUID;
-        this.actionID = Entities.addAction("spring", this.grabbedEntity, {
+        this.actionID = Entities.addAction("spring", this.grabbedThingID, {
             targetPosition: this.currentObjectPosition,
             linearTimeScale: timeScale,
             targetRotation: this.currentObjectRotation,
@@ -2053,12 +2105,12 @@ function MyController(hand) {
     this.ensureDynamic = function() {
         // if we distance hold something and keep it very still before releasing it, it ends up
         // non-dynamic in bullet.  If it's too still, give it a little bounce so it will fall.
-        var props = Entities.getEntityProperties(this.grabbedEntity, ["velocity", "dynamic", "parentID"]);
+        var props = Entities.getEntityProperties(this.grabbedThingID, ["velocity", "dynamic", "parentID"]);
         if (props.dynamic && props.parentID == NULL_UUID) {
             var velocity = props.velocity;
             if (Vec3.length(velocity) < 0.05) { // see EntityMotionState.cpp DYNAMIC_LINEAR_VELOCITY_THRESHOLD
                 velocity = { x: 0.0, y: 0.2, z:0.0 };
-                Entities.editEntity(this.grabbedEntity, { velocity: velocity });
+                Entities.editEntity(this.grabbedThingID, { velocity: velocity });
             }
         }
     };
@@ -2080,7 +2132,7 @@ function MyController(hand) {
         var worldToSensorMat = Mat4.inverse(MyAvatar.getSensorToWorldMatrix());
         var roomControllerPosition = Mat4.transformPoint(worldToSensorMat, worldControllerPosition);
 
-        var grabbedProperties = Entities.getEntityProperties(this.grabbedEntity, GRABBABLE_PROPERTIES);
+        var grabbedProperties = Entities.getEntityProperties(this.grabbedThingID, GRABBABLE_PROPERTIES);
 
         var now = Date.now();
         var deltaObjectTime = (now - this.currentObjectTime) / MSECS_PER_SEC; // convert to seconds
@@ -2135,7 +2187,7 @@ function MyController(hand) {
         newTargetPosition = Vec3.sum(newTargetPosition, this.offsetPosition);
 
         var objectToAvatar = Vec3.subtract(this.currentObjectPosition, MyAvatar.position);
-        var handControllerData = getEntityCustomData('handControllerKey', this.grabbedEntity, defaultMoveWithHeadData);
+        var handControllerData = getEntityCustomData('handControllerKey', this.grabbedThingID, defaultMoveWithHeadData);
         if (handControllerData.disableMoveWithHead !== true) {
             // mix in head motion
             if (MOVE_WITH_HEAD) {
@@ -2164,7 +2216,7 @@ function MyController(hand) {
         this.overlayLineOn(rayPickInfo.searchRay.origin, Vec3.subtract(grabbedProperties.position, this.offsetPosition), COLORS_GRAB_DISTANCE_HOLD);
 
         var distanceToObject = Vec3.length(Vec3.subtract(MyAvatar.position, this.currentObjectPosition));
-        var success = Entities.updateAction(this.grabbedEntity, this.actionID, {
+        var success = Entities.updateAction(this.grabbedThingID, this.actionID, {
             targetPosition: newTargetPosition,
             linearTimeScale: this.distanceGrabTimescale(this.mass, distanceToObject),
             targetRotation: this.currentObjectRotation,
@@ -2181,7 +2233,7 @@ function MyController(hand) {
     };
 
     this.setupHoldAction = function() {
-        this.actionID = Entities.addAction("hold", this.grabbedEntity, {
+        this.actionID = Entities.addAction("hold", this.grabbedThingID, {
             hand: this.hand === RIGHT_HAND ? "right" : "left",
             timeScale: NEAR_GRABBING_ACTION_TIMEFRAME,
             relativePosition: this.offsetPosition,
@@ -2271,17 +2323,30 @@ function MyController(hand) {
         Controller.triggerHapticPulse(HAPTIC_PULSE_STRENGTH, HAPTIC_PULSE_DURATION, this.hand);
 
         if (this.entityActivated) {
-            var saveGrabbedID = this.grabbedEntity;
+            var saveGrabbedID = this.grabbedThingID;
             this.release();
-            this.grabbedEntity = saveGrabbedID;
+            this.grabbedThingID = saveGrabbedID;
         }
 
-        var grabbedProperties = Entities.getEntityProperties(this.grabbedEntity, GRABBABLE_PROPERTIES);
-        if (FORCE_IGNORE_IK) {
+        var grabbedProperties;
+        if (this.grabbedIsOverlay) {
+            grabbedProperties = {
+                position: Overlays.getProperty(this.grabbedThingID, "position"),
+                rotation: Overlays.getProperty(this.grabbedThingID, "rotation"),
+                parentID: Overlays.getProperty(this.grabbedThingID, "parentID"),
+                parentJointIndex: Overlays.getProperty(this.grabbedThingID, "parentJointIndex"),
+                dynamic: false,
+                shapeType: "none"
+            };
             this.ignoreIK = true;
         } else {
-            var grabbableData = getEntityCustomData(GRABBABLE_DATA_KEY, this.grabbedEntity, DEFAULT_GRABBABLE_DATA);
-            this.ignoreIK = (grabbableData.ignoreIK !== undefined) ? grabbableData.ignoreIK : true;
+            grabbedProperties = Entities.getEntityProperties(this.grabbedThingID, GRABBABLE_PROPERTIES);
+            if (FORCE_IGNORE_IK) {
+                this.ignoreIK = true;
+            } else {
+                var grabbableData = getEntityCustomData(GRABBABLE_DATA_KEY, this.grabbedThingID, DEFAULT_GRABBABLE_DATA);
+                this.ignoreIK = (grabbableData.ignoreIK !== undefined) ? grabbableData.ignoreIK : true;
+            }
         }
 
         var handRotation;
@@ -2320,7 +2385,8 @@ function MyController(hand) {
             this.offsetPosition = Vec3.multiplyQbyV(Quat.inverse(Quat.multiply(handRotation, this.offsetRotation)), offset);
         }
 
-        var isPhysical = propsArePhysical(grabbedProperties) || entityHasActions(this.grabbedEntity);
+        var isPhysical = propsArePhysical(grabbedProperties) ||
+            (!this.grabbedIsOverlay && entityHasActions(this.grabbedThingID));
         if (isPhysical && this.state == STATE_NEAR_GRABBING && grabbedProperties.parentID === NULL_UUID) {
             // grab entity via action
             if (!this.setupHoldAction()) {
@@ -2328,7 +2394,7 @@ function MyController(hand) {
             }
             Messages.sendMessage('Hifi-Object-Manipulation', JSON.stringify({
                 action: 'grab',
-                grabbedEntity: this.grabbedEntity,
+                grabbedEntity: this.grabbedThingID,
                 joint: this.hand === RIGHT_HAND ? "RightHand" : "LeftHand"
             }));
         } else {
@@ -2353,29 +2419,36 @@ function MyController(hand) {
                 reparentProps.localPosition = this.offsetPosition;
                 reparentProps.localRotation = this.offsetRotation;
             }
-            Entities.editEntity(this.grabbedEntity, reparentProps);
+
+            if (this.grabbedIsOverlay) {
+                Overlays.editOverlay(this.grabbedThingID, reparentProps);
+            } else {
+                Entities.editEntity(this.grabbedThingID, reparentProps);
+            }
 
             if (this.thisHandIsParent(grabbedProperties)) {
                 // this should never happen, but if it does, don't set previous parent to be this hand.
-                // this.previousParentID[this.grabbedEntity] = NULL;
-                // this.previousParentJointIndex[this.grabbedEntity] = -1;
+                // this.previousParentID[this.grabbedThingID] = NULL;
+                // this.previousParentJointIndex[this.grabbedThingID] = -1;
             } else {
-                this.previousParentID[this.grabbedEntity] = grabbedProperties.parentID;
-                this.previousParentJointIndex[this.grabbedEntity] = grabbedProperties.parentJointIndex;
+                this.previousParentID[this.grabbedThingID] = grabbedProperties.parentID;
+                this.previousParentJointIndex[this.grabbedThingID] = grabbedProperties.parentJointIndex;
             }
 
             Messages.sendMessage('Hifi-Object-Manipulation', JSON.stringify({
                 action: 'equip',
-                grabbedEntity: this.grabbedEntity,
+                grabbedEntity: this.grabbedThingID,
                 joint: this.hand === RIGHT_HAND ? "RightHand" : "LeftHand"
             }));
         }
 
-        Entities.editEntity(this.grabbedEntity, {
-            velocity: { x: 0, y: 0, z: 0 },
-            angularVelocity: { x: 0, y: 0, z: 0 },
-            // dynamic: false
-        });
+        if (!this.grabbedIsOverlay) {
+            Entities.editEntity(this.grabbedThingID, {
+                velocity: { x: 0, y: 0, z: 0 },
+                angularVelocity: { x: 0, y: 0, z: 0 },
+                // dynamic: false
+            });
+        }
 
         if (this.state == STATE_NEAR_GRABBING) {
             this.callEntityMethodOnGrabbed("startNearGrab");
@@ -2441,26 +2514,39 @@ function MyController(hand) {
 
             if (dropDetected && !this.waitForTriggerRelease && this.triggerSmoothedGrab()) {
                 // store the offset attach points into preferences.
-                if (USE_ATTACH_POINT_SETTINGS && this.grabbedHotspot && this.grabbedEntity) {
-                    var prefprops = Entities.getEntityProperties(this.grabbedEntity, ["localPosition", "localRotation"]);
+                if (USE_ATTACH_POINT_SETTINGS && this.grabbedHotspot && this.grabbedThingID) {
+                    var prefprops = Entities.getEntityProperties(this.grabbedThingID, ["localPosition", "localRotation"]);
                     if (prefprops && prefprops.localPosition && prefprops.localRotation) {
                         storeAttachPointForHotspotInSettings(this.grabbedHotspot, this.hand,
                                                              prefprops.localPosition, prefprops.localRotation);
                     }
                 }
 
-                var grabbedEntity = this.grabbedEntity;
+                var grabbedEntity = this.grabbedThingID;
                 this.release();
-                this.grabbedEntity = grabbedEntity;
+                this.grabbedThingID = grabbedEntity;
                 this.setState(STATE_NEAR_GRABBING, "drop gesture detected");
                 return;
             }
             this.prevDropDetected = dropDetected;
         }
 
-        var props = Entities.getEntityProperties(this.grabbedEntity, ["localPosition", "parentID", "parentJointIndex",
+        var props;
+        if (this.grabbedIsOverlay) {
+            props = {
+                localPosition: Overlays.getProperty(this.grabbedThingID, "localPosition"),
+                parentID: Overlays.getProperty(this.grabbedThingID, "parentID"),
+                parentJointIndex: Overlays.getProperty(this.grabbedThingID, "parentJointIndex"),
+                position: Overlays.getProperty(this.grabbedThingID, "position"),
+                rotation: Overlays.getProperty(this.grabbedThingID, "rotation"),
+                dimensions: Overlays.getProperty(this.grabbedThingID, "dimensions"),
+                registrationPoint: { x: 0.5, y: 0.5, z: 0.5 }
+            };
+        } else {
+            props = Entities.getEntityProperties(this.grabbedThingID, ["localPosition", "parentID", "parentJointIndex",
                                                                       "position", "rotation", "dimensions",
                                                                       "registrationPoint"]);
+        }
         if (!props.position) {
             // server may have reset, taking our equipped entity with it.  move back to "off" state
             this.callEntityMethodOnGrabbed("releaseGrab");
@@ -2472,7 +2558,7 @@ function MyController(hand) {
             // someone took it from us or otherwise edited the parentID.  end the grab.  We don't do this
             // for equipped things so that they can be adjusted while equipped.
             this.callEntityMethodOnGrabbed("releaseGrab");
-            this.grabbedEntity = null;
+            this.grabbedThingID = null;
             this.setState(STATE_OFF, "someone took it");
             return;
         }
@@ -2554,7 +2640,7 @@ function MyController(hand) {
 
         if (this.actionID && this.actionTimeout - now < ACTION_TTL_REFRESH * MSECS_PER_SEC) {
             // if less than a 5 seconds left, refresh the actions ttl
-            var success = Entities.updateAction(this.grabbedEntity, this.actionID, {
+            var success = Entities.updateAction(this.grabbedThingID, this.actionID, {
                 hand: this.hand === RIGHT_HAND ? "right" : "left",
                 timeScale: NEAR_GRABBING_ACTION_TIMEFRAME,
                 relativePosition: this.offsetPosition,
@@ -2568,14 +2654,14 @@ function MyController(hand) {
                 this.actionTimeout = now + (ACTION_TTL * MSECS_PER_SEC);
             } else {
                 print("continueNearGrabbing -- updateAction failed");
-                Entities.deleteAction(this.grabbedEntity, this.actionID);
+                Entities.deleteAction(this.grabbedThingID, this.actionID);
                 this.setupHoldAction();
             }
         }
     };
 
     this.maybeScale = function(props) {
-        if (!objectScalingEnabled || this.isTablet(this.grabbedEntity)) {
+        if (!objectScalingEnabled || this.isTablet(this.grabbedThingID) || this.grabbedIsOverlay) {
             return;
         }
 
@@ -2597,7 +2683,7 @@ function MyController(hand) {
                                                                    this.getOtherHandController().getHandPosition()));
             var currentRescale = scalingCurrentDistance / this.scalingStartDistance;
             var newDimensions = Vec3.multiply(currentRescale, this.scalingStartDimensions);
-            Entities.editEntity(this.grabbedEntity, { dimensions: newDimensions });
+            Entities.editEntity(this.grabbedThingID, { dimensions: newDimensions });
         }
     };
 
@@ -2648,7 +2734,7 @@ function MyController(hand) {
     this.nearTrigger = function(deltaTime, timestamp) {
         if (this.triggerSmoothedReleased()) {
             this.callEntityMethodOnGrabbed("stopNearTrigger");
-            this.grabbedEntity = null;
+            this.grabbedThingID = null;
             this.setState(STATE_OFF, "trigger released");
             return;
         }
@@ -2658,7 +2744,7 @@ function MyController(hand) {
     this.farTrigger = function(deltaTime, timestamp) {
         if (this.triggerSmoothedReleased()) {
             this.callEntityMethodOnGrabbed("stopFarTrigger");
-            this.grabbedEntity = null;
+            this.grabbedThingID = null;
             this.setState(STATE_OFF, "trigger released");
             return;
         }
@@ -2673,9 +2759,9 @@ function MyController(hand) {
             var intersection = findRayIntersection(pickRay, true, [], [], true);
             if (intersection.accurate || intersection.overlayID) {
                 this.lastPickTime = now;
-                if (intersection.entityID != this.grabbedEntity) {
+                if (intersection.entityID != this.grabbedThingID) {
                     this.callEntityMethodOnGrabbed("stopFarTrigger");
-                    this.grabbedEntity = null;
+                    this.grabbedThingID = null;
                     this.setState(STATE_OFF, "laser moved off of entity");
                     return;
                 }
@@ -2697,13 +2783,13 @@ function MyController(hand) {
 
     this.entityTouchingEnter = function() {
         // test for intersection between controller laser and web entity plane.
-        var intersectInfo = handLaserIntersectEntity(this.grabbedEntity,
+        var intersectInfo = handLaserIntersectEntity(this.grabbedThingID,
                                                      getControllerWorldLocation(this.handToController(), true));
         if (intersectInfo) {
             var pointerEvent = {
                 type: "Press",
                 id: this.hand + 1, // 0 is reserved for hardware mouse
-                pos2D: projectOntoEntityXYPlane(this.grabbedEntity, intersectInfo.point),
+                pos2D: projectOntoEntityXYPlane(this.grabbedThingID, intersectInfo.point),
                 pos3D: intersectInfo.point,
                 normal: intersectInfo.normal,
                 direction: intersectInfo.searchRay.direction,
@@ -2711,8 +2797,8 @@ function MyController(hand) {
                 isPrimaryHeld: true
             };
 
-            Entities.sendMousePressOnEntity(this.grabbedEntity, pointerEvent);
-            Entities.sendClickDownOnEntity(this.grabbedEntity, pointerEvent);
+            Entities.sendMousePressOnEntity(this.grabbedThingID, pointerEvent);
+            Entities.sendClickDownOnEntity(this.grabbedThingID, pointerEvent);
 
             this.touchingEnterTimer = 0;
             this.touchingEnterPointerEvent = pointerEvent;
@@ -2734,7 +2820,7 @@ function MyController(hand) {
 
     this.entityTouchingExit = function() {
         // test for intersection between controller laser and web entity plane.
-        var intersectInfo = handLaserIntersectEntity(this.grabbedEntity,
+        var intersectInfo = handLaserIntersectEntity(this.grabbedThingID,
                                                      getControllerWorldLocation(this.handToController(), true));
         if (intersectInfo) {
             var pointerEvent;
@@ -2742,7 +2828,7 @@ function MyController(hand) {
                 pointerEvent = {
                     type: "Release",
                     id: this.hand + 1, // 0 is reserved for hardware mouse
-                    pos2D: projectOntoEntityXYPlane(this.grabbedEntity, intersectInfo.point),
+                    pos2D: projectOntoEntityXYPlane(this.grabbedThingID, intersectInfo.point),
                     pos3D: intersectInfo.point,
                     normal: intersectInfo.normal,
                     direction: intersectInfo.searchRay.direction,
@@ -2755,11 +2841,11 @@ function MyController(hand) {
                 pointerEvent.isPrimaryHeld = false;
             }
 
-            Entities.sendMouseReleaseOnEntity(this.grabbedEntity, pointerEvent);
-            Entities.sendClickReleaseOnEntity(this.grabbedEntity, pointerEvent);
-            Entities.sendHoverLeaveEntity(this.grabbedEntity, pointerEvent);
+            Entities.sendMouseReleaseOnEntity(this.grabbedThingID, pointerEvent);
+            Entities.sendClickReleaseOnEntity(this.grabbedThingID, pointerEvent);
+            Entities.sendHoverLeaveEntity(this.grabbedThingID, pointerEvent);
         }
-        this.grabbedEntity = null;
+        this.grabbedThingID = null;
         this.grabbedOverlay = null;
     };
 
@@ -2767,7 +2853,7 @@ function MyController(hand) {
 
         this.touchingEnterTimer += dt;
 
-        entityPropertiesCache.addEntity(this.grabbedEntity);
+        entityPropertiesCache.addEntity(this.grabbedThingID);
 
         if (this.state == STATE_ENTITY_LASER_TOUCHING && !this.triggerSmoothedGrab()) {
             this.setState(STATE_OFF, "released trigger");
@@ -2775,7 +2861,7 @@ function MyController(hand) {
         }
 
         // test for intersection between controller laser and web entity plane.
-        var intersectInfo = handLaserIntersectEntity(this.grabbedEntity,
+        var intersectInfo = handLaserIntersectEntity(this.grabbedThingID,
                                                      getControllerWorldLocation(this.handToController(), true));
         if (intersectInfo) {
 
@@ -2785,15 +2871,15 @@ function MyController(hand) {
                 return;
             }
 
-            if (Entities.keyboardFocusEntity != this.grabbedEntity) {
+            if (Entities.keyboardFocusEntity != this.grabbedThingID) {
                 Overlays.keyboardFocusOverlay = 0;
-                Entities.keyboardFocusEntity = this.grabbedEntity;
+                Entities.keyboardFocusEntity = this.grabbedThingID;
             }
 
             var pointerEvent = {
                 type: "Move",
                 id: this.hand + 1, // 0 is reserved for hardware mouse
-                pos2D: projectOntoEntityXYPlane(this.grabbedEntity, intersectInfo.point),
+                pos2D: projectOntoEntityXYPlane(this.grabbedThingID, intersectInfo.point),
                 pos3D: intersectInfo.point,
                 normal: intersectInfo.normal,
                 direction: intersectInfo.searchRay.direction,
@@ -2804,8 +2890,8 @@ function MyController(hand) {
             var POINTER_PRESS_TO_MOVE_DELAY = 0.25; // seconds
             if (this.deadspotExpired || this.touchingEnterTimer > POINTER_PRESS_TO_MOVE_DELAY ||
                 Vec3.distance(intersectInfo.point, this.touchingEnterPointerEvent.pos3D) > this.deadspotRadius) {
-                Entities.sendMouseMoveOnEntity(this.grabbedEntity, pointerEvent);
-                Entities.sendHoldingClickOnEntity(this.grabbedEntity, pointerEvent);
+                Entities.sendMouseMoveOnEntity(this.grabbedThingID, pointerEvent);
+                Entities.sendHoldingClickOnEntity(this.grabbedThingID, pointerEvent);
                 this.deadspotExpired = true;
             }
 
@@ -2815,7 +2901,7 @@ function MyController(hand) {
             }
             Reticle.setVisible(false);
         } else {
-            this.grabbedEntity = null;
+            this.grabbedThingID = null;
             this.setState(STATE_OFF, "grabbed entity was destroyed");
             return;
         }
@@ -2900,7 +2986,7 @@ function MyController(hand) {
             Overlays.sendMouseReleaseOnOverlay(this.grabbedOverlay, pointerEvent);
             Overlays.sendHoverLeaveOverlay(this.grabbedOverlay, pointerEvent);
         }
-        this.grabbedEntity = null;
+        this.grabbedThingID = null;
         this.grabbedOverlay = null;
     };
 
@@ -2923,7 +3009,7 @@ function MyController(hand) {
 
             if (this.state == STATE_OVERLAY_STYLUS_TOUCHING &&
                 intersectInfo.distance > WEB_STYLUS_LENGTH / 2.0 + WEB_TOUCH_Y_OFFSET + WEB_TOUCH_Y_TOUCH_DEADZONE_SIZE) {
-                this.grabbedEntity = null;
+                this.grabbedThingID = null;
                 this.setState(STATE_OFF, "pulled away from overlay");
                 return;
             }
@@ -2980,7 +3066,7 @@ function MyController(hand) {
             }
             Reticle.setVisible(false);
         } else {
-            this.grabbedEntity = null;
+            this.grabbedThingID = null;
             this.setState(STATE_OFF, "grabbed overlay was destroyed");
             return;
         }
@@ -2989,7 +3075,7 @@ function MyController(hand) {
     this.release = function() {
         this.turnOffVisualizations();
 
-        if (this.grabbedEntity !== null) {
+        if (this.grabbedThingID !== null) {
             if (this.state === STATE_HOLD) {
                 this.callEntityMethodOnGrabbed("releaseEquip");
             }
@@ -2997,35 +3083,49 @@ function MyController(hand) {
             // Make a small release haptic pulse if we really were holding something
             Controller.triggerHapticPulse(HAPTIC_PULSE_STRENGTH, HAPTIC_PULSE_DURATION, this.hand);
             if (this.actionID !== null) {
-                Entities.deleteAction(this.grabbedEntity, this.actionID);
+                Entities.deleteAction(this.grabbedThingID, this.actionID);
             } else {
                 // no action, so it's a parenting grab
-                if (this.previousParentID[this.grabbedEntity] === NULL_UUID) {
-                    Entities.editEntity(this.grabbedEntity, {
-                        parentID: this.previousParentID[this.grabbedEntity],
-                        parentJointIndex: this.previousParentJointIndex[this.grabbedEntity]
-                    });
-                    this.ensureDynamic();
+                if (this.previousParentID[this.grabbedThingID] === NULL_UUID) {
+                    if (this.grabbedIsOverlay) {
+                        Overlays.editOverlay(this.grabbedThingID, {
+                            parentID: NULL_UUID,
+                            parentJointIndex: -1
+                        });
+                    } else {
+                        Entities.editEntity(this.grabbedThingID, {
+                            parentID: this.previousParentID[this.grabbedThingID],
+                            parentJointIndex: this.previousParentJointIndex[this.grabbedThingID]
+                        });
+                        this.ensureDynamic();
+                    }
                 } else {
-                    // we're putting this back as a child of some other parent, so zero its velocity
-                    Entities.editEntity(this.grabbedEntity, {
-                        parentID: this.previousParentID[this.grabbedEntity],
-                        parentJointIndex: this.previousParentJointIndex[this.grabbedEntity],
-                        velocity: {x: 0, y: 0, z: 0},
-                        angularVelocity: {x: 0, y: 0, z: 0}
-                    });
+                    if (this.grabbedIsOverlay) {
+                        Overlays.editOverlay(this.grabbedThingID, {
+                            parentID: this.previousParentID[this.grabbedThingID],
+                            parentJointIndex: this.previousParentJointIndex[this.grabbedThingID],
+                        });
+                    } else {
+                        // we're putting this back as a child of some other parent, so zero its velocity
+                        Entities.editEntity(this.grabbedThingID, {
+                            parentID: this.previousParentID[this.grabbedThingID],
+                            parentJointIndex: this.previousParentJointIndex[this.grabbedThingID],
+                            velocity: {x: 0, y: 0, z: 0},
+                            angularVelocity: {x: 0, y: 0, z: 0}
+                        });
+                    }
                 }
             }
 
             Messages.sendMessage('Hifi-Object-Manipulation', JSON.stringify({
                 action: 'release',
-                grabbedEntity: this.grabbedEntity,
+                grabbedEntity: this.grabbedThingID,
                 joint: this.hand === RIGHT_HAND ? "RightHand" : "LeftHand"
             }));
         }
 
         this.actionID = null;
-        this.grabbedEntity = null;
+        this.grabbedThingID = null;
         this.grabbedOverlay = null;
         this.grabbedHotspot = null;
 
@@ -3113,9 +3213,13 @@ function MyController(hand) {
                     }
                     _this.previouslyUnhooked[childID] = now;
 
+                    // we don't know if it's an entity or an overlay
                     Entities.editEntity(childID, { parentID: previousParentID, parentJointIndex: previousParentJointIndex });
+                    Overlays.editOverlay(childID, { parentID: previousParentID, parentJointIndex: previousParentJointIndex });
+
                 } else {
                     Entities.editEntity(childID, { parentID: NULL_UUID });
+                    Overlays.editOverlay(childID, { parentID: NULL_UUID });
                 }
             }
         });
@@ -3234,8 +3338,8 @@ var handleHandMessages = function(channel, message, sender) {
                 selectedController.release();
                 var wearableEntity = data.entityID;
                 entityPropertiesCache.addEntity(wearableEntity);
-                selectedController.grabbedEntity = wearableEntity;
-                var hotspots = selectedController.collectEquipHotspots(selectedController.grabbedEntity);
+                selectedController.grabbedThingID = wearableEntity;
+                var hotspots = selectedController.collectEquipHotspots(selectedController.grabbedThingID);
                 if (hotspots.length > 0) {
                     if (hotspotIndex >= hotspots.length) {
                         hotspotIndex = 0;

scripts/system/controllers/squeezeHands.js

@@ -25,10 +25,13 @@ var OVERLAY_RAMP_RATE = 8.0;
 
 var animStateHandlerID;
 
-var isPointingIndex = false;
+var isBothIndexesPointing = false;
 var HIFI_POINT_INDEX_MESSAGE_CHANNEL = "Hifi-Point-Index";
 
-var indexfingerJointNames = ["LeftHandIndex1", "LeftHandIndex2", "LeftHandIndex3", "RightHandIndex1", "RightHandIndex2", "RightHandIndex3"];
+var isLeftIndexPointing = false;
+var isRightIndexPointing = false;
+var isLeftThumbRaised = false;
+var isRightThumbRaised = false;
 
 function clamp(val, min, max) {
     return Math.min(Math.max(val, min), max);
@@ -46,17 +49,32 @@ function init() {
     Script.update.connect(update);
     animStateHandlerID = MyAvatar.addAnimationStateHandler(
         animStateHandler,
-        ["leftHandOverlayAlpha", "rightHandOverlayAlpha", "leftHandGraspAlpha", "rightHandGraspAlpha"]
+        [
+            "leftHandOverlayAlpha", "leftHandGraspAlpha",
+            "rightHandOverlayAlpha", "rightHandGraspAlpha",
+            "isLeftHandGrasp", "isLeftIndexPoint", "isLeftThumbRaise", "isLeftIndexPointAndThumbRaise",
+            "isRightHandGrasp", "isRightIndexPoint", "isRightThumbRaise", "isRightIndexPointAndThumbRaise",
+        ]
     );
     Messages.subscribe(HIFI_POINT_INDEX_MESSAGE_CHANNEL);
     Messages.messageReceived.connect(handleMessages);
 }
 
 function animStateHandler(props) {
-    return { leftHandOverlayAlpha: leftHandOverlayAlpha,
-             leftHandGraspAlpha: lastLeftTrigger,
-             rightHandOverlayAlpha: rightHandOverlayAlpha,
-             rightHandGraspAlpha: lastRightTrigger };
+    return {
+        leftHandOverlayAlpha: leftHandOverlayAlpha,
+        leftHandGraspAlpha: lastLeftTrigger,
+        rightHandOverlayAlpha: rightHandOverlayAlpha,
+        rightHandGraspAlpha: lastRightTrigger,
+        isLeftHandGrasp: !isBothIndexesPointing && !isLeftIndexPointing && !isLeftThumbRaised,
+        isLeftIndexPoint: (isBothIndexesPointing || isLeftIndexPointing) && !isLeftThumbRaised,
+        isLeftThumbRaise: !isBothIndexesPointing && !isLeftIndexPointing && isLeftThumbRaised,
+        isLeftIndexPointAndThumbRaise: (isBothIndexesPointing || isLeftIndexPointing) && isLeftThumbRaised,
+        isRightHandGrasp: !isBothIndexesPointing && !isRightIndexPointing && !isRightThumbRaised,
+        isRightIndexPoint: (isBothIndexesPointing || isRightIndexPointing) && !isRightThumbRaised,
+        isRightThumbRaise: !isBothIndexesPointing && !isRightIndexPointing && isRightThumbRaised,
+        isRightIndexPointAndThumbRaise: (isBothIndexesPointing || isRightIndexPointing) && isRightThumbRaised
+    };
 }
 
 function update(dt) {
@@ -84,13 +102,11 @@ function update(dt) {
         rightHandOverlayAlpha = clamp(rightHandOverlayAlpha - OVERLAY_RAMP_RATE * dt, 0, 1);
     }
 
-    // Point index finger.
-    if (isPointingIndex) {
-        var zeroRotation = { x: 0, y: 0, z: 0, w: 1 };
-        for (var i = 0; i < indexfingerJointNames.length; i++) {
-            MyAvatar.setJointRotation(indexfingerJointNames[i], zeroRotation);
-        }
-    }
+    // Pointing index fingers and raising thumbs
+    isLeftIndexPointing = leftHandPose.valid && Controller.getValue(Controller.Standard.LeftIndexPoint) === 1;
+    isRightIndexPointing = rightHandPose.valid && Controller.getValue(Controller.Standard.RightIndexPoint) === 1;
+    isLeftThumbRaised = leftHandPose.valid && Controller.getValue(Controller.Standard.LeftThumbUp) === 1;
+    isRightThumbRaised = rightHandPose.valid && Controller.getValue(Controller.Standard.RightThumbUp) === 1;
 }
 
 function handleMessages(channel, message, sender) {
@@ -98,13 +114,7 @@ function handleMessages(channel, message, sender) {
         var data = JSON.parse(message);
         if (data.pointIndex !== undefined) {
             print("pointIndex: " + data.pointIndex);
-            isPointingIndex = data.pointIndex;
-
-            if (!isPointingIndex) {
-                for (var i = 0; i < indexfingerJointNames.length; i++) {
-                    MyAvatar.clearJointData(indexfingerJointNames[i]);
-                }
-            }
+            isBothIndexesPointing = data.pointIndex;
         }
     }
 }

scripts/system/edit.js

@@ -564,6 +564,11 @@ function findClickedEntity(event) {
 
     var pickRay = Camera.computePickRay(event.x, event.y);
 
+    var overlayResult = Overlays.findRayIntersection(pickRay, true, [HMD.tabletID, HMD.tabletScreenID, HMD.homeButtonID]);
+    if (overlayResult.intersects) {
+        return null;
+    }
+
     var entityResult = Entities.findRayIntersection(pickRay, true); // want precision picking
     var lightResult = lightOverlayManager.findRayIntersection(pickRay);
     lightResult.accurate = true;

scripts/system/libraries/WebTablet.js

@@ -7,8 +7,8 @@
 //  Distributed under the Apache License, Version 2.0.
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
-/* global getControllerWorldLocation, setEntityCustomData, Tablet, WebTablet:true, HMD, Settings, Script,
-   Vec3, Quat, MyAvatar, Entities, Overlays, Camera, Messages, Xform, clamp */
+/* global getControllerWorldLocation, Tablet, WebTablet:true, HMD, Settings, Script,
+   Vec3, Quat, MyAvatar, Entities, Overlays, Camera, Messages, Xform, clamp, Controller, Mat4 */
 
 Script.include(Script.resolvePath("../libraries/utils.js"));
 Script.include(Script.resolvePath("../libraries/controllers.js"));
@@ -34,7 +34,7 @@ var TABLET_NATURAL_DIMENSIONS = {x: 33.797, y: 50.129, z: 2.269};
 var HOME_BUTTON_TEXTURE = "http://hifi-content.s3.amazonaws.com/alan/dev/tablet-with-home-button.fbx/tablet-with-home-button.fbm/button-close.png";
 // var HOME_BUTTON_TEXTURE = Script.resourcesPath() + "meshes/tablet-with-home-button.fbx/tablet-with-home-button.fbm/button-close.png";
 var TABLET_MODEL_PATH = "http://hifi-content.s3.amazonaws.com/alan/dev/tablet-with-home-button.fbx";
-// var TABLET_MODEL_PATH = Script.resourcesPath() + "meshes/tablet-with-home-button.fbx";
+var LOCAL_TABLET_MODEL_PATH = Script.resourcesPath() + "meshes/tablet-with-home-button.fbx";
 
 // returns object with two fields:
 //    * position - position in front of the user
@@ -49,24 +49,30 @@ function calcSpawnInfo(hand, height) {
         var handController = getControllerWorldLocation(hand, true);
         var controllerPosition = handController.position;
 
-        // compute the angle of the chord with length (height / 2)
-        var theta = Math.asin(height / (2 * Vec3.distance(headPos, controllerPosition)));
+        // base of the tablet is slightly above controller position
+        var TABLET_BASE_DISPLACEMENT = {x: 0, y: 0.1, z: 0};
+        var tabletBase = Vec3.sum(controllerPosition, TABLET_BASE_DISPLACEMENT);
 
-        // then we can use this angle to rotate the vector between the HMD position and the center of the tablet.
-        // this vector, u, will become our new look at direction.
-        var d = Vec3.normalize(Vec3.subtract(headPos, controllerPosition));
+        var d = Vec3.subtract(headPos, tabletBase);
+        var theta = Math.acos(d.y / Vec3.length(d));
+        d.y = 0;
+        if (Vec3.length(d) < 0.0001) {
+            d = {x: 1, y: 0, z: 0};
+        } else {
+            d = Vec3.normalize(d);
+        }
         var w = Vec3.normalize(Vec3.cross(Y_AXIS, d));
-        var q = Quat.angleAxis(theta * (180 / Math.PI), w);
+        var ANGLE_OFFSET = 25;
+        var q = Quat.angleAxis(theta * (180 / Math.PI) - (90 - ANGLE_OFFSET), w);
         var u = Vec3.multiplyQbyV(q, d);
 
         // use u to compute a full lookAt quaternion.
-        var lookAtRot = Quat.lookAt(controllerPosition, Vec3.sum(controllerPosition, u), Y_AXIS);
-
-        // adjust the tablet position by a small amount.
-        var yDisplacement = (height / 2) + 0.1;
+        var lookAtRot = Quat.lookAt(tabletBase, Vec3.sum(tabletBase, u), Y_AXIS);
+        var yDisplacement = (height / 2);
         var zDisplacement = 0.05;
         var tabletOffset = Vec3.multiplyQbyV(lookAtRot, {x: 0, y: yDisplacement, z: zDisplacement});
-        finalPosition = Vec3.sum(controllerPosition, tabletOffset);
+        finalPosition = Vec3.sum(tabletBase, tabletOffset);
+
         return {
             position: finalPosition,
             rotation: lookAtRot
@@ -106,10 +112,19 @@ WebTablet = function (url, width, dpi, hand, clientOnly) {
         this.dpi = DEFAULT_DPI * (DEFAULT_WIDTH / this.width);
     }
 
+    var modelURL;
+    if (Settings.getValue("tabletVisibleToOthers")) {
+        modelURL = TABLET_MODEL_PATH;
+    } else {
+        modelURL = LOCAL_TABLET_MODEL_PATH;
+    }
+
     var tabletProperties = {
         name: "WebTablet Tablet",
         type: "Model",
-        modelURL: TABLET_MODEL_PATH,
+        modelURL: modelURL,
+        url: modelURL, // for overlay
+        grabbable: true, // for overlay
         userData: JSON.stringify({
             "grabbableKey": {"grabbable": true}
         }),
@@ -121,7 +136,14 @@ WebTablet = function (url, width, dpi, hand, clientOnly) {
     this.calculateTabletAttachmentProperties(hand, true, tabletProperties);
 
     this.cleanUpOldTablets();
-    this.tabletEntityID = Entities.addEntity(tabletProperties, clientOnly);
+
+    if (Settings.getValue("tabletVisibleToOthers")) {
+        this.tabletEntityID = Entities.addEntity(tabletProperties, clientOnly);
+        this.tabletIsOverlay = false;
+    } else {
+        this.tabletEntityID = Overlays.addOverlay("model", tabletProperties);
+        this.tabletIsOverlay = true;
+    }
 
     if (this.webOverlayID) {
         Overlays.deleteOverlay(this.webOverlayID);
@@ -145,12 +167,12 @@ WebTablet = function (url, width, dpi, hand, clientOnly) {
         isAA: HMD.active
     });
 
-    var HOME_BUTTON_Y_OFFSET = (this.height / 2) - 0.035;
-    this.homeButtonEntity = Overlays.addOverlay("sphere", {
+    var HOME_BUTTON_Y_OFFSET = (this.height / 2) - 0.009;
+    this.homeButtonID = Overlays.addOverlay("sphere", {
         name: "homeButton",
-        localPosition: {x: 0.0, y: -HOME_BUTTON_Y_OFFSET, z: -0.01},
+        localPosition: {x: -0.001, y: -HOME_BUTTON_Y_OFFSET, z: 0.0},
         localRotation: Quat.angleAxis(0, Y_AXIS),
-        dimensions: { x: 0.04, y: 0.04, z: 0.02},
+        dimensions: { x: 4 * tabletScaleFactor, y: 4 * tabletScaleFactor, z: 4 * tabletScaleFactor},
         alpha: 0.0,
         visible: true,
         drawInFront: false,
@@ -159,7 +181,7 @@ WebTablet = function (url, width, dpi, hand, clientOnly) {
     });
 
     this.receive = function (channel, senderID, senderUUID, localOnly) {
-        if (_this.homeButtonEntity == senderID) {
+        if (_this.homeButtonID == senderID) {
             var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
             var onHomeScreen = tablet.onHomeScreen();
             if (onHomeScreen) {
@@ -178,7 +200,16 @@ WebTablet = function (url, width, dpi, hand, clientOnly) {
     };
 
     this.getLocation = function() {
-        return Entities.getEntityProperties(_this.tabletEntityID, ["localPosition", "localRotation"]);
+        if (this.tabletIsOverlay) {
+            var location = Overlays.getProperty(this.tabletEntityID, "localPosition");
+            var orientation = Overlays.getProperty(this.tabletEntityID, "localOrientation");
+            return {
+                localPosition: location,
+                localRotation: orientation
+            };
+        } else {
+            return Entities.getEntityProperties(_this.tabletEntityID, ["localPosition", "localRotation"]);
+        }
     };
     this.clicked = false;
 
@@ -236,8 +267,12 @@ WebTablet.prototype.getOverlayObject = function () {
 
 WebTablet.prototype.destroy = function () {
     Overlays.deleteOverlay(this.webOverlayID);
-    Entities.deleteEntity(this.tabletEntityID);
-    Overlays.deleteOverlay(this.homeButtonEntity);
+    if (this.tabletIsOverlay) {
+        Overlays.deleteOverlay(this.tabletEntityID);
+    } else {
+        Entities.deleteEntity(this.tabletEntityID);
+    }
+    Overlays.deleteOverlay(this.homeButtonID);
     HMD.displayModeChanged.disconnect(this.myOnHmdChanged);
 
     Controller.mousePressEvent.disconnect(this.myMousePressEvent);
@@ -420,10 +455,16 @@ WebTablet.prototype.getPosition = function () {
 
 WebTablet.prototype.mousePressEvent = function (event) {
     var pickRay = Camera.computePickRay(event.x, event.y);
-    var entityPickResults = Entities.findRayIntersection(pickRay, true, [this.tabletEntityID]); // non-accurate picking
-    if (entityPickResults.intersects && entityPickResults.entityID === this.tabletEntityID) {
-        var overlayPickResults = Overlays.findRayIntersection(pickRay);
-        if (overlayPickResults.intersects && overlayPickResults.overlayID === HMD.homeButtonID) {
+    var entityPickResults;
+    if (this.tabletIsOverlay) {
+        entityPickResults = Overlays.findRayIntersection(pickRay, true, [this.tabletEntityID]);
+    } else {
+        entityPickResults = Entities.findRayIntersection(pickRay, true, [this.tabletEntityID]);
+    }
+    if (entityPickResults.intersects && (entityPickResults.entityID === this.tabletEntityID ||
+                                         entityPickResults.overlayID === this.tabletEntityID)) {
+        var overlayPickResults = Overlays.findRayIntersection(pickRay, true, [this.webOverlayID, this.homeButtonID], []);
+        if (overlayPickResults.intersects && overlayPickResults.overlayID === this.homeButtonID) {
             var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
             var onHomeScreen = tablet.onHomeScreen();
             if (onHomeScreen) {
@@ -432,11 +473,15 @@ WebTablet.prototype.mousePressEvent = function (event) {
                 tablet.gotoHomeScreen();
                 this.setHomeButtonTexture();
             }
-        } else if (!HMD.active && (!overlayPickResults.intersects || !overlayPickResults.overlayID === this.webOverlayID)) {
+        } else if (!HMD.active && (!overlayPickResults.intersects || overlayPickResults.overlayID !== this.webOverlayID)) {
             this.dragging = true;
             var invCameraXform = new Xform(Camera.orientation, Camera.position).inv();
             this.initialLocalIntersectionPoint = invCameraXform.xformPoint(entityPickResults.intersection);
-            this.initialLocalPosition = Entities.getEntityProperties(this.tabletEntityID, ["localPosition"]).localPosition;
+            if (this.tabletIsOverlay) {
+                this.initialLocalPosition = Overlays.getProperty(this.tabletEntityID, "localPosition");
+            } else {
+                this.initialLocalPosition = Entities.getEntityProperties(this.tabletEntityID, ["localPosition"]).localPosition;
+            }
         }
     }
 };
@@ -482,9 +527,15 @@ WebTablet.prototype.mouseMoveEvent = function (event) {
             var localIntersectionPoint = Vec3.sum(localPickRay.origin, Vec3.multiply(localPickRay.direction, result.distance));
             var localOffset = Vec3.subtract(localIntersectionPoint, this.initialLocalIntersectionPoint);
             var localPosition = Vec3.sum(this.initialLocalPosition, localOffset);
-            Entities.editEntity(this.tabletEntityID, {
-                localPosition: localPosition
-            });
+            if (this.tabletIsOverlay) {
+                Overlays.editOverlay(this.tabletEntityID, {
+                    localPosition: localPosition
+                });
+            } else {
+                Entities.editEntity(this.tabletEntityID, {
+                    localPosition: localPosition
+                });
+            }
         }
     }
 };

scripts/system/libraries/entitySelectionTool.js

@@ -3866,6 +3866,12 @@ SelectionDisplay = (function() {
         var somethingClicked = false;
         var pickRay = generalComputePickRay(event.x, event.y);
 
+        var result = Overlays.findRayIntersection(pickRay, true, [HMD.tabletID, HMD.tabletScreenID, HMD.homeButtonID]);
+        if (result.intersects) {
+            // mouse clicks on the tablet should override the edit affordances
+            return false;
+        }
+
         // before we do a ray test for grabbers, disable the ray intersection for our selection box
         Overlays.editOverlay(selectionBox, {
             ignoreRayIntersection: true

scripts/system/marketplaces/marketplaces.js

@@ -52,10 +52,16 @@ function onMessageBoxClosed(id, button) {
 
 Window.messageBoxClosed.connect(onMessageBoxClosed);
 
+var shouldActivateButton = false;
+var onMarketplaceScreen = false;
+
 function showMarketplace() {
     UserActivityLogger.openedMarketplace();
 
+    shouldActivateButton = true;
     tablet.gotoWebScreen(MARKETPLACE_URL_INITIAL, MARKETPLACES_INJECT_SCRIPT_URL);
+    onMarketplaceScreen = true;
+
     tablet.webEventReceived.connect(function (message) {
 
         if (message === GOTO_DIRECTORY) {
@@ -98,15 +104,10 @@ function showMarketplace() {
     });
 }
 
-function toggleMarketplace() {
-    var entity = HMD.tabletID;
-    Entities.editEntity(entity, {textures: JSON.stringify({"tex.close": HOME_BUTTON_TEXTURE})});
-    showMarketplace();
-}
-
 var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
 var marketplaceButton = tablet.addButton({
     icon: "icons/tablet-icons/market-i.svg",
+    activeIcon: "icons/tablet-icons/market-a.svg",
     text: "MARKET",
     sortOrder: 9
 });
@@ -117,16 +118,30 @@ function onCanWriteAssetsChanged() {
 }
 
 function onClick() {
-    toggleMarketplace();
+    if (onMarketplaceScreen) {
+        // for toolbar-mode: go back to home screen, this will close the window.
+        tablet.gotoHomeScreen();
+    } else {
+        var entity = HMD.tabletID;
+        Entities.editEntity(entity, {textures: JSON.stringify({"tex.close": HOME_BUTTON_TEXTURE})});
+        showMarketplace();
+    }
+}
+
+function onScreenChanged(type, url) {
+    // for toolbar mode: change button to active when window is first openend, false otherwise.
+    marketplaceButton.editProperties({isActive: shouldActivateButton});
+    shouldActivateButton = false;
+    onMarketplaceScreen = false;
 }
 
 marketplaceButton.clicked.connect(onClick);
+tablet.screenChanged.connect(onScreenChanged);
 Entities.canWriteAssetsChanged.connect(onCanWriteAssetsChanged);
 
 Script.scriptEnding.connect(function () {
-    if (tablet) {
-        tablet.removeButton(marketplaceButton);
-    }
+    tablet.removeButton(marketplaceButton);
+    tablet.screenChanged.disconnect(onScreenChanged);
     Entities.canWriteAssetsChanged.disconnect(onCanWriteAssetsChanged);
 });
 

scripts/system/menu.js

@@ -10,26 +10,46 @@
 //
 
 var HOME_BUTTON_TEXTURE = "http://hifi-content.s3.amazonaws.com/alan/dev/tablet-with-home-button.fbx/tablet-with-home-button.fbm/button-root.png";
-//var HOME_BUTTON_TEXTURE = Script.resourcesPath() + "meshes/tablet-with-home-button.fbx/tablet-with-home-button.fbm/button-root.png";
+// var HOME_BUTTON_TEXTURE = Script.resourcesPath() + "meshes/tablet-with-home-button.fbx/tablet-with-home-button.fbm/button-root.png";
 
 (function() {
     var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
     var button = tablet.addButton({
         icon: "icons/tablet-icons/menu-i.svg",
+        activeIcon: "icons/tablet-icons/menu-a.svg",
         text: "MENU",
         sortOrder: 3
     });
 
+    var shouldActivateButton = false;
+    var onMenuScreen = false;
+
     function onClicked() {
-        var entity = HMD.tabletID;
-        Entities.editEntity(entity, {textures: JSON.stringify({"tex.close": HOME_BUTTON_TEXTURE})});
-        tablet.gotoMenuScreen();
+        if (onMenuScreen) {
+            // for toolbar-mode: go back to home screen, this will close the window.
+            tablet.gotoHomeScreen();
+        } else {
+            var entity = HMD.tabletID;
+            Entities.editEntity(entity, {textures: JSON.stringify({"tex.close": HOME_BUTTON_TEXTURE})});
+            shouldActivateButton = true;
+            tablet.gotoMenuScreen();
+            onMenuScreen = true;
+        }
+    }
+
+    function onScreenChanged(type, url) {
+        // for toolbar mode: change button to active when window is first openend, false otherwise.
+        button.editProperties({isActive: shouldActivateButton});
+        shouldActivateButton = false;
+        onMenuScreen = false;
     }
 
     button.clicked.connect(onClicked);
+    tablet.screenChanged.connect(onScreenChanged);
 
     Script.scriptEnding.connect(function () {
         button.clicked.disconnect(onClicked);
         tablet.removeButton(button);
+        tablet.screenChanged.disconnect(onScreenChanged);
     });
 }());

scripts/system/pal.js

@@ -37,6 +37,15 @@ var conserveResources = true;
 
 Script.include("/~/system/libraries/controllers.js");
 
+function projectVectorOntoPlane(normalizedVector, planeNormal) {
+    return Vec3.cross(planeNormal, Vec3.cross(normalizedVector, planeNormal));
+}
+function angleBetweenVectorsInPlane(from, to, normal) {
+    var projectedFrom = projectVectorOntoPlane(from, normal);
+    var projectedTo = projectVectorOntoPlane(to, normal);
+    return Vec3.orientedAngle(projectedFrom, projectedTo, normal);
+}
+
 //
 // Overlays.
 //
@@ -229,7 +238,11 @@ function fromQml(message) { // messages are {method, params}, like json-rpc. See
         break;
     case 'refresh':
         removeOverlays();
-        populateUserList(message.params);
+        // If filter is specified from .qml instead of through settings, update the settings.
+        if (message.params.filter !== undefined) {
+            Settings.setValue('pal/filtered', !!message.params.filter);
+        }
+        populateUserList(message.params.selected);
         UserActivityLogger.palAction("refresh", "");
         break;
     case 'updateGain':
@@ -271,13 +284,42 @@ function addAvatarNode(id) {
         color: color(selected, false, 0.0),
         ignoreRayIntersection: false}, selected, !conserveResources);
 }
+// Each open/refresh will capture a stable set of avatarsOfInterest, within the specified filter.
+var avatarsOfInterest = {};
 function populateUserList(selectData) {
+    var filter = Settings.getValue('pal/filtered') && {distance: Settings.getValue('pal/nearDistance')};
     var data = [], avatars = AvatarList.getAvatarIdentifiers();
-    conserveResources = avatars.length > 20;
+    avatarsOfInterest = {};
+    var myPosition = filter && Camera.position,
+        frustum = filter && Camera.frustum,
+        verticalHalfAngle = filter && (frustum.fieldOfView / 2),
+        horizontalHalfAngle = filter && (verticalHalfAngle * frustum.aspectRatio),
+        orientation = filter && Camera.orientation,
+        front = filter && Quat.getFront(orientation),
+        verticalAngleNormal = filter && Quat.getRight(orientation),
+        horizontalAngleNormal = filter && Quat.getUp(orientation);
     avatars.forEach(function (id) { // sorting the identifiers is just an aid for debugging
         var avatar = AvatarList.getAvatar(id);
+        var name = avatar.sessionDisplayName;
+        if (!name) {
+            // Either we got a data packet but no identity yet, or something is really messed up. In any case,
+            // we won't be able to do anything with this user, so don't include them.
+            // In normal circumstances, a refresh will bring in the new user, but if we're very heavily loaded,
+            // we could be losing and gaining people randomly.
+            print('No avatar identity data for', id);
+            return;
+        }
+        if (id && myPosition && (Vec3.distance(avatar.position, myPosition) > filter.distance)) {
+            return;
+        }
+        var normal = id && filter && Vec3.normalize(Vec3.subtract(avatar.position, myPosition));
+        var horizontal = normal && angleBetweenVectorsInPlane(normal, front, horizontalAngleNormal);
+        var vertical = normal && angleBetweenVectorsInPlane(normal, front, verticalAngleNormal);
+        if (id && filter && ((Math.abs(horizontal) > horizontalHalfAngle) || (Math.abs(vertical) > verticalHalfAngle))) {
+            return;
+        }
         var avatarPalDatum = {
-            displayName: avatar.sessionDisplayName,
+            displayName: name,
             userName: '',
             sessionId: id || '',
             audioLevel: 0.0,
@@ -289,10 +331,12 @@ function populateUserList(selectData) {
             addAvatarNode(id); // No overlay for ourselves
             // Everyone needs to see admin status. Username and fingerprint returns default constructor output if the requesting user isn't an admin.
             Users.requestUsernameFromID(id);
+            avatarsOfInterest[id] = true;
         }
         data.push(avatarPalDatum);
         print('PAL data:', JSON.stringify(avatarPalDatum));
     });
+    conserveResources = Object.keys(avatarsOfInterest).length > 20;
     sendToQml({ method: 'users', params: data });
     if (selectData) {
         selectData[2] = true;
@@ -317,8 +361,8 @@ var pingPong = true;
 function updateOverlays() {
     var eye = Camera.position;
     AvatarList.getAvatarIdentifiers().forEach(function (id) {
-        if (!id) {
-            return; // don't update ourself
+        if (!id || !avatarsOfInterest[id]) {
+            return; // don't update ourself, or avatars we're not interested in
         }
         var avatar = AvatarList.getAvatar(id);
         if (!avatar) {
@@ -477,23 +521,17 @@ var button;
 var buttonName = "PEOPLE";
 var tablet = null;
 
-function onTabletScreenChanged(type, url) {
-    if (type !== "QML" || url !== "../Pal.qml") {
-        off();
-    }
-}
-
 function startup() {
     tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
     button = tablet.addButton({
         text: buttonName,
         icon: "icons/tablet-icons/people-i.svg",
+        activeIcon: "icons/tablet-icons/people-a.svg",
         sortOrder: 7
     });
     tablet.fromQml.connect(fromQml);
     button.clicked.connect(onTabletButtonClicked);
     tablet.screenChanged.connect(onTabletScreenChanged);
-
     Users.usernameFromIDReply.connect(usernameFromIDReply);
     Window.domainChanged.connect(clearLocalQMLDataAndClosePAL);
     Window.domainConnectionRefused.connect(clearLocalQMLDataAndClosePAL);
@@ -524,17 +562,39 @@ function off() {
     Users.requestsDomainListData = false;
 }
 
+var onPalScreen = false;
+var shouldActivateButton = false;
+
 function onTabletButtonClicked() {
-    tablet.loadQMLSource("../Pal.qml");
-    Users.requestsDomainListData = true;
-    populateUserList();
-    isWired = true;
-    Script.update.connect(updateOverlays);
-    Controller.mousePressEvent.connect(handleMouseEvent);
-    Controller.mouseMoveEvent.connect(handleMouseMoveEvent);
-    triggerMapping.enable();
-    triggerPressMapping.enable();
-    audioTimer = createAudioInterval(conserveResources ? AUDIO_LEVEL_CONSERVED_UPDATE_INTERVAL_MS : AUDIO_LEVEL_UPDATE_INTERVAL_MS);
+    if (onPalScreen) {
+        // for toolbar-mode: go back to home screen, this will close the window.
+        tablet.gotoHomeScreen();
+    } else {
+        shouldActivateButton = true;
+        tablet.loadQMLSource("../Pal.qml");
+        onPalScreen = true;
+        Users.requestsDomainListData = true;
+        populateUserList();
+        isWired = true;
+        Script.update.connect(updateOverlays);
+        Controller.mousePressEvent.connect(handleMouseEvent);
+        Controller.mouseMoveEvent.connect(handleMouseMoveEvent);
+        triggerMapping.enable();
+        triggerPressMapping.enable();
+        audioTimer = createAudioInterval(conserveResources ? AUDIO_LEVEL_CONSERVED_UPDATE_INTERVAL_MS : AUDIO_LEVEL_UPDATE_INTERVAL_MS);
+    }
+}
+
+function onTabletScreenChanged(type, url) {
+    // for toolbar mode: change button to active when window is first openend, false otherwise.
+    button.editProperties({isActive: shouldActivateButton});
+    shouldActivateButton = false;
+    onPalScreen = false;
+
+    // disable sphere overlays when not on pal screen.
+    if (type !== "QML" || url !== "../Pal.qml") {
+        off();
+    }
 }
 
 //
@@ -621,14 +681,12 @@ function shutdown() {
     button.clicked.disconnect(onTabletButtonClicked);
     tablet.removeButton(button);
     tablet.screenChanged.disconnect(onTabletScreenChanged);
-
     Users.usernameFromIDReply.disconnect(usernameFromIDReply);
     Window.domainChanged.disconnect(clearLocalQMLDataAndClosePAL);
     Window.domainConnectionRefused.disconnect(clearLocalQMLDataAndClosePAL);
     Messages.subscribe(CHANNEL);
     Messages.messageReceived.disconnect(receiveMessage);
     Users.avatarDisconnected.disconnect(avatarDisconnected);
-
     off();
 }
 

scripts/system/tablet-goto.js

@@ -14,10 +14,27 @@
 (function() { // BEGIN LOCAL_SCOPE
     var gotoQmlSource = "TabletAddressDialog.qml";
     var buttonName = "GOTO";
+    var onGotoScreen = false;
+    var shouldActivateButton = false;
 
-    function onClicked(){
-        tablet.loadQMLSource(gotoQmlSource);
+    function onClicked() {
+        if (onGotoScreen) {
+            // for toolbar-mode: go back to home screen, this will close the window.
+            tablet.gotoHomeScreen();
+        } else {
+            shouldActivateButton = true;
+            tablet.loadQMLSource(gotoQmlSource);
+            onGotoScreen = true;
+        }
     }
+
+    function onScreenChanged(type, url) {
+        // for toolbar mode: change button to active when window is first openend, false otherwise.
+        button.editProperties({isActive: shouldActivateButton});
+        shouldActivateButton = false;
+        onGotoScreen = false;
+    }
+
     var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
     var button = tablet.addButton({
         icon: "icons/tablet-icons/goto-i.svg",
@@ -27,12 +44,12 @@
     });
 
     button.clicked.connect(onClicked);
+    tablet.screenChanged.connect(onScreenChanged);
 
     Script.scriptEnding.connect(function () {
         button.clicked.disconnect(onClicked);
-        if (tablet) {
-            tablet.removeButton(button);
-        }
+        tablet.removeButton(button);
+        tablet.screenChanged.disconnect(onScreenChanged);
     });
 
 }()); // END LOCAL_SCOPE

scripts/system/tablet-ui/tabletUI.js

@@ -26,12 +26,19 @@
         print("show tablet-ui");
 
         var DEFAULT_WIDTH = 0.4375;
-        var DEFAULT_HMD_TABLET_SCALE = 100;
-        var HMD_TABLET_SCALE = Settings.getValue("hmdTabletScale") || DEFAULT_HMD_TABLET_SCALE;
-        UIWebTablet = new WebTablet("qml/hifi/tablet/TabletRoot.qml", DEFAULT_WIDTH * (HMD_TABLET_SCALE / 100), null, activeHand, true);
+        var DEFAULT_TABLET_SCALE = 100;
+        var toolbarMode = Tablet.getTablet("com.highfidelity.interface.tablet.system").toolbarMode;
+        var TABLET_SCALE = DEFAULT_TABLET_SCALE;
+        if (toolbarMode) {
+            TABLET_SCALE = Settings.getValue("desktopTabletScale") || DEFAULT_TABLET_SCALE;
+        } else {
+            TABLET_SCALE = Settings.getValue("hmdTabletScale") || DEFAULT_TABLET_SCALE;
+        }
+        UIWebTablet = new WebTablet("qml/hifi/tablet/TabletRoot.qml", DEFAULT_WIDTH * (TABLET_SCALE / 100), null, activeHand, true);
         UIWebTablet.register();
         HMD.tabletID = UIWebTablet.tabletEntityID;
-        HMD.homeButtonID = UIWebTablet.homeButtonEntity;
+        HMD.homeButtonID = UIWebTablet.homeButtonID;
+        HMD.tabletScreenID = UIWebTablet.webOverlayID;
     }
 
     function hideTabletUI() {
@@ -48,6 +55,7 @@
             UIWebTablet = null;
             HMD.tabletID = null;
             HMD.homeButtonID = null;
+            HMD.tabletScreenID = null;
         }
     }
 
@@ -77,7 +85,7 @@
             hideTabletUI();
             HMD.closeTablet();
         } else if (HMD.showTablet && !tabletShown && !toolbarMode) {
-            UserActivityLogger.openedTablet();
+            UserActivityLogger.openedTablet(Settings.getValue("tabletVisibleToOthers"));
             showTabletUI();
         } else if (!HMD.showTablet && tabletShown) {
             UserActivityLogger.closedTablet();
@@ -126,5 +134,6 @@
         Entities.deleteEntity(HMD.tabletID);
         HMD.tabletID = null;
         HMD.homeButtonID = null;
+        HMD.tabletScreenID = null;
     });
 }()); // END LOCAL_SCOPE

scripts/system/tablet-users.js

@@ -1,7 +1,7 @@
 "use strict";
 
 //
-//  users.js
+//  tablet-users.js
 //
 //  Created by Faye Li on 18 Jan 2017.
 //  Copyright 2017 High Fidelity, Inc.
@@ -36,16 +36,34 @@
     var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
     var button = tablet.addButton({
         icon: "icons/tablet-icons/users-i.svg",
+        activeIcon: "icons/tablet-icons/users-a.svg",
         text: "USERS",
         sortOrder: 11
     });
 
+    var onUsersScreen = false;
+    var shouldActivateButton = false;
+
     function onClicked() {
-        var tabletEntity = HMD.tabletID;
-        if (tabletEntity) {
-            Entities.editEntity(tabletEntity, {textures: JSON.stringify({"tex.close" : HOME_BUTTON_TEXTURE})});
+        if (onUsersScreen) {
+            // for toolbar-mode: go back to home screen, this will close the window.
+            tablet.gotoHomeScreen();
+        } else {
+            var tabletEntity = HMD.tabletID;
+            if (tabletEntity) {
+                Entities.editEntity(tabletEntity, {textures: JSON.stringify({"tex.close" : HOME_BUTTON_TEXTURE})});
+            }
+            shouldActivateButton = true;
+            tablet.gotoWebScreen(USERS_URL);
+            onUsersScreen = true;
         }
-        tablet.gotoWebScreen(USERS_URL);
+    }
+
+    function onScreenChanged() {
+        // for toolbar mode: change button to active when window is first openend, false otherwise.
+        button.editProperties({isActive: shouldActivateButton});
+        shouldActivateButton = false;
+        onUsersScreen = false;
     }
 
     function onWebEventReceived(event) {
@@ -88,12 +106,13 @@
                 // update your visibility (all, friends, or none)
                 myVisibility = event.data.visibility;
                 GlobalServices.findableBy = myVisibility;
-            } 
+            }
         }
     }
 
     button.clicked.connect(onClicked);
     tablet.webEventReceived.connect(onWebEventReceived);
+    tablet.screenChanged.connect(onScreenChanged);
 
     function cleanup() {
         button.clicked.disconnect(onClicked);

unpublishedScripts/marketplace/shortbow/README.md

@@ -0,0 +1,111 @@
+# Shortbow
+
+Shortbow is a wave-based archery game.
+
+## Notes
+
+There are several design decisions that are based on certain characteristics of the High Fidelity platform,
+and in particular, [server entity scripts](https://wiki.highfidelity.com/wiki/Creating_Entity_Server_Scripts),
+which are touched on below.
+It is recommended that you understand the basics of client entity scripts and server entity scripts (and their
+differences) if you plan on digging into the shortbow code.
+
+ * Client entity scripts end in `ClientEntity.js` and server entity scripts end in `ServerEntity.js`.
+ * Server entity scripts are not guaranteed to have access to *other* entities, and should not rely on it.
+   * You should not rely on using `Entities.getEntityProperties` to access the properties of entities
+     other than the entity that the server entity script is running on. This also applies to other
+     functions like `Entities.findEntities`. This means that something like the `ShortGameManager` (described below)
+     will not know the entity IDs of entities like the start button or scoreboard text entities, so it
+     has to find ways to work around that limitation.
+   * You can, however, use `Entities.editEntity` to edit other entities.
+   * *NOTE*: It is likely that this will change in the future, and server entity scripts will be able to
+     query the existence of other entities in some way. This will obviate the need for some of the workarounds
+     used in shortbow.
+ * The Entity Script Server (where server entity scripts) does not run a physics simulation
+   * Server entity scripts do not generate collision events like would be used with
+     `Script.addEventHandler(entityID, "collisionWithEntity", collideHandler)`.
+     * High Fidelity distributes its physics simulation to clients, so collisions need to be handled
+       there. In the case of enemies in shortbow, for instance, the collisions are handled in the
+       client entity script `enemeyClientEntity.js`.
+   * If no client is present to run the physics simulation, entities will not physically interact with other
+     entities.
+   * But, entities will still apply their basic physical properties.
+
+## Shortbow Game Manager
+
+This section describes both `shortbowServerEntity.js` and `shortbowGameManager.js`. The `shortbowServerEntity.js` script
+exists on the main arena model entity, and is the parent of all of the other parts of the game, other than the
+enemies and bows that are spawned during gameplay.
+
+The `shortbowServerEntity.js` script is a server entity script that runs the shortbow game. The actual logic for
+the game is stored inside `shortbowGameManager.js`, in the `ShortbowGameManager` prototype.
+
+## Enemy Scripts
+
+These scripts exist on each enemy that is spawned.
+
+### enemyClientEntity.js
+
+This script handles collision events on the client. There are two collisions events that it is interested in:
+
+ 1. Collisions with arrows
+   1. Arrow entities have "projectile" in their name
+   1. Any other entity with "projectile" in its name could be used to destroy the enemy
+ 1. Collisions with the gate that the enemies roll towards
+   1. The gate entity has "GateCollider" in its name
+
+### enemyServerEntity.js
+
+This script acts as a fail-safe to work around some of the limitations that are mentioned under [Notes](#notes).
+In particular, if no client is running the physics simulation of an enemy entity, it may fall through the floor
+of the arena or never have a collision event generated against the gate. A server entity script also
+cannot access the properties of another entity, so it can't know if the entity has moved far away from
+the arena.
+
+To handle this, this script is used to periodically send heartbeats to the [ShortbowGameManager](#shortbow-game-manager)
+that runs the game. The heartbeats include the position of the enemy. If the script that received the
+heartbeats hasn't heard from `enemyServerEntity.js` in too long, or the entity has moved too far away, it
+will remove it from it's local list of enemies that still need to be destroyed. This ensure that the game
+doesn't get into a "hung" state where it's waiting for an enemy that will never escape through the gate or be
+destroyed.
+
+## Start Button
+
+These scripts exist on the start button.
+
+### startGameButtonClientEntity.js
+
+This script sends a message to the [ShortbowGameManager](#shortbow-game-manager) to request that the game be started.
+
+### startGameButtonServerEntity.js
+
+When the shortbow game starts the start button is hidden, and when the shortbow game ends it is shown again.
+As described in the [Notes](#notes), server entity scripts cannot access other entities, including their IDs.
+Because of this, the [ShortbowGameManager](#shortbow-game-manager) has no way of knowing the id of the start button,
+and thus being able to use `Entities.editEntity` to set its `visible` property to `true` or `false`. One way around
+this, and is what is used here, is to use `Messages` on a common channel to send messages to a server entity script
+on the start button to request that it be shown or hidden.
+
+## Display (Scoreboard)
+
+This script exists on each text entity that scoreboard is composed of. The scoreboard area is composed of a text entity for each of the following: Score, High Score, Wave, Lives.
+
+### displayServerEntity.js
+
+The same problem that exists for [startGameButtonServerEntity.js](#startgamebuttonserverentityjs) exists for
+the text entities on the scoreboard. This script works around the problem in the same way, but instead of
+receiving a message that tells it whether to hide or show itself, it receives a message that contains the
+text to update the text entity with. For intance, the "lives" display entity will receive a message each
+time a life is lost with the current number of lives.
+
+## How is the "common channel" determined that is used in some of the client and server entity scripts?
+
+Imagine that there are two instances of shortbow next to each. If the channel being used is always the same,
+and not differentiated in some way between the two instances, a "start game" message sent from [startGameButtonClientEntity.js](#startgamebuttoncliententityjs)
+on one game will be received and handled by both games, causing both of them to start. We need a way to create
+a channel that is unique to the scripts that are running for a particular instance of shortbow.
+
+All of the entities in shortbow, besides the enemy entities, are children of the main arena entity that
+runs the game. All of the scripts on these entities can access their parentID, so they can use
+a prefix plus the parentID to generate a unique channel for that instance of shortbow.
+

unpublishedScripts/marketplace/shortbow/bow/bow.js

@@ -0,0 +1,671 @@
+//
+//  Created by Seth Alves on 2016-9-7
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+/* global MyAvatar, Vec3, Controller, Quat */
+
+var GRAB_COMMUNICATIONS_SETTING = "io.highfidelity.isFarGrabbing";
+setGrabCommunications = function setFarGrabCommunications(on) {
+    Settings.setValue(GRAB_COMMUNICATIONS_SETTING, on ? "on" : "");
+}
+getGrabCommunications = function getFarGrabCommunications() {
+    return !!Settings.getValue(GRAB_COMMUNICATIONS_SETTING, "");
+}
+
+// this offset needs to match the one in libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp:378
+var GRAB_POINT_SPHERE_OFFSET = { x: 0.04, y: 0.13, z: 0.039 };  // x = upward, y = forward, z = lateral
+
+getGrabPointSphereOffset = function(handController) {
+    if (handController === Controller.Standard.RightHand) {
+        return GRAB_POINT_SPHERE_OFFSET;
+    }
+    return {
+        x: GRAB_POINT_SPHERE_OFFSET.x * -1,
+        y: GRAB_POINT_SPHERE_OFFSET.y,
+        z: GRAB_POINT_SPHERE_OFFSET.z
+    };
+};
+
+// controllerWorldLocation is where the controller would be, in-world, with an added offset
+getControllerWorldLocation = function (handController, doOffset) {
+    var orientation;
+    var position;
+    var pose = Controller.getPoseValue(handController);
+    var valid = pose.valid;
+    var controllerJointIndex;
+    if (pose.valid) {
+        if (handController === Controller.Standard.RightHand) {
+            controllerJointIndex = MyAvatar.getJointIndex("_CAMERA_RELATIVE_CONTROLLER_RIGHTHAND");
+        } else {
+            controllerJointIndex = MyAvatar.getJointIndex("_CAMERA_RELATIVE_CONTROLLER_LEFTHAND");
+        }
+        orientation = Quat.multiply(MyAvatar.orientation, MyAvatar.getAbsoluteJointRotationInObjectFrame(controllerJointIndex));
+        position = Vec3.sum(MyAvatar.position, Vec3.multiplyQbyV(MyAvatar.orientation, MyAvatar.getAbsoluteJointTranslationInObjectFrame(controllerJointIndex)));
+
+        // add to the real position so the grab-point is out in front of the hand, a bit
+        if (doOffset) {
+            var offset = getGrabPointSphereOffset(handController);
+            position = Vec3.sum(position, Vec3.multiplyQbyV(orientation, offset));
+        }
+
+    } else if (!HMD.isHandControllerAvailable()) {
+        // NOTE: keep this offset in sync with scripts/system/controllers/handControllerPointer.js:493
+        var VERTICAL_HEAD_LASER_OFFSET = 0.1;
+        position = Vec3.sum(Camera.position, Vec3.multiplyQbyV(Camera.orientation, {x: 0, y: VERTICAL_HEAD_LASER_OFFSET, z: 0}));
+        orientation = Quat.multiply(Camera.orientation, Quat.angleAxis(-90, { x: 1, y: 0, z: 0 }));
+        valid = true;
+    }
+
+    return {position: position,
+            translation: position,
+            orientation: orientation,
+            rotation: orientation,
+            valid: valid};
+};
+
+
+
+//
+//
+//
+//
+//
+//
+//  bow.js
+//
+//  This script attaches to a bow that you can pick up with a hand controller.
+//  Created by James B. Pollack @imgntn on 10/19/2015
+//  Copyright 2015 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+/*global Script, Controller, SoundCache, Entities, getEntityCustomData, setEntityCustomData, MyAvatar, Vec3, Quat, Messages */
+
+
+function getControllerLocation(controllerHand) {
+    var standardControllerValue =
+        (controllerHand === "right") ? Controller.Standard.RightHand : Controller.Standard.LeftHand;
+    return getControllerWorldLocation(standardControllerValue, true);
+};
+
+(function() {
+
+    Script.include("/~/system/libraries/utils.js");
+
+    const NULL_UUID = "{00000000-0000-0000-0000-000000000000}";
+
+    const NOTCH_ARROW_SOUND_URL = Script.resolvePath('notch.wav');
+    const SHOOT_ARROW_SOUND_URL = Script.resolvePath('String_release2.L.wav');
+    const STRING_PULL_SOUND_URL = Script.resolvePath('Bow_draw.1.L.wav');
+    const ARROW_HIT_SOUND_URL = Script.resolvePath('Arrow_impact1.L.wav');
+
+    const ARROW_MODEL_URL = Script.resolvePath('arrow.fbx');
+    const ARROW_DIMENSIONS = {
+        x: 0.20,
+        y: 0.19,
+        z: 0.93
+    };
+
+    const MIN_ARROW_SPEED = 3.0;
+    const MAX_ARROW_SPEED = 30.0;
+
+    const ARROW_TIP_OFFSET = 0.47;
+    const ARROW_GRAVITY = {
+        x: 0,
+        y: -9.8,
+        z: 0
+    };
+
+
+    const ARROW_LIFETIME = 15; // seconds
+    const ARROW_PARTICLE_LIFESPAN = 2; // seconds
+
+    const TOP_NOTCH_OFFSET = 0.6;
+    const BOTTOM_NOTCH_OFFSET = 0.6;
+
+    const LINE_DIMENSIONS = {
+        x: 5.0,
+        y: 5.0,
+        z: 5.0
+    };
+
+    const DRAW_STRING_THRESHOLD = 0.80;
+    const DRAW_STRING_PULL_DELTA_HAPTIC_PULSE = 0.09;
+    const DRAW_STRING_MAX_DRAW = 0.7;
+
+
+    const MIN_ARROW_DISTANCE_FROM_BOW_REST = 0.2;
+    const MAX_ARROW_DISTANCE_FROM_BOW_REST = ARROW_DIMENSIONS.z - 0.2;
+    const MIN_HAND_DISTANCE_FROM_BOW_TO_KNOCK_ARROW = 0.25;
+    const MIN_ARROW_DISTANCE_FROM_BOW_REST_TO_SHOOT = 0.30;
+
+    const NOTCH_OFFSET_FORWARD = 0.08;
+    const NOTCH_OFFSET_UP = 0.035;
+
+    const SHOT_SCALE = {
+        min1: 0.0,
+        max1: 0.6,
+        min2: 1.0,
+        max2: 15.0
+    };
+
+    const USE_DEBOUNCE = false;
+
+    const TRIGGER_CONTROLS = [
+        Controller.Standard.LT,
+        Controller.Standard.RT,
+    ];
+
+    function interval() {
+        var lastTime = new Date().getTime();
+
+        return function getInterval() {
+            var newTime = new Date().getTime();
+            var delta = newTime - lastTime;
+            lastTime = newTime;
+            return delta;
+        };
+    }
+
+    var checkInterval = interval();
+
+    var _this;
+
+    function Bow() {
+        _this = this;
+        return;
+    }
+
+    const STRING_NAME = 'Hifi-Bow-String';
+    const ARROW_NAME = 'Hifi-Arrow-projectile';
+
+    const STATE_IDLE = 0;
+    const STATE_ARROW_GRABBED = 1;
+
+    Bow.prototype = {
+        topString: null,
+        aiming: false,
+        arrowTipPosition: null,
+        preNotchString: null,
+        stringID: null,
+        arrow: null,
+        stringData: {
+            currentColor: {
+                red: 255,
+                green: 255,
+                blue: 255
+            }
+        },
+
+        state: STATE_IDLE,
+        sinceLastUpdate: 0,
+        preload: function(entityID) {
+            this.entityID = entityID;
+            this.stringPullSound = SoundCache.getSound(STRING_PULL_SOUND_URL);
+            this.shootArrowSound = SoundCache.getSound(SHOOT_ARROW_SOUND_URL);
+            this.arrowHitSound = SoundCache.getSound(ARROW_HIT_SOUND_URL);
+            this.arrowNotchSound = SoundCache.getSound(NOTCH_ARROW_SOUND_URL);
+            var userData = Entities.getEntityProperties(this.entityID, ["userData"]).userData;
+            print(userData);
+            this.userData = JSON.parse(userData);
+            this.stringID = null;
+        },
+
+        unload: function() {
+            Messages.sendLocalMessage('Hifi-Hand-Disabler', "none");
+            Entities.deleteEntity(this.arrow);
+        },
+
+        startEquip: function(entityID, args) {
+            this.hand = args[0];
+            this.bowHand = args[0];
+            this.stringHand = this.bowHand === "right" ? "left" : "right";
+
+            Entities.editEntity(_this.entityID, {
+                collidesWith: "",
+            });
+
+            var data = getEntityCustomData('grabbableKey', this.entityID, {});
+            data.grabbable = false;
+            setEntityCustomData('grabbableKey', this.entityID, data);
+
+            this.initString();
+
+            var self = this;
+            this.updateIntervalID = Script.setInterval(function() { self.update(); }, 11);
+        },
+
+        getStringHandPosition: function() {
+            return getControllerLocation(this.stringHand).position;
+        },
+
+        releaseEquip: function(entityID, args) {
+            Script.clearInterval(this.updateIntervalID);
+            this.updateIntervalID = null;
+
+            Messages.sendLocalMessage('Hifi-Hand-Disabler', "none");
+
+            var data = getEntityCustomData('grabbableKey', this.entityID, {});
+            data.grabbable = true;
+            setEntityCustomData('grabbableKey', this.entityID, data);
+            Entities.deleteEntity(this.arrow);
+            this.resetStringToIdlePosition();
+            this.destroyArrow();
+            Entities.editEntity(_this.entityID, {
+                collidesWith: "static,dynamic,kinematic,otherAvatar,myAvatar"
+            });
+        },
+
+        update: function(entityID) {
+            var self = this;
+            self.deltaTime = checkInterval();
+            //debounce during debugging -- maybe we're updating too fast?
+            if (USE_DEBOUNCE === true) {
+                self.sinceLastUpdate = self.sinceLastUpdate + self.deltaTime;
+
+                if (self.sinceLastUpdate > 60) {
+                    self.sinceLastUpdate = 0;
+                } else {
+                    return;
+                }
+            }
+
+            //invert the hands because our string will be held with the opposite hand of the first one we pick up the bow with
+            this.triggerValue = Controller.getValue(TRIGGER_CONTROLS[(this.hand === 'left') ? 1 : 0]);
+
+            this.bowProperties = Entities.getEntityProperties(this.entityID, ['position', 'rotation']);
+            var notchPosition = this.getNotchPosition(this.bowProperties);
+            var stringHandPosition = this.getStringHandPosition();
+            var handToNotch = Vec3.subtract(notchPosition, stringHandPosition);
+            var pullBackDistance = Vec3.length(handToNotch);
+
+            if (this.state === STATE_IDLE) {
+                this.pullBackDistance = 0;
+
+                this.resetStringToIdlePosition();
+                if (this.triggerValue >= DRAW_STRING_THRESHOLD && pullBackDistance < MIN_HAND_DISTANCE_FROM_BOW_TO_KNOCK_ARROW && !this.backHandBusy) {
+                    //the first time aiming the arrow
+                    var handToDisable = (this.hand === 'right' ? 'left' : 'right');
+                    this.state = STATE_ARROW_GRABBED;
+                }
+            }
+
+            if (this.state === STATE_ARROW_GRABBED) {
+                if (!this.arrow) {
+                    var handToDisable = (this.hand === 'right' ? 'left' : 'right');
+                    Messages.sendLocalMessage('Hifi-Hand-Disabler', handToDisable);
+                    this.playArrowNotchSound();
+                    this.arrow = this.createArrow();
+                    this.playStringPullSound();
+                }
+
+                if (this.triggerValue < DRAW_STRING_THRESHOLD) {
+                    if (pullBackDistance >= (MIN_ARROW_DISTANCE_FROM_BOW_REST_TO_SHOOT)) {
+                        // The arrow has been pulled far enough back that we can release it
+                        Messages.sendLocalMessage('Hifi-Hand-Disabler', "none");
+                        this.updateArrowPositionInNotch(true, true);
+                        this.arrow = null;
+                        this.state = STATE_IDLE;
+                        this.resetStringToIdlePosition();
+                    } else {
+                        // The arrow has not been pulled far enough back so we just remove the arrow
+                        Messages.sendLocalMessage('Hifi-Hand-Disabler', "none");
+                        Entities.deleteEntity(this.arrow);
+                        this.arrow = null;
+                        this.state = STATE_IDLE;
+                        this.resetStringToIdlePosition();
+                    }
+                } else {
+                    this.updateArrowPositionInNotch(false, true);
+                    this.updateString();
+                }
+            }
+        },
+
+        destroyArrow: function() {
+            var children = Entities.getChildrenIDs(this.entityID);
+            children.forEach(function(childID) {
+                var childName = Entities.getEntityProperties(childID, ["name"]).name;
+                if (childName == ARROW_NAME) {
+                    Entities.deleteEntity(childID);
+                    // Continue iterating through children in case we've ended up in
+                    // a bad state where there are multiple arrows.
+                }
+            });
+        },
+
+        createArrow: function() {
+            this.playArrowNotchSound();
+
+            var arrow = Entities.addEntity({
+                name: ARROW_NAME,
+                type: 'Model',
+                modelURL: ARROW_MODEL_URL,
+                shapeType: 'simple-compound',
+                dimensions: ARROW_DIMENSIONS,
+                position: this.bowProperties.position,
+                parentID: this.entityID,
+                dynamic: false,
+                collisionless: true,
+                collisionSoundURL: ARROW_HIT_SOUND_URL,
+                damping: 0.01,
+                userData: JSON.stringify({
+                    grabbableKey: {
+                        grabbable: false
+                    },
+                    creatorSessionUUID: MyAvatar.sessionUUID
+                })
+            });
+
+            var makeArrowStick = function(entityA, entityB, collision) {
+                Entities.editEntity(entityA, {
+                    localAngularVelocity: {
+                        x: 0,
+                        y: 0,
+                        z: 0
+                    },
+                    localVelocity: {
+                        x: 0,
+                        y: 0,
+                        z: 0
+                    },
+                    gravity: {
+                        x: 0,
+                        y: 0,
+                        z: 0
+                    },
+                    parentID: entityB,
+                    dynamic: false,
+                    collisionless: true,
+                    collidesWith: ""
+                });
+                Script.removeEventHandler(arrow, "collisionWithEntity", makeArrowStick);
+            };
+
+            Script.addEventHandler(arrow, "collisionWithEntity", makeArrowStick);
+
+            return arrow;
+        },
+
+        initString: function() {
+            // Check for existence of string
+            var children = Entities.getChildrenIDs(this.entityID);
+            children.forEach(function(childID) {
+                var childName = Entities.getEntityProperties(childID, ["name"]).name;
+                if (childName == STRING_NAME) {
+                    this.stringID = childID;
+                }
+            });
+
+            // If thie string wasn't found, create it
+            if (this.stringID === null) {
+                this.stringID = Entities.addEntity({
+                    collisionless: true,
+                    dimensions: { "x": 5, "y": 5, "z": 5 },
+                    ignoreForCollisions: 1,
+                    linePoints: [ { "x": 0, "y": 0, "z": 0 }, { "x": 0, "y": -1.2, "z": 0 } ],
+                    lineWidth: 5,
+                    color: { red: 153, green: 102, blue: 51 },
+                    name: STRING_NAME,
+                    parentID: this.entityID,
+                    localPosition: { "x": 0, "y": 0.6, "z": 0.1 },
+                    localRotation: { "w": 1, "x": 0, "y": 0, "z": 0 },
+                    type: 'Line',
+                    userData: JSON.stringify({
+                        grabbableKey: {
+                            grabbable: false
+                        }
+                    })
+                });
+            }
+
+            this.resetStringToIdlePosition();
+        },
+
+        // This resets the string to a straight line
+        resetStringToIdlePosition: function() {
+            Entities.editEntity(this.stringID, {
+                linePoints: [ { "x": 0, "y": 0, "z": 0 }, { "x": 0, "y": -1.2, "z": 0 } ],
+                lineWidth: 5,
+                localPosition: { "x": 0, "y": 0.6, "z": 0.1 },
+                localRotation: { "w": 1, "x": 0, "y": 0, "z": 0 },
+            });
+        },
+
+        updateString: function() {
+            var upVector = Quat.getUp(this.bowProperties.rotation);
+            var upOffset = Vec3.multiply(upVector, TOP_NOTCH_OFFSET);
+            var downVector = Vec3.multiply(-1, Quat.getUp(this.bowProperties.rotation));
+            var downOffset = Vec3.multiply(downVector, BOTTOM_NOTCH_OFFSET);
+            var backOffset = Vec3.multiply(-0.1, Quat.getFront(this.bowProperties.rotation));
+
+            var topStringPosition = Vec3.sum(this.bowProperties.position, upOffset);
+            this.topStringPosition = Vec3.sum(topStringPosition, backOffset);
+            var bottomStringPosition = Vec3.sum(this.bowProperties.position, downOffset);
+            this.bottomStringPosition = Vec3.sum(bottomStringPosition, backOffset);
+
+            var stringProps = Entities.getEntityProperties(this.stringID, ['position', 'rotation']);
+            var handPositionLocal = Vec3.subtract(this.arrowRearPosition, stringProps.position);
+            handPositionLocal = Vec3.multiplyQbyV(Quat.inverse(stringProps.rotation), handPositionLocal);
+
+            var linePoints = [
+                { x: 0, y: 0, z: 0 },
+                handPositionLocal,
+                { x: 0, y: -1.2, z: 0 },
+            ];
+
+            Entities.editEntity(this.stringID, {
+                linePoints: linePoints,
+            });
+        },
+
+        getNotchPosition: function(bowProperties) {
+            var frontVector = Quat.getFront(bowProperties.rotation);
+            var notchVectorForward = Vec3.multiply(frontVector, NOTCH_OFFSET_FORWARD);
+            var upVector = Quat.getUp(bowProperties.rotation);
+            var notchVectorUp = Vec3.multiply(upVector, NOTCH_OFFSET_UP);
+            var notchPosition = Vec3.sum(bowProperties.position, notchVectorForward);
+            notchPosition = Vec3.sum(notchPosition, notchVectorUp);
+            return notchPosition;
+        },
+
+        updateArrowPositionInNotch: function(shouldReleaseArrow, doHapticPulses) {
+            //set the notch that the arrow should go through
+            var notchPosition = this.getNotchPosition(this.bowProperties);
+            //set the arrow rotation to be between the notch and other hand
+            var stringHandPosition = this.getStringHandPosition();
+            var handToNotch = Vec3.subtract(notchPosition, stringHandPosition);
+            var arrowRotation = Quat.rotationBetween(Vec3.FRONT, handToNotch);
+
+            var backHand = this.hand === 'left' ? 1 : 0;
+            var pullBackDistance = Vec3.length(handToNotch);
+            // pulse as arrow is drawn
+            if (doHapticPulses &&
+                Math.abs(pullBackDistance - this.pullBackDistance) > DRAW_STRING_PULL_DELTA_HAPTIC_PULSE) {
+                Controller.triggerHapticPulse(1, 20, backHand);
+                this.pullBackDistance = pullBackDistance;
+            }
+
+            if (pullBackDistance > DRAW_STRING_MAX_DRAW) {
+                pullBackDistance = DRAW_STRING_MAX_DRAW;
+            }
+
+            var handToNotchDistance = Vec3.length(handToNotch);
+            var stringToNotchDistance = Math.max(MIN_ARROW_DISTANCE_FROM_BOW_REST, Math.min(MAX_ARROW_DISTANCE_FROM_BOW_REST, handToNotchDistance));
+            var halfArrowVec = Vec3.multiply(Vec3.normalize(handToNotch), ARROW_DIMENSIONS.z / 2.0);
+            var offset = Vec3.subtract(notchPosition, Vec3.multiply(Vec3.normalize(handToNotch), stringToNotchDistance - ARROW_DIMENSIONS.z / 2.0));
+
+            var arrowPosition = offset;
+
+            // Set arrow rear position
+            var frontVector = Quat.getFront(arrowRotation);
+            var frontOffset = Vec3.multiply(frontVector, -ARROW_TIP_OFFSET);
+            var arrorRearPosition = Vec3.sum(arrowPosition, frontOffset);
+            this.arrowRearPosition = arrorRearPosition;
+
+            //if we're not shooting, we're updating the arrow's orientation
+            if (shouldReleaseArrow !== true) {
+                Entities.editEntity(this.arrow, {
+                    position: arrowPosition,
+                    rotation: arrowRotation
+                });
+            } else {
+                //shoot the arrow
+                var arrowAge = Entities.getEntityProperties(this.arrow, ["age"]).age;
+
+                //scale the shot strength by the distance you've pulled the arrow back and set its release velocity to be
+                // in the direction of the v
+                var arrowForce = this.scaleArrowShotStrength(stringToNotchDistance);
+                var handToNotchNorm = Vec3.normalize(handToNotch);
+
+                var releaseVelocity = Vec3.multiply(handToNotchNorm, arrowForce);
+
+                //make the arrow physical, give it gravity, a lifetime, and set our velocity
+                var arrowProperties = {
+                    dynamic: true,
+                    collisionless: false,
+                    collidesWith: "static,dynamic,otherAvatar", // workaround: not with kinematic --> no collision with bow
+                    velocity: releaseVelocity,
+                    parentID: NULL_UUID,
+                    gravity: ARROW_GRAVITY,
+                    lifetime: arrowAge + ARROW_LIFETIME,
+                };
+
+                // add a particle effect to make the arrow easier to see as it flies
+                var arrowParticleProperties = {
+                    accelerationSpread: { x: 0, y: 0, z: 0 },
+                    alpha: 1,
+                    alphaFinish: 0,
+                    alphaSpread: 0,
+                    alphaStart: 0.3,
+                    azimuthFinish: 3.1,
+                    azimuthStart: -3.14159,
+                    color: { red: 255, green: 255, blue: 255 },
+                    colorFinish: { red: 255, green: 255, blue: 255 },
+                    colorSpread: { red: 0, green: 0, blue: 0 },
+                    colorStart: { red: 255, green: 255, blue: 255 },
+                    emitAcceleration: { x: 0, y: 0, z: 0 },
+                    emitDimensions: { x: 0, y: 0, z: 0 },
+                    emitOrientation: { x: -0.7, y: 0.0, z: 0.0, w: 0.7 },
+                    emitRate: 0.01,
+                    emitSpeed: 0,
+                    emitterShouldTrail: 0,
+                    isEmitting: 1,
+                    lifespan: ARROW_PARTICLE_LIFESPAN,
+                    lifetime: ARROW_PARTICLE_LIFESPAN + 1,
+                    maxParticles: 1000,
+                    name: 'arrow-particles',
+                    parentID: this.arrow,
+                    particleRadius: 0.132,
+                    polarFinish: 0,
+                    polarStart: 0,
+                    radiusFinish: 0.35,
+                    radiusSpread: 0,
+                    radiusStart: 0.132,
+                    speedSpread: 0,
+                    textures: Script.resolvePath('arrow-sparkle.png'),
+                    type: 'ParticleEffect'
+                };
+
+                Entities.addEntity(arrowParticleProperties);
+
+                // actually shoot the arrow
+                Entities.editEntity(this.arrow, arrowProperties);
+
+                // play the sound of a shooting arrow
+                this.playShootArrowSound();
+
+                Entities.addAction("travel-oriented", this.arrow, {
+                    forward: { x: 0, y: 0, z: -1 },
+                    angularTimeScale: 0.1,
+                    tag: "arrow from hifi-bow",
+                    ttl: ARROW_LIFETIME
+                });
+
+
+            }
+        },
+
+        scaleArrowShotStrength: function(value) {
+            var percentage = (value - MIN_ARROW_DISTANCE_FROM_BOW_REST)
+                / (MAX_ARROW_DISTANCE_FROM_BOW_REST - MIN_ARROW_DISTANCE_FROM_BOW_REST);
+            return MIN_ARROW_SPEED + (percentage * (MAX_ARROW_SPEED - MIN_ARROW_SPEED)) ;
+        },
+
+        playStringPullSound: function() {
+            var audioProperties = {
+                volume: 0.10,
+                position: this.bowProperties.position
+            };
+            this.stringPullInjector = Audio.playSound(this.stringPullSound, audioProperties);
+        },
+
+        playShootArrowSound: function(sound) {
+            var audioProperties = {
+                volume: 0.15,
+                position: this.bowProperties.position
+            };
+            Audio.playSound(this.shootArrowSound, audioProperties);
+        },
+
+        playArrowNotchSound: function() {
+            var audioProperties = {
+                volume: 0.15,
+                position: this.bowProperties.position
+            };
+            Audio.playSound(this.arrowNotchSound, audioProperties);
+        },
+
+        changeStringPullSoundVolume: function(pullBackDistance) {
+            var audioProperties = {
+                volume: this.scaleSoundVolume(pullBackDistance),
+                position: this.bowProperties.position
+            };
+
+            this.stringPullInjector.options = audioProperties;
+        },
+
+        scaleSoundVolume: function(value) {
+            var min1 = SHOT_SCALE.min1;
+            var max1 = SHOT_SCALE.max1;
+            var min2 = 0;
+            var max2 = 0.2;
+            return min2 + (max2 - min2) * ((value - min1) / (max1 - min1));
+        },
+
+        handleMessages: function(channel, message, sender) {
+            if (sender !== MyAvatar.sessionUUID) {
+                return;
+            }
+            if (channel !== 'Hifi-Object-Manipulation') {
+                return;
+            }
+            try {
+                var data = JSON.parse(message);
+                var action = data.action;
+                var hand = data.joint;
+                var isBackHand = ((_this.hand == "left" && hand == "RightHand") ||
+                                  (_this.hand == "right" && hand == "LeftHand"));
+                if ((action == "equip" || action == "grab") && isBackHand) {
+                    _this.backHandBusy = true;
+                }
+                if (action == "release" && isBackHand) {
+                    _this.backHandBusy = false;
+                }
+            }  catch (e) {
+                print("WARNING: bow.js -- error parsing Hifi-Object-Manipulation message: " + message);
+            }
+        }
+    };
+
+    var bow = new Bow();
+
+    Messages.subscribe('Hifi-Object-Manipulation');
+    Messages.messageReceived.connect(bow.handleMessages);
+
+    return bow;
+});

unpublishedScripts/marketplace/shortbow/bow/bow.json

@@ -0,0 +1,44 @@
+{
+    "Entities": [
+        {
+            "clientOnly": 0,
+            "collisionsWillMove": 1,
+            "compoundShapeURL": "http://mpassets.highfidelity.com/a9221d01-95eb-4b2e-85e5-d9e0970a7f51-v1/bow_collision_hull.obj",
+            "created": "2017-02-14T18:54:38Z",
+            "dimensions": {
+                "x": 0.039999999105930328,
+                "y": 1.2999999523162842,
+                "z": 0.20000000298023224
+            },
+            "dynamic": 1,
+            "gravity": {
+                "x": 0,
+                "y": -9.8000001907348633,
+                "z": 0
+            },
+            "id": "{73954924-2e18-4787-91a7-092c2afb6242}",
+            "lastEdited": 1487098438422164,
+            "lastEditedBy": "{d2da5e17-9125-414d-ac4e-cd7fba6c22f8}",
+            "modelURL": "http://mpassets.highfidelity.com/a9221d01-95eb-4b2e-85e5-d9e0970a7f51-v1/bow-deadly.fbx",
+            "name": "WG.Hifi-Bow",
+            "owningAvatarID": "{00000000-0000-0000-0000-000000000000}",
+            "queryAACube": {
+                "scale": 1.3159027099609375,
+                "x": -0.65795135498046875,
+                "y": -0.65795135498046875,
+                "z": -0.65795135498046875
+            },
+            "rotation": {
+                "w": 0.9717707633972168,
+                "x": 0.15437555313110352,
+                "y": -0.10472267866134644,
+                "z": -0.14421302080154419
+            },
+            "script": "http://mpassets.highfidelity.com/a9221d01-95eb-4b2e-85e5-d9e0970a7f51-v1/bow.js",
+            "shapeType": "compound",
+            "type": "Model",
+            "userData": "{\"grabbableKey\":{\"grabbable\":true},\"wearable\":{\"joints\":{\"RightHand\":[{\"x\":0.0813,\"y\":0.0452,\"z\":0.0095},{\"x\":-0.3946,\"y\":-0.6604,\"z\":0.4748,\"w\":-0.4275}],\"LeftHand\":[{\"x\":-0.0881,\"y\":0.0259,\"z\":0.0159},{\"x\":0.4427,\"y\":-0.6519,\"z\":0.4592,\"w\":0.4099}]}}}"
+        }
+    ],
+    "Version": 67
+}

unpublishedScripts/marketplace/shortbow/bow/bow.svo.json

@@ -0,0 +1,32 @@
+{
+  "Entities": [        {
+            "collisionsWillMove": 1,
+            "compoundShapeURL": "http://hifi-content.s3.amazonaws.com/caitlyn/production/bow/bow_collision_hull.obj",
+            "created": "2016-09-01T23:57:55Z",
+            "dimensions": {
+                "x": 0.039999999105930328,
+                "y": 1.2999999523162842,
+                "z": 0.20000000298023224
+            },
+            "dynamic": 1,
+            "gravity": {
+                "x": 0,
+                "y": -1,
+                "z": 0
+            },
+            "modelURL": "http://hifi-content.s3.amazonaws.com/caitlyn/production/bow/bow-deadly.fbx",
+            "name": "Hifi-Bow",
+            "rotation": {
+                "w": 0.9718012809753418,
+                "x": 0.15440607070922852,
+                "y": -0.10469216108322144,
+                "z": -0.14418250322341919
+            },
+            "script": "http://hifi-content.s3.amazonaws.com/caitlyn/production/bow/bow.js",
+            "shapeType": "compound",
+            "type": "Model",
+            "userData": "{\"grabbableKey\":{\"grabbable\":true},\"wearable\":{\"joints\":{\"RightHand\":[{\"x\":0.0813,\"y\":0.0452,\"z\":0.0095},{\"x\":-0.3946,\"y\":-0.6604,\"z\":0.4748,\"w\":-0.4275}],\"LeftHand\":[{\"x\":-0.0881,\"y\":0.0259,\"z\":0.0159},{\"x\":0.4427,\"y\":-0.6519,\"z\":0.4592,\"w\":0.4099}]}}}"
+        }
+    ],
+    "Version": 57
+}

unpublishedScripts/marketplace/shortbow/bow/bow_collision_hull.obj

@@ -0,0 +1,21 @@
+v -0.016461 -0.431491 -0.033447
+v -0.007624 0.437384 -0.046243
+v 0.011984 -0.424659 -0.03691
+v 0.015514 0.425913 -0.028648
+v -0.010788 -0.421429 0.093711
+v 0.007135 -0.423115 0.098735
+v -0.010208 0.425558 0.096005
+v 0.006734 0.43913 0.088902
+
+f 1 2 3
+f 3 2 4
+f 5 6 7
+f 7 6 8
+f 1 5 2
+f 2 5 7
+f 3 4 6
+f 6 4 8
+f 1 3 5
+f 5 3 6
+f 2 7 4
+f 4 7 8

unpublishedScripts/marketplace/shortbow/bow/spawnBow.js

@@ -0,0 +1,67 @@
+//
+//  Created by Ryan Huffman on 1/10/2017
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+
+var leftHandPosition = {
+    "x": 0,
+    "y": 0.0559,
+    "z": 0.0159
+};
+var leftHandRotation = Quat.fromPitchYawRollDegrees(90, -90, 0);
+var rightHandPosition = Vec3.multiplyVbyV(leftHandPosition, { x: -1, y: 0, z: 0 });
+var rightHandRotation = Quat.fromPitchYawRollDegrees(90, 90, 0);
+
+var userData = {
+    "grabbableKey": {
+        "grabbable": true
+    },
+    "wearable": {
+        "joints": {
+            "LeftHand": [
+                leftHandPosition,
+                leftHandRotation
+            ],
+            "RightHand": [
+                rightHandPosition,
+                rightHandRotation
+            ]
+        }
+    }
+};
+
+var id = Entities.addEntity({
+    "position": MyAvatar.position,
+    "collisionsWillMove": 1,
+    "compoundShapeURL": Script.resolvePath("bow_collision_hull.obj"),
+    "created": "2016-09-01T23:57:55Z",
+    "dimensions": {
+        "x": 0.039999999105930328,
+        "y": 1.2999999523162842,
+        "z": 0.20000000298023224
+    },
+    "dynamic": 1,
+    "gravity": {
+        "x": 0,
+        "y": -9.8,
+        "z": 0
+    },
+    "modelURL": Script.resolvePath("bow-deadly.fbx"),
+    "name": "Hifi-Bow",
+    "rotation": {
+        "w": 0.9718012809753418,
+        "x": 0.15440607070922852,
+        "y": -0.10469216108322144,
+        "z": -0.14418250322341919
+    },
+    "script": Script.resolvePath("bow.js"),
+    "shapeType": "compound",
+    "type": "Model",
+    "userData": "{\"grabbableKey\":{\"grabbable\":true},\"wearable\":{\"joints\":{\"RightHand\":[{\"x\":0.0813,\"y\":0.0452,\"z\":0.0095},{\"x\":-0.3946,\"y\":-0.6604,\"z\":0.4748,\"w\":-0.4275}],\"LeftHand\":[{\"x\":-0.0881,\"y\":0.0259,\"z\":0.0159},{\"x\":0.4427,\"y\":-0.6519,\"z\":0.4592,\"w\":0.4099}]}}}",
+    "lifetime": 600
+});
+print("Created bow:", id);

unpublishedScripts/marketplace/shortbow/enemyClientEntity.js

@@ -0,0 +1,61 @@
+//
+//  Created by Ryan Huffman on 1/10/2017
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+/* globals utils */
+
+(function() {
+    Script.include('utils.js');
+
+    function Enemy() {
+    }
+    Enemy.prototype = {
+        preload: function(entityID) {
+            this.entityID = entityID;
+
+            // To avoid sending extraneous messages and checking entities that we've already
+            // seen, we keep track of which entities we've collided with previously.
+            this.entityIDsThatHaveCollidedWithMe = [];
+
+            Script.addEventHandler(entityID, "collisionWithEntity", this.onCollide.bind(this));
+
+            var userData = Entities.getEntityProperties(this.entityID, 'userData').userData;
+            var data = utils.parseJSON(userData);
+            if (data !== undefined && data.gameChannel !== undefined) {
+                this.gameChannel = data.gameChannel;
+            } else {
+                print("enemyEntity.js | ERROR: userData does not contain a game channel and/or team number");
+            }
+        },
+        onCollide: function(entityA, entityB, collision) {
+            if (this.entityIDsThatHaveCollidedWithMe.indexOf(entityB) > -1) {
+                return;
+            }
+            this.entityIDsThatHaveCollidedWithMe.push(entityB);
+
+            var colliderName = Entities.getEntityProperties(entityB, 'name').name;
+
+            if (colliderName.indexOf("projectile") > -1) {
+                Messages.sendMessage(this.gameChannel, JSON.stringify({
+                    type: "enemy-killed",
+                    entityID: this.entityID,
+                    position: Entities.getEntityProperties(this.entityID, 'position').position
+                }));
+                Entities.deleteEntity(this.entityID);
+            } else if (colliderName.indexOf("GateCollider") > -1) {
+                Messages.sendMessage(this.gameChannel, JSON.stringify({
+                    type: "enemy-escaped",
+                    entityID: this.entityID,
+                    position: Entities.getEntityProperties(this.entityID, 'position').position
+                }));
+                Entities.deleteEntity(this.entityID);
+            }
+        }
+    };
+
+    return new Enemy();
+});