From 6d7fab40530c781ffed6f6b6506352c678435033 Mon Sep 17 00:00:00 2001
From: David Kelly <david@highfidelity.io>
Date: Tue, 21 Feb 2017 16:59:08 -0800
Subject: [PATCH 01/26] Agent Avatars sending loudness in AvatarData

---
 assignment-client/src/Agent.cpp | 53 +++++++++++++++++++++++++--------
 assignment-client/src/Agent.h   |  1 +
 2 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp
index bea677aeb6..cd8c8189b2 100644
--- a/assignment-client/src/Agent.cpp
+++ b/assignment-client/src/Agent.cpp
@@ -88,9 +88,9 @@ void Agent::playAvatarSound(SharedSoundPointer sound) {
         QMetaObject::invokeMethod(this, "playAvatarSound", Q_ARG(SharedSoundPointer, sound));
         return;
     } else {
-        // TODO: seems to add occasional artifact in tests.  I believe it is 
+        // TODO: seems to add occasional artifact in tests.  I believe it is
         // correct to do this, but need to figure out for sure, so commenting this
-        // out until I verify.  
+        // out until I verify.
         // _numAvatarSoundSentBytes = 0;
         setAvatarSound(sound);
     }
@@ -105,7 +105,7 @@ void Agent::handleOctreePacket(QSharedPointer<ReceivedMessage> message, SharedNo
         if (message->getSize() > statsMessageLength) {
             // pull out the piggybacked packet and create a new QSharedPointer<NLPacket> for it
             int piggyBackedSizeWithHeader = message->getSize() - statsMessageLength;
-            
+
             auto buffer = std::unique_ptr<char[]>(new char[piggyBackedSizeWithHeader]);
             memcpy(buffer.get(), message->getRawMessage() + statsMessageLength, piggyBackedSizeWithHeader);
 
@@ -284,7 +284,7 @@ void Agent::selectAudioFormat(const QString& selectedCodecName) {
     for (auto& plugin : codecPlugins) {
         if (_selectedCodecName == plugin->getName()) {
             _codec = plugin;
-            _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO); 
+            _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO);
             _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
             qDebug() << "Selected Codec Plugin:" << _codec.get();
             break;
@@ -424,16 +424,16 @@ void Agent::executeScript() {
     entityScriptingInterface->setEntityTree(_entityViewer.getTree());
 
     DependencyManager::set<AssignmentParentFinder>(_entityViewer.getTree());
-    
+
     // 100Hz timer for audio
     AvatarAudioTimer* audioTimerWorker = new AvatarAudioTimer();
     audioTimerWorker->moveToThread(&_avatarAudioTimerThread);
     connect(audioTimerWorker, &AvatarAudioTimer::avatarTick, this, &Agent::processAgentAvatarAudio);
     connect(this, &Agent::startAvatarAudioTimer, audioTimerWorker, &AvatarAudioTimer::start);
     connect(this, &Agent::stopAvatarAudioTimer, audioTimerWorker, &AvatarAudioTimer::stop);
-    connect(&_avatarAudioTimerThread, &QThread::finished, audioTimerWorker, &QObject::deleteLater); 
+    connect(&_avatarAudioTimerThread, &QThread::finished, audioTimerWorker, &QObject::deleteLater);
     _avatarAudioTimerThread.start();
-    
+
     // 60Hz timer for avatar
     QObject::connect(_scriptEngine.get(), &ScriptEngine::update, this, &Agent::processAgentAvatar);
     _scriptEngine->run();
@@ -448,14 +448,14 @@ QUuid Agent::getSessionUUID() const {
     return DependencyManager::get<NodeList>()->getSessionUUID();
 }
 
-void Agent::setIsListeningToAudioStream(bool isListeningToAudioStream) { 
+void Agent::setIsListeningToAudioStream(bool isListeningToAudioStream) {
     // this must happen on Agent's main thread
     if (QThread::currentThread() != thread()) {
         QMetaObject::invokeMethod(this, "setIsListeningToAudioStream", Q_ARG(bool, isListeningToAudioStream));
         return;
     }
     if (_isListeningToAudioStream) {
-        // have to tell just the audio mixer to KillAvatar. 
+        // have to tell just the audio mixer to KillAvatar.
 
         auto nodeList = DependencyManager::get<NodeList>();
         nodeList->eachMatchingNode(
@@ -471,7 +471,7 @@ void Agent::setIsListeningToAudioStream(bool isListeningToAudioStream) {
         });
 
     }
-    _isListeningToAudioStream = isListeningToAudioStream; 
+    _isListeningToAudioStream = isListeningToAudioStream;
 }
 
 void Agent::setIsAvatar(bool isAvatar) {
@@ -552,6 +552,7 @@ void Agent::processAgentAvatar() {
         nodeList->broadcastToNodes(std::move(avatarPacket), NodeSet() << NodeType::AvatarMixer);
     }
 }
+
 void Agent::encodeFrameOfZeros(QByteArray& encodedZeros) {
     _flushEncoder = false;
     static const QByteArray zeros(AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL, 0);
@@ -562,6 +563,23 @@ void Agent::encodeFrameOfZeros(QByteArray& encodedZeros) {
     }
 }
 
+void Agent::computeLoudness(QByteArray* decodedBuffer) {
+    float loudness = 0.0f;
+    auto scriptedAvatar = DependencyManager::get<ScriptableAvatar>();
+    if (decodedBuffer) {
+        auto soundData = reinterpret_cast<const int16_t*>(decodedBuffer->constData());
+        auto numFrames = decodedBuffer->size() / sizeof(int16_t);
+        // now iterate and come up with average
+        if (numFrames > 0) {
+            for(int i = 0; i < numFrames; i++) {
+                loudness += (float) std::abs(soundData[i]);
+            }
+            loudness /= numFrames;
+        }
+    }
+    scriptedAvatar->setAudioLoudness(loudness);
+}
+
 void Agent::processAgentAvatarAudio() {
     auto recordingInterface = DependencyManager::get<RecordingScriptingInterface>();
     bool isPlayingRecording = recordingInterface->isPlaying();
@@ -611,6 +629,9 @@ void Agent::processAgentAvatarAudio() {
         audioPacket->seek(sizeof(quint16));
 
         if (silentFrame) {
+            // no matter what, the loudness should be set to 0
+            computeLoudness(nullptr);
+
             if (!_isListeningToAudioStream) {
                 // if we have a silent frame and we're not listening then just send nothing and break out of here
                 return;
@@ -618,7 +639,7 @@ void Agent::processAgentAvatarAudio() {
 
             // write the codec
             audioPacket->writeString(_selectedCodecName);
-            
+
             // write the number of silent samples so the audio-mixer can uphold timing
             audioPacket->writePrimitive(numAvailableSamples);
 
@@ -628,8 +649,9 @@ void Agent::processAgentAvatarAudio() {
             audioPacket->writePrimitive(headOrientation);
             audioPacket->writePrimitive(scriptedAvatar->getPosition());
             audioPacket->writePrimitive(glm::vec3(0));
+
         } else if (nextSoundOutput) {
-            
+
             // write the codec
             audioPacket->writeString(_selectedCodecName);
 
@@ -646,6 +668,8 @@ void Agent::processAgentAvatarAudio() {
             QByteArray encodedBuffer;
             if (_flushEncoder) {
                 encodeFrameOfZeros(encodedBuffer);
+                // loudness is 0
+                computeLoudness(nullptr);
             } else {
                 QByteArray decodedBuffer(reinterpret_cast<const char*>(nextSoundOutput), numAvailableSamples*sizeof(int16_t));
                 if (_encoder) {
@@ -654,10 +678,15 @@ void Agent::processAgentAvatarAudio() {
                 } else {
                     encodedBuffer = decodedBuffer;
                 }
+                computeLoudness(&decodedBuffer);
             }
             audioPacket->write(encodedBuffer.constData(), encodedBuffer.size());
         }
 
+        // we should never have both nextSoundOutput being null and silentFrame being false, but lets
+        // assert on it in case things above change in a bad way
+        assert(nextSoundOutput || silentFrame);
+
         // write audio packet to AudioMixer nodes
         auto nodeList = DependencyManager::get<NodeList>();
         nodeList->eachNode([this, &nodeList, &audioPacket](const SharedNodePointer& node) {
diff --git a/assignment-client/src/Agent.h b/assignment-client/src/Agent.h
index c9b1707101..7f04b4746f 100644
--- a/assignment-client/src/Agent.h
+++ b/assignment-client/src/Agent.h
@@ -82,6 +82,7 @@ private:
     void negotiateAudioFormat();
     void selectAudioFormat(const QString& selectedCodecName);
     void encodeFrameOfZeros(QByteArray& encodedZeros);
+    void computeLoudness(QByteArray* decodedBuffer);
 
     std::unique_ptr<ScriptEngine> _scriptEngine;
     EntityEditPacketSender _entityEditSender;

From 397eb89c143d2e746ca6e5ecd3980c210ea95e04 Mon Sep 17 00:00:00 2001
From: David Kelly <david@highfidelity.io>
Date: Wed, 22 Feb 2017 08:38:49 -0800
Subject: [PATCH 02/26] compiler warning - odd

---
 assignment-client/src/Agent.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp
index cd8c8189b2..2f91571d8b 100644
--- a/assignment-client/src/Agent.cpp
+++ b/assignment-client/src/Agent.cpp
@@ -568,7 +568,7 @@ void Agent::computeLoudness(QByteArray* decodedBuffer) {
     auto scriptedAvatar = DependencyManager::get<ScriptableAvatar>();
     if (decodedBuffer) {
         auto soundData = reinterpret_cast<const int16_t*>(decodedBuffer->constData());
-        auto numFrames = decodedBuffer->size() / sizeof(int16_t);
+        int numFrames = decodedBuffer->size() / sizeof(int16_t);
         // now iterate and come up with average
         if (numFrames > 0) {
             for(int i = 0; i < numFrames; i++) {

From d9a716bf3de4bc95ddd0d98f5ecd0e5a3487cc10 Mon Sep 17 00:00:00 2001
From: David Rowe <david@ctrlaltstudio.com>
Date: Fri, 24 Feb 2017 18:42:52 +1300
Subject: [PATCH 03/26] If someone else is grabbing entity you want to grab
 show their grab beam

---
 .../system/controllers/handControllerGrab.js  | 62 ++++++++++++++++---
 1 file changed, 54 insertions(+), 8 deletions(-)

diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js
index ea76490b7b..5064705f78 100644
--- a/scripts/system/controllers/handControllerGrab.js
+++ b/scripts/system/controllers/handControllerGrab.js
@@ -419,10 +419,10 @@ function entityIsGrabbedByOther(entityID) {
         }
         if (tag.slice(0, 5) == "grab-") {
             // we see a grab-*uuid* shaped tag and it's not ours, so someone else is grabbing it.
-            return true;
+            return tag.slice(5, 42);
         }
     }
-    return false;
+    return null;
 }
 
 function propsArePhysical(props) {
@@ -790,10 +790,10 @@ function MyController(hand) {
 
     // for visualizations
     this.overlayLine = null;
-
-    // for lights
-    this.overlayLine = null;
     this.searchSphere = null;
+    this.otherGrabbingLine = null;
+
+    this.otherGrabbingUUID = null;
 
     this.waitForTriggerRelease = false;
 
@@ -1050,6 +1050,29 @@ function MyController(hand) {
         }
     };
 
+    this.otherGrabbingLineOn = function(avatarPosition, entityPosition, color) {
+        if (this.otherGrabbingLine === null) {
+            var lineProperties = {
+                lineWidth: 5,
+                start: avatarPosition,
+                end: entityPosition,
+                color: color,
+                glow: 1.0,
+                ignoreRayIntersection: true,
+                drawInFront: true,
+                visible: true,
+                alpha: 1
+            };
+            this.otherGrabbingLine = Overlays.addOverlay("line3d", lineProperties);
+        } else {
+            Overlays.editOverlay(this.otherGrabbingLine, {
+                start: avatarPosition,
+                end: entityPosition,
+                color: color
+            });
+        }
+    };
+
     this.evalLightWorldTransform = function(modelPos, modelRot) {
 
         var MODEL_LIGHT_POSITION = {
@@ -1093,14 +1116,20 @@ function MyController(hand) {
         }
     };
 
-    this.turnOffVisualizations = function() {
+    this.otherGrabbingLineOff = function() {
+        if (this.otherGrabbingLine !== null) {
+            Overlays.deleteOverlay(this.otherGrabbingLine);
+        }
+        this.otherGrabbingLine = null;
+    };
 
+    this.turnOffVisualizations = function() {
         this.overlayLineOff();
         this.grabPointSphereOff();
         this.lineOff();
         this.searchSphereOff();
+        this.otherGrabbingLineOff();
         restore2DMode();
-
     };
 
     this.triggerPress = function(value) {
@@ -1504,7 +1533,8 @@ function MyController(hand) {
             return false;
         }
 
-        if (entityIsGrabbedByOther(entityID)) {
+        this.otherGrabbingUUID = entityIsGrabbedByOther(entityID);
+        if (this.otherGrabbingUUID !== null) {
             // don't distance grab something that is already grabbed.
             if (debug) {
                 print("distance grab is skipping '" + props.name + "': already grabbed by another.");
@@ -1683,6 +1713,7 @@ function MyController(hand) {
                 } else {
                     // potentialFarTriggerEntity = entity;
                 }
+                this.otherGrabbingLineOff();
             } else if (this.entityIsDistanceGrabbable(rayPickInfo.entityID, handPosition)) {
                 if (this.triggerSmoothedGrab() && !isEditing() && farGrabEnabled && farSearching) {
                     this.grabbedEntity = entity;
@@ -1692,7 +1723,21 @@ function MyController(hand) {
                 } else {
                     // potentialFarGrabEntity = entity;
                 }
+                this.otherGrabbingLineOff();
+            } else if (this.otherGrabbingUUID !== null) {
+                if (this.triggerSmoothedGrab() && !isEditing() && farGrabEnabled && farSearching) {
+                    var avatar = AvatarList.getAvatar(this.otherGrabbingUUID);
+                    var IN_FRONT_OF_AVATAR = { x: 0, y: 0, z: 0.2 };
+                    var startPosition = Vec3.sum(avatar.position, Vec3.multiplyQbyV(avatar.rotation, IN_FRONT_OF_AVATAR));
+                    this.otherGrabbingLineOn(startPosition, rayPickInfo.properties.position, COLORS_GRAB_DISTANCE_HOLD);
+                } else {
+                    this.otherGrabbingLineOff();
+                }
+            } else {
+                this.otherGrabbingLineOff();
             }
+        } else {
+            this.otherGrabbingLineOff();
         }
 
         this.updateEquipHaptics(potentialEquipHotspot, handPosition);
@@ -2268,6 +2313,7 @@ function MyController(hand) {
         this.lineOff();
         this.overlayLineOff();
         this.searchSphereOff();
+        this.otherGrabbingLineOff();
 
         this.dropGestureReset();
         this.clearEquipHaptics();

From a98824f48300179684c5e3817545cbf67de1bfc6 Mon Sep 17 00:00:00 2001
From: David Rowe <david@ctrlaltstudio.com>
Date: Sat, 25 Feb 2017 08:13:15 +1300
Subject: [PATCH 04/26] Code review

---
 scripts/system/controllers/handControllerGrab.js | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js
index 5064705f78..fe0fe19ae3 100644
--- a/scripts/system/controllers/handControllerGrab.js
+++ b/scripts/system/controllers/handControllerGrab.js
@@ -417,9 +417,11 @@ function entityIsGrabbedByOther(entityID) {
             // we see a grab-*uuid* shaped tag, but it's our tag, so that's okay.
             continue;
         }
-        if (tag.slice(0, 5) == "grab-") {
+        var GRAB_PREFIX_LENGTH = 5;
+        var UUID_LENGTH = 38;
+        if (tag.slice(0, GRAB_PREFIX_LENGTH) == "grab-") {
             // we see a grab-*uuid* shaped tag and it's not ours, so someone else is grabbing it.
-            return tag.slice(5, 42);
+            return tag.slice(GRAB_PREFIX_LENGTH, GRAB_PREFIX_LENGTH + UUID_LENGTH - 1);
         }
     }
     return null;

From f4a3627b767294133ef060ad5e74630721d29093 Mon Sep 17 00:00:00 2001
From: David Rowe <david@ctrlaltstudio.com>
Date: Tue, 28 Feb 2017 09:47:36 +1300
Subject: [PATCH 05/26] Make other avatar's grab beam start at more natural
 position

---
 scripts/system/controllers/handControllerGrab.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js
index fe0fe19ae3..1ab06927ac 100644
--- a/scripts/system/controllers/handControllerGrab.js
+++ b/scripts/system/controllers/handControllerGrab.js
@@ -1729,7 +1729,7 @@ function MyController(hand) {
             } else if (this.otherGrabbingUUID !== null) {
                 if (this.triggerSmoothedGrab() && !isEditing() && farGrabEnabled && farSearching) {
                     var avatar = AvatarList.getAvatar(this.otherGrabbingUUID);
-                    var IN_FRONT_OF_AVATAR = { x: 0, y: 0, z: 0.2 };
+                    var IN_FRONT_OF_AVATAR = { x: 0, y: 0.2, z: 0.4 };  // Up from hips and in front of avatar.
                     var startPosition = Vec3.sum(avatar.position, Vec3.multiplyQbyV(avatar.rotation, IN_FRONT_OF_AVATAR));
                     this.otherGrabbingLineOn(startPosition, rayPickInfo.properties.position, COLORS_GRAB_DISTANCE_HOLD);
                 } else {

From 08cae1d3f23c74b34a4bcb9349b0a8b68a3fc61e Mon Sep 17 00:00:00 2001
From: David Rowe <david@ctrlaltstudio.com>
Date: Tue, 28 Feb 2017 10:02:06 +1300
Subject: [PATCH 06/26] Make other avatar's grab beam finish at entity's
 centroid

---
 scripts/system/controllers/handControllerGrab.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js
index 1ab06927ac..cdfb33ee06 100644
--- a/scripts/system/controllers/handControllerGrab.js
+++ b/scripts/system/controllers/handControllerGrab.js
@@ -1731,7 +1731,11 @@ function MyController(hand) {
                     var avatar = AvatarList.getAvatar(this.otherGrabbingUUID);
                     var IN_FRONT_OF_AVATAR = { x: 0, y: 0.2, z: 0.4 };  // Up from hips and in front of avatar.
                     var startPosition = Vec3.sum(avatar.position, Vec3.multiplyQbyV(avatar.rotation, IN_FRONT_OF_AVATAR));
-                    this.otherGrabbingLineOn(startPosition, rayPickInfo.properties.position, COLORS_GRAB_DISTANCE_HOLD);
+                    var finishPisition = Vec3.sum(rayPickInfo.properties.position,  // Entity's centroid.
+                        Vec3.multiplyQbyV(rayPickInfo.properties.rotation ,
+                        Vec3.multiplyVbyV(rayPickInfo.properties.dimensions,
+                        Vec3.subtract(DEFAULT_REGISTRATION_POINT, rayPickInfo.properties.registrationPoint))));
+                    this.otherGrabbingLineOn(startPosition, finishPisition, COLORS_GRAB_DISTANCE_HOLD);
                 } else {
                     this.otherGrabbingLineOff();
                 }

From 6fcc096bcf57cd2d3429906e3ceec3e16dd5e140 Mon Sep 17 00:00:00 2001
From: David Rowe <david@ctrlaltstudio.com>
Date: Tue, 28 Feb 2017 10:14:38 +1300
Subject: [PATCH 07/26] Fix JavaScript error

---
 scripts/system/controllers/handControllerGrab.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js
index cdfb33ee06..3df7b91b6a 100644
--- a/scripts/system/controllers/handControllerGrab.js
+++ b/scripts/system/controllers/handControllerGrab.js
@@ -413,13 +413,13 @@ function entityIsGrabbedByOther(entityID) {
         var actionID = actionIDs[actionIndex];
         var actionArguments = Entities.getActionArguments(entityID, actionID);
         var tag = actionArguments.tag;
-        if (tag == getTag()) {
+        if (tag === getTag()) {
             // we see a grab-*uuid* shaped tag, but it's our tag, so that's okay.
             continue;
         }
         var GRAB_PREFIX_LENGTH = 5;
         var UUID_LENGTH = 38;
-        if (tag.slice(0, GRAB_PREFIX_LENGTH) == "grab-") {
+        if (tag && tag.slice(0, GRAB_PREFIX_LENGTH) == "grab-") {
             // we see a grab-*uuid* shaped tag and it's not ours, so someone else is grabbing it.
             return tag.slice(GRAB_PREFIX_LENGTH, GRAB_PREFIX_LENGTH + UUID_LENGTH - 1);
         }

From 2635657456a097bfc99d28eb44711f3b7a6b0797 Mon Sep 17 00:00:00 2001
From: David Kelly <david@highfidelity.io>
Date: Tue, 28 Feb 2017 13:24:25 -0700
Subject: [PATCH 08/26] zappoman's feedback, plus added const corrrectness to
 computeLoudness

---
 assignment-client/src/Agent.cpp |  4 +++-
 assignment-client/src/Agent.h   | 10 +++++-----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp
index 5211c3beaa..9c830ef391 100644
--- a/assignment-client/src/Agent.cpp
+++ b/assignment-client/src/Agent.cpp
@@ -380,6 +380,8 @@ void Agent::executeScript() {
         audioTransform.setTranslation(scriptedAvatar->getPosition());
         audioTransform.setRotation(headOrientation);
 
+        computeLoudness(&audio);
+
         QByteArray encodedBuffer;
         if (_encoder) {
             _encoder->encode(audio, encodedBuffer);
@@ -571,7 +573,7 @@ void Agent::encodeFrameOfZeros(QByteArray& encodedZeros) {
     }
 }
 
-void Agent::computeLoudness(QByteArray* decodedBuffer) {
+void Agent::computeLoudness(const QByteArray* decodedBuffer) {
     float loudness = 0.0f;
     auto scriptedAvatar = DependencyManager::get<ScriptableAvatar>();
     if (decodedBuffer) {
diff --git a/assignment-client/src/Agent.h b/assignment-client/src/Agent.h
index 7f04b4746f..ce7393011f 100644
--- a/assignment-client/src/Agent.h
+++ b/assignment-client/src/Agent.h
@@ -68,10 +68,10 @@ private slots:
     void handleAudioPacket(QSharedPointer<ReceivedMessage> message);
     void handleOctreePacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleJurisdictionPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
-    void handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message); 
+    void handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message);
 
     void nodeActivated(SharedNodePointer activatedNode);
-    
+
     void processAgentAvatar();
     void processAgentAvatarAudio();
 
@@ -82,7 +82,7 @@ private:
     void negotiateAudioFormat();
     void selectAudioFormat(const QString& selectedCodecName);
     void encodeFrameOfZeros(QByteArray& encodedZeros);
-    void computeLoudness(QByteArray* decodedBuffer);
+    void computeLoudness(const QByteArray* decodedBuffer);
 
     std::unique_ptr<ScriptEngine> _scriptEngine;
     EntityEditPacketSender _entityEditSender;
@@ -104,10 +104,10 @@ private:
     bool _isAvatar = false;
     QTimer* _avatarIdentityTimer = nullptr;
     QHash<QUuid, quint16> _outgoingScriptAudioSequenceNumbers;
-    
+
     CodecPluginPointer _codec;
     QString _selectedCodecName;
-    Encoder* _encoder { nullptr }; 
+    Encoder* _encoder { nullptr };
     QThread _avatarAudioTimerThread;
     bool _flushEncoder { false };
 };

From 01abb4bdb61ed9e6eff63da995cd74400f4cbad1 Mon Sep 17 00:00:00 2001
From: "Anthony J. Thibault" <tony@highfidelity.io>
Date: Fri, 24 Feb 2017 15:34:20 -0800
Subject: [PATCH 09/26] Exposed DebugDraw interface to Java Script

---
 libraries/script-engine/src/Quat.h           | 10 ++++
 libraries/script-engine/src/ScriptEngine.cpp |  3 +
 libraries/script-engine/src/Vec3.h           |  9 +++
 libraries/shared/src/DebugDraw.cpp           |  8 +--
 libraries/shared/src/DebugDraw.h             | 63 ++++++++++++++++----
 tools/jsdoc/plugins/hifi.js                  |  1 +
 6 files changed, 80 insertions(+), 14 deletions(-)

diff --git a/libraries/script-engine/src/Quat.h b/libraries/script-engine/src/Quat.h
index bb81f24586..b51f1cb47e 100644
--- a/libraries/script-engine/src/Quat.h
+++ b/libraries/script-engine/src/Quat.h
@@ -19,6 +19,16 @@
 #include <QObject>
 #include <QString>
 
+/**jsdoc
+ * A Quaternion
+ *
+ * @typedef Quat
+ * @property {float} x imaginary component i.
+ * @property {float} y imaginary component j.
+ * @property {float} z imaginary component k.
+ * @property {float} w real component.
+ */
+
 /// Scriptable interface a Quaternion helper class object. Used exclusively in the JavaScript API
 class Quat : public QObject {
     Q_OBJECT
diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp
index 83f2f5ccc0..2147374367 100644
--- a/libraries/script-engine/src/ScriptEngine.cpp
+++ b/libraries/script-engine/src/ScriptEngine.cpp
@@ -34,6 +34,7 @@
 #include <AudioConstants.h>
 #include <AudioEffectOptions.h>
 #include <AvatarData.h>
+#include <DebugDraw.h>
 #include <EntityScriptingInterface.h>
 #include <MessagesClient.h>
 #include <NetworkAccessManager.h>
@@ -630,6 +631,8 @@ void ScriptEngine::init() {
     registerGlobalObject("Tablet", DependencyManager::get<TabletScriptingInterface>().data());
     registerGlobalObject("Assets", &_assetScriptingInterface);
     registerGlobalObject("Resources", DependencyManager::get<ResourceScriptingInterface>().data());
+
+    registerGlobalObject("DebugDraw", &DebugDraw::getInstance());
 }
 
 void ScriptEngine::registerValue(const QString& valueName, QScriptValue value) {
diff --git a/libraries/script-engine/src/Vec3.h b/libraries/script-engine/src/Vec3.h
index 5f524eaf74..b3a3dc3035 100644
--- a/libraries/script-engine/src/Vec3.h
+++ b/libraries/script-engine/src/Vec3.h
@@ -37,6 +37,15 @@
  * @property {float} z Z-coordinate of the vector.
  */
 
+/**jsdoc
+ * A 4-dimensional vector.
+ *
+ * @typedef Vec4
+ * @property {float} x X-coordinate of the vector.
+ * @property {float} y Y-coordinate of the vector.
+ * @property {float} z Z-coordinate of the vector.
+ * @property {float} w W-coordinate of the vector.
+ */
 
 /// Scriptable interface a Vec3ernion helper class object. Used exclusively in the JavaScript API
 class Vec3 : public QObject {
diff --git a/libraries/shared/src/DebugDraw.cpp b/libraries/shared/src/DebugDraw.cpp
index 04759e6187..549dbb7293 100644
--- a/libraries/shared/src/DebugDraw.cpp
+++ b/libraries/shared/src/DebugDraw.cpp
@@ -28,19 +28,19 @@ void DebugDraw::drawRay(const glm::vec3& start, const glm::vec3& end, const glm:
     _rays.push_back(Ray(start, end, color));
 }
 
-void DebugDraw::addMarker(const std::string& key, const glm::quat& rotation, const glm::vec3& position, const glm::vec4& color) {
+void DebugDraw::addMarker(const QString& key, const glm::quat& rotation, const glm::vec3& position, const glm::vec4& color) {
     _markers[key] = MarkerInfo(rotation, position, color);
 }
 
-void DebugDraw::removeMarker(const std::string& key) {
+void DebugDraw::removeMarker(const QString& key) {
     _markers.erase(key);
 }
 
-void DebugDraw::addMyAvatarMarker(const std::string& key, const glm::quat& rotation, const glm::vec3& position, const glm::vec4& color) {
+void DebugDraw::addMyAvatarMarker(const QString& key, const glm::quat& rotation, const glm::vec3& position, const glm::vec4& color) {
     _myAvatarMarkers[key] = MarkerInfo(rotation, position, color);
 }
 
-void DebugDraw::removeMyAvatarMarker(const std::string& key) {
+void DebugDraw::removeMyAvatarMarker(const QString& key) {
     _myAvatarMarkers.erase(key);
 }
 
diff --git a/libraries/shared/src/DebugDraw.h b/libraries/shared/src/DebugDraw.h
index f77e281e06..ac7e8b3cbc 100644
--- a/libraries/shared/src/DebugDraw.h
+++ b/libraries/shared/src/DebugDraw.h
@@ -17,26 +17,69 @@
 #include <glm/glm.hpp>
 #include <glm/gtc/quaternion.hpp>
 
-class DebugDraw {
+#include <QObject>
+#include <QString>
+
+/**jsdoc
+ * Helper functions to render ephemeral debug markers and lines.
+ * DebugDraw markers and lines are only visible locally, they are not visible by other users.
+ * @namespace DebugDraw
+ */
+class DebugDraw : public QObject {
+    Q_OBJECT
 public:
     static DebugDraw& getInstance();
 
     DebugDraw();
     ~DebugDraw();
 
-    // world space line, drawn only once
-    void drawRay(const glm::vec3& start, const glm::vec3& end, const glm::vec4& color);
+    /**jsdoc
+     * Draws a line in world space, but it will only be visible for a single frame.
+     * @function DebugDraw.drawRay
+     * @param {Vec3} start - start position of line in world space.
+     * @param {Vec3} end - end position of line in world space.
+     * @param {Vec4} color - color of line, each component should be in the zero to one range.  x = red, y = blue, z = green, w = alpha.
+     */
+    Q_INVOKABLE void drawRay(const glm::vec3& start, const glm::vec3& end, const glm::vec4& color);
 
-    // world space maker, marker drawn every frame until it is removed.
-    void addMarker(const std::string& key, const glm::quat& rotation, const glm::vec3& position, const glm::vec4& color);
-    void removeMarker(const std::string& key);
+    /**jsdoc
+     * Adds a debug marker to the world. This marker will be drawn every frame until it is removed with DebugDraw.removeMarker.
+     * This can be called repeatedly to change the position of the marker.
+     * @function DebugDraw.addMarker
+     * @param {string} key - name to uniquely identify this marker, later used for DebugDraw.removeMarker.
+     * @param {Quat} rotation - start position of line in world space.
+     * @param {Vec3} position - position of the marker in world space.
+     * @param {Vec4} color - color of the marker.
+     */
+    Q_INVOKABLE void addMarker(const QString& key, const glm::quat& rotation, const glm::vec3& position, const glm::vec4& color);
 
-    // myAvatar relative marker, maker is drawn every frame until it is removed.
-    void addMyAvatarMarker(const std::string& key, const glm::quat& rotation, const glm::vec3& position, const glm::vec4& color);
-    void removeMyAvatarMarker(const std::string& key);
+    /**jsdoc
+     * Removes debug marker from the world.  Once a marker is removed, it will no longer be visible.
+     * @function DebugDraw.removeMarker
+     * @param {string} key - name of marker to remove.
+     */
+    Q_INVOKABLE void removeMarker(const QString& key);
+
+    /**jsdoc
+     * Adds a debug marker to the world, this marker will be drawn every frame until it is removed with DebugDraw.removeMyAvatarMarker.
+     * This can be called repeatedly to change the position of the marker.
+     * @function DebugDraw.addMyAvatarMarker
+     * @param {string} key - name to uniquely identify this marker, later used for DebugDraw.removeMyAvatarMarker.
+     * @param {Quat} rotation - start position of line in avatar space.
+     * @param {Vec3} position - position of the marker in avatar space.
+     * @param {Vec4} color - color of the marker.
+     */
+    Q_INVOKABLE void addMyAvatarMarker(const QString& key, const glm::quat& rotation, const glm::vec3& position, const glm::vec4& color);
+
+    /**jsdoc
+     * Removes debug marker from the world.  Once a marker is removed, it will no longer be visible
+     * @function DebugDraw.removeMyAvatarMarker
+     * @param {string} key - name of marker to remove.
+     */
+    Q_INVOKABLE void removeMyAvatarMarker(const QString& key);
 
     using MarkerInfo = std::tuple<glm::quat, glm::vec3, glm::vec4>;
-    using MarkerMap = std::unordered_map<std::string, MarkerInfo>;
+    using MarkerMap = std::map<QString, MarkerInfo>;
     using Ray = std::tuple<glm::vec3, glm::vec3, glm::vec4>;
     using Rays = std::vector<Ray>;
 
diff --git a/tools/jsdoc/plugins/hifi.js b/tools/jsdoc/plugins/hifi.js
index 8a6d2bf0f2..8be15c4103 100644
--- a/tools/jsdoc/plugins/hifi.js
+++ b/tools/jsdoc/plugins/hifi.js
@@ -21,6 +21,7 @@ exports.handlers = {
             '../../libraries/networking/src',
             '../../libraries/animation/src',
             '../../libraries/entities/src',
+            '../../libraries/shared/src'
         ];
         var exts = ['.h', '.cpp'];
 

From f41845710042c935d078ef0d04111ca99480053b Mon Sep 17 00:00:00 2001
From: "Anthony J. Thibault" <tony@highfidelity.io>
Date: Fri, 24 Feb 2017 15:50:41 -0800
Subject: [PATCH 10/26] Index finger touch support for the tablet-ui.

---
 .../system/controllers/handControllerGrab.js  | 215 ++++++++++++++----
 1 file changed, 169 insertions(+), 46 deletions(-)

diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js
index d313d1cfa1..5f854cd5ab 100644
--- a/scripts/system/controllers/handControllerGrab.js
+++ b/scripts/system/controllers/handControllerGrab.js
@@ -28,7 +28,7 @@ Script.include("/~/system/libraries/controllers.js");
 //
 
 var WANT_DEBUG = false;
-var WANT_DEBUG_STATE = false;
+var WANT_DEBUG_STATE = true;
 var WANT_DEBUG_SEARCH_NAME = null;
 
 var FORCE_IGNORE_IK = false;
@@ -46,6 +46,8 @@ var BUMPER_ON_VALUE = 0.5;
 
 var THUMB_ON_VALUE = 0.5;
 
+var USE_FINGER_AS_STYLUS = true;
+
 var HAPTIC_PULSE_STRENGTH = 1.0;
 var HAPTIC_PULSE_DURATION = 13.0;
 var HAPTIC_TEXTURE_STRENGTH = 0.1;
@@ -74,6 +76,10 @@ var WEB_TOUCH_Y_OFFSET = 0.05; // how far forward (or back with a negative numbe
 var WEB_TOUCH_TOO_CLOSE = 0.03; // if the stylus is pushed far though the web surface, don't consider it touching
 var WEB_TOUCH_Y_TOUCH_DEADZONE_SIZE = 0.01;
 
+var FINGER_TOUCH_Y_OFFSET = -0.02;
+var FINGER_TOUCH_MIN = -0.01 - FINGER_TOUCH_Y_OFFSET;
+var FINGER_TOUCH_MAX = 0.01 - FINGER_TOUCH_Y_OFFSET;
+
 //
 // distant manipulation
 //
@@ -252,20 +258,51 @@ CONTROLLER_STATE_MACHINE[STATE_FAR_TRIGGER] = {
     updateMethod: "farTrigger"
 };
 CONTROLLER_STATE_MACHINE[STATE_ENTITY_STYLUS_TOUCHING] = {
-    name: "entityTouching",
+    name: "entityStylusTouching",
+    enterMethod: "entityTouchingEnter",
+    exitMethod: "entityTouchingExit",
+    updateMethod: "entityTouching"
+};
+CONTROLLER_STATE_MACHINE[STATE_ENTITY_LASER_TOUCHING] = {
+    name: "entityLaserTouching",
     enterMethod: "entityTouchingEnter",
     exitMethod: "entityTouchingExit",
     updateMethod: "entityTouching"
 };
-CONTROLLER_STATE_MACHINE[STATE_ENTITY_LASER_TOUCHING] = CONTROLLER_STATE_MACHINE[STATE_ENTITY_STYLUS_TOUCHING];
 CONTROLLER_STATE_MACHINE[STATE_OVERLAY_STYLUS_TOUCHING] = {
-    name: "overlayTouching",
+    name: "overlayStylusTouching",
+    enterMethod: "overlayTouchingEnter",
+    exitMethod: "overlayTouchingExit",
+    updateMethod: "overlayTouching"
+};
+CONTROLLER_STATE_MACHINE[STATE_OVERLAY_LASER_TOUCHING] = {
+    name: "overlayLaserTouching",
     enterMethod: "overlayTouchingEnter",
     exitMethod: "overlayTouchingExit",
     updateMethod: "overlayTouching"
 };
-CONTROLLER_STATE_MACHINE[STATE_OVERLAY_LASER_TOUCHING] = CONTROLLER_STATE_MACHINE[STATE_OVERLAY_STYLUS_TOUCHING];
 
+function getFingerWorldLocation(hand) {
+    var fingerJointName = (hand === RIGHT_HAND) ? "RightHandIndex4" : "LeftHandIndex4";
+
+    var fingerJointIndex = MyAvatar.getJointIndex(fingerJointName);
+    var fingerPosition = MyAvatar.getAbsoluteJointTranslationInObjectFrame(fingerJointIndex);
+    var fingerRotation = MyAvatar.getAbsoluteJointRotationInObjectFrame(fingerJointIndex);
+    var worldFingerRotation = Quat.multiply(MyAvatar.orientation, fingerRotation);
+    var worldFingerPosition = Vec3.sum(MyAvatar.position, Vec3.multiplyQbyV(MyAvatar.orientation, fingerPosition));
+
+    // local y offset.
+    var localYOffset = Vec3.multiplyQbyV(worldFingerRotation, {x: 0, y: FINGER_TOUCH_Y_OFFSET, z: 0});
+
+    var offsetWorldFingerPosition = Vec3.sum(worldFingerPosition, localYOffset);
+
+    return {
+        position: offsetWorldFingerPosition,
+        orientation: worldFingerRotation,
+        rotation: worldFingerRotation,
+        valid: true
+    };
+}
 
 function distanceBetweenPointAndEntityBoundingBox(point, entityProps) {
     var entityXform = new Xform(entityProps.rotation, entityProps.position);
@@ -347,6 +384,7 @@ function handLaserIntersectItem(position, rotation, start) {
             direction: rayDirection,
             length: PICK_MAX_DISTANCE
         };
+
         return intersectionInfo;
     } else {
         // entity has been destroyed? or is no longer in cache
@@ -816,6 +854,8 @@ function MyController(hand) {
     this.tabletStabbedPos2D = null;
     this.tabletStabbedPos3D = null;
 
+    this.useFingerInsteadOfStylus = false;
+
     var _this = this;
 
     var suppressedIn2D = [STATE_OFF, STATE_SEARCHING];
@@ -829,10 +869,17 @@ function MyController(hand) {
         this.updateSmoothedTrigger();
         this.maybeScaleMyAvatar();
 
+        if (USE_FINGER_AS_STYLUS && MyAvatar.getJointIndex("LeftHandIndex4") !== -1) {
+            this.useFingerInsteadOfStylus = true;
+        } else {
+            this.useFingerInsteadOfStylus = false;
+        }
+
         if (this.ignoreInput()) {
 
             // Most hand input is disabled, because we are interacting with the 2d hud.
             // However, we still should check for collisions of the stylus with the web overlay.
+
             var controllerLocation = getControllerWorldLocation(this.handToController(), true);
             this.processStylus(controllerLocation.position);
 
@@ -1174,30 +1221,54 @@ function MyController(hand) {
     };
 
     this.processStylus = function(worldHandPosition) {
-        // see if the hand is near a tablet or web-entity
-        var candidateEntities = Entities.findEntities(worldHandPosition, WEB_DISPLAY_STYLUS_DISTANCE);
-        entityPropertiesCache.addEntities(candidateEntities);
-        var nearWeb = false;
-        for (var i = 0; i < candidateEntities.length; i++) {
-            var props = entityPropertiesCache.getProps(candidateEntities[i]);
-            if (props && (props.type == "Web" || this.isTablet(candidateEntities[i]))) {
-                nearWeb = true;
-                break;
+
+        var performRayTest = false;
+        if (this.useFingerInsteadOfStylus) {
+            this.hideStylus();
+            performRayTest = true;
+        } else {
+            var i;
+
+            // see if the hand is near a tablet or web-entity
+            var candidateEntities = Entities.findEntities(worldHandPosition, WEB_DISPLAY_STYLUS_DISTANCE);
+            entityPropertiesCache.addEntities(candidateEntities);
+            for (i = 0; i < candidateEntities.length; i++) {
+                var props = entityPropertiesCache.getProps(candidateEntities[i]);
+                if (props && (props.type == "Web" || this.isTablet(candidateEntities[i]))) {
+                    performRayTest = true;
+                    break;
+                }
+            }
+
+            if (!performRayTest) {
+                var candidateOverlays = Overlays.findOverlays(worldHandPosition, WEB_DISPLAY_STYLUS_DISTANCE);
+                for (i = 0; i < candidateOverlays.length; i++) {
+                    if (this.isTablet(candidateOverlays[i])) {
+                        performRayTest = true;
+                        break;
+                    }
+                }
+            }
+
+            if (performRayTest) {
+                this.showStylus();
+            } else {
+                this.hideStylus();
             }
         }
 
-        var candidateOverlays = Overlays.findOverlays(worldHandPosition, WEB_DISPLAY_STYLUS_DISTANCE);
-        for (var j = 0; j < candidateOverlays.length; j++) {
-            if (this.isTablet(candidateOverlays[j])) {
-                nearWeb = true;
+        if (performRayTest) {
+            var rayPickInfo = this.calcRayPickInfo(this.hand, this.useFingerInsteadOfStylus);
+            var max, min;
+            if (this.useFingerInsteadOfStylus) {
+                max = FINGER_TOUCH_MAX;
+                min = FINGER_TOUCH_MIN;
+            } else {
+                max = WEB_STYLUS_LENGTH / 2.0 + WEB_TOUCH_Y_OFFSET;
+                min = WEB_STYLUS_LENGTH / 2.0 + WEB_TOUCH_TOO_CLOSE;
             }
-        }
 
-        if (nearWeb) {
-            this.showStylus();
-            var rayPickInfo = this.calcRayPickInfo(this.hand);
-            if (rayPickInfo.distance < WEB_STYLUS_LENGTH / 2.0 + WEB_TOUCH_Y_OFFSET &&
-                rayPickInfo.distance > WEB_STYLUS_LENGTH / 2.0 + WEB_TOUCH_TOO_CLOSE) {
+            if (rayPickInfo.distance < max && rayPickInfo.distance > min) {
                 this.handleStylusOnHomeButton(rayPickInfo);
                 if (this.handleStylusOnWebEntity(rayPickInfo)) {
                     return;
@@ -1206,10 +1277,8 @@ function MyController(hand) {
                     return;
                 }
             } else {
-        this.homeButtonTouched = false;
-        }
-        } else {
-            this.hideStylus();
+                this.homeButtonTouched = false;
+            }
         }
     };
 
@@ -1324,10 +1393,17 @@ function MyController(hand) {
 
     // Performs ray pick test from the hand controller into the world
     // @param {number} which hand to use, RIGHT_HAND or LEFT_HAND
+    // @param {bool} if true use the world position/orientation of the index finger to cast the ray from.
     // @returns {object} returns object with two keys entityID and distance
     //
-    this.calcRayPickInfo = function(hand) {
-        var controllerLocation = getControllerWorldLocation(this.handToController(), true);
+    this.calcRayPickInfo = function(hand, useFingerInsteadOfController) {
+
+        var controllerLocation;
+        if (useFingerInsteadOfController) {
+            controllerLocation = getFingerWorldLocation(hand);
+        } else {
+            controllerLocation = getControllerWorldLocation(this.handToController(), true);
+        }
         var worldHandPosition = controllerLocation.position;
         var worldHandRotation = controllerLocation.orientation;
 
@@ -2783,8 +2859,13 @@ function MyController(hand) {
 
     this.entityTouchingEnter = function() {
         // test for intersection between controller laser and web entity plane.
-        var intersectInfo = handLaserIntersectEntity(this.grabbedThingID,
-                                                     getControllerWorldLocation(this.handToController(), true));
+        var controllerLocation;
+        if (this.useFingerInsteadOfStylus && this.state === STATE_ENTITY_STYLUS_TOUCHING) {
+            controllerLocation = getFingerWorldLocation(this.hand);
+        } else {
+            controllerLocation = getControllerWorldLocation(this.handToController(), true);
+        }
+        var intersectInfo = handLaserIntersectEntity(this.grabbedThingID, controllerLocation);
         if (intersectInfo) {
             var pointerEvent = {
                 type: "Press",
@@ -2820,8 +2901,13 @@ function MyController(hand) {
 
     this.entityTouchingExit = function() {
         // test for intersection between controller laser and web entity plane.
-        var intersectInfo = handLaserIntersectEntity(this.grabbedThingID,
-                                                     getControllerWorldLocation(this.handToController(), true));
+        var controllerLocation;
+        if (this.useFingerInsteadOfStylus && this.state === STATE_ENTITY_STYLUS_TOUCHING) {
+            controllerLocation = getFingerWorldLocation(this.hand);
+        } else {
+            controllerLocation = getControllerWorldLocation(this.handToController(), true);
+        }
+        var intersectInfo = handLaserIntersectEntity(this.grabbedThingID, controllerLocation);
         if (intersectInfo) {
             var pointerEvent;
             if (this.deadspotExpired) {
@@ -2861,12 +2947,26 @@ function MyController(hand) {
         }
 
         // test for intersection between controller laser and web entity plane.
-        var intersectInfo = handLaserIntersectEntity(this.grabbedThingID,
-                                                     getControllerWorldLocation(this.handToController(), true));
+        var controllerLocation;
+        if (this.useFingerInsteadOfStylus && this.state === STATE_ENTITY_STYLUS_TOUCHING) {
+            controllerLocation = getFingerWorldLocation(this.hand);
+        } else {
+            controllerLocation = getControllerWorldLocation(this.handToController(), true);
+        }
+        var intersectInfo = handLaserIntersectEntity(this.grabbedThingID, controllerLocation);
         if (intersectInfo) {
 
+            var max, min;
+            if (this.useFingerInsteadOfStylus && this.state === STATE_ENTITY_STYLUS_TOUCHING) {
+                max = FINGER_TOUCH_MAX;
+                min = FINGER_TOUCH_MIN;
+            } else {
+                max = WEB_STYLUS_LENGTH / 2.0 + WEB_TOUCH_Y_OFFSET;
+                min = WEB_STYLUS_LENGTH / 2.0 + WEB_TOUCH_TOO_CLOSE;
+            }
+
             if (this.state == STATE_ENTITY_STYLUS_TOUCHING &&
-                intersectInfo.distance > WEB_STYLUS_LENGTH / 2.0 + WEB_TOUCH_Y_OFFSET) {
+                intersectInfo.distance > max) {
                 this.setState(STATE_OFF, "pulled away from web entity");
                 return;
             }
@@ -2909,8 +3009,13 @@ function MyController(hand) {
 
     this.overlayTouchingEnter = function () {
         // Test for intersection between controller laser and Web overlay plane.
-        var intersectInfo =
-            handLaserIntersectOverlay(this.grabbedOverlay, getControllerWorldLocation(this.handToController(), true));
+        var controllerLocation;
+        if (this.useFingerInsteadOfStylus && this.state === STATE_OVERLAY_STYLUS_TOUCHING) {
+            controllerLocation = getFingerWorldLocation(this.hand);
+        } else {
+            controllerLocation = getControllerWorldLocation(this.handToController(), true);
+        }
+        var intersectInfo = handLaserIntersectOverlay(this.grabbedOverlay, controllerLocation);
         if (intersectInfo) {
             var pointerEvent = {
                 type: "Press",
@@ -2945,8 +3050,13 @@ function MyController(hand) {
 
     this.overlayTouchingExit = function () {
         // Test for intersection between controller laser and Web overlay plane.
-        var intersectInfo =
-            handLaserIntersectOverlay(this.grabbedOverlay, getControllerWorldLocation(this.handToController(), true));
+        var controllerLocation;
+        if (this.useFingerInsteadOfStylus && this.state === STATE_OVERLAY_STYLUS_TOUCHING) {
+            controllerLocation = getFingerWorldLocation(this.hand);
+        } else {
+            controllerLocation = getControllerWorldLocation(this.handToController(), true);
+        }
+        var intersectInfo = handLaserIntersectOverlay(this.grabbedOverlay, controllerLocation);
         if (intersectInfo) {
             var pointerEvent;
 
@@ -3003,12 +3113,25 @@ function MyController(hand) {
         }
 
         // Test for intersection between controller laser and Web overlay plane.
-        var intersectInfo =
-            handLaserIntersectOverlay(this.grabbedOverlay, getControllerWorldLocation(this.handToController(), true));
+        var controllerLocation;
+        if (this.useFingerInsteadOfStylus && this.state === STATE_OVERLAY_STYLUS_TOUCHING) {
+            controllerLocation = getFingerWorldLocation(this.hand);
+        } else {
+            controllerLocation = getControllerWorldLocation(this.handToController(), true);
+        }
+        var intersectInfo = handLaserIntersectOverlay(this.grabbedOverlay, controllerLocation);
         if (intersectInfo) {
 
-            if (this.state == STATE_OVERLAY_STYLUS_TOUCHING &&
-                intersectInfo.distance > WEB_STYLUS_LENGTH / 2.0 + WEB_TOUCH_Y_OFFSET + WEB_TOUCH_Y_TOUCH_DEADZONE_SIZE) {
+            var max, min;
+            if (this.useFingerInsteadOfStylus && this.state === STATE_OVERLAY_STYLUS_TOUCHING) {
+                max = FINGER_TOUCH_MAX;
+                min = FINGER_TOUCH_MIN;
+            } else {
+                max = WEB_STYLUS_LENGTH / 2.0 + WEB_TOUCH_Y_OFFSET + WEB_TOUCH_Y_TOUCH_DEADZONE_SIZE;
+                min = WEB_STYLUS_LENGTH / 2.0 + WEB_TOUCH_TOO_CLOSE;
+            }
+
+            if (this.state == STATE_OVERLAY_STYLUS_TOUCHING && intersectInfo.distance > max) {
                 this.grabbedThingID = null;
                 this.setState(STATE_OFF, "pulled away from overlay");
                 return;
@@ -3019,7 +3142,7 @@ function MyController(hand) {
 
             if (this.state == STATE_OVERLAY_STYLUS_TOUCHING &&
                 !this.tabletStabbed &&
-                intersectInfo.distance < WEB_STYLUS_LENGTH / 2.0 + WEB_TOUCH_TOO_CLOSE) {
+                intersectInfo.distance < min) {
                 // they've stabbed the tablet, don't send events until they pull back
                 this.tabletStabbed = true;
                 this.tabletStabbedPos2D = pos2D;

From d142c3d69bfd851b57f6901bf24440b3c0ec4697 Mon Sep 17 00:00:00 2001
From: "Anthony J. Thibault" <tony@highfidelity.io>
Date: Wed, 1 Mar 2017 13:29:24 -0800
Subject: [PATCH 11/26] eslint fix

---
 scripts/system/controllers/handControllerGrab.js | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js
index 5f854cd5ab..dbc2bebd31 100644
--- a/scripts/system/controllers/handControllerGrab.js
+++ b/scripts/system/controllers/handControllerGrab.js
@@ -2956,13 +2956,11 @@ function MyController(hand) {
         var intersectInfo = handLaserIntersectEntity(this.grabbedThingID, controllerLocation);
         if (intersectInfo) {
 
-            var max, min;
+            var max;
             if (this.useFingerInsteadOfStylus && this.state === STATE_ENTITY_STYLUS_TOUCHING) {
                 max = FINGER_TOUCH_MAX;
-                min = FINGER_TOUCH_MIN;
             } else {
                 max = WEB_STYLUS_LENGTH / 2.0 + WEB_TOUCH_Y_OFFSET;
-                min = WEB_STYLUS_LENGTH / 2.0 + WEB_TOUCH_TOO_CLOSE;
             }
 
             if (this.state == STATE_ENTITY_STYLUS_TOUCHING &&

From 465a9e2008f18e893b27e33e87e4c58c5c475bd2 Mon Sep 17 00:00:00 2001
From: "Anthony J. Thibault" <tony@highfidelity.io>
Date: Wed, 1 Mar 2017 14:31:04 -0800
Subject: [PATCH 12/26] Responsiveness improvement to tablet

PointerEvents sent from handControllerGrab.js to the Qml window are now direct connections, instead of queued.
This should reduce latency of laser and finger pressed onto the tablet.
---
 interface/src/ui/overlays/Web3DOverlay.cpp | 33 ++++++++++++++--------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/interface/src/ui/overlays/Web3DOverlay.cpp b/interface/src/ui/overlays/Web3DOverlay.cpp
index bfc37ccf60..7b9e075d64 100644
--- a/interface/src/ui/overlays/Web3DOverlay.cpp
+++ b/interface/src/ui/overlays/Web3DOverlay.cpp
@@ -198,18 +198,27 @@ void Web3DOverlay::render(RenderArgs* args) {
         _webSurface->getRootItem()->setProperty("scriptURL", _scriptURL);
         currentContext->makeCurrent(currentSurface);
 
+        auto selfOverlayID = getOverlayID();
+        std::weak_ptr<Web3DOverlay> weakSelf = std::dynamic_pointer_cast<Web3DOverlay>(qApp->getOverlays().getOverlay(selfOverlayID));
         auto forwardPointerEvent = [=](OverlayID overlayID, const PointerEvent& event) {
-            if (overlayID == getOverlayID()) {
-                handlePointerEvent(event);
+            auto self = weakSelf.lock();
+            if (!self) {
+                return;
+            }
+            if (overlayID == selfOverlayID) {
+                self->handlePointerEvent(event);
             }
         };
 
-        _mousePressConnection = connect(&(qApp->getOverlays()), &Overlays::mousePressOnOverlay, forwardPointerEvent);
-        _mouseReleaseConnection = connect(&(qApp->getOverlays()), &Overlays::mouseReleaseOnOverlay, forwardPointerEvent);
-        _mouseMoveConnection = connect(&(qApp->getOverlays()), &Overlays::mouseMoveOnOverlay, forwardPointerEvent);
-        _hoverLeaveConnection = connect(&(qApp->getOverlays()), &Overlays::hoverLeaveOverlay,
-            [=](OverlayID overlayID, const PointerEvent& event) {
-            if (this->_pressed && this->getOverlayID() == overlayID) {
+        _mousePressConnection = connect(&(qApp->getOverlays()), &Overlays::mousePressOnOverlay, this, forwardPointerEvent, Qt::DirectConnection);
+        _mouseReleaseConnection = connect(&(qApp->getOverlays()), &Overlays::mouseReleaseOnOverlay, this, forwardPointerEvent, Qt::DirectConnection);
+        _mouseMoveConnection = connect(&(qApp->getOverlays()), &Overlays::mouseMoveOnOverlay, this, forwardPointerEvent, Qt::DirectConnection);
+        _hoverLeaveConnection = connect(&(qApp->getOverlays()), &Overlays::hoverLeaveOverlay, this, [=](OverlayID overlayID, const PointerEvent& event) {
+            auto self = weakSelf.lock();
+            if (!self) {
+                return;
+            }
+            if (self->_pressed && overlayID == selfOverlayID) {
                 // If the user mouses off the overlay while the button is down, simulate a touch end.
                 QTouchEvent::TouchPoint point;
                 point.setId(event.getID());
@@ -222,12 +231,12 @@ void Web3DOverlay::render(RenderArgs* args) {
                 touchPoints.push_back(point);
                 QTouchEvent* touchEvent = new QTouchEvent(QEvent::TouchEnd, nullptr, Qt::NoModifier, Qt::TouchPointReleased,
                     touchPoints);
-                touchEvent->setWindow(_webSurface->getWindow());
+                touchEvent->setWindow(self->_webSurface->getWindow());
                 touchEvent->setDevice(&_touchDevice);
-                touchEvent->setTarget(_webSurface->getRootItem());
-                QCoreApplication::postEvent(_webSurface->getWindow(), touchEvent);
+                touchEvent->setTarget(self->_webSurface->getRootItem());
+                QCoreApplication::postEvent(self->_webSurface->getWindow(), touchEvent);
             }
-        });
+        }, Qt::DirectConnection);
 
         _emitScriptEventConnection = connect(this, &Web3DOverlay::scriptEventReceived, _webSurface.data(), &OffscreenQmlSurface::emitScriptEvent);
         _webEventReceivedConnection = connect(_webSurface.data(), &OffscreenQmlSurface::webEventReceived, this, &Web3DOverlay::webEventReceived);

From 81451191c1fef266e2b78b9549b1bb855710caea Mon Sep 17 00:00:00 2001
From: David Kelly <david@highfidelity.io>
Date: Wed, 1 Mar 2017 18:06:03 -0700
Subject: [PATCH 13/26] no need to get the ScriptableAvatar from
 DependencyManager every time

---
 assignment-client/src/Agent.cpp | 14 ++++++--------
 assignment-client/src/Agent.h   |  3 ++-
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp
index 9c830ef391..d20ef2e687 100644
--- a/assignment-client/src/Agent.cpp
+++ b/assignment-client/src/Agent.cpp
@@ -43,7 +43,6 @@
 #include <WebSocketServerClass.h>
 #include <EntityScriptingInterface.h> // TODO: consider moving to scriptengine.h
 
-#include "avatars/ScriptableAvatar.h"
 #include "entities/AssignmentParentFinder.h"
 #include "RecordingScriptingInterface.h"
 #include "AbstractAudioInterface.h"
@@ -380,7 +379,7 @@ void Agent::executeScript() {
         audioTransform.setTranslation(scriptedAvatar->getPosition());
         audioTransform.setRotation(headOrientation);
 
-        computeLoudness(&audio);
+        computeLoudness(&audio, scriptedAvatar);
 
         QByteArray encodedBuffer;
         if (_encoder) {
@@ -573,9 +572,8 @@ void Agent::encodeFrameOfZeros(QByteArray& encodedZeros) {
     }
 }
 
-void Agent::computeLoudness(const QByteArray* decodedBuffer) {
+void Agent::computeLoudness(const QByteArray* decodedBuffer, QSharedPointer<ScriptableAvatar> scriptableAvatar) {
     float loudness = 0.0f;
-    auto scriptedAvatar = DependencyManager::get<ScriptableAvatar>();
     if (decodedBuffer) {
         auto soundData = reinterpret_cast<const int16_t*>(decodedBuffer->constData());
         int numFrames = decodedBuffer->size() / sizeof(int16_t);
@@ -587,7 +585,7 @@ void Agent::computeLoudness(const QByteArray* decodedBuffer) {
             loudness /= numFrames;
         }
     }
-    scriptedAvatar->setAudioLoudness(loudness);
+    scriptableAvatar->setAudioLoudness(loudness);
 }
 
 void Agent::processAgentAvatarAudio() {
@@ -640,7 +638,7 @@ void Agent::processAgentAvatarAudio() {
 
         if (silentFrame) {
             // no matter what, the loudness should be set to 0
-            computeLoudness(nullptr);
+            computeLoudness(nullptr, scriptedAvatar);
 
             if (!_isListeningToAudioStream) {
                 // if we have a silent frame and we're not listening then just send nothing and break out of here
@@ -679,7 +677,7 @@ void Agent::processAgentAvatarAudio() {
             if (_flushEncoder) {
                 encodeFrameOfZeros(encodedBuffer);
                 // loudness is 0
-                computeLoudness(nullptr);
+                computeLoudness(nullptr, scriptedAvatar);
             } else {
                 QByteArray decodedBuffer(reinterpret_cast<const char*>(nextSoundOutput), numAvailableSamples*sizeof(int16_t));
                 if (_encoder) {
@@ -688,7 +686,7 @@ void Agent::processAgentAvatarAudio() {
                 } else {
                     encodedBuffer = decodedBuffer;
                 }
-                computeLoudness(&decodedBuffer);
+                computeLoudness(&decodedBuffer, scriptedAvatar);
             }
             audioPacket->write(encodedBuffer.constData(), encodedBuffer.size());
         }
diff --git a/assignment-client/src/Agent.h b/assignment-client/src/Agent.h
index ce7393011f..0ce7b71d5d 100644
--- a/assignment-client/src/Agent.h
+++ b/assignment-client/src/Agent.h
@@ -30,6 +30,7 @@
 #include <plugins/CodecPlugin.h>
 
 #include "MixedAudioStream.h"
+#include "avatars/ScriptableAvatar.h"
 
 class Agent : public ThreadedAssignment {
     Q_OBJECT
@@ -82,7 +83,7 @@ private:
     void negotiateAudioFormat();
     void selectAudioFormat(const QString& selectedCodecName);
     void encodeFrameOfZeros(QByteArray& encodedZeros);
-    void computeLoudness(const QByteArray* decodedBuffer);
+    void computeLoudness(const QByteArray* decodedBuffer, QSharedPointer<ScriptableAvatar>);
 
     std::unique_ptr<ScriptEngine> _scriptEngine;
     EntityEditPacketSender _entityEditSender;

From 4f03c06a948ce37283f07247e73ffc793fa24f2e Mon Sep 17 00:00:00 2001
From: "Anthony J. Thibault" <tony@highfidelity.io>
Date: Wed, 1 Mar 2017 18:07:53 -0800
Subject: [PATCH 14/26] Added General Preference to control stylus vs finger
 usage

By default the finger is preferred over the stylus.
---
 interface/src/Application.cpp                    | 6 ++++++
 interface/src/Application.h                      | 3 +++
 interface/src/ui/PreferencesDialog.cpp           | 6 ++++++
 scripts/system/controllers/handControllerGrab.js | 7 +++++--
 4 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp
index d48fe19a99..d37c1a259e 100644
--- a/interface/src/Application.cpp
+++ b/interface/src/Application.cpp
@@ -549,6 +549,7 @@ const float DEFAULT_DESKTOP_TABLET_SCALE_PERCENT = 75.0f;
 const bool DEFAULT_DESKTOP_TABLET_BECOMES_TOOLBAR = true;
 const bool DEFAULT_HMD_TABLET_BECOMES_TOOLBAR = false;
 const bool DEFAULT_TABLET_VISIBLE_TO_OTHERS = false;
+const bool DEFAULT_PREFER_AVATAR_FINGER_OVER_STYLUS = true;
 
 Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bool runServer, QString runServerPathOption) :
     QApplication(argc, argv),
@@ -572,6 +573,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
     _desktopTabletBecomesToolbarSetting("desktopTabletBecomesToolbar", DEFAULT_DESKTOP_TABLET_BECOMES_TOOLBAR),
     _hmdTabletBecomesToolbarSetting("hmdTabletBecomesToolbar", DEFAULT_HMD_TABLET_BECOMES_TOOLBAR),
     _tabletVisibleToOthersSetting("tabletVisibleToOthers", DEFAULT_TABLET_VISIBLE_TO_OTHERS),
+    _preferAvatarFingerOverStylusSetting("preferAvatarFingerOverStylus", DEFAULT_PREFER_AVATAR_FINGER_OVER_STYLUS),
     _constrainToolbarPosition("toolbar/constrainToolbarToCenterX", true),
     _scaleMirror(1.0f),
     _rotateMirror(0.0f),
@@ -2362,6 +2364,10 @@ void Application::setTabletVisibleToOthersSetting(bool value) {
     updateSystemTabletMode();
 }
 
+void Application::setPreferAvatarFingerOverStylus(bool value) {
+    _preferAvatarFingerOverStylusSetting.set(value);
+}
+
 void Application::setSettingConstrainToolbarPosition(bool setting) {
     _constrainToolbarPosition.set(setting);
     DependencyManager::get<OffscreenUi>()->setConstrainToolbarToCenterX(setting);
diff --git a/interface/src/Application.h b/interface/src/Application.h
index 13c1458aee..ec6d9b19f7 100644
--- a/interface/src/Application.h
+++ b/interface/src/Application.h
@@ -220,6 +220,8 @@ public:
     void setHmdTabletBecomesToolbarSetting(bool value);
     bool getTabletVisibleToOthersSetting() { return _tabletVisibleToOthersSetting.get(); }
     void setTabletVisibleToOthersSetting(bool value);
+    bool getPreferAvatarFingerOverStylus() { return _preferAvatarFingerOverStylusSetting.get(); }
+    void setPreferAvatarFingerOverStylus(bool value);
 
     float getSettingConstrainToolbarPosition() { return _constrainToolbarPosition.get(); }
     void setSettingConstrainToolbarPosition(bool setting);
@@ -565,6 +567,7 @@ private:
     Setting::Handle<bool> _desktopTabletBecomesToolbarSetting;
     Setting::Handle<bool> _hmdTabletBecomesToolbarSetting;
     Setting::Handle<bool> _tabletVisibleToOthersSetting;
+    Setting::Handle<bool> _preferAvatarFingerOverStylusSetting;
     Setting::Handle<bool> _constrainToolbarPosition;
 
     float _scaleMirror;
diff --git a/interface/src/ui/PreferencesDialog.cpp b/interface/src/ui/PreferencesDialog.cpp
index d291510556..c2caf91045 100644
--- a/interface/src/ui/PreferencesDialog.cpp
+++ b/interface/src/ui/PreferencesDialog.cpp
@@ -107,6 +107,12 @@ void setupPreferences() {
         auto setter = [](bool value) { qApp->setTabletVisibleToOthersSetting(value); };
         preferences->addPreference(new CheckPreference(UI_CATEGORY, "Tablet Is Visible To Others", getter, setter));
     }
+    {
+        auto getter = []()->bool { return qApp->getPreferAvatarFingerOverStylus(); };
+        auto setter = [](bool value) { qApp->setPreferAvatarFingerOverStylus(value); };
+        preferences->addPreference(new CheckPreference(UI_CATEGORY, "Prefer Avatar Finger Over Stylus", getter, setter));
+    }
+
     // Snapshots
     static const QString SNAPSHOTS { "Snapshots" };
     {
diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js
index dbc2bebd31..e52c3344e6 100644
--- a/scripts/system/controllers/handControllerGrab.js
+++ b/scripts/system/controllers/handControllerGrab.js
@@ -46,8 +46,6 @@ var BUMPER_ON_VALUE = 0.5;
 
 var THUMB_ON_VALUE = 0.5;
 
-var USE_FINGER_AS_STYLUS = true;
-
 var HAPTIC_PULSE_STRENGTH = 1.0;
 var HAPTIC_PULSE_DURATION = 13.0;
 var HAPTIC_TEXTURE_STRENGTH = 0.1;
@@ -869,6 +867,11 @@ function MyController(hand) {
         this.updateSmoothedTrigger();
         this.maybeScaleMyAvatar();
 
+        var DEFAULT_USE_FINGER_AS_STYLUS = true;
+        var USE_FINGER_AS_STYLUS = Settings.getValue("preferAvatarFingerOverStylus");
+        if (USE_FINGER_AS_STYLUS === "") {
+            USE_FINGER_AS_STYLUS = DEFAULT_USE_FINGER_AS_STYLUS;
+        }
         if (USE_FINGER_AS_STYLUS && MyAvatar.getJointIndex("LeftHandIndex4") !== -1) {
             this.useFingerInsteadOfStylus = true;
         } else {

From 29f263a29668e5901865c8d60aadf772183b4026 Mon Sep 17 00:00:00 2001
From: David Kelly <david@highfidelity.io>
Date: Thu, 2 Mar 2017 12:43:41 -0700
Subject: [PATCH 15/26] oh man, this was hard to find

---
 assignment-client/src/Agent.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp
index d20ef2e687..be23dcfa25 100644
--- a/assignment-client/src/Agent.cpp
+++ b/assignment-client/src/Agent.cpp
@@ -637,8 +637,6 @@ void Agent::processAgentAvatarAudio() {
         audioPacket->seek(sizeof(quint16));
 
         if (silentFrame) {
-            // no matter what, the loudness should be set to 0
-            computeLoudness(nullptr, scriptedAvatar);
 
             if (!_isListeningToAudioStream) {
                 // if we have a silent frame and we're not listening then just send nothing and break out of here
@@ -658,6 +656,8 @@ void Agent::processAgentAvatarAudio() {
             audioPacket->writePrimitive(scriptedAvatar->getPosition());
             audioPacket->writePrimitive(glm::vec3(0));
 
+            // no matter what, the loudness should be set to 0
+            computeLoudness(nullptr, scriptedAvatar);
         } else if (nextSoundOutput) {
 
             // write the codec

From 644e29a43d63bf8029e4f71bf4d6d576d693eaab Mon Sep 17 00:00:00 2001
From: "Anthony J. Thibault" <tony@highfidelity.io>
Date: Thu, 2 Mar 2017 14:15:05 -0800
Subject: [PATCH 16/26] disable WANT_STATE_DEBUG in handControllerGrab.js

---
 scripts/system/controllers/handControllerGrab.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js
index e52c3344e6..33254e9da2 100644
--- a/scripts/system/controllers/handControllerGrab.js
+++ b/scripts/system/controllers/handControllerGrab.js
@@ -28,7 +28,7 @@ Script.include("/~/system/libraries/controllers.js");
 //
 
 var WANT_DEBUG = false;
-var WANT_DEBUG_STATE = true;
+var WANT_DEBUG_STATE = false;
 var WANT_DEBUG_SEARCH_NAME = null;
 
 var FORCE_IGNORE_IK = false;

From bc256f3e8f01ce07b5740f00099e7dac59c1b4ef Mon Sep 17 00:00:00 2001
From: "Anthony J. Thibault" <tony@highfidelity.io>
Date: Thu, 2 Mar 2017 14:42:06 -0800
Subject: [PATCH 17/26] Fix for multi-threaded access to maps in DebugDraw.

---
 libraries/shared/src/DebugDraw.cpp | 30 ++++++++++++++++++++++++++++++
 libraries/shared/src/DebugDraw.h   | 10 ++++++----
 2 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/libraries/shared/src/DebugDraw.cpp b/libraries/shared/src/DebugDraw.cpp
index 549dbb7293..f17671da4d 100644
--- a/libraries/shared/src/DebugDraw.cpp
+++ b/libraries/shared/src/DebugDraw.cpp
@@ -10,6 +10,8 @@
 #include "DebugDraw.h"
 #include "SharedUtil.h"
 
+using Lock = std::unique_lock<std::mutex>;
+
 DebugDraw& DebugDraw::getInstance() {
     static DebugDraw* instance = globalInstance<DebugDraw>("com.highfidelity.DebugDraw");
     return *instance;
@@ -25,22 +27,50 @@ DebugDraw::~DebugDraw() {
 
 // world space line, drawn only once
 void DebugDraw::drawRay(const glm::vec3& start, const glm::vec3& end, const glm::vec4& color) {
+    Lock lock(_mapMutex);
     _rays.push_back(Ray(start, end, color));
 }
 
 void DebugDraw::addMarker(const QString& key, const glm::quat& rotation, const glm::vec3& position, const glm::vec4& color) {
+    Lock lock(_mapMutex);
     _markers[key] = MarkerInfo(rotation, position, color);
 }
 
 void DebugDraw::removeMarker(const QString& key) {
+    Lock lock(_mapMutex);
     _markers.erase(key);
 }
 
 void DebugDraw::addMyAvatarMarker(const QString& key, const glm::quat& rotation, const glm::vec3& position, const glm::vec4& color) {
+    Lock lock(_mapMutex);
     _myAvatarMarkers[key] = MarkerInfo(rotation, position, color);
 }
 
 void DebugDraw::removeMyAvatarMarker(const QString& key) {
+    Lock lock(_mapMutex);
     _myAvatarMarkers.erase(key);
 }
 
+//
+// accessors used by renderer
+//
+
+DebugDraw::MarkerMap DebugDraw::getMarkerMap() const {
+    Lock lock(_mapMutex);
+    return _markers;
+}
+
+DebugDraw::MarkerMap DebugDraw::getMyAvatarMarkerMap() const {
+    Lock lock(_mapMutex);
+    return _myAvatarMarkers;
+}
+
+DebugDraw::Rays DebugDraw::getRays() const {
+    Lock lock(_mapMutex);
+    return _rays;
+}
+
+void DebugDraw::clearRays() {
+    Lock lock(_mapMutex);
+    _rays.clear();
+}
diff --git a/libraries/shared/src/DebugDraw.h b/libraries/shared/src/DebugDraw.h
index ac7e8b3cbc..64327585fb 100644
--- a/libraries/shared/src/DebugDraw.h
+++ b/libraries/shared/src/DebugDraw.h
@@ -10,6 +10,7 @@
 #ifndef hifi_DebugDraw_h
 #define hifi_DebugDraw_h
 
+#include <mutex>
 #include <unordered_map>
 #include <tuple>
 #include <string>
@@ -87,16 +88,17 @@ public:
     // accessors used by renderer
     //
 
-    const MarkerMap& getMarkerMap() const { return _markers; }
-    const MarkerMap& getMyAvatarMarkerMap() const { return _myAvatarMarkers; }
+    MarkerMap getMarkerMap() const;
+    MarkerMap getMyAvatarMarkerMap() const;
     void updateMyAvatarPos(const glm::vec3& pos) { _myAvatarPos = pos; }
     const glm::vec3& getMyAvatarPos() const { return _myAvatarPos; }
     void updateMyAvatarRot(const glm::quat& rot) { _myAvatarRot = rot; }
     const glm::quat& getMyAvatarRot() const { return _myAvatarRot; }
-    const Rays getRays() const { return _rays; }
-    void clearRays() { _rays.clear(); }
+    Rays getRays() const;
+    void clearRays();
 
 protected:
+    mutable std::mutex _mapMutex;
     MarkerMap _markers;
     MarkerMap _myAvatarMarkers;
     glm::quat _myAvatarRot;

From e56f02d94f8c499acb50b099830d4dce05be0875 Mon Sep 17 00:00:00 2001
From: "Anthony J. Thibault" <tony@highfidelity.io>
Date: Fri, 3 Mar 2017 14:12:04 -0800
Subject: [PATCH 18/26] Changed default for preferFingerOverStylus to false.

---
 interface/src/Application.cpp                    | 2 +-
 scripts/system/controllers/handControllerGrab.js | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp
index 0335743360..56b93e57e4 100644
--- a/interface/src/Application.cpp
+++ b/interface/src/Application.cpp
@@ -549,7 +549,7 @@ const float DEFAULT_DESKTOP_TABLET_SCALE_PERCENT = 75.0f;
 const bool DEFAULT_DESKTOP_TABLET_BECOMES_TOOLBAR = true;
 const bool DEFAULT_HMD_TABLET_BECOMES_TOOLBAR = false;
 const bool DEFAULT_TABLET_VISIBLE_TO_OTHERS = false;
-const bool DEFAULT_PREFER_AVATAR_FINGER_OVER_STYLUS = true;
+const bool DEFAULT_PREFER_AVATAR_FINGER_OVER_STYLUS = false;
 
 Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bool runServer, QString runServerPathOption) :
     QApplication(argc, argv),
diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js
index db678044b5..f53512b7a7 100644
--- a/scripts/system/controllers/handControllerGrab.js
+++ b/scripts/system/controllers/handControllerGrab.js
@@ -896,7 +896,7 @@ function MyController(hand) {
         this.updateSmoothedTrigger();
         this.maybeScaleMyAvatar();
 
-        var DEFAULT_USE_FINGER_AS_STYLUS = true;
+        var DEFAULT_USE_FINGER_AS_STYLUS = false;
         var USE_FINGER_AS_STYLUS = Settings.getValue("preferAvatarFingerOverStylus");
         if (USE_FINGER_AS_STYLUS === "") {
             USE_FINGER_AS_STYLUS = DEFAULT_USE_FINGER_AS_STYLUS;

From 15d680d4f9fd1ed08ccddd7396ad4a18051fe5b1 Mon Sep 17 00:00:00 2001
From: howard-stearns <howard.stearns@gmail.com>
Date: Fri, 3 Mar 2017 16:56:12 -0800
Subject: [PATCH 19/26] preserve audio data during pal refresh

---
 scripts/system/pal.js | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/scripts/system/pal.js b/scripts/system/pal.js
index 70b2739c96..4914cbe34c 100644
--- a/scripts/system/pal.js
+++ b/scripts/system/pal.js
@@ -248,12 +248,16 @@ function fromQml(message) { // messages are {method, params}, like json-rpc. See
         }
         break;
     case 'refresh':
+        data = {};
+        ExtendedOverlay.some(function (overlay) { // capture the audio data
+            data[overlay.key] = overlay;
+        });
         removeOverlays();
         // If filter is specified from .qml instead of through settings, update the settings.
         if (message.params.filter !== undefined) {
             Settings.setValue('pal/filtered', !!message.params.filter);
         }
-        populateUserList(message.params.selected);
+        populateUserList(message.params.selected, data);
         UserActivityLogger.palAction("refresh", "");
         break;
     case 'displayNameUpdate':
@@ -285,7 +289,7 @@ function addAvatarNode(id) {
 }
 // Each open/refresh will capture a stable set of avatarsOfInterest, within the specified filter.
 var avatarsOfInterest = {};
-function populateUserList(selectData) {
+function populateUserList(selectData, oldAudioData) {
     var filter = Settings.getValue('pal/filtered') && {distance: Settings.getValue('pal/nearDistance')};
     var data = [], avatars = AvatarList.getAvatarIdentifiers();
     avatarsOfInterest = {};
@@ -317,12 +321,13 @@ function populateUserList(selectData) {
         if (id && filter && ((Math.abs(horizontal) > horizontalHalfAngle) || (Math.abs(vertical) > verticalHalfAngle))) {
             return;
         }
+        var oldAudio = oldAudioData && oldAudioData[id];
         var avatarPalDatum = {
             displayName: name,
             userName: '',
             sessionId: id || '',
-            audioLevel: 0.0,
-            avgAudioLevel: 0.0,
+            audioLevel: (oldAudio && oldAudio.audioLevel) || 0.0,
+            avgAudioLevel: (oldAudio && oldAudio.avgAudioLevel) || 0.0,
             admin: false,
             personalMute: !!id && Users.getPersonalMuteStatus(id), // expects proper boolean, not null
             ignore: !!id && Users.getIgnoreStatus(id) // ditto

From 0bababf1f5866dff5f73c4650a87648cb427b5a1 Mon Sep 17 00:00:00 2001
From: Ken Cooke <ken@highfidelity.io>
Date: Sat, 4 Mar 2017 14:53:07 -0800
Subject: [PATCH 20/26] Safe replacement of glm_mat4_mul() for unaligned
 arguments instead of __m128

---
 libraries/shared/src/GLMHelpers.h | 49 +++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/libraries/shared/src/GLMHelpers.h b/libraries/shared/src/GLMHelpers.h
index 4aac913768..609c3ab08b 100644
--- a/libraries/shared/src/GLMHelpers.h
+++ b/libraries/shared/src/GLMHelpers.h
@@ -245,4 +245,53 @@ inline bool isNaN(const glm::quat& value) { return isNaN(value.w) || isNaN(value
 
 glm::mat4 orthoInverse(const glm::mat4& m);
 
+//
+// Safe replacement of glm_mat4_mul() for unaligned arguments instead of __m128
+//
+inline void glm_mat4u_mul(const glm::mat4& m1, const glm::mat4& m2, glm::mat4& r) {
+
+#if GLM_ARCH & GLM_ARCH_SSE2_BIT
+    __m128 u0 = _mm_loadu_ps((float*)&m1[0][0]);
+    __m128 u1 = _mm_loadu_ps((float*)&m1[1][0]);
+    __m128 u2 = _mm_loadu_ps((float*)&m1[2][0]);
+    __m128 u3 = _mm_loadu_ps((float*)&m1[3][0]);
+
+    __m128 v0 = _mm_loadu_ps((float*)&m2[0][0]);
+    __m128 v1 = _mm_loadu_ps((float*)&m2[1][0]);
+    __m128 v2 = _mm_loadu_ps((float*)&m2[2][0]);
+    __m128 v3 = _mm_loadu_ps((float*)&m2[3][0]);
+
+    __m128 t0 = _mm_mul_ps(_mm_shuffle_ps(v0, v0, _MM_SHUFFLE(0,0,0,0)), u0);
+    __m128 t1 = _mm_mul_ps(_mm_shuffle_ps(v0, v0, _MM_SHUFFLE(1,1,1,1)), u1);
+    __m128 t2 = _mm_mul_ps(_mm_shuffle_ps(v0, v0, _MM_SHUFFLE(2,2,2,2)), u2);
+    __m128 t3 = _mm_mul_ps(_mm_shuffle_ps(v0, v0, _MM_SHUFFLE(3,3,3,3)), u3);
+    v0 = _mm_add_ps(_mm_add_ps(t0, t1), _mm_add_ps(t2, t3));
+
+    t0 = _mm_mul_ps(_mm_shuffle_ps(v1, v1, _MM_SHUFFLE(0,0,0,0)), u0);
+    t1 = _mm_mul_ps(_mm_shuffle_ps(v1, v1, _MM_SHUFFLE(1,1,1,1)), u1);
+    t2 = _mm_mul_ps(_mm_shuffle_ps(v1, v1, _MM_SHUFFLE(2,2,2,2)), u2);
+    t3 = _mm_mul_ps(_mm_shuffle_ps(v1, v1, _MM_SHUFFLE(3,3,3,3)), u3);
+    v1 = _mm_add_ps(_mm_add_ps(t0, t1), _mm_add_ps(t2, t3));
+
+    t0 = _mm_mul_ps(_mm_shuffle_ps(v2, v2, _MM_SHUFFLE(0,0,0,0)), u0);
+    t1 = _mm_mul_ps(_mm_shuffle_ps(v2, v2, _MM_SHUFFLE(1,1,1,1)), u1);
+    t2 = _mm_mul_ps(_mm_shuffle_ps(v2, v2, _MM_SHUFFLE(2,2,2,2)), u2);
+    t3 = _mm_mul_ps(_mm_shuffle_ps(v2, v2, _MM_SHUFFLE(3,3,3,3)), u3);
+    v2 = _mm_add_ps(_mm_add_ps(t0, t1), _mm_add_ps(t2, t3));
+
+    t0 = _mm_mul_ps(_mm_shuffle_ps(v3, v3, _MM_SHUFFLE(0,0,0,0)), u0);
+    t1 = _mm_mul_ps(_mm_shuffle_ps(v3, v3, _MM_SHUFFLE(1,1,1,1)), u1);
+    t2 = _mm_mul_ps(_mm_shuffle_ps(v3, v3, _MM_SHUFFLE(2,2,2,2)), u2);
+    t3 = _mm_mul_ps(_mm_shuffle_ps(v3, v3, _MM_SHUFFLE(3,3,3,3)), u3);
+    v3 = _mm_add_ps(_mm_add_ps(t0, t1), _mm_add_ps(t2, t3));
+
+    _mm_storeu_ps((float*)&r[0][0], v0);
+    _mm_storeu_ps((float*)&r[1][0], v1);
+    _mm_storeu_ps((float*)&r[2][0], v2);
+    _mm_storeu_ps((float*)&r[3][0], v3);
+#else
+    r = m1 * m2;
+#endif
+}
+
 #endif // hifi_GLMHelpers_h

From 117bba8b6a79473190220e15b77617f32eb7171b Mon Sep 17 00:00:00 2001
From: Ken Cooke <ken@highfidelity.io>
Date: Sat, 4 Mar 2017 15:17:09 -0800
Subject: [PATCH 21/26] redo unsafe optimization

---
 interface/src/avatar/CauterizedModel.cpp | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/interface/src/avatar/CauterizedModel.cpp b/interface/src/avatar/CauterizedModel.cpp
index 0c3d863649..7faf89dec5 100644
--- a/interface/src/avatar/CauterizedModel.cpp
+++ b/interface/src/avatar/CauterizedModel.cpp
@@ -110,13 +110,7 @@ void CauterizedModel::updateClusterMatrices() {
         for (int j = 0; j < mesh.clusters.size(); j++) {
             const FBXCluster& cluster = mesh.clusters.at(j);
             auto jointMatrix = _rig->getJointTransform(cluster.jointIndex);
-#if (GLM_ARCH & GLM_ARCH_SSE2) && !(defined Q_OS_MAC)
-            glm::mat4 out, inverseBindMatrix = cluster.inverseBindMatrix;
-            glm_mat4_mul((glm_vec4*)&jointMatrix, (glm_vec4*)&inverseBindMatrix, (glm_vec4*)&out);
-            state.clusterMatrices[j] = out;
-#else
-            state.clusterMatrices[j] = jointMatrix * cluster.inverseBindMatrix;
-#endif
+            glm_mat4u_mul(jointMatrix, cluster.inverseBindMatrix, state.clusterMatrices[j]);
         }
 
         // Once computed the cluster matrices, update the buffer(s)

From 46c5f961130a42e0434927f33d75470ce7b8323e Mon Sep 17 00:00:00 2001
From: Ken Cooke <ken@highfidelity.io>
Date: Sat, 4 Mar 2017 15:24:03 -0800
Subject: [PATCH 22/26] redo unsafe optimization

---
 interface/src/avatar/CauterizedModel.cpp | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/interface/src/avatar/CauterizedModel.cpp b/interface/src/avatar/CauterizedModel.cpp
index 7faf89dec5..1ca87a498a 100644
--- a/interface/src/avatar/CauterizedModel.cpp
+++ b/interface/src/avatar/CauterizedModel.cpp
@@ -143,13 +143,7 @@ void CauterizedModel::updateClusterMatrices() {
                 if (_cauterizeBoneSet.find(cluster.jointIndex) != _cauterizeBoneSet.end()) {
                     jointMatrix = cauterizeMatrix;
                 }
-#if (GLM_ARCH & GLM_ARCH_SSE2) && !(defined Q_OS_MAC)
-                glm::mat4 out, inverseBindMatrix = cluster.inverseBindMatrix;
-                glm_mat4_mul((glm_vec4*)&jointMatrix, (glm_vec4*)&inverseBindMatrix, (glm_vec4*)&out);
-                state.clusterMatrices[j] = out;
-#else
-                state.clusterMatrices[j] = jointMatrix * cluster.inverseBindMatrix;
-#endif
+                glm_mat4u_mul(jointMatrix, cluster.inverseBindMatrix, state.clusterMatrices[j]);
             }
 
             if (!_cauterizeBoneSet.empty() && (state.clusterMatrices.size() > 1)) {

From 50f92cb934034767416cc5946e886b1cdb4048e8 Mon Sep 17 00:00:00 2001
From: Ken Cooke <ken@highfidelity.io>
Date: Sat, 4 Mar 2017 15:29:23 -0800
Subject: [PATCH 23/26] redo unsafe optimization

---
 interface/src/avatar/SoftAttachmentModel.cpp | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/interface/src/avatar/SoftAttachmentModel.cpp b/interface/src/avatar/SoftAttachmentModel.cpp
index 6ed54afb27..0521f7a893 100644
--- a/interface/src/avatar/SoftAttachmentModel.cpp
+++ b/interface/src/avatar/SoftAttachmentModel.cpp
@@ -60,13 +60,7 @@ void SoftAttachmentModel::updateClusterMatrices() {
             } else {
                 jointMatrix = _rig->getJointTransform(cluster.jointIndex);
             }
-#if (GLM_ARCH & GLM_ARCH_SSE2) && !(defined Q_OS_MAC)
-            glm::mat4 out, inverseBindMatrix = cluster.inverseBindMatrix;
-            glm_mat4_mul((glm_vec4*)&jointMatrix, (glm_vec4*)&inverseBindMatrix, (glm_vec4*)&out);
-            state.clusterMatrices[j] = out;
-#else
-            state.clusterMatrices[j] = jointMatrix * cluster.inverseBindMatrix;
-#endif
+            glm_mat4u_mul(jointMatrix, cluster.inverseBindMatrix, state.clusterMatrices[j]);
         }
 
         // Once computed the cluster matrices, update the buffer(s)

From 44c1f8500dfafc8de6ee0338baaeee40790f1451 Mon Sep 17 00:00:00 2001
From: Ken Cooke <ken@highfidelity.io>
Date: Sat, 4 Mar 2017 15:55:53 -0800
Subject: [PATCH 24/26] redo unsafe optimization

---
 libraries/animation/src/AnimPose.cpp | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/libraries/animation/src/AnimPose.cpp b/libraries/animation/src/AnimPose.cpp
index 5638cacabc..e1c8528e0b 100644
--- a/libraries/animation/src/AnimPose.cpp
+++ b/libraries/animation/src/AnimPose.cpp
@@ -50,15 +50,9 @@ glm::vec3 AnimPose::xformVector(const glm::vec3& rhs) const {
 }
 
 AnimPose AnimPose::operator*(const AnimPose& rhs) const {
-#if (GLM_ARCH & GLM_ARCH_SSE2) && !(defined Q_OS_MAC)
     glm::mat4 result;
-    glm::mat4 lhsMat = *this;
-    glm::mat4 rhsMat = rhs;
-    glm_mat4_mul((glm_vec4*)&lhsMat, (glm_vec4*)&rhsMat, (glm_vec4*)&result);
+    glm_mat4u_mul(*this, rhs, result);
     return AnimPose(result);
-#else 
-    return AnimPose(static_cast<glm::mat4>(*this) * static_cast<glm::mat4>(rhs));
-#endif
 }
 
 AnimPose AnimPose::inverse() const {

From a5571bd49dbd4e71f236e95d8bf637aa9574f67c Mon Sep 17 00:00:00 2001
From: Ken Cooke <ken@highfidelity.io>
Date: Sat, 4 Mar 2017 16:02:39 -0800
Subject: [PATCH 25/26] redo unsafe optimization

---
 libraries/render-utils/src/Model.cpp | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/libraries/render-utils/src/Model.cpp b/libraries/render-utils/src/Model.cpp
index adfffe2614..d4de05c84d 100644
--- a/libraries/render-utils/src/Model.cpp
+++ b/libraries/render-utils/src/Model.cpp
@@ -1178,13 +1178,7 @@ void Model::updateClusterMatrices() {
         for (int j = 0; j < mesh.clusters.size(); j++) {
             const FBXCluster& cluster = mesh.clusters.at(j);
             auto jointMatrix = _rig->getJointTransform(cluster.jointIndex);
-#if (GLM_ARCH & GLM_ARCH_SSE2) && !(defined Q_OS_MAC)
-            glm::mat4 out, inverseBindMatrix = cluster.inverseBindMatrix;
-            glm_mat4_mul((glm_vec4*)&jointMatrix, (glm_vec4*)&inverseBindMatrix, (glm_vec4*)&out);
-            state.clusterMatrices[j] = out;
-#else
-            state.clusterMatrices[j] = jointMatrix * cluster.inverseBindMatrix;
-#endif
+            glm_mat4u_mul(jointMatrix, cluster.inverseBindMatrix, state.clusterMatrices[j]);
         }
 
         // Once computed the cluster matrices, update the buffer(s)

From 818425707b182343ecbd0739fcc2d9df5b5cd689 Mon Sep 17 00:00:00 2001
From: Ken Cooke <ken@highfidelity.io>
Date: Sat, 4 Mar 2017 16:14:31 -0800
Subject: [PATCH 26/26] update unit tests

---
 tests/shared/src/GLMHelpersTests.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/shared/src/GLMHelpersTests.cpp b/tests/shared/src/GLMHelpersTests.cpp
index 8d26d35c69..b4af4729a3 100644
--- a/tests/shared/src/GLMHelpersTests.cpp
+++ b/tests/shared/src/GLMHelpersTests.cpp
@@ -115,8 +115,8 @@ void GLMHelpersTests::testSimd() {
 
     a1 = a * b;
     b1 = b * a;
-    glm_mat4_mul((glm_vec4*)&a, (glm_vec4*)&b, (glm_vec4*)&a2);
-    glm_mat4_mul((glm_vec4*)&b, (glm_vec4*)&a, (glm_vec4*)&b2);
+    glm_mat4u_mul(a, b, a2);
+    glm_mat4u_mul(b, a, b2);
 
 
     {
@@ -133,8 +133,8 @@ void GLMHelpersTests::testSimd() {
         QElapsedTimer timer;
         timer.start();
         for (size_t i = 0; i < LOOPS; ++i) {
-            glm_mat4_mul((glm_vec4*)&a, (glm_vec4*)&b, (glm_vec4*)&a2);
-            glm_mat4_mul((glm_vec4*)&b, (glm_vec4*)&a, (glm_vec4*)&b2);
+            glm_mat4u_mul(a, b, a2);
+            glm_mat4u_mul(b, a, b2);
         }
         qDebug() << "SIMD " << timer.elapsed();
     }