merge from upstream

BUILD_LINUX.md

@@ -3,4 +3,4 @@ Please read the [general build guide](BUILD.md) for information on dependencies
 ###Qt5 Dependencies
 Should you choose not to install Qt5 via a package manager that handles dependencies for you, you may be missing some Qt5 dependencies. On Ubuntu, for example, the following additional packages are required:
 
-    libasound2 libxmu-dev libxi-dev freeglut3-dev libasound2-dev libjack0 libjack-dev libxrandr-dev libudev-dev
+    libasound2 libxmu-dev libxi-dev freeglut3-dev libasound2-dev libjack0 libjack-dev libxrandr-dev libudev-dev libssl-dev

interface/src/Application.cpp

@@ -89,6 +89,7 @@
 #include <OctalCode.h>
 #include <OctreeSceneStats.h>
 #include <OffscreenUi.h>
+#include <gl/OffscreenQmlSurfaceCache.h>
 #include <gl/OffscreenGLCanvas.h>
 #include <PathUtils.h>
 #include <PerfStat.h>
@@ -514,6 +515,7 @@ bool setupEssentials(int& argc, char** argv) {
     DependencyManager::set<InterfaceParentFinder>();
     DependencyManager::set<EntityTreeRenderer>(true, qApp, qApp);
     DependencyManager::set<CompositorHelper>();
+    DependencyManager::set<OffscreenQmlSurfaceCache>();
     return previousSessionCrashed;
 }
 
@@ -2003,6 +2005,10 @@ void Application::initializeUi() {
             showCursor(compositorHelper->getAllowMouseCapture() ? Qt::BlankCursor : Qt::ArrowCursor);
         }
     });
+
+    // Pre-create a couple of Web3D overlays to speed up tablet UI
+    auto offscreenSurfaceCache = DependencyManager::get<OffscreenQmlSurfaceCache>();
+    offscreenSurfaceCache->reserve(Web3DOverlay::QML, 2);
 }
 
 void Application::paintGL() {
@@ -3294,6 +3300,66 @@ bool Application::shouldPaint(float nsecsElapsed) {
     return true;
 }
 
+#ifdef Q_OS_WIN
+#include <Windows.h>
+#include <TCHAR.h>
+#include <pdh.h>
+#pragma comment(lib, "pdh.lib")
+
+static ULARGE_INTEGER lastCPU, lastSysCPU, lastUserCPU;
+static int numProcessors;
+static HANDLE self;
+static PDH_HQUERY cpuQuery;
+static PDH_HCOUNTER cpuTotal;
+
+void initCpuUsage() {
+    SYSTEM_INFO sysInfo;
+    FILETIME ftime, fsys, fuser;
+
+    GetSystemInfo(&sysInfo);
+    numProcessors = sysInfo.dwNumberOfProcessors;
+
+    GetSystemTimeAsFileTime(&ftime);
+    memcpy(&lastCPU, &ftime, sizeof(FILETIME));
+
+    self = GetCurrentProcess();
+    GetProcessTimes(self, &ftime, &ftime, &fsys, &fuser);
+    memcpy(&lastSysCPU, &fsys, sizeof(FILETIME));
+    memcpy(&lastUserCPU, &fuser, sizeof(FILETIME));
+
+    PdhOpenQuery(NULL, NULL, &cpuQuery);
+    PdhAddCounter(cpuQuery, "\\Processor(_Total)\\% Processor Time", NULL, &cpuTotal);
+    PdhCollectQueryData(cpuQuery);
+}
+
+void getCpuUsage(vec3& systemAndUser) {
+    FILETIME ftime, fsys, fuser;
+    ULARGE_INTEGER now, sys, user;
+
+    GetSystemTimeAsFileTime(&ftime);
+    memcpy(&now, &ftime, sizeof(FILETIME));
+
+    GetProcessTimes(self, &ftime, &ftime, &fsys, &fuser);
+    memcpy(&sys, &fsys, sizeof(FILETIME));
+    memcpy(&user, &fuser, sizeof(FILETIME));
+    systemAndUser.x = (sys.QuadPart - lastSysCPU.QuadPart);
+    systemAndUser.y = (user.QuadPart - lastUserCPU.QuadPart);
+    systemAndUser /= (float)(now.QuadPart - lastCPU.QuadPart);
+    systemAndUser /= (float)numProcessors;
+    systemAndUser *= 100.0f;
+    lastCPU = now;
+    lastUserCPU = user;
+    lastSysCPU = sys;
+
+    PDH_FMT_COUNTERVALUE counterVal;
+    PdhCollectQueryData(cpuQuery);
+    PdhGetFormattedCounterValue(cpuTotal, PDH_FMT_DOUBLE, NULL, &counterVal);
+    systemAndUser.z = (float)counterVal.doubleValue;
+}
+
+#endif
+
+
 void Application::idle(float nsecsElapsed) {
     PerformanceTimer perfTimer("idle");
 
@@ -3310,6 +3376,18 @@ void Application::idle(float nsecsElapsed) {
         connect(offscreenUi.data(), &OffscreenUi::showDesktop, this, &Application::showDesktop);
     }
 
+#ifdef Q_OS_WIN
+    static std::once_flag once;
+    std::call_once(once, [] {
+        initCpuUsage(); 
+    });
+
+    vec3 kernelUserAndSystem;
+    getCpuUsage(kernelUserAndSystem);
+    PROFILE_COUNTER(app, "cpuProcess", { { "system", kernelUserAndSystem.x }, { "user", kernelUserAndSystem.y } });
+    PROFILE_COUNTER(app, "cpuSystem", { { "system", kernelUserAndSystem.z } });
+#endif
+
     auto displayPlugin = getActiveDisplayPlugin();
     if (displayPlugin) {
         PROFILE_COUNTER_IF_CHANGED(app, "present", float, displayPlugin->presentRate());
@@ -3320,6 +3398,9 @@ void Application::idle(float nsecsElapsed) {
     PROFILE_COUNTER_IF_CHANGED(app, "currentProcessing", int, DependencyManager::get<StatTracker>()->getStat("Processing").toInt());
     PROFILE_COUNTER_IF_CHANGED(app, "pendingProcessing", int, DependencyManager::get<StatTracker>()->getStat("PendingProcessing").toInt());
 
+
+
+
     PROFILE_RANGE(app, __FUNCTION__);
 
     if (auto steamClient = PluginManager::getInstance()->getSteamClientPlugin()) {

interface/src/avatar/SoftAttachmentModel.cpp

@@ -37,7 +37,7 @@ int SoftAttachmentModel::getJointIndexOverride(int i) const {
 
 // virtual
 // use the _rigOverride matrices instead of the Model::_rig
-void SoftAttachmentModel::updateClusterMatrices(glm::vec3 modelPosition, glm::quat modelOrientation) {
+void SoftAttachmentModel::updateClusterMatrices() {
     if (!_needsUpdateClusterMatrices) {
         return;
     }
@@ -45,7 +45,6 @@ void SoftAttachmentModel::updateClusterMatrices(glm::vec3 modelPosition, glm::qu
 
     const FBXGeometry& geometry = getFBXGeometry();
 
-    glm::mat4 modelToWorld = glm::mat4_cast(modelOrientation);
     for (int i = 0; i < _meshStates.size(); i++) {
         MeshState& state = _meshStates[i];
         const FBXMesh& mesh = geometry.meshes.at(i);
@@ -53,7 +52,7 @@ void SoftAttachmentModel::updateClusterMatrices(glm::vec3 modelPosition, glm::qu
         for (int j = 0; j < mesh.clusters.size(); j++) {
             const FBXCluster& cluster = mesh.clusters.at(j);
 
-            // TODO: cache these look ups as an optimization
+            // TODO: cache these look-ups as an optimization
             int jointIndexOverride = getJointIndexOverride(cluster.jointIndex);
             glm::mat4 jointMatrix;
             if (jointIndexOverride >= 0 && jointIndexOverride < _rigOverride->getJointStateCount()) {
@@ -61,7 +60,13 @@ void SoftAttachmentModel::updateClusterMatrices(glm::vec3 modelPosition, glm::qu
             } else {
                 jointMatrix = _rig->getJointTransform(cluster.jointIndex);
             }
-            state.clusterMatrices[j] = modelToWorld * jointMatrix * cluster.inverseBindMatrix;
+#if GLM_ARCH & GLM_ARCH_SSE2
+            glm::mat4 out, inverseBindMatrix = cluster.inverseBindMatrix;
+            glm_mat4_mul((glm_vec4*)&jointMatrix, (glm_vec4*)&inverseBindMatrix, (glm_vec4*)&out);
+            state.clusterMatrices[j] = out;
+#else
+            state.clusterMatrices[j] = jointMatrix * cluster.inverseBindMatrix;
+#endif
         }
 
         // Once computed the cluster matrices, update the buffer(s)

interface/src/avatar/SoftAttachmentModel.h

@@ -31,7 +31,7 @@ public:
     ~SoftAttachmentModel();
 
     virtual void updateRig(float deltaTime, glm::mat4 parentTransform) override;
-    virtual void updateClusterMatrices(glm::vec3 modelPosition, glm::quat modelOrientation) override;
+    virtual void updateClusterMatrices() override;
 
 protected:
     int getJointIndexOverride(int i) const;

interface/src/ui/overlays/Web3DOverlay.cpp

@@ -28,14 +28,17 @@
 #include <RegisteredMetaTypes.h>
 #include <TabletScriptingInterface.h>
 #include <TextureCache.h>
+#include <AbstractViewStateInterface.h>
+#include <gl/OffscreenQmlSurface.h>
+#include <gl/OffscreenQmlSurfaceCache.h>
 
 static const float DPI = 30.47f;
 static const float INCHES_TO_METERS = 1.0f / 39.3701f;
 static const float METERS_TO_INCHES = 39.3701f;
 static const float OPAQUE_ALPHA_THRESHOLD = 0.99f;
 
-QString const Web3DOverlay::TYPE = "web3d";
+const QString Web3DOverlay::TYPE = "web3d";
-
+const QString Web3DOverlay::QML = "Web3DOverlay.qml";
 Web3DOverlay::Web3DOverlay() : _dpi(DPI) {
     _touchDevice.setCapabilities(QTouchDevice::Position);
     _touchDevice.setType(QTouchDevice::TouchScreen);
@@ -98,8 +101,9 @@ Web3DOverlay::~Web3DOverlay() {
         // is no longer valid
         auto webSurface = _webSurface;
         AbstractViewStateInterface::instance()->postLambdaEvent([webSurface] {
-            webSurface->deleteLater();
+            DependencyManager::get<OffscreenQmlSurfaceCache>()->release(QML, webSurface);
         });
+        _webSurface.reset();
     }
     auto geometryCache = DependencyManager::get<GeometryCache>();
     if (geometryCache) {
@@ -148,20 +152,15 @@ void Web3DOverlay::render(RenderArgs* args) {
     QOpenGLContext * currentContext = QOpenGLContext::currentContext();
     QSurface * currentSurface = currentContext->surface();
     if (!_webSurface) {
-        auto deleter = [](OffscreenQmlSurface* webSurface) {
+        _webSurface = DependencyManager::get<OffscreenQmlSurfaceCache>()->acquire(QML);
-            AbstractViewStateInterface::instance()->postLambdaEvent([webSurface] {
+        _webSurface->setMaxFps(10);
-                webSurface->deleteLater();
-            });
-        };
-        _webSurface = QSharedPointer<OffscreenQmlSurface>(new OffscreenQmlSurface(), deleter);
         // FIXME, the max FPS could be better managed by being dynamic (based on the number of current surfaces
         // and the current rendering load)
-        _webSurface->setMaxFps(10);
-        _webSurface->create(currentContext);
-
         loadSourceURL();
-
+        _webSurface->resume();
         _webSurface->resize(QSize(_resolution.x, _resolution.y));
+        _webSurface->getRootItem()->setProperty("url", _url);
+        _webSurface->getRootItem()->setProperty("scriptURL", _scriptURL);
         currentContext->makeCurrent(currentSurface);
 
         auto forwardPointerEvent = [=](unsigned int overlayID, const PointerEvent& event) {

interface/src/ui/overlays/Web3DOverlay.h

@@ -21,6 +21,7 @@ class Web3DOverlay : public Billboard3DOverlay {
     Q_OBJECT
 
 public:
+    static const QString QML;
     static QString const TYPE;
     virtual QString getType() const override { return TYPE; }
 

libraries/audio/src/AudioHRTF.cpp

@@ -392,8 +392,8 @@ static void crossfade_4x2(float* src, float* dst, const float* win, int numFrame
 // linear interpolation with gain
 static void interpolate(float* dst, const float* src0, const float* src1, float frac, float gain) {
 
-    __m128 f0 = _mm_set1_ps(HRTF_GAIN * gain * (1.0f - frac));
+    __m128 f0 = _mm_set1_ps(gain * (1.0f - frac));
-    __m128 f1 = _mm_set1_ps(HRTF_GAIN * gain * frac);
+    __m128 f1 = _mm_set1_ps(gain * frac);
 
     assert(HRTF_TAPS % 4 == 0);
 
@@ -861,6 +861,9 @@ void AudioHRTF::render(int16_t* input, float* output, int index, float azimuth,
     ALIGN32 float bqBuffer[4 * HRTF_BLOCK];                 // 4-channel (interleaved)
     int delay[4];                                           // 4-channel (interleaved)
 
+    // apply global and local gain adjustment
+    gain *= _gainAdjust;
+
     // to avoid polluting the cache, old filters are recomputed instead of stored
     setFilters(firCoef, bqCoef, delay, index, _azimuthState, _distanceState, _gainState, L0);
 

libraries/audio/src/AudioHRTF.h

@@ -44,6 +44,11 @@ public:
     //
     void renderSilent(int16_t* input, float* output, int index, float azimuth, float distance, float gain, int numFrames);
 
+    //
+    // HRTF local gain adjustment
+    //
+    void setGainAdjustment(float gain) { _gainAdjust = HRTF_GAIN * gain; };
+
 private:
     AudioHRTF(const AudioHRTF&) = delete;
     AudioHRTF& operator=(const AudioHRTF&) = delete;
@@ -73,6 +78,9 @@ private:
     float _distanceState = 0.0f;
     float _gainState = 0.0f;
 
+    // global and local gain adjustment
+    float _gainAdjust = HRTF_GAIN;
+
     bool _silentState = false;
 };
 

libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.h

@@ -114,10 +114,10 @@ protected:
     bool _vsyncEnabled { true };
     QThread* _presentThread{ nullptr };
     std::queue<gpu::FramePointer> _newFrameQueue;
-    RateCounter<> _droppedFrameRate;
+    RateCounter<100> _droppedFrameRate;
-    RateCounter<> _newFrameRate;
+    RateCounter<100> _newFrameRate;
-    RateCounter<> _presentRate;
+    RateCounter<100> _presentRate;
-    RateCounter<> _renderRate;
+    RateCounter<100> _renderRate;
 
     gpu::FramePointer _currentFrame;
     gpu::Frame* _lastFrame { nullptr };

libraries/gl/src/gl/OffscreenQmlSurfaceCache.cpp

@@ -0,0 +1,57 @@
+//
+//  Created by Bradley Austin Davis on 2017-01-11
+//  Copyright 2013-2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#include "OffscreenQmlSurfaceCache.h"
+
+#include <QtGui/QOpenGLContext>
+#include <QtQml/QQmlContext>
+
+#include <PathUtils.h>
+#include "OffscreenQmlSurface.h"
+
+OffscreenQmlSurfaceCache::OffscreenQmlSurfaceCache() {
+}
+
+OffscreenQmlSurfaceCache::~OffscreenQmlSurfaceCache() {
+    _cache.clear();
+}
+
+QSharedPointer<OffscreenQmlSurface> OffscreenQmlSurfaceCache::acquire(const QString& rootSource) {
+    auto& list = _cache[rootSource];
+    if (list.empty()) {
+        list.push_back(buildSurface(rootSource));
+    }
+    auto result = list.front();
+    list.pop_front();
+    return result;
+}
+
+void OffscreenQmlSurfaceCache::reserve(const QString& rootSource, int count) {
+    auto& list = _cache[rootSource];
+    while (list.size() < count) {
+        list.push_back(buildSurface(rootSource));
+    }
+}
+
+void OffscreenQmlSurfaceCache::release(const QString& rootSource, const QSharedPointer<OffscreenQmlSurface>& surface) {
+    surface->pause();
+    _cache[rootSource].push_back(surface);
+}
+
+QSharedPointer<OffscreenQmlSurface> OffscreenQmlSurfaceCache::buildSurface(const QString& rootSource) {
+    auto surface = QSharedPointer<OffscreenQmlSurface>(new OffscreenQmlSurface());
+    QOpenGLContext* currentContext = QOpenGLContext::currentContext();
+    QSurface* currentSurface = currentContext->surface();
+    surface->create(currentContext);
+    surface->setBaseUrl(QUrl::fromLocalFile(PathUtils::resourcesPath() + "/qml/"));
+    surface->load(rootSource);
+    surface->getRootContext()->setContextProperty("ApplicationInterface", qApp);
+    surface->resize(QSize(100, 100));
+    currentContext->makeCurrent(currentSurface);
+    return surface;
+}
+

libraries/gl/src/gl/OffscreenQmlSurfaceCache.h

@@ -0,0 +1,34 @@
+//
+//  Created by Bradley Austin Davis on 2017-01-11
+//  Copyright 2013-2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#pragma once
+#ifndef hifi_OffscreenQmlSurfaceCache_h
+#define hifi_OffscreenQmlSurfaceCahce_h
+
+#include "DependencyManager.h"
+
+#include <QtCore/QSharedPointer>
+
+class OffscreenQmlSurface;
+
+class OffscreenQmlSurfaceCache : public Dependency {
+    SINGLETON_DEPENDENCY
+
+public:
+    OffscreenQmlSurfaceCache();
+    virtual ~OffscreenQmlSurfaceCache();
+
+    QSharedPointer<OffscreenQmlSurface> acquire(const QString& rootSource);
+    void release(const QString& rootSource, const QSharedPointer<OffscreenQmlSurface>& surface);
+    void reserve(const QString& rootSource, int count = 1);
+
+private:
+    QSharedPointer<OffscreenQmlSurface> buildSurface(const QString& rootSource);
+    QHash<QString, QList<QSharedPointer<OffscreenQmlSurface>>> _cache;
+};
+
+#endif

libraries/networking/src/udt/SendQueue.cpp

@@ -88,8 +88,6 @@ SendQueue::SendQueue(Socket* socket, HifiSockAddr dest) :
     _socket(socket),
     _destination(dest)
 {
-    PROFILE_ASYNC_BEGIN(network, "SendQueue", _destination.toString());
-
     // setup psuedo-random number generation for all instances of SendQueue
     static std::random_device rd;
     static std::mt19937 generator(rd());
@@ -108,7 +106,6 @@ SendQueue::SendQueue(Socket* socket, HifiSockAddr dest) :
 }
 
 SendQueue::~SendQueue() {
-    PROFILE_ASYNC_END(network, "SendQueue", _destination.toString());
 }
 
 void SendQueue::queuePacket(std::unique_ptr<Packet> packet) {
@@ -229,8 +226,6 @@ void SendQueue::sendHandshake() {
     if (!_hasReceivedHandshakeACK) {
         // we haven't received a handshake ACK from the client, send another now
         auto handshakePacket = ControlPacket::create(ControlPacket::Handshake, sizeof(SequenceNumber));
-        PROFILE_ASYNC_BEGIN(network, "SendQueue:Handshake", _destination.toString());
-
         handshakePacket->writePrimitive(_initialSequenceNumber);
         _socket->writeBasePacket(*handshakePacket, _destination);
         
@@ -246,8 +241,6 @@ void SendQueue::handshakeACK(SequenceNumber initialSequenceNumber) {
             std::lock_guard<std::mutex> locker { _handshakeMutex };
             _hasReceivedHandshakeACK = true;
         }
-        PROFILE_ASYNC_END(network, "SendQueue:Handshake", _destination.toString());
-
         // Notify on the handshake ACK condition
         _handshakeACKCondition.notify_one();
     }

libraries/render-utils/src/MeshPartPayload.cpp

@@ -63,8 +63,7 @@ void MeshPartPayload::updateMeshPart(const std::shared_ptr<const model::Mesh>& d
 
 void MeshPartPayload::updateTransform(const Transform& transform, const Transform& offsetTransform) {
     _transform = transform;
-    _offsetTransform = offsetTransform;
+    Transform::mult(_drawTransform, _transform, offsetTransform);
-    Transform::mult(_drawTransform, _transform, _offsetTransform);
     _worldBound = _localBound;
     _worldBound.transform(_drawTransform);
 }
@@ -360,8 +359,8 @@ void ModelMeshPartPayload::notifyLocationChanged() {
 
 }
 
-void ModelMeshPartPayload::updateTransformForSkinnedMesh(const Transform& transform, const Transform& offsetTransform, const QVector<glm::mat4>& clusterMatrices) {
+void ModelMeshPartPayload::updateTransformForSkinnedMesh(const Transform& transform, const QVector<glm::mat4>& clusterMatrices) {
-    ModelMeshPartPayload::updateTransform(transform, offsetTransform);
+    _transform = transform;
 
     if (clusterMatrices.size() > 0) {
         _worldBound = AABox();
@@ -371,8 +370,10 @@ void ModelMeshPartPayload::updateTransformForSkinnedMesh(const Transform& transf
             _worldBound += clusterBound;
         }
 
-        // clusterMatrix has world rotation but not world translation.
+        _worldBound.transform(transform);
-        _worldBound.translate(transform.getTranslation());
+        if (clusterMatrices.size() == 1) {
+            _transform = _transform.worldTransform(Transform(clusterMatrices[0]));
+        }
     }
 }
 
@@ -520,23 +521,15 @@ void ModelMeshPartPayload::bindTransform(gpu::Batch& batch, const ShapePipeline:
     // Still relying on the raw data from the model
     const Model::MeshState& state = _model->_meshStates.at(_meshIndex);
 
-    Transform transform;
     if (state.clusterBuffer) {
         if (canCauterize && _model->getCauterizeBones()) {
             batch.setUniformBuffer(ShapePipeline::Slot::BUFFER::SKINNING, state.cauterizedClusterBuffer);
         } else {
             batch.setUniformBuffer(ShapePipeline::Slot::BUFFER::SKINNING, state.clusterBuffer);
         }
-    } else {
-        if (canCauterize && _model->getCauterizeBones()) {
-            transform = Transform(state.cauterizedClusterMatrices[0]);
-        } else {
-            transform = Transform(state.clusterMatrices[0]);
-        }
     }
 
-    transform.preTranslate(_transform.getTranslation());
+    batch.setModelTransform(_transform);
-    batch.setModelTransform(transform);
 }
 
 void ModelMeshPartPayload::startFade() {
@@ -569,8 +562,7 @@ void ModelMeshPartPayload::render(RenderArgs* args) const {
         return;
     }
 
-
+    // When an individual mesh parts like this finishes its fade, we will mark the Model as
-    // When an individual mesh parts like this finishes its fade, we will mark the Model as 
     // having render items that need updating
     bool nextIsFading = _isFading ? isStillFading() : false;
     bool startFading = !_isFading && !_hasFinishedFade && _hasStartedFade;
@@ -592,7 +584,7 @@ void ModelMeshPartPayload::render(RenderArgs* args) const {
 
     // Bind the model transform and the skinCLusterMatrices if needed
     bool canCauterize = args->_renderMode != RenderArgs::SHADOW_RENDER_MODE;
-    _model->updateClusterMatrices(_transform.getTranslation(), _transform.getRotation());
+    _model->updateClusterMatrices();
     bindTransform(batch, locations, canCauterize);
 
     //Bind the index buffer and vertex buffer and Blend shapes if needed

libraries/render-utils/src/MeshPartPayload.h

@@ -34,7 +34,7 @@ public:
     virtual void updateMeshPart(const std::shared_ptr<const model::Mesh>& drawMesh, int partIndex);
 
     virtual void notifyLocationChanged() {}
-    virtual void updateTransform(const Transform& transform, const Transform& offsetTransform);
+    void updateTransform(const Transform& transform, const Transform& offsetTransform);
 
     virtual void updateMaterial(model::MaterialPointer drawMaterial);
 
@@ -56,13 +56,12 @@ public:
     model::Mesh::Part _drawPart;
 
     std::shared_ptr<const model::Material> _drawMaterial;
-    
+
     model::Box _localBound;
     Transform _drawTransform;
     Transform _transform;
-    Transform _offsetTransform;
     mutable model::Box _worldBound;
-    
+
     bool _hasColorAttrib = false;
 
     size_t getVerticesCount() const { return _drawMesh ? _drawMesh->getNumVertices() : 0; }
@@ -86,7 +85,7 @@ public:
     typedef Payload::DataPointer Pointer;
 
     void notifyLocationChanged() override;
-    void updateTransformForSkinnedMesh(const Transform& transform, const Transform& offsetTransform, const QVector<glm::mat4>& clusterMatrices);
+    void updateTransformForSkinnedMesh(const Transform& transform, const QVector<glm::mat4>& clusterMatrices);
 
     // Entity fade in
     void startFade();
@@ -101,7 +100,7 @@ public:
 
     // ModelMeshPartPayload functions to perform render
     void bindMesh(gpu::Batch& batch) const override;
-    void bindTransform(gpu::Batch& batch, const render::ShapePipeline::LocationsPointer locations, bool canCauterize) const override;
+    void bindTransform(gpu::Batch& batch, const render::ShapePipeline::LocationsPointer locations, bool canCauterize = true) const override;
 
     void initCache();
 

libraries/render-utils/src/Model.cpp

@@ -92,7 +92,6 @@ Model::Model(RigPointer rig, QObject* parent) :
     _snapModelToRegistrationPoint(false),
     _snappedToRegistrationPoint(false),
     _cauterizeBones(false),
-    _pupilDilation(0.0f),
     _url(HTTP_INVALID_COM),
     _isVisible(true),
     _blendNumber(0),
@@ -234,19 +233,11 @@ void Model::updateRenderItems() {
 
         render::ScenePointer scene = AbstractViewStateInterface::instance()->getMain3DScene();
 
-        Transform modelMeshOffset;
-        if (self->isLoaded()) {
-            // includes model offset and unitScale.
-            modelMeshOffset = Transform(self->_rig->getGeometryToRigTransform());
-        } else {
-            modelMeshOffset.postTranslate(self->_offset);
-        }
-
         uint32_t deleteGeometryCounter = self->_deleteGeometryCounter;
 
         render::PendingChanges pendingChanges;
         foreach (auto itemID, self->_modelMeshRenderItems.keys()) {
-            pendingChanges.updateItem<ModelMeshPartPayload>(itemID, [modelMeshOffset, deleteGeometryCounter](ModelMeshPartPayload& data) {
+            pendingChanges.updateItem<ModelMeshPartPayload>(itemID, [deleteGeometryCounter](ModelMeshPartPayload& data) {
                 if (data._model && data._model->isLoaded()) {
                     if (!data.hasStartedFade() && data._model->getGeometry()->areTexturesLoaded()) {
                         data.startFade();
@@ -256,11 +247,11 @@ void Model::updateRenderItems() {
                         Transform modelTransform = data._model->getTransform();
 
                         // lazy update of cluster matrices used for rendering.  We need to update them here, so we can correctly update the bounding box.
-                        data._model->updateClusterMatrices(modelTransform.getTranslation(), modelTransform.getRotation());
+                        data._model->updateClusterMatrices();
 
                         // update the model transform and bounding box for this render item.
                         const Model::MeshState& state = data._model->_meshStates.at(data._meshIndex);
-                        data.updateTransformForSkinnedMesh(modelTransform, modelMeshOffset, state.clusterMatrices);
+                        data.updateTransformForSkinnedMesh(modelTransform, state.clusterMatrices);
                     }
                 }
             });
@@ -311,6 +302,7 @@ bool Model::updateGeometry() {
 
     if (_rig->jointStatesEmpty() && getFBXGeometry().joints.size() > 0) {
         initJointStates();
+        assert(_meshStates.empty());
 
         const FBXGeometry& fbxGeometry = getFBXGeometry();
         foreach (const FBXMesh& mesh, fbxGeometry.meshes) {
@@ -320,6 +312,9 @@ bool Model::updateGeometry() {
 
             _meshStates.append(state);
 
+            // Note: we add empty buffers for meshes that lack blendshapes so we can access the buffers by index
+            // later in ModelMeshPayload, however the vast majority of meshes will not have them.
+            // TODO? make _blendedVertexBuffers a map instead of vector and only add for meshes with blendshapes?
             auto buffer = std::make_shared<gpu::Buffer>();
             if (!mesh.blendshapes.isEmpty()) {
                 buffer->resize((mesh.vertices.size() + mesh.normals.size()) * sizeof(glm::vec3));
@@ -1168,7 +1163,7 @@ void Model::simulateInternal(float deltaTime) {
 }
 
 // virtual
-void Model::updateClusterMatrices(glm::vec3 modelPosition, glm::quat modelOrientation) {
+void Model::updateClusterMatrices() {
     PerformanceTimer perfTimer("Model::updateClusterMatrices");
 
     if (!_needsUpdateClusterMatrices || !isLoaded()) {
@@ -1183,7 +1178,6 @@ void Model::updateClusterMatrices(glm::vec3 modelPosition, glm::quat modelOrient
         glm::vec4(0.0f, 0.0f, 0.0f, 1.0f));
     auto cauterizeMatrix = _rig->getJointTransform(geometry.neckJointIndex) * zeroScale;
 
-    glm::mat4 modelToWorld = glm::mat4_cast(modelOrientation);
     for (int i = 0; i < _meshStates.size(); i++) {
         MeshState& state = _meshStates[i];
         const FBXMesh& mesh = geometry.meshes.at(i);
@@ -1191,12 +1185,11 @@ void Model::updateClusterMatrices(glm::vec3 modelPosition, glm::quat modelOrient
             const FBXCluster& cluster = mesh.clusters.at(j);
             auto jointMatrix = _rig->getJointTransform(cluster.jointIndex);
 #if GLM_ARCH & GLM_ARCH_SSE2
-            glm::mat4 temp, out, inverseBindMatrix = cluster.inverseBindMatrix;
+            glm::mat4 out, inverseBindMatrix = cluster.inverseBindMatrix;
-            glm_mat4_mul((glm_vec4*)&modelToWorld, (glm_vec4*)&jointMatrix, (glm_vec4*)&temp);
+            glm_mat4_mul((glm_vec4*)&jointMatrix, (glm_vec4*)&inverseBindMatrix, (glm_vec4*)&out);
-            glm_mat4_mul((glm_vec4*)&temp, (glm_vec4*)&inverseBindMatrix, (glm_vec4*)&out);
             state.clusterMatrices[j] = out;
-#else 
+#else
-            state.clusterMatrices[j] = modelToWorld * jointMatrix * cluster.inverseBindMatrix;
+            state.clusterMatrices[j] = jointMatrix * cluster.inverseBindMatrix;
 #endif
 
             // as an optimization, don't build cautrizedClusterMatrices if the boneSet is empty.
@@ -1204,7 +1197,13 @@ void Model::updateClusterMatrices(glm::vec3 modelPosition, glm::quat modelOrient
                 if (_cauterizeBoneSet.find(cluster.jointIndex) != _cauterizeBoneSet.end()) {
                     jointMatrix = cauterizeMatrix;
                 }
-                state.cauterizedClusterMatrices[j] = modelToWorld * jointMatrix * cluster.inverseBindMatrix;
+#if GLM_ARCH & GLM_ARCH_SSE2
+                glm::mat4 out, inverseBindMatrix = cluster.inverseBindMatrix;
+                glm_mat4_mul((glm_vec4*)&jointMatrix, (glm_vec4*)&inverseBindMatrix, (glm_vec4*)&out);
+                state.cauterizedClusterMatrices[j] = out;
+#else
+                state.cauterizedClusterMatrices[j] = jointMatrix * cluster.inverseBindMatrix;
+#endif
             }
         }
 

libraries/render-utils/src/Model.h

@@ -127,7 +127,7 @@ public:
     bool getSnapModelToRegistrationPoint() { return _snapModelToRegistrationPoint; }
 
     virtual void simulate(float deltaTime, bool fullUpdate = true);
-    virtual void updateClusterMatrices(glm::vec3 modelPosition, glm::quat modelOrientation);
+    virtual void updateClusterMatrices();
 
     /// Returns a reference to the shared geometry.
     const Geometry::Pointer& getGeometry() const { return _renderGeometry; }
@@ -255,9 +255,6 @@ signals:
 protected:
     bool addedToScene() const { return _addedToScene; }
 
-    void setPupilDilation(float dilation) { _pupilDilation = dilation; }
-    float getPupilDilation() const { return _pupilDilation; }
-
     void setBlendshapeCoefficients(const QVector<float>& coefficients) { _blendshapeCoefficients = coefficients; }
     const QVector<float>& getBlendshapeCoefficients() const { return _blendshapeCoefficients; }
 
@@ -354,7 +351,6 @@ protected:
     void deleteGeometry();
     void initJointTransforms();
 
-    float _pupilDilation;
     QVector<float> _blendshapeCoefficients;
 
     QUrl _url;

scripts/system/controllers/handControllerPointer.js

@@ -20,9 +20,10 @@
 // When partially squeezing over a HUD element, a laser or the reticle is shown where the active hand
 // controller beam intersects the HUD.
 
-var systemLaserOn = false;
+var activeTrigger;
-
+function isLaserOn() {
-
+    return activeTrigger.partial();
+}
 Script.include("../libraries/controllers.js");
 
 // UTILITIES -------------
@@ -124,12 +125,6 @@ function ignoreMouseActivity() {
     if (!Reticle.allowMouseCapture) {
         return true;
     }
-
-    // if the lasers are on, then reticle/mouse should be hidden and we can ignore it for seeking or depth updating
-    if (systemLaserOn) {
-        return true;
-    }
-
     var pos = Reticle.position;
     if (!pos || (pos.x == -1 && pos.y == -1)) {
         return true;
@@ -270,12 +265,6 @@ var ONE_MINUS_WEIGHTING = 1 - WEIGHTING;
 var AVERAGE_MOUSE_VELOCITY_FOR_SEEK_TO = 20;
 function isShakingMouse() { // True if the person is waving the mouse around trying to find it.
     var now = Date.now(), mouse = Reticle.position, isShaking = false;
-
-    // if the lasers are on, then we ignore mouse shaking
-    if (systemLaserOn) {
-        return false;
-    }
-
     if (lastIntegration && (lastIntegration !== now)) {
         var velocity = Vec3.length(Vec3.subtract(mouse, lastMouse)) / (now - lastIntegration);
         averageMouseVelocity = (ONE_MINUS_WEIGHTING * averageMouseVelocity) + (WEIGHTING * velocity);
@@ -290,16 +279,8 @@ function isShakingMouse() { // True if the person is waving the mouse around try
 var NON_LINEAR_DIVISOR = 2;
 var MINIMUM_SEEK_DISTANCE = 0.1;
 function updateSeeking(doNotStartSeeking) {
-    // if the lasers are on, then we never do seeking
+    if (!doNotStartSeeking && !isLaserOn() && (!Reticle.visible || isShakingMouse())) {
-    if (systemLaserOn) {
+        isSeeking = true;
-        isSeeking = false;
-        return;
-    }
-
-    if (!doNotStartSeeking && (!Reticle.visible || isShakingMouse())) {
-        if (!isSeeking) {
-            isSeeking = true;
-        }
     } // e.g., if we're about to turn it on with first movement.
     if (!isSeeking) {
         return;
@@ -340,7 +321,7 @@ function updateMouseActivity(isClick) {
         return;
     } // Bug: mouse clicks should keep going. Just not hand controller clicks
     handControllerLockOut.update(now);
-    Reticle.visible = !systemLaserOn;
+    Reticle.visible = true;
 }
 function expireMouseCursor(now) {
     if (!isPointingAtOverlay() && mouseCursorActivity.expired(now)) {
@@ -392,7 +373,7 @@ setupHandler(Controller.mouseDoublePressEvent, onMouseClick);
 
 var leftTrigger = new Trigger('left');
 var rightTrigger = new Trigger('right');
-var activeTrigger = rightTrigger;
+activeTrigger = rightTrigger;
 var activeHand = Controller.Standard.RightHand;
 var LEFT_HUD_LASER = 1;
 var RIGHT_HUD_LASER = 2;
@@ -498,6 +479,7 @@ var LASER_ALPHA = 0.5;
 var LASER_SEARCH_COLOR_XYZW = {x: 10 / 255, y: 10 / 255, z: 255 / 255, w: LASER_ALPHA};
 var LASER_TRIGGER_COLOR_XYZW = {x: 250 / 255, y: 10 / 255, z: 10 / 255, w: LASER_ALPHA};
 var SYSTEM_LASER_DIRECTION = {x: 0, y: 0, z: -1};
+var systemLaserOn = false;
 function clearSystemLaser() {
     if (!systemLaserOn) {
         return;
@@ -506,7 +488,6 @@ function clearSystemLaser() {
     HMD.disableExtraLaser();
     systemLaserOn = false;
     weMovedReticle = true;
-    Reticle.position = { x: -1, y: -1 };
 }
 function setColoredLaser() { // answer trigger state if lasers supported, else falsey.
     var color = (activeTrigger.state === 'full') ? LASER_TRIGGER_COLOR_XYZW : LASER_SEARCH_COLOR_XYZW;
@@ -590,12 +571,6 @@ function update() {
     Reticle.visible = false;
 }
 
-var BASIC_TIMER_INTERVAL = 20; // 20ms = 50hz good enough
-var updateIntervalTimer = Script.setInterval(function(){
-    update();
-}, BASIC_TIMER_INTERVAL);
-
-
 // Check periodically for changes to setup.
 var SETTINGS_CHANGE_RECHECK_INTERVAL = 10 * 1000; // 10 seconds
 function checkSettings() {
@@ -605,9 +580,10 @@ function checkSettings() {
 checkSettings();
 
 var settingsChecker = Script.setInterval(checkSettings, SETTINGS_CHANGE_RECHECK_INTERVAL);
+Script.update.connect(update);
 Script.scriptEnding.connect(function () {
     Script.clearInterval(settingsChecker);
-    Script.clearInterval(updateIntervalTimer);
+    Script.update.disconnect(update);
     OffscreenFlags.navigationFocusDisabled = false;
 });
 

scripts/system/pal.js

@@ -19,9 +19,13 @@ const UNSELECTED_TEXTURES = {"idle-D": Script.resolvePath("./assets/models/Avata
 const SELECTED_TEXTURES = { "idle-D": Script.resolvePath("./assets/models/Avatar-Overlay-v1.fbx/Avatar-Overlay-v1.fbm/avatar-overlay-selected.png"),
                             "idle-E": Script.resolvePath("./assets/models/Avatar-Overlay-v1.fbx/Avatar-Overlay-v1.fbm/avatar-overlay-selected.png")
 };
+const HOVER_TEXTURES = { "idle-D": Script.resolvePath("./assets/models/Avatar-Overlay-v1.fbx/Avatar-Overlay-v1.fbm/avatar-overlay-hover.png"),
+                         "idle-E": Script.resolvePath("./assets/models/Avatar-Overlay-v1.fbx/Avatar-Overlay-v1.fbm/avatar-overlay-hover.png")
+};
 
 const UNSELECTED_COLOR = { red: 0x1F, green: 0xC6, blue: 0xA6};
-const SELECTED_COLOR = {red: 0xf3, green: 0x91, blue: 0x29};
+const SELECTED_COLOR = {red: 0xF3, green: 0x91, blue: 0x29};
+const HOVER_COLOR = {red: 0xD0, green: 0xD0, blue: 0xD0}; // almost white for now
 
 (function() { // BEGIN LOCAL_SCOPE
 
@@ -46,6 +50,7 @@ function ExtendedOverlay(key, type, properties, selected, hasModel) { // A wrapp
     }
     this.key = key;
     this.selected = selected || false; // not undefined
+    this.hovering = false;
     this.activeOverlay = Overlays.addOverlay(type, properties); // We could use different overlays for (un)selected...
 }
 // Instance methods:
@@ -58,8 +63,8 @@ ExtendedOverlay.prototype.editOverlay = function (properties) { // change displa
     Overlays.editOverlay(this.activeOverlay, properties);
 };
 
-function color(selected, level) {
+function color(selected, hovering, level) {
-    var base = selected ? SELECTED_COLOR : UNSELECTED_COLOR;
+    var base = hovering ? HOVER_COLOR : selected ? SELECTED_COLOR : UNSELECTED_COLOR;
     function scale(component) {
         var delta = 0xFF - component;
         return component + (delta * level);
@@ -67,16 +72,38 @@ function color(selected, level) {
     return {red: scale(base.red), green: scale(base.green), blue: scale(base.blue)};
 }
 
-function textures(selected) {
+function textures(selected, hovering) {
-    return selected ? SELECTED_TEXTURES : UNSELECTED_TEXTURES;
+    return hovering ? HOVER_TEXTURES : selected ? SELECTED_TEXTURES : UNSELECTED_TEXTURES;
+}
+// so we don't have to traverse the overlays to get the last one
+var lastHoveringId = 0;
+ExtendedOverlay.prototype.hover = function (hovering) {
+    this.hovering = hovering;
+    if (this.key === lastHoveringId) {
+        if (hovering) {
+            return;
+        } else {
+            lastHoveringId = 0;
+        }
+    } 
+    this.editOverlay({color: color(this.selected, hovering, this.audioLevel)});
+    if (this.model) {
+        this.model.editOverlay({textures: textures(this.selected, hovering)});
+    }
+    if (hovering) {
+        // un-hover the last hovering overlay
+        if (lastHoveringId && lastHoveringId != this.key) {
+            ExtendedOverlay.get(lastHoveringId).hover(false);
+        }
+        lastHoveringId = this.key;
+    }
 }
-
 ExtendedOverlay.prototype.select = function (selected) {
     if (this.selected === selected) {
         return;
     }
     
-    this.editOverlay({color: color(selected, this.audioLevel)});
+    this.editOverlay({color: color(selected, this.hovering, this.audioLevel)});
     if (this.model) {
         this.model.editOverlay({textures: textures(selected)});
     }
@@ -98,14 +125,25 @@ ExtendedOverlay.some = function (iterator) { // Bails early as soon as iterator
         }
     }
 };
-ExtendedOverlay.applyPickRay = function (pickRay, cb) { // cb(overlay) on the one overlay intersected by pickRay, if any.
+ExtendedOverlay.unHover = function () { // calls hover(false) on lastHoveringId (if any)
+    if (lastHoveringId) {
+        ExtendedOverlay.get(lastHoveringId).hover(false);
+    }
+};
+
+// hit(overlay) on the one overlay intersected by pickRay, if any.
+// noHit() if no ExtendedOverlay was intersected (helps with hover)
+ExtendedOverlay.applyPickRay = function (pickRay, hit, noHit) {
     var pickedOverlay = Overlays.findRayIntersection(pickRay); // Depends on nearer coverOverlays to extend closer to us than farther ones.
     if (!pickedOverlay.intersects) {
+        if (noHit) {
+            return noHit();
+        }
         return;
     }
     ExtendedOverlay.some(function (overlay) { // See if pickedOverlay is one of ours.
         if ((overlay.activeOverlay) === pickedOverlay.overlayID) {
-            cb(overlay);
+            hit(overlay);
             return true;
         }
     });
@@ -209,7 +247,7 @@ function addAvatarNode(id) {
          drawInFront: true, 
          solid: true, 
          alpha: 0.8, 
-         color: color(selected, 0.0),
+         color: color(selected, false, 0.0),
          ignoreRayIntersection: false}, selected, true);
 }
 function populateUserList() {
@@ -293,7 +331,7 @@ function updateOverlays() {
 
         overlay.ping = pingPong;
         overlay.editOverlay({
-            color: color(ExtendedOverlay.isSelected(id), overlay.audioLevel),
+            color: color(ExtendedOverlay.isSelected(id), overlay.hovering, overlay.audioLevel),
             position: target,
             dimensions: 0.032 * distance 
         });
@@ -317,6 +355,7 @@ function updateOverlays() {
 }
 function removeOverlays() {
     selectedIds = [];
+    lastHoveringId = 0;
     HighlightedEntity.clearOverlays();
     ExtendedOverlay.some(function (overlay) { overlay.deleteOverlay(); });
 }
@@ -338,9 +377,54 @@ function handleMouseEvent(mousePressEvent) { // handleClick if we get one.
     }
     handleClick(Camera.computePickRay(mousePressEvent.x, mousePressEvent.y));
 }
+function handleMouseMove(pickRay) { // given the pickRay, just do the hover logic
+    ExtendedOverlay.applyPickRay(pickRay, function (overlay) {
+        overlay.hover(true);
+    }, function () {
+        ExtendedOverlay.unHover();
+    });
+}
+
+// handy global to keep track of which hand is the mouse (if any)
+var currentHandPressed = 0;
+const TRIGGER_CLICK_THRESHOLD = 0.85;
+const TRIGGER_PRESS_THRESHOLD = 0.05;
+
+function handleMouseMoveEvent(event) { // find out which overlay (if any) is over the mouse position
+    if (HMD.active) {
+        if (currentHandPressed != 0) {
+            pickRay = controllerComputePickRay(currentHandPressed);
+        } else {
+            // nothing should hover, so
+            ExtendedOverlay.unHover();
+            return;
+        }
+    } else {
+        pickRay = Camera.computePickRay(event.x, event.y);
+    }
+    handleMouseMove(pickRay);
+}
+function handleTriggerPressed(hand, value) {
+    // The idea is if you press one trigger, it is the one 
+    // we will consider the mouse.  Even if the other is pressed,
+    // we ignore it until this one is no longer pressed.
+    isPressed = value > TRIGGER_PRESS_THRESHOLD;
+    if (currentHandPressed == 0) {
+        currentHandPressed = isPressed ? hand : 0;
+        return;
+    }
+    if (currentHandPressed == hand) { 
+        currentHandPressed = isPressed ? hand : 0;
+        return;
+    } 
+    // otherwise, the other hand is still triggered
+    // so do nothing.
+}
+
 // We get mouseMoveEvents from the handControllers, via handControllerPointer.
 // But we don't get mousePressEvents.
 var triggerMapping = Controller.newMapping(Script.resolvePath('') + '-click');
+var triggerPressMapping = Controller.newMapping(Script.resolvePath('') + '-press');
 function controllerComputePickRay(hand) {
     var controllerPose = getControllerWorldLocation(hand, true);
     if (controllerPose.valid) {
@@ -349,15 +433,21 @@ function controllerComputePickRay(hand) {
 }
 function makeClickHandler(hand) {
     return function (clicked) {
-        if (clicked > 0.85) {
+        if (clicked > TRIGGER_CLICK_THRESHOLD) {
             var pickRay = controllerComputePickRay(hand);
             handleClick(pickRay);
         }
     };
 }
+function makePressHandler(hand) {
+    return function (value) {
+        handleTriggerPressed(hand, value);
+    }
+}
 triggerMapping.from(Controller.Standard.RTClick).peek().to(makeClickHandler(Controller.Standard.RightHand));
 triggerMapping.from(Controller.Standard.LTClick).peek().to(makeClickHandler(Controller.Standard.LeftHand));
-
+triggerPressMapping.from(Controller.Standard.RT).peek().to(makePressHandler(Controller.Standard.RightHand));
+triggerPressMapping.from(Controller.Standard.LT).peek().to(makePressHandler(Controller.Standard.LeftHand));
 //
 // Manage the connection between the button and the window.
 //
@@ -372,9 +462,11 @@ function off() {
     if (isWired) { // It is not ok to disconnect these twice, hence guard.
         Script.update.disconnect(updateOverlays);
         Controller.mousePressEvent.disconnect(handleMouseEvent);
+        Controller.mouseMoveEvent.disconnect(handleMouseMoveEvent);
         isWired = false;
     }
     triggerMapping.disable(); // It's ok if we disable twice.
+    triggerPressMapping.disable(); // see above
     removeOverlays();
     Users.requestsDomainListData = false;
 }
@@ -386,7 +478,9 @@ function onClicked() {
         isWired = true;
         Script.update.connect(updateOverlays);
         Controller.mousePressEvent.connect(handleMouseEvent);
+        Controller.mouseMoveEvent.connect(handleMouseMoveEvent);
         triggerMapping.enable();
+        triggerPressMapping.enable();
     } else {
         off();
     }