diff --git a/BUILD_LINUX.md b/BUILD_LINUX.md index 55cd647f66..34ba41894c 100644 --- a/BUILD_LINUX.md +++ b/BUILD_LINUX.md @@ -3,4 +3,4 @@ Please read the [general build guide](BUILD.md) for information on dependencies ###Qt5 Dependencies Should you choose not to install Qt5 via a package manager that handles dependencies for you, you may be missing some Qt5 dependencies. On Ubuntu, for example, the following additional packages are required: - libasound2 libxmu-dev libxi-dev freeglut3-dev libasound2-dev libjack0 libjack-dev libxrandr-dev libudev-dev + libasound2 libxmu-dev libxi-dev freeglut3-dev libasound2-dev libjack0 libjack-dev libxrandr-dev libudev-dev libssl-dev diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index a97f79e2a7..0e5c52bef9 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -89,6 +89,7 @@ #include #include #include +#include #include #include #include @@ -514,6 +515,7 @@ bool setupEssentials(int& argc, char** argv) { DependencyManager::set(); DependencyManager::set(true, qApp, qApp); DependencyManager::set(); + DependencyManager::set(); return previousSessionCrashed; } @@ -2003,6 +2005,10 @@ void Application::initializeUi() { showCursor(compositorHelper->getAllowMouseCapture() ? Qt::BlankCursor : Qt::ArrowCursor); } }); + + // Pre-create a couple of Web3D overlays to speed up tablet UI + auto offscreenSurfaceCache = DependencyManager::get(); + offscreenSurfaceCache->reserve(Web3DOverlay::QML, 2); } void Application::paintGL() { @@ -3294,6 +3300,66 @@ bool Application::shouldPaint(float nsecsElapsed) { return true; } +#ifdef Q_OS_WIN +#include +#include +#include +#pragma comment(lib, "pdh.lib") + +static ULARGE_INTEGER lastCPU, lastSysCPU, lastUserCPU; +static int numProcessors; +static HANDLE self; +static PDH_HQUERY cpuQuery; +static PDH_HCOUNTER cpuTotal; + +void initCpuUsage() { + SYSTEM_INFO sysInfo; + FILETIME ftime, fsys, fuser; + + GetSystemInfo(&sysInfo); + numProcessors = sysInfo.dwNumberOfProcessors; + + GetSystemTimeAsFileTime(&ftime); + memcpy(&lastCPU, &ftime, sizeof(FILETIME)); + + self = GetCurrentProcess(); + GetProcessTimes(self, &ftime, &ftime, &fsys, &fuser); + memcpy(&lastSysCPU, &fsys, sizeof(FILETIME)); + memcpy(&lastUserCPU, &fuser, sizeof(FILETIME)); + + PdhOpenQuery(NULL, NULL, &cpuQuery); + PdhAddCounter(cpuQuery, "\\Processor(_Total)\\% Processor Time", NULL, &cpuTotal); + PdhCollectQueryData(cpuQuery); +} + +void getCpuUsage(vec3& systemAndUser) { + FILETIME ftime, fsys, fuser; + ULARGE_INTEGER now, sys, user; + + GetSystemTimeAsFileTime(&ftime); + memcpy(&now, &ftime, sizeof(FILETIME)); + + GetProcessTimes(self, &ftime, &ftime, &fsys, &fuser); + memcpy(&sys, &fsys, sizeof(FILETIME)); + memcpy(&user, &fuser, sizeof(FILETIME)); + systemAndUser.x = (sys.QuadPart - lastSysCPU.QuadPart); + systemAndUser.y = (user.QuadPart - lastUserCPU.QuadPart); + systemAndUser /= (float)(now.QuadPart - lastCPU.QuadPart); + systemAndUser /= (float)numProcessors; + systemAndUser *= 100.0f; + lastCPU = now; + lastUserCPU = user; + lastSysCPU = sys; + + PDH_FMT_COUNTERVALUE counterVal; + PdhCollectQueryData(cpuQuery); + PdhGetFormattedCounterValue(cpuTotal, PDH_FMT_DOUBLE, NULL, &counterVal); + systemAndUser.z = (float)counterVal.doubleValue; +} + +#endif + + void Application::idle(float nsecsElapsed) { PerformanceTimer perfTimer("idle"); @@ -3310,6 +3376,18 @@ void Application::idle(float nsecsElapsed) { connect(offscreenUi.data(), &OffscreenUi::showDesktop, this, &Application::showDesktop); } +#ifdef Q_OS_WIN + static std::once_flag once; + std::call_once(once, [] { + initCpuUsage(); + }); + + vec3 kernelUserAndSystem; + getCpuUsage(kernelUserAndSystem); + PROFILE_COUNTER(app, "cpuProcess", { { "system", kernelUserAndSystem.x }, { "user", kernelUserAndSystem.y } }); + PROFILE_COUNTER(app, "cpuSystem", { { "system", kernelUserAndSystem.z } }); +#endif + auto displayPlugin = getActiveDisplayPlugin(); if (displayPlugin) { PROFILE_COUNTER_IF_CHANGED(app, "present", float, displayPlugin->presentRate()); @@ -3320,6 +3398,9 @@ void Application::idle(float nsecsElapsed) { PROFILE_COUNTER_IF_CHANGED(app, "currentProcessing", int, DependencyManager::get()->getStat("Processing").toInt()); PROFILE_COUNTER_IF_CHANGED(app, "pendingProcessing", int, DependencyManager::get()->getStat("PendingProcessing").toInt()); + + + PROFILE_RANGE(app, __FUNCTION__); if (auto steamClient = PluginManager::getInstance()->getSteamClientPlugin()) { diff --git a/interface/src/avatar/SoftAttachmentModel.cpp b/interface/src/avatar/SoftAttachmentModel.cpp index c2e8dab2a1..6351495598 100644 --- a/interface/src/avatar/SoftAttachmentModel.cpp +++ b/interface/src/avatar/SoftAttachmentModel.cpp @@ -37,7 +37,7 @@ int SoftAttachmentModel::getJointIndexOverride(int i) const { // virtual // use the _rigOverride matrices instead of the Model::_rig -void SoftAttachmentModel::updateClusterMatrices(glm::vec3 modelPosition, glm::quat modelOrientation) { +void SoftAttachmentModel::updateClusterMatrices() { if (!_needsUpdateClusterMatrices) { return; } @@ -45,7 +45,6 @@ void SoftAttachmentModel::updateClusterMatrices(glm::vec3 modelPosition, glm::qu const FBXGeometry& geometry = getFBXGeometry(); - glm::mat4 modelToWorld = glm::mat4_cast(modelOrientation); for (int i = 0; i < _meshStates.size(); i++) { MeshState& state = _meshStates[i]; const FBXMesh& mesh = geometry.meshes.at(i); @@ -53,7 +52,7 @@ void SoftAttachmentModel::updateClusterMatrices(glm::vec3 modelPosition, glm::qu for (int j = 0; j < mesh.clusters.size(); j++) { const FBXCluster& cluster = mesh.clusters.at(j); - // TODO: cache these look ups as an optimization + // TODO: cache these look-ups as an optimization int jointIndexOverride = getJointIndexOverride(cluster.jointIndex); glm::mat4 jointMatrix; if (jointIndexOverride >= 0 && jointIndexOverride < _rigOverride->getJointStateCount()) { @@ -61,7 +60,13 @@ void SoftAttachmentModel::updateClusterMatrices(glm::vec3 modelPosition, glm::qu } else { jointMatrix = _rig->getJointTransform(cluster.jointIndex); } - state.clusterMatrices[j] = modelToWorld * jointMatrix * cluster.inverseBindMatrix; +#if GLM_ARCH & GLM_ARCH_SSE2 + glm::mat4 out, inverseBindMatrix = cluster.inverseBindMatrix; + glm_mat4_mul((glm_vec4*)&jointMatrix, (glm_vec4*)&inverseBindMatrix, (glm_vec4*)&out); + state.clusterMatrices[j] = out; +#else + state.clusterMatrices[j] = jointMatrix * cluster.inverseBindMatrix; +#endif } // Once computed the cluster matrices, update the buffer(s) diff --git a/interface/src/avatar/SoftAttachmentModel.h b/interface/src/avatar/SoftAttachmentModel.h index 84a8b4cc66..cdf957514c 100644 --- a/interface/src/avatar/SoftAttachmentModel.h +++ b/interface/src/avatar/SoftAttachmentModel.h @@ -31,7 +31,7 @@ public: ~SoftAttachmentModel(); virtual void updateRig(float deltaTime, glm::mat4 parentTransform) override; - virtual void updateClusterMatrices(glm::vec3 modelPosition, glm::quat modelOrientation) override; + virtual void updateClusterMatrices() override; protected: int getJointIndexOverride(int i) const; diff --git a/interface/src/ui/overlays/Web3DOverlay.cpp b/interface/src/ui/overlays/Web3DOverlay.cpp index 988c839d84..15e208fa95 100644 --- a/interface/src/ui/overlays/Web3DOverlay.cpp +++ b/interface/src/ui/overlays/Web3DOverlay.cpp @@ -28,14 +28,17 @@ #include #include #include +#include +#include +#include static const float DPI = 30.47f; static const float INCHES_TO_METERS = 1.0f / 39.3701f; static const float METERS_TO_INCHES = 39.3701f; static const float OPAQUE_ALPHA_THRESHOLD = 0.99f; -QString const Web3DOverlay::TYPE = "web3d"; - +const QString Web3DOverlay::TYPE = "web3d"; +const QString Web3DOverlay::QML = "Web3DOverlay.qml"; Web3DOverlay::Web3DOverlay() : _dpi(DPI) { _touchDevice.setCapabilities(QTouchDevice::Position); _touchDevice.setType(QTouchDevice::TouchScreen); @@ -98,8 +101,9 @@ Web3DOverlay::~Web3DOverlay() { // is no longer valid auto webSurface = _webSurface; AbstractViewStateInterface::instance()->postLambdaEvent([webSurface] { - webSurface->deleteLater(); + DependencyManager::get()->release(QML, webSurface); }); + _webSurface.reset(); } auto geometryCache = DependencyManager::get(); if (geometryCache) { @@ -148,20 +152,15 @@ void Web3DOverlay::render(RenderArgs* args) { QOpenGLContext * currentContext = QOpenGLContext::currentContext(); QSurface * currentSurface = currentContext->surface(); if (!_webSurface) { - auto deleter = [](OffscreenQmlSurface* webSurface) { - AbstractViewStateInterface::instance()->postLambdaEvent([webSurface] { - webSurface->deleteLater(); - }); - }; - _webSurface = QSharedPointer(new OffscreenQmlSurface(), deleter); + _webSurface = DependencyManager::get()->acquire(QML); + _webSurface->setMaxFps(10); // FIXME, the max FPS could be better managed by being dynamic (based on the number of current surfaces // and the current rendering load) - _webSurface->setMaxFps(10); - _webSurface->create(currentContext); - loadSourceURL(); - + _webSurface->resume(); _webSurface->resize(QSize(_resolution.x, _resolution.y)); + _webSurface->getRootItem()->setProperty("url", _url); + _webSurface->getRootItem()->setProperty("scriptURL", _scriptURL); currentContext->makeCurrent(currentSurface); auto forwardPointerEvent = [=](unsigned int overlayID, const PointerEvent& event) { diff --git a/interface/src/ui/overlays/Web3DOverlay.h b/interface/src/ui/overlays/Web3DOverlay.h index 18fefb6885..2ea2bb5b6c 100644 --- a/interface/src/ui/overlays/Web3DOverlay.h +++ b/interface/src/ui/overlays/Web3DOverlay.h @@ -21,6 +21,7 @@ class Web3DOverlay : public Billboard3DOverlay { Q_OBJECT public: + static const QString QML; static QString const TYPE; virtual QString getType() const override { return TYPE; } diff --git a/libraries/audio/src/AudioHRTF.cpp b/libraries/audio/src/AudioHRTF.cpp index 84e3622498..2a191b5821 100644 --- a/libraries/audio/src/AudioHRTF.cpp +++ b/libraries/audio/src/AudioHRTF.cpp @@ -392,8 +392,8 @@ static void crossfade_4x2(float* src, float* dst, const float* win, int numFrame // linear interpolation with gain static void interpolate(float* dst, const float* src0, const float* src1, float frac, float gain) { - __m128 f0 = _mm_set1_ps(HRTF_GAIN * gain * (1.0f - frac)); - __m128 f1 = _mm_set1_ps(HRTF_GAIN * gain * frac); + __m128 f0 = _mm_set1_ps(gain * (1.0f - frac)); + __m128 f1 = _mm_set1_ps(gain * frac); assert(HRTF_TAPS % 4 == 0); @@ -861,6 +861,9 @@ void AudioHRTF::render(int16_t* input, float* output, int index, float azimuth, ALIGN32 float bqBuffer[4 * HRTF_BLOCK]; // 4-channel (interleaved) int delay[4]; // 4-channel (interleaved) + // apply global and local gain adjustment + gain *= _gainAdjust; + // to avoid polluting the cache, old filters are recomputed instead of stored setFilters(firCoef, bqCoef, delay, index, _azimuthState, _distanceState, _gainState, L0); diff --git a/libraries/audio/src/AudioHRTF.h b/libraries/audio/src/AudioHRTF.h index 08ae82b854..c9d053bec4 100644 --- a/libraries/audio/src/AudioHRTF.h +++ b/libraries/audio/src/AudioHRTF.h @@ -44,6 +44,11 @@ public: // void renderSilent(int16_t* input, float* output, int index, float azimuth, float distance, float gain, int numFrames); + // + // HRTF local gain adjustment + // + void setGainAdjustment(float gain) { _gainAdjust = HRTF_GAIN * gain; }; + private: AudioHRTF(const AudioHRTF&) = delete; AudioHRTF& operator=(const AudioHRTF&) = delete; @@ -73,6 +78,9 @@ private: float _distanceState = 0.0f; float _gainState = 0.0f; + // global and local gain adjustment + float _gainAdjust = HRTF_GAIN; + bool _silentState = false; }; diff --git a/libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.h b/libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.h index ee0bcf91a9..f4efc0267b 100644 --- a/libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.h +++ b/libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.h @@ -114,10 +114,10 @@ protected: bool _vsyncEnabled { true }; QThread* _presentThread{ nullptr }; std::queue _newFrameQueue; - RateCounter<> _droppedFrameRate; - RateCounter<> _newFrameRate; - RateCounter<> _presentRate; - RateCounter<> _renderRate; + RateCounter<100> _droppedFrameRate; + RateCounter<100> _newFrameRate; + RateCounter<100> _presentRate; + RateCounter<100> _renderRate; gpu::FramePointer _currentFrame; gpu::Frame* _lastFrame { nullptr }; diff --git a/libraries/gl/src/gl/OffscreenQmlSurfaceCache.cpp b/libraries/gl/src/gl/OffscreenQmlSurfaceCache.cpp new file mode 100644 index 0000000000..ad370a1f43 --- /dev/null +++ b/libraries/gl/src/gl/OffscreenQmlSurfaceCache.cpp @@ -0,0 +1,57 @@ +// +// Created by Bradley Austin Davis on 2017-01-11 +// Copyright 2013-2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#include "OffscreenQmlSurfaceCache.h" + +#include +#include + +#include +#include "OffscreenQmlSurface.h" + +OffscreenQmlSurfaceCache::OffscreenQmlSurfaceCache() { +} + +OffscreenQmlSurfaceCache::~OffscreenQmlSurfaceCache() { + _cache.clear(); +} + +QSharedPointer OffscreenQmlSurfaceCache::acquire(const QString& rootSource) { + auto& list = _cache[rootSource]; + if (list.empty()) { + list.push_back(buildSurface(rootSource)); + } + auto result = list.front(); + list.pop_front(); + return result; +} + +void OffscreenQmlSurfaceCache::reserve(const QString& rootSource, int count) { + auto& list = _cache[rootSource]; + while (list.size() < count) { + list.push_back(buildSurface(rootSource)); + } +} + +void OffscreenQmlSurfaceCache::release(const QString& rootSource, const QSharedPointer& surface) { + surface->pause(); + _cache[rootSource].push_back(surface); +} + +QSharedPointer OffscreenQmlSurfaceCache::buildSurface(const QString& rootSource) { + auto surface = QSharedPointer(new OffscreenQmlSurface()); + QOpenGLContext* currentContext = QOpenGLContext::currentContext(); + QSurface* currentSurface = currentContext->surface(); + surface->create(currentContext); + surface->setBaseUrl(QUrl::fromLocalFile(PathUtils::resourcesPath() + "/qml/")); + surface->load(rootSource); + surface->getRootContext()->setContextProperty("ApplicationInterface", qApp); + surface->resize(QSize(100, 100)); + currentContext->makeCurrent(currentSurface); + return surface; +} + diff --git a/libraries/gl/src/gl/OffscreenQmlSurfaceCache.h b/libraries/gl/src/gl/OffscreenQmlSurfaceCache.h new file mode 100644 index 0000000000..02382a791b --- /dev/null +++ b/libraries/gl/src/gl/OffscreenQmlSurfaceCache.h @@ -0,0 +1,34 @@ +// +// Created by Bradley Austin Davis on 2017-01-11 +// Copyright 2013-2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#pragma once +#ifndef hifi_OffscreenQmlSurfaceCache_h +#define hifi_OffscreenQmlSurfaceCahce_h + +#include "DependencyManager.h" + +#include + +class OffscreenQmlSurface; + +class OffscreenQmlSurfaceCache : public Dependency { + SINGLETON_DEPENDENCY + +public: + OffscreenQmlSurfaceCache(); + virtual ~OffscreenQmlSurfaceCache(); + + QSharedPointer acquire(const QString& rootSource); + void release(const QString& rootSource, const QSharedPointer& surface); + void reserve(const QString& rootSource, int count = 1); + +private: + QSharedPointer buildSurface(const QString& rootSource); + QHash>> _cache; +}; + +#endif diff --git a/libraries/networking/src/udt/SendQueue.cpp b/libraries/networking/src/udt/SendQueue.cpp index 82f4084bb2..31d61bde5d 100644 --- a/libraries/networking/src/udt/SendQueue.cpp +++ b/libraries/networking/src/udt/SendQueue.cpp @@ -88,8 +88,6 @@ SendQueue::SendQueue(Socket* socket, HifiSockAddr dest) : _socket(socket), _destination(dest) { - PROFILE_ASYNC_BEGIN(network, "SendQueue", _destination.toString()); - // setup psuedo-random number generation for all instances of SendQueue static std::random_device rd; static std::mt19937 generator(rd()); @@ -108,7 +106,6 @@ SendQueue::SendQueue(Socket* socket, HifiSockAddr dest) : } SendQueue::~SendQueue() { - PROFILE_ASYNC_END(network, "SendQueue", _destination.toString()); } void SendQueue::queuePacket(std::unique_ptr packet) { @@ -229,8 +226,6 @@ void SendQueue::sendHandshake() { if (!_hasReceivedHandshakeACK) { // we haven't received a handshake ACK from the client, send another now auto handshakePacket = ControlPacket::create(ControlPacket::Handshake, sizeof(SequenceNumber)); - PROFILE_ASYNC_BEGIN(network, "SendQueue:Handshake", _destination.toString()); - handshakePacket->writePrimitive(_initialSequenceNumber); _socket->writeBasePacket(*handshakePacket, _destination); @@ -246,8 +241,6 @@ void SendQueue::handshakeACK(SequenceNumber initialSequenceNumber) { std::lock_guard locker { _handshakeMutex }; _hasReceivedHandshakeACK = true; } - PROFILE_ASYNC_END(network, "SendQueue:Handshake", _destination.toString()); - // Notify on the handshake ACK condition _handshakeACKCondition.notify_one(); } diff --git a/libraries/render-utils/src/MeshPartPayload.cpp b/libraries/render-utils/src/MeshPartPayload.cpp index 037d58a67b..5c4c0890a7 100644 --- a/libraries/render-utils/src/MeshPartPayload.cpp +++ b/libraries/render-utils/src/MeshPartPayload.cpp @@ -63,8 +63,7 @@ void MeshPartPayload::updateMeshPart(const std::shared_ptr& d void MeshPartPayload::updateTransform(const Transform& transform, const Transform& offsetTransform) { _transform = transform; - _offsetTransform = offsetTransform; - Transform::mult(_drawTransform, _transform, _offsetTransform); + Transform::mult(_drawTransform, _transform, offsetTransform); _worldBound = _localBound; _worldBound.transform(_drawTransform); } @@ -360,8 +359,8 @@ void ModelMeshPartPayload::notifyLocationChanged() { } -void ModelMeshPartPayload::updateTransformForSkinnedMesh(const Transform& transform, const Transform& offsetTransform, const QVector& clusterMatrices) { - ModelMeshPartPayload::updateTransform(transform, offsetTransform); +void ModelMeshPartPayload::updateTransformForSkinnedMesh(const Transform& transform, const QVector& clusterMatrices) { + _transform = transform; if (clusterMatrices.size() > 0) { _worldBound = AABox(); @@ -371,8 +370,10 @@ void ModelMeshPartPayload::updateTransformForSkinnedMesh(const Transform& transf _worldBound += clusterBound; } - // clusterMatrix has world rotation but not world translation. - _worldBound.translate(transform.getTranslation()); + _worldBound.transform(transform); + if (clusterMatrices.size() == 1) { + _transform = _transform.worldTransform(Transform(clusterMatrices[0])); + } } } @@ -520,23 +521,15 @@ void ModelMeshPartPayload::bindTransform(gpu::Batch& batch, const ShapePipeline: // Still relying on the raw data from the model const Model::MeshState& state = _model->_meshStates.at(_meshIndex); - Transform transform; if (state.clusterBuffer) { if (canCauterize && _model->getCauterizeBones()) { batch.setUniformBuffer(ShapePipeline::Slot::BUFFER::SKINNING, state.cauterizedClusterBuffer); } else { batch.setUniformBuffer(ShapePipeline::Slot::BUFFER::SKINNING, state.clusterBuffer); } - } else { - if (canCauterize && _model->getCauterizeBones()) { - transform = Transform(state.cauterizedClusterMatrices[0]); - } else { - transform = Transform(state.clusterMatrices[0]); - } } - transform.preTranslate(_transform.getTranslation()); - batch.setModelTransform(transform); + batch.setModelTransform(_transform); } void ModelMeshPartPayload::startFade() { @@ -569,8 +562,7 @@ void ModelMeshPartPayload::render(RenderArgs* args) const { return; } - - // When an individual mesh parts like this finishes its fade, we will mark the Model as + // When an individual mesh parts like this finishes its fade, we will mark the Model as // having render items that need updating bool nextIsFading = _isFading ? isStillFading() : false; bool startFading = !_isFading && !_hasFinishedFade && _hasStartedFade; @@ -592,7 +584,7 @@ void ModelMeshPartPayload::render(RenderArgs* args) const { // Bind the model transform and the skinCLusterMatrices if needed bool canCauterize = args->_renderMode != RenderArgs::SHADOW_RENDER_MODE; - _model->updateClusterMatrices(_transform.getTranslation(), _transform.getRotation()); + _model->updateClusterMatrices(); bindTransform(batch, locations, canCauterize); //Bind the index buffer and vertex buffer and Blend shapes if needed diff --git a/libraries/render-utils/src/MeshPartPayload.h b/libraries/render-utils/src/MeshPartPayload.h index 3b882a9bb9..b048dc903f 100644 --- a/libraries/render-utils/src/MeshPartPayload.h +++ b/libraries/render-utils/src/MeshPartPayload.h @@ -34,7 +34,7 @@ public: virtual void updateMeshPart(const std::shared_ptr& drawMesh, int partIndex); virtual void notifyLocationChanged() {} - virtual void updateTransform(const Transform& transform, const Transform& offsetTransform); + void updateTransform(const Transform& transform, const Transform& offsetTransform); virtual void updateMaterial(model::MaterialPointer drawMaterial); @@ -56,13 +56,12 @@ public: model::Mesh::Part _drawPart; std::shared_ptr _drawMaterial; - + model::Box _localBound; Transform _drawTransform; Transform _transform; - Transform _offsetTransform; mutable model::Box _worldBound; - + bool _hasColorAttrib = false; size_t getVerticesCount() const { return _drawMesh ? _drawMesh->getNumVertices() : 0; } @@ -86,7 +85,7 @@ public: typedef Payload::DataPointer Pointer; void notifyLocationChanged() override; - void updateTransformForSkinnedMesh(const Transform& transform, const Transform& offsetTransform, const QVector& clusterMatrices); + void updateTransformForSkinnedMesh(const Transform& transform, const QVector& clusterMatrices); // Entity fade in void startFade(); @@ -101,7 +100,7 @@ public: // ModelMeshPartPayload functions to perform render void bindMesh(gpu::Batch& batch) const override; - void bindTransform(gpu::Batch& batch, const render::ShapePipeline::LocationsPointer locations, bool canCauterize) const override; + void bindTransform(gpu::Batch& batch, const render::ShapePipeline::LocationsPointer locations, bool canCauterize = true) const override; void initCache(); diff --git a/libraries/render-utils/src/Model.cpp b/libraries/render-utils/src/Model.cpp index 55a2b1c76a..9a85004d93 100644 --- a/libraries/render-utils/src/Model.cpp +++ b/libraries/render-utils/src/Model.cpp @@ -92,7 +92,6 @@ Model::Model(RigPointer rig, QObject* parent) : _snapModelToRegistrationPoint(false), _snappedToRegistrationPoint(false), _cauterizeBones(false), - _pupilDilation(0.0f), _url(HTTP_INVALID_COM), _isVisible(true), _blendNumber(0), @@ -234,19 +233,11 @@ void Model::updateRenderItems() { render::ScenePointer scene = AbstractViewStateInterface::instance()->getMain3DScene(); - Transform modelMeshOffset; - if (self->isLoaded()) { - // includes model offset and unitScale. - modelMeshOffset = Transform(self->_rig->getGeometryToRigTransform()); - } else { - modelMeshOffset.postTranslate(self->_offset); - } - uint32_t deleteGeometryCounter = self->_deleteGeometryCounter; render::PendingChanges pendingChanges; foreach (auto itemID, self->_modelMeshRenderItems.keys()) { - pendingChanges.updateItem(itemID, [modelMeshOffset, deleteGeometryCounter](ModelMeshPartPayload& data) { + pendingChanges.updateItem(itemID, [deleteGeometryCounter](ModelMeshPartPayload& data) { if (data._model && data._model->isLoaded()) { if (!data.hasStartedFade() && data._model->getGeometry()->areTexturesLoaded()) { data.startFade(); @@ -256,11 +247,11 @@ void Model::updateRenderItems() { Transform modelTransform = data._model->getTransform(); // lazy update of cluster matrices used for rendering. We need to update them here, so we can correctly update the bounding box. - data._model->updateClusterMatrices(modelTransform.getTranslation(), modelTransform.getRotation()); + data._model->updateClusterMatrices(); // update the model transform and bounding box for this render item. const Model::MeshState& state = data._model->_meshStates.at(data._meshIndex); - data.updateTransformForSkinnedMesh(modelTransform, modelMeshOffset, state.clusterMatrices); + data.updateTransformForSkinnedMesh(modelTransform, state.clusterMatrices); } } }); @@ -311,6 +302,7 @@ bool Model::updateGeometry() { if (_rig->jointStatesEmpty() && getFBXGeometry().joints.size() > 0) { initJointStates(); + assert(_meshStates.empty()); const FBXGeometry& fbxGeometry = getFBXGeometry(); foreach (const FBXMesh& mesh, fbxGeometry.meshes) { @@ -320,6 +312,9 @@ bool Model::updateGeometry() { _meshStates.append(state); + // Note: we add empty buffers for meshes that lack blendshapes so we can access the buffers by index + // later in ModelMeshPayload, however the vast majority of meshes will not have them. + // TODO? make _blendedVertexBuffers a map instead of vector and only add for meshes with blendshapes? auto buffer = std::make_shared(); if (!mesh.blendshapes.isEmpty()) { buffer->resize((mesh.vertices.size() + mesh.normals.size()) * sizeof(glm::vec3)); @@ -1168,7 +1163,7 @@ void Model::simulateInternal(float deltaTime) { } // virtual -void Model::updateClusterMatrices(glm::vec3 modelPosition, glm::quat modelOrientation) { +void Model::updateClusterMatrices() { PerformanceTimer perfTimer("Model::updateClusterMatrices"); if (!_needsUpdateClusterMatrices || !isLoaded()) { @@ -1183,7 +1178,6 @@ void Model::updateClusterMatrices(glm::vec3 modelPosition, glm::quat modelOrient glm::vec4(0.0f, 0.0f, 0.0f, 1.0f)); auto cauterizeMatrix = _rig->getJointTransform(geometry.neckJointIndex) * zeroScale; - glm::mat4 modelToWorld = glm::mat4_cast(modelOrientation); for (int i = 0; i < _meshStates.size(); i++) { MeshState& state = _meshStates[i]; const FBXMesh& mesh = geometry.meshes.at(i); @@ -1191,12 +1185,11 @@ void Model::updateClusterMatrices(glm::vec3 modelPosition, glm::quat modelOrient const FBXCluster& cluster = mesh.clusters.at(j); auto jointMatrix = _rig->getJointTransform(cluster.jointIndex); #if GLM_ARCH & GLM_ARCH_SSE2 - glm::mat4 temp, out, inverseBindMatrix = cluster.inverseBindMatrix; - glm_mat4_mul((glm_vec4*)&modelToWorld, (glm_vec4*)&jointMatrix, (glm_vec4*)&temp); - glm_mat4_mul((glm_vec4*)&temp, (glm_vec4*)&inverseBindMatrix, (glm_vec4*)&out); + glm::mat4 out, inverseBindMatrix = cluster.inverseBindMatrix; + glm_mat4_mul((glm_vec4*)&jointMatrix, (glm_vec4*)&inverseBindMatrix, (glm_vec4*)&out); state.clusterMatrices[j] = out; -#else - state.clusterMatrices[j] = modelToWorld * jointMatrix * cluster.inverseBindMatrix; +#else + state.clusterMatrices[j] = jointMatrix * cluster.inverseBindMatrix; #endif // as an optimization, don't build cautrizedClusterMatrices if the boneSet is empty. @@ -1204,7 +1197,13 @@ void Model::updateClusterMatrices(glm::vec3 modelPosition, glm::quat modelOrient if (_cauterizeBoneSet.find(cluster.jointIndex) != _cauterizeBoneSet.end()) { jointMatrix = cauterizeMatrix; } - state.cauterizedClusterMatrices[j] = modelToWorld * jointMatrix * cluster.inverseBindMatrix; +#if GLM_ARCH & GLM_ARCH_SSE2 + glm::mat4 out, inverseBindMatrix = cluster.inverseBindMatrix; + glm_mat4_mul((glm_vec4*)&jointMatrix, (glm_vec4*)&inverseBindMatrix, (glm_vec4*)&out); + state.cauterizedClusterMatrices[j] = out; +#else + state.cauterizedClusterMatrices[j] = jointMatrix * cluster.inverseBindMatrix; +#endif } } diff --git a/libraries/render-utils/src/Model.h b/libraries/render-utils/src/Model.h index b161f1d92a..1bb8f46103 100644 --- a/libraries/render-utils/src/Model.h +++ b/libraries/render-utils/src/Model.h @@ -127,7 +127,7 @@ public: bool getSnapModelToRegistrationPoint() { return _snapModelToRegistrationPoint; } virtual void simulate(float deltaTime, bool fullUpdate = true); - virtual void updateClusterMatrices(glm::vec3 modelPosition, glm::quat modelOrientation); + virtual void updateClusterMatrices(); /// Returns a reference to the shared geometry. const Geometry::Pointer& getGeometry() const { return _renderGeometry; } @@ -255,9 +255,6 @@ signals: protected: bool addedToScene() const { return _addedToScene; } - void setPupilDilation(float dilation) { _pupilDilation = dilation; } - float getPupilDilation() const { return _pupilDilation; } - void setBlendshapeCoefficients(const QVector& coefficients) { _blendshapeCoefficients = coefficients; } const QVector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } @@ -354,7 +351,6 @@ protected: void deleteGeometry(); void initJointTransforms(); - float _pupilDilation; QVector _blendshapeCoefficients; QUrl _url; diff --git a/scripts/system/controllers/handControllerPointer.js b/scripts/system/controllers/handControllerPointer.js index 6a4c71f2ee..6bebbf0498 100644 --- a/scripts/system/controllers/handControllerPointer.js +++ b/scripts/system/controllers/handControllerPointer.js @@ -20,9 +20,10 @@ // When partially squeezing over a HUD element, a laser or the reticle is shown where the active hand // controller beam intersects the HUD. -var systemLaserOn = false; - - +var activeTrigger; +function isLaserOn() { + return activeTrigger.partial(); +} Script.include("../libraries/controllers.js"); // UTILITIES ------------- @@ -124,12 +125,6 @@ function ignoreMouseActivity() { if (!Reticle.allowMouseCapture) { return true; } - - // if the lasers are on, then reticle/mouse should be hidden and we can ignore it for seeking or depth updating - if (systemLaserOn) { - return true; - } - var pos = Reticle.position; if (!pos || (pos.x == -1 && pos.y == -1)) { return true; @@ -270,12 +265,6 @@ var ONE_MINUS_WEIGHTING = 1 - WEIGHTING; var AVERAGE_MOUSE_VELOCITY_FOR_SEEK_TO = 20; function isShakingMouse() { // True if the person is waving the mouse around trying to find it. var now = Date.now(), mouse = Reticle.position, isShaking = false; - - // if the lasers are on, then we ignore mouse shaking - if (systemLaserOn) { - return false; - } - if (lastIntegration && (lastIntegration !== now)) { var velocity = Vec3.length(Vec3.subtract(mouse, lastMouse)) / (now - lastIntegration); averageMouseVelocity = (ONE_MINUS_WEIGHTING * averageMouseVelocity) + (WEIGHTING * velocity); @@ -290,16 +279,8 @@ function isShakingMouse() { // True if the person is waving the mouse around try var NON_LINEAR_DIVISOR = 2; var MINIMUM_SEEK_DISTANCE = 0.1; function updateSeeking(doNotStartSeeking) { - // if the lasers are on, then we never do seeking - if (systemLaserOn) { - isSeeking = false; - return; - } - - if (!doNotStartSeeking && (!Reticle.visible || isShakingMouse())) { - if (!isSeeking) { - isSeeking = true; - } + if (!doNotStartSeeking && !isLaserOn() && (!Reticle.visible || isShakingMouse())) { + isSeeking = true; } // e.g., if we're about to turn it on with first movement. if (!isSeeking) { return; @@ -340,7 +321,7 @@ function updateMouseActivity(isClick) { return; } // Bug: mouse clicks should keep going. Just not hand controller clicks handControllerLockOut.update(now); - Reticle.visible = !systemLaserOn; + Reticle.visible = true; } function expireMouseCursor(now) { if (!isPointingAtOverlay() && mouseCursorActivity.expired(now)) { @@ -392,7 +373,7 @@ setupHandler(Controller.mouseDoublePressEvent, onMouseClick); var leftTrigger = new Trigger('left'); var rightTrigger = new Trigger('right'); -var activeTrigger = rightTrigger; +activeTrigger = rightTrigger; var activeHand = Controller.Standard.RightHand; var LEFT_HUD_LASER = 1; var RIGHT_HUD_LASER = 2; @@ -498,6 +479,7 @@ var LASER_ALPHA = 0.5; var LASER_SEARCH_COLOR_XYZW = {x: 10 / 255, y: 10 / 255, z: 255 / 255, w: LASER_ALPHA}; var LASER_TRIGGER_COLOR_XYZW = {x: 250 / 255, y: 10 / 255, z: 10 / 255, w: LASER_ALPHA}; var SYSTEM_LASER_DIRECTION = {x: 0, y: 0, z: -1}; +var systemLaserOn = false; function clearSystemLaser() { if (!systemLaserOn) { return; @@ -506,7 +488,6 @@ function clearSystemLaser() { HMD.disableExtraLaser(); systemLaserOn = false; weMovedReticle = true; - Reticle.position = { x: -1, y: -1 }; } function setColoredLaser() { // answer trigger state if lasers supported, else falsey. var color = (activeTrigger.state === 'full') ? LASER_TRIGGER_COLOR_XYZW : LASER_SEARCH_COLOR_XYZW; @@ -590,12 +571,6 @@ function update() { Reticle.visible = false; } -var BASIC_TIMER_INTERVAL = 20; // 20ms = 50hz good enough -var updateIntervalTimer = Script.setInterval(function(){ - update(); -}, BASIC_TIMER_INTERVAL); - - // Check periodically for changes to setup. var SETTINGS_CHANGE_RECHECK_INTERVAL = 10 * 1000; // 10 seconds function checkSettings() { @@ -605,9 +580,10 @@ function checkSettings() { checkSettings(); var settingsChecker = Script.setInterval(checkSettings, SETTINGS_CHANGE_RECHECK_INTERVAL); +Script.update.connect(update); Script.scriptEnding.connect(function () { Script.clearInterval(settingsChecker); - Script.clearInterval(updateIntervalTimer); + Script.update.disconnect(update); OffscreenFlags.navigationFocusDisabled = false; }); diff --git a/scripts/system/pal.js b/scripts/system/pal.js index 2e6c179f95..9a73b2a18e 100644 --- a/scripts/system/pal.js +++ b/scripts/system/pal.js @@ -19,9 +19,13 @@ const UNSELECTED_TEXTURES = {"idle-D": Script.resolvePath("./assets/models/Avata const SELECTED_TEXTURES = { "idle-D": Script.resolvePath("./assets/models/Avatar-Overlay-v1.fbx/Avatar-Overlay-v1.fbm/avatar-overlay-selected.png"), "idle-E": Script.resolvePath("./assets/models/Avatar-Overlay-v1.fbx/Avatar-Overlay-v1.fbm/avatar-overlay-selected.png") }; +const HOVER_TEXTURES = { "idle-D": Script.resolvePath("./assets/models/Avatar-Overlay-v1.fbx/Avatar-Overlay-v1.fbm/avatar-overlay-hover.png"), + "idle-E": Script.resolvePath("./assets/models/Avatar-Overlay-v1.fbx/Avatar-Overlay-v1.fbm/avatar-overlay-hover.png") +}; const UNSELECTED_COLOR = { red: 0x1F, green: 0xC6, blue: 0xA6}; -const SELECTED_COLOR = {red: 0xf3, green: 0x91, blue: 0x29}; +const SELECTED_COLOR = {red: 0xF3, green: 0x91, blue: 0x29}; +const HOVER_COLOR = {red: 0xD0, green: 0xD0, blue: 0xD0}; // almost white for now (function() { // BEGIN LOCAL_SCOPE @@ -46,6 +50,7 @@ function ExtendedOverlay(key, type, properties, selected, hasModel) { // A wrapp } this.key = key; this.selected = selected || false; // not undefined + this.hovering = false; this.activeOverlay = Overlays.addOverlay(type, properties); // We could use different overlays for (un)selected... } // Instance methods: @@ -58,8 +63,8 @@ ExtendedOverlay.prototype.editOverlay = function (properties) { // change displa Overlays.editOverlay(this.activeOverlay, properties); }; -function color(selected, level) { - var base = selected ? SELECTED_COLOR : UNSELECTED_COLOR; +function color(selected, hovering, level) { + var base = hovering ? HOVER_COLOR : selected ? SELECTED_COLOR : UNSELECTED_COLOR; function scale(component) { var delta = 0xFF - component; return component + (delta * level); @@ -67,16 +72,38 @@ function color(selected, level) { return {red: scale(base.red), green: scale(base.green), blue: scale(base.blue)}; } -function textures(selected) { - return selected ? SELECTED_TEXTURES : UNSELECTED_TEXTURES; +function textures(selected, hovering) { + return hovering ? HOVER_TEXTURES : selected ? SELECTED_TEXTURES : UNSELECTED_TEXTURES; +} +// so we don't have to traverse the overlays to get the last one +var lastHoveringId = 0; +ExtendedOverlay.prototype.hover = function (hovering) { + this.hovering = hovering; + if (this.key === lastHoveringId) { + if (hovering) { + return; + } else { + lastHoveringId = 0; + } + } + this.editOverlay({color: color(this.selected, hovering, this.audioLevel)}); + if (this.model) { + this.model.editOverlay({textures: textures(this.selected, hovering)}); + } + if (hovering) { + // un-hover the last hovering overlay + if (lastHoveringId && lastHoveringId != this.key) { + ExtendedOverlay.get(lastHoveringId).hover(false); + } + lastHoveringId = this.key; + } } - ExtendedOverlay.prototype.select = function (selected) { if (this.selected === selected) { return; } - this.editOverlay({color: color(selected, this.audioLevel)}); + this.editOverlay({color: color(selected, this.hovering, this.audioLevel)}); if (this.model) { this.model.editOverlay({textures: textures(selected)}); } @@ -98,14 +125,25 @@ ExtendedOverlay.some = function (iterator) { // Bails early as soon as iterator } } }; -ExtendedOverlay.applyPickRay = function (pickRay, cb) { // cb(overlay) on the one overlay intersected by pickRay, if any. +ExtendedOverlay.unHover = function () { // calls hover(false) on lastHoveringId (if any) + if (lastHoveringId) { + ExtendedOverlay.get(lastHoveringId).hover(false); + } +}; + +// hit(overlay) on the one overlay intersected by pickRay, if any. +// noHit() if no ExtendedOverlay was intersected (helps with hover) +ExtendedOverlay.applyPickRay = function (pickRay, hit, noHit) { var pickedOverlay = Overlays.findRayIntersection(pickRay); // Depends on nearer coverOverlays to extend closer to us than farther ones. if (!pickedOverlay.intersects) { + if (noHit) { + return noHit(); + } return; } ExtendedOverlay.some(function (overlay) { // See if pickedOverlay is one of ours. if ((overlay.activeOverlay) === pickedOverlay.overlayID) { - cb(overlay); + hit(overlay); return true; } }); @@ -209,7 +247,7 @@ function addAvatarNode(id) { drawInFront: true, solid: true, alpha: 0.8, - color: color(selected, 0.0), + color: color(selected, false, 0.0), ignoreRayIntersection: false}, selected, true); } function populateUserList() { @@ -293,7 +331,7 @@ function updateOverlays() { overlay.ping = pingPong; overlay.editOverlay({ - color: color(ExtendedOverlay.isSelected(id), overlay.audioLevel), + color: color(ExtendedOverlay.isSelected(id), overlay.hovering, overlay.audioLevel), position: target, dimensions: 0.032 * distance }); @@ -317,6 +355,7 @@ function updateOverlays() { } function removeOverlays() { selectedIds = []; + lastHoveringId = 0; HighlightedEntity.clearOverlays(); ExtendedOverlay.some(function (overlay) { overlay.deleteOverlay(); }); } @@ -338,9 +377,54 @@ function handleMouseEvent(mousePressEvent) { // handleClick if we get one. } handleClick(Camera.computePickRay(mousePressEvent.x, mousePressEvent.y)); } +function handleMouseMove(pickRay) { // given the pickRay, just do the hover logic + ExtendedOverlay.applyPickRay(pickRay, function (overlay) { + overlay.hover(true); + }, function () { + ExtendedOverlay.unHover(); + }); +} + +// handy global to keep track of which hand is the mouse (if any) +var currentHandPressed = 0; +const TRIGGER_CLICK_THRESHOLD = 0.85; +const TRIGGER_PRESS_THRESHOLD = 0.05; + +function handleMouseMoveEvent(event) { // find out which overlay (if any) is over the mouse position + if (HMD.active) { + if (currentHandPressed != 0) { + pickRay = controllerComputePickRay(currentHandPressed); + } else { + // nothing should hover, so + ExtendedOverlay.unHover(); + return; + } + } else { + pickRay = Camera.computePickRay(event.x, event.y); + } + handleMouseMove(pickRay); +} +function handleTriggerPressed(hand, value) { + // The idea is if you press one trigger, it is the one + // we will consider the mouse. Even if the other is pressed, + // we ignore it until this one is no longer pressed. + isPressed = value > TRIGGER_PRESS_THRESHOLD; + if (currentHandPressed == 0) { + currentHandPressed = isPressed ? hand : 0; + return; + } + if (currentHandPressed == hand) { + currentHandPressed = isPressed ? hand : 0; + return; + } + // otherwise, the other hand is still triggered + // so do nothing. +} + // We get mouseMoveEvents from the handControllers, via handControllerPointer. // But we don't get mousePressEvents. var triggerMapping = Controller.newMapping(Script.resolvePath('') + '-click'); +var triggerPressMapping = Controller.newMapping(Script.resolvePath('') + '-press'); function controllerComputePickRay(hand) { var controllerPose = getControllerWorldLocation(hand, true); if (controllerPose.valid) { @@ -349,15 +433,21 @@ function controllerComputePickRay(hand) { } function makeClickHandler(hand) { return function (clicked) { - if (clicked > 0.85) { + if (clicked > TRIGGER_CLICK_THRESHOLD) { var pickRay = controllerComputePickRay(hand); handleClick(pickRay); } }; } +function makePressHandler(hand) { + return function (value) { + handleTriggerPressed(hand, value); + } +} triggerMapping.from(Controller.Standard.RTClick).peek().to(makeClickHandler(Controller.Standard.RightHand)); triggerMapping.from(Controller.Standard.LTClick).peek().to(makeClickHandler(Controller.Standard.LeftHand)); - +triggerPressMapping.from(Controller.Standard.RT).peek().to(makePressHandler(Controller.Standard.RightHand)); +triggerPressMapping.from(Controller.Standard.LT).peek().to(makePressHandler(Controller.Standard.LeftHand)); // // Manage the connection between the button and the window. // @@ -372,9 +462,11 @@ function off() { if (isWired) { // It is not ok to disconnect these twice, hence guard. Script.update.disconnect(updateOverlays); Controller.mousePressEvent.disconnect(handleMouseEvent); + Controller.mouseMoveEvent.disconnect(handleMouseMoveEvent); isWired = false; } triggerMapping.disable(); // It's ok if we disable twice. + triggerPressMapping.disable(); // see above removeOverlays(); Users.requestsDomainListData = false; } @@ -386,7 +478,9 @@ function onClicked() { isWired = true; Script.update.connect(updateOverlays); Controller.mousePressEvent.connect(handleMouseEvent); + Controller.mouseMoveEvent.connect(handleMouseMoveEvent); triggerMapping.enable(); + triggerPressMapping.enable(); } else { off(); }