From c892d8a81a21d609bea31f2533b9120cb243fd57 Mon Sep 17 00:00:00 2001 From: Bradley Austin Davis Date: Fri, 3 Mar 2017 10:41:47 -0800 Subject: [PATCH 01/64] Update Oculus SDK to 1.11 --- cmake/externals/LibOVR/CMakeLists.txt | 26 ++++++--------------- cmake/externals/LibOVR/LibOVRCMakeLists.txt | 15 ++++++++++++ 2 files changed, 22 insertions(+), 19 deletions(-) create mode 100644 cmake/externals/LibOVR/LibOVRCMakeLists.txt diff --git a/cmake/externals/LibOVR/CMakeLists.txt b/cmake/externals/LibOVR/CMakeLists.txt index 54a4a47929..d8b8307082 100644 --- a/cmake/externals/LibOVR/CMakeLists.txt +++ b/cmake/externals/LibOVR/CMakeLists.txt @@ -12,35 +12,23 @@ string(TOUPPER ${EXTERNAL_NAME} EXTERNAL_NAME_UPPER) # 0.5 public # URL http://static.oculus.com/sdk-downloads/ovr_sdk_win_0.5.0.1.zip # URL_MD5 d3fc4c02db9be5ff08af4ef4c97b32f9 -# 1.3 public -# URL http://hifi-public.s3.amazonaws.com/dependencies/ovr_sdk_win_1.3.0_public.zip -# URL_MD5 4d26faba0c1f35ff80bf674c96ed9259 if (WIN32) ExternalProject_Add( ${EXTERNAL_NAME} - URL https://hifi-public.s3.amazonaws.com/dependencies/ovr_sdk_win_1.8.0_public.zip - URL_MD5 bea17e04acc1dd8cf7cabefa1b28cc3c - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" + URL https://static.oculus.com/sdk-downloads/1.11.0/Public/1486063832/ovr_sdk_win_1.11.0_public.zip + URL_MD5 ea484403757cbfdfa743b6577fb1f9d2 + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH= + PATCH_COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/LibOVRCMakeLists.txt" /CMakeLists.txt LOG_DOWNLOAD 1 ) ExternalProject_Get_Property(${EXTERNAL_NAME} SOURCE_DIR) - message("LIBOVR dir ${SOURCE_DIR}") - set(LIBOVR_DIR ${SOURCE_DIR}/LibOVR) - if ("${CMAKE_SIZEOF_VOID_P}" EQUAL "8") - set(LIBOVR_LIB_DIR ${LIBOVR_DIR}/Lib/Windows/x64/Release/VS2013 CACHE TYPE INTERNAL) - else() - set(LIBOVR_LIB_DIR ${LIBOVR_DIR}/Lib/Windows/Win32/Release/VS2013 CACHE TYPE INTERNAL) - endif() - + ExternalProject_Get_Property(${EXTERNAL_NAME} INSTALL_DIR) + set(LIBOVR_DIR ${INSTALL_DIR}) set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${LIBOVR_DIR}/Include CACHE TYPE INTERNAL) - message("LIBOVR include dir ${${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS}") - set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${LIBOVR_LIB_DIR}/LibOVR.lib CACHE TYPE INTERNAL) - + set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${LIBOVR_DIR}/Lib/LibOVR.lib CACHE TYPE INTERNAL) elseif(APPLE) ExternalProject_Add( diff --git a/cmake/externals/LibOVR/LibOVRCMakeLists.txt b/cmake/externals/LibOVR/LibOVRCMakeLists.txt new file mode 100644 index 0000000000..bab6337ae2 --- /dev/null +++ b/cmake/externals/LibOVR/LibOVRCMakeLists.txt @@ -0,0 +1,15 @@ +cmake_minimum_required(VERSION 3.2) +project(LibOVR) + +#set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DOVR_BUILD_DEBUG") + +include_directories(LibOVR/Include LibOVR/Src) +#include_directories(LibOVRKernel/Src/) +file(GLOB HEADER_FILES LibOVR/Include/*.h) +file(GLOB EXTRA_HEADER_FILES LibOVR/Include/Extras/*.h) +file(GLOB_RECURSE SOURCE_FILES LibOVR/Src/*.c LibOVR/Src/*.cpp) +add_library(LibOVR STATIC ${SOURCE_FILES} ${HEADER_FILES} ${EXTRA_HEADER_FILES}) + +install(TARGETS LibOVR DESTINATION Lib) +install(FILES ${HEADER_FILES} DESTINATION Include) +install(FILES ${EXTRA_HEADER_FILES} DESTINATION Include/Extras) \ No newline at end of file From ce9d637b3f65194bb8b0c3140fe5ee84cde12bec Mon Sep 17 00:00:00 2001 From: Bradley Austin Davis Date: Fri, 3 Mar 2017 10:46:05 -0800 Subject: [PATCH 02/64] Expose ASW activation status, provide normalized present rate information --- .../src/display-plugins/OpenGLDisplayPlugin.cpp | 5 +++++ .../src/display-plugins/OpenGLDisplayPlugin.h | 2 ++ libraries/plugins/src/plugins/DisplayPlugin.h | 7 ++++++- plugins/oculus/src/OculusDisplayPlugin.cpp | 17 +++++++++++++++-- plugins/oculus/src/OculusDisplayPlugin.h | 4 +++- plugins/oculus/src/OculusHelpers.cpp | 6 +++++- 6 files changed, 36 insertions(+), 5 deletions(-) diff --git a/libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.cpp b/libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.cpp index cf6b39812a..b23b59d3f0 100644 --- a/libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.cpp +++ b/libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.cpp @@ -654,6 +654,11 @@ float OpenGLDisplayPlugin::presentRate() const { return _presentRate.rate(); } +void OpenGLDisplayPlugin::resetPresentRate() { + // FIXME + // _presentRate = RateCounter<100>(); +} + float OpenGLDisplayPlugin::renderRate() const { return _renderRate.rate(); } diff --git a/libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.h b/libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.h index f4efc0267b..e1eea5de6c 100644 --- a/libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.h +++ b/libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.h @@ -59,6 +59,8 @@ public: float presentRate() const override; + void resetPresentRate() override; + float newFramePresentRate() const override; float droppedFrameRate() const override; diff --git a/libraries/plugins/src/plugins/DisplayPlugin.h b/libraries/plugins/src/plugins/DisplayPlugin.h index 2491aed817..754c919fd4 100644 --- a/libraries/plugins/src/plugins/DisplayPlugin.h +++ b/libraries/plugins/src/plugins/DisplayPlugin.h @@ -139,7 +139,7 @@ public: /// By default, all HMDs are stereo virtual bool isStereo() const { return isHmd(); } virtual bool isThrottled() const { return false; } - virtual float getTargetFrameRate() const { return 0.0f; } + virtual float getTargetFrameRate() const { return 1.0f; } virtual bool hasAsyncReprojection() const { return false; } /// Returns a boolean value indicating whether the display is currently visible @@ -189,6 +189,11 @@ public: virtual float renderRate() const { return -1.0f; } // Rate at which we present to the display device virtual float presentRate() const { return -1.0f; } + // Reset the present rate tracking (useful for if the target frame rate changes as in ASW for Oculus) + virtual void resetPresentRate() {} + // Return the present rate as fraction of the target present rate (hopefully 0.0 and 1.0) + virtual float normalizedPresentRate() const { return presentRate() / getTargetFrameRate(); } + // Rate at which old frames are presented to the device display virtual float stutterRate() const { return -1.0f; } // Rate at which new frames are being presented to the display device diff --git a/plugins/oculus/src/OculusDisplayPlugin.cpp b/plugins/oculus/src/OculusDisplayPlugin.cpp index b076170ae5..db8c92ac23 100644 --- a/plugins/oculus/src/OculusDisplayPlugin.cpp +++ b/plugins/oculus/src/OculusDisplayPlugin.cpp @@ -28,6 +28,12 @@ OculusDisplayPlugin::OculusDisplayPlugin() { _compositorDroppedFrames.store(0); } +float OculusDisplayPlugin::getTargetFrameRate() const { + if (_aswActive) { + return _hmdDesc.DisplayRefreshRate / 2.0f; + } + return _hmdDesc.DisplayRefreshRate; +} bool OculusDisplayPlugin::internalActivate() { bool result = Parent::internalActivate(); @@ -185,8 +191,6 @@ void OculusDisplayPlugin::hmdPresent() { } } - - if (!OVR_SUCCESS(result)) { logWarning("Failed to present"); } @@ -195,12 +199,20 @@ void OculusDisplayPlugin::hmdPresent() { static int appDroppedFrames = 0; ovrPerfStats perfStats; ovr_GetPerfStats(_session, &perfStats); + bool shouldResetPresentRate = false; for (int i = 0; i < perfStats.FrameStatsCount; ++i) { const auto& frameStats = perfStats.FrameStats[i]; int delta = frameStats.CompositorDroppedFrameCount - compositorDroppedFrames; _stutterRate.increment(delta); compositorDroppedFrames = frameStats.CompositorDroppedFrameCount; appDroppedFrames = frameStats.AppDroppedFrameCount; + bool newAswState = ovrTrue == frameStats.AswIsActive; + if (_aswActive.exchange(newAswState) != newAswState) { + shouldResetPresentRate = true; + } + } + if (shouldResetPresentRate) { + resetPresentRate(); } _appDroppedFrames.store(appDroppedFrames); _compositorDroppedFrames.store(compositorDroppedFrames); @@ -212,6 +224,7 @@ void OculusDisplayPlugin::hmdPresent() { QJsonObject OculusDisplayPlugin::getHardwareStats() const { QJsonObject hardwareStats; + hardwareStats["asw_active"] = _aswActive.load(); hardwareStats["app_dropped_frame_count"] = _appDroppedFrames.load(); hardwareStats["compositor_dropped_frame_count"] = _compositorDroppedFrames.load(); hardwareStats["long_render_count"] = _longRenders.load(); diff --git a/plugins/oculus/src/OculusDisplayPlugin.h b/plugins/oculus/src/OculusDisplayPlugin.h index 6fc50b829f..9209fd373e 100644 --- a/plugins/oculus/src/OculusDisplayPlugin.h +++ b/plugins/oculus/src/OculusDisplayPlugin.h @@ -20,7 +20,8 @@ public: QString getPreferredAudioInDevice() const override; QString getPreferredAudioOutDevice() const override; - + float getTargetFrameRate() const override; + virtual QJsonObject getHardwareStats() const; protected: @@ -39,6 +40,7 @@ private: gpu::FramebufferPointer _outputFramebuffer; bool _customized { false }; + std::atomic_bool _aswActive; std::atomic_int _compositorDroppedFrames; std::atomic_int _appDroppedFrames; std::atomic_int _longSubmits; diff --git a/plugins/oculus/src/OculusHelpers.cpp b/plugins/oculus/src/OculusHelpers.cpp index 340b804404..767d191c03 100644 --- a/plugins/oculus/src/OculusHelpers.cpp +++ b/plugins/oculus/src/OculusHelpers.cpp @@ -88,7 +88,11 @@ ovrSession acquireOculusSession() { } if (!session) { - if (!OVR_SUCCESS(ovr_Initialize(nullptr))) { + ovrInitParams initParams { + ovrInit_RequestVersion | ovrInit_MixedRendering, OVR_MINOR_VERSION, nullptr, 0, 0 + }; + + if (!OVR_SUCCESS(ovr_Initialize(&initParams))) { logWarning("Failed to initialize Oculus SDK"); return session; } From 9908723bb9192da863862671d474630d02d7e952 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Fri, 3 Mar 2017 22:27:38 -0800 Subject: [PATCH 03/64] fist cut at basic TriangleSet class --- libraries/render-utils/src/Model.cpp | 65 +++++++++++++++----------- libraries/render-utils/src/Model.h | 3 +- libraries/shared/src/AABox.h | 2 + libraries/shared/src/TriangleSet.cpp | 69 ++++++++++++++++++++++++++++ libraries/shared/src/TriangleSet.h | 33 +++++++++++++ 5 files changed, 144 insertions(+), 28 deletions(-) create mode 100644 libraries/shared/src/TriangleSet.cpp create mode 100644 libraries/shared/src/TriangleSet.h diff --git a/libraries/render-utils/src/Model.cpp b/libraries/render-utils/src/Model.cpp index adfffe2614..9dd3563887 100644 --- a/libraries/render-utils/src/Model.cpp +++ b/libraries/render-utils/src/Model.cpp @@ -376,24 +376,26 @@ bool Model::findRayIntersectionAgainstSubMeshes(const glm::vec3& origin, const g } for (const auto& subMeshBox : _calculatedMeshBoxes) { + bool intersectedSubMesh = false; + float subMeshDistance = std::numeric_limits::max(); if (subMeshBox.findRayIntersection(origin, direction, distanceToSubMesh, subMeshFace, subMeshSurfaceNormal)) { if (distanceToSubMesh < bestDistance) { if (pickAgainstTriangles) { - // check our triangles here.... - const QVector& meshTriangles = _calculatedMeshTriangles[subMeshIndex]; - for(const auto& triangle : meshTriangles) { - float thisTriangleDistance; - if (findRayTriangleIntersection(origin, direction, triangle, thisTriangleDistance)) { - if (thisTriangleDistance < bestDistance) { - bestDistance = thisTriangleDistance; - intersectedSomething = true; - face = subMeshFace; - surfaceNormal = triangle.getNormal(); - extraInfo = geometry.getModelNameOfMesh(subMeshIndex); - } - } + + float subMeshDistance; + glm::vec3 subMeshNormal; + const auto& meshTriangleSet = _meshTriangleSets[subMeshIndex]; + bool intersectedMesh = meshTriangleSet.findRayIntersection(origin, direction, subMeshDistance, subMeshNormal); + + if (intersectedMesh && subMeshDistance < bestDistance) { + bestDistance = subMeshDistance; + intersectedSomething = true; + face = subMeshFace; + surfaceNormal = subMeshNormal; + extraInfo = geometry.getModelNameOfMesh(subMeshIndex); } + } else { // this is the non-triangle picking case... bestDistance = distanceToSubMesh; @@ -401,9 +403,13 @@ bool Model::findRayIntersectionAgainstSubMeshes(const glm::vec3& origin, const g face = subMeshFace; surfaceNormal = subMeshSurfaceNormal; extraInfo = geometry.getModelNameOfMesh(subMeshIndex); + + intersectedSubMesh = true; } } } + + subMeshIndex++; } _mutex.unlock(); @@ -451,18 +457,8 @@ bool Model::convexHullContains(glm::vec3 point) { int subMeshIndex = 0; foreach(const AABox& subMeshBox, _calculatedMeshBoxes) { if (subMeshBox.contains(point)) { - bool insideMesh = true; - // To be inside the sub mesh, we need to be behind every triangles' planes - const QVector& meshTriangles = _calculatedMeshTriangles[subMeshIndex]; - foreach (const Triangle& triangle, meshTriangles) { - if (!isPointBehindTrianglesPlane(point, triangle.v0, triangle.v1, triangle.v2)) { - // it's not behind at least one so we bail - insideMesh = false; - break; - } - } - if (insideMesh) { + if (_meshTriangleSets[subMeshIndex].convexHullContains(point)) { // It's inside this mesh, return true. _mutex.unlock(); return true; @@ -486,12 +482,20 @@ void Model::recalculateMeshBoxes(bool pickAgainstTriangles) { bool calculatedMeshTrianglesNeeded = pickAgainstTriangles && !_calculatedMeshTrianglesValid; if (!_calculatedMeshBoxesValid || calculatedMeshTrianglesNeeded || (!_calculatedMeshPartBoxesValid && pickAgainstTriangles) ) { + + if (pickAgainstTriangles) { + qDebug() << "RECALCULATING triangles!"; + } else { + qDebug() << "RECALCULATING boxes!"; + } + const FBXGeometry& geometry = getFBXGeometry(); int numberOfMeshes = geometry.meshes.size(); _calculatedMeshBoxes.resize(numberOfMeshes); - _calculatedMeshTriangles.clear(); - _calculatedMeshTriangles.resize(numberOfMeshes); _calculatedMeshPartBoxes.clear(); + _meshTriangleSets.clear(); + _meshTriangleSets.resize(numberOfMeshes); + for (int i = 0; i < numberOfMeshes; i++) { const FBXMesh& mesh = geometry.meshes.at(i); Extents scaledMeshExtents = calculateScaledOffsetExtents(mesh.meshExtents, _translation, _rotation); @@ -499,6 +503,8 @@ void Model::recalculateMeshBoxes(bool pickAgainstTriangles) { _calculatedMeshBoxes[i] = AABox(scaledMeshExtents); if (pickAgainstTriangles) { + auto& thisMeshTriangleSet = _meshTriangleSets[i]; + QVector thisMeshTriangles; for (int j = 0; j < mesh.parts.size(); j++) { const FBXMeshPart& part = mesh.parts.at(j); @@ -550,6 +556,9 @@ void Model::recalculateMeshBoxes(bool pickAgainstTriangles) { thisMeshTriangles.push_back(tri1); thisMeshTriangles.push_back(tri2); + thisMeshTriangleSet.insertTriangle(tri1); + thisMeshTriangleSet.insertTriangle(tri2); + } } @@ -582,11 +591,13 @@ void Model::recalculateMeshBoxes(bool pickAgainstTriangles) { Triangle tri = { v0, v1, v2 }; thisMeshTriangles.push_back(tri); + + thisMeshTriangleSet.insertTriangle(tri); + } } _calculatedMeshPartBoxes[QPair(i, j)] = thisPartBounds; } - _calculatedMeshTriangles[i] = thisMeshTriangles; _calculatedMeshPartBoxesValid = true; } } diff --git a/libraries/render-utils/src/Model.h b/libraries/render-utils/src/Model.h index 7c373274e4..bc3cf0e521 100644 --- a/libraries/render-utils/src/Model.h +++ b/libraries/render-utils/src/Model.h @@ -28,6 +28,7 @@ #include #include #include +#include #include "GeometryCache.h" #include "TextureCache.h" @@ -362,8 +363,8 @@ protected: bool _calculatedMeshPartBoxesValid; QVector _calculatedMeshBoxes; // world coordinate AABoxes for all sub mesh boxes bool _calculatedMeshBoxesValid; + QVector< TriangleSet > _meshTriangleSets; // world coordinate triangles for all sub meshes - QVector< QVector > _calculatedMeshTriangles; // world coordinate triangles for all sub meshes bool _calculatedMeshTrianglesValid; QMutex _mutex; diff --git a/libraries/shared/src/AABox.h b/libraries/shared/src/AABox.h index 2f0b09d67a..ccc7b6e302 100644 --- a/libraries/shared/src/AABox.h +++ b/libraries/shared/src/AABox.h @@ -109,6 +109,8 @@ public: bool isInvalid() const { return _corner == INFINITY_VECTOR; } + void clear() { _corner = INFINITY_VECTOR; _scale = glm::vec3(0.0f); } + private: glm::vec3 getClosestPointOnFace(const glm::vec3& point, BoxFace face) const; glm::vec3 getClosestPointOnFace(const glm::vec4& origin, const glm::vec4& direction, BoxFace face) const; diff --git a/libraries/shared/src/TriangleSet.cpp b/libraries/shared/src/TriangleSet.cpp new file mode 100644 index 0000000000..3264c259d3 --- /dev/null +++ b/libraries/shared/src/TriangleSet.cpp @@ -0,0 +1,69 @@ +// +// TriangleSet.cpp +// libraries/entities/src +// +// Created by Brad Hefta-Gaub on 3/2/2017. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "GLMHelpers.h" +#include "TriangleSet.h" + +void TriangleSet::insertTriangle(const Triangle& t) { + _triangles.push_back(t); + + _bounds += t.v0; + _bounds += t.v1; + _bounds += t.v2; +} + +void TriangleSet::clear() { + _triangles.clear(); + _bounds.clear(); +} + +// Determine of the given ray (origin/direction) in model space intersects with any triangles +// in the set. If an intersection occurs, the distance and surface normal will be provided. +bool TriangleSet::findRayIntersection(const glm::vec3& origin, const glm::vec3& direction, + float& distance, glm::vec3& surfaceNormal) const { + + float bestDistance = std::numeric_limits::max(); + float thisTriangleDistance; + bool intersectedSomething = false; + + for (const auto& triangle : _triangles) { + if (findRayTriangleIntersection(origin, direction, triangle, thisTriangleDistance)) { + if (thisTriangleDistance < bestDistance) { + bestDistance = thisTriangleDistance; + intersectedSomething = true; + surfaceNormal = triangle.getNormal(); + distance = bestDistance; + //face = subMeshFace; + //extraInfo = geometry.getModelNameOfMesh(subMeshIndex); + } + } + } + return intersectedSomething; +} + + +bool TriangleSet::convexHullContains(const glm::vec3& point) const { + if (!_bounds.contains(point)) { + return false; + } + + bool insideMesh = true; // optimistic + for (const auto& triangle : _triangles) { + if (!isPointBehindTrianglesPlane(point, triangle.v0, triangle.v1, triangle.v2)) { + // it's not behind at least one so we bail + insideMesh = false; + break; + } + + } + return insideMesh; +} + diff --git a/libraries/shared/src/TriangleSet.h b/libraries/shared/src/TriangleSet.h new file mode 100644 index 0000000000..3aef3966b6 --- /dev/null +++ b/libraries/shared/src/TriangleSet.h @@ -0,0 +1,33 @@ +// +// TriangleSet.h +// libraries/entities/src +// +// Created by Brad Hefta-Gaub on 3/2/2017. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +#include "AABox.h" +#include "GeometryUtil.h" + +class TriangleSet { +public: + void insertTriangle(const Triangle& t); + void clear(); + + // Determine of the given ray (origin/direction) in model space intersects with any triangles + // in the set. If an intersection occurs, the distance and surface normal will be provided. + bool findRayIntersection(const glm::vec3& origin, const glm::vec3& direction, + float& distance, glm::vec3& surfaceNormal) const; + + bool convexHullContains(const glm::vec3& point) const; // this point is "inside" all triangles + const AABox& getBounds() const; + +private: + QVector _triangles; + AABox _bounds; +}; From d85cb645b0301338f5cedad9b547772d8e4b6890 Mon Sep 17 00:00:00 2001 From: Menithal Date: Sat, 4 Mar 2017 11:42:43 +0200 Subject: [PATCH 04/64] Changed limit logic, default limit is now 0 Clones now have a named based on the original entity id Limit is now calculated from the source instance, instead of just clone name to avoid a single box being calculated as something else Default limit is now 0, which disables limit --- .../system/controllers/handControllerGrab.js | 76 +++++++++---------- scripts/system/html/js/entityProperties.js | 6 +- 2 files changed, 39 insertions(+), 43 deletions(-) diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js index 2e1d538db9..26178773e9 100644 --- a/scripts/system/controllers/handControllerGrab.js +++ b/scripts/system/controllers/handControllerGrab.js @@ -274,24 +274,24 @@ CONTROLLER_STATE_MACHINE[STATE_OVERLAY_LASER_TOUCHING] = CONTROLLER_STATE_MACHIN // Object assign polyfill if (typeof Object.assign != 'function') { - Object.assign = function(target, varArgs) { - 'use strict'; - if (target == null) { - throw new TypeError('Cannot convert undefined or null to object'); - } - var to = Object(target); - for (var index = 1; index < arguments.length; index++) { - var nextSource = arguments[index]; - if (nextSource != null) { - for (var nextKey in nextSource) { - if (Object.prototype.hasOwnProperty.call(nextSource, nextKey)) { - to[nextKey] = nextSource[nextKey]; - } + Object.assign = function(target, varArgs) { + 'use strict'; + if (target == null) { + throw new TypeError('Cannot convert undefined or null to object'); } - } - } - return to; - }; + var to = Object(target); + for (var index = 1; index < arguments.length; index++) { + var nextSource = arguments[index]; + if (nextSource != null) { + for (var nextKey in nextSource) { + if (Object.prototype.hasOwnProperty.call(nextSource, nextKey)) { + to[nextKey] = nextSource[nextKey]; + } + } + } + } + return to; + }; } function distanceBetweenPointAndEntityBoundingBox(point, entityProps) { @@ -1524,16 +1524,16 @@ function MyController(hand) { return true; }; this.entityIsCloneable = function(entityID) { - var entityProps = entityPropertiesCache.getGrabbableProps(entityID); - var props = entityPropertiesCache.getProps(entityID); - if (!props) { - return false; - } + var entityProps = entityPropertiesCache.getGrabbableProps(entityID); + var props = entityPropertiesCache.getProps(entityID); + if (!props) { + return false; + } - if (entityProps.hasOwnProperty("cloneable")) { - return entityProps.cloneable; - } - return false; + if (entityProps.hasOwnProperty("cloneable")) { + return entityProps.cloneable; + } + return false; } this.entityIsGrabbable = function(entityID) { var grabbableProps = entityPropertiesCache.getGrabbableProps(entityID); @@ -2229,7 +2229,7 @@ function MyController(hand) { // Can't set state of other controller to STATE_DISTANCE_HOLDING because then either: // (a) The entity would jump to line up with the formerly rotating controller's orientation, or // (b) The grab beam would need an orientation offset to the controller's true orientation. - // Neither of these options is good, so instead set STATE_SEARCHING and subsequently let the formerly distance + // Neither of these options is good, so instead set STATE_SEARCHING and subsequently let the formerly distance // rotating controller start distance holding the entity if it happens to be pointing at the entity. } return; @@ -2601,31 +2601,29 @@ function MyController(hand) { var userData = JSON.parse(grabbedProperties.userData); var grabInfo = userData.grabbableKey; if (grabInfo && grabInfo.cloneable) { - // Check if - var worldEntities = Entities.findEntitiesInBox(Vec3.subtract(MyAvatar.position, {x:25,y:25, z:25}), {x:50, y: 50, z: 50}) + var worldEntities = Entities.findEntities(MyAvatar.position, 50); var count = 0; worldEntities.forEach(function(item) { var item = Entities.getEntityProperties(item, ["name"]); - if (item.name === grabbedProperties.name) { + if (item.name.indexOf('-clone-' + grabbedProperties.id) !== -1) { count++; } }) + + var limit = grabInfo.cloneLimit ? grabInfo.cloneLimit : 0; + if (count >= limit && limit !== 0) { + delete limit; + return; + } + var cloneableProps = Entities.getEntityProperties(grabbedProperties.id); + cloneableProps.name = cloneableProps.name + '-clone-' + grabbedProperties.id; var lifetime = grabInfo.cloneLifetime ? grabInfo.cloneLifetime : 300; - var limit = grabInfo.cloneLimit ? grabInfo.cloneLimit : 10; var dynamic = grabInfo.cloneDynamic ? grabInfo.cloneDynamic : false; var cUserData = Object.assign({}, userData); var cProperties = Object.assign({}, cloneableProps); isClone = true; - if (count > limit) { - delete cloneableProps; - delete lifetime; - delete cUserData; - delete cProperties; - return; - } - delete cUserData.grabbableKey.cloneLifetime; delete cUserData.grabbableKey.cloneable; delete cUserData.grabbableKey.cloneDynamic; diff --git a/scripts/system/html/js/entityProperties.js b/scripts/system/html/js/entityProperties.js index 3280e1f196..66abcaa67a 100644 --- a/scripts/system/html/js/entityProperties.js +++ b/scripts/system/html/js/entityProperties.js @@ -879,7 +879,7 @@ function loaded() { elCloneable.checked = false; elCloneableDynamic.checked = false; elCloneableGroup.style.display = elCloneable.checked ? "block": "none"; - elCloneableLimit.value = 10; + elCloneableLimit.value = 0; elCloneableLifetime.value = 300; var parsedUserData = {} @@ -899,8 +899,6 @@ function loaded() { if ("cloneable" in parsedUserData["grabbableKey"]) { elCloneable.checked = parsedUserData["grabbableKey"].cloneable; elCloneableGroup.style.display = elCloneable.checked ? "block": "none"; - elCloneableLimit.value = elCloneable.checked ? 10: 0; - elCloneableLifetime.value = elCloneable.checked ? 300: 0; elCloneableDynamic.checked = parsedUserData["grabbableKey"].cloneDynamic ? parsedUserData["grabbableKey"].cloneDynamic : properties.dynamic; elDynamic.checked = elCloneable.checked ? false: properties.dynamic; if (elCloneable.checked) { @@ -908,7 +906,7 @@ function loaded() { elCloneableLifetime.value = parsedUserData["grabbableKey"].cloneLifetime ? parsedUserData["grabbableKey"].cloneLifetime : 300; } if ("cloneLimit" in parsedUserData["grabbableKey"]) { - elCloneableLimit.value = parsedUserData["grabbableKey"].cloneLimit ? parsedUserData["grabbableKey"].cloneLimit : 10; + elCloneableLimit.value = parsedUserData["grabbableKey"].cloneLimit ? parsedUserData["grabbableKey"].cloneLimit : 0; } } } From 32add6635d8588106eddf2d1a235b6e118d2450d Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Sat, 4 Mar 2017 23:33:17 -0800 Subject: [PATCH 05/64] checkpoint on reducing raypicking recalcs --- libraries/fbx/src/FBXReader.h | 2 +- libraries/render-utils/src/Model.cpp | 98 ++++++++++++++++------------ libraries/render-utils/src/Model.h | 6 +- 3 files changed, 62 insertions(+), 44 deletions(-) diff --git a/libraries/fbx/src/FBXReader.h b/libraries/fbx/src/FBXReader.h index 5910b8d312..6e51c413dc 100644 --- a/libraries/fbx/src/FBXReader.h +++ b/libraries/fbx/src/FBXReader.h @@ -300,7 +300,7 @@ public: QHash materials; - glm::mat4 offset; + glm::mat4 offset; // This includes offset, rotation, and scale as specified by the FST file int leftEyeJointIndex = -1; int rightEyeJointIndex = -1; diff --git a/libraries/render-utils/src/Model.cpp b/libraries/render-utils/src/Model.cpp index 9dd3563887..d94bb67590 100644 --- a/libraries/render-utils/src/Model.cpp +++ b/libraries/render-utils/src/Model.cpp @@ -375,6 +375,14 @@ bool Model::findRayIntersectionAgainstSubMeshes(const glm::vec3& origin, const g recalculateMeshBoxes(pickAgainstTriangles); } + glm::mat4 meshToModelMatrix = glm::scale(_scale) * glm::translate(_offset); + glm::mat4 meshToWorldMatrix = meshToModelMatrix * glm::translate(_translation) * glm::mat4_cast(_rotation); + glm::mat4 worldToMeshMatrix = glm::inverse(meshToWorldMatrix); + + glm::vec3 meshFrameOrigin = glm::vec3(worldToMeshMatrix * glm::vec4(origin, 1.0f)); + glm::vec3 meshFrameDirection = glm::vec3(worldToMeshMatrix * glm::vec4(direction, 0.0f)); + + for (const auto& subMeshBox : _calculatedMeshBoxes) { bool intersectedSubMesh = false; float subMeshDistance = std::numeric_limits::max(); @@ -383,19 +391,24 @@ bool Model::findRayIntersectionAgainstSubMeshes(const glm::vec3& origin, const g if (distanceToSubMesh < bestDistance) { if (pickAgainstTriangles) { - float subMeshDistance; + float subMeshDistance = 0.0f; glm::vec3 subMeshNormal; - const auto& meshTriangleSet = _meshTriangleSets[subMeshIndex]; - bool intersectedMesh = meshTriangleSet.findRayIntersection(origin, direction, subMeshDistance, subMeshNormal); + const auto& meshTriangleSet = _modelSpaceMeshTriangleSets[subMeshIndex]; + bool intersectedMesh = meshTriangleSet.findRayIntersection(meshFrameOrigin, meshFrameDirection, subMeshDistance, subMeshNormal); - if (intersectedMesh && subMeshDistance < bestDistance) { - bestDistance = subMeshDistance; - intersectedSomething = true; - face = subMeshFace; - surfaceNormal = subMeshNormal; - extraInfo = geometry.getModelNameOfMesh(subMeshIndex); + if (intersectedMesh) { + glm::vec3 meshIntersectionPoint = meshFrameOrigin + (meshFrameDirection * subMeshDistance); + glm::vec3 worldIntersectionPoint = glm::vec3(meshToWorldMatrix * glm::vec4(meshIntersectionPoint, 1.0f)); + float worldDistance = glm::distance(origin, worldIntersectionPoint); + + if (worldDistance < bestDistance) { + bestDistance = subMeshDistance; + intersectedSomething = true; + face = subMeshFace; + surfaceNormal = glm::vec3(meshToWorldMatrix * glm::vec4(subMeshNormal, 0.0f)); + extraInfo = geometry.getModelNameOfMesh(subMeshIndex); + } } - } else { // this is the non-triangle picking case... bestDistance = distanceToSubMesh; @@ -458,7 +471,13 @@ bool Model::convexHullContains(glm::vec3 point) { foreach(const AABox& subMeshBox, _calculatedMeshBoxes) { if (subMeshBox.contains(point)) { - if (_meshTriangleSets[subMeshIndex].convexHullContains(point)) { + glm::mat4 meshToModelMatrix = glm::scale(_scale) * glm::translate(_offset); + glm::mat4 meshToWorldMatrix = meshToModelMatrix * glm::translate(_translation) * glm::mat4_cast(_rotation); + glm::mat4 worldToMeshMatrix = glm::inverse(meshToWorldMatrix); + + glm::vec3 meshFramePoint = glm::vec3(worldToMeshMatrix * glm::vec4(point, 1.0f)); + + if (_modelSpaceMeshTriangleSets[subMeshIndex].convexHullContains(meshFramePoint)) { // It's inside this mesh, return true. _mutex.unlock(); return true; @@ -493,8 +512,9 @@ void Model::recalculateMeshBoxes(bool pickAgainstTriangles) { int numberOfMeshes = geometry.meshes.size(); _calculatedMeshBoxes.resize(numberOfMeshes); _calculatedMeshPartBoxes.clear(); - _meshTriangleSets.clear(); - _meshTriangleSets.resize(numberOfMeshes); + + _modelSpaceMeshTriangleSets.clear(); + _modelSpaceMeshTriangleSets.resize(numberOfMeshes); for (int i = 0; i < numberOfMeshes; i++) { const FBXMesh& mesh = geometry.meshes.at(i); @@ -503,9 +523,7 @@ void Model::recalculateMeshBoxes(bool pickAgainstTriangles) { _calculatedMeshBoxes[i] = AABox(scaledMeshExtents); if (pickAgainstTriangles) { - auto& thisMeshTriangleSet = _meshTriangleSets[i]; - QVector thisMeshTriangles; for (int j = 0; j < mesh.parts.size(); j++) { const FBXMeshPart& part = mesh.parts.at(j); @@ -540,24 +558,21 @@ void Model::recalculateMeshBoxes(bool pickAgainstTriangles) { thisPartBounds += mv2; thisPartBounds += mv3; - glm::vec3 v0 = calculateScaledOffsetPoint(mv0); - glm::vec3 v1 = calculateScaledOffsetPoint(mv1); - glm::vec3 v2 = calculateScaledOffsetPoint(mv2); - glm::vec3 v3 = calculateScaledOffsetPoint(mv3); + // let's also track the model space version... (eventually using only this!) + // these points will be transformed by the FST's offset, which includes the + // scaling, rotation, and translation specified by the FST/FBX, this can't change + // at runtime, so we can safely store these in our TriangleSet + { + glm::vec3 v0 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i0], 1.0f)); + glm::vec3 v1 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i1], 1.0f)); + glm::vec3 v2 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i2], 1.0f)); + glm::vec3 v3 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i3], 1.0f)); - // Sam's recommended triangle slices - Triangle tri1 = { v0, v1, v3 }; - Triangle tri2 = { v1, v2, v3 }; - - // NOTE: Random guy on the internet's recommended triangle slices - //Triangle tri1 = { v0, v1, v2 }; - //Triangle tri2 = { v2, v3, v0 }; - - thisMeshTriangles.push_back(tri1); - thisMeshTriangles.push_back(tri2); - - thisMeshTriangleSet.insertTriangle(tri1); - thisMeshTriangleSet.insertTriangle(tri2); + Triangle tri1 = { v0, v1, v3 }; + Triangle tri2 = { v1, v2, v3 }; + _modelSpaceMeshTriangleSets[i].insertTriangle(tri1); + _modelSpaceMeshTriangleSets[i].insertTriangle(tri2); + } } } @@ -584,15 +599,18 @@ void Model::recalculateMeshBoxes(bool pickAgainstTriangles) { thisPartBounds += mv1; thisPartBounds += mv2; - glm::vec3 v0 = calculateScaledOffsetPoint(mv0); - glm::vec3 v1 = calculateScaledOffsetPoint(mv1); - glm::vec3 v2 = calculateScaledOffsetPoint(mv2); + // let's also track the model space version... (eventually using only this!) + // these points will be transformed by the FST's offset, which includes the + // scaling, rotation, and translation specified by the FST/FBX, this can't change + // at runtime, so we can safely store these in our TriangleSet + { + glm::vec3 v0 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i0], 1.0f)); + glm::vec3 v1 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i1], 1.0f)); + glm::vec3 v2 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i2], 1.0f)); - Triangle tri = { v0, v1, v2 }; - - thisMeshTriangles.push_back(tri); - - thisMeshTriangleSet.insertTriangle(tri); + Triangle tri = { v0, v1, v2 }; + _modelSpaceMeshTriangleSets[i].insertTriangle(tri); + } } } diff --git a/libraries/render-utils/src/Model.h b/libraries/render-utils/src/Model.h index bc3cf0e521..763112cebc 100644 --- a/libraries/render-utils/src/Model.h +++ b/libraries/render-utils/src/Model.h @@ -294,10 +294,10 @@ protected: SpatiallyNestable* _spatiallyNestableOverride; - glm::vec3 _translation; + glm::vec3 _translation; // this is the translation in world coordinates to the model's registration point glm::quat _rotation; glm::vec3 _scale; - glm::vec3 _offset; + glm::vec3 _offset; // this is the translation for the minimum extent of the model (in original mesh coordinate space) to the model's registration point static float FAKE_DIMENSION_PLACEHOLDER; @@ -363,7 +363,7 @@ protected: bool _calculatedMeshPartBoxesValid; QVector _calculatedMeshBoxes; // world coordinate AABoxes for all sub mesh boxes bool _calculatedMeshBoxesValid; - QVector< TriangleSet > _meshTriangleSets; // world coordinate triangles for all sub meshes + QVector _modelSpaceMeshTriangleSets; // model space triangles for all sub meshes bool _calculatedMeshTrianglesValid; QMutex _mutex; From 87bcced409ea42d4e414ac308b0823afb1e3feed Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Sun, 5 Mar 2017 19:23:55 -0800 Subject: [PATCH 06/64] cleanup use of triangleSet for picking --- .../src/RenderableModelEntityItem.cpp | 6 + libraries/render-utils/src/Model.cpp | 333 ++++++------------ libraries/render-utils/src/Model.h | 32 +- libraries/shared/src/TriangleSet.cpp | 31 +- libraries/shared/src/TriangleSet.h | 4 +- 5 files changed, 134 insertions(+), 272 deletions(-) diff --git a/libraries/entities-renderer/src/RenderableModelEntityItem.cpp b/libraries/entities-renderer/src/RenderableModelEntityItem.cpp index e6902228c5..1a6daf3d43 100644 --- a/libraries/entities-renderer/src/RenderableModelEntityItem.cpp +++ b/libraries/entities-renderer/src/RenderableModelEntityItem.cpp @@ -418,6 +418,12 @@ void RenderableModelEntityItem::render(RenderArgs* args) { // Enqueue updates for the next frame if (_model) { +#if 1 //def WANT_EXTRA_RENDER_DEBUGGING + // debugging... + gpu::Batch& batch = *args->_batch; + _model->renderDebugMeshBoxes(batch); +#endif + render::ScenePointer scene = AbstractViewStateInterface::instance()->getMain3DScene(); // FIXME: this seems like it could be optimized if we tracked our last known visible state in diff --git a/libraries/render-utils/src/Model.cpp b/libraries/render-utils/src/Model.cpp index d94bb67590..172290eae5 100644 --- a/libraries/render-utils/src/Model.cpp +++ b/libraries/render-utils/src/Model.cpp @@ -96,9 +96,6 @@ Model::Model(RigPointer rig, QObject* parent, SpatiallyNestable* spatiallyNestab _isVisible(true), _blendNumber(0), _appliedBlendNumber(0), - _calculatedMeshPartBoxesValid(false), - _calculatedMeshBoxesValid(false), - _calculatedMeshTrianglesValid(false), _isWireframe(false), _rig(rig) { @@ -360,19 +357,14 @@ bool Model::findRayIntersectionAgainstSubMeshes(const glm::vec3& origin, const g // we can use the AABox's ray intersection by mapping our origin and direction into the model frame // and testing intersection there. if (modelFrameBox.findRayIntersection(modelFrameOrigin, modelFrameDirection, distance, face, surfaceNormal)) { + QMutexLocker locker(&_mutex); + float bestDistance = std::numeric_limits::max(); - - float distanceToSubMesh; - BoxFace subMeshFace; - glm::vec3 subMeshSurfaceNormal; int subMeshIndex = 0; - const FBXGeometry& geometry = getFBXGeometry(); - // If we hit the models box, then consider the submeshes... - _mutex.lock(); - if (!_calculatedMeshBoxesValid || (pickAgainstTriangles && !_calculatedMeshTrianglesValid)) { - recalculateMeshBoxes(pickAgainstTriangles); + if (!_triangleSetsValid) { + calculateTriangleSets(); } glm::mat4 meshToModelMatrix = glm::scale(_scale) * glm::translate(_offset); @@ -382,50 +374,26 @@ bool Model::findRayIntersectionAgainstSubMeshes(const glm::vec3& origin, const g glm::vec3 meshFrameOrigin = glm::vec3(worldToMeshMatrix * glm::vec4(origin, 1.0f)); glm::vec3 meshFrameDirection = glm::vec3(worldToMeshMatrix * glm::vec4(direction, 0.0f)); + for (const auto& triangleSet : _modelSpaceMeshTriangleSets) { + float triangleSetDistance = 0.0f; + BoxFace triangleSetFace; + glm::vec3 triangleSetNormal; + if (triangleSet.findRayIntersection(meshFrameOrigin, meshFrameDirection, triangleSetDistance, triangleSetFace, triangleSetNormal, pickAgainstTriangles)) { - for (const auto& subMeshBox : _calculatedMeshBoxes) { - bool intersectedSubMesh = false; - float subMeshDistance = std::numeric_limits::max(); + glm::vec3 meshIntersectionPoint = meshFrameOrigin + (meshFrameDirection * triangleSetDistance); + glm::vec3 worldIntersectionPoint = glm::vec3(meshToWorldMatrix * glm::vec4(meshIntersectionPoint, 1.0f)); + float worldDistance = glm::distance(origin, worldIntersectionPoint); - if (subMeshBox.findRayIntersection(origin, direction, distanceToSubMesh, subMeshFace, subMeshSurfaceNormal)) { - if (distanceToSubMesh < bestDistance) { - if (pickAgainstTriangles) { - - float subMeshDistance = 0.0f; - glm::vec3 subMeshNormal; - const auto& meshTriangleSet = _modelSpaceMeshTriangleSets[subMeshIndex]; - bool intersectedMesh = meshTriangleSet.findRayIntersection(meshFrameOrigin, meshFrameDirection, subMeshDistance, subMeshNormal); - - if (intersectedMesh) { - glm::vec3 meshIntersectionPoint = meshFrameOrigin + (meshFrameDirection * subMeshDistance); - glm::vec3 worldIntersectionPoint = glm::vec3(meshToWorldMatrix * glm::vec4(meshIntersectionPoint, 1.0f)); - float worldDistance = glm::distance(origin, worldIntersectionPoint); - - if (worldDistance < bestDistance) { - bestDistance = subMeshDistance; - intersectedSomething = true; - face = subMeshFace; - surfaceNormal = glm::vec3(meshToWorldMatrix * glm::vec4(subMeshNormal, 0.0f)); - extraInfo = geometry.getModelNameOfMesh(subMeshIndex); - } - } - } else { - // this is the non-triangle picking case... - bestDistance = distanceToSubMesh; - intersectedSomething = true; - face = subMeshFace; - surfaceNormal = subMeshSurfaceNormal; - extraInfo = geometry.getModelNameOfMesh(subMeshIndex); - - intersectedSubMesh = true; - } + if (worldDistance < bestDistance) { + bestDistance = worldDistance; + intersectedSomething = true; + face = triangleSetFace; + surfaceNormal = glm::vec3(meshToWorldMatrix * glm::vec4(triangleSetNormal, 0.0f)); + extraInfo = geometry.getModelNameOfMesh(subMeshIndex); } } - - subMeshIndex++; } - _mutex.unlock(); if (intersectedSomething) { distance = bestDistance; @@ -461,182 +429,97 @@ bool Model::convexHullContains(glm::vec3 point) { // we can use the AABox's contains() by mapping our point into the model frame // and testing there. if (modelFrameBox.contains(modelFramePoint)){ - _mutex.lock(); - if (!_calculatedMeshTrianglesValid) { - recalculateMeshBoxes(true); + QMutexLocker locker(&_mutex); + + if (!_triangleSetsValid) { + calculateTriangleSets(); } // If we are inside the models box, then consider the submeshes... - int subMeshIndex = 0; - foreach(const AABox& subMeshBox, _calculatedMeshBoxes) { - if (subMeshBox.contains(point)) { + glm::mat4 meshToModelMatrix = glm::scale(_scale) * glm::translate(_offset); + glm::mat4 meshToWorldMatrix = meshToModelMatrix * glm::translate(_translation) * glm::mat4_cast(_rotation); + glm::mat4 worldToMeshMatrix = glm::inverse(meshToWorldMatrix); + glm::vec3 meshFramePoint = glm::vec3(worldToMeshMatrix * glm::vec4(point, 1.0f)); - glm::mat4 meshToModelMatrix = glm::scale(_scale) * glm::translate(_offset); - glm::mat4 meshToWorldMatrix = meshToModelMatrix * glm::translate(_translation) * glm::mat4_cast(_rotation); - glm::mat4 worldToMeshMatrix = glm::inverse(meshToWorldMatrix); - - glm::vec3 meshFramePoint = glm::vec3(worldToMeshMatrix * glm::vec4(point, 1.0f)); - - if (_modelSpaceMeshTriangleSets[subMeshIndex].convexHullContains(meshFramePoint)) { + for (const auto& triangleSet : _modelSpaceMeshTriangleSets) { + const AABox& box = triangleSet.getBounds(); + if (box.contains(meshFramePoint)) { + if (triangleSet.convexHullContains(meshFramePoint)) { // It's inside this mesh, return true. - _mutex.unlock(); return true; } } - subMeshIndex++; } - _mutex.unlock(); + + } // It wasn't in any mesh, return false. return false; } -// TODO: we seem to call this too often when things haven't actually changed... look into optimizing this -// Any script might trigger findRayIntersectionAgainstSubMeshes (and maybe convexHullContains), so these -// can occur multiple times. In addition, rendering does it's own ray picking in order to decide which -// entity-scripts to call. I think it would be best to do the picking once-per-frame (in cpu, or gpu if possible) -// and then the calls use the most recent such result. -void Model::recalculateMeshBoxes(bool pickAgainstTriangles) { +void Model::calculateTriangleSets() { PROFILE_RANGE(render, __FUNCTION__); - bool calculatedMeshTrianglesNeeded = pickAgainstTriangles && !_calculatedMeshTrianglesValid; - if (!_calculatedMeshBoxesValid || calculatedMeshTrianglesNeeded || (!_calculatedMeshPartBoxesValid && pickAgainstTriangles) ) { + const FBXGeometry& geometry = getFBXGeometry(); + int numberOfMeshes = geometry.meshes.size(); - if (pickAgainstTriangles) { - qDebug() << "RECALCULATING triangles!"; - } else { - qDebug() << "RECALCULATING boxes!"; - } + _triangleSetsValid = true; + _modelSpaceMeshTriangleSets.clear(); + _modelSpaceMeshTriangleSets.resize(numberOfMeshes); - const FBXGeometry& geometry = getFBXGeometry(); - int numberOfMeshes = geometry.meshes.size(); - _calculatedMeshBoxes.resize(numberOfMeshes); - _calculatedMeshPartBoxes.clear(); + for (int i = 0; i < numberOfMeshes; i++) { + const FBXMesh& mesh = geometry.meshes.at(i); - _modelSpaceMeshTriangleSets.clear(); - _modelSpaceMeshTriangleSets.resize(numberOfMeshes); + for (int j = 0; j < mesh.parts.size(); j++) { + const FBXMeshPart& part = mesh.parts.at(j); - for (int i = 0; i < numberOfMeshes; i++) { - const FBXMesh& mesh = geometry.meshes.at(i); - Extents scaledMeshExtents = calculateScaledOffsetExtents(mesh.meshExtents, _translation, _rotation); + const int INDICES_PER_TRIANGLE = 3; + const int INDICES_PER_QUAD = 4; - _calculatedMeshBoxes[i] = AABox(scaledMeshExtents); + if (part.quadIndices.size() > 0) { + int numberOfQuads = part.quadIndices.size() / INDICES_PER_QUAD; + int vIndex = 0; + for (int q = 0; q < numberOfQuads; q++) { + int i0 = part.quadIndices[vIndex++]; + int i1 = part.quadIndices[vIndex++]; + int i2 = part.quadIndices[vIndex++]; + int i3 = part.quadIndices[vIndex++]; - if (pickAgainstTriangles) { + // track the model space version... these points will be transformed by the FST's offset, + // which includes the scaling, rotation, and translation specified by the FST/FBX, + // this can't change at runtime, so we can safely store these in our TriangleSet + glm::vec3 v0 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i0], 1.0f)); + glm::vec3 v1 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i1], 1.0f)); + glm::vec3 v2 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i2], 1.0f)); + glm::vec3 v3 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i3], 1.0f)); - for (int j = 0; j < mesh.parts.size(); j++) { - const FBXMeshPart& part = mesh.parts.at(j); - - bool atLeastOnePointInBounds = false; - AABox thisPartBounds; - - const int INDICES_PER_TRIANGLE = 3; - const int INDICES_PER_QUAD = 4; - - if (part.quadIndices.size() > 0) { - int numberOfQuads = part.quadIndices.size() / INDICES_PER_QUAD; - int vIndex = 0; - for (int q = 0; q < numberOfQuads; q++) { - int i0 = part.quadIndices[vIndex++]; - int i1 = part.quadIndices[vIndex++]; - int i2 = part.quadIndices[vIndex++]; - int i3 = part.quadIndices[vIndex++]; - - glm::vec3 mv0 = glm::vec3(mesh.modelTransform * glm::vec4(mesh.vertices[i0], 1.0f)); - glm::vec3 mv1 = glm::vec3(mesh.modelTransform * glm::vec4(mesh.vertices[i1], 1.0f)); - glm::vec3 mv2 = glm::vec3(mesh.modelTransform * glm::vec4(mesh.vertices[i2], 1.0f)); - glm::vec3 mv3 = glm::vec3(mesh.modelTransform * glm::vec4(mesh.vertices[i3], 1.0f)); - - // track the mesh parts in model space - if (!atLeastOnePointInBounds) { - thisPartBounds.setBox(mv0, 0.0f); - atLeastOnePointInBounds = true; - } else { - thisPartBounds += mv0; - } - thisPartBounds += mv1; - thisPartBounds += mv2; - thisPartBounds += mv3; - - // let's also track the model space version... (eventually using only this!) - // these points will be transformed by the FST's offset, which includes the - // scaling, rotation, and translation specified by the FST/FBX, this can't change - // at runtime, so we can safely store these in our TriangleSet - { - glm::vec3 v0 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i0], 1.0f)); - glm::vec3 v1 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i1], 1.0f)); - glm::vec3 v2 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i2], 1.0f)); - glm::vec3 v3 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i3], 1.0f)); - - Triangle tri1 = { v0, v1, v3 }; - Triangle tri2 = { v1, v2, v3 }; - _modelSpaceMeshTriangleSets[i].insertTriangle(tri1); - _modelSpaceMeshTriangleSets[i].insertTriangle(tri2); - } - - } - } - - if (part.triangleIndices.size() > 0) { - int numberOfTris = part.triangleIndices.size() / INDICES_PER_TRIANGLE; - int vIndex = 0; - for (int t = 0; t < numberOfTris; t++) { - int i0 = part.triangleIndices[vIndex++]; - int i1 = part.triangleIndices[vIndex++]; - int i2 = part.triangleIndices[vIndex++]; - - glm::vec3 mv0 = glm::vec3(mesh.modelTransform * glm::vec4(mesh.vertices[i0], 1.0f)); - glm::vec3 mv1 = glm::vec3(mesh.modelTransform * glm::vec4(mesh.vertices[i1], 1.0f)); - glm::vec3 mv2 = glm::vec3(mesh.modelTransform * glm::vec4(mesh.vertices[i2], 1.0f)); - - // track the mesh parts in model space - if (!atLeastOnePointInBounds) { - thisPartBounds.setBox(mv0, 0.0f); - atLeastOnePointInBounds = true; - } else { - thisPartBounds += mv0; - } - thisPartBounds += mv1; - thisPartBounds += mv2; - - // let's also track the model space version... (eventually using only this!) - // these points will be transformed by the FST's offset, which includes the - // scaling, rotation, and translation specified by the FST/FBX, this can't change - // at runtime, so we can safely store these in our TriangleSet - { - glm::vec3 v0 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i0], 1.0f)); - glm::vec3 v1 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i1], 1.0f)); - glm::vec3 v2 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i2], 1.0f)); - - Triangle tri = { v0, v1, v2 }; - _modelSpaceMeshTriangleSets[i].insertTriangle(tri); - } - - } - } - _calculatedMeshPartBoxes[QPair(i, j)] = thisPartBounds; + Triangle tri1 = { v0, v1, v3 }; + Triangle tri2 = { v1, v2, v3 }; + _modelSpaceMeshTriangleSets[i].insertTriangle(tri1); + _modelSpaceMeshTriangleSets[i].insertTriangle(tri2); + } + } + + if (part.triangleIndices.size() > 0) { + int numberOfTris = part.triangleIndices.size() / INDICES_PER_TRIANGLE; + int vIndex = 0; + for (int t = 0; t < numberOfTris; t++) { + int i0 = part.triangleIndices[vIndex++]; + int i1 = part.triangleIndices[vIndex++]; + int i2 = part.triangleIndices[vIndex++]; + + // track the model space version... these points will be transformed by the FST's offset, + // which includes the scaling, rotation, and translation specified by the FST/FBX, + // this can't change at runtime, so we can safely store these in our TriangleSet + glm::vec3 v0 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i0], 1.0f)); + glm::vec3 v1 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i1], 1.0f)); + glm::vec3 v2 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i2], 1.0f)); + + Triangle tri = { v0, v1, v2 }; + _modelSpaceMeshTriangleSets[i].insertTriangle(tri); } - _calculatedMeshPartBoxesValid = true; } } - _calculatedMeshBoxesValid = true; - _calculatedMeshTrianglesValid = pickAgainstTriangles; - } -} - -void Model::renderSetup(RenderArgs* args) { - // set up dilated textures on first render after load/simulate - const FBXGeometry& geometry = getFBXGeometry(); - if (_dilatedTextures.isEmpty()) { - foreach (const FBXMesh& mesh, geometry.meshes) { - QVector > dilated; - dilated.resize(mesh.parts.size()); - _dilatedTextures.append(dilated); - } - } - - if (!_addedToScene && isLoaded()) { - createRenderItemSet(); } } @@ -752,7 +635,15 @@ void Model::removeFromScene(std::shared_ptr scene, render::Pendin void Model::renderDebugMeshBoxes(gpu::Batch& batch) { int colorNdx = 0; _mutex.lock(); - foreach(AABox box, _calculatedMeshBoxes) { + + glm::mat4 meshToModelMatrix = glm::scale(_scale) * glm::translate(_offset); + glm::mat4 meshToWorldMatrix = meshToModelMatrix * glm::translate(_translation) * glm::mat4_cast(_rotation); + Transform meshToWorld(meshToWorldMatrix); + batch.setModelTransform(meshToWorld); + + for(const auto& triangleSet : _modelSpaceMeshTriangleSets) { + auto box = triangleSet.getBounds(); + if (_debugMeshBoxesID == GeometryCache::UNKNOWN_ID) { _debugMeshBoxesID = DependencyManager::get()->allocateID(); } @@ -784,8 +675,8 @@ void Model::renderDebugMeshBoxes(gpu::Batch& batch) { points << blf << tlf; glm::vec4 color[] = { - { 1.0f, 0.0f, 0.0f, 1.0f }, // red { 0.0f, 1.0f, 0.0f, 1.0f }, // green + { 1.0f, 0.0f, 0.0f, 1.0f }, // red { 0.0f, 0.0f, 1.0f, 1.0f }, // blue { 1.0f, 0.0f, 1.0f, 1.0f }, // purple { 1.0f, 1.0f, 0.0f, 1.0f }, // yellow @@ -843,37 +734,6 @@ Extents Model::getUnscaledMeshExtents() const { return scaledExtents; } -Extents Model::calculateScaledOffsetExtents(const Extents& extents, - glm::vec3 modelPosition, glm::quat modelOrientation) const { - // we need to include any fst scaling, translation, and rotation, which is captured in the offset matrix - glm::vec3 minimum = glm::vec3(getFBXGeometry().offset * glm::vec4(extents.minimum, 1.0f)); - glm::vec3 maximum = glm::vec3(getFBXGeometry().offset * glm::vec4(extents.maximum, 1.0f)); - - Extents scaledOffsetExtents = { ((minimum + _offset) * _scale), - ((maximum + _offset) * _scale) }; - - Extents rotatedExtents = scaledOffsetExtents.getRotated(modelOrientation); - - Extents translatedExtents = { rotatedExtents.minimum + modelPosition, - rotatedExtents.maximum + modelPosition }; - - return translatedExtents; -} - -/// Returns the world space equivalent of some box in model space. -AABox Model::calculateScaledOffsetAABox(const AABox& box, glm::vec3 modelPosition, glm::quat modelOrientation) const { - return AABox(calculateScaledOffsetExtents(Extents(box), modelPosition, modelOrientation)); -} - -glm::vec3 Model::calculateScaledOffsetPoint(const glm::vec3& point) const { - // we need to include any fst scaling, translation, and rotation, which is captured in the offset matrix - glm::vec3 offsetPoint = glm::vec3(getFBXGeometry().offset * glm::vec4(point, 1.0f)); - glm::vec3 scaledPoint = ((offsetPoint + _offset) * _scale); - glm::vec3 rotatedPoint = _rotation * scaledPoint; - glm::vec3 translatedPoint = rotatedPoint + _translation; - return translatedPoint; -} - void Model::clearJointState(int index) { _rig->clearJointState(index); } @@ -1159,8 +1019,11 @@ void Model::simulate(float deltaTime, bool fullUpdate) { // they really only become invalid if something about the transform to world space has changed. This is // not too bad at this point, because it doesn't impact rendering. However it does slow down ray picking // because ray picking needs valid boxes to work - _calculatedMeshBoxesValid = false; - _calculatedMeshTrianglesValid = false; + //_calculatedMeshBoxesValid = false; + //_calculatedMeshTrianglesValid = false; + + // FIXME -- if the model URL changes, then we need to recalculate the triangle sets??!!!! + onInvalidate(); // check for scale to fit diff --git a/libraries/render-utils/src/Model.h b/libraries/render-utils/src/Model.h index 763112cebc..05626d1ebe 100644 --- a/libraries/render-utils/src/Model.h +++ b/libraries/render-utils/src/Model.h @@ -96,7 +96,6 @@ public: render::PendingChanges& pendingChanges, render::Item::Status::Getters& statusGetters); void removeFromScene(std::shared_ptr scene, render::PendingChanges& pendingChanges); - void renderSetup(RenderArgs* args); bool isRenderable() const; bool isVisible() const { return _isVisible; } @@ -251,6 +250,9 @@ public: uint32_t getGeometryCounter() const { return _deleteGeometryCounter; } const QMap& getRenderItems() const { return _modelMeshRenderItems; } + void renderDebugMeshBoxes(gpu::Batch& batch); + + public slots: void loadURLFinished(bool success); @@ -267,15 +269,6 @@ protected: /// Returns the unscaled extents of the model's mesh Extents getUnscaledMeshExtents() const; - /// Returns the scaled equivalent of some extents in model space. - Extents calculateScaledOffsetExtents(const Extents& extents, glm::vec3 modelPosition, glm::quat modelOrientation) const; - - /// Returns the world space equivalent of some box in model space. - AABox calculateScaledOffsetAABox(const AABox& box, glm::vec3 modelPosition, glm::quat modelOrientation) const; - - /// Returns the scaled equivalent of a point in model space. - glm::vec3 calculateScaledOffsetPoint(const glm::vec3& point) const; - /// Clear the joint states void clearJointState(int index); @@ -332,14 +325,13 @@ protected: /// Allow sub classes to force invalidating the bboxes void invalidCalculatedMeshBoxes() { - _calculatedMeshBoxesValid = false; - _calculatedMeshPartBoxesValid = false; - _calculatedMeshTrianglesValid = false; + _triangleSetsValid = false; } // hook for derived classes to be notified when setUrl invalidates the current model. virtual void onInvalidate() {}; + protected: virtual void deleteGeometry(); @@ -358,17 +350,12 @@ protected: int _blendNumber; int _appliedBlendNumber; - QHash, AABox> _calculatedMeshPartBoxes; // world coordinate AABoxes for all sub mesh part boxes - - bool _calculatedMeshPartBoxesValid; - QVector _calculatedMeshBoxes; // world coordinate AABoxes for all sub mesh boxes - bool _calculatedMeshBoxesValid; - QVector _modelSpaceMeshTriangleSets; // model space triangles for all sub meshes - - bool _calculatedMeshTrianglesValid; QMutex _mutex; - void recalculateMeshBoxes(bool pickAgainstTriangles = false); + bool _triangleSetsValid { false }; + void calculateTriangleSets(); + QVector _modelSpaceMeshTriangleSets; // model space triangles for all sub meshes + void createRenderItemSet(); virtual void createVisibleRenderItemSet(); @@ -377,7 +364,6 @@ protected: bool _isWireframe; // debug rendering support - void renderDebugMeshBoxes(gpu::Batch& batch); int _debugMeshBoxesID = GeometryCache::UNKNOWN_ID; diff --git a/libraries/shared/src/TriangleSet.cpp b/libraries/shared/src/TriangleSet.cpp index 3264c259d3..5a634acc77 100644 --- a/libraries/shared/src/TriangleSet.cpp +++ b/libraries/shared/src/TriangleSet.cpp @@ -28,24 +28,31 @@ void TriangleSet::clear() { // Determine of the given ray (origin/direction) in model space intersects with any triangles // in the set. If an intersection occurs, the distance and surface normal will be provided. bool TriangleSet::findRayIntersection(const glm::vec3& origin, const glm::vec3& direction, - float& distance, glm::vec3& surfaceNormal) const { + float& distance, BoxFace& face, glm::vec3& surfaceNormal, bool precision) const { - float bestDistance = std::numeric_limits::max(); - float thisTriangleDistance; bool intersectedSomething = false; + float boxDistance = std::numeric_limits::max(); + float bestDistance = std::numeric_limits::max(); - for (const auto& triangle : _triangles) { - if (findRayTriangleIntersection(origin, direction, triangle, thisTriangleDistance)) { - if (thisTriangleDistance < bestDistance) { - bestDistance = thisTriangleDistance; - intersectedSomething = true; - surfaceNormal = triangle.getNormal(); - distance = bestDistance; - //face = subMeshFace; - //extraInfo = geometry.getModelNameOfMesh(subMeshIndex); + if (_bounds.findRayIntersection(origin, direction, boxDistance, face, surfaceNormal)) { + if (precision) { + for (const auto& triangle : _triangles) { + float thisTriangleDistance; + if (findRayTriangleIntersection(origin, direction, triangle, thisTriangleDistance)) { + if (thisTriangleDistance < bestDistance) { + bestDistance = thisTriangleDistance; + intersectedSomething = true; + surfaceNormal = triangle.getNormal(); + distance = bestDistance; + } + } } + } else { + intersectedSomething = true; + distance = boxDistance; } } + return intersectedSomething; } diff --git a/libraries/shared/src/TriangleSet.h b/libraries/shared/src/TriangleSet.h index 3aef3966b6..8024057174 100644 --- a/libraries/shared/src/TriangleSet.h +++ b/libraries/shared/src/TriangleSet.h @@ -22,10 +22,10 @@ public: // Determine of the given ray (origin/direction) in model space intersects with any triangles // in the set. If an intersection occurs, the distance and surface normal will be provided. bool findRayIntersection(const glm::vec3& origin, const glm::vec3& direction, - float& distance, glm::vec3& surfaceNormal) const; + float& distance, BoxFace& face, glm::vec3& surfaceNormal, bool precision) const; bool convexHullContains(const glm::vec3& point) const; // this point is "inside" all triangles - const AABox& getBounds() const; + const AABox& getBounds() const { return _bounds; } private: QVector _triangles; From 2a663cbcb16352c043e38f6f6744e1615045df5d Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Sun, 5 Mar 2017 19:28:15 -0800 Subject: [PATCH 07/64] cleanup --- .../entities-renderer/src/RenderableModelEntityItem.cpp | 2 +- libraries/render-utils/src/Model.cpp | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/libraries/entities-renderer/src/RenderableModelEntityItem.cpp b/libraries/entities-renderer/src/RenderableModelEntityItem.cpp index 1a6daf3d43..935dd4e796 100644 --- a/libraries/entities-renderer/src/RenderableModelEntityItem.cpp +++ b/libraries/entities-renderer/src/RenderableModelEntityItem.cpp @@ -418,7 +418,7 @@ void RenderableModelEntityItem::render(RenderArgs* args) { // Enqueue updates for the next frame if (_model) { -#if 1 //def WANT_EXTRA_RENDER_DEBUGGING +#ifdef WANT_EXTRA_RENDER_DEBUGGING // debugging... gpu::Batch& batch = *args->_batch; _model->renderDebugMeshBoxes(batch); diff --git a/libraries/render-utils/src/Model.cpp b/libraries/render-utils/src/Model.cpp index 172290eae5..6be23ad615 100644 --- a/libraries/render-utils/src/Model.cpp +++ b/libraries/render-utils/src/Model.cpp @@ -1015,15 +1015,6 @@ void Model::simulate(float deltaTime, bool fullUpdate) { || (_snapModelToRegistrationPoint && !_snappedToRegistrationPoint); if (isActive() && fullUpdate) { - // NOTE: This is overly aggressive and we are invalidating the MeshBoxes when in fact they may not be invalid - // they really only become invalid if something about the transform to world space has changed. This is - // not too bad at this point, because it doesn't impact rendering. However it does slow down ray picking - // because ray picking needs valid boxes to work - //_calculatedMeshBoxesValid = false; - //_calculatedMeshTrianglesValid = false; - - // FIXME -- if the model URL changes, then we need to recalculate the triangle sets??!!!! - onInvalidate(); // check for scale to fit From af97e9bdd9f1ec8bece9166253f04044b828f8c7 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 6 Mar 2017 10:49:38 -0800 Subject: [PATCH 08/64] CR feedback --- libraries/render-utils/src/Model.cpp | 21 +++++++++++++-------- libraries/render-utils/src/Model.h | 6 +++++- libraries/shared/src/TriangleSet.cpp | 2 +- libraries/shared/src/TriangleSet.h | 13 +++++++++---- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/libraries/render-utils/src/Model.cpp b/libraries/render-utils/src/Model.cpp index 6be23ad615..ae20823138 100644 --- a/libraries/render-utils/src/Model.cpp +++ b/libraries/render-utils/src/Model.cpp @@ -368,7 +368,7 @@ bool Model::findRayIntersectionAgainstSubMeshes(const glm::vec3& origin, const g } glm::mat4 meshToModelMatrix = glm::scale(_scale) * glm::translate(_offset); - glm::mat4 meshToWorldMatrix = meshToModelMatrix * glm::translate(_translation) * glm::mat4_cast(_rotation); + glm::mat4 meshToWorldMatrix = meshToModelMatrix * createMatFromQuatAndPos(_rotation, _translation); glm::mat4 worldToMeshMatrix = glm::inverse(meshToWorldMatrix); glm::vec3 meshFrameOrigin = glm::vec3(worldToMeshMatrix * glm::vec4(origin, 1.0f)); @@ -437,7 +437,7 @@ bool Model::convexHullContains(glm::vec3 point) { // If we are inside the models box, then consider the submeshes... glm::mat4 meshToModelMatrix = glm::scale(_scale) * glm::translate(_offset); - glm::mat4 meshToWorldMatrix = meshToModelMatrix * glm::translate(_translation) * glm::mat4_cast(_rotation); + glm::mat4 meshToWorldMatrix = meshToModelMatrix * createMatFromQuatAndPos(_rotation, _translation); glm::mat4 worldToMeshMatrix = glm::inverse(meshToWorldMatrix); glm::vec3 meshFramePoint = glm::vec3(worldToMeshMatrix * glm::vec4(point, 1.0f)); @@ -475,9 +475,15 @@ void Model::calculateTriangleSets() { const int INDICES_PER_TRIANGLE = 3; const int INDICES_PER_QUAD = 4; + const int TRIANGLES_PER_QUAD = 2; + + // tell our triangleSet how many triangles to expect. + int numberOfQuads = part.quadIndices.size() / INDICES_PER_QUAD; + int numberOfTris = part.triangleIndices.size() / INDICES_PER_TRIANGLE; + int totalTriangles = (numberOfQuads * TRIANGLES_PER_QUAD) + numberOfTris; + _modelSpaceMeshTriangleSets[i].reserve(totalTriangles); if (part.quadIndices.size() > 0) { - int numberOfQuads = part.quadIndices.size() / INDICES_PER_QUAD; int vIndex = 0; for (int q = 0; q < numberOfQuads; q++) { int i0 = part.quadIndices[vIndex++]; @@ -495,13 +501,12 @@ void Model::calculateTriangleSets() { Triangle tri1 = { v0, v1, v3 }; Triangle tri2 = { v1, v2, v3 }; - _modelSpaceMeshTriangleSets[i].insertTriangle(tri1); - _modelSpaceMeshTriangleSets[i].insertTriangle(tri2); + _modelSpaceMeshTriangleSets[i].insert(tri1); + _modelSpaceMeshTriangleSets[i].insert(tri2); } } if (part.triangleIndices.size() > 0) { - int numberOfTris = part.triangleIndices.size() / INDICES_PER_TRIANGLE; int vIndex = 0; for (int t = 0; t < numberOfTris; t++) { int i0 = part.triangleIndices[vIndex++]; @@ -516,7 +521,7 @@ void Model::calculateTriangleSets() { glm::vec3 v2 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i2], 1.0f)); Triangle tri = { v0, v1, v2 }; - _modelSpaceMeshTriangleSets[i].insertTriangle(tri); + _modelSpaceMeshTriangleSets[i].insert(tri); } } } @@ -637,7 +642,7 @@ void Model::renderDebugMeshBoxes(gpu::Batch& batch) { _mutex.lock(); glm::mat4 meshToModelMatrix = glm::scale(_scale) * glm::translate(_offset); - glm::mat4 meshToWorldMatrix = meshToModelMatrix * glm::translate(_translation) * glm::mat4_cast(_rotation); + glm::mat4 meshToWorldMatrix = meshToModelMatrix * createMatFromQuatAndPos(_rotation, _translation); Transform meshToWorld(meshToWorldMatrix); batch.setModelTransform(meshToWorld); diff --git a/libraries/render-utils/src/Model.h b/libraries/render-utils/src/Model.h index 05626d1ebe..41821736f7 100644 --- a/libraries/render-utils/src/Model.h +++ b/libraries/render-utils/src/Model.h @@ -290,7 +290,11 @@ protected: glm::vec3 _translation; // this is the translation in world coordinates to the model's registration point glm::quat _rotation; glm::vec3 _scale; - glm::vec3 _offset; // this is the translation for the minimum extent of the model (in original mesh coordinate space) to the model's registration point + + // For entity models this is the translation for the minimum extent of the model (in original mesh coordinate space) + // to the model's registration point. For avatar models this is the translation from the avatar's hips, as determined + // by the default pose, to the origin. + glm::vec3 _offset; static float FAKE_DIMENSION_PLACEHOLDER; diff --git a/libraries/shared/src/TriangleSet.cpp b/libraries/shared/src/TriangleSet.cpp index 5a634acc77..cdb3fd6b2c 100644 --- a/libraries/shared/src/TriangleSet.cpp +++ b/libraries/shared/src/TriangleSet.cpp @@ -12,7 +12,7 @@ #include "GLMHelpers.h" #include "TriangleSet.h" -void TriangleSet::insertTriangle(const Triangle& t) { +void TriangleSet::insert(const Triangle& t) { _triangles.push_back(t); _bounds += t.v0; diff --git a/libraries/shared/src/TriangleSet.h b/libraries/shared/src/TriangleSet.h index 8024057174..cc9dd1cc6d 100644 --- a/libraries/shared/src/TriangleSet.h +++ b/libraries/shared/src/TriangleSet.h @@ -16,15 +16,20 @@ class TriangleSet { public: - void insertTriangle(const Triangle& t); + void reserve(size_t size) { _triangles.reserve((int)size); } // reserve space in the datastructure for size number of triangles + + void insert(const Triangle& t); void clear(); - // Determine of the given ray (origin/direction) in model space intersects with any triangles - // in the set. If an intersection occurs, the distance and surface normal will be provided. + // Determine if the given ray (origin/direction) in model space intersects with any triangles in the set. If an + // intersection occurs, the distance and surface normal will be provided. bool findRayIntersection(const glm::vec3& origin, const glm::vec3& direction, float& distance, BoxFace& face, glm::vec3& surfaceNormal, bool precision) const; - bool convexHullContains(const glm::vec3& point) const; // this point is "inside" all triangles + // Determine if a point is "inside" all the triangles of a convex hull. It is the responsibility of the caller to + // determine that the triangle set is indeed a convex hull. If the triangles added to this set are not in fact a + // convex hull, the result of this method is meaningless and undetermined. + bool convexHullContains(const glm::vec3& point) const; const AABox& getBounds() const { return _bounds; } private: From faba16033103ae971b91a01f8ddd739dfd7b9995 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 6 Mar 2017 11:10:04 -0800 Subject: [PATCH 09/64] more CR feedback --- libraries/shared/src/TriangleSet.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/shared/src/TriangleSet.h b/libraries/shared/src/TriangleSet.h index cc9dd1cc6d..87336c77af 100644 --- a/libraries/shared/src/TriangleSet.h +++ b/libraries/shared/src/TriangleSet.h @@ -9,14 +9,14 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#include +#include #include "AABox.h" #include "GeometryUtil.h" class TriangleSet { public: - void reserve(size_t size) { _triangles.reserve((int)size); } // reserve space in the datastructure for size number of triangles + void reserve(size_t size) { _triangles.reserve(size); } // reserve space in the datastructure for size number of triangles void insert(const Triangle& t); void clear(); @@ -33,6 +33,6 @@ public: const AABox& getBounds() const { return _bounds; } private: - QVector _triangles; + std::vector _triangles; AABox _bounds; }; From e076b32f9c29160a744d29e167428f52446e52e8 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Mon, 6 Mar 2017 10:15:53 -0800 Subject: [PATCH 10/64] don't automatically unhook overlays from hands unless they were grabbable overlays --- scripts/system/controllers/handControllerGrab.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js index d982a032cc..cd18d6f932 100644 --- a/scripts/system/controllers/handControllerGrab.js +++ b/scripts/system/controllers/handControllerGrab.js @@ -3213,13 +3213,21 @@ function MyController(hand) { } _this.previouslyUnhooked[childID] = now; - // we don't know if it's an entity or an overlay + if (Overlays.getProperty(childID, "grabbable")) { + // only auto-unhook overlays that were flagged as grabbable. this avoids unhooking overlays + // used in tutorial. + Overlays.editOverlay(childID, { + parentID: previousParentID, + parentJointIndex: previousParentJointIndex + }); + } Entities.editEntity(childID, { parentID: previousParentID, parentJointIndex: previousParentJointIndex }); - Overlays.editOverlay(childID, { parentID: previousParentID, parentJointIndex: previousParentJointIndex }); } else { Entities.editEntity(childID, { parentID: NULL_UUID }); - Overlays.editOverlay(childID, { parentID: NULL_UUID }); + if (Overlays.getProperty(childID, "grabbable")) { + Overlays.editOverlay(childID, { parentID: NULL_UUID }); + } } } }); From 74323cb1f0065e3e41c97b8c615cdb12de50f370 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Mon, 6 Mar 2017 11:45:49 -0800 Subject: [PATCH 11/64] remove debug print --- scripts/system/controllers/handControllerGrab.js | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js index cd18d6f932..5b99738a78 100644 --- a/scripts/system/controllers/handControllerGrab.js +++ b/scripts/system/controllers/handControllerGrab.js @@ -3195,7 +3195,6 @@ function MyController(hand) { // we appear to be holding something and this script isn't in a state that would be holding something. // unhook it. if we previously took note of this entity's parent, put it back where it was. This // works around some problems that happen when more than one hand or avatar is passing something around. - print("disconnecting stray child of hand: (" + _this.hand + ") " + childID); if (_this.previousParentID[childID]) { var previousParentID = _this.previousParentID[childID]; var previousParentJointIndex = _this.previousParentJointIndex[childID]; From f9dcfa54aab2d5fa998dcd69df9dac87eb462640 Mon Sep 17 00:00:00 2001 From: Brad Davis Date: Mon, 6 Mar 2017 16:08:21 -0800 Subject: [PATCH 12/64] Add debug/release builds to oculus lib --- cmake/externals/LibOVR/CMakeLists.txt | 8 +++++++- cmake/externals/LibOVR/LibOVRCMakeLists.txt | 5 ++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cmake/externals/LibOVR/CMakeLists.txt b/cmake/externals/LibOVR/CMakeLists.txt index d8b8307082..c98aa8a04a 100644 --- a/cmake/externals/LibOVR/CMakeLists.txt +++ b/cmake/externals/LibOVR/CMakeLists.txt @@ -28,7 +28,13 @@ if (WIN32) ExternalProject_Get_Property(${EXTERNAL_NAME} INSTALL_DIR) set(LIBOVR_DIR ${INSTALL_DIR}) set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${LIBOVR_DIR}/Include CACHE TYPE INTERNAL) - set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${LIBOVR_DIR}/Lib/LibOVR.lib CACHE TYPE INTERNAL) + set(${EXTERNAL_NAME_UPPER}_LIBRARY_DEBUG ${LIBOVR_DIR}/Lib/LibOVRd.lib CACHE TYPE INTERNAL) + set(${EXTERNAL_NAME_UPPER}_LIBRARY_RELEASE ${LIBOVR_DIR}/Lib/LibOVR.lib CACHE TYPE INTERNAL) + include(SelectLibraryConfigurations) + select_library_configurations(LIBOVR) + set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${${EXTERNAL_NAME_UPPER}_LIBRARIES} CACHE TYPE INTERNAL) + message("Libs ${EXTERNAL_NAME_UPPER}_LIBRARIES ${${EXTERNAL_NAME_UPPER}_LIBRARIES}") + elseif(APPLE) ExternalProject_Add( diff --git a/cmake/externals/LibOVR/LibOVRCMakeLists.txt b/cmake/externals/LibOVR/LibOVRCMakeLists.txt index bab6337ae2..556533f0c2 100644 --- a/cmake/externals/LibOVR/LibOVRCMakeLists.txt +++ b/cmake/externals/LibOVR/LibOVRCMakeLists.txt @@ -1,14 +1,13 @@ cmake_minimum_required(VERSION 3.2) project(LibOVR) -#set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DOVR_BUILD_DEBUG") - include_directories(LibOVR/Include LibOVR/Src) -#include_directories(LibOVRKernel/Src/) file(GLOB HEADER_FILES LibOVR/Include/*.h) file(GLOB EXTRA_HEADER_FILES LibOVR/Include/Extras/*.h) file(GLOB_RECURSE SOURCE_FILES LibOVR/Src/*.c LibOVR/Src/*.cpp) +set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DOVR_BUILD_DEBUG") add_library(LibOVR STATIC ${SOURCE_FILES} ${HEADER_FILES} ${EXTRA_HEADER_FILES}) +set_target_properties(LibOVR PROPERTIES DEBUG_POSTFIX "d") install(TARGETS LibOVR DESTINATION Lib) install(FILES ${HEADER_FILES} DESTINATION Include) From 2f51e635543a731fbd267eb810ce2a12c6757e3f Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Tue, 7 Mar 2017 01:29:20 +0100 Subject: [PATCH 13/64] Since the deck is currently only capable of handling one clip, add temporary _length reset in FIXME section --- libraries/recording/src/recording/Deck.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/recording/src/recording/Deck.cpp b/libraries/recording/src/recording/Deck.cpp index 61eb86c91f..5a32ff4984 100644 --- a/libraries/recording/src/recording/Deck.cpp +++ b/libraries/recording/src/recording/Deck.cpp @@ -33,6 +33,7 @@ void Deck::queueClip(ClipPointer clip, float timeOffset) { // FIXME disabling multiple clips for now _clips.clear(); + _length = 0.0f; // if the time offset is not zero, wrap in an OffsetClip if (timeOffset != 0.0f) { From 87934ee82d1c1a247311ec3886d3c278aa40d8a3 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Mon, 6 Mar 2017 21:50:32 -0800 Subject: [PATCH 14/64] fix bugs in some meshes --- libraries/render-utils/src/Model.cpp | 24 ++++++++++++++---------- libraries/shared/src/TriangleSet.h | 5 ++++- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/libraries/render-utils/src/Model.cpp b/libraries/render-utils/src/Model.cpp index 2af42c046e..48c1d29b68 100644 --- a/libraries/render-utils/src/Model.cpp +++ b/libraries/render-utils/src/Model.cpp @@ -368,7 +368,7 @@ bool Model::findRayIntersectionAgainstSubMeshes(const glm::vec3& origin, const g } glm::mat4 meshToModelMatrix = glm::scale(_scale) * glm::translate(_offset); - glm::mat4 meshToWorldMatrix = meshToModelMatrix * createMatFromQuatAndPos(_rotation, _translation); + glm::mat4 meshToWorldMatrix = createMatFromQuatAndPos(_rotation, _translation) * meshToModelMatrix; glm::mat4 worldToMeshMatrix = glm::inverse(meshToWorldMatrix); glm::vec3 meshFrameOrigin = glm::vec3(worldToMeshMatrix * glm::vec4(origin, 1.0f)); @@ -437,7 +437,7 @@ bool Model::convexHullContains(glm::vec3 point) { // If we are inside the models box, then consider the submeshes... glm::mat4 meshToModelMatrix = glm::scale(_scale) * glm::translate(_offset); - glm::mat4 meshToWorldMatrix = meshToModelMatrix * createMatFromQuatAndPos(_rotation, _translation); + glm::mat4 meshToWorldMatrix = createMatFromQuatAndPos(_rotation, _translation) * meshToModelMatrix; glm::mat4 worldToMeshMatrix = glm::inverse(meshToWorldMatrix); glm::vec3 meshFramePoint = glm::vec3(worldToMeshMatrix * glm::vec4(point, 1.0f)); @@ -483,6 +483,8 @@ void Model::calculateTriangleSets() { int totalTriangles = (numberOfQuads * TRIANGLES_PER_QUAD) + numberOfTris; _modelSpaceMeshTriangleSets[i].reserve(totalTriangles); + auto meshTransform = getFBXGeometry().offset * mesh.modelTransform; + if (part.quadIndices.size() > 0) { int vIndex = 0; for (int q = 0; q < numberOfQuads; q++) { @@ -494,10 +496,10 @@ void Model::calculateTriangleSets() { // track the model space version... these points will be transformed by the FST's offset, // which includes the scaling, rotation, and translation specified by the FST/FBX, // this can't change at runtime, so we can safely store these in our TriangleSet - glm::vec3 v0 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i0], 1.0f)); - glm::vec3 v1 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i1], 1.0f)); - glm::vec3 v2 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i2], 1.0f)); - glm::vec3 v3 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i3], 1.0f)); + glm::vec3 v0 = glm::vec3(meshTransform * glm::vec4(mesh.vertices[i0], 1.0f)); + glm::vec3 v1 = glm::vec3(meshTransform * glm::vec4(mesh.vertices[i1], 1.0f)); + glm::vec3 v2 = glm::vec3(meshTransform * glm::vec4(mesh.vertices[i2], 1.0f)); + glm::vec3 v3 = glm::vec3(meshTransform * glm::vec4(mesh.vertices[i3], 1.0f)); Triangle tri1 = { v0, v1, v3 }; Triangle tri2 = { v1, v2, v3 }; @@ -516,9 +518,9 @@ void Model::calculateTriangleSets() { // track the model space version... these points will be transformed by the FST's offset, // which includes the scaling, rotation, and translation specified by the FST/FBX, // this can't change at runtime, so we can safely store these in our TriangleSet - glm::vec3 v0 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i0], 1.0f)); - glm::vec3 v1 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i1], 1.0f)); - glm::vec3 v2 = glm::vec3(getFBXGeometry().offset * glm::vec4(mesh.vertices[i2], 1.0f)); + glm::vec3 v0 = glm::vec3(meshTransform * glm::vec4(mesh.vertices[i0], 1.0f)); + glm::vec3 v1 = glm::vec3(meshTransform * glm::vec4(mesh.vertices[i1], 1.0f)); + glm::vec3 v2 = glm::vec3(meshTransform * glm::vec4(mesh.vertices[i2], 1.0f)); Triangle tri = { v0, v1, v2 }; _modelSpaceMeshTriangleSets[i].insert(tri); @@ -642,10 +644,12 @@ void Model::renderDebugMeshBoxes(gpu::Batch& batch) { _mutex.lock(); glm::mat4 meshToModelMatrix = glm::scale(_scale) * glm::translate(_offset); - glm::mat4 meshToWorldMatrix = meshToModelMatrix * createMatFromQuatAndPos(_rotation, _translation); + glm::mat4 meshToWorldMatrix = createMatFromQuatAndPos(_rotation, _translation) * meshToModelMatrix; Transform meshToWorld(meshToWorldMatrix); batch.setModelTransform(meshToWorld); + DependencyManager::get()->bindSimpleProgram(batch, false, false, false, true, true); + for(const auto& triangleSet : _modelSpaceMeshTriangleSets) { auto box = triangleSet.getBounds(); diff --git a/libraries/shared/src/TriangleSet.h b/libraries/shared/src/TriangleSet.h index 87336c77af..b54f1a642a 100644 --- a/libraries/shared/src/TriangleSet.h +++ b/libraries/shared/src/TriangleSet.h @@ -16,7 +16,10 @@ class TriangleSet { public: - void reserve(size_t size) { _triangles.reserve(size); } // reserve space in the datastructure for size number of triangles + void reserve(size_t size) { _triangles.reserve(size); } // reserve space in the datastructure for size number of triangles + size_t size() const { return _triangles.size(); } + + const Triangle& getTriangle(size_t t) const { return _triangles[t]; } void insert(const Triangle& t); void clear(); From c52c43e23d27142118fae920b37970f61f67cfc1 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 6 Mar 2017 19:35:00 -0800 Subject: [PATCH 15/64] Remove weighted offset, special case downward pressure --- .../animation/src/AnimInverseKinematics.cpp | 48 +++++++------------ 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/libraries/animation/src/AnimInverseKinematics.cpp b/libraries/animation/src/AnimInverseKinematics.cpp index 173af3fdf6..fa8e4654f6 100644 --- a/libraries/animation/src/AnimInverseKinematics.cpp +++ b/libraries/animation/src/AnimInverseKinematics.cpp @@ -488,13 +488,7 @@ const AnimPoseVec& AnimInverseKinematics::overlay(const AnimVariantMap& animVars // measure new _hipsOffset for next frame // by looking for discrepancies between where a targeted endEffector is // and where it wants to be (after IK solutions are done) - - // use weighted average between HMD and other targets - float HMD_WEIGHT = 10.0f; - float OTHER_WEIGHT = 1.0f; - float totalWeight = 0.0f; - - glm::vec3 additionalHipsOffset = Vectors::ZERO; + glm::vec3 newHipsOffset = Vectors::ZERO; for (auto& target: targets) { int targetIndex = target.getIndex(); if (targetIndex == _headIndex && _headIndex != -1) { @@ -505,42 +499,34 @@ const AnimPoseVec& AnimInverseKinematics::overlay(const AnimVariantMap& animVars glm::vec3 under = _skeleton->getAbsolutePose(_headIndex, underPoses).trans(); glm::vec3 actual = _skeleton->getAbsolutePose(_headIndex, _relativePoses).trans(); const float HEAD_OFFSET_SLAVE_FACTOR = 0.65f; - additionalHipsOffset += (OTHER_WEIGHT * HEAD_OFFSET_SLAVE_FACTOR) * (under- actual); - totalWeight += OTHER_WEIGHT; + newHipsOffset += HEAD_OFFSET_SLAVE_FACTOR * (actual - under); } else if (target.getType() == IKTarget::Type::HmdHead) { + // we want to shift the hips to bring the head to its designated position glm::vec3 actual = _skeleton->getAbsolutePose(_headIndex, _relativePoses).trans(); - glm::vec3 thisOffset = target.getTranslation() - actual; - glm::vec3 futureHipsOffset = _hipsOffset + thisOffset; - if (glm::length(glm::vec2(futureHipsOffset.x, futureHipsOffset.z)) < _maxHipsOffsetLength) { - // it is imperative to shift the hips and bring the head to its designated position - // so we slam newHipsOffset here and ignore all other targets - additionalHipsOffset = futureHipsOffset - _hipsOffset; - totalWeight = 0.0f; - break; - } else { - additionalHipsOffset += HMD_WEIGHT * (target.getTranslation() - actual); - totalWeight += HMD_WEIGHT; - } + _hipsOffset += target.getTranslation() - actual; + // and ignore all other targets + newHipsOffset = _hipsOffset; + break; + } else if (target.getType() == IKTarget::Type::RotationAndPosition) { + glm::vec3 actualPosition = _skeleton->getAbsolutePose(targetIndex, _relativePoses).trans(); + glm::vec3 targetPosition = target.getTranslation(); + newHipsOffset += targetPosition - actualPosition; + + // Add downward pressure on the hips + newHipsOffset *= 0.95f; + newHipsOffset -= 1.0f; } } else if (target.getType() == IKTarget::Type::RotationAndPosition) { glm::vec3 actualPosition = _skeleton->getAbsolutePose(targetIndex, _relativePoses).trans(); glm::vec3 targetPosition = target.getTranslation(); - additionalHipsOffset += OTHER_WEIGHT * (targetPosition - actualPosition); - totalWeight += OTHER_WEIGHT; + newHipsOffset += targetPosition - actualPosition; } } - if (totalWeight > 1.0f) { - additionalHipsOffset /= totalWeight; - } - - // Add downward pressure on the hips - additionalHipsOffset *= 0.95f; - additionalHipsOffset -= 1.0f; // smooth transitions by relaxing _hipsOffset toward the new value const float HIPS_OFFSET_SLAVE_TIMESCALE = 0.10f; float tau = dt < HIPS_OFFSET_SLAVE_TIMESCALE ? dt / HIPS_OFFSET_SLAVE_TIMESCALE : 1.0f; - _hipsOffset += additionalHipsOffset * tau; + _hipsOffset += (newHipsOffset - _hipsOffset) * tau; // clamp the hips offset float hipsOffsetLength = glm::length(_hipsOffset); From d2c3b0b7066ccb6fcf5e752376df27abf5c5e5bb Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Tue, 7 Mar 2017 22:37:54 +0100 Subject: [PATCH 16/64] Fix coding standards link --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 86ea351609..a0d867ade9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,7 +16,7 @@ Contributing git checkout -b new_branch_name ``` 4. Code - * Follow the [coding standard](https://readme.highfidelity.com/v1.0/docs/coding-standard) + * Follow the [coding standard](https://wiki.highfidelity.com/wiki/Coding_Standards) 5. Commit * Use [well formed commit messages](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html) 6. Update your branch From b54d6cb0e1fda141c8c81749ef1a35b78b31df3f Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Wed, 8 Mar 2017 00:07:49 +0100 Subject: [PATCH 17/64] stop avatar controller while playing back a recording, which makes it not collide anymore --- interface/src/avatar/MyAvatar.cpp | 3 +++ interface/src/avatar/MyAvatar.h | 1 + 2 files changed, 4 insertions(+) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 969268c549..318608e3a8 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -154,9 +154,12 @@ MyAvatar::MyAvatar(RigPointer rig) : if (recordingInterface->getPlayFromCurrentLocation()) { setRecordingBasis(); } + _wasCharacterControllerEnabled = _characterController.isEnabled(); + _characterController.setEnabled(false); } else { clearRecordingBasis(); useFullAvatarURL(_fullAvatarURLFromPreferences, _fullAvatarModelName); + _characterController.setEnabled(_wasCharacterControllerEnabled); } auto audioIO = DependencyManager::get(); diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 3cc665b533..00923e78cc 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -411,6 +411,7 @@ private: SharedSoundPointer _collisionSound; MyCharacterController _characterController; + bool _wasCharacterControllerEnabled { true }; AvatarWeakPointer _lookAtTargetAvatar; glm::vec3 _targetAvatarPosition; From b954d5b0e94276d752b14ab4ca4fb321d826784a Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Wed, 8 Mar 2017 00:08:57 +0100 Subject: [PATCH 18/64] stop the playback after the clip stopped playing rather than pause. Allows you to click play again without having to move the position back. --- libraries/recording/src/recording/Deck.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/recording/src/recording/Deck.cpp b/libraries/recording/src/recording/Deck.cpp index 5a32ff4984..186516e01c 100644 --- a/libraries/recording/src/recording/Deck.cpp +++ b/libraries/recording/src/recording/Deck.cpp @@ -154,8 +154,8 @@ void Deck::processFrames() { // if doing relative movement emit looped(); } else { - // otherwise pause playback - pause(); + // otherwise stop playback + stop(); } return; } From 6b26f5f3a30595f2b16389800fe3325ded88b616 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Wed, 8 Mar 2017 00:09:56 +0100 Subject: [PATCH 19/64] Fix the timer display --- .../developer/utilities/record/recorder.js | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/scripts/developer/utilities/record/recorder.js b/scripts/developer/utilities/record/recorder.js index 0e335116d5..7338105483 100644 --- a/scripts/developer/utilities/record/recorder.js +++ b/scripts/developer/utilities/record/recorder.js @@ -9,6 +9,8 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +/* globals HIFI_PUBLIC_BUCKET:true, Tool, ToolBar */ + HIFI_PUBLIC_BUCKET = "http://s3.amazonaws.com/hifi-public/"; Script.include("/~/system/libraries/toolBars.js"); @@ -47,7 +49,7 @@ setupTimer(); var watchStop = false; function setupToolBar() { - if (toolBar != null) { + if (toolBar !== null) { print("Multiple calls to Recorder.js:setupToolBar()"); return; } @@ -112,15 +114,15 @@ function setupTimer() { text: (0.00).toFixed(3), backgroundColor: COLOR_OFF, x: 0, y: 0, - width: 0, height: 0, - leftMargin: 10, topMargin: 10, + width: 200, height: 37, + leftMargin: 5, topMargin: 10, alpha: 1.0, backgroundAlpha: 1.0, visible: true }); slider = { x: 0, y: 0, w: 200, h: 20, - pos: 0.0, // 0.0 <= pos <= 1.0 + pos: 0.0 // 0.0 <= pos <= 1.0 }; slider.background = Overlays.addOverlay("text", { text: "", @@ -148,16 +150,17 @@ function updateTimer() { var text = ""; if (Recording.isRecording()) { text = formatTime(Recording.recorderElapsed()); - } else { - text = formatTime(Recording.playerElapsed()) + " / " + - formatTime(Recording.playerLength()); + text = formatTime(Recording.playerElapsed()) + " / " + formatTime(Recording.playerLength()); } + var timerWidth = text.length * 8 + ((Recording.isRecording()) ? 15 : 0); + Overlays.editOverlay(timer, { - text: text - }) - toolBar.changeSpacing(text.length * 8 + ((Recording.isRecording()) ? 15 : 0), spacing); + text: text, + width: timerWidth + }); + toolBar.changeSpacing(timerWidth, spacing); if (Recording.isRecording()) { slider.pos = 1.0; @@ -173,7 +176,7 @@ function updateTimer() { function formatTime(time) { var MIN_PER_HOUR = 60; var SEC_PER_MIN = 60; - var MSEC_PER_SEC = 1000; + var MSEC_DIGITS = 3; var hours = Math.floor(time / (SEC_PER_MIN * MIN_PER_HOUR)); time -= hours * (SEC_PER_MIN * MIN_PER_HOUR); @@ -184,11 +187,9 @@ function formatTime(time) { var seconds = time; var text = ""; - text += (hours > 0) ? hours + ":" : - ""; - text += (minutes > 0) ? ((minutes < 10 && text != "") ? "0" : "") + minutes + ":" : - ""; - text += ((seconds < 10 && text != "") ? "0" : "") + seconds.toFixed(3); + text += (hours > 0) ? hours + ":" : ""; + text += (minutes > 0) ? ((minutes < 10 && text !== "") ? "0" : "") + minutes + ":" : ""; + text += ((seconds < 10 && text !== "") ? "0" : "") + seconds.toFixed(MSEC_DIGITS); return text; } @@ -205,16 +206,16 @@ function moveUI() { Overlays.editOverlay(slider.background, { x: slider.x, - y: slider.y, + y: slider.y }); Overlays.editOverlay(slider.foreground, { x: slider.x, - y: slider.y, + y: slider.y }); } function mousePressEvent(event) { - clickedOverlay = Overlays.getOverlayAtPoint({ x: event.x, y: event.y }); + var clickedOverlay = Overlays.getOverlayAtPoint({ x: event.x, y: event.y }); if (recordIcon === toolBar.clicked(clickedOverlay, false) && !Recording.isPlaying()) { if (!Recording.isRecording()) { @@ -226,7 +227,8 @@ function mousePressEvent(event) { toolBar.setAlpha(ALPHA_OFF, loadIcon); } else { Recording.stopRecording(); - toolBar.selectTool(recordIcon, true ); + toolBar.selectTool(recordIcon, true); + Recording.setPlayerTime(0); Recording.loadLastRecording(); toolBar.setAlpha(ALPHA_ON, playIcon); toolBar.setAlpha(ALPHA_ON, playLoopIcon); @@ -263,7 +265,7 @@ function mousePressEvent(event) { toolBar.setAlpha(ALPHA_OFF, loadIcon); } } else if (saveIcon === toolBar.clicked(clickedOverlay)) { - if (!Recording.isRecording() && !Recording.isPlaying() && Recording.playerLength() != 0) { + if (!Recording.isRecording() && !Recording.isPlaying() && Recording.playerLength() !== 0) { recordingFile = Window.save("Save recording to file", ".", "Recordings (*.hfr)"); if (!(recordingFile === "null" || recordingFile === null || recordingFile === "")) { Recording.saveRecording(recordingFile); @@ -282,8 +284,8 @@ function mousePressEvent(event) { } } } else if (Recording.playerLength() > 0 && - slider.x < event.x && event.x < slider.x + slider.w && - slider.y < event.y && event.y < slider.y + slider.h) { + slider.x < event.x && event.x < slider.x + slider.w && + slider.y < event.y && event.y < slider.y + slider.h) { isSliding = true; slider.pos = (event.x - slider.x) / slider.w; Recording.setPlayerTime(slider.pos * Recording.playerLength()); @@ -308,7 +310,7 @@ function mouseReleaseEvent(event) { function update() { var newDimensions = Controller.getViewportDimensions(); - if (windowDimensions.x != newDimensions.x || windowDimensions.y != newDimensions.y) { + if (windowDimensions.x !== newDimensions.x || windowDimensions.y !== newDimensions.y) { windowDimensions = newDimensions; moveUI(); } From b21bd3afa9d22cd87ce2193493ddee3c65789e51 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Wed, 8 Mar 2017 01:14:11 +0100 Subject: [PATCH 20/64] Make the time and slider follow the record.js toolbar --- .../developer/utilities/record/recorder.js | 50 +++++++++++-------- scripts/system/libraries/toolBars.js | 4 ++ 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/scripts/developer/utilities/record/recorder.js b/scripts/developer/utilities/record/recorder.js index 7338105483..dcbbe2145f 100644 --- a/scripts/developer/utilities/record/recorder.js +++ b/scripts/developer/utilities/record/recorder.js @@ -16,7 +16,7 @@ Script.include("/~/system/libraries/toolBars.js"); var recordingFile = "recording.hfr"; -function setPlayerOptions() { +function setDefaultPlayerOptions() { Recording.setPlayFromCurrentLocation(true); Recording.setPlayerUseDisplayName(false); Recording.setPlayerUseAttachments(false); @@ -40,10 +40,10 @@ var saveIcon; var loadIcon; var spacing; var timerOffset; -setupToolBar(); - var timer = null; var slider = null; + +setupToolBar(); setupTimer(); var watchStop = false; @@ -58,6 +58,8 @@ function setupToolBar() { toolBar = new ToolBar(0, 0, ToolBar.HORIZONTAL); + toolBar.onMove = onToolbarMove; + toolBar.setBack(COLOR_TOOL_BAR, ALPHA_OFF); recordIcon = toolBar.addTool({ @@ -146,6 +148,26 @@ function setupTimer() { }); } +function onToolbarMove(newX, newY, deltaX, deltaY) { + print(newX); + Overlays.editOverlay(timer, { + x: newX + timerOffset - ToolBar.SPACING, + y: newY - ToolBar.SPACING + }); + + slider.x = newX - ToolBar.SPACING; + slider.y = newY - slider.h - ToolBar.SPACING; + + Overlays.editOverlay(slider.background, { + x: slider.x, + y: slider.y + }); + Overlays.editOverlay(slider.foreground, { + x: slider.x, + y: slider.y + }); +} + function updateTimer() { var text = ""; if (Recording.isRecording()) { @@ -196,22 +218,6 @@ function formatTime(time) { function moveUI() { var relative = { x: 70, y: 40 }; toolBar.move(relative.x, windowDimensions.y - relative.y); - Overlays.editOverlay(timer, { - x: relative.x + timerOffset - ToolBar.SPACING, - y: windowDimensions.y - relative.y - ToolBar.SPACING - }); - - slider.x = relative.x - ToolBar.SPACING; - slider.y = windowDimensions.y - relative.y - slider.h - ToolBar.SPACING; - - Overlays.editOverlay(slider.background, { - x: slider.x, - y: slider.y - }); - Overlays.editOverlay(slider.foreground, { - x: slider.x, - y: slider.y - }); } function mousePressEvent(event) { @@ -228,6 +234,9 @@ function mousePressEvent(event) { } else { Recording.stopRecording(); toolBar.selectTool(recordIcon, true); + setDefaultPlayerOptions(); + // Plays the recording at the same spot as you recorded it + Recording.setPlayFromCurrentLocation(false); Recording.setPlayerTime(0); Recording.loadLastRecording(); toolBar.setAlpha(ALPHA_ON, playIcon); @@ -242,7 +251,6 @@ function mousePressEvent(event) { toolBar.setAlpha(ALPHA_ON, saveIcon); toolBar.setAlpha(ALPHA_ON, loadIcon); } else if (Recording.playerLength() > 0) { - setPlayerOptions(); Recording.setPlayerLoop(false); Recording.startPlaying(); toolBar.setAlpha(ALPHA_OFF, recordIcon); @@ -257,7 +265,6 @@ function mousePressEvent(event) { toolBar.setAlpha(ALPHA_ON, saveIcon); toolBar.setAlpha(ALPHA_ON, loadIcon); } else if (Recording.playerLength() > 0) { - setPlayerOptions(); Recording.setPlayerLoop(true); Recording.startPlaying(); toolBar.setAlpha(ALPHA_OFF, recordIcon); @@ -276,6 +283,7 @@ function mousePressEvent(event) { recordingFile = Window.browse("Load recording from file", ".", "Recordings (*.hfr *.rec *.HFR *.REC)"); if (!(recordingFile === "null" || recordingFile === null || recordingFile === "")) { Recording.loadRecording(recordingFile); + setDefaultPlayerOptions(); } if (Recording.playerLength() > 0) { toolBar.setAlpha(ALPHA_ON, playIcon); diff --git a/scripts/system/libraries/toolBars.js b/scripts/system/libraries/toolBars.js index e49f8c4004..351f10e7bd 100644 --- a/scripts/system/libraries/toolBars.js +++ b/scripts/system/libraries/toolBars.js @@ -160,6 +160,7 @@ ToolBar = function(x, y, direction, optionalPersistenceKey, optionalInitialPosit visible: false }); this.spacing = []; + this.onMove = null; this.addTool = function(properties, selectable, selected) { if (direction == ToolBar.HORIZONTAL) { @@ -254,6 +255,9 @@ ToolBar = function(x, y, direction, optionalPersistenceKey, optionalInitialPosit y: y - ToolBar.SPACING }); } + if (this.onMove !== null) { + this.onMove(x, y, dx, dy); + }; } this.setAlpha = function(alpha, tool) { From 097db9a7f98bbab309cf30033e26c4b2d461df17 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Wed, 8 Mar 2017 01:41:20 +0100 Subject: [PATCH 21/64] - debug message removal (oops) - smaller timer textbox, allows you to grab around the edges and still drag the toolbar --- scripts/developer/utilities/record/recorder.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/developer/utilities/record/recorder.js b/scripts/developer/utilities/record/recorder.js index dcbbe2145f..ba1c8b0393 100644 --- a/scripts/developer/utilities/record/recorder.js +++ b/scripts/developer/utilities/record/recorder.js @@ -90,7 +90,7 @@ function setupToolBar() { visible: true }, false); - timerOffset = toolBar.width; + timerOffset = toolBar.width + ToolBar.SPACING; spacing = toolBar.addSpacing(0); saveIcon = toolBar.addTool({ @@ -116,8 +116,8 @@ function setupTimer() { text: (0.00).toFixed(3), backgroundColor: COLOR_OFF, x: 0, y: 0, - width: 200, height: 37, - leftMargin: 5, topMargin: 10, + width: 200, height: 25, + leftMargin: 5, topMargin: 3, alpha: 1.0, backgroundAlpha: 1.0, visible: true }); @@ -149,10 +149,9 @@ function setupTimer() { } function onToolbarMove(newX, newY, deltaX, deltaY) { - print(newX); Overlays.editOverlay(timer, { x: newX + timerOffset - ToolBar.SPACING, - y: newY - ToolBar.SPACING + y: newY }); slider.x = newX - ToolBar.SPACING; @@ -182,7 +181,7 @@ function updateTimer() { text: text, width: timerWidth }); - toolBar.changeSpacing(timerWidth, spacing); + toolBar.changeSpacing(timerWidth + ToolBar.SPACING, spacing); if (Recording.isRecording()) { slider.pos = 1.0; From d2a2ba18462f93a02eaffa5f7ac7f97949dc35b0 Mon Sep 17 00:00:00 2001 From: kunalgosar Date: Tue, 7 Mar 2017 17:19:21 -0800 Subject: [PATCH 22/64] Fixed recording bug --- libraries/avatars/src/AvatarData.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 06df75d451..3c33451f02 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -2000,6 +2000,11 @@ void AvatarData::fromJson(const QJsonObject& json, bool useFrameSkeleton) { } } + auto currentBasis = getRecordingBasis(); + if (!currentBasis) { + currentBasis = std::make_shared(Transform::fromJson(json[JSON_AVATAR_BASIS])); + } + if (json.contains(JSON_AVATAR_RELATIVE)) { // During playback you can either have the recording basis set to the avatar current state // meaning that all playback is relative to this avatars starting position, or @@ -2008,15 +2013,14 @@ void AvatarData::fromJson(const QJsonObject& json, bool useFrameSkeleton) { // The first is more useful for playing back recordings on your own avatar, while // the latter is more useful for playing back other avatars within your scene. - auto currentBasis = getRecordingBasis(); - if (!currentBasis) { - currentBasis = std::make_shared(Transform::fromJson(json[JSON_AVATAR_BASIS])); - } - auto relativeTransform = Transform::fromJson(json[JSON_AVATAR_RELATIVE]); auto worldTransform = currentBasis->worldTransform(relativeTransform); setPosition(worldTransform.getTranslation()); setOrientation(worldTransform.getRotation()); + } else { + // We still set the position in the case that there is no movement. + setPosition(currentBasis->getTranslation()); + setOrientation(currentBasis->getRotation()); } if (json.contains(JSON_AVATAR_SCALE)) { From 8ae6f2727d4e0faef0af43bb6b853fdd3dac17c5 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 7 Mar 2017 17:52:55 -0800 Subject: [PATCH 23/64] add support for double click messages to overlays and entities --- interface/src/Application.cpp | 18 +++++++- interface/src/Application.h | 2 +- interface/src/ui/overlays/Overlays.cpp | 20 ++++++++ interface/src/ui/overlays/Overlays.h | 3 ++ .../src/EntityTreeRenderer.cpp | 46 +++++++++++++++++++ .../src/EntityTreeRenderer.h | 3 ++ libraries/shared/src/PointerEvent.cpp | 5 ++ libraries/shared/src/PointerEvent.h | 7 +-- 8 files changed, 99 insertions(+), 5 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index f870bd9f83..671c530b27 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -3174,7 +3174,23 @@ void Application::mousePressEvent(QMouseEvent* event) { } } -void Application::mouseDoublePressEvent(QMouseEvent* event) const { +void Application::mouseDoublePressEvent(QMouseEvent* event) { + auto offscreenUi = DependencyManager::get(); + auto eventPosition = getApplicationCompositor().getMouseEventPosition(event); + QPointF transformedPos = offscreenUi->mapToVirtualScreen(eventPosition, _glWidget); + QMouseEvent mappedEvent(event->type(), + transformedPos, + event->screenPos(), event->button(), + event->buttons(), event->modifiers()); + + if (!_aboutToQuit) { + getOverlays().mouseDoublePressEvent(&mappedEvent); + if (!_controllerScriptingInterface->areEntityClicksCaptured()) { + getEntities()->mouseDoublePressEvent(&mappedEvent); + } + } + + // if one of our scripts have asked to capture this event, then stop processing it if (_controllerScriptingInterface->isMouseCaptured()) { return; diff --git a/interface/src/Application.h b/interface/src/Application.h index c4ba760153..98080783a6 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -494,7 +494,7 @@ private: void mouseMoveEvent(QMouseEvent* event); void mousePressEvent(QMouseEvent* event); - void mouseDoublePressEvent(QMouseEvent* event) const; + void mouseDoublePressEvent(QMouseEvent* event); void mouseReleaseEvent(QMouseEvent* event); void touchBeginEvent(QTouchEvent* event); diff --git a/interface/src/ui/overlays/Overlays.cpp b/interface/src/ui/overlays/Overlays.cpp index ad7fbd6cc2..f40dd522c4 100644 --- a/interface/src/ui/overlays/Overlays.cpp +++ b/interface/src/ui/overlays/Overlays.cpp @@ -769,6 +769,26 @@ bool Overlays::mousePressEvent(QMouseEvent* event) { return false; } +bool Overlays::mouseDoublePressEvent(QMouseEvent* event) { + PerformanceTimer perfTimer("Overlays::mouseDoublePressEvent"); + + PickRay ray = qApp->computePickRay(event->x(), event->y()); + RayToOverlayIntersectionResult rayPickResult = findRayIntersectionForMouseEvent(ray); + if (rayPickResult.intersects) { + _currentClickingOnOverlayID = rayPickResult.overlayID; + + // Only Web overlays can have focus. + auto thisOverlay = std::dynamic_pointer_cast(getOverlay(_currentClickingOnOverlayID)); + if (thisOverlay) { + auto pointerEvent = calculatePointerEvent(thisOverlay, ray, rayPickResult, event, PointerEvent::Press); + emit mouseDoublePressOnOverlay(_currentClickingOnOverlayID, pointerEvent); + return true; + } + } + emit mouseDoublePressOffOverlay(); + return false; +} + bool Overlays::mouseReleaseEvent(QMouseEvent* event) { PerformanceTimer perfTimer("Overlays::mouseReleaseEvent"); diff --git a/interface/src/ui/overlays/Overlays.h b/interface/src/ui/overlays/Overlays.h index 5c22e46880..c35c7c1ced 100644 --- a/interface/src/ui/overlays/Overlays.h +++ b/interface/src/ui/overlays/Overlays.h @@ -101,6 +101,7 @@ public: OverlayID addOverlay(Overlay::Pointer overlay); bool mousePressEvent(QMouseEvent* event); + bool mouseDoublePressEvent(QMouseEvent* event); bool mouseReleaseEvent(QMouseEvent* event); bool mouseMoveEvent(QMouseEvent* event); @@ -300,9 +301,11 @@ signals: void panelDeleted(OverlayID id); void mousePressOnOverlay(OverlayID overlayID, const PointerEvent& event); + void mouseDoublePressOnOverlay(OverlayID overlayID, const PointerEvent& event); void mouseReleaseOnOverlay(OverlayID overlayID, const PointerEvent& event); void mouseMoveOnOverlay(OverlayID overlayID, const PointerEvent& event); void mousePressOffOverlay(); + void mouseDoublePressOffOverlay(); void hoverEnterOverlay(OverlayID overlayID, const PointerEvent& event); void hoverOverOverlay(OverlayID overlayID, const PointerEvent& event); diff --git a/libraries/entities-renderer/src/EntityTreeRenderer.cpp b/libraries/entities-renderer/src/EntityTreeRenderer.cpp index bd25bcf905..27e00b47c6 100644 --- a/libraries/entities-renderer/src/EntityTreeRenderer.cpp +++ b/libraries/entities-renderer/src/EntityTreeRenderer.cpp @@ -735,6 +735,52 @@ void EntityTreeRenderer::mousePressEvent(QMouseEvent* event) { } } +void EntityTreeRenderer::mouseDoublePressEvent(QMouseEvent* event) { + // If we don't have a tree, or we're in the process of shutting down, then don't + // process these events. + if (!_tree || _shuttingDown) { + return; + } + PerformanceTimer perfTimer("EntityTreeRenderer::mouseDoublePressEvent"); + PickRay ray = _viewState->computePickRay(event->x(), event->y()); + + bool precisionPicking = !_dontDoPrecisionPicking; + RayToEntityIntersectionResult rayPickResult = findRayIntersectionWorker(ray, Octree::Lock, precisionPicking); + if (rayPickResult.intersects) { + //qCDebug(entitiesrenderer) << "mouseDoublePressEvent over entity:" << rayPickResult.entityID; + + QString urlString = rayPickResult.properties.getHref(); + QUrl url = QUrl(urlString, QUrl::StrictMode); + if (url.isValid() && !url.isEmpty()){ + DependencyManager::get()->handleLookupString(urlString); + } + + glm::vec2 pos2D = projectOntoEntityXYPlane(rayPickResult.entity, ray, rayPickResult); + PointerEvent pointerEvent(PointerEvent::Press, MOUSE_POINTER_ID, + pos2D, rayPickResult.intersection, + rayPickResult.surfaceNormal, ray.direction, + toPointerButton(*event), toPointerButtons(*event)); + + emit mouseDoublePressOnEntity(rayPickResult.entityID, pointerEvent); + + if (_entitiesScriptEngine) { + _entitiesScriptEngine->callEntityScriptMethod(rayPickResult.entityID, "mouseDoublePressOnEntity", pointerEvent); + } + + _currentClickingOnEntityID = rayPickResult.entityID; + emit clickDownOnEntity(_currentClickingOnEntityID, pointerEvent); + if (_entitiesScriptEngine) { + _entitiesScriptEngine->callEntityScriptMethod(_currentClickingOnEntityID, "doubleclickOnEntity", pointerEvent); + } + + _lastPointerEvent = pointerEvent; + _lastPointerEventValid = true; + + } else { + emit mouseDoublePressOffEntity(); + } +} + void EntityTreeRenderer::mouseReleaseEvent(QMouseEvent* event) { // If we don't have a tree, or we're in the process of shutting down, then don't // process these events. diff --git a/libraries/entities-renderer/src/EntityTreeRenderer.h b/libraries/entities-renderer/src/EntityTreeRenderer.h index c11738c459..753f25310c 100644 --- a/libraries/entities-renderer/src/EntityTreeRenderer.h +++ b/libraries/entities-renderer/src/EntityTreeRenderer.h @@ -90,6 +90,7 @@ public: // event handles which may generate entity related events void mouseReleaseEvent(QMouseEvent* event); void mousePressEvent(QMouseEvent* event); + void mouseDoublePressEvent(QMouseEvent* event); void mouseMoveEvent(QMouseEvent* event); /// connect our signals to anEntityScriptingInterface for firing of events related clicking, @@ -103,9 +104,11 @@ public: signals: void mousePressOnEntity(const EntityItemID& entityItemID, const PointerEvent& event); + void mouseDoublePressOnEntity(const EntityItemID& entityItemID, const PointerEvent& event); void mouseMoveOnEntity(const EntityItemID& entityItemID, const PointerEvent& event); void mouseReleaseOnEntity(const EntityItemID& entityItemID, const PointerEvent& event); void mousePressOffEntity(); + void mouseDoublePressOffEntity(); void clickDownOnEntity(const EntityItemID& entityItemID, const PointerEvent& event); void holdingClickOnEntity(const EntityItemID& entityItemID, const PointerEvent& event); diff --git a/libraries/shared/src/PointerEvent.cpp b/libraries/shared/src/PointerEvent.cpp index ed9acb9ada..e429204e94 100644 --- a/libraries/shared/src/PointerEvent.cpp +++ b/libraries/shared/src/PointerEvent.cpp @@ -47,6 +47,9 @@ QScriptValue PointerEvent::toScriptValue(QScriptEngine* engine, const PointerEve case Press: obj.setProperty("type", "Press"); break; + case DoublePress: + obj.setProperty("type", "DoublePress"); + break; case Release: obj.setProperty("type", "Release"); break; @@ -128,6 +131,8 @@ void PointerEvent::fromScriptValue(const QScriptValue& object, PointerEvent& eve QString typeStr = type.isString() ? type.toString() : "Move"; if (typeStr == "Press") { event._type = Press; + } else if (typeStr == "DoublePress") { + event._type = DoublePress; } else if (typeStr == "Release") { event._type = Release; } else { diff --git a/libraries/shared/src/PointerEvent.h b/libraries/shared/src/PointerEvent.h index 054835c4fc..166c4be592 100644 --- a/libraries/shared/src/PointerEvent.h +++ b/libraries/shared/src/PointerEvent.h @@ -26,9 +26,10 @@ public: }; enum EventType { - Press, // A button has just been pressed - Release, // A button has just been released - Move // The pointer has just moved + Press, // A button has just been pressed + DoublePress, // A button has just been double pressed + Release, // A button has just been released + Move // The pointer has just moved }; PointerEvent(); From d7c3677b2211bd3b906bbf40c1d23b61f827b2d5 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 7 Mar 2017 17:55:03 -0800 Subject: [PATCH 24/64] add example --- .../entityScripts/doubleClickExample.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 script-archive/entityScripts/doubleClickExample.js diff --git a/script-archive/entityScripts/doubleClickExample.js b/script-archive/entityScripts/doubleClickExample.js new file mode 100644 index 0000000000..daff2668ed --- /dev/null +++ b/script-archive/entityScripts/doubleClickExample.js @@ -0,0 +1,19 @@ +(function() { + var _this; + function DoubleClickExample() { + _this = this; + return; + } + + DoubleClickExample.prototype = { + clickDownOnEntity: function() { + print("clickDownOnEntity"); + }, + + doubleclickOnEntity: function() { + print("doubleclickOnEntity"); + } + + }; + return new DoubleClickExample(); +}); \ No newline at end of file From adc3581d63cad51d359e556d9f6e82b65991ada2 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Tue, 7 Mar 2017 19:32:34 -0800 Subject: [PATCH 25/64] remove override of locationChanged which was keeping the render-bounds of Line3DOverlays from updating during calls to editOverlay --- interface/src/ui/overlays/Line3DOverlay.cpp | 5 ----- interface/src/ui/overlays/Line3DOverlay.h | 2 -- 2 files changed, 7 deletions(-) diff --git a/interface/src/ui/overlays/Line3DOverlay.cpp b/interface/src/ui/overlays/Line3DOverlay.cpp index e1ea06c599..900c79fc3f 100644 --- a/interface/src/ui/overlays/Line3DOverlay.cpp +++ b/interface/src/ui/overlays/Line3DOverlay.cpp @@ -258,8 +258,3 @@ QVariant Line3DOverlay::getProperty(const QString& property) { Line3DOverlay* Line3DOverlay::createClone() const { return new Line3DOverlay(this); } - - -void Line3DOverlay::locationChanged(bool tellPhysics) { - // do nothing -} diff --git a/interface/src/ui/overlays/Line3DOverlay.h b/interface/src/ui/overlays/Line3DOverlay.h index aceecff6b2..c9ceac55a9 100644 --- a/interface/src/ui/overlays/Line3DOverlay.h +++ b/interface/src/ui/overlays/Line3DOverlay.h @@ -48,8 +48,6 @@ public: virtual Line3DOverlay* createClone() const override; - virtual void locationChanged(bool tellPhysics = true) override; - glm::vec3 getDirection() const { return _direction; } float getLength() const { return _length; } glm::vec3 getLocalStart() const { return getLocalPosition(); } From 59b4fa5fd74a6f820126b1d6563d48539b9ed16a Mon Sep 17 00:00:00 2001 From: Dante Ruiz Date: Wed, 8 Mar 2017 21:44:04 +0000 Subject: [PATCH 26/64] Close toolbar windows when scripts are reloaded --- scripts/system/audio.js | 3 + scripts/system/goto.js | 6 +- scripts/system/help.js | 3 + scripts/system/marketplaces/marketplaces.js | 4 + scripts/system/menu.js | 3 + scripts/system/pal.js | 3 + scripts/system/tablet-users.js | 3 + scripts/system/users.js | 1281 ------------------- 8 files changed, 24 insertions(+), 1282 deletions(-) delete mode 100644 scripts/system/users.js diff --git a/scripts/system/audio.js b/scripts/system/audio.js index 6e7e95d659..beeb8609d8 100644 --- a/scripts/system/audio.js +++ b/scripts/system/audio.js @@ -72,6 +72,9 @@ tablet.screenChanged.connect(onScreenChanged); AudioDevice.muteToggled.connect(onMuteToggled); Script.scriptEnding.connect(function () { + if (onAudioScreen) { + tablet.gotoHomeScreen(); + } button.clicked.disconnect(onClicked); tablet.screenChanged.disconnect(onScreenChanged); AudioDevice.muteToggled.disconnect(onMuteToggled); diff --git a/scripts/system/goto.js b/scripts/system/goto.js index 0e09ea3d79..d364bf579e 100644 --- a/scripts/system/goto.js +++ b/scripts/system/goto.js @@ -18,13 +18,14 @@ var button; var buttonName = "GOTO"; var toolBar = null; var tablet = null; - +var onGotoScreen = false; function onAddressBarShown(visible) { button.editProperties({isActive: visible}); } function onClicked(){ DialogsManager.toggleAddressBar(); + onGotoScreen = !onGotoScreen; } if (Settings.getValue("HUDUIEnabled")) { @@ -49,6 +50,9 @@ button.clicked.connect(onClicked); DialogsManager.addressBarShown.connect(onAddressBarShown); Script.scriptEnding.connect(function () { + if (onGotoScreen) { + DialogsManager.toggleAddressBar(); + } button.clicked.disconnect(onClicked); if (tablet) { tablet.removeButton(button); diff --git a/scripts/system/help.js b/scripts/system/help.js index 5a1b712fb5..3923b922fc 100644 --- a/scripts/system/help.js +++ b/scripts/system/help.js @@ -48,6 +48,9 @@ }, POLL_RATE); Script.scriptEnding.connect(function () { + if (enabled) { + Menu.closeInfoView('InfoView_html/help.html'); + } button.clicked.disconnect(onClicked); Script.clearInterval(interval); if (tablet) { diff --git a/scripts/system/marketplaces/marketplaces.js b/scripts/system/marketplaces/marketplaces.js index 68da7696be..f130e3cedb 100644 --- a/scripts/system/marketplaces/marketplaces.js +++ b/scripts/system/marketplaces/marketplaces.js @@ -121,6 +121,7 @@ function onClick() { if (onMarketplaceScreen) { // for toolbar-mode: go back to home screen, this will close the window. tablet.gotoHomeScreen(); + onMarketplaceScreen = false; } else { var entity = HMD.tabletID; Entities.editEntity(entity, {textures: JSON.stringify({"tex.close": HOME_BUTTON_TEXTURE})}); @@ -140,6 +141,9 @@ tablet.screenChanged.connect(onScreenChanged); Entities.canWriteAssetsChanged.connect(onCanWriteAssetsChanged); Script.scriptEnding.connect(function () { + if (onMarketplaceScreen) { + tablet.gotoHomeScreen(); + } tablet.removeButton(marketplaceButton); tablet.screenChanged.disconnect(onScreenChanged); Entities.canWriteAssetsChanged.disconnect(onCanWriteAssetsChanged); diff --git a/scripts/system/menu.js b/scripts/system/menu.js index 1d5f8bccd6..4ad5958144 100644 --- a/scripts/system/menu.js +++ b/scripts/system/menu.js @@ -48,6 +48,9 @@ var HOME_BUTTON_TEXTURE = "http://hifi-content.s3.amazonaws.com/alan/dev/tablet- tablet.screenChanged.connect(onScreenChanged); Script.scriptEnding.connect(function () { + if (onMenuScreen) { + tablet.gotoHomeScreen(); + } button.clicked.disconnect(onClicked); tablet.removeButton(button); tablet.screenChanged.disconnect(onScreenChanged); diff --git a/scripts/system/pal.js b/scripts/system/pal.js index 4914cbe34c..fdb6cbcaf5 100644 --- a/scripts/system/pal.js +++ b/scripts/system/pal.js @@ -696,6 +696,9 @@ function clearLocalQMLDataAndClosePAL() { } function shutdown() { + if (onPalScreen) { + tablet.gotoHomeScreen(); + } button.clicked.disconnect(onTabletButtonClicked); tablet.removeButton(button); tablet.screenChanged.disconnect(onTabletScreenChanged); diff --git a/scripts/system/tablet-users.js b/scripts/system/tablet-users.js index 8e89ac74b7..88ffa33a88 100644 --- a/scripts/system/tablet-users.js +++ b/scripts/system/tablet-users.js @@ -115,6 +115,9 @@ tablet.screenChanged.connect(onScreenChanged); function cleanup() { + if (onUsersScreen) { + tablet.gotoHomeScreen(); + } button.clicked.disconnect(onClicked); tablet.removeButton(button); } diff --git a/scripts/system/users.js b/scripts/system/users.js deleted file mode 100644 index 480b9f07a2..0000000000 --- a/scripts/system/users.js +++ /dev/null @@ -1,1281 +0,0 @@ -"use strict"; - -// -// users.js -// examples -// -// Created by David Rowe on 9 Mar 2015. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// -/*globals HMD, Toolbars, Script, Menu, Overlays, Tablet, Controller, Settings, OverlayWebWindow, Account, GlobalServices */ - -(function() { // BEGIN LOCAL_SCOPE -var button; -var buttonName = "USERS"; -var toolBar = null; -var tablet = null; - -var MENU_ITEM = "Users Online"; - -if (Settings.getValue("HUDUIEnabled")) { - toolBar = Toolbars.getToolbar("com.highfidelity.interface.toolbar.system"); - button = toolBar.addButton({ - objectName: buttonName, - imageURL: Script.resolvePath("assets/images/tools/people.svg"), - visible: true, - alpha: 0.9 - }); -} else { - tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system"); - button = tablet.addButton({ - icon: "icons/tablet-icons/users-i.svg", - text: "USERS", - isActive: Menu.isOptionChecked(MENU_ITEM), - sortOrder: 11 - }); -} - - -function onClicked() { - Menu.setIsOptionChecked(MENU_ITEM, !Menu.isOptionChecked(MENU_ITEM)); - button.editProperties({isActive: Menu.isOptionChecked(MENU_ITEM)}); -} -button.clicked.connect(onClicked); - -// resolve these paths immediately -var MIN_MAX_BUTTON_SVG = Script.resolvePath("assets/images/tools/min-max-toggle.svg"); -var BASE_URL = Script.resolvePath("assets/images/tools/"); - -var PopUpMenu = function (properties) { - var value = properties.value, - promptOverlay, - valueOverlay, - buttonOverlay, - optionOverlays = [], - isDisplayingOptions = false, - OPTION_MARGIN = 4, - - MIN_MAX_BUTTON_SVG_WIDTH = 17.1, - MIN_MAX_BUTTON_SVG_HEIGHT = 32.5, - MIN_MAX_BUTTON_WIDTH = 14, - MIN_MAX_BUTTON_HEIGHT = MIN_MAX_BUTTON_WIDTH; - - function positionDisplayOptions() { - var y, - i; - - y = properties.y - (properties.values.length - 1) * properties.lineHeight - OPTION_MARGIN; - - for (i = 0; i < properties.values.length; i += 1) { - Overlays.editOverlay(optionOverlays[i], { - y: y - }); - y += properties.lineHeight; - } - } - - function showDisplayOptions() { - var i, - yOffScreen = Controller.getViewportDimensions().y; - - for (i = 0; i < properties.values.length; i += 1) { - optionOverlays[i] = Overlays.addOverlay("text", { - x: properties.x + properties.promptWidth, - y: yOffScreen, - width: properties.width - properties.promptWidth, - height: properties.textHeight + OPTION_MARGIN, // Only need to add margin at top to balance descenders - topMargin: OPTION_MARGIN, - leftMargin: OPTION_MARGIN, - color: properties.optionColor, - alpha: properties.optionAlpha, - backgroundColor: properties.popupBackgroundColor, - backgroundAlpha: properties.popupBackgroundAlpha, - text: properties.displayValues[i], - font: properties.font, - visible: true - }); - } - - positionDisplayOptions(); - - isDisplayingOptions = true; - } - - function deleteDisplayOptions() { - var i; - - for (i = 0; i < optionOverlays.length; i += 1) { - Overlays.deleteOverlay(optionOverlays[i]); - } - - isDisplayingOptions = false; - } - - function handleClick(overlay) { - var clicked = false, - i; - - if (overlay === valueOverlay || overlay === buttonOverlay) { - showDisplayOptions(); - return true; - } - - if (isDisplayingOptions) { - for (i = 0; i < optionOverlays.length; i += 1) { - if (overlay === optionOverlays[i]) { - value = properties.values[i]; - Overlays.editOverlay(valueOverlay, { - text: properties.displayValues[i] - }); - clicked = true; - } - } - - deleteDisplayOptions(); - } - - return clicked; - } - - function updatePosition(x, y) { - properties.x = x; - properties.y = y; - Overlays.editOverlay(promptOverlay, { - x: x, - y: y - }); - Overlays.editOverlay(valueOverlay, { - x: x + properties.promptWidth, - y: y - OPTION_MARGIN - }); - Overlays.editOverlay(buttonOverlay, { - x: x + properties.width - MIN_MAX_BUTTON_WIDTH - 1, - y: y - OPTION_MARGIN + 1 - }); - if (isDisplayingOptions) { - positionDisplayOptions(); - } - } - - function setVisible(visible) { - Overlays.editOverlay(promptOverlay, { - visible: visible - }); - Overlays.editOverlay(valueOverlay, { - visible: visible - }); - Overlays.editOverlay(buttonOverlay, { - visible: visible - }); - } - - function tearDown() { - Overlays.deleteOverlay(promptOverlay); - Overlays.deleteOverlay(valueOverlay); - Overlays.deleteOverlay(buttonOverlay); - if (isDisplayingOptions) { - deleteDisplayOptions(); - } - } - - function getValue() { - return value; - } - - function setValue(newValue) { - var index; - - index = properties.values.indexOf(newValue); - if (index !== -1) { - value = newValue; - Overlays.editOverlay(valueOverlay, { - text: properties.displayValues[index] - }); - } - } - - promptOverlay = Overlays.addOverlay("text", { - x: properties.x, - y: properties.y, - width: properties.promptWidth, - height: properties.textHeight, - topMargin: 0, - leftMargin: 0, - color: properties.promptColor, - alpha: properties.promptAlpha, - backgroundColor: properties.promptBackgroundColor, - backgroundAlpha: properties.promptBackgroundAlpha, - text: properties.prompt, - font: properties.font, - visible: properties.visible - }); - - valueOverlay = Overlays.addOverlay("text", { - x: properties.x + properties.promptWidth, - y: properties.y, - width: properties.width - properties.promptWidth, - height: properties.textHeight + OPTION_MARGIN, // Only need to add margin at top to balance descenders - topMargin: OPTION_MARGIN, - leftMargin: OPTION_MARGIN, - color: properties.optionColor, - alpha: properties.optionAlpha, - backgroundColor: properties.optionBackgroundColor, - backgroundAlpha: properties.optionBackgroundAlpha, - text: properties.displayValues[properties.values.indexOf(value)], - font: properties.font, - visible: properties.visible - }); - - buttonOverlay = Overlays.addOverlay("image", { - x: properties.x + properties.width - MIN_MAX_BUTTON_WIDTH - 1, - y: properties.y, - width: MIN_MAX_BUTTON_WIDTH, - height: MIN_MAX_BUTTON_HEIGHT, - imageURL: MIN_MAX_BUTTON_SVG, - subImage: { - x: 0, - y: 0, - width: MIN_MAX_BUTTON_SVG_WIDTH, - height: MIN_MAX_BUTTON_SVG_HEIGHT / 2 - }, - //color: properties.buttonColor, - alpha: properties.buttonAlpha, - visible: properties.visible - }); - - return { - updatePosition: updatePosition, - setVisible: setVisible, - handleClick: handleClick, - tearDown: tearDown, - getValue: getValue, - setValue: setValue - }; -}; - -var usersWindow = (function () { - - var WINDOW_WIDTH = 260, - WINDOW_MARGIN = 12, - WINDOW_BASE_MARGIN = 24, // A little less is needed in order look correct - WINDOW_FONT = { - size: 12 - }, - WINDOW_FOREGROUND_COLOR = { - red: 240, - green: 240, - blue: 240 - }, - WINDOW_FOREGROUND_ALPHA = 0.95, - WINDOW_HEADING_COLOR = { - red: 180, - green: 180, - blue: 180 - }, - WINDOW_HEADING_ALPHA = 0.95, - WINDOW_BACKGROUND_COLOR = { - red: 80, - green: 80, - blue: 80 - }, - WINDOW_BACKGROUND_ALPHA = 0.8, - windowPane, - windowHeading, - - // Margin on the left and right side of the window to keep - // it from getting too close to the edge of the screen which - // is unclickable. - WINDOW_MARGIN_X = 20, - - // Window border is similar to that of edit.js. - WINDOW_MARGIN_HALF = WINDOW_MARGIN / 2, - WINDOW_BORDER_WIDTH = WINDOW_WIDTH + 2 * WINDOW_MARGIN_HALF, - WINDOW_BORDER_TOP_MARGIN = 2 * WINDOW_MARGIN_HALF, - WINDOW_BORDER_BOTTOM_MARGIN = WINDOW_MARGIN_HALF, - WINDOW_BORDER_LEFT_MARGIN = WINDOW_MARGIN_HALF, - WINDOW_BORDER_RADIUS = 4, - WINDOW_BORDER_COLOR = { red: 255, green: 255, blue: 255 }, - WINDOW_BORDER_ALPHA = 0.5, - windowBorder, - - MIN_MAX_BUTTON_SVG = BASE_URL + "min-max-toggle.svg", - MIN_MAX_BUTTON_SVG_WIDTH = 17.1, - MIN_MAX_BUTTON_SVG_HEIGHT = 32.5, - MIN_MAX_BUTTON_WIDTH = 14, - MIN_MAX_BUTTON_HEIGHT = MIN_MAX_BUTTON_WIDTH, - MIN_MAX_BUTTON_COLOR = { - red: 255, - green: 255, - blue: 255 - }, - MIN_MAX_BUTTON_ALPHA = 0.9, - minimizeButton, - SCROLLBAR_BACKGROUND_WIDTH = 12, - SCROLLBAR_BACKGROUND_COLOR = { - red: 70, - green: 70, - blue: 70 - }, - SCROLLBAR_BACKGROUND_ALPHA = 0.8, - scrollbarBackground, - SCROLLBAR_BAR_MIN_HEIGHT = 5, - SCROLLBAR_BAR_COLOR = { - red: 170, - green: 170, - blue: 170 - }, - SCROLLBAR_BAR_ALPHA = 0.8, - SCROLLBAR_BAR_SELECTED_ALPHA = 0.95, - scrollbarBar, - scrollbarBackgroundHeight, - scrollbarBarHeight, - FRIENDS_BUTTON_SPACER = 6, // Space before add/remove friends button - FRIENDS_BUTTON_SVG = BASE_URL + "add-remove-friends.svg", - FRIENDS_BUTTON_SVG_WIDTH = 107, - FRIENDS_BUTTON_SVG_HEIGHT = 27, - FRIENDS_BUTTON_WIDTH = FRIENDS_BUTTON_SVG_WIDTH, - FRIENDS_BUTTON_HEIGHT = FRIENDS_BUTTON_SVG_HEIGHT, - FRIENDS_BUTTON_COLOR = { - red: 225, - green: 225, - blue: 225 - }, - FRIENDS_BUTTON_ALPHA = 0.95, - FRIENDS_WINDOW_URL = "https://metaverse.highfidelity.com/user/friends", - FRIENDS_WINDOW_WIDTH = 290, - FRIENDS_WINDOW_HEIGHT = 500, - FRIENDS_WINDOW_TITLE = "Add/Remove Friends", - friendsButton, - friendsWindow, - - OPTION_BACKGROUND_COLOR = { - red: 60, - green: 60, - blue: 60 - }, - OPTION_BACKGROUND_ALPHA = 0.1, - - DISPLAY_SPACER = 12, // Space before display control - DISPLAY_PROMPT = "Show me:", - DISPLAY_PROMPT_WIDTH = 60, - DISPLAY_EVERYONE = "everyone", - DISPLAY_FRIENDS = "friends", - DISPLAY_VALUES = [DISPLAY_EVERYONE, DISPLAY_FRIENDS], - DISPLAY_DISPLAY_VALUES = DISPLAY_VALUES, - DISPLAY_OPTIONS_BACKGROUND_COLOR = { - red: 120, - green: 120, - blue: 120 - }, - DISPLAY_OPTIONS_BACKGROUND_ALPHA = 0.9, - displayControl, - - VISIBILITY_SPACER = 6, // Space before visibility control - VISIBILITY_PROMPT = "Visible to:", - VISIBILITY_PROMPT_WIDTH = 60, - VISIBILITY_ALL = "all", - VISIBILITY_FRIENDS = "friends", - VISIBILITY_NONE = "none", - VISIBILITY_VALUES = [VISIBILITY_ALL, VISIBILITY_FRIENDS, VISIBILITY_NONE], - VISIBILITY_DISPLAY_VALUES = ["everyone", "friends", "no one"], - visibilityControl, - - windowHeight, - windowBorderHeight, - windowTextHeight, - windowLineSpacing, - windowLineHeight, // = windowTextHeight + windowLineSpacing - windowMinimumHeight, - - usersOnline, // Raw users data - linesOfUsers = [], // Array of indexes pointing into usersOnline - numUsersToDisplay = 0, - firstUserToDisplay = 0, - - API_URL = "https://metaverse.highfidelity.com/api/v1/users?status=online", - API_FRIENDS_FILTER = "&filter=friends", - HTTP_GET_TIMEOUT = 60000, // ms = 1 minute - usersRequest, - processUsers, - pollUsersTimedOut, - usersTimer = null, - USERS_UPDATE_TIMEOUT = 5000, // ms = 5s - - showMe, - myVisibility, - - MENU_NAME = "View", - MENU_ITEM = "Users Online", - MENU_ITEM_OVERLAYS = "Overlays", - MENU_ITEM_AFTER = MENU_ITEM_OVERLAYS, - - SETTING_USERS_SHOW_ME = "UsersWindow.ShowMe", - SETTING_USERS_VISIBLE_TO = "UsersWindow.VisibleTo", - SETTING_USERS_WINDOW_MINIMIZED = "UsersWindow.Minimized", - SETTING_USERS_WINDOW_OFFSET = "UsersWindow.Offset", - // +ve x, y values are offset from left, top of screen; -ve from right, bottom. - - isLoggedIn = false, - isVisible = true, - isMinimized = false, - isBorderVisible = false, - - viewport, - isMirrorDisplay = false, - isFullscreenMirror = false, - - windowPosition = {}, // Bottom left corner of window pane. - isMovingWindow = false, - movingClickOffset = { x: 0, y: 0 }, - - isUsingScrollbars = false, - isMovingScrollbar = false, - scrollbarBackgroundPosition = {}, - scrollbarBarPosition = {}, - scrollbarBarClickedAt, // 0.0 .. 1.0 - scrollbarValue = 0.0; // 0.0 .. 1.0 - - function isWindowDisabled() { - return !Menu.isOptionChecked(MENU_ITEM) || !Menu.isOptionChecked(MENU_ITEM_OVERLAYS); - } - - function isValueTrue(value) { - // Work around Boolean Settings values being read as string when Interface starts up but as Booleans when re-read after - // Being written if refresh script. - return value === true || value === "true"; - } - - function calculateWindowHeight() { - var AUDIO_METER_HEIGHT = 52, - MIRROR_HEIGHT = 220, - nonUsersHeight, - maxWindowHeight; - - if (isMinimized) { - windowHeight = windowTextHeight + WINDOW_MARGIN + WINDOW_BASE_MARGIN; - windowBorderHeight = windowHeight + WINDOW_BORDER_TOP_MARGIN + WINDOW_BORDER_BOTTOM_MARGIN; - return; - } - - // Reserve space for title, friends button, and option controls - nonUsersHeight = WINDOW_MARGIN + windowLineHeight + - (shouldShowFriendsButton() ? FRIENDS_BUTTON_SPACER + FRIENDS_BUTTON_HEIGHT : 0) + - DISPLAY_SPACER + - windowLineHeight + VISIBILITY_SPACER + - windowLineHeight + WINDOW_BASE_MARGIN; - - // Limit window to height of viewport above window position minus VU meter and mirror if displayed - windowHeight = linesOfUsers.length * windowLineHeight - windowLineSpacing + nonUsersHeight; - maxWindowHeight = windowPosition.y - AUDIO_METER_HEIGHT; - if (isMirrorDisplay && !isFullscreenMirror) { - maxWindowHeight -= MIRROR_HEIGHT; - } - windowHeight = Math.max(Math.min(windowHeight, maxWindowHeight), nonUsersHeight); - windowBorderHeight = windowHeight + WINDOW_BORDER_TOP_MARGIN + WINDOW_BORDER_BOTTOM_MARGIN; - - // Corresponding number of users to actually display - numUsersToDisplay = Math.max(Math.round((windowHeight - nonUsersHeight) / windowLineHeight), 0); - isUsingScrollbars = 0 < numUsersToDisplay && numUsersToDisplay < linesOfUsers.length; - if (isUsingScrollbars) { - firstUserToDisplay = Math.floor(scrollbarValue * (linesOfUsers.length - numUsersToDisplay)); - } else { - firstUserToDisplay = 0; - scrollbarValue = 0.0; - } - } - - function saturateWindowPosition() { - windowPosition.x = Math.max(WINDOW_MARGIN_X, Math.min(viewport.x - WINDOW_WIDTH - WINDOW_MARGIN_X, windowPosition.x)); - windowPosition.y = Math.max(windowMinimumHeight, Math.min(viewport.y, windowPosition.y)); - } - - function updateOverlayPositions() { - // Overlay positions are all relative to windowPosition; windowPosition is the position of the windowPane overlay. - var windowLeft = windowPosition.x, - windowTop = windowPosition.y - windowHeight, - x, - y; - - Overlays.editOverlay(windowBorder, { - x: windowPosition.x - WINDOW_BORDER_LEFT_MARGIN, - y: windowTop - WINDOW_BORDER_TOP_MARGIN - }); - Overlays.editOverlay(windowPane, { - x: windowLeft, - y: windowTop - }); - Overlays.editOverlay(windowHeading, { - x: windowLeft + WINDOW_MARGIN, - y: windowTop + WINDOW_MARGIN - }); - - Overlays.editOverlay(minimizeButton, { - x: windowLeft + WINDOW_WIDTH - WINDOW_MARGIN / 2 - MIN_MAX_BUTTON_WIDTH, - y: windowTop + WINDOW_MARGIN - }); - - scrollbarBackgroundPosition.x = windowLeft + WINDOW_WIDTH - 0.5 * WINDOW_MARGIN - SCROLLBAR_BACKGROUND_WIDTH; - scrollbarBackgroundPosition.y = windowTop + WINDOW_MARGIN + windowTextHeight; - Overlays.editOverlay(scrollbarBackground, { - x: scrollbarBackgroundPosition.x, - y: scrollbarBackgroundPosition.y - }); - scrollbarBarPosition.y = scrollbarBackgroundPosition.y + 1 + - scrollbarValue * (scrollbarBackgroundHeight - scrollbarBarHeight - 2); - Overlays.editOverlay(scrollbarBar, { - x: scrollbarBackgroundPosition.x + 1, - y: scrollbarBarPosition.y - }); - - - x = windowLeft + WINDOW_MARGIN; - y = windowPosition.y - - DISPLAY_SPACER - - windowLineHeight - VISIBILITY_SPACER - - windowLineHeight - WINDOW_BASE_MARGIN; - if (shouldShowFriendsButton()) { - y -= FRIENDS_BUTTON_HEIGHT; - Overlays.editOverlay(friendsButton, { - x: x, - y: y - }); - y += FRIENDS_BUTTON_HEIGHT; - } - - y += DISPLAY_SPACER; - displayControl.updatePosition(x, y); - - y += windowLineHeight + VISIBILITY_SPACER; - visibilityControl.updatePosition(x, y); - } - - function updateUsersDisplay() { - var displayText = "", - user, - userText, - textWidth, - maxTextWidth, - ellipsisWidth, - reducedTextWidth, - i; - - if (!isMinimized) { - maxTextWidth = WINDOW_WIDTH - (isUsingScrollbars ? SCROLLBAR_BACKGROUND_WIDTH : 0) - 2 * WINDOW_MARGIN; - ellipsisWidth = Overlays.textSize(windowPane, "...").width; - reducedTextWidth = maxTextWidth - ellipsisWidth; - - for (i = 0; i < numUsersToDisplay; i += 1) { - user = usersOnline[linesOfUsers[firstUserToDisplay + i]]; - userText = user.text; - textWidth = user.textWidth; - - if (textWidth > maxTextWidth) { - // Trim and append "..." to fit window width - maxTextWidth = maxTextWidth - Overlays.textSize(windowPane, "...").width; - while (textWidth > reducedTextWidth) { - userText = userText.slice(0, -1); - textWidth = Overlays.textSize(windowPane, userText).width; - } - userText += "..."; - } - - displayText += "\n" + userText; - } - - displayText = displayText.slice(1); // Remove leading "\n". - - scrollbarBackgroundHeight = numUsersToDisplay * windowLineHeight - windowLineSpacing / 2; - Overlays.editOverlay(scrollbarBackground, { - height: scrollbarBackgroundHeight, - visible: isLoggedIn && isUsingScrollbars - }); - scrollbarBarHeight = Math.max(numUsersToDisplay / linesOfUsers.length * scrollbarBackgroundHeight, - SCROLLBAR_BAR_MIN_HEIGHT); - Overlays.editOverlay(scrollbarBar, { - height: scrollbarBarHeight, - visible: isLoggedIn && isUsingScrollbars - }); - } - - Overlays.editOverlay(windowBorder, { - height: windowBorderHeight - }); - - Overlays.editOverlay(windowPane, { - height: windowHeight, - text: displayText - }); - - Overlays.editOverlay(windowHeading, { - text: isLoggedIn ? (linesOfUsers.length > 0 ? "Users online" : "No users online") : "Users online - log in to view" - }); - } - - function shouldShowFriendsButton() { - return isVisible && isLoggedIn && !isMinimized; - } - - function updateOverlayVisibility() { - Overlays.editOverlay(windowBorder, { - visible: isVisible && isBorderVisible - }); - Overlays.editOverlay(windowPane, { - visible: isVisible - }); - Overlays.editOverlay(windowHeading, { - visible: isVisible - }); - Overlays.editOverlay(minimizeButton, { - visible: isVisible - }); - Overlays.editOverlay(scrollbarBackground, { - visible: isVisible && isUsingScrollbars && !isMinimized - }); - Overlays.editOverlay(scrollbarBar, { - visible: isVisible && isUsingScrollbars && !isMinimized - }); - Overlays.editOverlay(friendsButton, { - visible: shouldShowFriendsButton() - }); - displayControl.setVisible(isVisible && !isMinimized); - visibilityControl.setVisible(isVisible && !isMinimized); - } - - function checkLoggedIn() { - var wasLoggedIn = isLoggedIn; - - isLoggedIn = Account.isLoggedIn(); - if (isLoggedIn !== wasLoggedIn) { - if (wasLoggedIn) { - setMinimized(true); - calculateWindowHeight(); - updateOverlayPositions(); - updateUsersDisplay(); - } - - updateOverlayVisibility(); - } - } - - function pollUsers() { - var url = API_URL; - - if (showMe === DISPLAY_FRIENDS) { - url += API_FRIENDS_FILTER; - } - - usersRequest = new XMLHttpRequest(); - usersRequest.open("GET", url, true); - usersRequest.timeout = HTTP_GET_TIMEOUT; - usersRequest.ontimeout = pollUsersTimedOut; - usersRequest.onreadystatechange = processUsers; - usersRequest.send(); - } - - processUsers = function () { - var response, - myUsername, - user, - userText, - i; - - if (usersRequest.readyState === usersRequest.DONE) { - if (usersRequest.status === 200) { - response = JSON.parse(usersRequest.responseText); - if (response.status !== "success") { - print("Error: Request for users status returned status = " + response.status); - usersTimer = Script.setTimeout(pollUsers, HTTP_GET_TIMEOUT); // Try again after a longer delay. - return; - } - if (!response.hasOwnProperty("data") || !response.data.hasOwnProperty("users")) { - print("Error: Request for users status returned invalid data"); - usersTimer = Script.setTimeout(pollUsers, HTTP_GET_TIMEOUT); // Try again after a longer delay. - return; - } - - usersOnline = response.data.users; - myUsername = GlobalServices.username; - linesOfUsers = []; - for (i = 0; i < usersOnline.length; i += 1) { - user = usersOnline[i]; - if (user.username !== myUsername && user.online) { - userText = user.username; - if (user.location.root) { - userText += " @ " + user.location.root.name; - } - - usersOnline[i].text = userText; - usersOnline[i].textWidth = Overlays.textSize(windowPane, userText).width; - - linesOfUsers.push(i); - } - } - - checkLoggedIn(); - calculateWindowHeight(); - updateUsersDisplay(); - updateOverlayPositions(); - - } else { - print("Error: Request for users status returned " + usersRequest.status + " " + usersRequest.statusText); - usersTimer = Script.setTimeout(pollUsers, HTTP_GET_TIMEOUT); // Try again after a longer delay. - return; - } - - usersTimer = Script.setTimeout(pollUsers, USERS_UPDATE_TIMEOUT); // Update after finished processing. - } - }; - - pollUsersTimedOut = function () { - print("Error: Request for users status timed out"); - usersTimer = Script.setTimeout(pollUsers, HTTP_GET_TIMEOUT); // Try again after a longer delay. - }; - - function setVisible(visible) { - isVisible = visible; - - if (isVisible) { - if (usersTimer === null) { - pollUsers(); - } - } else { - Script.clearTimeout(usersTimer); - usersTimer = null; - } - - updateOverlayVisibility(); - } - - function setMinimized(minimized) { - isMinimized = minimized; - Overlays.editOverlay(minimizeButton, { - subImage: { - y: isMinimized ? MIN_MAX_BUTTON_SVG_HEIGHT / 2 : 0 - } - }); - updateOverlayVisibility(); - Settings.setValue(SETTING_USERS_WINDOW_MINIMIZED, isMinimized); - } - - function onMenuItemEvent(event) { - if (event === MENU_ITEM) { - setVisible(Menu.isOptionChecked(MENU_ITEM)); - } - } - - function onFindableByChanged(event) { - if (VISIBILITY_VALUES.indexOf(event) !== -1) { - myVisibility = event; - visibilityControl.setValue(event); - Settings.setValue(SETTING_USERS_VISIBLE_TO, myVisibility); - } else { - print("Error: Unrecognized onFindableByChanged value: " + event); - } - } - - function onMousePressEvent(event) { - var clickedOverlay, - numLinesBefore, - overlayX, - overlayY, - minY, - maxY, - lineClicked, - userClicked, - delta; - - if (!isVisible || isWindowDisabled()) { - return; - } - - clickedOverlay = Overlays.getOverlayAtPoint({ - x: event.x, - y: event.y - }); - - if (displayControl.handleClick(clickedOverlay)) { - if (usersTimer !== null) { - Script.clearTimeout(usersTimer); - usersTimer = null; - } - pollUsers(); - showMe = displayControl.getValue(); - Settings.setValue(SETTING_USERS_SHOW_ME, showMe); - return; - } - - if (visibilityControl.handleClick(clickedOverlay)) { - myVisibility = visibilityControl.getValue(); - GlobalServices.findableBy = myVisibility; - Settings.setValue(SETTING_USERS_VISIBLE_TO, myVisibility); - return; - } - - if (clickedOverlay === windowPane) { - - overlayX = event.x - windowPosition.x - WINDOW_MARGIN; - overlayY = event.y - windowPosition.y + windowHeight - WINDOW_MARGIN - windowLineHeight; - - numLinesBefore = Math.round(overlayY / windowLineHeight); - minY = numLinesBefore * windowLineHeight; - maxY = minY + windowTextHeight; - - lineClicked = -1; - if (minY <= overlayY && overlayY <= maxY) { - lineClicked = numLinesBefore; - } - - userClicked = firstUserToDisplay + lineClicked; - - if (0 <= userClicked && userClicked < linesOfUsers.length && 0 <= overlayX && - overlayX <= usersOnline[linesOfUsers[userClicked]].textWidth) { - //print("Go to " + usersOnline[linesOfUsers[userClicked]].username); - location.goToUser(usersOnline[linesOfUsers[userClicked]].username); - } - - return; - } - - if (clickedOverlay === minimizeButton) { - setMinimized(!isMinimized); - calculateWindowHeight(); - updateOverlayPositions(); - updateUsersDisplay(); - return; - } - - if (clickedOverlay === scrollbarBar) { - scrollbarBarClickedAt = (event.y - scrollbarBarPosition.y) / scrollbarBarHeight; - Overlays.editOverlay(scrollbarBar, { - backgroundAlpha: SCROLLBAR_BAR_SELECTED_ALPHA - }); - isMovingScrollbar = true; - return; - } - - if (clickedOverlay === scrollbarBackground) { - delta = scrollbarBarHeight / (scrollbarBackgroundHeight - scrollbarBarHeight); - - if (event.y < scrollbarBarPosition.y) { - scrollbarValue = Math.max(scrollbarValue - delta, 0.0); - } else { - scrollbarValue = Math.min(scrollbarValue + delta, 1.0); - } - - firstUserToDisplay = Math.floor(scrollbarValue * (linesOfUsers.length - numUsersToDisplay)); - updateOverlayPositions(); - updateUsersDisplay(); - return; - } - - if (clickedOverlay === friendsButton) { - if (!friendsWindow) { - friendsWindow = new OverlayWebWindow({ - title: FRIENDS_WINDOW_TITLE, - width: FRIENDS_WINDOW_WIDTH, - height: FRIENDS_WINDOW_HEIGHT, - visible: false - }); - } - friendsWindow.setURL(FRIENDS_WINDOW_URL); - friendsWindow.setVisible(true); - friendsWindow.raise(); - return; - } - - if (clickedOverlay === windowBorder) { - movingClickOffset = { - x: event.x - windowPosition.x, - y: event.y - windowPosition.y - }; - - isMovingWindow = true; - } - } - - function onMouseMoveEvent(event) { - var isVisible; - - if (!isLoggedIn || isWindowDisabled()) { - return; - } - - if (isMovingScrollbar) { - if (scrollbarBackgroundPosition.x - WINDOW_MARGIN <= event.x && - event.x <= scrollbarBackgroundPosition.x + SCROLLBAR_BACKGROUND_WIDTH + WINDOW_MARGIN && - scrollbarBackgroundPosition.y - WINDOW_MARGIN <= event.y && - event.y <= scrollbarBackgroundPosition.y + scrollbarBackgroundHeight + WINDOW_MARGIN) { - scrollbarValue = (event.y - scrollbarBarClickedAt * scrollbarBarHeight - scrollbarBackgroundPosition.y) / - (scrollbarBackgroundHeight - scrollbarBarHeight - 2); - scrollbarValue = Math.min(Math.max(scrollbarValue, 0.0), 1.0); - firstUserToDisplay = Math.floor(scrollbarValue * (linesOfUsers.length - numUsersToDisplay)); - updateOverlayPositions(); - updateUsersDisplay(); - } else { - Overlays.editOverlay(scrollbarBar, { - backgroundAlpha: SCROLLBAR_BAR_ALPHA - }); - isMovingScrollbar = false; - } - } - - if (isMovingWindow) { - windowPosition = { - x: event.x - movingClickOffset.x, - y: event.y - movingClickOffset.y - }; - - saturateWindowPosition(); - calculateWindowHeight(); - updateOverlayPositions(); - updateUsersDisplay(); - - } else { - - isVisible = isBorderVisible; - if (isVisible) { - isVisible = windowPosition.x - WINDOW_BORDER_LEFT_MARGIN <= event.x && - event.x <= windowPosition.x - WINDOW_BORDER_LEFT_MARGIN + WINDOW_BORDER_WIDTH && - windowPosition.y - windowHeight - WINDOW_BORDER_TOP_MARGIN <= event.y && - event.y <= windowPosition.y + WINDOW_BORDER_BOTTOM_MARGIN; - } else { - isVisible = windowPosition.x <= event.x && event.x <= windowPosition.x + WINDOW_WIDTH && - windowPosition.y - windowHeight <= event.y && event.y <= windowPosition.y; - } - if (isVisible !== isBorderVisible) { - isBorderVisible = isVisible; - Overlays.editOverlay(windowBorder, { - visible: isBorderVisible - }); - } - } - } - - function onMouseReleaseEvent() { - var offset = {}; - - if (isWindowDisabled()) { - return; - } - - if (isMovingScrollbar) { - Overlays.editOverlay(scrollbarBar, { - backgroundAlpha: SCROLLBAR_BAR_ALPHA - }); - isMovingScrollbar = false; - } - - if (isMovingWindow) { - // Save offset of bottom of window to nearest edge of the window. - offset.x = (windowPosition.x + WINDOW_WIDTH / 2 < viewport.x / 2) ? - windowPosition.x : windowPosition.x - viewport.x; - offset.y = (windowPosition.y < viewport.y / 2) ? - windowPosition.y : windowPosition.y - viewport.y; - Settings.setValue(SETTING_USERS_WINDOW_OFFSET, JSON.stringify(offset)); - isMovingWindow = false; - } - } - - function onScriptUpdate() { - var oldViewport = viewport, - oldIsMirrorDisplay = isMirrorDisplay, - oldIsFullscreenMirror = isFullscreenMirror, - MIRROR_MENU_ITEM = "Mirror", - FULLSCREEN_MIRROR_MENU_ITEM = "Fullscreen Mirror"; - - if (isWindowDisabled()) { - return; - } - - viewport = Controller.getViewportDimensions(); - isMirrorDisplay = Menu.isOptionChecked(MIRROR_MENU_ITEM); - isFullscreenMirror = Menu.isOptionChecked(FULLSCREEN_MIRROR_MENU_ITEM); - - if (viewport.y !== oldViewport.y || isMirrorDisplay !== oldIsMirrorDisplay || - isFullscreenMirror !== oldIsFullscreenMirror) { - calculateWindowHeight(); - updateUsersDisplay(); - } - - if (viewport.y !== oldViewport.y) { - if (windowPosition.y > oldViewport.y / 2) { - // Maintain position w.r.t. bottom of window. - windowPosition.y = viewport.y - (oldViewport.y - windowPosition.y); - } - } - - if (viewport.x !== oldViewport.x) { - if (windowPosition.x + (WINDOW_WIDTH / 2) > oldViewport.x / 2) { - // Maintain position w.r.t. right of window. - windowPosition.x = viewport.x - (oldViewport.x - windowPosition.x); - } - } - - updateOverlayPositions(); - } - - function setUp() { - var textSizeOverlay, - offsetSetting, - offset = {}, - hmdViewport; - - textSizeOverlay = Overlays.addOverlay("text", { - font: WINDOW_FONT, - visible: false - }); - windowTextHeight = Math.floor(Overlays.textSize(textSizeOverlay, "1").height); - windowLineSpacing = Math.floor(Overlays.textSize(textSizeOverlay, "1\n2").height - 2 * windowTextHeight); - windowLineHeight = windowTextHeight + windowLineSpacing; - windowMinimumHeight = windowTextHeight + WINDOW_MARGIN + WINDOW_BASE_MARGIN; - Overlays.deleteOverlay(textSizeOverlay); - - viewport = Controller.getViewportDimensions(); - - offsetSetting = Settings.getValue(SETTING_USERS_WINDOW_OFFSET); - if (offsetSetting !== "") { - offset = JSON.parse(Settings.getValue(SETTING_USERS_WINDOW_OFFSET)); - } - if (offset.hasOwnProperty("x") && offset.hasOwnProperty("y")) { - windowPosition.x = offset.x < 0 ? viewport.x + offset.x : offset.x; - windowPosition.y = offset.y <= 0 ? viewport.y + offset.y : offset.y; - } else { - hmdViewport = Controller.getRecommendedOverlayRect(); - windowPosition = { - x: (viewport.x - hmdViewport.width) / 2, // HMD viewport is narrower than screen. - y: hmdViewport.height // HMD viewport starts at top of screen but only extends down so far. - }; - } - - saturateWindowPosition(); - calculateWindowHeight(); - - windowBorder = Overlays.addOverlay("rectangle", { - x: 0, - y: viewport.y, // Start up off-screen - width: WINDOW_BORDER_WIDTH, - height: windowBorderHeight, - radius: WINDOW_BORDER_RADIUS, - color: WINDOW_BORDER_COLOR, - alpha: WINDOW_BORDER_ALPHA, - visible: false - }); - - windowPane = Overlays.addOverlay("text", { - x: 0, - y: viewport.y, - width: WINDOW_WIDTH, - height: windowHeight, - topMargin: WINDOW_MARGIN + windowLineHeight, - leftMargin: WINDOW_MARGIN, - color: WINDOW_FOREGROUND_COLOR, - alpha: WINDOW_FOREGROUND_ALPHA, - backgroundColor: WINDOW_BACKGROUND_COLOR, - backgroundAlpha: WINDOW_BACKGROUND_ALPHA, - text: "", - font: WINDOW_FONT, - visible: false - }); - - windowHeading = Overlays.addOverlay("text", { - x: 0, - y: viewport.y, - width: WINDOW_WIDTH - 2 * WINDOW_MARGIN, - height: windowTextHeight, - topMargin: 0, - leftMargin: 0, - color: WINDOW_HEADING_COLOR, - alpha: WINDOW_HEADING_ALPHA, - backgroundAlpha: 0.0, - text: "Users online", - font: WINDOW_FONT, - visible: false - }); - - minimizeButton = Overlays.addOverlay("image", { - x: 0, - y: viewport.y, - width: MIN_MAX_BUTTON_WIDTH, - height: MIN_MAX_BUTTON_HEIGHT, - imageURL: MIN_MAX_BUTTON_SVG, - subImage: { - x: 0, - y: 0, - width: MIN_MAX_BUTTON_SVG_WIDTH, - height: MIN_MAX_BUTTON_SVG_HEIGHT / 2 - }, - color: MIN_MAX_BUTTON_COLOR, - alpha: MIN_MAX_BUTTON_ALPHA, - visible: false - }); - - scrollbarBackgroundPosition = { - x: 0, - y: viewport.y - }; - scrollbarBackground = Overlays.addOverlay("text", { - x: 0, - y: scrollbarBackgroundPosition.y, - width: SCROLLBAR_BACKGROUND_WIDTH, - height: windowTextHeight, - backgroundColor: SCROLLBAR_BACKGROUND_COLOR, - backgroundAlpha: SCROLLBAR_BACKGROUND_ALPHA, - text: "", - visible: false - }); - - scrollbarBarPosition = { - x: 0, - y: viewport.y - }; - scrollbarBar = Overlays.addOverlay("text", { - x: 0, - y: scrollbarBarPosition.y, - width: SCROLLBAR_BACKGROUND_WIDTH - 2, - height: windowTextHeight, - backgroundColor: SCROLLBAR_BAR_COLOR, - backgroundAlpha: SCROLLBAR_BAR_ALPHA, - text: "", - visible: false - }); - - friendsButton = Overlays.addOverlay("image", { - x: 0, - y: viewport.y, - width: FRIENDS_BUTTON_WIDTH, - height: FRIENDS_BUTTON_HEIGHT, - imageURL: FRIENDS_BUTTON_SVG, - subImage: { - x: 0, - y: 0, - width: FRIENDS_BUTTON_SVG_WIDTH, - height: FRIENDS_BUTTON_SVG_HEIGHT - }, - color: FRIENDS_BUTTON_COLOR, - alpha: FRIENDS_BUTTON_ALPHA, - visible: false - }); - - showMe = Settings.getValue(SETTING_USERS_SHOW_ME, ""); - if (DISPLAY_VALUES.indexOf(showMe) === -1) { - showMe = DISPLAY_EVERYONE; - } - - displayControl = new PopUpMenu({ - prompt: DISPLAY_PROMPT, - value: showMe, - values: DISPLAY_VALUES, - displayValues: DISPLAY_DISPLAY_VALUES, - x: 0, - y: viewport.y, - width: WINDOW_WIDTH - 1.5 * WINDOW_MARGIN, - promptWidth: DISPLAY_PROMPT_WIDTH, - lineHeight: windowLineHeight, - textHeight: windowTextHeight, - font: WINDOW_FONT, - promptColor: WINDOW_HEADING_COLOR, - promptAlpha: WINDOW_HEADING_ALPHA, - promptBackgroundColor: WINDOW_BACKGROUND_COLOR, - promptBackgroundAlpha: 0.0, - optionColor: WINDOW_FOREGROUND_COLOR, - optionAlpha: WINDOW_FOREGROUND_ALPHA, - optionBackgroundColor: OPTION_BACKGROUND_COLOR, - optionBackgroundAlpha: OPTION_BACKGROUND_ALPHA, - popupBackgroundColor: DISPLAY_OPTIONS_BACKGROUND_COLOR, - popupBackgroundAlpha: DISPLAY_OPTIONS_BACKGROUND_ALPHA, - buttonColor: MIN_MAX_BUTTON_COLOR, - buttonAlpha: MIN_MAX_BUTTON_ALPHA, - visible: false - }); - - myVisibility = Settings.getValue(SETTING_USERS_VISIBLE_TO, ""); - if (VISIBILITY_VALUES.indexOf(myVisibility) === -1) { - myVisibility = VISIBILITY_FRIENDS; - } - GlobalServices.findableBy = myVisibility; - - visibilityControl = new PopUpMenu({ - prompt: VISIBILITY_PROMPT, - value: myVisibility, - values: VISIBILITY_VALUES, - displayValues: VISIBILITY_DISPLAY_VALUES, - x: 0, - y: viewport.y, - width: WINDOW_WIDTH - 1.5 * WINDOW_MARGIN, - promptWidth: VISIBILITY_PROMPT_WIDTH, - lineHeight: windowLineHeight, - textHeight: windowTextHeight, - font: WINDOW_FONT, - promptColor: WINDOW_HEADING_COLOR, - promptAlpha: WINDOW_HEADING_ALPHA, - promptBackgroundColor: WINDOW_BACKGROUND_COLOR, - promptBackgroundAlpha: 0.0, - optionColor: WINDOW_FOREGROUND_COLOR, - optionAlpha: WINDOW_FOREGROUND_ALPHA, - optionBackgroundColor: OPTION_BACKGROUND_COLOR, - optionBackgroundAlpha: OPTION_BACKGROUND_ALPHA, - popupBackgroundColor: DISPLAY_OPTIONS_BACKGROUND_COLOR, - popupBackgroundAlpha: DISPLAY_OPTIONS_BACKGROUND_ALPHA, - buttonColor: MIN_MAX_BUTTON_COLOR, - buttonAlpha: MIN_MAX_BUTTON_ALPHA, - visible: false - }); - - Controller.mousePressEvent.connect(onMousePressEvent); - Controller.mouseMoveEvent.connect(onMouseMoveEvent); - Controller.mouseReleaseEvent.connect(onMouseReleaseEvent); - - Menu.addMenuItem({ - menuName: MENU_NAME, - menuItemName: MENU_ITEM, - afterItem: MENU_ITEM_AFTER, - isCheckable: true, - isChecked: isVisible - }); - Menu.menuItemEvent.connect(onMenuItemEvent); - - GlobalServices.findableByChanged.connect(onFindableByChanged); - - Script.update.connect(onScriptUpdate); - - pollUsers(); - - // Set minimized at end - setup code does not handle `minimized == false` correctly - setMinimized(isValueTrue(Settings.getValue(SETTING_USERS_WINDOW_MINIMIZED, true))); - } - - function tearDown() { - Menu.removeMenuItem(MENU_NAME, MENU_ITEM); - - Script.clearTimeout(usersTimer); - Overlays.deleteOverlay(windowBorder); - Overlays.deleteOverlay(windowPane); - Overlays.deleteOverlay(windowHeading); - Overlays.deleteOverlay(minimizeButton); - Overlays.deleteOverlay(scrollbarBackground); - Overlays.deleteOverlay(scrollbarBar); - Overlays.deleteOverlay(friendsButton); - displayControl.tearDown(); - visibilityControl.tearDown(); - } - - setUp(); - Script.scriptEnding.connect(tearDown); -}()); - -function cleanup () { - //remove tablet button - button.clicked.disconnect(onClicked); - if (tablet) { - tablet.removeButton(button); - } - if (toolBar) { - toolBar.removeButton(buttonName); - } -} -Script.scriptEnding.connect(cleanup); - -}()); // END LOCAL_SCOPE From ce105e0329b03ac8086afcea6e79d714417b4546 Mon Sep 17 00:00:00 2001 From: Dante Ruiz Date: Wed, 8 Mar 2017 22:21:57 +0000 Subject: [PATCH 27/64] fix double tablets when reloading scripts --- scripts/system/tablet-ui/tabletUI.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/system/tablet-ui/tabletUI.js b/scripts/system/tablet-ui/tabletUI.js index d5065cb826..fd73578328 100644 --- a/scripts/system/tablet-ui/tabletUI.js +++ b/scripts/system/tablet-ui/tabletUI.js @@ -131,7 +131,9 @@ } Script.scriptEnding.connect(function () { - Entities.deleteEntity(HMD.tabletID); + var tabletID = HMD.tabletID; + Entities.deleteEntity(tabletID); + Overlays.deleteOverlay(tabletID) HMD.tabletID = null; HMD.homeButtonID = null; HMD.tabletScreenID = null; From 5f7a0ea8fc0857723cc7f579eb54e49d06e355f7 Mon Sep 17 00:00:00 2001 From: Dante Ruiz Date: Thu, 9 Mar 2017 00:05:26 +0000 Subject: [PATCH 28/64] Close tablet when switching form HMD to desktop and vice versa --- scripts/system/hmd.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/system/hmd.js b/scripts/system/hmd.js index c206a76e3f..8174cd960c 100644 --- a/scripts/system/hmd.js +++ b/scripts/system/hmd.js @@ -21,10 +21,12 @@ var desktopMenuItemName = "Desktop"; ['OpenVR (Vive)', 'Oculus Rift'].forEach(function (name) { if (!headset && Menu.menuItemExists(displayMenuName, name)) { headset = name; + } }); -var controllerDisplay = false; + + var controllerDisplay = false; function updateControllerDisplay() { if (HMD.active && Menu.isOptionChecked("Third Person")) { if (!controllerDisplay) { @@ -37,7 +39,9 @@ function updateControllerDisplay() { } } -var button; + + var button; + var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system"); // Independent and Entity mode make people sick. Third Person and Mirror have traps that we need to work through. @@ -45,6 +49,7 @@ var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system"); var desktopOnlyViews = ['Mirror', 'Independent Mode', 'Entity Mode']; function onHmdChanged(isHmd) { + HMD.closeTablet(); if (isHmd) { button.editProperties({ icon: "icons/tablet-icons/switch-desk-i.svg", From 561bd27c38e4997e4deba36fd1dd483120c36894 Mon Sep 17 00:00:00 2001 From: Dante Ruiz Date: Thu, 9 Mar 2017 00:08:50 +0000 Subject: [PATCH 29/64] minimize diff --- scripts/system/hmd.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/system/hmd.js b/scripts/system/hmd.js index 8174cd960c..452801c786 100644 --- a/scripts/system/hmd.js +++ b/scripts/system/hmd.js @@ -21,7 +21,6 @@ var desktopMenuItemName = "Desktop"; ['OpenVR (Vive)', 'Oculus Rift'].forEach(function (name) { if (!headset && Menu.menuItemExists(displayMenuName, name)) { headset = name; - } }); @@ -40,7 +39,7 @@ function updateControllerDisplay() { } - var button; +var button; var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system"); From f6d799c048818e4bd7d043c6154a0ae96c370b51 Mon Sep 17 00:00:00 2001 From: Dante Ruiz Date: Thu, 9 Mar 2017 00:09:32 +0000 Subject: [PATCH 30/64] minimize diff --- scripts/system/hmd.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/system/hmd.js b/scripts/system/hmd.js index 452801c786..af4abf0788 100644 --- a/scripts/system/hmd.js +++ b/scripts/system/hmd.js @@ -25,7 +25,7 @@ var desktopMenuItemName = "Desktop"; }); - var controllerDisplay = false; +var controllerDisplay = false; function updateControllerDisplay() { if (HMD.active && Menu.isOptionChecked("Third Person")) { if (!controllerDisplay) { From b878f164fbf0a18191bb73a56d4e523141d2f2a0 Mon Sep 17 00:00:00 2001 From: Dante Ruiz Date: Thu, 9 Mar 2017 00:10:38 +0000 Subject: [PATCH 31/64] minimize diff again --- scripts/system/hmd.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/system/hmd.js b/scripts/system/hmd.js index af4abf0788..39f8338d72 100644 --- a/scripts/system/hmd.js +++ b/scripts/system/hmd.js @@ -24,7 +24,6 @@ var desktopMenuItemName = "Desktop"; } }); - var controllerDisplay = false; function updateControllerDisplay() { if (HMD.active && Menu.isOptionChecked("Third Person")) { @@ -38,9 +37,7 @@ function updateControllerDisplay() { } } - -var button; - +var button; var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system"); // Independent and Entity mode make people sick. Third Person and Mirror have traps that we need to work through. From 5847f86db8e3598bd6be32bb5110a62991d1adb7 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Wed, 8 Mar 2017 16:12:03 -0800 Subject: [PATCH 32/64] fix crash when unpacking too many joints --- libraries/animation/src/Rig.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/libraries/animation/src/Rig.cpp b/libraries/animation/src/Rig.cpp index b70d28fc30..5573f5c0fe 100644 --- a/libraries/animation/src/Rig.cpp +++ b/libraries/animation/src/Rig.cpp @@ -1310,17 +1310,18 @@ void Rig::copyJointsFromJointData(const QVector& jointDataVec) { if (!_animSkeleton) { return; } - if (jointDataVec.size() != (int)_internalPoseSet._relativePoses.size()) { - // animations haven't fully loaded yet. - _internalPoseSet._relativePoses = _animSkeleton->getRelativeDefaultPoses(); + int numJoints = jointDataVec.size(); + const AnimPoseVec& absoluteDefaultPoses = _animSkeleton->getAbsoluteDefaultPoses(); + if (numJoints != (int)absoluteDefaultPoses.size()) { + // jointData is incompatible + return; } // make a vector of rotations in absolute-geometry-frame - const AnimPoseVec& absoluteDefaultPoses = _animSkeleton->getAbsoluteDefaultPoses(); std::vector rotations; - rotations.reserve(absoluteDefaultPoses.size()); + rotations.reserve(numJoints); const glm::quat rigToGeometryRot(glmExtractRotation(_rigToGeometryTransform)); - for (int i = 0; i < jointDataVec.size(); i++) { + for (int i = 0; i < numJoints; i++) { const JointData& data = jointDataVec.at(i); if (data.rotationSet) { // JointData rotations are in absolute rig-frame so we rotate them to absolute geometry-frame @@ -1334,8 +1335,11 @@ void Rig::copyJointsFromJointData(const QVector& jointDataVec) { _animSkeleton->convertAbsoluteRotationsToRelative(rotations); // store new relative poses + if (numJoints != _internalPoseSet._relativePoses.size()) { + _internalPoseSet._relativePoses = _animSkeleton->getRelativeDefaultPoses(); + } const AnimPoseVec& relativeDefaultPoses = _animSkeleton->getRelativeDefaultPoses(); - for (int i = 0; i < jointDataVec.size(); i++) { + for (int i = 0; i < numJoints; i++) { const JointData& data = jointDataVec.at(i); _internalPoseSet._relativePoses[i].scale() = Vectors::ONE; _internalPoseSet._relativePoses[i].rot() = rotations[i]; From 2a275a4f4b7dbb448a3b23a16f7cd3f30a12e7e0 Mon Sep 17 00:00:00 2001 From: Dante Ruiz Date: Thu, 9 Mar 2017 00:15:02 +0000 Subject: [PATCH 33/64] removed tab at the end of word --- scripts/system/hmd.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/system/hmd.js b/scripts/system/hmd.js index 39f8338d72..c545e6bcee 100644 --- a/scripts/system/hmd.js +++ b/scripts/system/hmd.js @@ -37,7 +37,7 @@ function updateControllerDisplay() { } } -var button; +var button; var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system"); // Independent and Entity mode make people sick. Third Person and Mirror have traps that we need to work through. From 30d7ffb303925a2f0c7db6ed9e010742ffe43f61 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Wed, 8 Mar 2017 17:17:49 -0800 Subject: [PATCH 34/64] fix warning about signed unsigned compare --- libraries/animation/src/Rig.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/animation/src/Rig.cpp b/libraries/animation/src/Rig.cpp index 5573f5c0fe..9ecd0f6352 100644 --- a/libraries/animation/src/Rig.cpp +++ b/libraries/animation/src/Rig.cpp @@ -1335,7 +1335,7 @@ void Rig::copyJointsFromJointData(const QVector& jointDataVec) { _animSkeleton->convertAbsoluteRotationsToRelative(rotations); // store new relative poses - if (numJoints != _internalPoseSet._relativePoses.size()) { + if (numJoints != (int)_internalPoseSet._relativePoses.size()) { _internalPoseSet._relativePoses = _animSkeleton->getRelativeDefaultPoses(); } const AnimPoseVec& relativeDefaultPoses = _animSkeleton->getRelativeDefaultPoses(); From 1bdad89cbe2bd95b1af910ee6d4c10d232c86012 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 8 Mar 2017 17:27:46 -0800 Subject: [PATCH 35/64] properly handle silent packet transitions --- libraries/audio-client/src/AudioClient.cpp | 7 +++- libraries/audio-client/src/AudioNoiseGate.cpp | 7 +++- libraries/audio-client/src/AudioNoiseGate.h | 5 +++ libraries/audio/src/InboundAudioStream.cpp | 34 ++++++++++++++++--- 4 files changed, 46 insertions(+), 7 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index bd141cfb12..9cd87d2e70 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1052,7 +1052,12 @@ void AudioClient::handleAudioInput() { auto packetType = _shouldEchoToServer ? PacketType::MicrophoneAudioWithEcho : PacketType::MicrophoneAudioNoEcho; - if (_lastInputLoudness == 0) { + // if the _inputGate closed in this last frame, then we don't actually want + // to send a silent packet, instead, we want to go ahead and encode and send + // the output from the input gate (eventually, this could be crossfaded) + // and allow the codec to properly encode down to silent/zero. If we still + // have _lastInputLoudness of 0 in our NEXT frame, we will send a silent packet + if (_lastInputLoudness == 0 && !_inputGate.closedInLastFrame()) { packetType = PacketType::SilentAudioFrame; } Transform audioTransform; diff --git a/libraries/audio-client/src/AudioNoiseGate.cpp b/libraries/audio-client/src/AudioNoiseGate.cpp index 8766a20cdf..9458f47d8c 100644 --- a/libraries/audio-client/src/AudioNoiseGate.cpp +++ b/libraries/audio-client/src/AudioNoiseGate.cpp @@ -58,6 +58,7 @@ void AudioNoiseGate::removeDCOffset(int16_t* samples, int numSamples) { } } +#include void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { // @@ -77,7 +78,8 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { // NOISE_GATE_FRAMES_TO_AVERAGE: How many audio frames should we average together to compute noise floor. // More means better rejection but also can reject continuous things like singing. // NUMBER_OF_NOISE_SAMPLE_FRAMES: How often should we re-evaluate the noise floor? - + + _closedInLastFrame = false; float loudness = 0; int thisSample = 0; @@ -147,6 +149,9 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { _framesToClose = NOISE_GATE_CLOSE_FRAME_DELAY; } else { if (--_framesToClose == 0) { + if (_isOpen) { + _closedInLastFrame = true; + } _isOpen = false; } } diff --git a/libraries/audio-client/src/AudioNoiseGate.h b/libraries/audio-client/src/AudioNoiseGate.h index 8cb1155938..18b5a77056 100644 --- a/libraries/audio-client/src/AudioNoiseGate.h +++ b/libraries/audio-client/src/AudioNoiseGate.h @@ -24,6 +24,10 @@ public: void removeDCOffset(int16_t* samples, int numSamples); bool clippedInLastFrame() const { return _didClipInLastFrame; } + + bool closedInLastFrame() const { return _closedInLastFrame; } + + bool isOpen() const { return _isOpen; } float getMeasuredFloor() const { return _measuredFloor; } float getLastLoudness() const { return _lastLoudness; } @@ -40,6 +44,7 @@ private: float _sampleFrames[NUMBER_OF_NOISE_SAMPLE_FRAMES]; int _sampleCounter; bool _isOpen; + bool _closedInLastFrame { false }; int _framesToClose; }; diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 57c344adaf..12181cb8e2 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -136,9 +136,10 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { break; } case SequenceNumberStats::Early: { - // Packet is early; write droppable silent samples for each of the skipped packets. - // NOTE: we assume that each dropped packet contains the same number of samples - // as the packet we just received. + // Packet is early treat the packets as if all the packets between the last + // OnTime packet and this packet was lost. If we're using a codec this will + // also result in allowing the codec to flush its internal state. Then + // fall through to the "on time" logic to actually handle this packet int packetsDropped = arrivalInfo._seqDiffFromExpected; lostAudioData(packetsDropped); @@ -147,7 +148,10 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { case SequenceNumberStats::OnTime: { // Packet is on time; parse its data to the ringbuffer if (message.getType() == PacketType::SilentAudioFrame) { - // FIXME - Some codecs need to know about these silent frames... and can produce better output + // If we recieved a SilentAudioFrame from our sender, we might want to drop + // some of the samples in order to catch up to our desired jitter buffer size. + // NOTE: If we're using a codec we will be calling the codec's lostFrame() + // method to allow the codec to flush its internal state. writeDroppableSilentFrames(networkFrames); } else { // note: PCM and no codec are identical @@ -158,7 +162,12 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { parseAudioData(message.getType(), afterProperties); } else { qDebug(audio) << "Codec mismatch: expected" << _selectedCodecName << "got" << codecInPacket << "writing silence"; - writeDroppableSilentFrames(networkFrames); + + // Since the data in the stream is using a codec that we're not prepared for, + // we need to let the codec know that we don't have data for it, this will + // flush any internal codec state and produce fade to silence. + lostAudioData(1); + // inform others of the mismatch auto sendingNode = DependencyManager::get()->nodeWithUUID(message.getSourceID()); emit mismatchedAudioCodec(sendingNode, _selectedCodecName, codecInPacket); @@ -240,6 +249,21 @@ int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packet int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) { + // if we have a decoder, we still want to tell the decoder about our + // lost frame. this will flush the internal state of the decoder + // we can safely ignore the output of the codec in this case, because + // we've enforced that on the sending side, the encoder ran at least + // one frame of truly silent audio before we sent the "droppable" silent + // frame. Technically we could leave this out, if we know for certain + // that the sender has really sent us an encoded packet of zeros, but + // since we can trust all encoders to always encode at least one silent + // frame (open source, someone code modify it), we will go ahead and + // tell our decoder about the lost frame. + if (_decoder) { + QByteArray decodedBuffer; + _decoder->lostFrame(decodedBuffer); + } + // calculate how many silent frames we should drop. int silentSamples = silentFrames * _numChannels; int samplesPerFrame = _ringBuffer.getNumFrameSamples(); From 60daaba52511e79eac826685df77893c50c1eb2d Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 8 Mar 2017 17:30:44 -0800 Subject: [PATCH 36/64] small cleanup --- libraries/audio-client/src/AudioNoiseGate.cpp | 2 -- libraries/audio-client/src/AudioNoiseGate.h | 3 --- 2 files changed, 5 deletions(-) diff --git a/libraries/audio-client/src/AudioNoiseGate.cpp b/libraries/audio-client/src/AudioNoiseGate.cpp index 9458f47d8c..ac45f98f0b 100644 --- a/libraries/audio-client/src/AudioNoiseGate.cpp +++ b/libraries/audio-client/src/AudioNoiseGate.cpp @@ -58,8 +58,6 @@ void AudioNoiseGate::removeDCOffset(int16_t* samples, int numSamples) { } } -#include - void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { // // Impose Noise Gate diff --git a/libraries/audio-client/src/AudioNoiseGate.h b/libraries/audio-client/src/AudioNoiseGate.h index 18b5a77056..774a4157bb 100644 --- a/libraries/audio-client/src/AudioNoiseGate.h +++ b/libraries/audio-client/src/AudioNoiseGate.h @@ -24,10 +24,7 @@ public: void removeDCOffset(int16_t* samples, int numSamples); bool clippedInLastFrame() const { return _didClipInLastFrame; } - bool closedInLastFrame() const { return _closedInLastFrame; } - - bool isOpen() const { return _isOpen; } float getMeasuredFloor() const { return _measuredFloor; } float getLastLoudness() const { return _lastLoudness; } From 7acfcf6750704b8b76523cf23d64f2291477df6a Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 8 Mar 2017 18:15:32 -0800 Subject: [PATCH 37/64] fade in and out when opening/closing the noise gate --- libraries/audio-client/src/AudioNoiseGate.cpp | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/libraries/audio-client/src/AudioNoiseGate.cpp b/libraries/audio-client/src/AudioNoiseGate.cpp index ac45f98f0b..0ced518097 100644 --- a/libraries/audio-client/src/AudioNoiseGate.cpp +++ b/libraries/audio-client/src/AudioNoiseGate.cpp @@ -142,7 +142,11 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { _sampleCounter = 0; } + + bool isOpeningGate = false; + if (samplesOverNoiseGate > NOISE_GATE_WIDTH) { + isOpeningGate = !_isOpen; _isOpen = true; _framesToClose = NOISE_GATE_CLOSE_FRAME_DELAY; } else { @@ -154,7 +158,23 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { } } if (!_isOpen) { - memset(samples, 0, numSamples * sizeof(int16_t)); + if (_closedInLastFrame) { + // would be nice to do a little crossfade from silence + for (int i = 0; i < numSamples; i++) { + float fadedSample = (1.0f - (float)i / (float)numSamples) * (float)samples[i]; + samples[i] = (int16_t)fadedSample; + } + } else { + memset(samples, 0, numSamples * sizeof(int16_t)); + } _lastLoudness = 0; } + + if (isOpeningGate) { + // would be nice to do a little crossfade from silence + for (int i = 0; i < numSamples; i++) { + float fadedSample = ((float)i / (float)numSamples) * (float)samples[i]; + samples[i] = (int16_t)fadedSample; + } + } } From ffe69acb5cb0ec4acc53a3a2ab4d5c25c2b633dd Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Wed, 8 Mar 2017 19:26:12 -0800 Subject: [PATCH 38/64] fix rare physics crash --- libraries/physics/src/PhysicsEngine.cpp | 27 +++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/libraries/physics/src/PhysicsEngine.cpp b/libraries/physics/src/PhysicsEngine.cpp index f57be4eab3..1073f4b1cf 100644 --- a/libraries/physics/src/PhysicsEngine.cpp +++ b/libraries/physics/src/PhysicsEngine.cpp @@ -143,12 +143,35 @@ void PhysicsEngine::addObjectToDynamicsWorld(ObjectMotionState* motionState) { } void PhysicsEngine::removeObjects(const VectorOfMotionStates& objects) { - // first bump and prune contacts for all objects in the list + // bump and prune contacts for all objects in the list for (auto object : objects) { bumpAndPruneContacts(object); } - // then remove them + if (_activeStaticBodies.size() > 0) { + // very unlikely to get here but its theoretically possible: + // immediately after a static entity was moved we skipped a simulation step and never got around to + // updating the static entity's RigidBody AABB. Now we're deleting bodies ==> we must scan + // _activeStaticBodies for bodies being removed and clear any that we find. + for (auto object : objects) { + btRigidBody* body = object->getRigidBody(); + + std::vector::reverse_iterator itr = _activeStaticBodies.rbegin(); + while (itr != _activeStaticBodies.rend()) { + if (body == *itr) { + if (*itr != *(_activeStaticBodies.rbegin())) { + // swap with rbegin + *itr = *(_activeStaticBodies.rbegin()); + } + _activeStaticBodies.pop_back(); + break; + } + ++itr; + } + } + } + + // remove bodies for (auto object : objects) { btRigidBody* body = object->getRigidBody(); if (body) { From 2d595c1f879ccc75709b164567b7e4c4c00c7010 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 8 Mar 2017 20:29:32 -0800 Subject: [PATCH 39/64] CR feedback --- libraries/audio-client/src/AudioNoiseGate.cpp | 7 ++--- libraries/audio/src/InboundAudioStream.cpp | 30 ++++++++++--------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/libraries/audio-client/src/AudioNoiseGate.cpp b/libraries/audio-client/src/AudioNoiseGate.cpp index ac45f98f0b..98ce8cc9e8 100644 --- a/libraries/audio-client/src/AudioNoiseGate.cpp +++ b/libraries/audio-client/src/AudioNoiseGate.cpp @@ -77,8 +77,6 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { // More means better rejection but also can reject continuous things like singing. // NUMBER_OF_NOISE_SAMPLE_FRAMES: How often should we re-evaluate the noise floor? - _closedInLastFrame = false; - float loudness = 0; int thisSample = 0; int samplesOverNoiseGate = 0; @@ -142,14 +140,13 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { _sampleCounter = 0; } + if (samplesOverNoiseGate > NOISE_GATE_WIDTH) { _isOpen = true; _framesToClose = NOISE_GATE_CLOSE_FRAME_DELAY; } else { if (--_framesToClose == 0) { - if (_isOpen) { - _closedInLastFrame = true; - } + _closedInLastFrame = !_isOpen; _isOpen = false; } } diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 12181cb8e2..3a5829e8fb 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -136,9 +136,9 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { break; } case SequenceNumberStats::Early: { - // Packet is early treat the packets as if all the packets between the last + // Packet is early. Treat the packets as if all the packets between the last // OnTime packet and this packet was lost. If we're using a codec this will - // also result in allowing the codec to flush its internal state. Then + // also result in allowing the codec to interpolate lost data. Then // fall through to the "on time" logic to actually handle this packet int packetsDropped = arrivalInfo._seqDiffFromExpected; lostAudioData(packetsDropped); @@ -150,8 +150,6 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { if (message.getType() == PacketType::SilentAudioFrame) { // If we recieved a SilentAudioFrame from our sender, we might want to drop // some of the samples in order to catch up to our desired jitter buffer size. - // NOTE: If we're using a codec we will be calling the codec's lostFrame() - // method to allow the codec to flush its internal state. writeDroppableSilentFrames(networkFrames); } else { // note: PCM and no codec are identical @@ -249,17 +247,21 @@ int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packet int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) { - // if we have a decoder, we still want to tell the decoder about our - // lost frame. this will flush the internal state of the decoder - // we can safely ignore the output of the codec in this case, because - // we've enforced that on the sending side, the encoder ran at least - // one frame of truly silent audio before we sent the "droppable" silent - // frame. Technically we could leave this out, if we know for certain - // that the sender has really sent us an encoded packet of zeros, but - // since we can trust all encoders to always encode at least one silent - // frame (open source, someone code modify it), we will go ahead and - // tell our decoder about the lost frame. + // We can't guarentee that all clients have faded the stream down + // to silence and encode that silence before sending us a + // SilentAudioFrame. The encoder may have truncated the stream and + // left the decoder holding some loud state. To handle this case + // we will call the decoder's lostFrame() method, which indicates + // that it should interpolate from it's last known state (which may + // be some unknown loud state) down toward silence. if (_decoder) { + // FIXME - We could potentially use the output from the codec, in which + // case we might get a cleaner fade toward silence. NOTE: The below logic + // attempts to catch up in the event that the jitter buffers have grown. + // The better long term fix is to use the output from the decode, detect + // when it actually reaches silence, and then delete the silent portions + // of the jitter buffers. Or petentially do a cross fade from the decode + // output to silence. QByteArray decodedBuffer; _decoder->lostFrame(decodedBuffer); } From 79d8d90581103152926775f5cdd312bc1e2d0ecd Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 8 Mar 2017 20:35:24 -0800 Subject: [PATCH 40/64] CR feedback --- libraries/audio/src/InboundAudioStream.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 3a5829e8fb..70884c9fd1 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -137,7 +137,7 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { } case SequenceNumberStats::Early: { // Packet is early. Treat the packets as if all the packets between the last - // OnTime packet and this packet was lost. If we're using a codec this will + // OnTime packet and this packet were lost. If we're using a codec this will // also result in allowing the codec to interpolate lost data. Then // fall through to the "on time" logic to actually handle this packet int packetsDropped = arrivalInfo._seqDiffFromExpected; @@ -161,9 +161,9 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { } else { qDebug(audio) << "Codec mismatch: expected" << _selectedCodecName << "got" << codecInPacket << "writing silence"; - // Since the data in the stream is using a codec that we're not prepared for, + // Since the data in the stream is using a codec that we aren't prepared for, // we need to let the codec know that we don't have data for it, this will - // flush any internal codec state and produce fade to silence. + // allowe the codec to interpolate missing data and produce a fade to silence. lostAudioData(1); // inform others of the mismatch @@ -248,12 +248,12 @@ int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packet int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) { // We can't guarentee that all clients have faded the stream down - // to silence and encode that silence before sending us a - // SilentAudioFrame. The encoder may have truncated the stream and - // left the decoder holding some loud state. To handle this case - // we will call the decoder's lostFrame() method, which indicates - // that it should interpolate from it's last known state (which may - // be some unknown loud state) down toward silence. + // to silence and encoding that silence before sending us a + // SilentAudioFrame. If the encoder has truncated the stream it will + // leave the decoder holding some unknown loud state. To handle this + // case we will call the decoder's lostFrame() method, which indicates + // that it should interpolate from its last known state down toward + // silence. if (_decoder) { // FIXME - We could potentially use the output from the codec, in which // case we might get a cleaner fade toward silence. NOTE: The below logic From bdab24d9bfc731b81db02ffdf409a7768881adb7 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 9 Mar 2017 08:25:56 -0800 Subject: [PATCH 41/64] fix spelling --- libraries/audio/src/InboundAudioStream.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 70884c9fd1..88ec7e7bc0 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -163,7 +163,7 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { // Since the data in the stream is using a codec that we aren't prepared for, // we need to let the codec know that we don't have data for it, this will - // allowe the codec to interpolate missing data and produce a fade to silence. + // allow the codec to interpolate missing data and produce a fade to silence. lostAudioData(1); // inform others of the mismatch @@ -248,7 +248,7 @@ int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packet int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) { // We can't guarentee that all clients have faded the stream down - // to silence and encoding that silence before sending us a + // to silence and encoded that silence before sending us a // SilentAudioFrame. If the encoder has truncated the stream it will // leave the decoder holding some unknown loud state. To handle this // case we will call the decoder's lostFrame() method, which indicates From e69d6d8b5f83cb1de3d9ca60312b99bb28bedb70 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 9 Mar 2017 14:20:57 -0500 Subject: [PATCH 42/64] track silent listeners in audio-mixer --- assignment-client/src/audio/AudioMixer.cpp | 1 + assignment-client/src/audio/AudioMixerSlave.cpp | 1 + assignment-client/src/audio/AudioMixerStats.cpp | 2 ++ assignment-client/src/audio/AudioMixerStats.h | 1 + 4 files changed, 5 insertions(+) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 2583e15760..b95c429b2d 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -241,6 +241,7 @@ void AudioMixer::sendStatsPacket() { statsObject["avg_streams_per_frame"] = (float)_stats.sumStreams / (float)_numStatFrames; statsObject["avg_listeners_per_frame"] = (float)_stats.sumListeners / (float)_numStatFrames; + statsObject["avg_listeners_(silent)_per_frame"] = (float)_stats.sumListenersSilent / (float)_numStatFrames; statsObject["silent_packets_per_frame"] = (float)_numSilentPackets / (float)_numStatFrames; diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 6b53de89c2..8edef785b8 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -106,6 +106,7 @@ void AudioMixerSlave::mix(const SharedNodePointer& node) { sendMixPacket(node, *data, encodedBuffer); } else { + ++stats.sumListenersSilent; sendSilentPacket(node, *data); } diff --git a/assignment-client/src/audio/AudioMixerStats.cpp b/assignment-client/src/audio/AudioMixerStats.cpp index a831210871..4cfdd55167 100644 --- a/assignment-client/src/audio/AudioMixerStats.cpp +++ b/assignment-client/src/audio/AudioMixerStats.cpp @@ -14,6 +14,7 @@ void AudioMixerStats::reset() { sumStreams = 0; sumListeners = 0; + sumListenersSilent = 0; totalMixes = 0; hrtfRenders = 0; hrtfSilentRenders = 0; @@ -28,6 +29,7 @@ void AudioMixerStats::reset() { void AudioMixerStats::accumulate(const AudioMixerStats& otherStats) { sumStreams += otherStats.sumStreams; sumListeners += otherStats.sumListeners; + sumListenersSilent += otherStats.sumListenersSilent; totalMixes += otherStats.totalMixes; hrtfRenders += otherStats.hrtfRenders; hrtfSilentRenders += otherStats.hrtfSilentRenders; diff --git a/assignment-client/src/audio/AudioMixerStats.h b/assignment-client/src/audio/AudioMixerStats.h index 77ac8b985d..f4ba9db769 100644 --- a/assignment-client/src/audio/AudioMixerStats.h +++ b/assignment-client/src/audio/AudioMixerStats.h @@ -19,6 +19,7 @@ struct AudioMixerStats { int sumStreams { 0 }; int sumListeners { 0 }; + int sumListenersSilent { 0 }; int totalMixes { 0 }; From 7e2f1a6455deb0263aced407154c3f88f7451316 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 8 Mar 2017 02:59:48 -0500 Subject: [PATCH 43/64] check for silent samples befor limiting --- assignment-client/src/audio/AudioMixerSlave.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 8edef785b8..d01d961e33 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -222,17 +222,19 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { stats.mixTime += mixTime.count(); #endif - // use the per listener AudioLimiter to render the mixed data... - listenerData->audioLimiter.render(_mixSamples, _bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - // check for silent audio after the peak limiter has converted the samples + // check for silent audio before limiting + // limiting uses a dither and can only guarantee abs(sample) <= 1 bool hasAudio = false; for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) { - if (_bufferSamples[i] != 0) { + if (_mixSamples[i] != 0.0f) { hasAudio = true; break; } } + + // use the per listener AudioLimiter to render the mixed data + listenerData->audioLimiter.render(_mixSamples, _bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + return hasAudio; } From c6c0c6d3829ca3a75ea8e63ed818471df98f8109 Mon Sep 17 00:00:00 2001 From: Dante Ruiz Date: Thu, 9 Mar 2017 22:54:22 +0000 Subject: [PATCH 44/64] toolbar becomes visible after taking a snapshot --- interface/src/scripting/HMDScriptingInterface.cpp | 4 ++++ interface/src/scripting/HMDScriptingInterface.h | 2 ++ scripts/system/snapshot.js | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/interface/src/scripting/HMDScriptingInterface.cpp b/interface/src/scripting/HMDScriptingInterface.cpp index 2bca793d80..65ac506a88 100644 --- a/interface/src/scripting/HMDScriptingInterface.cpp +++ b/interface/src/scripting/HMDScriptingInterface.cpp @@ -81,6 +81,10 @@ void HMDScriptingInterface::closeTablet() { _showTablet = false; } +void HMDScriptingInterface::openTablet() { + _showTablet = true; +} + QScriptValue HMDScriptingInterface::getHUDLookAtPosition2D(QScriptContext* context, QScriptEngine* engine) { glm::vec3 hudIntersection; auto instance = DependencyManager::get(); diff --git a/interface/src/scripting/HMDScriptingInterface.h b/interface/src/scripting/HMDScriptingInterface.h index d895d5da4c..276e23d2d5 100644 --- a/interface/src/scripting/HMDScriptingInterface.h +++ b/interface/src/scripting/HMDScriptingInterface.h @@ -76,6 +76,8 @@ public: Q_INVOKABLE void closeTablet(); + Q_INVOKABLE void openTablet(); + signals: bool shouldShowHandControllersChanged(); diff --git a/scripts/system/snapshot.js b/scripts/system/snapshot.js index 8f918c9cb2..b16aa9e765 100644 --- a/scripts/system/snapshot.js +++ b/scripts/system/snapshot.js @@ -191,12 +191,12 @@ function resetButtons(pathStillSnapshot, pathAnimatedSnapshot, notify) { if (clearOverlayWhenMoving) { MyAvatar.setClearOverlayWhenMoving(true); // not until after the share dialog } + HMD.openTablet(); } function processingGif() { // show hud Reticle.visible = reticleVisible; - button.clicked.disconnect(onClicked); buttonConnected = false; // show overlays if they were on From d54005b62ec5ba5d41d49d2490d454031cf6fb92 Mon Sep 17 00:00:00 2001 From: Dante Ruiz Date: Fri, 10 Mar 2017 00:17:44 +0000 Subject: [PATCH 45/64] no more duplicate snapshot buttons after taking reloading scripts --- scripts/system/snapshot.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/system/snapshot.js b/scripts/system/snapshot.js index 8f918c9cb2..55152ef245 100644 --- a/scripts/system/snapshot.js +++ b/scripts/system/snapshot.js @@ -211,8 +211,10 @@ Window.snapshotShared.connect(snapshotShared); Window.processingGif.connect(processingGif); Script.scriptEnding.connect(function () { - button.clicked.disconnect(onClicked); - buttonConnected = false; + if (buttonConnected) { + button.clicked.disconnect(onClicked); + buttonConnected = false; + } if (tablet) { tablet.removeButton(button); } From 2c141fd5552565a8c5e14fa6f2382b3080113c25 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Fri, 10 Mar 2017 09:18:29 -0800 Subject: [PATCH 46/64] more correct comment --- libraries/physics/src/PhysicsEngine.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/physics/src/PhysicsEngine.cpp b/libraries/physics/src/PhysicsEngine.cpp index 1073f4b1cf..363887de25 100644 --- a/libraries/physics/src/PhysicsEngine.cpp +++ b/libraries/physics/src/PhysicsEngine.cpp @@ -149,10 +149,10 @@ void PhysicsEngine::removeObjects(const VectorOfMotionStates& objects) { } if (_activeStaticBodies.size() > 0) { - // very unlikely to get here but its theoretically possible: - // immediately after a static entity was moved we skipped a simulation step and never got around to - // updating the static entity's RigidBody AABB. Now we're deleting bodies ==> we must scan - // _activeStaticBodies for bodies being removed and clear any that we find. + // _activeStaticBodies was not cleared last frame. + // The only way to get here is if a static object were moved but we did not actually step the simulation last + // frame (because the framerate is faster than our physics simulation rate). When this happens we must scan + // _activeStaticBodies for objects that were recently deleted so we don't try to access a dangling pointer. for (auto object : objects) { btRigidBody* body = object->getRigidBody(); From 69fdea36216e0f957fb3812dd0ee84046243ce86 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 10 Mar 2017 11:44:53 -0800 Subject: [PATCH 47/64] fix typo in comment --- libraries/audio-client/src/AudioNoiseGate.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio-client/src/AudioNoiseGate.cpp b/libraries/audio-client/src/AudioNoiseGate.cpp index 45a70c4a28..5224849c5b 100644 --- a/libraries/audio-client/src/AudioNoiseGate.cpp +++ b/libraries/audio-client/src/AudioNoiseGate.cpp @@ -155,7 +155,7 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { } if (!_isOpen) { if (_closedInLastFrame) { - // would be nice to do a little crossfade from silence + // would be nice to do a little crossfade to silence for (int i = 0; i < numSamples; i++) { float fadedSample = (1.0f - (float)i / (float)numSamples) * (float)samples[i]; samples[i] = (int16_t)fadedSample; From eaea718de1994149d5a2bb774bc60f88bbc98cc7 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 10 Mar 2017 17:08:19 -0800 Subject: [PATCH 48/64] add various scriping interfaces for managing the audio scope and detecting noise gate transitions --- interface/src/Application.cpp | 8 ++++++++ interface/src/audio/AudioScope.cpp | 14 ++++++++------ interface/src/audio/AudioScope.h | 9 +++++++-- libraries/audio-client/src/AudioClient.cpp | 8 ++++++++ libraries/audio-client/src/AudioClient.h | 2 ++ libraries/audio-client/src/AudioNoiseGate.cpp | 9 +++++---- libraries/audio-client/src/AudioNoiseGate.h | 2 ++ .../script-engine/src/AudioScriptingInterface.h | 11 +++++++---- 8 files changed, 47 insertions(+), 16 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index f870bd9f83..7e3f61e0ec 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1182,6 +1182,10 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo // set the local loopback interface for local sounds AudioInjector::setLocalAudioInterface(audioIO.data()); AudioScriptingInterface::getInstance().setLocalAudioInterface(audioIO.data()); + connect(audioIO.data(), &AudioClient::noiseGateOpened, &AudioScriptingInterface::getInstance(), &AudioScriptingInterface::noiseGateOpened); + connect(audioIO.data(), &AudioClient::noiseGateClosed, &AudioScriptingInterface::getInstance(), &AudioScriptingInterface::noiseGateClosed); + connect(audioIO.data(), &AudioClient::inputReceived, &AudioScriptingInterface::getInstance(), &AudioScriptingInterface::inputReceived); + this->installEventFilter(this); @@ -1947,6 +1951,9 @@ void Application::initializeUi() { rootContext->setContextProperty("ApplicationInterface", this); rootContext->setContextProperty("Audio", &AudioScriptingInterface::getInstance()); rootContext->setContextProperty("AudioStats", DependencyManager::get()->getStats().data()); + //rootContext->setContextProperty("AudioScope", DependencyManager::get.data()); + + rootContext->setContextProperty("Controller", DependencyManager::get().data()); rootContext->setContextProperty("Entities", DependencyManager::get().data()); _fileDownload = new FileScriptingInterface(engine); @@ -5521,6 +5528,7 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEngine* scri scriptEngine->registerGlobalObject("Settings", SettingsScriptingInterface::getInstance()); scriptEngine->registerGlobalObject("AudioDevice", AudioDeviceScriptingInterface::getInstance()); scriptEngine->registerGlobalObject("AudioStats", DependencyManager::get()->getStats().data()); + scriptEngine->registerGlobalObject("AudioScope", DependencyManager::get().data()); // Caches scriptEngine->registerGlobalObject("AnimationCache", DependencyManager::get().data()); diff --git a/interface/src/audio/AudioScope.cpp b/interface/src/audio/AudioScope.cpp index 346fbd11f4..cf9984e32b 100644 --- a/interface/src/audio/AudioScope.cpp +++ b/interface/src/audio/AudioScope.cpp @@ -52,12 +52,14 @@ AudioScope::AudioScope() : connect(audioIO.data(), &AudioClient::inputReceived, this, &AudioScope::addInputToScope); } -void AudioScope::toggle() { - _isEnabled = !_isEnabled; - if (_isEnabled) { - allocateScope(); - } else { - freeScope(); +void AudioScope::setVisible(bool visible) { + if (_isEnabled != visible) { + _isEnabled = visible; + if (_isEnabled) { + allocateScope(); + } else { + freeScope(); + } } } diff --git a/interface/src/audio/AudioScope.h b/interface/src/audio/AudioScope.h index 0b716d7666..615bdaf17f 100644 --- a/interface/src/audio/AudioScope.h +++ b/interface/src/audio/AudioScope.h @@ -34,8 +34,14 @@ public: void render(RenderArgs* renderArgs, int width, int height); public slots: - void toggle(); + void toggle() { setVisible(!_isEnabled); } + void setVisible(bool visible); + bool getVisible() const { return _isEnabled; } + void togglePause() { _isPaused = !_isPaused; } + void setPause(bool paused) { _isPaused = paused; } + bool getPause() { return _isPaused; } + void selectAudioScopeFiveFrames(); void selectAudioScopeTwentyFrames(); void selectAudioScopeFiftyFrames(); @@ -74,7 +80,6 @@ private: int _inputID; int _outputLeftID; int _outputRightD; - }; #endif // hifi_AudioScope_h diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 9cd87d2e70..6d135dc571 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1024,6 +1024,7 @@ void AudioClient::handleAudioInput() { if (_inputGate.clippedInLastFrame()) { _timeSinceLastClip = 0.0f; } + } else { float loudness = 0.0f; @@ -1041,6 +1042,13 @@ void AudioClient::handleAudioInput() { emit inputReceived({ reinterpret_cast(networkAudioSamples), numNetworkBytes }); + if (_inputGate.openedInLastFrame()) { + emit noiseGateOpened(); + } + if (_inputGate.closedInLastFrame()) { + emit noiseGateClosed(); + } + } else { // our input loudness is 0, since we're muted _lastInputLoudness = 0; diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 5619051eaf..57b19acceb 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -218,6 +218,8 @@ signals: void inputReceived(const QByteArray& inputSamples); void outputBytesToNetwork(int numBytes); void inputBytesFromNetwork(int numBytes); + void noiseGateOpened(); + void noiseGateClosed(); void changeDevice(const QAudioDeviceInfo& outputDeviceInfo); void deviceChanged(); diff --git a/libraries/audio-client/src/AudioNoiseGate.cpp b/libraries/audio-client/src/AudioNoiseGate.cpp index 5224849c5b..17bbd1a509 100644 --- a/libraries/audio-client/src/AudioNoiseGate.cpp +++ b/libraries/audio-client/src/AudioNoiseGate.cpp @@ -141,15 +141,16 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { } - bool isOpeningGate = false; + _closedInLastFrame = false; + _openedInLastFrame = false; if (samplesOverNoiseGate > NOISE_GATE_WIDTH) { - isOpeningGate = !_isOpen; + _openedInLastFrame = !_isOpen; _isOpen = true; _framesToClose = NOISE_GATE_CLOSE_FRAME_DELAY; } else { if (--_framesToClose == 0) { - _closedInLastFrame = !_isOpen; + _closedInLastFrame = _isOpen; _isOpen = false; } } @@ -166,7 +167,7 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { _lastLoudness = 0; } - if (isOpeningGate) { + if (_openedInLastFrame) { // would be nice to do a little crossfade from silence for (int i = 0; i < numSamples; i++) { float fadedSample = ((float)i / (float)numSamples) * (float)samples[i]; diff --git a/libraries/audio-client/src/AudioNoiseGate.h b/libraries/audio-client/src/AudioNoiseGate.h index 774a4157bb..ba1ab8599c 100644 --- a/libraries/audio-client/src/AudioNoiseGate.h +++ b/libraries/audio-client/src/AudioNoiseGate.h @@ -25,6 +25,7 @@ public: bool clippedInLastFrame() const { return _didClipInLastFrame; } bool closedInLastFrame() const { return _closedInLastFrame; } + bool openedInLastFrame() const { return _openedInLastFrame; } float getMeasuredFloor() const { return _measuredFloor; } float getLastLoudness() const { return _lastLoudness; } @@ -42,6 +43,7 @@ private: int _sampleCounter; bool _isOpen; bool _closedInLastFrame { false }; + bool _openedInLastFrame { false }; int _framesToClose; }; diff --git a/libraries/script-engine/src/AudioScriptingInterface.h b/libraries/script-engine/src/AudioScriptingInterface.h index 07a6b171f4..6cce78d48f 100644 --- a/libraries/script-engine/src/AudioScriptingInterface.h +++ b/libraries/script-engine/src/AudioScriptingInterface.h @@ -32,10 +32,13 @@ protected: Q_INVOKABLE void setStereoInput(bool stereo); signals: - void mutedByMixer(); - void environmentMuted(); - void receivedFirstPacket(); - void disconnected(); + void mutedByMixer(); /// the client has been muted by the mixer + void environmentMuted(); /// the entire environment has been muted by the mixer + void receivedFirstPacket(); /// the client has received its first packet from the audio mixer + void disconnected(); /// the client has been disconnected from the audio mixer + void noiseGateOpened(); /// the noise gate has opened + void noiseGateClosed(); /// the noise gate has closed + void inputReceived(const QByteArray& inputSamples); /// a frame of mic input audio has been received and processed private: AudioScriptingInterface(); From 5f3ee2e98f1285f64d392b07d8832caf5b9aa95b Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 10 Mar 2017 17:08:54 -0800 Subject: [PATCH 49/64] add a basic tablet app for the audio scope --- scripts/system/audioScope.js | 97 ++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 scripts/system/audioScope.js diff --git a/scripts/system/audioScope.js b/scripts/system/audioScope.js new file mode 100644 index 0000000000..0bcff7d5be --- /dev/null +++ b/scripts/system/audioScope.js @@ -0,0 +1,97 @@ +"use strict"; +// +// audioScope.js +// scripts/system/ +// +// Created by Brad Hefta-Gaub on 3/10/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +/* global Script, Tablet, AudioScope, Audio */ + +(function () { // BEGIN LOCAL_SCOPE + + var framesSinceOpened = 0; + var scopeVisibile = AudioScope.getVisible(); + var scopePaused = AudioScope.getPause(); + var autoPause = false; + + var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system"); + var showScopeButton = tablet.addButton({ + icon: "http://s3.amazonaws.com/hifi-public/tony/icons/star.png", + text: "Audio Scope", + isActive: scopeVisibile + }); + + var pauseScopeButton = tablet.addButton({ + icon: "http://s3.amazonaws.com/hifi-public/tony/icons/star.png", + text: "Pause", + isActive: scopePaused + }); + + var autoPauseScopeButton = tablet.addButton({ + icon: "http://s3.amazonaws.com/hifi-public/tony/icons/star.png", + text: "Auto Pause", + isActive: autoPause + }); + + function setScopePause(paused) { + scopePaused = paused; + pauseScopeButton.editProperties({ + isActive: scopePaused, + text: scopePaused ? "Unpause" : "Pause" + }); + AudioScope.setPause(scopePaused); + } + + showScopeButton.clicked.connect(function () { + // toggle button active state + scopeVisibile = !scopeVisibile; + showScopeButton.editProperties({ + isActive: scopeVisibile + }); + + AudioScope.setVisible(scopeVisibile); + }); + + pauseScopeButton.clicked.connect(function () { + // toggle button active state + setScopePause(!scopePaused); + }); + + autoPauseScopeButton.clicked.connect(function () { + // toggle button active state + autoPause = !autoPause; + autoPauseScopeButton.editProperties({ + isActive: autoPause, + text: autoPause ? "Auto Pause" : "Manual" + }); + }); + + Script.scriptEnding.connect(function () { + tablet.removeButton(showScopeButton); + tablet.removeButton(pauseScopeButton); + tablet.removeButton(autoPauseScopeButton); + }); + + Audio.noiseGateOpened.connect(function(){ + framesSinceOpened = 0; + }); + + Audio.noiseGateClosed.connect(function(){ + // noise gate closed + }); + + Audio.inputReceived.connect(function(){ + if (autoPause && AudioScope.getVisible()) { + framesSinceOpened++; + if (framesSinceOpened > 50) { + setScopePause(true); + } + } + }); + + +}()); // END LOCAL_SCOPE \ No newline at end of file From b8a2525f9e04a7c8bec8c56463f089d15375daa9 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 10 Mar 2017 17:38:00 -0800 Subject: [PATCH 50/64] add auto pause/resume to the Audio Scope App --- scripts/system/audioScope.js | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/scripts/system/audioScope.js b/scripts/system/audioScope.js index 0bcff7d5be..85060173d4 100644 --- a/scripts/system/audioScope.js +++ b/scripts/system/audioScope.js @@ -13,7 +13,6 @@ (function () { // BEGIN LOCAL_SCOPE - var framesSinceOpened = 0; var scopeVisibile = AudioScope.getVisible(); var scopePaused = AudioScope.getPause(); var autoPause = false; @@ -77,21 +76,16 @@ }); Audio.noiseGateOpened.connect(function(){ - framesSinceOpened = 0; + if (autoPause) { + setScopePause(false); + } }); Audio.noiseGateClosed.connect(function(){ // noise gate closed - }); - - Audio.inputReceived.connect(function(){ - if (autoPause && AudioScope.getVisible()) { - framesSinceOpened++; - if (framesSinceOpened > 50) { - setScopePause(true); - } + if (autoPause) { + setScopePause(true); } }); - }()); // END LOCAL_SCOPE \ No newline at end of file From 7eb4b8f5f73f8c39c5cdc5219d96c2555a196c25 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 10 Mar 2017 17:38:13 -0800 Subject: [PATCH 51/64] expose AudioScope to QML --- interface/src/Application.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 7e3f61e0ec..645c6ab15f 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1951,7 +1951,7 @@ void Application::initializeUi() { rootContext->setContextProperty("ApplicationInterface", this); rootContext->setContextProperty("Audio", &AudioScriptingInterface::getInstance()); rootContext->setContextProperty("AudioStats", DependencyManager::get()->getStats().data()); - //rootContext->setContextProperty("AudioScope", DependencyManager::get.data()); + rootContext->setContextProperty("AudioScope", DependencyManager::get().data()); rootContext->setContextProperty("Controller", DependencyManager::get().data()); From 23e47f2fdc8de48290a0a7dd8f0847213b2e121c Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 10 Mar 2017 18:13:35 -0800 Subject: [PATCH 52/64] added icons to the audioScope app --- .../icons/tablet-icons/scope-auto.svg | 46 +++++++++++++++++++ .../icons/tablet-icons/scope-pause.svg | 30 ++++++++++++ .../icons/tablet-icons/scope-play.svg | 30 ++++++++++++ scripts/system/audioScope.js | 12 +++-- 4 files changed, 114 insertions(+), 4 deletions(-) create mode 100644 interface/resources/icons/tablet-icons/scope-auto.svg create mode 100644 interface/resources/icons/tablet-icons/scope-pause.svg create mode 100644 interface/resources/icons/tablet-icons/scope-play.svg diff --git a/interface/resources/icons/tablet-icons/scope-auto.svg b/interface/resources/icons/tablet-icons/scope-auto.svg new file mode 100644 index 0000000000..85ef3f0e38 --- /dev/null +++ b/interface/resources/icons/tablet-icons/scope-auto.svg @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/interface/resources/icons/tablet-icons/scope-pause.svg b/interface/resources/icons/tablet-icons/scope-pause.svg new file mode 100644 index 0000000000..3fe74fcc9f --- /dev/null +++ b/interface/resources/icons/tablet-icons/scope-pause.svg @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + diff --git a/interface/resources/icons/tablet-icons/scope-play.svg b/interface/resources/icons/tablet-icons/scope-play.svg new file mode 100644 index 0000000000..56d90ef38a --- /dev/null +++ b/interface/resources/icons/tablet-icons/scope-play.svg @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + diff --git a/scripts/system/audioScope.js b/scripts/system/audioScope.js index 85060173d4..81d8e8fbd4 100644 --- a/scripts/system/audioScope.js +++ b/scripts/system/audioScope.js @@ -19,19 +19,22 @@ var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system"); var showScopeButton = tablet.addButton({ - icon: "http://s3.amazonaws.com/hifi-public/tony/icons/star.png", + icon: "icons/tablet-icons/scope.svg", text: "Audio Scope", isActive: scopeVisibile }); + var scopePauseImage = "icons/tablet-icons/scope-pause.svg"; + var scopePlayImage = "icons/tablet-icons/scope-play.svg"; + var pauseScopeButton = tablet.addButton({ - icon: "http://s3.amazonaws.com/hifi-public/tony/icons/star.png", - text: "Pause", + icon: scopePaused ? scopePlayImage : scopePauseImage, + text: scopePaused ? "Unpause" : "Pause", isActive: scopePaused }); var autoPauseScopeButton = tablet.addButton({ - icon: "http://s3.amazonaws.com/hifi-public/tony/icons/star.png", + icon: "icons/tablet-icons/scope-auto.svg", text: "Auto Pause", isActive: autoPause }); @@ -40,6 +43,7 @@ scopePaused = paused; pauseScopeButton.editProperties({ isActive: scopePaused, + icon: scopePaused ? scopePlayImage : scopePauseImage, text: scopePaused ? "Unpause" : "Pause" }); AudioScope.setPause(scopePaused); From aa65f03509f9a56f5e060c1e95025283b78b377f Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 10 Mar 2017 18:14:00 -0800 Subject: [PATCH 53/64] fix mac warning and remove accidentally added whitespace --- assignment-client/src/avatars/AvatarMixer.cpp | 1 - interface/src/Application.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index c10a616818..d2bfdde7ea 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -37,7 +37,6 @@ const QString AVATAR_MIXER_LOGGING_NAME = "avatar-mixer"; // FIXME - what we'd actually like to do is send to users at ~50% of their present rate down to 30hz. Assume 90 for now. const int AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND = 45; -const unsigned int AVATAR_DATA_SEND_INTERVAL_MSECS = (1.0f / (float) AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND) * 1000; AvatarMixer::AvatarMixer(ReceivedMessage& message) : ThreadedAssignment(message) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 645c6ab15f..60d9e815f6 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1953,7 +1953,6 @@ void Application::initializeUi() { rootContext->setContextProperty("AudioStats", DependencyManager::get()->getStats().data()); rootContext->setContextProperty("AudioScope", DependencyManager::get().data()); - rootContext->setContextProperty("Controller", DependencyManager::get().data()); rootContext->setContextProperty("Entities", DependencyManager::get().data()); _fileDownload = new FileScriptingInterface(engine); From 6ce50269ac62278f13d1d4253b0427b575234766 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 10 Mar 2017 18:42:18 -0800 Subject: [PATCH 54/64] add audio mixer in/out stats to the client stats --- interface/resources/qml/Stats.qml | 10 ++++++++++ interface/src/ui/Stats.cpp | 6 ++++++ interface/src/ui/Stats.h | 10 ++++++++++ 3 files changed, 26 insertions(+) diff --git a/interface/resources/qml/Stats.qml b/interface/resources/qml/Stats.qml index 58d589b667..ec170dfd08 100644 --- a/interface/resources/qml/Stats.qml +++ b/interface/resources/qml/Stats.qml @@ -181,6 +181,16 @@ Item { root.avatarMixerOutPps + "pps, " + root.myAvatarSendRate.toFixed(2) + "hz"; } + StatText { + visible: root.expanded; + text: "Audio Mixer In: " + root.audioMixerInKbps + " kbps, " + + root.audioMixerInPps + "pps"; + } + StatText { + visible: root.expanded; + text: "Audio Mixer Out: " + root.audioMixerOutKbps + " kbps, " + + root.audioMixerOutPps + "pps"; + } StatText { visible: root.expanded; text: "Downloads: " + root.downloads + "/" + root.downloadLimit + diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 1075bbdaa4..038dcd42cc 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -214,6 +214,12 @@ void Stats::updateStats(bool force) { STAT_UPDATE(audioMixerPps, roundf( bandwidthRecorder->getAverageInputPacketsPerSecond(NodeType::AudioMixer) + bandwidthRecorder->getAverageOutputPacketsPerSecond(NodeType::AudioMixer))); + + STAT_UPDATE(audioMixerInKbps, roundf(bandwidthRecorder->getAverageInputKilobitsPerSecond(NodeType::AudioMixer))); + STAT_UPDATE(audioMixerInPps, roundf(bandwidthRecorder->getAverageInputPacketsPerSecond(NodeType::AudioMixer))); + STAT_UPDATE(audioMixerOutKbps, roundf(bandwidthRecorder->getAverageOutputKilobitsPerSecond(NodeType::AudioMixer))); + STAT_UPDATE(audioMixerOutPps, roundf(bandwidthRecorder->getAverageOutputPacketsPerSecond(NodeType::AudioMixer))); + } else { STAT_UPDATE(audioMixerKbps, -1); STAT_UPDATE(audioMixerPps, -1); diff --git a/interface/src/ui/Stats.h b/interface/src/ui/Stats.h index 6be084100c..96c5d374d3 100644 --- a/interface/src/ui/Stats.h +++ b/interface/src/ui/Stats.h @@ -70,6 +70,12 @@ class Stats : public QQuickItem { STATS_PROPERTY(int, avatarMixerOutKbps, 0) STATS_PROPERTY(int, avatarMixerOutPps, 0) STATS_PROPERTY(float, myAvatarSendRate, 0) + + STATS_PROPERTY(int, audioMixerInKbps, 0) + STATS_PROPERTY(int, audioMixerInPps, 0) + STATS_PROPERTY(int, audioMixerOutKbps, 0) + STATS_PROPERTY(int, audioMixerOutPps, 0) + STATS_PROPERTY(int, audioMixerKbps, 0) STATS_PROPERTY(int, audioMixerPps, 0) STATS_PROPERTY(int, downloads, 0) @@ -180,6 +186,10 @@ signals: void avatarMixerOutKbpsChanged(); void avatarMixerOutPpsChanged(); void myAvatarSendRateChanged(); + void audioMixerInKbpsChanged(); + void audioMixerInPpsChanged(); + void audioMixerOutKbpsChanged(); + void audioMixerOutPpsChanged(); void audioMixerKbpsChanged(); void audioMixerPpsChanged(); void downloadsChanged(); From a38965d628b09dfe179c1dccef4c56059a8e37fb Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Sat, 11 Mar 2017 09:50:30 -0800 Subject: [PATCH 55/64] add more audio stats --- interface/resources/qml/Stats.qml | 15 ++++++++++ interface/src/ui/Stats.cpp | 21 ++++++++++++-- interface/src/ui/Stats.h | 15 +++++++++- libraries/audio-client/src/AudioClient.cpp | 14 ++++++++++ libraries/audio-client/src/AudioClient.h | 14 ++++++++++ libraries/audio-client/src/AudioNoiseGate.h | 1 + libraries/shared/src/shared/RateCounter.h | 31 ++++++++++++--------- 7 files changed, 94 insertions(+), 17 deletions(-) diff --git a/interface/resources/qml/Stats.qml b/interface/resources/qml/Stats.qml index ec170dfd08..564c74b526 100644 --- a/interface/resources/qml/Stats.qml +++ b/interface/resources/qml/Stats.qml @@ -186,11 +186,26 @@ Item { text: "Audio Mixer In: " + root.audioMixerInKbps + " kbps, " + root.audioMixerInPps + "pps"; } + StatText { + visible: root.expanded; + text: "Audio In Audio: " + root.audioAudioInboundPPS + " pps, " + + "Silent: " + root.audioSilentInboundPPS + " pps"; + } StatText { visible: root.expanded; text: "Audio Mixer Out: " + root.audioMixerOutKbps + " kbps, " + root.audioMixerOutPps + "pps"; } + StatText { + visible: root.expanded; + text: "Audio Out Mic: " + root.audioMicOutboundPPS + " pps, " + + "Silent: " + root.audioSilentOutboundPPS + " pps"; + } + StatText { + visible: root.expanded; + text: "Audio Codec: " + root.audioCodec + " Noise Gate: " + + root.audioNoiseGate; + } StatText { visible: root.expanded; text: "Downloads: " + root.downloads + "/" + root.downloadLimit + diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 038dcd42cc..923d9f642d 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -198,15 +198,16 @@ void Stats::updateStats(bool force) { STAT_UPDATE(avatarMixerInPps, roundf(bandwidthRecorder->getAverageInputPacketsPerSecond(NodeType::AvatarMixer))); STAT_UPDATE(avatarMixerOutKbps, roundf(bandwidthRecorder->getAverageOutputKilobitsPerSecond(NodeType::AvatarMixer))); STAT_UPDATE(avatarMixerOutPps, roundf(bandwidthRecorder->getAverageOutputPacketsPerSecond(NodeType::AvatarMixer))); - STAT_UPDATE(myAvatarSendRate, avatarManager->getMyAvatarSendRate()); } else { STAT_UPDATE(avatarMixerInKbps, -1); STAT_UPDATE(avatarMixerInPps, -1); STAT_UPDATE(avatarMixerOutKbps, -1); STAT_UPDATE(avatarMixerOutPps, -1); - STAT_UPDATE(myAvatarSendRate, avatarManager->getMyAvatarSendRate()); } + STAT_UPDATE(myAvatarSendRate, avatarManager->getMyAvatarSendRate()); + SharedNodePointer audioMixerNode = nodeList->soloNodeOfType(NodeType::AudioMixer); + auto audioClient = DependencyManager::get(); if (audioMixerNode || force) { STAT_UPDATE(audioMixerKbps, roundf( bandwidthRecorder->getAverageInputKilobitsPerSecond(NodeType::AudioMixer) + @@ -219,11 +220,25 @@ void Stats::updateStats(bool force) { STAT_UPDATE(audioMixerInPps, roundf(bandwidthRecorder->getAverageInputPacketsPerSecond(NodeType::AudioMixer))); STAT_UPDATE(audioMixerOutKbps, roundf(bandwidthRecorder->getAverageOutputKilobitsPerSecond(NodeType::AudioMixer))); STAT_UPDATE(audioMixerOutPps, roundf(bandwidthRecorder->getAverageOutputPacketsPerSecond(NodeType::AudioMixer))); - + STAT_UPDATE(audioMicOutboundPPS, audioClient->getMicAudioOutboundPPS()); + STAT_UPDATE(audioSilentOutboundPPS, audioClient->getSilentOutboundPPS()); + STAT_UPDATE(audioAudioInboundPPS, audioClient->getAudioInboundPPS()); + STAT_UPDATE(audioSilentInboundPPS, audioClient->getSilentInboundPPS()); } else { STAT_UPDATE(audioMixerKbps, -1); STAT_UPDATE(audioMixerPps, -1); + STAT_UPDATE(audioMixerInKbps, -1); + STAT_UPDATE(audioMixerInPps, -1); + STAT_UPDATE(audioMixerOutKbps, -1); + STAT_UPDATE(audioMixerOutPps, -1); + STAT_UPDATE(audioMicOutboundPPS, -1); + STAT_UPDATE(audioSilentOutboundPPS, -1); + STAT_UPDATE(audioAudioInboundPPS, -1); + STAT_UPDATE(audioSilentInboundPPS, -1); } + STAT_UPDATE(audioCodec, audioClient->getSelectedAudioFormat()); + STAT_UPDATE(audioNoiseGate, audioClient->getNoiseGateOpen() ? "Open" : "Closed"); + auto loadingRequests = ResourceCache::getLoadingRequests(); STAT_UPDATE(downloads, loadingRequests.size()); diff --git a/interface/src/ui/Stats.h b/interface/src/ui/Stats.h index 96c5d374d3..0ce113e0a0 100644 --- a/interface/src/ui/Stats.h +++ b/interface/src/ui/Stats.h @@ -75,9 +75,15 @@ class Stats : public QQuickItem { STATS_PROPERTY(int, audioMixerInPps, 0) STATS_PROPERTY(int, audioMixerOutKbps, 0) STATS_PROPERTY(int, audioMixerOutPps, 0) - STATS_PROPERTY(int, audioMixerKbps, 0) STATS_PROPERTY(int, audioMixerPps, 0) + STATS_PROPERTY(int, audioMicOutboundPPS, 0) + STATS_PROPERTY(int, audioSilentOutboundPPS, 0) + STATS_PROPERTY(int, audioAudioInboundPPS, 0) + STATS_PROPERTY(int, audioSilentInboundPPS, 0) + STATS_PROPERTY(QString, audioCodec, QString()) + STATS_PROPERTY(QString, audioNoiseGate, QString()) + STATS_PROPERTY(int, downloads, 0) STATS_PROPERTY(int, downloadLimit, 0) STATS_PROPERTY(int, downloadsPending, 0) @@ -192,6 +198,13 @@ signals: void audioMixerOutPpsChanged(); void audioMixerKbpsChanged(); void audioMixerPpsChanged(); + void audioMicOutboundPPSChanged(); + void audioSilentOutboundPPSChanged(); + void audioAudioInboundPPSChanged(); + void audioSilentInboundPPSChanged(); + void audioCodecChanged(); + void audioNoiseGateChanged(); + void downloadsChanged(); void downloadLimitChanged(); void downloadsPendingChanged(); diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 6d135dc571..9fb8bae6ca 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -608,6 +608,13 @@ void AudioClient::handleAudioEnvironmentDataPacket(QSharedPointer message) { + + if (message->getType() == PacketType::SilentAudioFrame) { + _silentInbound.increment(); + } else { + _audioInbound.increment(); + } + auto nodeList = DependencyManager::get(); nodeList->flagTimeForConnectionStep(LimitedNodeList::ConnectionStep::ReceiveFirstAudioPacket); @@ -1067,7 +1074,11 @@ void AudioClient::handleAudioInput() { // have _lastInputLoudness of 0 in our NEXT frame, we will send a silent packet if (_lastInputLoudness == 0 && !_inputGate.closedInLastFrame()) { packetType = PacketType::SilentAudioFrame; + _silentOutbound.increment(); + } else { + _micAudioOutbound.increment(); } + Transform audioTransform; audioTransform.setTranslation(_positionGetter()); audioTransform.setRotation(_orientationGetter()); @@ -1092,6 +1103,7 @@ void AudioClient::handleAudioInput() { } } +// FIXME - should this go through the noise gate and honor mute and echo? void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { Transform audioTransform; audioTransform.setTranslation(_positionGetter()); @@ -1104,6 +1116,8 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { encodedBuffer = audio; } + _micAudioOutbound.increment(); + // FIXME check a flag to see if we should echo audio? emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, avatarBoundingBoxCorner, avatarBoundingBoxScale, diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 57b19acceb..512b4bb3c1 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -46,6 +46,8 @@ #include #include +#include + #include #include "AudioIOStats.h" @@ -121,6 +123,13 @@ public: void negotiateAudioFormat(); void selectAudioFormat(const QString& selectedCodecName); + Q_INVOKABLE QString getSelectedAudioFormat() const { return _selectedCodecName; } + Q_INVOKABLE bool getNoiseGateOpen() const { return _inputGate.isOpen(); } + Q_INVOKABLE float getSilentOutboundPPS() const { return _silentOutbound.rate(); } + Q_INVOKABLE float getMicAudioOutboundPPS() const { return _micAudioOutbound.rate(); } + Q_INVOKABLE float getSilentInboundPPS() const { return _silentInbound.rate(); } + Q_INVOKABLE float getAudioInboundPPS() const { return _audioInbound.rate(); } + const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; } MixedProcessedAudioStream& getReceivedAudioStream() { return _receivedAudioStream; } @@ -384,6 +393,11 @@ private: Encoder* _encoder { nullptr }; // for outbound mic stream QThread* _checkDevicesThread { nullptr }; + + RateCounter<> _silentOutbound; + RateCounter<> _micAudioOutbound; + RateCounter<> _silentInbound; + RateCounter<> _audioInbound; }; diff --git a/libraries/audio-client/src/AudioNoiseGate.h b/libraries/audio-client/src/AudioNoiseGate.h index ba1ab8599c..fb31561994 100644 --- a/libraries/audio-client/src/AudioNoiseGate.h +++ b/libraries/audio-client/src/AudioNoiseGate.h @@ -26,6 +26,7 @@ public: bool clippedInLastFrame() const { return _didClipInLastFrame; } bool closedInLastFrame() const { return _closedInLastFrame; } bool openedInLastFrame() const { return _openedInLastFrame; } + bool isOpen() const { return _isOpen; } float getMeasuredFloor() const { return _measuredFloor; } float getLastLoudness() const { return _lastLoudness; } diff --git a/libraries/shared/src/shared/RateCounter.h b/libraries/shared/src/shared/RateCounter.h index d04d87493a..3cf509b6bf 100644 --- a/libraries/shared/src/shared/RateCounter.h +++ b/libraries/shared/src/shared/RateCounter.h @@ -24,29 +24,34 @@ public: RateCounter() { _rate = 0; } // avoid use of std::atomic copy ctor void increment(size_t count = 1) { - auto now = usecTimestampNow(); - float currentIntervalMs = (now - _start) / (float) USECS_PER_MSEC; - if (currentIntervalMs > (float) INTERVAL) { - float currentCount = _count; - float intervalSeconds = currentIntervalMs / (float) MSECS_PER_SECOND; - _rate = roundf(currentCount / intervalSeconds * _scale) / _scale; - _start = now; - _count = 0; - }; + checkRate(); _count += count; } - float rate() const { return _rate; } + float rate() const { checkRate(); return _rate; } uint8_t precision() const { return PRECISION; } uint32_t interval() const { return INTERVAL; } private: - uint64_t _start { usecTimestampNow() }; - size_t _count { 0 }; + mutable uint64_t _start { usecTimestampNow() }; + mutable size_t _count { 0 }; const float _scale { powf(10, PRECISION) }; - std::atomic _rate; + mutable std::atomic _rate; + + void checkRate() const { + auto now = usecTimestampNow(); + float currentIntervalMs = (now - _start) / (float)USECS_PER_MSEC; + if (currentIntervalMs > (float)INTERVAL) { + float currentCount = _count; + float intervalSeconds = currentIntervalMs / (float)MSECS_PER_SECOND; + _rate = roundf(currentCount / intervalSeconds * _scale) / _scale; + _start = now; + _count = 0; + }; + } + }; #endif From b67b17d3c01bbdfb957a8feba6506cbcea02b14b Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 13 Mar 2017 09:10:34 -0800 Subject: [PATCH 56/64] Fix JS exception messages formatting --- libraries/script-engine/src/ScriptEngine.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index 73a79f1bc6..d721d1c86f 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -142,7 +142,7 @@ QString encodeEntityIdIntoEntityUrl(const QString& url, const QString& entityID) QString ScriptEngine::logException(const QScriptValue& exception) { auto message = formatException(exception); - scriptErrorMessage(qPrintable(message)); + scriptErrorMessage(message); return message; } @@ -453,7 +453,7 @@ void ScriptEngine::loadURL(const QUrl& scriptURL, bool reload) { } void ScriptEngine::scriptErrorMessage(const QString& message) { - qCCritical(scriptengine) << message; + qCCritical(scriptengine) << qPrintable(message); emit errorMessage(message); } From 3e020ede78ba0584114a0c402f4d93af0ae0fc32 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 13 Mar 2017 11:58:43 -0700 Subject: [PATCH 57/64] CR feedback --- libraries/audio-client/src/AudioClient.cpp | 9 +- libraries/audio-client/src/AudioNoiseGate.cpp | 97 ++++++++++--------- libraries/audio-client/src/AudioNoiseGate.h | 24 ++--- 3 files changed, 68 insertions(+), 62 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 9fb8bae6ca..c32b5600d9 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1028,7 +1028,7 @@ void AudioClient::handleAudioInput() { // if we performed the noise gate we can get values from it instead of enumerating the samples again _lastInputLoudness = _inputGate.getLastLoudness(); - if (_inputGate.clippedInLastFrame()) { + if (_inputGate.clippedInLastBlock()) { _timeSinceLastClip = 0.0f; } @@ -1049,10 +1049,9 @@ void AudioClient::handleAudioInput() { emit inputReceived({ reinterpret_cast(networkAudioSamples), numNetworkBytes }); - if (_inputGate.openedInLastFrame()) { + if (_inputGate.openedInLastBlock()) { emit noiseGateOpened(); - } - if (_inputGate.closedInLastFrame()) { + } else if (_inputGate.closedInLastBlock()) { emit noiseGateClosed(); } @@ -1072,7 +1071,7 @@ void AudioClient::handleAudioInput() { // the output from the input gate (eventually, this could be crossfaded) // and allow the codec to properly encode down to silent/zero. If we still // have _lastInputLoudness of 0 in our NEXT frame, we will send a silent packet - if (_lastInputLoudness == 0 && !_inputGate.closedInLastFrame()) { + if (_lastInputLoudness == 0 && !_inputGate.closedInLastBlock()) { packetType = PacketType::SilentAudioFrame; _silentOutbound.increment(); } else { diff --git a/libraries/audio-client/src/AudioNoiseGate.cpp b/libraries/audio-client/src/AudioNoiseGate.cpp index 17bbd1a509..2acdea6f0c 100644 --- a/libraries/audio-client/src/AudioNoiseGate.cpp +++ b/libraries/audio-client/src/AudioNoiseGate.cpp @@ -19,16 +19,16 @@ const float AudioNoiseGate::CLIPPING_THRESHOLD = 0.90f; AudioNoiseGate::AudioNoiseGate() : - _inputFrameCounter(0), + _inputBlockCounter(0), _lastLoudness(0.0f), - _quietestFrame(std::numeric_limits::max()), - _loudestFrame(0.0f), - _didClipInLastFrame(false), + _quietestBlock(std::numeric_limits::max()), + _loudestBlock(0.0f), + _didClipInLastBlock(false), _dcOffset(0.0f), _measuredFloor(0.0f), _sampleCounter(0), _isOpen(false), - _framesToClose(0) + _buffersToClose(0) { } @@ -37,7 +37,7 @@ void AudioNoiseGate::removeDCOffset(int16_t* samples, int numSamples) { // // DC Offset correction // - // Measure the DC offset over a trailing number of frames, and remove it from the input signal. + // Measure the DC offset over a trailing number of buffers, and remove it from the input signal. // This causes the noise background measurements and server muting to be more accurate. Many off-board // ADC's have a noticeable DC offset. // @@ -51,7 +51,7 @@ void AudioNoiseGate::removeDCOffset(int16_t* samples, int numSamples) { // Update measured DC offset measuredDcOffset /= numSamples; if (_dcOffset == 0.0f) { - // On first frame, copy over measured offset + // On first buffer, copy over measured offset _dcOffset = measuredDcOffset; } else { _dcOffset = DC_OFFSET_AVERAGING * _dcOffset + (1.0f - DC_OFFSET_AVERAGING) * measuredDcOffset; @@ -69,13 +69,13 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { // // NOISE_GATE_HEIGHT: How loud you have to speak relative to noise background to open the gate. // Make this value lower for more sensitivity and less rejection of noise. - // NOISE_GATE_WIDTH: The number of samples in an audio frame for which the height must be exceeded + // NOISE_GATE_WIDTH: The number of samples in an audio buffer for which the height must be exceeded // to open the gate. - // NOISE_GATE_CLOSE_FRAME_DELAY: Once the noise is below the gate height for the frame, how many frames + // NOISE_GATE_CLOSE_BLOCK_DELAY: Once the noise is below the gate height for the buffer, how many buffers // will we wait before closing the gate. - // NOISE_GATE_FRAMES_TO_AVERAGE: How many audio frames should we average together to compute noise floor. + // NOISE_GATE_BLOCKSS_TO_AVERAGE: How many audio buffers should we average together to compute noise floor. // More means better rejection but also can reject continuous things like singing. - // NUMBER_OF_NOISE_SAMPLE_FRAMES: How often should we re-evaluate the noise floor? + // NUMBER_OF_NOISE_SAMPLE_BLOCKS: How often should we re-evaluate the noise floor? float loudness = 0; int thisSample = 0; @@ -83,16 +83,16 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { const float NOISE_GATE_HEIGHT = 7.0f; const int NOISE_GATE_WIDTH = 5; - const int NOISE_GATE_CLOSE_FRAME_DELAY = 5; - const int NOISE_GATE_FRAMES_TO_AVERAGE = 5; + const int NOISE_GATE_CLOSE_BLOCK_DELAY = 5; + const int NOISE_GATE_BLOCKS_TO_AVERAGE = 5; // Check clipping, and check if should open noise gate - _didClipInLastFrame = false; + _didClipInLastBlock = false; for (int i = 0; i < numSamples; i++) { thisSample = std::abs(samples[i]); if (thisSample >= ((float) AudioConstants::MAX_SAMPLE_VALUE * CLIPPING_THRESHOLD)) { - _didClipInLastFrame = true; + _didClipInLastBlock = true; } loudness += thisSample; @@ -104,61 +104,64 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { _lastLoudness = fabs(loudness / numSamples); - if (_quietestFrame > _lastLoudness) { - _quietestFrame = _lastLoudness; + if (_quietestBlock > _lastLoudness) { + _quietestBlock = _lastLoudness; } - if (_loudestFrame < _lastLoudness) { - _loudestFrame = _lastLoudness; + if (_loudestBlock < _lastLoudness) { + _loudestBlock = _lastLoudness; } const int FRAMES_FOR_NOISE_DETECTION = 400; - if (_inputFrameCounter++ > FRAMES_FOR_NOISE_DETECTION) { - _quietestFrame = std::numeric_limits::max(); - _loudestFrame = 0.0f; - _inputFrameCounter = 0; + if (_inputBlockCounter++ > FRAMES_FOR_NOISE_DETECTION) { + _quietestBlock = std::numeric_limits::max(); + _loudestBlock = 0.0f; + _inputBlockCounter = 0; } // If Noise Gate is enabled, check and turn the gate on and off - float averageOfAllSampleFrames = 0.0f; - _sampleFrames[_sampleCounter++] = _lastLoudness; - if (_sampleCounter == NUMBER_OF_NOISE_SAMPLE_FRAMES) { + float averageOfAllSampleBlocks = 0.0f; + _sampleBlocks[_sampleCounter++] = _lastLoudness; + if (_sampleCounter == NUMBER_OF_NOISE_SAMPLE_BLOCKS) { float smallestSample = std::numeric_limits::max(); - for (int i = 0; i <= NUMBER_OF_NOISE_SAMPLE_FRAMES - NOISE_GATE_FRAMES_TO_AVERAGE; i += NOISE_GATE_FRAMES_TO_AVERAGE) { + for (int i = 0; i <= NUMBER_OF_NOISE_SAMPLE_BLOCKS - NOISE_GATE_BLOCKS_TO_AVERAGE; i += NOISE_GATE_BLOCKS_TO_AVERAGE) { float thisAverage = 0.0f; - for (int j = i; j < i + NOISE_GATE_FRAMES_TO_AVERAGE; j++) { - thisAverage += _sampleFrames[j]; - averageOfAllSampleFrames += _sampleFrames[j]; + for (int j = i; j < i + NOISE_GATE_BLOCKS_TO_AVERAGE; j++) { + thisAverage += _sampleBlocks[j]; + averageOfAllSampleBlocks += _sampleBlocks[j]; } - thisAverage /= NOISE_GATE_FRAMES_TO_AVERAGE; + thisAverage /= NOISE_GATE_BLOCKS_TO_AVERAGE; if (thisAverage < smallestSample) { smallestSample = thisAverage; } } - averageOfAllSampleFrames /= NUMBER_OF_NOISE_SAMPLE_FRAMES; + averageOfAllSampleBlocks /= NUMBER_OF_NOISE_SAMPLE_BLOCKS; _measuredFloor = smallestSample; _sampleCounter = 0; } - _closedInLastFrame = false; - _openedInLastFrame = false; + _closedInLastBlock = false; + _openedInLastBlock = false; if (samplesOverNoiseGate > NOISE_GATE_WIDTH) { - _openedInLastFrame = !_isOpen; + _openedInLastBlock = !_isOpen; _isOpen = true; - _framesToClose = NOISE_GATE_CLOSE_FRAME_DELAY; + _buffersToClose = NOISE_GATE_CLOSE_BLOCK_DELAY; } else { - if (--_framesToClose == 0) { - _closedInLastFrame = _isOpen; + if (--_buffersToClose == 0) { + _closedInLastBlock = _isOpen; _isOpen = false; } } if (!_isOpen) { - if (_closedInLastFrame) { - // would be nice to do a little crossfade to silence + // First buffer after being closed gets faded to silence, we fade across + // the entire buffer on fading out. All subsequent buffers are muted by being slammed + // to zeros + if (_closedInLastBlock) { + float fadeSlope = (1.0f / numSamples); for (int i = 0; i < numSamples; i++) { - float fadedSample = (1.0f - (float)i / (float)numSamples) * (float)samples[i]; + float fadedSample = (1.0f - ((float)i * fadeSlope)) * (float)samples[i]; samples[i] = (int16_t)fadedSample; } } else { @@ -167,10 +170,14 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { _lastLoudness = 0; } - if (_openedInLastFrame) { - // would be nice to do a little crossfade from silence - for (int i = 0; i < numSamples; i++) { - float fadedSample = ((float)i / (float)numSamples) * (float)samples[i]; + if (_openedInLastBlock) { + // would be nice to do a little crossfade from silence, but we only want to fade + // across the first 1/10th of the buffer, because we don't want to miss early + // transients. + int fadeSamples = numSamples / 10; // fade over 1/10th of the samples + float fadeSlope = (1.0f / fadeSamples); + for (int i = 0; i < fadeSamples; i++) { + float fadedSample = (float)i * fadeSlope * (float)samples[i]; samples[i] = (int16_t)fadedSample; } } diff --git a/libraries/audio-client/src/AudioNoiseGate.h b/libraries/audio-client/src/AudioNoiseGate.h index fb31561994..81ef27664d 100644 --- a/libraries/audio-client/src/AudioNoiseGate.h +++ b/libraries/audio-client/src/AudioNoiseGate.h @@ -14,7 +14,7 @@ #include -const int NUMBER_OF_NOISE_SAMPLE_FRAMES = 300; +const int NUMBER_OF_NOISE_SAMPLE_BLOCKS = 300; class AudioNoiseGate { public: @@ -23,9 +23,9 @@ public: void gateSamples(int16_t* samples, int numSamples); void removeDCOffset(int16_t* samples, int numSamples); - bool clippedInLastFrame() const { return _didClipInLastFrame; } - bool closedInLastFrame() const { return _closedInLastFrame; } - bool openedInLastFrame() const { return _openedInLastFrame; } + bool clippedInLastBlock() const { return _didClipInLastBlock; } + bool closedInLastBlock() const { return _closedInLastBlock; } + bool openedInLastBlock() const { return _openedInLastBlock; } bool isOpen() const { return _isOpen; } float getMeasuredFloor() const { return _measuredFloor; } float getLastLoudness() const { return _lastLoudness; } @@ -33,19 +33,19 @@ public: static const float CLIPPING_THRESHOLD; private: - int _inputFrameCounter; + int _inputBlockCounter; float _lastLoudness; - float _quietestFrame; - float _loudestFrame; - bool _didClipInLastFrame; + float _quietestBlock; + float _loudestBlock; + bool _didClipInLastBlock; float _dcOffset; float _measuredFloor; - float _sampleFrames[NUMBER_OF_NOISE_SAMPLE_FRAMES]; + float _sampleBlocks[NUMBER_OF_NOISE_SAMPLE_BLOCKS]; int _sampleCounter; bool _isOpen; - bool _closedInLastFrame { false }; - bool _openedInLastFrame { false }; - int _framesToClose; + bool _closedInLastBlock { false }; + bool _openedInLastBlock { false }; + int _buffersToClose; }; #endif // hifi_AudioNoiseGate_h \ No newline at end of file From 099ee1db7f1ffda4248b3c3a4a4aabfd04b5a8be Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 13 Mar 2017 12:05:42 -0700 Subject: [PATCH 58/64] CR feedback --- libraries/audio-client/src/AudioNoiseGate.cpp | 22 +++++++++---------- libraries/audio-client/src/AudioNoiseGate.h | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/libraries/audio-client/src/AudioNoiseGate.cpp b/libraries/audio-client/src/AudioNoiseGate.cpp index 2acdea6f0c..7aef8086a0 100644 --- a/libraries/audio-client/src/AudioNoiseGate.cpp +++ b/libraries/audio-client/src/AudioNoiseGate.cpp @@ -28,7 +28,7 @@ AudioNoiseGate::AudioNoiseGate() : _measuredFloor(0.0f), _sampleCounter(0), _isOpen(false), - _buffersToClose(0) + _blocksToClose(0) { } @@ -37,7 +37,7 @@ void AudioNoiseGate::removeDCOffset(int16_t* samples, int numSamples) { // // DC Offset correction // - // Measure the DC offset over a trailing number of buffers, and remove it from the input signal. + // Measure the DC offset over a trailing number of blocks, and remove it from the input signal. // This causes the noise background measurements and server muting to be more accurate. Many off-board // ADC's have a noticeable DC offset. // @@ -51,7 +51,7 @@ void AudioNoiseGate::removeDCOffset(int16_t* samples, int numSamples) { // Update measured DC offset measuredDcOffset /= numSamples; if (_dcOffset == 0.0f) { - // On first buffer, copy over measured offset + // On first block, copy over measured offset _dcOffset = measuredDcOffset; } else { _dcOffset = DC_OFFSET_AVERAGING * _dcOffset + (1.0f - DC_OFFSET_AVERAGING) * measuredDcOffset; @@ -69,11 +69,11 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { // // NOISE_GATE_HEIGHT: How loud you have to speak relative to noise background to open the gate. // Make this value lower for more sensitivity and less rejection of noise. - // NOISE_GATE_WIDTH: The number of samples in an audio buffer for which the height must be exceeded + // NOISE_GATE_WIDTH: The number of samples in an audio block for which the height must be exceeded // to open the gate. - // NOISE_GATE_CLOSE_BLOCK_DELAY: Once the noise is below the gate height for the buffer, how many buffers + // NOISE_GATE_CLOSE_BLOCK_DELAY: Once the noise is below the gate height for the block, how many blocks // will we wait before closing the gate. - // NOISE_GATE_BLOCKSS_TO_AVERAGE: How many audio buffers should we average together to compute noise floor. + // NOISE_GATE_BLOCKSS_TO_AVERAGE: How many audio blocks should we average together to compute noise floor. // More means better rejection but also can reject continuous things like singing. // NUMBER_OF_NOISE_SAMPLE_BLOCKS: How often should we re-evaluate the noise floor? @@ -147,16 +147,16 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { if (samplesOverNoiseGate > NOISE_GATE_WIDTH) { _openedInLastBlock = !_isOpen; _isOpen = true; - _buffersToClose = NOISE_GATE_CLOSE_BLOCK_DELAY; + _blocksToClose = NOISE_GATE_CLOSE_BLOCK_DELAY; } else { - if (--_buffersToClose == 0) { + if (--_blocksToClose == 0) { _closedInLastBlock = _isOpen; _isOpen = false; } } if (!_isOpen) { - // First buffer after being closed gets faded to silence, we fade across - // the entire buffer on fading out. All subsequent buffers are muted by being slammed + // First block after being closed gets faded to silence, we fade across + // the entire block on fading out. All subsequent blocks are muted by being slammed // to zeros if (_closedInLastBlock) { float fadeSlope = (1.0f / numSamples); @@ -172,7 +172,7 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { if (_openedInLastBlock) { // would be nice to do a little crossfade from silence, but we only want to fade - // across the first 1/10th of the buffer, because we don't want to miss early + // across the first 1/10th of the block, because we don't want to miss early // transients. int fadeSamples = numSamples / 10; // fade over 1/10th of the samples float fadeSlope = (1.0f / fadeSamples); diff --git a/libraries/audio-client/src/AudioNoiseGate.h b/libraries/audio-client/src/AudioNoiseGate.h index 81ef27664d..f72e92b0d5 100644 --- a/libraries/audio-client/src/AudioNoiseGate.h +++ b/libraries/audio-client/src/AudioNoiseGate.h @@ -45,7 +45,7 @@ private: bool _isOpen; bool _closedInLastBlock { false }; bool _openedInLastBlock { false }; - int _buffersToClose; + int _blocksToClose; }; #endif // hifi_AudioNoiseGate_h \ No newline at end of file From 956fdd09339378ae25ecc17a3fbb262d3f0c59ad Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 13 Mar 2017 12:06:38 -0700 Subject: [PATCH 59/64] CR feedback --- libraries/audio-client/src/AudioNoiseGate.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio-client/src/AudioNoiseGate.cpp b/libraries/audio-client/src/AudioNoiseGate.cpp index 7aef8086a0..8a9134b5dc 100644 --- a/libraries/audio-client/src/AudioNoiseGate.cpp +++ b/libraries/audio-client/src/AudioNoiseGate.cpp @@ -73,7 +73,7 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { // to open the gate. // NOISE_GATE_CLOSE_BLOCK_DELAY: Once the noise is below the gate height for the block, how many blocks // will we wait before closing the gate. - // NOISE_GATE_BLOCKSS_TO_AVERAGE: How many audio blocks should we average together to compute noise floor. + // NOISE_GATE_BLOCKS_TO_AVERAGE: How many audio blocks should we average together to compute noise floor. // More means better rejection but also can reject continuous things like singing. // NUMBER_OF_NOISE_SAMPLE_BLOCKS: How often should we re-evaluate the noise floor? From efc7fbacb14a800c46982f9b870620f010930b97 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 7 Mar 2017 10:03:38 -0800 Subject: [PATCH 60/64] Bug fix for potential crash when getting DebugDraw::instance (cherry picked from commit b0ad9a8110b8206115700e30ddd48d26959218a7) --- interface/src/Application.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index f870bd9f83..e35dcc674d 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -608,6 +608,8 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo } } + // make sure the debug draw singleton is initialized on the main thread. + DebugDraw::getInstance().removeMarker(""); _runningMarker.startRunningMarker(); From d38b994f4c2a577b47adc9d5868ef098ec2d2d0e Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 7 Mar 2017 16:07:30 -0800 Subject: [PATCH 61/64] Fix for potential crash due to DebugDraw data race. (cherry picked from commit 1b8a624edb81e8f62e2cb257b74e588653daaab2) --- libraries/render-utils/src/AnimDebugDraw.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libraries/render-utils/src/AnimDebugDraw.cpp b/libraries/render-utils/src/AnimDebugDraw.cpp index c8746d5c60..6066385847 100644 --- a/libraries/render-utils/src/AnimDebugDraw.cpp +++ b/libraries/render-utils/src/AnimDebugDraw.cpp @@ -346,7 +346,9 @@ void AnimDebugDraw::update() { numVerts += (int)markerMap.size() * VERTICES_PER_BONE; auto myAvatarMarkerMap = DebugDraw::getInstance().getMyAvatarMarkerMap(); numVerts += (int)myAvatarMarkerMap.size() * VERTICES_PER_BONE; - numVerts += (int)DebugDraw::getInstance().getRays().size() * VERTICES_PER_RAY; + auto rays = DebugDraw::getInstance().getRays(); + DebugDraw::getInstance().clearRays(); + numVerts += (int)rays.size() * VERTICES_PER_RAY; // allocate verts! std::vector vertices; @@ -398,10 +400,9 @@ void AnimDebugDraw::update() { } // draw rays from shared DebugDraw singleton - for (auto& iter : DebugDraw::getInstance().getRays()) { + for (auto& iter : rays) { addLine(std::get<0>(iter), std::get<1>(iter), std::get<2>(iter), v); } - DebugDraw::getInstance().clearRays(); data._vertexBuffer->resize(sizeof(AnimDebugDrawData::Vertex) * numVerts); data._vertexBuffer->setSubData(0, vertices); From 6747f553bf507ec33e64e75d25d3422427b53970 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 7 Mar 2017 18:13:42 -0800 Subject: [PATCH 62/64] Add DebugDraw to .eslintrc config file (cherry picked from commit c7cd0fdc3821ac932ad4e86e02c2797c5800182c) --- .eslintrc.js | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc.js b/.eslintrc.js index c708decc51..9635142d1a 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -17,6 +17,7 @@ module.exports = { "Clipboard": false, "Controller": false, "DialogsManager": false, + "DebugDraw": false, "Entities": false, "FaceTracker": false, "GlobalServices": false, From 0abcda86a8b17cae2f8425cf2d24c0d725352fe3 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 13 Mar 2017 18:52:31 -0400 Subject: [PATCH 63/64] mv AudioNoiseGate to audio --- libraries/audio-client/src/AudioClient.h | 2 +- libraries/{audio-client => audio}/src/AudioNoiseGate.cpp | 8 ++++---- libraries/{audio-client => audio}/src/AudioNoiseGate.h | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) rename libraries/{audio-client => audio}/src/AudioNoiseGate.cpp (99%) rename libraries/{audio-client => audio}/src/AudioNoiseGate.h (96%) diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 512b4bb3c1..7e9acc0586 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -45,13 +45,13 @@ #include #include #include +#include #include #include #include "AudioIOStats.h" -#include "AudioNoiseGate.h" #ifdef _WIN32 #pragma warning( push ) diff --git a/libraries/audio-client/src/AudioNoiseGate.cpp b/libraries/audio/src/AudioNoiseGate.cpp similarity index 99% rename from libraries/audio-client/src/AudioNoiseGate.cpp rename to libraries/audio/src/AudioNoiseGate.cpp index 8a9134b5dc..c99b31b989 100644 --- a/libraries/audio-client/src/AudioNoiseGate.cpp +++ b/libraries/audio/src/AudioNoiseGate.cpp @@ -1,6 +1,6 @@ // // AudioNoiseGate.cpp -// interface/src/audio +// libraries/audio // // Created by Stephen Birarda on 2014-12-16. // Copyright 2014 High Fidelity, Inc. @@ -9,12 +9,12 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +#include "AudioNoiseGate.h" + #include #include -#include - -#include "AudioNoiseGate.h" +#include "AudioConstants.h" const float AudioNoiseGate::CLIPPING_THRESHOLD = 0.90f; diff --git a/libraries/audio-client/src/AudioNoiseGate.h b/libraries/audio/src/AudioNoiseGate.h similarity index 96% rename from libraries/audio-client/src/AudioNoiseGate.h rename to libraries/audio/src/AudioNoiseGate.h index f72e92b0d5..d47cebf853 100644 --- a/libraries/audio-client/src/AudioNoiseGate.h +++ b/libraries/audio/src/AudioNoiseGate.h @@ -1,6 +1,6 @@ // // AudioNoiseGate.h -// interface/src/audio +// libraries/audio // // Created by Stephen Birarda on 2014-12-16. // Copyright 2014 High Fidelity, Inc. @@ -48,4 +48,4 @@ private: int _blocksToClose; }; -#endif // hifi_AudioNoiseGate_h \ No newline at end of file +#endif // hifi_AudioNoiseGate_h From f43706dfe1f2f2ed87b5e8793b8f1b147b56b8e7 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 13 Mar 2017 18:56:54 -0400 Subject: [PATCH 64/64] clean AudioNoiseGate --- libraries/audio/src/AudioNoiseGate.cpp | 38 ++++++-------------------- libraries/audio/src/AudioNoiseGate.h | 11 +++----- 2 files changed, 13 insertions(+), 36 deletions(-) diff --git a/libraries/audio/src/AudioNoiseGate.cpp b/libraries/audio/src/AudioNoiseGate.cpp index c99b31b989..604897af8a 100644 --- a/libraries/audio/src/AudioNoiseGate.cpp +++ b/libraries/audio/src/AudioNoiseGate.cpp @@ -19,19 +19,13 @@ const float AudioNoiseGate::CLIPPING_THRESHOLD = 0.90f; AudioNoiseGate::AudioNoiseGate() : - _inputBlockCounter(0), _lastLoudness(0.0f), - _quietestBlock(std::numeric_limits::max()), - _loudestBlock(0.0f), _didClipInLastBlock(false), _dcOffset(0.0f), _measuredFloor(0.0f), _sampleCounter(0), _isOpen(false), - _blocksToClose(0) -{ - -} + _blocksToClose(0) {} void AudioNoiseGate::removeDCOffset(int16_t* samples, int numSamples) { // @@ -80,7 +74,7 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { float loudness = 0; int thisSample = 0; int samplesOverNoiseGate = 0; - + const float NOISE_GATE_HEIGHT = 7.0f; const int NOISE_GATE_WIDTH = 5; const int NOISE_GATE_CLOSE_BLOCK_DELAY = 5; @@ -88,36 +82,22 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { // Check clipping, and check if should open noise gate _didClipInLastBlock = false; - + for (int i = 0; i < numSamples; i++) { thisSample = std::abs(samples[i]); if (thisSample >= ((float) AudioConstants::MAX_SAMPLE_VALUE * CLIPPING_THRESHOLD)) { _didClipInLastBlock = true; } - + loudness += thisSample; // Noise Reduction: Count peaks above the average loudness if (thisSample > (_measuredFloor * NOISE_GATE_HEIGHT)) { samplesOverNoiseGate++; } } - + _lastLoudness = fabs(loudness / numSamples); - - if (_quietestBlock > _lastLoudness) { - _quietestBlock = _lastLoudness; - } - if (_loudestBlock < _lastLoudness) { - _loudestBlock = _lastLoudness; - } - - const int FRAMES_FOR_NOISE_DETECTION = 400; - if (_inputBlockCounter++ > FRAMES_FOR_NOISE_DETECTION) { - _quietestBlock = std::numeric_limits::max(); - _loudestBlock = 0.0f; - _inputBlockCounter = 0; - } - + // If Noise Gate is enabled, check and turn the gate on and off float averageOfAllSampleBlocks = 0.0f; _sampleBlocks[_sampleCounter++] = _lastLoudness; @@ -130,7 +110,7 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { averageOfAllSampleBlocks += _sampleBlocks[j]; } thisAverage /= NOISE_GATE_BLOCKS_TO_AVERAGE; - + if (thisAverage < smallestSample) { smallestSample = thisAverage; } @@ -138,7 +118,7 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { averageOfAllSampleBlocks /= NUMBER_OF_NOISE_SAMPLE_BLOCKS; _measuredFloor = smallestSample; _sampleCounter = 0; - + } _closedInLastBlock = false; @@ -156,7 +136,7 @@ void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { } if (!_isOpen) { // First block after being closed gets faded to silence, we fade across - // the entire block on fading out. All subsequent blocks are muted by being slammed + // the entire block on fading out. All subsequent blocks are muted by being slammed // to zeros if (_closedInLastBlock) { float fadeSlope = (1.0f / numSamples); diff --git a/libraries/audio/src/AudioNoiseGate.h b/libraries/audio/src/AudioNoiseGate.h index d47cebf853..8430f120e5 100644 --- a/libraries/audio/src/AudioNoiseGate.h +++ b/libraries/audio/src/AudioNoiseGate.h @@ -19,24 +19,21 @@ const int NUMBER_OF_NOISE_SAMPLE_BLOCKS = 300; class AudioNoiseGate { public: AudioNoiseGate(); - + void gateSamples(int16_t* samples, int numSamples); void removeDCOffset(int16_t* samples, int numSamples); - + bool clippedInLastBlock() const { return _didClipInLastBlock; } bool closedInLastBlock() const { return _closedInLastBlock; } bool openedInLastBlock() const { return _openedInLastBlock; } bool isOpen() const { return _isOpen; } float getMeasuredFloor() const { return _measuredFloor; } float getLastLoudness() const { return _lastLoudness; } - + static const float CLIPPING_THRESHOLD; - + private: - int _inputBlockCounter; float _lastLoudness; - float _quietestBlock; - float _loudestBlock; bool _didClipInLastBlock; float _dcOffset; float _measuredFloor;