From 8a5fb07ca77aea2c6f6bd5562cd7a501509e027d Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Thu, 8 Dec 2016 21:12:47 -0800 Subject: [PATCH 001/104] Merge pull request #9169 from samcake/red Fix the highlight issue introduced RC 29 - 5774 --- libraries/render-utils/src/local_lights_shading.slf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/render-utils/src/local_lights_shading.slf b/libraries/render-utils/src/local_lights_shading.slf index d3542fcdfa..0b884d8bc0 100644 --- a/libraries/render-utils/src/local_lights_shading.slf +++ b/libraries/render-utils/src/local_lights_shading.slf @@ -85,7 +85,7 @@ void main(void) { vec3 fragEyeDir = normalize(fragEyeVector.xyz); // COmpute the rougness into gloss2 once: - float fragGloss2 = pow(frag.roughness + 0.001, 2.0); + float fragGloss2 = pow(frag.roughness + 0.001, 4.0); bool withScattering = (frag.scattering * isScatteringEnabled() > 0.0); int numLightTouching = 0; From 9b65acc54e558ccf3bec8d98914e14c9933729a1 Mon Sep 17 00:00:00 2001 From: Brad Davis Date: Fri, 9 Dec 2016 09:28:31 -0800 Subject: [PATCH 002/104] Merge pull request #9177 from samcake/blue Fix the missing lighting on AMD --- .../render-utils/src/LightClusterGrid.slh | 14 +++++++++----- libraries/render-utils/src/LightClusters.cpp | 18 +++++++++--------- .../render-utils/src/local_lights_shading.slf | 5 ++--- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/libraries/render-utils/src/LightClusterGrid.slh b/libraries/render-utils/src/LightClusterGrid.slh index f4ec35a75c..722d37814d 100644 --- a/libraries/render-utils/src/LightClusterGrid.slh +++ b/libraries/render-utils/src/LightClusterGrid.slh @@ -24,7 +24,7 @@ struct FrustumGrid { mat4 eyeToWorldMat; }; -uniform frustumGridBuffer { +layout(std140) uniform frustumGridBuffer { FrustumGrid frustumGrid; }; @@ -51,16 +51,20 @@ float projection_getFar(mat4 projection) { #define GRID_INDEX_TYPE ivec4 #define GRID_FETCH_BUFFER(i) i / 4][i % 4 <@else@> -#define GRID_NUM_ELEMENTS 16384 +#define GRID_NUM_ELEMENTS 4096 +#define GRID_INDEX_TYPE ivec4 +#define GRID_FETCH_BUFFER(i) i / 4][i % 4 + + <@endif@> -uniform clusterGridBuffer { +layout(std140) uniform clusterGridBuffer { GRID_INDEX_TYPE _clusterGridTable[GRID_NUM_ELEMENTS]; }; -uniform clusterContentBuffer { +layout(std140) uniform clusterContentBuffer { GRID_INDEX_TYPE _clusterGridContent[GRID_NUM_ELEMENTS]; }; diff --git a/libraries/render-utils/src/LightClusters.cpp b/libraries/render-utils/src/LightClusters.cpp index d3a384f1df..8dc95da65b 100644 --- a/libraries/render-utils/src/LightClusters.cpp +++ b/libraries/render-utils/src/LightClusters.cpp @@ -27,21 +27,21 @@ enum LightClusterGridShader_MapSlot { DEFERRED_BUFFER_LINEAR_DEPTH_UNIT = 0, - DEFERRED_BUFFER_COLOR_UNIT, - DEFERRED_BUFFER_NORMAL_UNIT, - DEFERRED_BUFFER_EMISSIVE_UNIT, - DEFERRED_BUFFER_DEPTH_UNIT, + DEFERRED_BUFFER_COLOR_UNIT = 1, + DEFERRED_BUFFER_NORMAL_UNIT = 2, + DEFERRED_BUFFER_EMISSIVE_UNIT = 3, + DEFERRED_BUFFER_DEPTH_UNIT = 4, }; enum LightClusterGridShader_BufferSlot { LIGHT_CLUSTER_GRID_FRUSTUM_GRID_SLOT = 0, - DEFERRED_FRAME_TRANSFORM_BUFFER_SLOT, - CAMERA_CORRECTION_BUFFER_SLOT, + DEFERRED_FRAME_TRANSFORM_BUFFER_SLOT =1, + CAMERA_CORRECTION_BUFFER_SLOT = 2, LIGHT_GPU_SLOT = render::ShapePipeline::Slot::LIGHT, - LIGHT_INDEX_GPU_SLOT, + LIGHT_INDEX_GPU_SLOT = 5, - LIGHT_CLUSTER_GRID_CLUSTER_GRID_SLOT, - LIGHT_CLUSTER_GRID_CLUSTER_CONTENT_SLOT, + LIGHT_CLUSTER_GRID_CLUSTER_GRID_SLOT = 6, + LIGHT_CLUSTER_GRID_CLUSTER_CONTENT_SLOT = 7, }; FrustumGrid::FrustumGrid(const FrustumGrid& source) : diff --git a/libraries/render-utils/src/local_lights_shading.slf b/libraries/render-utils/src/local_lights_shading.slf index 0b884d8bc0..a4e28a9757 100644 --- a/libraries/render-utils/src/local_lights_shading.slf +++ b/libraries/render-utils/src/local_lights_shading.slf @@ -34,7 +34,6 @@ in vec2 _texCoord0; out vec4 _fragColor; void main(void) { - // Grab the fragment data from the uv vec2 texCoord = _texCoord0.st; @@ -49,7 +48,7 @@ void main(void) { // Frag pos in world mat4 invViewMat = getViewInverse(); vec4 fragPos = invViewMat * fragPosition; - + // From frag world pos find the cluster vec4 clusterEyePos = frustumGrid_worldToEye(fragPos); ivec3 clusterPos = frustumGrid_eyeToClusterPos(clusterEyePos.xyz); @@ -84,7 +83,7 @@ void main(void) { vec4 fragEyeVector = invViewMat * vec4(-frag.position.xyz, 0.0); vec3 fragEyeDir = normalize(fragEyeVector.xyz); - // COmpute the rougness into gloss2 once: + // Compute the rougness into gloss2 once: float fragGloss2 = pow(frag.roughness + 0.001, 4.0); bool withScattering = (frag.scattering * isScatteringEnabled() > 0.0); From a4126b3ee000054c76fd069b8a15e008632bb7f3 Mon Sep 17 00:00:00 2001 From: howard-stearns Date: Wed, 11 Jan 2017 14:23:18 -0800 Subject: [PATCH 003/104] revert "fix competition between seek mouse and lasers" --- .../controllers/handControllerPointer.js | 27 ++++--------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/scripts/system/controllers/handControllerPointer.js b/scripts/system/controllers/handControllerPointer.js index 4d478fbdd5..0623ddf100 100644 --- a/scripts/system/controllers/handControllerPointer.js +++ b/scripts/system/controllers/handControllerPointer.js @@ -20,9 +20,6 @@ // When partially squeezing over a HUD element, a laser or the reticle is shown where the active hand // controller beam intersects the HUD. -var systemLaserOn = false; - - Script.include("../libraries/controllers.js"); // UTILITIES ------------- @@ -124,12 +121,6 @@ function ignoreMouseActivity() { if (!Reticle.allowMouseCapture) { return true; } - - // if the lasers are on, then reticle/mouse should be hidden and we can ignore it for seeking or depth updating - if (systemLaserOn) { - return true; - } - var pos = Reticle.position; if (!pos || (pos.x == -1 && pos.y == -1)) { return true; @@ -270,12 +261,6 @@ var ONE_MINUS_WEIGHTING = 1 - WEIGHTING; var AVERAGE_MOUSE_VELOCITY_FOR_SEEK_TO = 20; function isShakingMouse() { // True if the person is waving the mouse around trying to find it. var now = Date.now(), mouse = Reticle.position, isShaking = false; - - // if the lasers are on, then we ignore mouse shaking - if (systemLaserOn) { - return false; - } - if (lastIntegration && (lastIntegration !== now)) { var velocity = Vec3.length(Vec3.subtract(mouse, lastMouse)) / (now - lastIntegration); averageMouseVelocity = (ONE_MINUS_WEIGHTING * averageMouseVelocity) + (WEIGHTING * velocity); @@ -290,14 +275,9 @@ function isShakingMouse() { // True if the person is waving the mouse around try var NON_LINEAR_DIVISOR = 2; var MINIMUM_SEEK_DISTANCE = 0.1; function updateSeeking(doNotStartSeeking) { - // if the lasers are on, then we never do seeking - if (systemLaserOn) { - isSeeking = false; - return; - } - if (!doNotStartSeeking && (!Reticle.visible || isShakingMouse())) { if (!isSeeking) { + print('Start seeking mouse.'); isSeeking = true; } } // e.g., if we're about to turn it on with first movement. @@ -307,6 +287,7 @@ function updateSeeking(doNotStartSeeking) { averageMouseVelocity = lastIntegration = 0; var lookAt2D = HMD.getHUDLookAtPosition2D(); if (!lookAt2D) { // If this happens, something has gone terribly wrong. + print('Cannot seek without lookAt position'); isSeeking = false; return; // E.g., if parallel to location in HUD } @@ -322,6 +303,7 @@ function updateSeeking(doNotStartSeeking) { } var okX = !updateDimension('x'), okY = !updateDimension('y'); // Evaluate both. Don't short-circuit. if (okX && okY) { + print('Finished seeking mouse'); isSeeking = false; } else { Reticle.setPosition(copy); // Not setReticlePosition @@ -340,7 +322,7 @@ function updateMouseActivity(isClick) { return; } // Bug: mouse clicks should keep going. Just not hand controller clicks handControllerLockOut.update(now); - Reticle.visible = !systemLaserOn; + Reticle.visible = true; } function expireMouseCursor(now) { if (!isPointingAtOverlay() && mouseCursorActivity.expired(now)) { @@ -492,6 +474,7 @@ var LASER_ALPHA = 0.5; var LASER_SEARCH_COLOR_XYZW = {x: 10 / 255, y: 10 / 255, z: 255 / 255, w: LASER_ALPHA}; var LASER_TRIGGER_COLOR_XYZW = {x: 250 / 255, y: 10 / 255, z: 10 / 255, w: LASER_ALPHA}; var SYSTEM_LASER_DIRECTION = {x: 0, y: 0, z: -1}; +var systemLaserOn = false; function clearSystemLaser() { if (!systemLaserOn) { return; From dc2a42df4f277c76a9adbb9d73471f34e36b1bee Mon Sep 17 00:00:00 2001 From: howard-stearns Date: Wed, 11 Jan 2017 14:24:24 -0800 Subject: [PATCH 004/104] fix mouse vs lasers, again --- scripts/system/controllers/handControllerPointer.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/system/controllers/handControllerPointer.js b/scripts/system/controllers/handControllerPointer.js index 0623ddf100..2a64b856bc 100644 --- a/scripts/system/controllers/handControllerPointer.js +++ b/scripts/system/controllers/handControllerPointer.js @@ -20,6 +20,10 @@ // When partially squeezing over a HUD element, a laser or the reticle is shown where the active hand // controller beam intersects the HUD. +var activeTrigger; +function isLaserOn() { + return activeTrigger.partial(); +} Script.include("../libraries/controllers.js"); // UTILITIES ------------- @@ -275,7 +279,7 @@ function isShakingMouse() { // True if the person is waving the mouse around try var NON_LINEAR_DIVISOR = 2; var MINIMUM_SEEK_DISTANCE = 0.1; function updateSeeking(doNotStartSeeking) { - if (!doNotStartSeeking && (!Reticle.visible || isShakingMouse())) { + if (!doNotStartSeeking && !isLaserOn() && (!Reticle.visible || isShakingMouse())) { if (!isSeeking) { print('Start seeking mouse.'); isSeeking = true; @@ -374,7 +378,7 @@ setupHandler(Controller.mouseDoublePressEvent, onMouseClick); var leftTrigger = new Trigger('left'); var rightTrigger = new Trigger('right'); -var activeTrigger = rightTrigger; +activeTrigger = rightTrigger; var activeHand = Controller.Standard.RightHand; var LEFT_HUD_LASER = 1; var RIGHT_HUD_LASER = 2; From 4813324b76ea03e0017d4bd2db04761c29672c84 Mon Sep 17 00:00:00 2001 From: howard-stearns Date: Wed, 11 Jan 2017 16:24:05 -0800 Subject: [PATCH 005/104] use update signal instead of timer --- scripts/system/controllers/handControllerPointer.js | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/scripts/system/controllers/handControllerPointer.js b/scripts/system/controllers/handControllerPointer.js index 2a64b856bc..1f8f0f53e6 100644 --- a/scripts/system/controllers/handControllerPointer.js +++ b/scripts/system/controllers/handControllerPointer.js @@ -571,12 +571,6 @@ function update() { Reticle.visible = false; } -var BASIC_TIMER_INTERVAL = 20; // 20ms = 50hz good enough -var updateIntervalTimer = Script.setInterval(function(){ - update(); -}, BASIC_TIMER_INTERVAL); - - // Check periodically for changes to setup. var SETTINGS_CHANGE_RECHECK_INTERVAL = 10 * 1000; // 10 seconds function checkSettings() { @@ -586,9 +580,10 @@ function checkSettings() { checkSettings(); var settingsChecker = Script.setInterval(checkSettings, SETTINGS_CHANGE_RECHECK_INTERVAL); +Script.update.connect(update); Script.scriptEnding.connect(function () { Script.clearInterval(settingsChecker); - Script.clearInterval(updateIntervalTimer); + Script.update.disconnect(update); OffscreenFlags.navigationFocusDisabled = false; }); From b3c64d4693fc662df773b9199b6a12d9da4e1e8a Mon Sep 17 00:00:00 2001 From: howard-stearns Date: Wed, 11 Jan 2017 16:32:15 -0800 Subject: [PATCH 006/104] remove log spam --- scripts/system/controllers/handControllerPointer.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scripts/system/controllers/handControllerPointer.js b/scripts/system/controllers/handControllerPointer.js index 1f8f0f53e6..b7dfcaeb56 100644 --- a/scripts/system/controllers/handControllerPointer.js +++ b/scripts/system/controllers/handControllerPointer.js @@ -280,10 +280,7 @@ var NON_LINEAR_DIVISOR = 2; var MINIMUM_SEEK_DISTANCE = 0.1; function updateSeeking(doNotStartSeeking) { if (!doNotStartSeeking && !isLaserOn() && (!Reticle.visible || isShakingMouse())) { - if (!isSeeking) { - print('Start seeking mouse.'); - isSeeking = true; - } + isSeeking = true; } // e.g., if we're about to turn it on with first movement. if (!isSeeking) { return; @@ -291,7 +288,6 @@ function updateSeeking(doNotStartSeeking) { averageMouseVelocity = lastIntegration = 0; var lookAt2D = HMD.getHUDLookAtPosition2D(); if (!lookAt2D) { // If this happens, something has gone terribly wrong. - print('Cannot seek without lookAt position'); isSeeking = false; return; // E.g., if parallel to location in HUD } @@ -307,7 +303,6 @@ function updateSeeking(doNotStartSeeking) { } var okX = !updateDimension('x'), okY = !updateDimension('y'); // Evaluate both. Don't short-circuit. if (okX && okY) { - print('Finished seeking mouse'); isSeeking = false; } else { Reticle.setPosition(copy); // Not setReticlePosition From aba5be4dd21c53c99eaaf870cde6856ba9f3df49 Mon Sep 17 00:00:00 2001 From: howard-stearns Date: Wed, 11 Jan 2017 17:01:46 -0800 Subject: [PATCH 007/104] don't set bogus values --- scripts/system/controllers/handControllerPointer.js | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/system/controllers/handControllerPointer.js b/scripts/system/controllers/handControllerPointer.js index b7dfcaeb56..2c732b4cdc 100644 --- a/scripts/system/controllers/handControllerPointer.js +++ b/scripts/system/controllers/handControllerPointer.js @@ -482,7 +482,6 @@ function clearSystemLaser() { HMD.disableExtraLaser(); systemLaserOn = false; weMovedReticle = true; - Reticle.position = { x: -1, y: -1 }; } function setColoredLaser() { // answer trigger state if lasers supported, else falsey. var color = (activeTrigger.state === 'full') ? LASER_TRIGGER_COLOR_XYZW : LASER_SEARCH_COLOR_XYZW; From 698d63bef9bd80a470879c2fe97a14021cdc14c6 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Sat, 14 Jan 2017 10:26:13 -0800 Subject: [PATCH 008/104] Fix audio failures on systems that return incorrect/unsupported mixFormat --- libraries/audio-client/src/AudioClient.cpp | 42 +++++++++------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 1aa4439de9..1e3dc11338 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -424,35 +424,25 @@ bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice, adjustedAudioFormat = desiredAudioFormat; // - // Attempt the device sample rate in decreasing order of preference. + // Attempt the device sample rate and channel count in decreasing order of preference. // - if (audioDevice.supportedSampleRates().contains(48000)) { - adjustedAudioFormat.setSampleRate(48000); - } else if (audioDevice.supportedSampleRates().contains(44100)) { - adjustedAudioFormat.setSampleRate(44100); - } else if (audioDevice.supportedSampleRates().contains(32000)) { - adjustedAudioFormat.setSampleRate(32000); - } else if (audioDevice.supportedSampleRates().contains(24000)) { - adjustedAudioFormat.setSampleRate(24000); - } else if (audioDevice.supportedSampleRates().contains(16000)) { - adjustedAudioFormat.setSampleRate(16000); - } else if (audioDevice.supportedSampleRates().contains(96000)) { - adjustedAudioFormat.setSampleRate(96000); - } else if (audioDevice.supportedSampleRates().contains(192000)) { - adjustedAudioFormat.setSampleRate(192000); - } else if (audioDevice.supportedSampleRates().contains(88200)) { - adjustedAudioFormat.setSampleRate(88200); - } else if (audioDevice.supportedSampleRates().contains(176400)) { - adjustedAudioFormat.setSampleRate(176400); + const int sampleRates[] = { 48000, 44100, 32000, 24000, 16000, 96000, 192000, 88200, 176400 }; + const int inputChannels[] = { 1, 2, 4, 6, 8 }; // prefer mono + const int outputChannels[] = { 2, 4, 6, 8, 1 }; // prefer stereo, downmix as last resort + + for (int channelCount : (desiredAudioFormat.channelCount() == 1 ? inputChannels : outputChannels)) { + for (int sampleRate : sampleRates) { + + adjustedAudioFormat.setChannelCount(channelCount); + adjustedAudioFormat.setSampleRate(sampleRate); + + if (audioDevice.isFormatSupported(adjustedAudioFormat)) { + return true; + } + } } - if (adjustedAudioFormat != desiredAudioFormat) { - // return the nearest in case it needs 2 channels - adjustedAudioFormat = audioDevice.nearestFormat(adjustedAudioFormat); - return true; - } else { - return false; - } + return false; // a supported format could not be found } bool sampleChannelConversion(const int16_t* sourceSamples, int16_t* destinationSamples, unsigned int numSourceSamples, From bb715bf7adb0c8b167709c85b970e47ae7ce41e0 Mon Sep 17 00:00:00 2001 From: Brad Davis Date: Mon, 16 Jan 2017 10:09:47 -0800 Subject: [PATCH 009/104] Fix bug in FBX parse changes --- libraries/fbx/src/FBXReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/fbx/src/FBXReader.cpp b/libraries/fbx/src/FBXReader.cpp index 5272969e6b..42922ce226 100644 --- a/libraries/fbx/src/FBXReader.cpp +++ b/libraries/fbx/src/FBXReader.cpp @@ -972,7 +972,7 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS static const QVariant EMISSIVE = QByteArray("Emissive"); static const QVariant AMBIENT_FACTOR = QByteArray("AmbientFactor"); static const QVariant SHININESS = QByteArray("Shininess"); - static const QVariant OPACITY = QByteArray("Shininess"); + static const QVariant OPACITY = QByteArray("Opacity"); static const QVariant MAYA_USE_NORMAL_MAP = QByteArray("Maya|use_normal_map"); static const QVariant MAYA_BASE_COLOR = QByteArray("Maya|base_color"); static const QVariant MAYA_USE_COLOR_MAP = QByteArray("Maya|use_color_map"); From 167aa7620e3f28d527e3454f3b5ec1df321b3624 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Mon, 16 Jan 2017 15:13:28 -0800 Subject: [PATCH 010/104] fix cauterization hack --- .../render-utils/src/MeshPartPayload.cpp | 19 ++++++++++++++++--- libraries/render-utils/src/MeshPartPayload.h | 5 ++++- libraries/render-utils/src/Model.cpp | 2 +- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/libraries/render-utils/src/MeshPartPayload.cpp b/libraries/render-utils/src/MeshPartPayload.cpp index 5c4c0890a7..e3b2527e67 100644 --- a/libraries/render-utils/src/MeshPartPayload.cpp +++ b/libraries/render-utils/src/MeshPartPayload.cpp @@ -359,8 +359,11 @@ void ModelMeshPartPayload::notifyLocationChanged() { } -void ModelMeshPartPayload::updateTransformForSkinnedMesh(const Transform& transform, const QVector& clusterMatrices) { +void ModelMeshPartPayload::updateTransformForSkinnedMesh(const Transform& transform, + const QVector& clusterMatrices, + const QVector& cauterizedClusterMatrices) { _transform = transform; + _cauterizedTransform = transform; if (clusterMatrices.size() > 0) { _worldBound = AABox(); @@ -373,6 +376,11 @@ void ModelMeshPartPayload::updateTransformForSkinnedMesh(const Transform& transf _worldBound.transform(transform); if (clusterMatrices.size() == 1) { _transform = _transform.worldTransform(Transform(clusterMatrices[0])); + if (cauterizedClusterMatrices.size() != 0) { + _cauterizedTransform = _cauterizedTransform.worldTransform(Transform(cauterizedClusterMatrices[0])); + } else { + _cauterizedTransform = _transform; + } } } } @@ -527,9 +535,14 @@ void ModelMeshPartPayload::bindTransform(gpu::Batch& batch, const ShapePipeline: } else { batch.setUniformBuffer(ShapePipeline::Slot::BUFFER::SKINNING, state.clusterBuffer); } + batch.setModelTransform(_transform); + } else { + if (canCauterize && _model->getCauterizeBones()) { + batch.setModelTransform(_cauterizedTransform); + } else { + batch.setModelTransform(_transform); + } } - - batch.setModelTransform(_transform); } void ModelMeshPartPayload::startFade() { diff --git a/libraries/render-utils/src/MeshPartPayload.h b/libraries/render-utils/src/MeshPartPayload.h index b048dc903f..b7a8cf63f0 100644 --- a/libraries/render-utils/src/MeshPartPayload.h +++ b/libraries/render-utils/src/MeshPartPayload.h @@ -85,7 +85,9 @@ public: typedef Payload::DataPointer Pointer; void notifyLocationChanged() override; - void updateTransformForSkinnedMesh(const Transform& transform, const QVector& clusterMatrices); + void updateTransformForSkinnedMesh(const Transform& transform, + const QVector& clusterMatrices, + const QVector& cauterizedClusterMatrices); // Entity fade in void startFade(); @@ -106,6 +108,7 @@ public: Model* _model; + Transform _cauterizedTransform; int _meshIndex; int _shapeID; diff --git a/libraries/render-utils/src/Model.cpp b/libraries/render-utils/src/Model.cpp index 7c1a6e14d0..3a3095458c 100644 --- a/libraries/render-utils/src/Model.cpp +++ b/libraries/render-utils/src/Model.cpp @@ -238,7 +238,7 @@ void Model::updateRenderItems() { // update the model transform and bounding box for this render item. const Model::MeshState& state = data._model->_meshStates.at(data._meshIndex); - data.updateTransformForSkinnedMesh(modelTransform, state.clusterMatrices); + data.updateTransformForSkinnedMesh(modelTransform, state.clusterMatrices, state.cauterizedClusterMatrices); } } }); From ef6e83788eb3f3c7451e39d49095a79e8e97974f Mon Sep 17 00:00:00 2001 From: howard-stearns Date: Mon, 16 Jan 2017 14:54:32 -0800 Subject: [PATCH 011/104] handControllerGrab update loop change --- scripts/system/controllers/handControllerGrab.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js index 2ed09232e6..57698bd0dc 100644 --- a/scripts/system/controllers/handControllerGrab.js +++ b/scripts/system/controllers/handControllerGrab.js @@ -3082,7 +3082,7 @@ var handleHandMessages = function(channel, message, sender) { Messages.messageReceived.connect(handleHandMessages); -var TARGET_UPDATE_HZ = 50; // 50hz good enough (no change in logic) +var TARGET_UPDATE_HZ = 60; // 50hz good enough, but we're using update var BASIC_TIMER_INTERVAL_MS = 1000 / TARGET_UPDATE_HZ; var lastInterval = Date.now(); @@ -3095,7 +3095,7 @@ var updateTotalWork = 0; var UPDATE_PERFORMANCE_DEBUGGING = false; -var updateIntervalTimer = Script.setInterval(function(){ +function updateWrapper(){ intervalCount++; var thisInterval = Date.now(); @@ -3141,11 +3141,12 @@ var updateIntervalTimer = Script.setInterval(function(){ updateTotalWork = 0; } -}, BASIC_TIMER_INTERVAL_MS); +} +Script.update.connect(updateWrapper); function cleanup() { Menu.removeMenuItem("Developer", "Show Grab Sphere"); - Script.clearInterval(updateIntervalTimer); + Script.update.disconnect(updateWrapper); rightController.cleanup(); leftController.cleanup(); Controller.disableMapping(MAPPING_NAME); From ec93a663e7c523902d20d3e04e0ae744eabbfbed Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Fri, 3 Feb 2017 14:05:23 -0800 Subject: [PATCH 012/104] don't send AVATAR_SELF_ID over wire --- libraries/entities/src/EntityEditPacketSender.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/libraries/entities/src/EntityEditPacketSender.cpp b/libraries/entities/src/EntityEditPacketSender.cpp index e05db07d0d..97648c7079 100644 --- a/libraries/entities/src/EntityEditPacketSender.cpp +++ b/libraries/entities/src/EntityEditPacketSender.cpp @@ -100,7 +100,18 @@ void EntityEditPacketSender::queueEditEntityMessage(PacketType type, QByteArray bufferOut(NLPacket::maxPayloadSize(type), 0); - if (EntityItemProperties::encodeEntityEditPacket(type, entityItemID, properties, bufferOut)) { + bool success; + if (properties.parentIDChanged() && properties.getParentID() == AVATAR_SELF_ID) { + EntityItemProperties propertiesCopy = properties; + auto nodeList = DependencyManager::get(); + const QUuid myNodeID = nodeList->getSessionUUID(); + propertiesCopy.setParentID(myNodeID); + success = EntityItemProperties::encodeEntityEditPacket(type, entityItemID, propertiesCopy, bufferOut); + } else { + success = EntityItemProperties::encodeEntityEditPacket(type, entityItemID, properties, bufferOut); + } + + if (success) { #ifdef WANT_DEBUG qCDebug(entities) << "calling queueOctreeEditMessage()..."; qCDebug(entities) << " id:" << entityItemID; From 6b4c800c2ee98e59ba1ba7aa6ea6b1e33e058d86 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Fri, 3 Feb 2017 15:50:59 -0800 Subject: [PATCH 013/104] don't call stepKinematicMotion on kinematic entities --- libraries/physics/src/EntityMotionState.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/libraries/physics/src/EntityMotionState.cpp b/libraries/physics/src/EntityMotionState.cpp index e9891020b3..c90ca5a1a9 100644 --- a/libraries/physics/src/EntityMotionState.cpp +++ b/libraries/physics/src/EntityMotionState.cpp @@ -199,15 +199,12 @@ void EntityMotionState::getWorldTransform(btTransform& worldTrans) const { return; } assert(entityTreeIsLocked()); - if (_motionType == MOTION_TYPE_KINEMATIC) { + if (_motionType == MOTION_TYPE_KINEMATIC && !_entity->hasAncestorOfType(NestableType::Avatar)) { BT_PROFILE("kinematicIntegration"); // This is physical kinematic motion which steps strictly by the subframe count // of the physics simulation and uses full gravity for acceleration. - if (_entity->hasAncestorOfType(NestableType::Avatar)) { - _entity->setAcceleration(glm::vec3(0.0f)); - } else { - _entity->setAcceleration(_entity->getGravity()); - } + _entity->setAcceleration(_entity->getGravity()); + uint32_t thisStep = ObjectMotionState::getWorldSimulationStep(); float dt = (thisStep - _lastKinematicStep) * PHYSICS_ENGINE_FIXED_SUBSTEP; _entity->stepKinematicMotion(dt); From c0e313830c829543e341eb0ae45b8fa1c2982605 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Fri, 3 Feb 2017 16:14:15 -0800 Subject: [PATCH 014/104] make hasAncestorOfType understand AVATAR_SELF_ID --- libraries/physics/src/EntityMotionState.cpp | 9 ++++++--- libraries/shared/src/SpatiallyNestable.cpp | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/libraries/physics/src/EntityMotionState.cpp b/libraries/physics/src/EntityMotionState.cpp index c90ca5a1a9..e9891020b3 100644 --- a/libraries/physics/src/EntityMotionState.cpp +++ b/libraries/physics/src/EntityMotionState.cpp @@ -199,12 +199,15 @@ void EntityMotionState::getWorldTransform(btTransform& worldTrans) const { return; } assert(entityTreeIsLocked()); - if (_motionType == MOTION_TYPE_KINEMATIC && !_entity->hasAncestorOfType(NestableType::Avatar)) { + if (_motionType == MOTION_TYPE_KINEMATIC) { BT_PROFILE("kinematicIntegration"); // This is physical kinematic motion which steps strictly by the subframe count // of the physics simulation and uses full gravity for acceleration. - _entity->setAcceleration(_entity->getGravity()); - + if (_entity->hasAncestorOfType(NestableType::Avatar)) { + _entity->setAcceleration(glm::vec3(0.0f)); + } else { + _entity->setAcceleration(_entity->getGravity()); + } uint32_t thisStep = ObjectMotionState::getWorldSimulationStep(); float dt = (thisStep - _lastKinematicStep) * PHYSICS_ENGINE_FIXED_SUBSTEP; _entity->stepKinematicMotion(dt); diff --git a/libraries/shared/src/SpatiallyNestable.cpp b/libraries/shared/src/SpatiallyNestable.cpp index 35e574bf06..f071bda71f 100644 --- a/libraries/shared/src/SpatiallyNestable.cpp +++ b/libraries/shared/src/SpatiallyNestable.cpp @@ -1034,6 +1034,13 @@ AACube SpatiallyNestable::getQueryAACube() const { bool SpatiallyNestable::hasAncestorOfType(NestableType nestableType) const { bool success; + if (nestableType == NestableType::Avatar) { + QUuid parentID = getParentID(); + if (parentID == AVATAR_SELF_ID) { + return true; + } + } + SpatiallyNestablePointer parent = getParentPointer(success); if (!success || !parent) { return false; From 933aed47194f5a9491561939c65f419709ae69d3 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Fri, 3 Feb 2017 16:36:06 -0800 Subject: [PATCH 015/104] don't call stepKinematicMotion on kinematic entities when they are children of avatars --- libraries/physics/src/EntityMotionState.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/libraries/physics/src/EntityMotionState.cpp b/libraries/physics/src/EntityMotionState.cpp index e9891020b3..c90ca5a1a9 100644 --- a/libraries/physics/src/EntityMotionState.cpp +++ b/libraries/physics/src/EntityMotionState.cpp @@ -199,15 +199,12 @@ void EntityMotionState::getWorldTransform(btTransform& worldTrans) const { return; } assert(entityTreeIsLocked()); - if (_motionType == MOTION_TYPE_KINEMATIC) { + if (_motionType == MOTION_TYPE_KINEMATIC && !_entity->hasAncestorOfType(NestableType::Avatar)) { BT_PROFILE("kinematicIntegration"); // This is physical kinematic motion which steps strictly by the subframe count // of the physics simulation and uses full gravity for acceleration. - if (_entity->hasAncestorOfType(NestableType::Avatar)) { - _entity->setAcceleration(glm::vec3(0.0f)); - } else { - _entity->setAcceleration(_entity->getGravity()); - } + _entity->setAcceleration(_entity->getGravity()); + uint32_t thisStep = ObjectMotionState::getWorldSimulationStep(); float dt = (thisStep - _lastKinematicStep) * PHYSICS_ENGINE_FIXED_SUBSTEP; _entity->stepKinematicMotion(dt); From f13fdf2a4c62b8cd4b3b70504e5d7403710ff030 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Fri, 3 Feb 2017 16:43:36 -0800 Subject: [PATCH 016/104] make findAncestorOfType understand AVATAR_SELF_ID --- libraries/shared/src/SpatiallyNestable.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libraries/shared/src/SpatiallyNestable.cpp b/libraries/shared/src/SpatiallyNestable.cpp index f071bda71f..ddc3f416e0 100644 --- a/libraries/shared/src/SpatiallyNestable.cpp +++ b/libraries/shared/src/SpatiallyNestable.cpp @@ -1055,6 +1055,14 @@ bool SpatiallyNestable::hasAncestorOfType(NestableType nestableType) const { const QUuid SpatiallyNestable::findAncestorOfType(NestableType nestableType) const { bool success; + + if (nestableType == NestableType::Avatar) { + QUuid parentID = getParentID(); + if (parentID == AVATAR_SELF_ID) { + return AVATAR_SELF_ID; // TODO -- can we put nodeID here? + } + } + SpatiallyNestablePointer parent = getParentPointer(success); if (!success || !parent) { return QUuid(); From b90d7e26899067b6c4ba56cde235fddced1b3372 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Fri, 3 Feb 2017 16:47:03 -0800 Subject: [PATCH 017/104] another AVATAR_SELF_ID adjustment --- libraries/entities/src/EntityItem.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/entities/src/EntityItem.cpp b/libraries/entities/src/EntityItem.cpp index 233ce7d88e..2503687dce 100644 --- a/libraries/entities/src/EntityItem.cpp +++ b/libraries/entities/src/EntityItem.cpp @@ -1828,7 +1828,8 @@ void EntityItem::computeCollisionGroupAndFinalMask(int16_t& group, int16_t& mask // "bootstrapping" problem where you can shoot yourself across the room by grabbing something // and holding it against your own avatar. QUuid ancestorID = findAncestorOfType(NestableType::Avatar); - if (!ancestorID.isNull() && ancestorID == Physics::getSessionUUID()) { + if (!ancestorID.isNull() && + (ancestorID == Physics::getSessionUUID() || ancestorID == AVATAR_SELF_ID)) { userMask &= ~USER_COLLISION_GROUP_MY_AVATAR; } } From 0dc7152dcca44bbba6237e4b82bf60e09ec03055 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Mon, 6 Feb 2017 09:50:48 -0800 Subject: [PATCH 018/104] if an entity has an avatar parent but ends up in EntityMotionState::setWorldTransform, ignore the update from bullet --- libraries/physics/src/EntityMotionState.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/physics/src/EntityMotionState.cpp b/libraries/physics/src/EntityMotionState.cpp index c90ca5a1a9..d652e78f6a 100644 --- a/libraries/physics/src/EntityMotionState.cpp +++ b/libraries/physics/src/EntityMotionState.cpp @@ -230,6 +230,9 @@ void EntityMotionState::setWorldTransform(const btTransform& worldTrans) { assert(entityTreeIsLocked()); measureBodyAcceleration(); bool positionSuccess; + if (_entity->hasAncestorOfType(NestableType::Avatar)) { + return; + } _entity->setPosition(bulletToGLM(worldTrans.getOrigin()) + ObjectMotionState::getWorldOffset(), positionSuccess, false); if (!positionSuccess) { static QString repeatedMessage = From f68fcbb9cd89a37da03f305044d1df8fa87a0421 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Mon, 6 Feb 2017 12:10:26 -0800 Subject: [PATCH 019/104] when the parentID is changed by a network update, set physics flags --- libraries/entities/src/EntityItem.cpp | 2 +- libraries/physics/src/EntityMotionState.cpp | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/libraries/entities/src/EntityItem.cpp b/libraries/entities/src/EntityItem.cpp index 2503687dce..a656827d5e 100644 --- a/libraries/entities/src/EntityItem.cpp +++ b/libraries/entities/src/EntityItem.cpp @@ -828,7 +828,7 @@ int EntityItem::readEntityDataFromBuffer(const unsigned char* data, int bytesLef { // parentID and parentJointIndex are also protected by simulation ownership bool oldOverwrite = overwriteLocalData; overwriteLocalData = overwriteLocalData && !weOwnSimulation; - READ_ENTITY_PROPERTY(PROP_PARENT_ID, QUuid, setParentID); + READ_ENTITY_PROPERTY(PROP_PARENT_ID, QUuid, updateParentID); READ_ENTITY_PROPERTY(PROP_PARENT_JOINT_INDEX, quint16, setParentJointIndex); overwriteLocalData = oldOverwrite; } diff --git a/libraries/physics/src/EntityMotionState.cpp b/libraries/physics/src/EntityMotionState.cpp index d652e78f6a..c90ca5a1a9 100644 --- a/libraries/physics/src/EntityMotionState.cpp +++ b/libraries/physics/src/EntityMotionState.cpp @@ -230,9 +230,6 @@ void EntityMotionState::setWorldTransform(const btTransform& worldTrans) { assert(entityTreeIsLocked()); measureBodyAcceleration(); bool positionSuccess; - if (_entity->hasAncestorOfType(NestableType::Avatar)) { - return; - } _entity->setPosition(bulletToGLM(worldTrans.getOrigin()) + ObjectMotionState::getWorldOffset(), positionSuccess, false); if (!positionSuccess) { static QString repeatedMessage = From 40730c1d19376b455e8350d671696529372fd970 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Mon, 6 Feb 2017 14:16:47 -0800 Subject: [PATCH 020/104] avoid bootstrapping self with something that has someone else as simulation owner --- libraries/entities/src/EntityItem.cpp | 50 +++++++++++++++------------ 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/libraries/entities/src/EntityItem.cpp b/libraries/entities/src/EntityItem.cpp index a656827d5e..6543af5355 100644 --- a/libraries/entities/src/EntityItem.cpp +++ b/libraries/entities/src/EntityItem.cpp @@ -1823,29 +1823,6 @@ void EntityItem::computeCollisionGroupAndFinalMask(int16_t& group, int16_t& mask } uint8_t userMask = getCollisionMask(); - if (userMask & USER_COLLISION_GROUP_MY_AVATAR) { - // if this entity is a descendant of MyAvatar, don't collide with MyAvatar. This avoids the - // "bootstrapping" problem where you can shoot yourself across the room by grabbing something - // and holding it against your own avatar. - QUuid ancestorID = findAncestorOfType(NestableType::Avatar); - if (!ancestorID.isNull() && - (ancestorID == Physics::getSessionUUID() || ancestorID == AVATAR_SELF_ID)) { - userMask &= ~USER_COLLISION_GROUP_MY_AVATAR; - } - } - if (userMask & USER_COLLISION_GROUP_MY_AVATAR) { - // also, don't bootstrap our own avatar with a hold action - QList holdActions = getActionsOfType(ACTION_TYPE_HOLD); - QList::const_iterator i = holdActions.begin(); - while (i != holdActions.end()) { - EntityActionPointer action = *i; - if (action->isMine()) { - userMask &= ~USER_COLLISION_GROUP_MY_AVATAR; - break; - } - i++; - } - } if ((bool)(userMask & USER_COLLISION_GROUP_MY_AVATAR) != (bool)(userMask & USER_COLLISION_GROUP_OTHER_AVATAR)) { @@ -1855,6 +1832,33 @@ void EntityItem::computeCollisionGroupAndFinalMask(int16_t& group, int16_t& mask userMask ^= USER_COLLISION_MASK_AVATARS | ~userMask; } } + + if (userMask & USER_COLLISION_GROUP_MY_AVATAR) { + bool iAmHoldingThis = false; + // if this entity is a descendant of MyAvatar, don't collide with MyAvatar. This avoids the + // "bootstrapping" problem where you can shoot yourself across the room by grabbing something + // and holding it against your own avatar. + QUuid ancestorID = findAncestorOfType(NestableType::Avatar); + if (!ancestorID.isNull() && + (ancestorID == Physics::getSessionUUID() || ancestorID == AVATAR_SELF_ID)) { + iAmHoldingThis = true; + } + // also, don't bootstrap our own avatar with a hold action + QList holdActions = getActionsOfType(ACTION_TYPE_HOLD); + QList::const_iterator i = holdActions.begin(); + while (i != holdActions.end()) { + EntityActionPointer action = *i; + if (action->isMine()) { + iAmHoldingThis = true; + break; + } + i++; + } + + if (iAmHoldingThis) { + userMask &= ~USER_COLLISION_GROUP_MY_AVATAR; + } + } mask = Physics::getDefaultCollisionMask(group) & (int16_t)(userMask); } } From 725917af0d29615b0fdfdbd321f99d3ee9d5dfff Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Mon, 6 Feb 2017 16:49:26 -0800 Subject: [PATCH 021/104] send UserActivityLogger events when tablet is opened or closed --- .../src/UserActivityLoggerScriptingInterface.cpp | 8 ++++++++ .../networking/src/UserActivityLoggerScriptingInterface.h | 2 ++ scripts/system/tablet-ui/tabletUI.js | 5 +++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/UserActivityLoggerScriptingInterface.cpp b/libraries/networking/src/UserActivityLoggerScriptingInterface.cpp index 02d1711230..f38d24c31f 100644 --- a/libraries/networking/src/UserActivityLoggerScriptingInterface.cpp +++ b/libraries/networking/src/UserActivityLoggerScriptingInterface.cpp @@ -16,6 +16,14 @@ void UserActivityLoggerScriptingInterface::enabledEdit() { logAction("enabled_edit"); } +void UserActivityLoggerScriptingInterface::openedTablet() { + logAction("opened_tablet"); +} + +void UserActivityLoggerScriptingInterface::closedTablet() { + logAction("closed_tablet"); +} + void UserActivityLoggerScriptingInterface::openedMarketplace() { logAction("opened_marketplace"); } diff --git a/libraries/networking/src/UserActivityLoggerScriptingInterface.h b/libraries/networking/src/UserActivityLoggerScriptingInterface.h index a202858a1c..b827b2262a 100644 --- a/libraries/networking/src/UserActivityLoggerScriptingInterface.h +++ b/libraries/networking/src/UserActivityLoggerScriptingInterface.h @@ -21,6 +21,8 @@ class UserActivityLoggerScriptingInterface : public QObject, public Dependency { Q_OBJECT public: Q_INVOKABLE void enabledEdit(); + Q_INVOKABLE void openedTablet(); + Q_INVOKABLE void closedTablet(); Q_INVOKABLE void openedMarketplace(); Q_INVOKABLE void toggledAway(bool isAway); Q_INVOKABLE void tutorialProgress(QString stepName, int stepNumber, float secondsToComplete, diff --git a/scripts/system/tablet-ui/tabletUI.js b/scripts/system/tablet-ui/tabletUI.js index eab3d85adc..dc1d71f402 100644 --- a/scripts/system/tablet-ui/tabletUI.js +++ b/scripts/system/tablet-ui/tabletUI.js @@ -12,7 +12,7 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -/* global Script, HMD, WebTablet, UIWebTablet */ +/* global Script, HMD, WebTablet, UIWebTablet, UserActivityLogger, Settings, Entities, Messages, Tablet, Overlays, MyAvatar */ (function() { // BEGIN LOCAL_SCOPE var tabletShown = false; @@ -65,8 +65,10 @@ hideTabletUI(); HMD.closeTablet(); } else if (HMD.showTablet && !tabletShown) { + UserActivityLogger.openedTablet(); showTabletUI(); } else if (!HMD.showTablet && tabletShown) { + UserActivityLogger.closedTablet(); hideTabletUI(); } } @@ -86,7 +88,6 @@ var accumulatedLevel = 0.0; // Note: Might have to tweak the following two based on the rate we're getting the data var AVERAGING_RATIO = 0.05; - var MIC_LEVEL_UPDATE_INTERVAL_MS = 100; // Calculate microphone level with the same scaling equation (log scale, exponentially averaged) in AvatarInputs and pal.js function getMicLevel() { From 6d1d36e0f0a01881b2033052367b690aa8d061ad Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Tue, 7 Feb 2017 10:22:43 -0800 Subject: [PATCH 022/104] fix for bad merge around cauterizedClusterMatrices --- libraries/render-utils/src/MeshPartPayload.cpp | 10 +--------- libraries/render-utils/src/MeshPartPayload.h | 4 +--- libraries/render-utils/src/Model.cpp | 2 +- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/libraries/render-utils/src/MeshPartPayload.cpp b/libraries/render-utils/src/MeshPartPayload.cpp index 17a8c07c39..333371ed76 100644 --- a/libraries/render-utils/src/MeshPartPayload.cpp +++ b/libraries/render-utils/src/MeshPartPayload.cpp @@ -370,22 +370,14 @@ void ModelMeshPartPayload::notifyLocationChanged() { } -void ModelMeshPartPayload::updateTransformForSkinnedMesh(const Transform& transform, - const QVector& clusterMatrices, - const QVector& cauterizedClusterMatrices) { +void ModelMeshPartPayload::updateTransformForSkinnedMesh(const Transform& transform, const QVector& clusterMatrices) { _transform = transform; - _cauterizedTransform = transform; if (clusterMatrices.size() > 0) { _worldBound = _adjustedLocalBound; _worldBound.transform(_transform); if (clusterMatrices.size() == 1) { _transform = _transform.worldTransform(Transform(clusterMatrices[0])); - if (cauterizedClusterMatrices.size() != 0) { - _cauterizedTransform = _cauterizedTransform.worldTransform(Transform(cauterizedClusterMatrices[0])); - } else { - _cauterizedTransform = _transform; - } } } else { _worldBound = _localBound; diff --git a/libraries/render-utils/src/MeshPartPayload.h b/libraries/render-utils/src/MeshPartPayload.h index bdd9d737e9..c585c95025 100644 --- a/libraries/render-utils/src/MeshPartPayload.h +++ b/libraries/render-utils/src/MeshPartPayload.h @@ -90,8 +90,7 @@ public: void notifyLocationChanged() override; void updateTransformForSkinnedMesh(const Transform& transform, - const QVector& clusterMatrices, - const QVector& cauterizedClusterMatrices); + const QVector& clusterMatrices); float computeFadeAlpha() const; @@ -111,7 +110,6 @@ public: Model* _model; - Transform _cauterizedTransform; int _meshIndex; int _shapeID; diff --git a/libraries/render-utils/src/Model.cpp b/libraries/render-utils/src/Model.cpp index 53e57541c1..e1627f2fd6 100644 --- a/libraries/render-utils/src/Model.cpp +++ b/libraries/render-utils/src/Model.cpp @@ -248,7 +248,7 @@ void Model::updateRenderItems() { // update the model transform and bounding box for this render item. const Model::MeshState& state = data._model->_meshStates.at(data._meshIndex); - data.updateTransformForSkinnedMesh(modelTransform, state.clusterMatrices, state.cauterizedClusterMatrices); + data.updateTransformForSkinnedMesh(modelTransform, state.clusterMatrices); } } }); From 8ed69f62fa5bea7a053b082f66de96573de945bc Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 10 Feb 2017 18:22:02 -0800 Subject: [PATCH 023/104] adjust stats so avatar list is at bottom --- assignment-client/src/avatars/AvatarMixer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 61164ee8d7..9bfc2b28e6 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -660,7 +660,7 @@ void AvatarMixer::sendStatsPacket() { avatarsObject[uuidStringWithoutCurlyBraces(node->getUUID())] = avatarStats; }); - statsObject["avatars"] = avatarsObject; + statsObject["z_avatars"] = avatarsObject; ThreadedAssignment::addPacketStatsAndSendStatsPacket(statsObject); _sumListeners = 0; From b48ff6f24f2b1afa811fea6d3f204df1671131c0 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Mon, 13 Feb 2017 18:17:01 -0800 Subject: [PATCH 024/104] add some various debug and stats code to avatarMixer --- assignment-client/src/avatars/AvatarMixer.cpp | 204 +++++++++++++++++- assignment-client/src/avatars/AvatarMixer.h | 20 ++ 2 files changed, 219 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 9bfc2b28e6..261fb1b76e 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -83,10 +83,99 @@ void AvatarMixer::sendIdentityPacket(AvatarMixerClientData* nodeData, const Shar ++_sumIdentityPackets; } +#include +#include + +void AvatarMixer::start() { + auto nodeList = DependencyManager::get(); + + +/**** + // prepare the NodeList + nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer }); + nodeList->linkedDataCreateCallback = [&](Node* node) { getOrCreateClientData(node); }; + + // parse out any AudioMixer settings + { + DomainHandler& domainHandler = nodeList->getDomainHandler(); + const QJsonObject& settingsObject = domainHandler.getSettingsObject(); + parseSettingsObject(settingsObject); + } + + // mix state + unsigned int frame = 1; + auto frameTimestamp = p_high_resolution_clock::now(); +****/ + + while (!_isFinished) { + _loopRate.increment(); + + auto sleepAmount = std::chrono::milliseconds(AVATAR_DATA_SEND_INTERVAL_MSECS); + std::this_thread::sleep_for(sleepAmount); + + /* + auto ticTimer = _ticTiming.timer(); + + { + auto timer = _sleepTiming.timer(); + auto frameDuration = timeFrame(frameTimestamp); + throttle(frameDuration, frame); + } + + auto frameTimer = _frameTiming.timer(); + + nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { + // prepare frames; pop off any new audio from their streams + { + auto prepareTimer = _prepareTiming.timer(); + std::for_each(cbegin, cend, [&](const SharedNodePointer& node) { + _stats.sumStreams += prepareFrame(node, frame); + }); + } + + // mix across slave threads + { + auto mixTimer = _mixTiming.timer(); + _slavePool.mix(cbegin, cend, frame, _throttlingRatio); + } + }); + */ + + // gather stats + /* + _slavePool.each([&](AudioMixerSlave& slave) { + _stats.accumulate(slave.stats); + slave.stats.reset(); + }); + */ + + /* + ++frame; + ++_numStatFrames; + */ + + // play nice with qt event-looping + { + //auto eventsTimer = _eventsTiming.timer(); + + // since we're a while loop we need to yield to qt's event processing + QCoreApplication::processEvents(); + + if (_isFinished) { + // alert qt eventing that this is finished + QCoreApplication::sendPostedEvents(this, QEvent::DeferredDelete); + break; + } + } + } +} + + // NOTE: some additional optimizations to consider. // 1) use the view frustum to cull those avatars that are out of view. Since avatar data doesn't need to be present // if the avatar is not in view or in the keyhole. void AvatarMixer::broadcastAvatarData() { + quint64 startBroadcastAvatarData = usecTimestampNow(); _broadcastRate.increment(); int idleTime = AVATAR_DATA_SEND_INTERVAL_MSECS; @@ -262,6 +351,7 @@ void AvatarMixer::broadcastAvatarData() { // setup a PacketList for the avatarPackets auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); + quint64 startDisplayNameManagement = usecTimestampNow(); if (nodeData->getAvatarSessionDisplayNameMustChange()) { const QString& existingBaseDisplayName = nodeData->getBaseDisplayName(); if (--_sessionDisplayNames[existingBaseDisplayName].second <= 0) { @@ -289,11 +379,18 @@ void AvatarMixer::broadcastAvatarData() { sendIdentityPacket(nodeData, node); // Tell node whose name changed about its new session display name. Others will find out below. qDebug() << "Giving session display name" << sessionDisplayName << "to node with ID" << node->getUUID(); } + quint64 endDisplayNameManagement = usecTimestampNow(); + _displayNameManagementElapsedTime += (endDisplayNameManagement - startDisplayNameManagement); + // this is an AGENT we have received head data from // send back a packet with other active node data to this node nodeList->eachMatchingNode( [&](const SharedNodePointer& otherNode)->bool { + + bool shouldConsider = false; + quint64 startIgnoreCalculation = usecTimestampNow(); + // make sure we have data for this avatar, that it isn't the same node, // and isn't an avatar that the viewing node has ignored // or that has ignored the viewing node @@ -301,7 +398,9 @@ void AvatarMixer::broadcastAvatarData() { || otherNode->getUUID() == node->getUUID() || (node->isIgnoringNodeWithID(otherNode->getUUID()) && !getsIgnoredByMe) || (otherNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { - return false; + + shouldConsider = false; + } else { AvatarMixerClientData* otherData = reinterpret_cast(otherNode->getLinkedData()); AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); @@ -333,20 +432,35 @@ void AvatarMixer::broadcastAvatarData() { // Perform the collision check between the two bounding boxes if (nodeBox.touches(otherNodeBox)) { nodeData->ignoreOther(node, otherNode); - return getsAnyIgnored; + shouldConsider = getsAnyIgnored; } } // Not close enough to ignore - nodeData->removeFromRadiusIgnoringSet(node, otherNode->getUUID()); - return true; + if (shouldConsider) { + nodeData->removeFromRadiusIgnoringSet(node, otherNode->getUUID()); + } + + shouldConsider = true; } + + quint64 endIgnoreCalculation = usecTimestampNow(); + _ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); + + return shouldConsider; }, [&](const SharedNodePointer& otherNode) { + + quint64 startAvatarDataPacking = usecTimestampNow(); + ++numOtherAvatars; AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); MutexTryLocker lock(otherNodeData->getMutex()); if (!lock.isLocked()) { + + // FIXME -- might want to track this lock failed... + quint64 endAvatarDataPacking = usecTimestampNow(); + _avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); return; } @@ -374,6 +488,9 @@ void AvatarMixer::broadcastAvatarData() { if (distanceToAvatar != 0.0f && !getsOutOfView && distribution(generator) > (nodeData->getFullRateDistance() / distanceToAvatar)) { + + quint64 endAvatarDataPacking = usecTimestampNow(); + _avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); return; } @@ -389,6 +506,9 @@ void AvatarMixer::broadcastAvatarData() { // or that somehow we haven't sent if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { ++numAvatarsHeldBack; + + quint64 endAvatarDataPacking = usecTimestampNow(); + _avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); return; } else if (lastSeqFromSender - lastSeqToReceiver > 1) { // this is a skip - we still send the packet but capture the presence of the skip so we see it happening @@ -411,6 +531,8 @@ void AvatarMixer::broadcastAvatarData() { // this throttles the extra data to only be sent every Nth message if (!isInView && getsOutOfView && (lastSeqToReceiver % EXTRA_AVATAR_DATA_FRAME_RATIO > 0)) { + quint64 endAvatarDataPacking = usecTimestampNow(); + _avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); return; } @@ -436,8 +558,13 @@ void AvatarMixer::broadcastAvatarData() { numAvatarDataBytes += avatarPacketList->write(bytes); avatarPacketList->endSegment(); + + quint64 endAvatarDataPacking = usecTimestampNow(); + _avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); }); + quint64 startPacketSending = usecTimestampNow(); + // close the current packet so that we're always sending something avatarPacketList->closeCurrentPacket(true); @@ -457,6 +584,10 @@ void AvatarMixer::broadcastAvatarData() { } else { nodeData->setMaxAvatarDistance(maxAvatarDistanceThisFrame); } + + quint64 endPacketSending = usecTimestampNow(); + _packetSendingElapsedTime += (endPacketSending - startPacketSending); + } ); @@ -501,6 +632,9 @@ void AvatarMixer::broadcastAvatarData() { } #endif + quint64 endBroadcastAvatarData = usecTimestampNow(); + _broadcastAvatarDataElapsedTime += (endBroadcastAvatarData - startBroadcastAvatarData); + } void AvatarMixer::nodeKilled(SharedNodePointer killedNode) { @@ -551,6 +685,7 @@ void AvatarMixer::nodeKilled(SharedNodePointer killedNode) { } void AvatarMixer::handleViewFrustumPacket(QSharedPointer message, SharedNodePointer senderNode) { + auto start = usecTimestampNow(); auto nodeList = DependencyManager::get(); nodeList->getOrCreateLinkedData(senderNode); @@ -560,9 +695,14 @@ void AvatarMixer::handleViewFrustumPacket(QSharedPointer messag nodeData->readViewFrustumPacket(message->getMessage()); } } + + auto end = usecTimestampNow(); + _handleViewFrustumPacketElapsedTime += (end - start); } void AvatarMixer::handleRequestsDomainListDataPacket(QSharedPointer message, SharedNodePointer senderNode) { + auto start = usecTimestampNow(); + auto nodeList = DependencyManager::get(); nodeList->getOrCreateLinkedData(senderNode); @@ -574,14 +714,20 @@ void AvatarMixer::handleRequestsDomainListDataPacket(QSharedPointersetRequestsDomainListData(isRequesting); } } + auto end = usecTimestampNow(); + _handleRequestsDomainListDataPacketElapsedTime += (end - start); } void AvatarMixer::handleAvatarDataPacket(QSharedPointer message, SharedNodePointer senderNode) { + auto start = usecTimestampNow(); auto nodeList = DependencyManager::get(); nodeList->updateNodeWithDataFromPacket(message, senderNode); + auto end = usecTimestampNow(); + _handleAvatarDataPacketElapsedTime += (end - start); } void AvatarMixer::handleAvatarIdentityPacket(QSharedPointer message, SharedNodePointer senderNode) { + auto start = usecTimestampNow(); auto nodeList = DependencyManager::get(); nodeList->getOrCreateLinkedData(senderNode); @@ -605,18 +751,29 @@ void AvatarMixer::handleAvatarIdentityPacket(QSharedPointer mes } } } + auto end = usecTimestampNow(); + _handleAvatarIdentityPacketElapsedTime += (end - start); } void AvatarMixer::handleKillAvatarPacket(QSharedPointer message) { + auto start = usecTimestampNow(); DependencyManager::get()->processKillNode(*message); + auto end = usecTimestampNow(); + _handleKillAvatarPacketElapsedTime += (end - start); } void AvatarMixer::handleNodeIgnoreRequestPacket(QSharedPointer message, SharedNodePointer senderNode) { + auto start = usecTimestampNow(); senderNode->parseIgnoreRequestMessage(message); + auto end = usecTimestampNow(); + _handleNodeIgnoreRequestPacketElapsedTime += (end - start); } void AvatarMixer::handleRadiusIgnoreRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode) { + auto start = usecTimestampNow(); sendingNode->parseIgnoreRadiusRequestMessage(packet); + auto end = usecTimestampNow(); + _handleRadiusIgnoreRequestPacketElapsedTime += (end - start); } void AvatarMixer::sendStatsPacket() { @@ -628,6 +785,33 @@ void AvatarMixer::sendStatsPacket() { statsObject["trailing_sleep_percentage"] = _trailingSleepRatio * 100; statsObject["performance_throttling_ratio"] = _performanceThrottlingRatio; statsObject["broadcast_loop_rate"] = _broadcastRate.rate(); + statsObject["tight_loop_rate"] = _loopRate.rate(); + + // broadcastAvatarDataElapsed timing details... + statsObject["timing_average_a_displayNameManagement"] = (float)_displayNameManagementElapsedTime / (float)_numStatFrames; + statsObject["timing_average_b_ignoreCalculation"] = (float)_ignoreCalculationElapsedTime / (float)_numStatFrames; + statsObject["timing_average_c_avatarDataPacking"] = (float)_avatarDataPackingElapsedTime / (float)_numStatFrames; + statsObject["timing_average_d_packetSending"] = (float)_packetSendingElapsedTime / (float)_numStatFrames; + + statsObject["timing_average_x_total_broadcastAvatarData"] = (float)_broadcastAvatarDataElapsedTime / (float)_numStatFrames; + + + statsObject["timing_average_z_handleViewFrustumPacket"] = (float)_handleViewFrustumPacketElapsedTime / (float)_numStatFrames; + statsObject["timing_average_z_handleAvatarDataPacket"] = (float)_handleAvatarDataPacketElapsedTime / (float)_numStatFrames; + statsObject["timing_average_z_handleAvatarIdentityPacket"] = (float)_handleAvatarIdentityPacketElapsedTime / (float)_numStatFrames; + statsObject["timing_average_z_handleKillAvatarPacket"] = (float)_handleKillAvatarPacketElapsedTime / (float)_numStatFrames; + statsObject["timing_average_z_handleNodeIgnoreRequestPacket"] = (float)_handleNodeIgnoreRequestPacketElapsedTime / (float)_numStatFrames; + statsObject["timing_average_z_handleRadiusIgnoreRequestPacket"] = (float)_handleRadiusIgnoreRequestPacketElapsedTime / (float)_numStatFrames; + statsObject["timing_average_z_handleRequestsDomainListDataPacket"] = (float)_handleRequestsDomainListDataPacketElapsedTime / (float)_numStatFrames; + + _handleViewFrustumPacketElapsedTime = 0; + _handleAvatarDataPacketElapsedTime = 0; + _handleAvatarIdentityPacketElapsedTime = 0; + _handleKillAvatarPacketElapsedTime = 0; + _handleNodeIgnoreRequestPacketElapsedTime = 0; + _handleRadiusIgnoreRequestPacketElapsedTime = 0; + _handleRequestsDomainListDataPacketElapsedTime = 0; + QJsonObject avatarsObject; @@ -666,6 +850,13 @@ void AvatarMixer::sendStatsPacket() { _sumListeners = 0; _sumIdentityPackets = 0; _numStatFrames = 0; + + _broadcastAvatarDataElapsedTime = 0; + _displayNameManagementElapsedTime = 0; + _ignoreCalculationElapsedTime = 0; + _avatarDataPackingElapsedTime = 0; + _packetSendingElapsedTime = 0; + } void AvatarMixer::run() { @@ -675,7 +866,7 @@ void AvatarMixer::run() { DomainHandler& domainHandler = DependencyManager::get()->getDomainHandler(); connect(&domainHandler, &DomainHandler::settingsReceived, this, &AvatarMixer::domainSettingsRequestComplete); connect(&domainHandler, &DomainHandler::settingsReceiveFail, this, &AvatarMixer::domainSettingsRequestFailed); - + ThreadedAssignment::commonInit(AVATAR_MIXER_LOGGING_NAME, NodeType::AvatarMixer); // setup the timer that will be fired on the broadcast thread @@ -687,6 +878,9 @@ void AvatarMixer::run() { // connect appropriate signals and slots connect(_broadcastTimer, &QTimer::timeout, this, &AvatarMixer::broadcastAvatarData, Qt::DirectConnection); connect(&_broadcastThread, SIGNAL(started()), _broadcastTimer, SLOT(start())); + + // start our tight loop... + start(); } void AvatarMixer::domainSettingsRequestComplete() { diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index 5d54622ac9..cb06362b7f 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -45,6 +45,7 @@ private slots: void handleRequestsDomainListDataPacket(QSharedPointer message, SharedNodePointer senderNode); void domainSettingsRequestComplete(); void handlePacketVersionMismatch(PacketType type, const HifiSockAddr& senderSockAddr, const QUuid& senderUUID); + void start(); private: @@ -73,6 +74,25 @@ private: RateCounter<> _broadcastRate; p_high_resolution_clock::time_point _lastDebugMessage; QHash> _sessionDisplayNames; + + quint64 _broadcastAvatarDataElapsedTime { 0 }; // total time spent in broadcastAvatarData since last stats window + quint64 _displayNameManagementElapsedTime { 0 }; // total time spent in broadcastAvatarData/display name management... since last stats window + quint64 _ignoreCalculationElapsedTime { 0 }; + quint64 _avatarDataPackingElapsedTime { 0 }; + quint64 _packetSendingElapsedTime { 0 }; + + + quint64 _handleViewFrustumPacketElapsedTime { 0 }; + quint64 _handleAvatarDataPacketElapsedTime { 0 }; + quint64 _handleAvatarIdentityPacketElapsedTime { 0 }; + quint64 _handleKillAvatarPacketElapsedTime { 0 }; + quint64 _handleNodeIgnoreRequestPacketElapsedTime { 0 }; + quint64 _handleRadiusIgnoreRequestPacketElapsedTime { 0 }; + quint64 _handleRequestsDomainListDataPacketElapsedTime { 0 }; + + + RateCounter<> _loopRate; // this is the rate that the main thread tight loop runs + }; #endif // hifi_AvatarMixer_h From e642f6f96ab78e634d77a98097911e4b3d839d32 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Mon, 13 Feb 2017 18:29:45 -0800 Subject: [PATCH 025/104] add some various debug and stats code to avatarMixer --- assignment-client/src/avatars/AvatarMixer.cpp | 6 ++++++ assignment-client/src/avatars/AvatarMixer.h | 1 + 2 files changed, 7 insertions(+) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 261fb1b76e..87952f3264 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -159,6 +159,7 @@ void AvatarMixer::start() { //auto eventsTimer = _eventsTiming.timer(); // since we're a while loop we need to yield to qt's event processing + auto start = usecTimestampNow(); QCoreApplication::processEvents(); if (_isFinished) { @@ -166,6 +167,8 @@ void AvatarMixer::start() { QCoreApplication::sendPostedEvents(this, QEvent::DeferredDelete); break; } + auto end = usecTimestampNow(); + _processEventsElapsedTime += (end - start); } } } @@ -804,6 +807,8 @@ void AvatarMixer::sendStatsPacket() { statsObject["timing_average_z_handleRadiusIgnoreRequestPacket"] = (float)_handleRadiusIgnoreRequestPacketElapsedTime / (float)_numStatFrames; statsObject["timing_average_z_handleRequestsDomainListDataPacket"] = (float)_handleRequestsDomainListDataPacketElapsedTime / (float)_numStatFrames; + statsObject["timing_average_z_processEvents"] = (float)_processEventsElapsedTime / (float)_numStatFrames; + _handleViewFrustumPacketElapsedTime = 0; _handleAvatarDataPacketElapsedTime = 0; _handleAvatarIdentityPacketElapsedTime = 0; @@ -811,6 +816,7 @@ void AvatarMixer::sendStatsPacket() { _handleNodeIgnoreRequestPacketElapsedTime = 0; _handleRadiusIgnoreRequestPacketElapsedTime = 0; _handleRequestsDomainListDataPacketElapsedTime = 0; + _processEventsElapsedTime = 0; QJsonObject avatarsObject; diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index cb06362b7f..4509b394f3 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -90,6 +90,7 @@ private: quint64 _handleRadiusIgnoreRequestPacketElapsedTime { 0 }; quint64 _handleRequestsDomainListDataPacketElapsedTime { 0 }; + quint64 _processEventsElapsedTime { 0 }; RateCounter<> _loopRate; // this is the rate that the main thread tight loop runs From 3d874f6ad2b4e2c9147e58ea5fb4e2f343509725 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 14 Feb 2017 14:12:22 -0800 Subject: [PATCH 026/104] more hacking --- assignment-client/src/avatars/AvatarMixer.cpp | 99 +++++++------------ assignment-client/src/avatars/AvatarMixer.h | 4 + .../src/avatars/AvatarMixerClientData.cpp | 8 ++ .../src/avatars/AvatarMixerClientData.h | 5 +- 4 files changed, 52 insertions(+), 64 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 87952f3264..0fada86499 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -89,75 +89,29 @@ void AvatarMixer::sendIdentityPacket(AvatarMixerClientData* nodeData, const Shar void AvatarMixer::start() { auto nodeList = DependencyManager::get(); - -/**** - // prepare the NodeList - nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer }); - nodeList->linkedDataCreateCallback = [&](Node* node) { getOrCreateClientData(node); }; - - // parse out any AudioMixer settings - { - DomainHandler& domainHandler = nodeList->getDomainHandler(); - const QJsonObject& settingsObject = domainHandler.getSettingsObject(); - parseSettingsObject(settingsObject); - } - - // mix state - unsigned int frame = 1; - auto frameTimestamp = p_high_resolution_clock::now(); -****/ - while (!_isFinished) { _loopRate.increment(); + // FIXME - we really should sleep for the remainder of what we haven't spent time processing auto sleepAmount = std::chrono::milliseconds(AVATAR_DATA_SEND_INTERVAL_MSECS); std::this_thread::sleep_for(sleepAmount); - /* - auto ticTimer = _ticTiming.timer(); - + // Allow nodes to process any pending/queued packets across our worker threads { - auto timer = _sleepTiming.timer(); - auto frameDuration = timeFrame(frameTimestamp); - throttle(frameDuration, frame); + auto start = usecTimestampNow(); + auto nodeList = DependencyManager::get(); + nodeList->eachNode([](const SharedNodePointer& node) { + auto nodeData = dynamic_cast(node->getLinkedData()); + if (nodeData) { + nodeData->processQueuedAvatarDataPackets(); + } + }); + auto end = usecTimestampNow(); + _processQueuedAvatarDataPacketsElapsedTime += (end - start); } - auto frameTimer = _frameTiming.timer(); - - nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { - // prepare frames; pop off any new audio from their streams - { - auto prepareTimer = _prepareTiming.timer(); - std::for_each(cbegin, cend, [&](const SharedNodePointer& node) { - _stats.sumStreams += prepareFrame(node, frame); - }); - } - - // mix across slave threads - { - auto mixTimer = _mixTiming.timer(); - _slavePool.mix(cbegin, cend, frame, _throttlingRatio); - } - }); - */ - - // gather stats - /* - _slavePool.each([&](AudioMixerSlave& slave) { - _stats.accumulate(slave.stats); - slave.stats.reset(); - }); - */ - - /* - ++frame; - ++_numStatFrames; - */ - // play nice with qt event-looping { - //auto eventsTimer = _eventsTiming.timer(); - // since we're a while loop we need to yield to qt's event processing auto start = usecTimestampNow(); QCoreApplication::processEvents(); @@ -723,8 +677,15 @@ void AvatarMixer::handleRequestsDomainListDataPacket(QSharedPointer message, SharedNodePointer senderNode) { auto start = usecTimestampNow(); + auto nodeList = DependencyManager::get(); - nodeList->updateNodeWithDataFromPacket(message, senderNode); + AvatarMixerClientData* nodeData = dynamic_cast(nodeList->getOrCreateLinkedData(senderNode)); + + if (nodeData) { + QMutexLocker linkedDataLocker(&nodeData->getMutex()); // NOTE: we might be able to safely assume this doesn't need to be locked! + nodeData->queueAvatarDataPacket(message); + } + auto end = usecTimestampNow(); _handleAvatarDataPacketElapsedTime += (end - start); } @@ -780,6 +741,9 @@ void AvatarMixer::handleRadiusIgnoreRequestPacket(QSharedPointer _loopRate; // this is the rate that the main thread tight loop runs + std::unordered_map>> _pendingAvatarDataPackets; + }; #endif // hifi_AvatarMixer_h diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index a7a506e1d8..ffa525bc35 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -16,6 +16,14 @@ #include "AvatarMixerClientData.h" +void AvatarMixerClientData::processQueuedAvatarDataPackets() { + for (auto message : _queuedAvatarDataPackets) { + parseData(*message); + } + _queuedAvatarDataPackets.clear(); +} + + int AvatarMixerClientData::parseData(ReceivedMessage& message) { // pull the sequence number from the data first message.readPrimitive(&_lastReceivedSequenceNumber); diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index aa011f8baf..611af1701d 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -118,9 +118,12 @@ public: return _lastOtherAvatarSentJoints[otherAvatar]; } - + void queueAvatarDataPacket(QSharedPointer message) { _queuedAvatarDataPackets.push_back(message); } + void processQueuedAvatarDataPackets(); private: + std::vector> _queuedAvatarDataPackets; + AvatarSharedPointer _avatar { new AvatarData() }; uint16_t _lastReceivedSequenceNumber { 0 }; From 67e6d654d904a98dba29669153295d6065490d13 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 14 Feb 2017 15:32:32 -0800 Subject: [PATCH 027/104] introducing AvatarMixerSlavePool concept --- assignment-client/src/avatars/AvatarMixer.cpp | 13 ++ assignment-client/src/avatars/AvatarMixer.h | 7 +- .../src/avatars/AvatarMixerSlave.cpp | 44 ++++ .../src/avatars/AvatarMixerSlave.h | 38 ++++ .../src/avatars/AvatarMixerSlavePool.cpp | 190 ++++++++++++++++++ .../src/avatars/AvatarMixerSlavePool.h | 101 ++++++++++ 6 files changed, 392 insertions(+), 1 deletion(-) create mode 100644 assignment-client/src/avatars/AvatarMixerSlave.cpp create mode 100644 assignment-client/src/avatars/AvatarMixerSlave.h create mode 100644 assignment-client/src/avatars/AvatarMixerSlavePool.cpp create mode 100644 assignment-client/src/avatars/AvatarMixerSlavePool.h diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index a422ce7b3b..0590210c0b 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -100,12 +100,23 @@ void AvatarMixer::start() { { auto start = usecTimestampNow(); auto nodeList = DependencyManager::get(); + + /** nodeList->eachNode([](const SharedNodePointer& node) { auto nodeData = dynamic_cast(node->getLinkedData()); if (nodeData) { nodeData->processQueuedAvatarDataPackets(); } }); + **/ + + nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { + // mix across slave threads + { + _slavePool.processIncomingPackets(cbegin, cend); + } + }); + auto end = usecTimestampNow(); _processQueuedAvatarDataPacketsElapsedTime += (end - start); } @@ -745,6 +756,8 @@ void AvatarMixer::sendStatsPacket() { QJsonObject statsObject; + statsObject["threads"] = _slavePool.numThreads(); + statsObject["average_listeners_last_second"] = (float) _sumListeners / (float) _numStatFrames; statsObject["average_identity_packets_per_frame"] = (float) _sumIdentityPackets / (float) _numStatFrames; diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index 7a668c0a7f..93e4087b36 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -15,12 +15,16 @@ #ifndef hifi_AvatarMixer_h #define hifi_AvatarMixer_h +#include + #include #include #include #include "AvatarMixerClientData.h" +#include "AvatarMixerSlavePool.h" + /// Handles assignments of type AvatarMixer - distribution of avatar data to various clients class AvatarMixer : public ThreadedAssignment { Q_OBJECT @@ -96,7 +100,8 @@ private: RateCounter<> _loopRate; // this is the rate that the main thread tight loop runs - std::unordered_map>> _pendingAvatarDataPackets; + + AvatarMixerSlavePool _slavePool; }; diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp new file mode 100644 index 0000000000..b7676dac58 --- /dev/null +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -0,0 +1,44 @@ +// +// AvatarMixerSlave.cpp +// assignment-client/src/avatar +// +// Created by Brad Hefta-Gaub on 2/14/2017. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "AvatarMixer.h" +#include "AvatarMixerClientData.h" +#include "AvatarMixerSlave.h" + + +void AvatarMixerSlave::configure(ConstIter begin, ConstIter end) { + _begin = begin; + _end = end; +} + +void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { + auto nodeData = dynamic_cast(node->getLinkedData()); + if (nodeData) { + nodeData->processQueuedAvatarDataPackets(); + } +} + diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h new file mode 100644 index 0000000000..6bd2f5acd4 --- /dev/null +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -0,0 +1,38 @@ +// +// AvatarMixerSlave.h +// assignment-client/src/avatar +// +// Created by Brad Hefta-Gaub on 2/14/2017. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AvatarMixerSlave_h +#define hifi_AvatarMixerSlave_h + +/* +#include +#include +#include +#include +#include +#include +*/ + +class AvatarMixerSlave { +public: + using ConstIter = NodeList::const_iterator; + + void configure(ConstIter begin, ConstIter end); + + void processIncomingPackets(const SharedNodePointer& node); + +private: + // frame state + ConstIter _begin; + ConstIter _end; +}; + +#endif // hifi_AvatarMixerSlave_h diff --git a/assignment-client/src/avatars/AvatarMixerSlavePool.cpp b/assignment-client/src/avatars/AvatarMixerSlavePool.cpp new file mode 100644 index 0000000000..e50a8475a0 --- /dev/null +++ b/assignment-client/src/avatars/AvatarMixerSlavePool.cpp @@ -0,0 +1,190 @@ +// +// AvatarMixerSlavePool.cpp +// assignment-client/src/avatar +// +// Created by Brad Hefta-Gaub on 2/14/2017. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include +#include + +#include "AvatarMixerSlavePool.h" + +void AvatarMixerSlaveThread::run() { + while (true) { + wait(); + + // iterate over all available nodes + SharedNodePointer node; + while (try_pop(node)) { + processIncomingPackets(node); + } + + bool stopping = _stop; + notify(stopping); + if (stopping) { + return; + } + } +} + +void AvatarMixerSlaveThread::wait() { + { + Lock lock(_pool._mutex); + _pool._slaveCondition.wait(lock, [&] { + assert(_pool._numStarted <= _pool._numThreads); + return _pool._numStarted != _pool._numThreads; + }); + ++_pool._numStarted; + } + configure(_pool._begin, _pool._end); +} + +void AvatarMixerSlaveThread::notify(bool stopping) { + { + Lock lock(_pool._mutex); + assert(_pool._numFinished < _pool._numThreads); + ++_pool._numFinished; + if (stopping) { + ++_pool._numStopped; + } + } + _pool._poolCondition.notify_one(); +} + +bool AvatarMixerSlaveThread::try_pop(SharedNodePointer& node) { + return _pool._queue.try_pop(node); +} + +#ifdef AVATAR_SINGLE_THREADED +static AvatarMixerSlave slave; +#endif + +void AvatarMixerSlavePool::processIncomingPackets(ConstIter begin, ConstIter end) { + _begin = begin; + _end = end; + + // ???? + //_frame = frame; + //_throttlingRatio = throttlingRatio; + +#ifdef AVATAR_SINGLE_THREADED + slave.configure(_begin, _end, frame, throttlingRatio); + std::for_each(begin, end, [&](const SharedNodePointer& node) { + slave.processIncomingPackets(node); + }); +#else + // fill the queue + std::for_each(_begin, _end, [&](const SharedNodePointer& node) { + _queue.emplace(node); + }); + + { + Lock lock(_mutex); + + // start the job + _numStarted = _numFinished = 0; + _slaveCondition.notify_all(); + + // wait + _poolCondition.wait(lock, [&] { + assert(_numFinished <= _numThreads); + return _numFinished == _numThreads; + }); + + assert(_numStarted == _numThreads); + } + + assert(_queue.empty()); +#endif +} + +void AvatarMixerSlavePool::each(std::function functor) { +#ifdef AVATAR_SINGLE_THREADED + functor(slave); +#else + for (auto& slave : _slaves) { + functor(*slave.get()); + } +#endif +} + +void AvatarMixerSlavePool::setNumThreads(int numThreads) { + // clamp to allowed size + { + int maxThreads = QThread::idealThreadCount(); + if (maxThreads == -1) { + // idealThreadCount returns -1 if cores cannot be detected + static const int MAX_THREADS_IF_UNKNOWN = 4; + maxThreads = MAX_THREADS_IF_UNKNOWN; + } + + int clampedThreads = std::min(std::max(1, numThreads), maxThreads); + if (clampedThreads != numThreads) { + qWarning("%s: clamped to %d (was %d)", __FUNCTION__, clampedThreads, numThreads); + numThreads = clampedThreads; + } + } + + resize(numThreads); +} + +void AvatarMixerSlavePool::resize(int numThreads) { + assert(_numThreads == (int)_slaves.size()); + +#ifdef AVATAR_SINGLE_THREADED + qDebug("%s: running single threaded", __FUNCTION__, numThreads); +#else + qDebug("%s: set %d threads (was %d)", __FUNCTION__, numThreads, _numThreads); + + Lock lock(_mutex); + + if (numThreads > _numThreads) { + // start new slaves + for (int i = 0; i < numThreads - _numThreads; ++i) { + auto slave = new AvatarMixerSlaveThread(*this); + slave->start(); + _slaves.emplace_back(slave); + } + } else if (numThreads < _numThreads) { + auto extraBegin = _slaves.begin() + numThreads; + + // mark slaves to stop... + auto slave = extraBegin; + while (slave != _slaves.end()) { + (*slave)->_stop = true; + ++slave; + } + + // ...cycle them until they do stop... + _numStopped = 0; + while (_numStopped != (_numThreads - numThreads)) { + _numStarted = _numFinished = _numStopped; + _slaveCondition.notify_all(); + _poolCondition.wait(lock, [&] { + assert(_numFinished <= _numThreads); + return _numFinished == _numThreads; + }); + } + + // ...wait for threads to finish... + slave = extraBegin; + while (slave != _slaves.end()) { + QThread* thread = reinterpret_cast(slave->get()); + static const int MAX_THREAD_WAIT_TIME = 10; + thread->wait(MAX_THREAD_WAIT_TIME); + ++slave; + } + + // ...and erase them + _slaves.erase(extraBegin, _slaves.end()); + } + + _numThreads = _numStarted = _numFinished = numThreads; + assert(_numThreads == (int)_slaves.size()); +#endif +} diff --git a/assignment-client/src/avatars/AvatarMixerSlavePool.h b/assignment-client/src/avatars/AvatarMixerSlavePool.h new file mode 100644 index 0000000000..ae4433d1c0 --- /dev/null +++ b/assignment-client/src/avatars/AvatarMixerSlavePool.h @@ -0,0 +1,101 @@ +// +// AvatarMixerSlavePool.h +// assignment-client/src/avatar +// +// Created by Brad Hefta-Gaub on 2/14/2017. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AvatarMixerSlavePool_h +#define hifi_AvatarMixerSlavePool_h + +#include +#include +#include + +#include + +#include + +#include + +#include "AvatarMixerSlave.h" + +class AvatarMixerSlavePool; + +class AvatarMixerSlaveThread : public QThread, public AvatarMixerSlave { + Q_OBJECT + using ConstIter = NodeList::const_iterator; + using Mutex = std::mutex; + using Lock = std::unique_lock; + +public: + AvatarMixerSlaveThread(AvatarMixerSlavePool& pool) : _pool(pool) {} + + void run() override final; + +private: + friend class AvatarMixerSlavePool; + + void wait(); + void notify(bool stopping); + bool try_pop(SharedNodePointer& node); + + AvatarMixerSlavePool& _pool; + bool _stop { false }; +}; + +// Slave pool for audio mixers +// AvatarMixerSlavePool is not thread-safe! It should be instantiated and used from a single thread. +class AvatarMixerSlavePool { + using Queue = tbb::concurrent_queue; + using Mutex = std::mutex; + using Lock = std::unique_lock; + using ConditionVariable = std::condition_variable; + +public: + using ConstIter = NodeList::const_iterator; + + AvatarMixerSlavePool(int numThreads = QThread::idealThreadCount()) { setNumThreads(numThreads); } + ~AvatarMixerSlavePool() { resize(0); } + + // iterate over all slaves + void each(std::function functor); + + void setNumThreads(int numThreads); + int numThreads() { return _numThreads; } + + // Jobs the slave pool can do... + void processIncomingPackets(ConstIter begin, ConstIter end); + + +private: + void resize(int numThreads); + + std::vector> _slaves; + + friend void AvatarMixerSlaveThread::wait(); + friend void AvatarMixerSlaveThread::notify(bool stopping); + friend bool AvatarMixerSlaveThread::try_pop(SharedNodePointer& node); + + // synchronization state + Mutex _mutex; + ConditionVariable _slaveCondition; + ConditionVariable _poolCondition; + int _numThreads { 0 }; + int _numStarted { 0 }; // guarded by _mutex + int _numFinished { 0 }; // guarded by _mutex + int _numStopped { 0 }; // guarded by _mutex + + // frame state + Queue _queue; + unsigned int _frame { 0 }; + float _throttlingRatio { 0.0f }; + ConstIter _begin; + ConstIter _end; +}; + +#endif // hifi_AvatarMixerSlavePool_h From 4f655bba3ff5b4ea49f11c97cb69bbf61d978c31 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 15 Feb 2017 09:29:45 -0800 Subject: [PATCH 028/104] more hacking --- assignment-client/src/Agent.cpp | 12 ++- assignment-client/src/avatars/AvatarMixer.cpp | 85 ++++++++----------- assignment-client/src/avatars/AvatarMixer.h | 6 +- .../src/avatars/AvatarMixerClientData.cpp | 38 ++++++++- .../src/avatars/AvatarMixerClientData.h | 10 ++- .../src/avatars/AvatarMixerSlave.cpp | 2 +- .../src/avatars/AvatarMixerSlavePool.h | 2 - 7 files changed, 93 insertions(+), 62 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index bea677aeb6..806608cd5f 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -434,8 +434,16 @@ void Agent::executeScript() { connect(&_avatarAudioTimerThread, &QThread::finished, audioTimerWorker, &QObject::deleteLater); _avatarAudioTimerThread.start(); - // 60Hz timer for avatar - QObject::connect(_scriptEngine.get(), &ScriptEngine::update, this, &Agent::processAgentAvatar); + // Agents should run at 45hz + static const int AVATAR_DATA_HZ = 45; + static const int AVATAR_DATA_IN_MSECS = MSECS_PER_SECOND / AVATAR_DATA_HZ; + QTimer* avatarDataTimer = new QTimer(this); + connect(avatarDataTimer, &QTimer::timeout, this, &Agent::processAgentAvatar); + avatarDataTimer->setSingleShot(false); + avatarDataTimer->setInterval(AVATAR_DATA_IN_MSECS); + avatarDataTimer->setTimerType(Qt::PreciseTimer); + avatarDataTimer->start(); + _scriptEngine->run(); Frame::clearFrameHandler(AUDIO_FRAME_TYPE); diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 0590210c0b..1405e40f1b 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -44,8 +44,9 @@ AvatarMixer::AvatarMixer(ReceivedMessage& message) : connect(DependencyManager::get().data(), &NodeList::nodeKilled, this, &AvatarMixer::nodeKilled); auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); + packetReceiver.registerListener(PacketType::AvatarData, this, "queueIncomingPacket"); + packetReceiver.registerListener(PacketType::ViewFrustum, this, "handleViewFrustumPacket"); - packetReceiver.registerListener(PacketType::AvatarData, this, "handleAvatarDataPacket"); packetReceiver.registerListener(PacketType::AvatarIdentity, this, "handleAvatarIdentityPacket"); packetReceiver.registerListener(PacketType::KillAvatar, this, "handleKillAvatarPacket"); packetReceiver.registerListener(PacketType::NodeIgnoreRequest, this, "handleNodeIgnoreRequestPacket"); @@ -56,6 +57,14 @@ AvatarMixer::AvatarMixer(ReceivedMessage& message) : connect(nodeList.data(), &NodeList::packetVersionMismatch, this, &AvatarMixer::handlePacketVersionMismatch); } +void AvatarMixer::queueIncomingPacket(QSharedPointer message, SharedNodePointer node) { + auto start = usecTimestampNow(); + getOrCreateClientData(node)->queuePacket(message, node); + auto end = usecTimestampNow(); + _queueIncomingPacketElapsedTime += (end - start); +} + + AvatarMixer::~AvatarMixer() { if (_broadcastTimer) { _broadcastTimer->deleteLater(); @@ -87,8 +96,11 @@ void AvatarMixer::sendIdentityPacket(AvatarMixerClientData* nodeData, const Shar #include void AvatarMixer::start() { + auto nodeList = DependencyManager::get(); + //_slavePool.setNumThreads(1); // grins + while (!_isFinished) { _loopRate.increment(); @@ -99,34 +111,21 @@ void AvatarMixer::start() { // Allow nodes to process any pending/queued packets across our worker threads { auto start = usecTimestampNow(); - auto nodeList = DependencyManager::get(); - - /** - nodeList->eachNode([](const SharedNodePointer& node) { - auto nodeData = dynamic_cast(node->getLinkedData()); - if (nodeData) { - nodeData->processQueuedAvatarDataPackets(); - } - }); - **/ - nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { - // mix across slave threads - { - _slavePool.processIncomingPackets(cbegin, cend); - } + _slavePool.processIncomingPackets(cbegin, cend); }); - auto end = usecTimestampNow(); _processQueuedAvatarDataPacketsElapsedTime += (end - start); } + + + // play nice with qt event-looping { // since we're a while loop we need to yield to qt's event processing auto start = usecTimestampNow(); QCoreApplication::processEvents(); - if (_isFinished) { // alert qt eventing that this is finished QCoreApplication::sendPostedEvents(this, QEvent::DeferredDelete); @@ -654,8 +653,7 @@ void AvatarMixer::nodeKilled(SharedNodePointer killedNode) { void AvatarMixer::handleViewFrustumPacket(QSharedPointer message, SharedNodePointer senderNode) { auto start = usecTimestampNow(); - auto nodeList = DependencyManager::get(); - nodeList->getOrCreateLinkedData(senderNode); + getOrCreateClientData(senderNode); if (senderNode->getLinkedData()) { AvatarMixerClientData* nodeData = dynamic_cast(senderNode->getLinkedData()); @@ -671,8 +669,7 @@ void AvatarMixer::handleViewFrustumPacket(QSharedPointer messag void AvatarMixer::handleRequestsDomainListDataPacket(QSharedPointer message, SharedNodePointer senderNode) { auto start = usecTimestampNow(); - auto nodeList = DependencyManager::get(); - nodeList->getOrCreateLinkedData(senderNode); + getOrCreateClientData(senderNode); if (senderNode->getLinkedData()) { AvatarMixerClientData* nodeData = dynamic_cast(senderNode->getLinkedData()); @@ -686,25 +683,10 @@ void AvatarMixer::handleRequestsDomainListDataPacket(QSharedPointer message, SharedNodePointer senderNode) { - auto start = usecTimestampNow(); - - auto nodeList = DependencyManager::get(); - AvatarMixerClientData* nodeData = dynamic_cast(nodeList->getOrCreateLinkedData(senderNode)); - - if (nodeData) { - QMutexLocker linkedDataLocker(&nodeData->getMutex()); // NOTE: we might be able to safely assume this doesn't need to be locked! - nodeData->queueAvatarDataPacket(message); - } - - auto end = usecTimestampNow(); - _handleAvatarDataPacketElapsedTime += (end - start); -} - void AvatarMixer::handleAvatarIdentityPacket(QSharedPointer message, SharedNodePointer senderNode) { auto start = usecTimestampNow(); auto nodeList = DependencyManager::get(); - nodeList->getOrCreateLinkedData(senderNode); + getOrCreateClientData(senderNode); if (senderNode->getLinkedData()) { AvatarMixerClientData* nodeData = dynamic_cast(senderNode->getLinkedData()); @@ -777,6 +759,8 @@ void AvatarMixer::sendStatsPacket() { statsObject["timing_average_y_processEvents"] = (float)_processEventsElapsedTime / (float)_numStatFrames; + statsObject["timing_average_y_queueIncomingPacket"] = (float)_queueIncomingPacketElapsedTime / (float)_numStatFrames; + statsObject["timing_average_z_handleAvatarDataPacket"] = (float)_handleAvatarDataPacketElapsedTime / (float)_numStatFrames; statsObject["timing_average_z_handleAvatarIdentityPacket"] = (float)_handleAvatarIdentityPacketElapsedTime / (float)_numStatFrames; @@ -799,6 +783,7 @@ void AvatarMixer::sendStatsPacket() { _handleRadiusIgnoreRequestPacketElapsedTime = 0; _handleRequestsDomainListDataPacketElapsedTime = 0; _processEventsElapsedTime = 0; + _queueIncomingPacketElapsedTime = 0; _processQueuedAvatarDataPacketsElapsedTime = 0; QJsonObject avatarsObject; @@ -875,23 +860,25 @@ void AvatarMixer::run() { start(); } +AvatarMixerClientData* AvatarMixer::getOrCreateClientData(SharedNodePointer node) { + auto clientData = dynamic_cast(node->getLinkedData()); + + if (!clientData) { + node->setLinkedData(std::unique_ptr { new AvatarMixerClientData(node->getUUID()) }); + clientData = dynamic_cast(node->getLinkedData()); + clientData->getAvatar().setDomainMinimumScale(_domainMinimumScale); + clientData->getAvatar().setDomainMaximumScale(_domainMaximumScale); + } + + return clientData; +} + void AvatarMixer::domainSettingsRequestComplete() { auto nodeList = DependencyManager::get(); nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer }); // parse the settings to pull out the values we need parseDomainServerSettings(nodeList->getDomainHandler().getSettingsObject()); - - float domainMinimumScale = _domainMinimumScale; - float domainMaximumScale = _domainMaximumScale; - - nodeList->linkedDataCreateCallback = [domainMinimumScale, domainMaximumScale] (Node* node) { - auto clientData = std::unique_ptr { new AvatarMixerClientData(node->getUUID()) }; - clientData->getAvatar().setDomainMinimumScale(domainMinimumScale); - clientData->getAvatar().setDomainMaximumScale(domainMaximumScale); - - node->setLinkedData(std::move(clientData)); - }; // start the broadcastThread _broadcastThread.start(); diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index 93e4087b36..b1d1dbef0b 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -40,8 +40,9 @@ public slots: void sendStatsPacket() override; private slots: + void queueIncomingPacket(QSharedPointer message, SharedNodePointer node); void handleViewFrustumPacket(QSharedPointer message, SharedNodePointer senderNode); - void handleAvatarDataPacket(QSharedPointer message, SharedNodePointer senderNode); + //void handleAvatarDataPacket(QSharedPointer message, SharedNodePointer senderNode); void handleAvatarIdentityPacket(QSharedPointer message, SharedNodePointer senderNode); void handleKillAvatarPacket(QSharedPointer message); void handleNodeIgnoreRequestPacket(QSharedPointer message, SharedNodePointer senderNode); @@ -53,6 +54,8 @@ private slots: private: + AvatarMixerClientData* getOrCreateClientData(SharedNodePointer node); + void broadcastAvatarData(); void parseDomainServerSettings(const QJsonObject& domainSettings); void sendIdentityPacket(AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode); @@ -97,6 +100,7 @@ private: quint64 _processEventsElapsedTime { 0 }; quint64 _sendStatsElapsedTime { 0 }; + quint64 _queueIncomingPacketElapsedTime { 0 }; RateCounter<> _loopRate; // this is the rate that the main thread tight loop runs diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index ffa525bc35..5ba346ade9 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -16,13 +16,43 @@ #include "AvatarMixerClientData.h" -void AvatarMixerClientData::processQueuedAvatarDataPackets() { - for (auto message : _queuedAvatarDataPackets) { - parseData(*message); + + +void AvatarMixerClientData::queuePacket(QSharedPointer message, SharedNodePointer node) { + if (!_packetQueue.node) { + _packetQueue.node = node; } - _queuedAvatarDataPackets.clear(); + _packetQueue.push(message); } +// +// packetReceiver.registerListener(PacketType::ViewFrustum, this, "handleViewFrustumPacket"); +// packetReceiver.registerListener(PacketType::AvatarData, this, "handleAvatarDataPacket"); +// packetReceiver.registerListener(PacketType::AvatarIdentity, this, "handleAvatarIdentityPacket"); +// packetReceiver.registerListener(PacketType::KillAvatar, this, "handleKillAvatarPacket"); +// packetReceiver.registerListener(PacketType::NodeIgnoreRequest, this, "handleNodeIgnoreRequestPacket"); +// packetReceiver.registerListener(PacketType::RadiusIgnoreRequest, this, "handleRadiusIgnoreRequestPacket"); +// packetReceiver.registerListener(PacketType::RequestsDomainListData, this, "handleRequestsDomainListDataPacket"); + +void AvatarMixerClientData::processPackets() { + SharedNodePointer node = _packetQueue.node; + assert(_packetQueue.empty() || node); + _packetQueue.node.clear(); + + while (!_packetQueue.empty()) { + auto& packet = _packetQueue.back(); + + switch (packet->getType()) { + case PacketType::AvatarData: + parseData(*packet); + break; + default: + Q_UNREACHABLE(); + } + _packetQueue.pop(); + } + assert(_packetQueue.empty()); +} int AvatarMixerClientData::parseData(ReceivedMessage& message) { // pull the sequence number from the data first diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index 611af1701d..f43df28052 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -118,11 +119,14 @@ public: return _lastOtherAvatarSentJoints[otherAvatar]; } - void queueAvatarDataPacket(QSharedPointer message) { _queuedAvatarDataPackets.push_back(message); } - void processQueuedAvatarDataPackets(); + void queuePacket(QSharedPointer message, SharedNodePointer node); + void processPackets(); private: - std::vector> _queuedAvatarDataPackets; + struct PacketQueue : public std::queue> { + QWeakPointer node; + }; + PacketQueue _packetQueue; AvatarSharedPointer _avatar { new AvatarData() }; diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index b7676dac58..44f2060b52 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -38,7 +38,7 @@ void AvatarMixerSlave::configure(ConstIter begin, ConstIter end) { void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { auto nodeData = dynamic_cast(node->getLinkedData()); if (nodeData) { - nodeData->processQueuedAvatarDataPackets(); + nodeData->processPackets(); } } diff --git a/assignment-client/src/avatars/AvatarMixerSlavePool.h b/assignment-client/src/avatars/AvatarMixerSlavePool.h index ae4433d1c0..9865f897b9 100644 --- a/assignment-client/src/avatars/AvatarMixerSlavePool.h +++ b/assignment-client/src/avatars/AvatarMixerSlavePool.h @@ -92,8 +92,6 @@ private: // frame state Queue _queue; - unsigned int _frame { 0 }; - float _throttlingRatio { 0.0f }; ConstIter _begin; ConstIter _end; }; From cddb72bbd77772374435309c03474b6120654bb7 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 15 Feb 2017 12:11:17 -0800 Subject: [PATCH 029/104] clean up stats, add slave stats --- assignment-client/src/avatars/AvatarMixer.cpp | 48 ++++++++++++------- assignment-client/src/avatars/AvatarMixer.h | 1 + .../src/avatars/AvatarMixerClientData.cpp | 7 ++- .../src/avatars/AvatarMixerClientData.h | 2 +- .../src/avatars/AvatarMixerSlave.cpp | 23 +++++++-- .../src/avatars/AvatarMixerSlave.h | 6 +++ 6 files changed, 63 insertions(+), 24 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 1405e40f1b..94e78968bb 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -102,6 +102,7 @@ void AvatarMixer::start() { //_slavePool.setNumThreads(1); // grins while (!_isFinished) { + _numTightLoopFrames++; _loopRate.increment(); // FIXME - we really should sleep for the remainder of what we haven't spent time processing @@ -754,24 +755,19 @@ void AvatarMixer::sendStatsPacket() { statsObject["timing_average_b_ignoreCalculation"] = (float)_ignoreCalculationElapsedTime / (float)_numStatFrames; statsObject["timing_average_c_avatarDataPacking"] = (float)_avatarDataPackingElapsedTime / (float)_numStatFrames; statsObject["timing_average_d_packetSending"] = (float)_packetSendingElapsedTime / (float)_numStatFrames; - statsObject["timing_average_e_total_broadcastAvatarData"] = (float)_broadcastAvatarDataElapsedTime / (float)_numStatFrames; - - statsObject["timing_average_y_processEvents"] = (float)_processEventsElapsedTime / (float)_numStatFrames; - statsObject["timing_average_y_queueIncomingPacket"] = (float)_queueIncomingPacketElapsedTime / (float)_numStatFrames; - - - statsObject["timing_average_z_handleAvatarDataPacket"] = (float)_handleAvatarDataPacketElapsedTime / (float)_numStatFrames; - statsObject["timing_average_z_handleAvatarIdentityPacket"] = (float)_handleAvatarIdentityPacketElapsedTime / (float)_numStatFrames; - statsObject["timing_average_z_handleKillAvatarPacket"] = (float)_handleKillAvatarPacketElapsedTime / (float)_numStatFrames; - statsObject["timing_average_z_handleNodeIgnoreRequestPacket"] = (float)_handleNodeIgnoreRequestPacketElapsedTime / (float)_numStatFrames; - statsObject["timing_average_z_handleRadiusIgnoreRequestPacket"] = (float)_handleRadiusIgnoreRequestPacketElapsedTime / (float)_numStatFrames; - statsObject["timing_average_z_handleRequestsDomainListDataPacket"] = (float)_handleRequestsDomainListDataPacketElapsedTime / (float)_numStatFrames; - statsObject["timing_average_z_handleViewFrustumPacket"] = (float)_handleViewFrustumPacketElapsedTime / (float)_numStatFrames; - - statsObject["timing_average_z_processQueuedAvatarDataPackets"] = (float)_processQueuedAvatarDataPacketsElapsedTime / (float)_numStatFrames; - + // this things all occur on the frequency of the tight loop + statsObject["timing_average_y_processEvents"] = (float)_processEventsElapsedTime / (float)_numTightLoopFrames; + statsObject["timing_average_y_queueIncomingPacket"] = (float)_queueIncomingPacketElapsedTime / (float)_numTightLoopFrames; + statsObject["timing_average_z_handleAvatarDataPacket"] = (float)_handleAvatarDataPacketElapsedTime / (float)_numTightLoopFrames; + statsObject["timing_average_z_handleAvatarIdentityPacket"] = (float)_handleAvatarIdentityPacketElapsedTime / (float)_numTightLoopFrames; + statsObject["timing_average_z_handleKillAvatarPacket"] = (float)_handleKillAvatarPacketElapsedTime / (float)_numTightLoopFrames; + statsObject["timing_average_z_handleNodeIgnoreRequestPacket"] = (float)_handleNodeIgnoreRequestPacketElapsedTime / (float)_numTightLoopFrames; + statsObject["timing_average_z_handleRadiusIgnoreRequestPacket"] = (float)_handleRadiusIgnoreRequestPacketElapsedTime / (float)_numTightLoopFrames; + statsObject["timing_average_z_handleRequestsDomainListDataPacket"] = (float)_handleRequestsDomainListDataPacketElapsedTime / (float)_numTightLoopFrames; + statsObject["timing_average_z_handleViewFrustumPacket"] = (float)_handleViewFrustumPacketElapsedTime / (float)_numTightLoopFrames; + statsObject["timing_average_z_processQueuedAvatarDataPackets"] = (float)_processQueuedAvatarDataPacketsElapsedTime / (float)_numTightLoopFrames; statsObject["timing_sendStats"] = (float)_sendStatsElapsedTime; @@ -786,8 +782,24 @@ void AvatarMixer::sendStatsPacket() { _queueIncomingPacketElapsedTime = 0; _processQueuedAvatarDataPacketsElapsedTime = 0; - QJsonObject avatarsObject; + QJsonObject slavesObject; + // gather stats + int slaveNumber = 1; + _slavePool.each([&](AvatarMixerSlave& slave) { + QJsonObject slaveObject; + int nodesProcessed, packetsProcessed; + quint64 processIncomingPacketsElapsedTime; + slave.harvestStats(nodesProcessed, packetsProcessed, processIncomingPacketsElapsedTime); + slaveObject["nodesProcessed"] = nodesProcessed; + slaveObject["packetsProcessed"] = packetsProcessed; + slaveObject["timing_average_processIncomingPackets"] = (float)processIncomingPacketsElapsedTime / (float)_numTightLoopFrames; + slavesObject[QString::number(slaveNumber)] = slaveObject; + slaveNumber++; + }); + statsObject["timing_slaves"] = slavesObject; + + QJsonObject avatarsObject; auto nodeList = DependencyManager::get(); // add stats for each listerner nodeList->eachNode([&](const SharedNodePointer& node) { @@ -818,11 +830,13 @@ void AvatarMixer::sendStatsPacket() { }); statsObject["z_avatars"] = avatarsObject; + ThreadedAssignment::addPacketStatsAndSendStatsPacket(statsObject); _sumListeners = 0; _sumIdentityPackets = 0; _numStatFrames = 0; + _numTightLoopFrames = 0; _broadcastAvatarDataElapsedTime = 0; _displayNameManagementElapsedTime = 0; diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index b1d1dbef0b..c8d1516e0d 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -69,6 +69,7 @@ private: int _sumListeners { 0 }; int _numStatFrames { 0 }; + std::atomic _numTightLoopFrames; int _sumIdentityPackets { 0 }; float _maxKbpsPerNode = 0.0f; diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index 5ba346ade9..09e3a6a87a 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -34,7 +34,8 @@ void AvatarMixerClientData::queuePacket(QSharedPointer message, // packetReceiver.registerListener(PacketType::RadiusIgnoreRequest, this, "handleRadiusIgnoreRequestPacket"); // packetReceiver.registerListener(PacketType::RequestsDomainListData, this, "handleRequestsDomainListDataPacket"); -void AvatarMixerClientData::processPackets() { +int AvatarMixerClientData::processPackets() { + int packetsProcessed = 0; SharedNodePointer node = _packetQueue.node; assert(_packetQueue.empty() || node); _packetQueue.node.clear(); @@ -42,6 +43,8 @@ void AvatarMixerClientData::processPackets() { while (!_packetQueue.empty()) { auto& packet = _packetQueue.back(); + packetsProcessed++; + switch (packet->getType()) { case PacketType::AvatarData: parseData(*packet); @@ -52,6 +55,8 @@ void AvatarMixerClientData::processPackets() { _packetQueue.pop(); } assert(_packetQueue.empty()); + + return packetsProcessed; } int AvatarMixerClientData::parseData(ReceivedMessage& message) { diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index f43df28052..c69599cc26 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -120,7 +120,7 @@ public: } void queuePacket(QSharedPointer message, SharedNodePointer node); - void processPackets(); + int processPackets(); // returns number of packets processed private: struct PacketQueue : public std::queue> { diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 44f2060b52..06d21b51b9 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -35,10 +35,23 @@ void AvatarMixerSlave::configure(ConstIter begin, ConstIter end) { _end = end; } -void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { - auto nodeData = dynamic_cast(node->getLinkedData()); - if (nodeData) { - nodeData->processPackets(); - } +void AvatarMixerSlave::harvestStats(int& nodesProcessed, int& packetsProcessed, quint64& processIncomingPacketsElapsedTime) { + nodesProcessed = _nodesProcessed; + packetsProcessed = _packetsProcessed; + processIncomingPacketsElapsedTime = _processIncomingPacketsElapsedTime; + _packetsProcessed = _nodesProcessed = 0; + _processIncomingPacketsElapsedTime = 0; +} + + +void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { + auto start = usecTimestampNow(); + auto nodeData = dynamic_cast(node->getLinkedData()); + if (nodeData) { + _nodesProcessed++; + _packetsProcessed += nodeData->processPackets(); + } + auto end = usecTimestampNow(); + _processIncomingPacketsElapsedTime += (end - start); } diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index 6bd2f5acd4..1abba8d46c 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -29,10 +29,16 @@ public: void processIncomingPackets(const SharedNodePointer& node); + void harvestStats(int& nodesProcessed, int& packetsProcessed, quint64& processIncomingPacketsElapsedTime); + private: // frame state ConstIter _begin; ConstIter _end; + + int _nodesProcessed { 0 }; + int _packetsProcessed { 0 }; + quint64 _processIncomingPacketsElapsedTime { 0 }; }; #endif // hifi_AvatarMixerSlave_h From 00086fcc061a2c1b095b8d0223e8b8911f705ef2 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 15 Feb 2017 13:00:30 -0800 Subject: [PATCH 030/104] wire up thread count to settings, tweak stats --- assignment-client/src/avatars/AvatarMixer.cpp | 31 ++++++++++++++++--- .../resources/describe-settings.json | 16 ++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 94e78968bb..73c9aef8e1 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -785,14 +785,17 @@ void AvatarMixer::sendStatsPacket() { QJsonObject slavesObject; // gather stats int slaveNumber = 1; + int tightLoopFrames = _numTightLoopFrames; + int tenTimesPerFrame = tightLoopFrames * 10; _slavePool.each([&](AvatarMixerSlave& slave) { QJsonObject slaveObject; int nodesProcessed, packetsProcessed; quint64 processIncomingPacketsElapsedTime; slave.harvestStats(nodesProcessed, packetsProcessed, processIncomingPacketsElapsedTime); - slaveObject["nodesProcessed"] = nodesProcessed; - slaveObject["packetsProcessed"] = packetsProcessed; - slaveObject["timing_average_processIncomingPackets"] = (float)processIncomingPacketsElapsedTime / (float)_numTightLoopFrames; + slaveObject["nodesProcessed"] = (nodesProcessed > tenTimesPerFrame) ? nodesProcessed / tightLoopFrames : ((float)nodesProcessed / (float)tightLoopFrames); + slaveObject["packetsProcessed"] = (packetsProcessed > tenTimesPerFrame) ? packetsProcessed / tightLoopFrames : ((float)packetsProcessed / (float)tightLoopFrames); + slaveObject["timing_average_processIncomingPackets"] = (processIncomingPacketsElapsedTime > tenTimesPerFrame) + ? processIncomingPacketsElapsedTime / tightLoopFrames : ((float)processIncomingPacketsElapsedTime / (float)tightLoopFrames); slavesObject[QString::number(slaveNumber)] = slaveObject; slaveNumber++; @@ -914,10 +917,13 @@ void AvatarMixer::handlePacketVersionMismatch(PacketType type, const HifiSockAdd void AvatarMixer::parseDomainServerSettings(const QJsonObject& domainSettings) { const QString AVATAR_MIXER_SETTINGS_KEY = "avatar_mixer"; + QJsonObject avatarMixerGroupObject = domainSettings[AVATAR_MIXER_SETTINGS_KEY].toObject(); + + const QString NODE_SEND_BANDWIDTH_KEY = "max_node_send_bandwidth"; const float DEFAULT_NODE_SEND_BANDWIDTH = 5.0f; - QJsonValue nodeBandwidthValue = domainSettings[AVATAR_MIXER_SETTINGS_KEY].toObject()[NODE_SEND_BANDWIDTH_KEY]; + QJsonValue nodeBandwidthValue = avatarMixerGroupObject[NODE_SEND_BANDWIDTH_KEY]; if (!nodeBandwidthValue.isDouble()) { qDebug() << NODE_SEND_BANDWIDTH_KEY << "is not a double - will continue with default value"; } @@ -925,6 +931,22 @@ void AvatarMixer::parseDomainServerSettings(const QJsonObject& domainSettings) { _maxKbpsPerNode = nodeBandwidthValue.toDouble(DEFAULT_NODE_SEND_BANDWIDTH) * KILO_PER_MEGA; qDebug() << "The maximum send bandwidth per node is" << _maxKbpsPerNode << "kbps."; + const QString AUTO_THREADS = "auto_threads"; + bool autoThreads = avatarMixerGroupObject[AUTO_THREADS].toBool(); + if (!autoThreads) { + bool ok; + const QString NUM_THREADS = "num_threads"; + int numThreads = avatarMixerGroupObject[NUM_THREADS].toString().toInt(&ok); + if (!ok) { + qWarning() << "Avatar mixer: Error reading thread count. Using 1 thread."; + numThreads = 1; + } + qDebug() << "Avatar mixer will use specified number of threads:" << numThreads; + _slavePool.setNumThreads(numThreads); + } else { + qDebug() << "Avatar mixer will automatically determine number of threads to use. Using:" << _slavePool.numThreads() << "threads."; + } + const QString AVATARS_SETTINGS_KEY = "avatars"; static const QString MIN_SCALE_OPTION = "min_avatar_scale"; @@ -942,4 +964,5 @@ void AvatarMixer::parseDomainServerSettings(const QJsonObject& domainSettings) { qDebug() << "This domain requires a minimum avatar scale of" << _domainMinimumScale << "and a maximum avatar scale of" << _domainMaximumScale; + } diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index 045af9dc09..12dcb90f47 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -1299,6 +1299,22 @@ "placeholder": 5.0, "default": 5.0, "advanced": true + }, + { + "name": "auto_threads", + "label": "Automatically determine thread count", + "type": "checkbox", + "help": "Allow system to determine number of threads (recommended)", + "default": false, + "advanced": true + }, + { + "name": "num_threads", + "label": "Number of Threads", + "help": "Threads to spin up for avatar mixing (if not automatically set)", + "placeholder": "1", + "default": "1", + "advanced": true } ] } From babeabede5ed0f37eacf6c7545606a779a15bbe1 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 15 Feb 2017 13:49:16 -0800 Subject: [PATCH 031/104] more cleanup and stats rework --- assignment-client/src/avatars/AvatarMixer.cpp | 146 ++++++++++-------- assignment-client/src/avatars/AvatarMixer.h | 3 + 2 files changed, 87 insertions(+), 62 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 73c9aef8e1..f5455cbe67 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -113,13 +113,26 @@ void AvatarMixer::start() { { auto start = usecTimestampNow(); nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { + auto end = usecTimestampNow(); + _processQueuedAvatarDataPacketsLockWaitElapsedTime += (end - start); + _slavePool.processIncomingPackets(cbegin, cend); }); auto end = usecTimestampNow(); _processQueuedAvatarDataPacketsElapsedTime += (end - start); } - + // this is where we need to put the real work... + { + /* + auto start = usecTimestampNow(); + nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { + _slavePool.processIncomingPackets(cbegin, cend); + }); + auto end = usecTimestampNow(); + _processQueuedAvatarDataPacketsElapsedTime += (end - start); + */ + } // play nice with qt event-looping @@ -139,6 +152,41 @@ void AvatarMixer::start() { } +void AvatarMixer::manageDisplayName(AvatarMixerClientData* nodeData, const SharedNodePointer& node) { + AvatarData& avatar = nodeData->getAvatar(); + + quint64 startDisplayNameManagement = usecTimestampNow(); + if (nodeData->getAvatarSessionDisplayNameMustChange()) { + const QString& existingBaseDisplayName = nodeData->getBaseDisplayName(); + if (--_sessionDisplayNames[existingBaseDisplayName].second <= 0) { + _sessionDisplayNames.remove(existingBaseDisplayName); + } + + QString baseName = avatar.getDisplayName().trimmed(); + const QRegularExpression curses { "fuck|shit|damn|cock|cunt" }; // POC. We may eventually want something much more elaborate (subscription?). + baseName = baseName.replace(curses, "*"); // Replace rather than remove, so that people have a clue that the person's a jerk. + const QRegularExpression trailingDigits { "\\s*_\\d+$" }; // whitespace "_123" + baseName = baseName.remove(trailingDigits); + if (baseName.isEmpty()) { + baseName = "anonymous"; + } + + QPair& soFar = _sessionDisplayNames[baseName]; // Inserts and answers 0, 0 if not already present, which is what we want. + int& highWater = soFar.first; + nodeData->setBaseDisplayName(baseName); + QString sessionDisplayName = (highWater > 0) ? baseName + "_" + QString::number(highWater) : baseName; + avatar.setSessionDisplayName(sessionDisplayName); + highWater++; + soFar.second++; // refcount + nodeData->flagIdentityChange(); + nodeData->setAvatarSessionDisplayNameMustChange(false); + sendIdentityPacket(nodeData, node); // Tell node whose name changed about its new session display name. Others will find out below. + qDebug() << "Giving session display name" << sessionDisplayName << "to node with ID" << node->getUUID(); + } + quint64 endDisplayNameManagement = usecTimestampNow(); + _displayNameManagementElapsedTime += (endDisplayNameManagement - startDisplayNameManagement); +} + // NOTE: some additional optimizations to consider. // 1) use the view frustum to cull those avatars that are out of view. Since avatar data doesn't need to be present // if the avatar is not in view or in the keyhole. @@ -319,37 +367,7 @@ void AvatarMixer::broadcastAvatarData() { // setup a PacketList for the avatarPackets auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); - quint64 startDisplayNameManagement = usecTimestampNow(); - if (nodeData->getAvatarSessionDisplayNameMustChange()) { - const QString& existingBaseDisplayName = nodeData->getBaseDisplayName(); - if (--_sessionDisplayNames[existingBaseDisplayName].second <= 0) { - _sessionDisplayNames.remove(existingBaseDisplayName); - } - - QString baseName = avatar.getDisplayName().trimmed(); - const QRegularExpression curses{ "fuck|shit|damn|cock|cunt" }; // POC. We may eventually want something much more elaborate (subscription?). - baseName = baseName.replace(curses, "*"); // Replace rather than remove, so that people have a clue that the person's a jerk. - const QRegularExpression trailingDigits{ "\\s*_\\d+$" }; // whitespace "_123" - baseName = baseName.remove(trailingDigits); - if (baseName.isEmpty()) { - baseName = "anonymous"; - } - - QPair& soFar = _sessionDisplayNames[baseName]; // Inserts and answers 0, 0 if not already present, which is what we want. - int& highWater = soFar.first; - nodeData->setBaseDisplayName(baseName); - QString sessionDisplayName = (highWater > 0) ? baseName + "_" + QString::number(highWater) : baseName; - avatar.setSessionDisplayName(sessionDisplayName); - highWater++; - soFar.second++; // refcount - nodeData->flagIdentityChange(); - nodeData->setAvatarSessionDisplayNameMustChange(false); - sendIdentityPacket(nodeData, node); // Tell node whose name changed about its new session display name. Others will find out below. - qDebug() << "Giving session display name" << sessionDisplayName << "to node with ID" << node->getUUID(); - } - quint64 endDisplayNameManagement = usecTimestampNow(); - _displayNameManagementElapsedTime += (endDisplayNameManagement - startDisplayNameManagement); - + manageDisplayName(nodeData, node); // this is an AGENT we have received head data from // send back a packet with other active node data to this node @@ -758,19 +776,42 @@ void AvatarMixer::sendStatsPacket() { statsObject["timing_average_e_total_broadcastAvatarData"] = (float)_broadcastAvatarDataElapsedTime / (float)_numStatFrames; // this things all occur on the frequency of the tight loop - statsObject["timing_average_y_processEvents"] = (float)_processEventsElapsedTime / (float)_numTightLoopFrames; - statsObject["timing_average_y_queueIncomingPacket"] = (float)_queueIncomingPacketElapsedTime / (float)_numTightLoopFrames; - statsObject["timing_average_z_handleAvatarDataPacket"] = (float)_handleAvatarDataPacketElapsedTime / (float)_numTightLoopFrames; - statsObject["timing_average_z_handleAvatarIdentityPacket"] = (float)_handleAvatarIdentityPacketElapsedTime / (float)_numTightLoopFrames; - statsObject["timing_average_z_handleKillAvatarPacket"] = (float)_handleKillAvatarPacketElapsedTime / (float)_numTightLoopFrames; - statsObject["timing_average_z_handleNodeIgnoreRequestPacket"] = (float)_handleNodeIgnoreRequestPacketElapsedTime / (float)_numTightLoopFrames; - statsObject["timing_average_z_handleRadiusIgnoreRequestPacket"] = (float)_handleRadiusIgnoreRequestPacketElapsedTime / (float)_numTightLoopFrames; - statsObject["timing_average_z_handleRequestsDomainListDataPacket"] = (float)_handleRequestsDomainListDataPacketElapsedTime / (float)_numTightLoopFrames; - statsObject["timing_average_z_handleViewFrustumPacket"] = (float)_handleViewFrustumPacketElapsedTime / (float)_numTightLoopFrames; - statsObject["timing_average_z_processQueuedAvatarDataPackets"] = (float)_processQueuedAvatarDataPacketsElapsedTime / (float)_numTightLoopFrames; + int tightLoopFrames = _numTightLoopFrames; + int tenTimesPerFrame = tightLoopFrames * 10; + #define TIGHT_LOOP_STAT(x) (x > tenTimesPerFrame) ? x / tightLoopFrames : ((float)x / (float)tightLoopFrames); + + statsObject["timing_average_y_processEvents"] = TIGHT_LOOP_STAT(_processEventsElapsedTime); + statsObject["timing_average_y_queueIncomingPacket"] = TIGHT_LOOP_STAT(_queueIncomingPacketElapsedTime); + statsObject["timing_average_z_handleAvatarDataPacket"] = TIGHT_LOOP_STAT(_handleAvatarDataPacketElapsedTime); + statsObject["timing_average_z_handleAvatarIdentityPacket"] = TIGHT_LOOP_STAT(_handleAvatarIdentityPacketElapsedTime); + statsObject["timing_average_z_handleKillAvatarPacket"] = TIGHT_LOOP_STAT(_handleKillAvatarPacketElapsedTime); + statsObject["timing_average_z_handleNodeIgnoreRequestPacket"] = TIGHT_LOOP_STAT(_handleNodeIgnoreRequestPacketElapsedTime); + statsObject["timing_average_z_handleRadiusIgnoreRequestPacket"] = TIGHT_LOOP_STAT(_handleRadiusIgnoreRequestPacketElapsedTime); + statsObject["timing_average_z_handleRequestsDomainListDataPacket"] = TIGHT_LOOP_STAT(_handleRequestsDomainListDataPacketElapsedTime); + statsObject["timing_average_z_handleViewFrustumPacket"] = TIGHT_LOOP_STAT(_handleViewFrustumPacketElapsedTime); + statsObject["timing_average_z_processQueuedAvatarDataPackets"] = TIGHT_LOOP_STAT(_processQueuedAvatarDataPacketsElapsedTime); + statsObject["timing_average_z_processQueuedAvatarDataPacketsLockWait"] = TIGHT_LOOP_STAT(_processQueuedAvatarDataPacketsLockWaitElapsedTime); statsObject["timing_sendStats"] = (float)_sendStatsElapsedTime; + + QJsonObject slavesObject; + // gather stats + int slaveNumber = 1; + _slavePool.each([&](AvatarMixerSlave& slave) { + QJsonObject slaveObject; + int nodesProcessed, packetsProcessed; + quint64 processIncomingPacketsElapsedTime; + slave.harvestStats(nodesProcessed, packetsProcessed, processIncomingPacketsElapsedTime); + slaveObject["nodesProcessed"] = TIGHT_LOOP_STAT(nodesProcessed); + slaveObject["packetsProcessed"] = TIGHT_LOOP_STAT(packetsProcessed); + slaveObject["timing_average_processIncomingPackets"] = TIGHT_LOOP_STAT(processIncomingPacketsElapsedTime); + + slavesObject[QString::number(slaveNumber)] = slaveObject; + slaveNumber++; + }); + statsObject["timing_slaves"] = slavesObject; + _handleViewFrustumPacketElapsedTime = 0; _handleAvatarDataPacketElapsedTime = 0; _handleAvatarIdentityPacketElapsedTime = 0; @@ -781,26 +822,7 @@ void AvatarMixer::sendStatsPacket() { _processEventsElapsedTime = 0; _queueIncomingPacketElapsedTime = 0; _processQueuedAvatarDataPacketsElapsedTime = 0; - - QJsonObject slavesObject; - // gather stats - int slaveNumber = 1; - int tightLoopFrames = _numTightLoopFrames; - int tenTimesPerFrame = tightLoopFrames * 10; - _slavePool.each([&](AvatarMixerSlave& slave) { - QJsonObject slaveObject; - int nodesProcessed, packetsProcessed; - quint64 processIncomingPacketsElapsedTime; - slave.harvestStats(nodesProcessed, packetsProcessed, processIncomingPacketsElapsedTime); - slaveObject["nodesProcessed"] = (nodesProcessed > tenTimesPerFrame) ? nodesProcessed / tightLoopFrames : ((float)nodesProcessed / (float)tightLoopFrames); - slaveObject["packetsProcessed"] = (packetsProcessed > tenTimesPerFrame) ? packetsProcessed / tightLoopFrames : ((float)packetsProcessed / (float)tightLoopFrames); - slaveObject["timing_average_processIncomingPackets"] = (processIncomingPacketsElapsedTime > tenTimesPerFrame) - ? processIncomingPacketsElapsedTime / tightLoopFrames : ((float)processIncomingPacketsElapsedTime / (float)tightLoopFrames); - - slavesObject[QString::number(slaveNumber)] = slaveObject; - slaveNumber++; - }); - statsObject["timing_slaves"] = slavesObject; + _processQueuedAvatarDataPacketsLockWaitElapsedTime = 0; QJsonObject avatarsObject; auto nodeList = DependencyManager::get(); diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index c8d1516e0d..8c06f7fc43 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -60,6 +60,8 @@ private: void parseDomainServerSettings(const QJsonObject& domainSettings); void sendIdentityPacket(AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode); + void manageDisplayName(AvatarMixerClientData* nodeData, const SharedNodePointer& node); + QThread _broadcastThread; p_high_resolution_clock::time_point _lastFrameTimestamp; @@ -98,6 +100,7 @@ private: quint64 _handleRadiusIgnoreRequestPacketElapsedTime { 0 }; quint64 _handleRequestsDomainListDataPacketElapsedTime { 0 }; quint64 _processQueuedAvatarDataPacketsElapsedTime { 0 }; + quint64 _processQueuedAvatarDataPacketsLockWaitElapsedTime { 0 }; quint64 _processEventsElapsedTime { 0 }; quint64 _sendStatsElapsedTime { 0 }; From faa8e629a03f39c4b5a0acd3da2431462f17006f Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 15 Feb 2017 20:28:32 -0800 Subject: [PATCH 032/104] checkpoint --- assignment-client/src/avatars/AvatarMixer.cpp | 33 +++++++++++------ assignment-client/src/avatars/AvatarMixer.h | 2 +- libraries/avatars/src/AvatarData.cpp | 36 ------------------- libraries/avatars/src/AvatarData.h | 30 ++++++++-------- libraries/shared/src/SpatiallyNestable.h | 6 ++-- 5 files changed, 40 insertions(+), 67 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index f5455cbe67..acb62e6225 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -99,8 +99,6 @@ void AvatarMixer::start() { auto nodeList = DependencyManager::get(); - //_slavePool.setNumThreads(1); // grins - while (!_isFinished) { _numTightLoopFrames++; _loopRate.increment(); @@ -122,6 +120,19 @@ void AvatarMixer::start() { _processQueuedAvatarDataPacketsElapsedTime += (end - start); } + // process pending display names... this doesn't currently run on multiple threads, because it + // side-effects the mixer's data, which is fine because it's a very low cost operation + { + auto start = usecTimestampNow(); + nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { + std::for_each(cbegin, cend, [&](const SharedNodePointer& node) { + manageDisplayName(node); + }); + }); + auto end = usecTimestampNow(); + _displayNameManagementElapsedTime += (end - start); + } + // this is where we need to put the real work... { /* @@ -152,11 +163,12 @@ void AvatarMixer::start() { } -void AvatarMixer::manageDisplayName(AvatarMixerClientData* nodeData, const SharedNodePointer& node) { - AvatarData& avatar = nodeData->getAvatar(); - - quint64 startDisplayNameManagement = usecTimestampNow(); - if (nodeData->getAvatarSessionDisplayNameMustChange()) { +// NOTE: nodeData->getAvatar() might be side effected, most be called when access to node/nodeData +// is guarenteed to not be accessed by other thread +void AvatarMixer::manageDisplayName(const SharedNodePointer& node) { + AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); + if (nodeData && nodeData->getAvatarSessionDisplayNameMustChange()) { + AvatarData& avatar = nodeData->getAvatar(); const QString& existingBaseDisplayName = nodeData->getBaseDisplayName(); if (--_sessionDisplayNames[existingBaseDisplayName].second <= 0) { _sessionDisplayNames.remove(existingBaseDisplayName); @@ -183,8 +195,6 @@ void AvatarMixer::manageDisplayName(AvatarMixerClientData* nodeData, const Share sendIdentityPacket(nodeData, node); // Tell node whose name changed about its new session display name. Others will find out below. qDebug() << "Giving session display name" << sessionDisplayName << "to node with ID" << node->getUUID(); } - quint64 endDisplayNameManagement = usecTimestampNow(); - _displayNameManagementElapsedTime += (endDisplayNameManagement - startDisplayNameManagement); } // NOTE: some additional optimizations to consider. @@ -367,7 +377,7 @@ void AvatarMixer::broadcastAvatarData() { // setup a PacketList for the avatarPackets auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); - manageDisplayName(nodeData, node); + //manageDisplayName(node); // this is an AGENT we have received head data from // send back a packet with other active node data to this node @@ -769,7 +779,6 @@ void AvatarMixer::sendStatsPacket() { statsObject["tight_loop_rate"] = _loopRate.rate(); // broadcastAvatarDataElapsed timing details... - statsObject["timing_average_a_displayNameManagement"] = (float)_displayNameManagementElapsedTime / (float)_numStatFrames; statsObject["timing_average_b_ignoreCalculation"] = (float)_ignoreCalculationElapsedTime / (float)_numStatFrames; statsObject["timing_average_c_avatarDataPacking"] = (float)_avatarDataPackingElapsedTime / (float)_numStatFrames; statsObject["timing_average_d_packetSending"] = (float)_packetSendingElapsedTime / (float)_numStatFrames; @@ -782,6 +791,8 @@ void AvatarMixer::sendStatsPacket() { statsObject["timing_average_y_processEvents"] = TIGHT_LOOP_STAT(_processEventsElapsedTime); statsObject["timing_average_y_queueIncomingPacket"] = TIGHT_LOOP_STAT(_queueIncomingPacketElapsedTime); + + statsObject["timing_average_z_displayNameManagement"] = TIGHT_LOOP_STAT(_displayNameManagementElapsedTime); statsObject["timing_average_z_handleAvatarDataPacket"] = TIGHT_LOOP_STAT(_handleAvatarDataPacketElapsedTime); statsObject["timing_average_z_handleAvatarIdentityPacket"] = TIGHT_LOOP_STAT(_handleAvatarIdentityPacketElapsedTime); statsObject["timing_average_z_handleKillAvatarPacket"] = TIGHT_LOOP_STAT(_handleKillAvatarPacketElapsedTime); diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index 8c06f7fc43..e0b03804a9 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -60,7 +60,7 @@ private: void parseDomainServerSettings(const QJsonObject& domainSettings); void sendIdentityPacket(AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode); - void manageDisplayName(AvatarMixerClientData* nodeData, const SharedNodePointer& node); + void manageDisplayName(const SharedNodePointer& node); QThread _broadcastThread; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 47a8cc6e6e..c55d06f2e7 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -120,10 +120,6 @@ void AvatarData::nextAttitude(glm::vec3 position, glm::quat orientation) { updateAttitude(); } -float AvatarData::getTargetScale() const { - return _targetScale; -} - void AvatarData::setTargetScale(float targetScale) { auto newValue = glm::clamp(targetScale, MIN_AVATAR_SCALE, MAX_AVATAR_SCALE); if (_targetScale != newValue) { @@ -152,38 +148,6 @@ void AvatarData::lazyInitHeadData() { } -bool AvatarData::avatarBoundingBoxChangedSince(quint64 time) { - return _avatarBoundingBoxChanged >= time; -} - -bool AvatarData::avatarScaleChangedSince(quint64 time) { - return _avatarScaleChanged >= time; -} - -bool AvatarData::lookAtPositionChangedSince(quint64 time) { - return _headData->lookAtPositionChangedSince(time); -} - -bool AvatarData::audioLoudnessChangedSince(quint64 time) { - return _headData->audioLoudnessChangedSince(time); -} - -bool AvatarData::sensorToWorldMatrixChangedSince(quint64 time) { - return _sensorToWorldMatrixChanged >= time; -} - -bool AvatarData::additionalFlagsChangedSince(quint64 time) { - return _additionalFlagsChanged >= time; -} - -bool AvatarData::parentInfoChangedSince(quint64 time) { - return _parentChanged >= time; -} - -bool AvatarData::faceTrackerInfoChangedSince(quint64 time) { - return true; // FIXME! -} - float AvatarData::getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) { auto distance = glm::distance(_globalPosition, viewerPosition); float result = ROTATION_CHANGE_179D; // assume worst diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index b28501eead..964bc4a6df 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -419,7 +419,6 @@ public: void setAudioAverageLoudness(float value) { _headData->setAudioAverageLoudness(value); } // Scale - float getTargetScale() const; virtual void setTargetScale(float targetScale); float getDomainLimitedScale() const { return glm::clamp(_targetScale, _domainMinimumScale, _domainMaximumScale); } @@ -534,8 +533,8 @@ public: QJsonObject toJson() const; void fromJson(const QJsonObject& json, bool useFrameSkeleton = true); - glm::vec3 getClientGlobalPosition() { return _globalPosition; } - glm::vec3 getGlobalBoundingBoxCorner() { return _globalPosition + _globalBoundingBoxOffset - _globalBoundingBoxDimensions; } + glm::vec3 getClientGlobalPosition() const { return _globalPosition; } + glm::vec3 getGlobalBoundingBoxCorner() const { return _globalPosition + _globalBoundingBoxOffset - _globalBoundingBoxDimensions; } Q_INVOKABLE AvatarEntityMap getAvatarEntityData() const; Q_INVOKABLE void setAvatarEntityData(const AvatarEntityMap& avatarEntityData); @@ -550,7 +549,7 @@ public: Q_INVOKABLE float getDataRate(const QString& rateName = QString("")) const; Q_INVOKABLE float getUpdateRate(const QString& rateName = QString("")) const; - int getJointCount() { return _jointData.size(); } + int getJointCount() const { return _jointData.size(); } QVector getLastSentJointData() { QReadLocker readLock(&_jointDataLock); @@ -571,7 +570,7 @@ public slots: virtual bool setAbsoluteJointRotationInObjectFrame(int index, const glm::quat& rotation) override { return false; } virtual bool setAbsoluteJointTranslationInObjectFrame(int index, const glm::vec3& translation) override { return false; } - float getTargetScale() { return _targetScale; } + float getTargetScale() const { return _targetScale; } // why is this a slot? void resetLastSent() { _lastToByteArray = 0; } @@ -581,18 +580,17 @@ protected: float getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition); float getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition); - bool avatarBoundingBoxChangedSince(quint64 time); - bool avatarScaleChangedSince(quint64 time); - bool lookAtPositionChangedSince(quint64 time); - bool audioLoudnessChangedSince(quint64 time); - bool sensorToWorldMatrixChangedSince(quint64 time); - bool additionalFlagsChangedSince(quint64 time); + bool avatarBoundingBoxChangedSince(quint64 time) const { return _avatarBoundingBoxChanged >= time; } + bool avatarScaleChangedSince(quint64 time) const { return _avatarScaleChanged >= time; } + bool lookAtPositionChangedSince(quint64 time) const { return _headData->lookAtPositionChangedSince(time); } + bool audioLoudnessChangedSince(quint64 time) const { return _headData->audioLoudnessChangedSince(time); } + bool sensorToWorldMatrixChangedSince(quint64 time) const { return _sensorToWorldMatrixChanged >= time; } + bool additionalFlagsChangedSince(quint64 time) const { return _additionalFlagsChanged >= time; } + bool parentInfoChangedSince(quint64 time) const { return _parentChanged >= time; } + bool faceTrackerInfoChangedSince(quint64 time) const { return true; } // FIXME - bool hasParent() { return !getParentID().isNull(); } - bool parentInfoChangedSince(quint64 time); - - bool hasFaceTracker() { return _headData ? _headData->_isFaceTrackerConnected : false; } - bool faceTrackerInfoChangedSince(quint64 time); + bool hasParent() const { return !getParentID().isNull(); } + bool hasFaceTracker() const { return _headData ? _headData->_isFaceTrackerConnected : false; } glm::vec3 _handPosition; virtual const QString& getSessionDisplayNameForTransport() const { return _sessionDisplayName; } diff --git a/libraries/shared/src/SpatiallyNestable.h b/libraries/shared/src/SpatiallyNestable.h index be285eff53..820c8685d7 100644 --- a/libraries/shared/src/SpatiallyNestable.h +++ b/libraries/shared/src/SpatiallyNestable.h @@ -179,9 +179,9 @@ public: const glm::vec3& localVelocity, const glm::vec3& localAngularVelocity); - bool scaleChangedSince(quint64 time) { return _scaleChanged > time; } - bool tranlationChangedSince(quint64 time) { return _translationChanged > time; } - bool rotationChangedSince(quint64 time) { return _rotationChanged > time; } + bool scaleChangedSince(quint64 time) const { return _scaleChanged > time; } + bool tranlationChangedSince(quint64 time) const { return _translationChanged > time; } + bool rotationChangedSince(quint64 time) const { return _rotationChanged > time; } protected: const NestableType _nestableType; // EntityItem or an AvatarData From bd722165743e0fd6ed65adf2b675f3874b420d90 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 15 Feb 2017 20:49:45 -0800 Subject: [PATCH 033/104] add support for multiple jobs types on the slave pool --- .../src/avatars/AvatarMixerSlave.cpp | 2 ++ .../src/avatars/AvatarMixerSlave.h | 1 + .../src/avatars/AvatarMixerSlavePool.cpp | 34 +++++++++++++------ .../src/avatars/AvatarMixerSlavePool.h | 12 ++++--- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 06d21b51b9..784b10ebfe 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -55,3 +55,5 @@ void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { _processIncomingPacketsElapsedTime += (end - start); } +void AvatarMixerSlave::anotherJob(const SharedNodePointer& node) { +} diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index 1abba8d46c..d75c6ae396 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -28,6 +28,7 @@ public: void configure(ConstIter begin, ConstIter end); void processIncomingPackets(const SharedNodePointer& node); + void anotherJob(const SharedNodePointer& node); void harvestStats(int& nodesProcessed, int& packetsProcessed, quint64& processIncomingPacketsElapsedTime); diff --git a/assignment-client/src/avatars/AvatarMixerSlavePool.cpp b/assignment-client/src/avatars/AvatarMixerSlavePool.cpp index e50a8475a0..1b335f8383 100644 --- a/assignment-client/src/avatars/AvatarMixerSlavePool.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlavePool.cpp @@ -21,7 +21,7 @@ void AvatarMixerSlaveThread::run() { // iterate over all available nodes SharedNodePointer node; while (try_pop(node)) { - processIncomingPackets(node); + (this->*_function)(node); } bool stopping = _stop; @@ -41,7 +41,10 @@ void AvatarMixerSlaveThread::wait() { }); ++_pool._numStarted; } - configure(_pool._begin, _pool._end); + if (_pool._configure) { + _pool._configure(*this); + } + _function = _pool._function; } void AvatarMixerSlaveThread::notify(bool stopping) { @@ -65,18 +68,26 @@ static AvatarMixerSlave slave; #endif void AvatarMixerSlavePool::processIncomingPackets(ConstIter begin, ConstIter end) { + _function = &AvatarMixerSlave::processIncomingPackets; + _configure = [](AvatarMixerSlave& slave) {}; + run(begin, end); +} + +void AvatarMixerSlavePool::anotherJob(ConstIter begin, ConstIter end) { + _function = &AvatarMixerSlave::anotherJob; + _configure = [](AvatarMixerSlave& slave) {}; + run(begin, end); +} + +void AvatarMixerSlavePool::run(ConstIter begin, ConstIter end) { _begin = begin; _end = end; - // ???? - //_frame = frame; - //_throttlingRatio = throttlingRatio; - -#ifdef AVATAR_SINGLE_THREADED - slave.configure(_begin, _end, frame, throttlingRatio); +#ifdef AUDIO_SINGLE_THREADED + _configure(slave); std::for_each(begin, end, [&](const SharedNodePointer& node) { - slave.processIncomingPackets(node); - }); + _function(slave, node); +}); #else // fill the queue std::for_each(_begin, _end, [&](const SharedNodePointer& node) { @@ -86,7 +97,7 @@ void AvatarMixerSlavePool::processIncomingPackets(ConstIter begin, ConstIter end { Lock lock(_mutex); - // start the job + // run _numStarted = _numFinished = 0; _slaveCondition.notify_all(); @@ -103,6 +114,7 @@ void AvatarMixerSlavePool::processIncomingPackets(ConstIter begin, ConstIter end #endif } + void AvatarMixerSlavePool::each(std::function functor) { #ifdef AVATAR_SINGLE_THREADED functor(slave); diff --git a/assignment-client/src/avatars/AvatarMixerSlavePool.h b/assignment-client/src/avatars/AvatarMixerSlavePool.h index 9865f897b9..8f13477a93 100644 --- a/assignment-client/src/avatars/AvatarMixerSlavePool.h +++ b/assignment-client/src/avatars/AvatarMixerSlavePool.h @@ -45,6 +45,7 @@ private: bool try_pop(SharedNodePointer& node); AvatarMixerSlavePool& _pool; + void (AvatarMixerSlave::*_function)(const SharedNodePointer& node) { nullptr }; bool _stop { false }; }; @@ -62,17 +63,18 @@ public: AvatarMixerSlavePool(int numThreads = QThread::idealThreadCount()) { setNumThreads(numThreads); } ~AvatarMixerSlavePool() { resize(0); } + // Jobs the slave pool can do... + void processIncomingPackets(ConstIter begin, ConstIter end); + void anotherJob(ConstIter begin, ConstIter end); + // iterate over all slaves void each(std::function functor); void setNumThreads(int numThreads); int numThreads() { return _numThreads; } - // Jobs the slave pool can do... - void processIncomingPackets(ConstIter begin, ConstIter end); - - private: + void run(ConstIter begin, ConstIter end); void resize(int numThreads); std::vector> _slaves; @@ -85,6 +87,8 @@ private: Mutex _mutex; ConditionVariable _slaveCondition; ConditionVariable _poolCondition; + void (AvatarMixerSlave::*_function)(const SharedNodePointer& node); + std::function _configure; int _numThreads { 0 }; int _numStarted { 0 }; // guarded by _mutex int _numFinished { 0 }; // guarded by _mutex From d22f4c1dd7be72600021de00bd289d0284b30ef3 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 15 Feb 2017 21:29:03 -0800 Subject: [PATCH 034/104] partial const migration work so mixer will not side-effect AvatarData --- assignment-client/src/avatars/AvatarMixer.cpp | 13 +- libraries/avatars/src/AvatarData.cpp | 406 +++++++++++++++++- libraries/avatars/src/AvatarData.h | 14 +- 3 files changed, 417 insertions(+), 16 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index acb62e6225..de5a9b3f2c 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -301,7 +301,7 @@ void AvatarMixer::broadcastAvatarData() { ++_sumListeners; nodeData->resetInViewStats(); - AvatarData& avatar = nodeData->getAvatar(); + const AvatarData& avatar = nodeData->getAvatar(); glm::vec3 myPosition = avatar.getClientGlobalPosition(); // reset the internal state for correct random number distribution @@ -377,8 +377,6 @@ void AvatarMixer::broadcastAvatarData() { // setup a PacketList for the avatarPackets auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); - //manageDisplayName(node); - // this is an AGENT we have received head data from // send back a packet with other active node data to this node nodeList->eachMatchingNode( @@ -470,7 +468,7 @@ void AvatarMixer::broadcastAvatarData() { sendIdentityPacket(otherNodeData, node); } - AvatarData& otherAvatar = otherNodeData->getAvatar(); + const AvatarData& otherAvatar = otherNodeData->getAvatar(); // Decide whether to send this avatar's data based on it's distance from us // The full rate distance is the distance at which EVERY update will be sent for this avatar @@ -550,7 +548,7 @@ void AvatarMixer::broadcastAvatarData() { QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); bool distanceAdjust = true; glm::vec3 viewerPosition = myPosition; - auto bytes = otherAvatar.toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, distanceAdjust, viewerPosition, &lastSentJointsForOther); + QByteArray bytes = otherAvatar.toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, distanceAdjust, viewerPosition, &lastSentJointsForOther); numAvatarDataBytes += avatarPacketList->write(bytes); avatarPacketList->endSegment(); @@ -916,8 +914,9 @@ AvatarMixerClientData* AvatarMixer::getOrCreateClientData(SharedNodePointer node if (!clientData) { node->setLinkedData(std::unique_ptr { new AvatarMixerClientData(node->getUUID()) }); clientData = dynamic_cast(node->getLinkedData()); - clientData->getAvatar().setDomainMinimumScale(_domainMinimumScale); - clientData->getAvatar().setDomainMaximumScale(_domainMaximumScale); + auto& avatar = clientData->getAvatar(); + avatar.setDomainMinimumScale(_domainMinimumScale); + avatar.setDomainMaximumScale(_domainMaximumScale); } return clientData; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index c55d06f2e7..cbd5d42263 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -137,10 +137,10 @@ void AvatarData::setHandPosition(const glm::vec3& handPosition) { _handPosition = glm::inverse(getOrientation()) * (handPosition - getPosition()); } -void AvatarData::lazyInitHeadData() { +void AvatarData::lazyInitHeadData() const { // lazily allocate memory for HeadData in case we're not an Avatar instance if (!_headData) { - _headData = new HeadData(this); + _headData = new HeadData(const_cast(this)); } if (_forceFaceTrackerConnected) { _headData->_isFaceTrackerConnected = true; @@ -148,7 +148,7 @@ void AvatarData::lazyInitHeadData() { } -float AvatarData::getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) { +float AvatarData::getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) const { auto distance = glm::distance(_globalPosition, viewerPosition); float result = ROTATION_CHANGE_179D; // assume worst if (distance < AVATAR_DISTANCE_LEVEL_1) { @@ -163,7 +163,7 @@ float AvatarData::getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) { return result; } -float AvatarData::getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition) { +float AvatarData::getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition) const { return AVATAR_MIN_TRANSLATION; // Eventually make this distance sensitive as well } @@ -574,6 +574,404 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent return avatarDataByteArray.left(avatarDataSize); } +QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, + bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) const { + + bool cullSmallChanges = (dataDetail == CullSmallData); + bool sendAll = (dataDetail == SendAllData); + bool sendMinimum = (dataDetail == MinimumData); + + lazyInitHeadData(); + + QByteArray avatarDataByteArray(udt::MAX_PACKET_SIZE, 0); + unsigned char* destinationBuffer = reinterpret_cast(avatarDataByteArray.data()); + unsigned char* startPosition = destinationBuffer; + + // FIXME - + // + // BUG -- if you enter a space bubble, and then back away, the avatar has wrong orientation until "send all" happens... + // this is an iFrame issue... what to do about that? + // + // BUG -- Resizing avatar seems to "take too long"... the avatar doesn't redraw at smaller size right away + // + // TODO consider these additional optimizations in the future + // 1) SensorToWorld - should we only send this for avatars with attachments?? - 20 bytes - 7.20 kbps + // 2) GUIID for the session change to 2byte index (savings) - 14 bytes - 5.04 kbps + // 3) Improve Joints -- currently we use rotational tolerances, but if we had skeleton/bone length data + // we could do a better job of determining if the change in joints actually translates to visible + // changes at distance. + // + // Potential savings: + // 63 rotations * 6 bytes = 136kbps + // 3 translations * 6 bytes = 6.48kbps + // + + auto parentID = getParentID(); + + bool hasAvatarGlobalPosition = true; // always include global position + bool hasAvatarOrientation = sendAll || rotationChangedSince(lastSentTime); + bool hasAvatarBoundingBox = sendAll || avatarBoundingBoxChangedSince(lastSentTime); + bool hasAvatarScale = sendAll || avatarScaleChangedSince(lastSentTime); + bool hasLookAtPosition = sendAll || lookAtPositionChangedSince(lastSentTime); + bool hasAudioLoudness = sendAll || audioLoudnessChangedSince(lastSentTime); + bool hasSensorToWorldMatrix = sendAll || sensorToWorldMatrixChangedSince(lastSentTime); + bool hasAdditionalFlags = sendAll || additionalFlagsChangedSince(lastSentTime); + + // local position, and parent info only apply to avatars that are parented. The local position + // and the parent info can change independently though, so we track their "changed since" + // separately + bool hasParentInfo = sendAll || parentInfoChangedSince(lastSentTime); + bool hasAvatarLocalPosition = hasParent() && (sendAll || + tranlationChangedSince(lastSentTime) || + parentInfoChangedSince(lastSentTime)); + + bool hasFaceTrackerInfo = hasFaceTracker() && (sendAll || faceTrackerInfoChangedSince(lastSentTime)); + bool hasJointData = sendAll || !sendMinimum; + + // Leading flags, to indicate how much data is actually included in the packet... + AvatarDataPacket::HasFlags packetStateFlags = + (hasAvatarGlobalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_GLOBAL_POSITION : 0) + | (hasAvatarBoundingBox ? AvatarDataPacket::PACKET_HAS_AVATAR_BOUNDING_BOX : 0) + | (hasAvatarOrientation ? AvatarDataPacket::PACKET_HAS_AVATAR_ORIENTATION : 0) + | (hasAvatarScale ? AvatarDataPacket::PACKET_HAS_AVATAR_SCALE : 0) + | (hasLookAtPosition ? AvatarDataPacket::PACKET_HAS_LOOK_AT_POSITION : 0) + | (hasAudioLoudness ? AvatarDataPacket::PACKET_HAS_AUDIO_LOUDNESS : 0) + | (hasSensorToWorldMatrix ? AvatarDataPacket::PACKET_HAS_SENSOR_TO_WORLD_MATRIX : 0) + | (hasAdditionalFlags ? AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS : 0) + | (hasParentInfo ? AvatarDataPacket::PACKET_HAS_PARENT_INFO : 0) + | (hasAvatarLocalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_LOCAL_POSITION : 0) + | (hasFaceTrackerInfo ? AvatarDataPacket::PACKET_HAS_FACE_TRACKER_INFO : 0) + | (hasJointData ? AvatarDataPacket::PACKET_HAS_JOINT_DATA : 0); + + memcpy(destinationBuffer, &packetStateFlags, sizeof(packetStateFlags)); + destinationBuffer += sizeof(packetStateFlags); + + if (hasAvatarGlobalPosition) { + auto startSection = destinationBuffer; + auto data = reinterpret_cast(destinationBuffer); + data->globalPosition[0] = _globalPosition.x; + data->globalPosition[1] = _globalPosition.y; + data->globalPosition[2] = _globalPosition.z; + destinationBuffer += sizeof(AvatarDataPacket::AvatarGlobalPosition); + + int numBytes = destinationBuffer - startSection; + + //_globalPositionRateOutbound.increment(numBytes); + } + + if (hasAvatarBoundingBox) { + auto startSection = destinationBuffer; + auto data = reinterpret_cast(destinationBuffer); + + data->avatarDimensions[0] = _globalBoundingBoxDimensions.x; + data->avatarDimensions[1] = _globalBoundingBoxDimensions.y; + data->avatarDimensions[2] = _globalBoundingBoxDimensions.z; + + data->boundOriginOffset[0] = _globalBoundingBoxOffset.x; + data->boundOriginOffset[1] = _globalBoundingBoxOffset.y; + data->boundOriginOffset[2] = _globalBoundingBoxOffset.z; + + destinationBuffer += sizeof(AvatarDataPacket::AvatarBoundingBox); + + int numBytes = destinationBuffer - startSection; + //_avatarBoundingBoxRateOutbound.increment(numBytes); + } + + if (hasAvatarOrientation) { + auto startSection = destinationBuffer; + auto localOrientation = getLocalOrientation(); + destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, localOrientation); + + int numBytes = destinationBuffer - startSection; + //_avatarOrientationRateOutbound.increment(numBytes); + } + + if (hasAvatarScale) { + auto startSection = destinationBuffer; + auto data = reinterpret_cast(destinationBuffer); + auto scale = getDomainLimitedScale(); + packFloatRatioToTwoByte((uint8_t*)(&data->scale), scale); + destinationBuffer += sizeof(AvatarDataPacket::AvatarScale); + + int numBytes = destinationBuffer - startSection; + //_avatarScaleRateOutbound.increment(numBytes); + } + + if (hasLookAtPosition) { + auto startSection = destinationBuffer; + auto data = reinterpret_cast(destinationBuffer); + auto lookAt = _headData->getLookAtPosition(); + data->lookAtPosition[0] = lookAt.x; + data->lookAtPosition[1] = lookAt.y; + data->lookAtPosition[2] = lookAt.z; + destinationBuffer += sizeof(AvatarDataPacket::LookAtPosition); + + int numBytes = destinationBuffer - startSection; + //_lookAtPositionRateOutbound.increment(numBytes); + } + + if (hasAudioLoudness) { + auto startSection = destinationBuffer; + auto data = reinterpret_cast(destinationBuffer); + data->audioLoudness = packFloatGainToByte(_headData->getAudioLoudness() / AUDIO_LOUDNESS_SCALE); + destinationBuffer += sizeof(AvatarDataPacket::AudioLoudness); + + int numBytes = destinationBuffer - startSection; + //_audioLoudnessRateOutbound.increment(numBytes); + } + + if (hasSensorToWorldMatrix) { + auto startSection = destinationBuffer; + auto data = reinterpret_cast(destinationBuffer); + glm::mat4 sensorToWorldMatrix = getSensorToWorldMatrix(); + packOrientationQuatToSixBytes(data->sensorToWorldQuat, glmExtractRotation(sensorToWorldMatrix)); + glm::vec3 scale = extractScale(sensorToWorldMatrix); + packFloatScalarToSignedTwoByteFixed((uint8_t*)&data->sensorToWorldScale, scale.x, SENSOR_TO_WORLD_SCALE_RADIX); + data->sensorToWorldTrans[0] = sensorToWorldMatrix[3][0]; + data->sensorToWorldTrans[1] = sensorToWorldMatrix[3][1]; + data->sensorToWorldTrans[2] = sensorToWorldMatrix[3][2]; + destinationBuffer += sizeof(AvatarDataPacket::SensorToWorldMatrix); + + int numBytes = destinationBuffer - startSection; + //_sensorToWorldRateOutbound.increment(numBytes); + } + + if (hasAdditionalFlags) { + auto startSection = destinationBuffer; + auto data = reinterpret_cast(destinationBuffer); + + uint8_t flags { 0 }; + + setSemiNibbleAt(flags, KEY_STATE_START_BIT, _keyState); + + // hand state + bool isFingerPointing = _handState & IS_FINGER_POINTING_FLAG; + setSemiNibbleAt(flags, HAND_STATE_START_BIT, _handState & ~IS_FINGER_POINTING_FLAG); + if (isFingerPointing) { + setAtBit(flags, HAND_STATE_FINGER_POINTING_BIT); + } + // faceshift state + if (_headData->_isFaceTrackerConnected) { + setAtBit(flags, IS_FACESHIFT_CONNECTED); + } + // eye tracker state + if (_headData->_isEyeTrackerConnected) { + setAtBit(flags, IS_EYE_TRACKER_CONNECTED); + } + // referential state + if (!parentID.isNull()) { + setAtBit(flags, HAS_REFERENTIAL); + } + data->flags = flags; + destinationBuffer += sizeof(AvatarDataPacket::AdditionalFlags); + + int numBytes = destinationBuffer - startSection; + //_additionalFlagsRateOutbound.increment(numBytes); + } + + if (hasParentInfo) { + auto startSection = destinationBuffer; + auto parentInfo = reinterpret_cast(destinationBuffer); + QByteArray referentialAsBytes = parentID.toRfc4122(); + memcpy(parentInfo->parentUUID, referentialAsBytes.data(), referentialAsBytes.size()); + parentInfo->parentJointIndex = getParentJointIndex(); + destinationBuffer += sizeof(AvatarDataPacket::ParentInfo); + + int numBytes = destinationBuffer - startSection; + //_parentInfoRateOutbound.increment(numBytes); + } + + if (hasAvatarLocalPosition) { + auto startSection = destinationBuffer; + auto data = reinterpret_cast(destinationBuffer); + auto localPosition = getLocalPosition(); + data->localPosition[0] = localPosition.x; + data->localPosition[1] = localPosition.y; + data->localPosition[2] = localPosition.z; + destinationBuffer += sizeof(AvatarDataPacket::AvatarLocalPosition); + + int numBytes = destinationBuffer - startSection; + //_localPositionRateOutbound.increment(numBytes); + } + + // If it is connected, pack up the data + if (hasFaceTrackerInfo) { + auto startSection = destinationBuffer; + auto faceTrackerInfo = reinterpret_cast(destinationBuffer); + + faceTrackerInfo->leftEyeBlink = _headData->_leftEyeBlink; + faceTrackerInfo->rightEyeBlink = _headData->_rightEyeBlink; + faceTrackerInfo->averageLoudness = _headData->_averageLoudness; + faceTrackerInfo->browAudioLift = _headData->_browAudioLift; + faceTrackerInfo->numBlendshapeCoefficients = _headData->_blendshapeCoefficients.size(); + destinationBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo); + + // followed by a variable number of float coefficients + memcpy(destinationBuffer, _headData->_blendshapeCoefficients.data(), _headData->_blendshapeCoefficients.size() * sizeof(float)); + destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); + + int numBytes = destinationBuffer - startSection; + //_faceTrackerRateOutbound.increment(numBytes); + } + + // If it is connected, pack up the data + if (hasJointData) { + auto startSection = destinationBuffer; + QReadLocker readLock(&_jointDataLock); + + // joint rotation data + int numJoints = _jointData.size(); + *destinationBuffer++ = (uint8_t)numJoints; + + unsigned char* validityPosition = destinationBuffer; + unsigned char validity = 0; + int validityBit = 0; + +#ifdef WANT_DEBUG + int rotationSentCount = 0; + unsigned char* beforeRotations = destinationBuffer; +#endif + + if (sentJointDataOut) { + sentJointDataOut->resize(_jointData.size()); // Make sure the destination is resized before using it + } + float minRotationDOT = !distanceAdjust ? AVATAR_MIN_ROTATION_DOT : getDistanceBasedMinRotationDOT(viewerPosition); + + for (int i = 0; i < _jointData.size(); i++) { + const JointData& data = _jointData[i]; + + // The dot product for smaller rotations is a smaller number. + // So if the dot() is less than the value, then the rotation is a larger angle of rotation + bool largeEnoughRotation = fabsf(glm::dot(data.rotation, lastSentJointData[i].rotation)) < minRotationDOT; + + if (sendAll || lastSentJointData[i].rotation != data.rotation) { + if (sendAll || !cullSmallChanges || largeEnoughRotation) { + if (data.rotationSet) { + validity |= (1 << validityBit); +#ifdef WANT_DEBUG + rotationSentCount++; +#endif + if (sentJointDataOut) { + auto jointDataOut = *sentJointDataOut; + jointDataOut[i].rotation = data.rotation; + } + + } + } + } + if (++validityBit == BITS_IN_BYTE) { + *destinationBuffer++ = validity; + validityBit = validity = 0; + } + } + if (validityBit != 0) { + *destinationBuffer++ = validity; + } + + validityBit = 0; + validity = *validityPosition++; + for (int i = 0; i < _jointData.size(); i++) { + const JointData& data = _jointData[i]; + if (validity & (1 << validityBit)) { + destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, data.rotation); + } + if (++validityBit == BITS_IN_BYTE) { + validityBit = 0; + validity = *validityPosition++; + } + } + + + // joint translation data + validityPosition = destinationBuffer; + validity = 0; + validityBit = 0; + +#ifdef WANT_DEBUG + int translationSentCount = 0; + unsigned char* beforeTranslations = destinationBuffer; +#endif + + float minTranslation = !distanceAdjust ? AVATAR_MIN_TRANSLATION : getDistanceBasedMinTranslationDistance(viewerPosition); + + float maxTranslationDimension = 0.0; + for (int i = 0; i < _jointData.size(); i++) { + const JointData& data = _jointData[i]; + if (sendAll || lastSentJointData[i].translation != data.translation) { + if (sendAll || + !cullSmallChanges || + glm::distance(data.translation, lastSentJointData[i].translation) > minTranslation) { + if (data.translationSet) { + validity |= (1 << validityBit); +#ifdef WANT_DEBUG + translationSentCount++; +#endif + maxTranslationDimension = glm::max(fabsf(data.translation.x), maxTranslationDimension); + maxTranslationDimension = glm::max(fabsf(data.translation.y), maxTranslationDimension); + maxTranslationDimension = glm::max(fabsf(data.translation.z), maxTranslationDimension); + + if (sentJointDataOut) { + auto jointDataOut = *sentJointDataOut; + jointDataOut[i].translation = data.translation; + } + + } + } + } + if (++validityBit == BITS_IN_BYTE) { + *destinationBuffer++ = validity; + validityBit = validity = 0; + } + } + + if (validityBit != 0) { + *destinationBuffer++ = validity; + } + + validityBit = 0; + validity = *validityPosition++; + for (int i = 0; i < _jointData.size(); i++) { + const JointData& data = _jointData[i]; + if (validity & (1 << validityBit)) { + destinationBuffer += + packFloatVec3ToSignedTwoByteFixed(destinationBuffer, data.translation, TRANSLATION_COMPRESSION_RADIX); + } + if (++validityBit == BITS_IN_BYTE) { + validityBit = 0; + validity = *validityPosition++; + } + } + + // faux joints + Transform controllerLeftHandTransform = Transform(getControllerLeftHandMatrix()); + destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, controllerLeftHandTransform.getRotation()); + destinationBuffer += packFloatVec3ToSignedTwoByteFixed(destinationBuffer, controllerLeftHandTransform.getTranslation(), + TRANSLATION_COMPRESSION_RADIX); + Transform controllerRightHandTransform = Transform(getControllerRightHandMatrix()); + destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, controllerRightHandTransform.getRotation()); + destinationBuffer += packFloatVec3ToSignedTwoByteFixed(destinationBuffer, controllerRightHandTransform.getTranslation(), + TRANSLATION_COMPRESSION_RADIX); + +#ifdef WANT_DEBUG + if (sendAll) { + qCDebug(avatars) << "AvatarData::toByteArray" << cullSmallChanges << sendAll + << "rotations:" << rotationSentCount << "translations:" << translationSentCount + << "largest:" << maxTranslationDimension + << "size:" + << (beforeRotations - startPosition) << "+" + << (beforeTranslations - beforeRotations) << "+" + << (destinationBuffer - beforeTranslations) << "=" + << (destinationBuffer - startPosition); + } +#endif + + int numBytes = destinationBuffer - startSection; + //_jointDataRateOutbound.increment(numBytes); + } + + int avatarDataSize = destinationBuffer - startPosition; + return avatarDataByteArray.left(avatarDataSize); +} // NOTE: This is never used in a "distanceAdjust" mode, so it's ok that it doesn't use a variable minimum rotation/translation void AvatarData::doneEncoding(bool cullSmallChanges) { // The server has finished sending this version of the joint-data to other nodes. Update _lastSentJointData. diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 964bc4a6df..45e62d4045 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -354,6 +354,10 @@ public: virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector* sentJointDataOut = nullptr); + // FIXME + virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, + bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) const; + virtual void doneEncoding(bool cullSmallChanges); /// \return true if an error should be logged @@ -380,7 +384,7 @@ public: void nextAttitude(glm::vec3 position, glm::quat orientation); // Can be safely called at any time. virtual void updateAttitude() {} // Tell skeleton mesh about changes - glm::quat getHeadOrientation() { + glm::quat getHeadOrientation() const { lazyInitHeadData(); return _headData->getOrientation(); } @@ -575,10 +579,10 @@ public slots: void resetLastSent() { _lastToByteArray = 0; } protected: - void lazyInitHeadData(); + void lazyInitHeadData() const; - float getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition); - float getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition); + float getDistanceBasedMinRotationDOT(glm::vec3 viewerPosition) const; + float getDistanceBasedMinTranslationDistance(glm::vec3 viewerPosition) const; bool avatarBoundingBoxChangedSince(quint64 time) const { return _avatarBoundingBoxChanged >= time; } bool avatarScaleChangedSince(quint64 time) const { return _avatarScaleChanged >= time; } @@ -614,7 +618,7 @@ protected: bool _forceFaceTrackerConnected; bool _hasNewJointData { true }; // set in AvatarData, cleared in Avatar - HeadData* _headData { nullptr }; + mutable HeadData* _headData { nullptr }; QUrl _skeletonModelURL; bool _firstSkeletonCheck { true }; From 2d300a6643b36687801c5a885c069fb9f361c162 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 15 Feb 2017 22:14:26 -0800 Subject: [PATCH 035/104] checkpoint with eachNode() changed to nestedEach() --- assignment-client/src/avatars/AvatarMixer.cpp | 277 +++++++++--------- assignment-client/src/avatars/AvatarMixer.h | 4 - 2 files changed, 137 insertions(+), 144 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index de5a9b3f2c..7aa0a61de7 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -37,8 +37,7 @@ const int AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND = 45; const unsigned int AVATAR_DATA_SEND_INTERVAL_MSECS = (1.0f / (float) AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND) * 1000; AvatarMixer::AvatarMixer(ReceivedMessage& message) : - ThreadedAssignment(message), - _broadcastThread() + ThreadedAssignment(message) { // make sure we hear about node kills so we can tell the other nodes connect(DependencyManager::get().data(), &NodeList::nodeKilled, this, &AvatarMixer::nodeKilled); @@ -66,12 +65,6 @@ void AvatarMixer::queueIncomingPacket(QSharedPointer message, S AvatarMixer::~AvatarMixer() { - if (_broadcastTimer) { - _broadcastTimer->deleteLater(); - } - - _broadcastThread.quit(); - _broadcastThread.wait(); } // An 80% chance of sending a identity packet within a 5 second interval. @@ -135,6 +128,10 @@ void AvatarMixer::start() { // this is where we need to put the real work... { + // for now, call the single threaded version + broadcastAvatarData(); + + /* auto start = usecTimestampNow(); nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { @@ -197,6 +194,16 @@ void AvatarMixer::manageDisplayName(const SharedNodePointer& node) { } } +void avatarLoops(); + +// only send extra avatar data (avatars out of view, ignored) every Nth AvatarData frame +// Extra avatar data will be sent (AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND/EXTRA_AVATAR_DATA_FRAME_RATIO) times +// per second. +// This value should be a power of two for performance purposes, as the mixer performs a modulo operation every frame +// to determine whether the extra data should be sent. +static const int EXTRA_AVATAR_DATA_FRAME_RATIO = 16; + + // NOTE: some additional optimizations to consider. // 1) use the view frustum to cull those avatars that are out of view. Since avatar data doesn't need to be present // if the avatar is not in view or in the keyhole. @@ -224,13 +231,6 @@ void AvatarMixer::broadcastAvatarData() { const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; - // only send extra avatar data (avatars out of view, ignored) every Nth AvatarData frame - // Extra avatar data will be sent (AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND/EXTRA_AVATAR_DATA_FRAME_RATIO) times - // per second. - // This value should be a power of two for performance purposes, as the mixer performs a modulo operation every frame - // to determine whether the extra data should be sent. - const int EXTRA_AVATAR_DATA_FRAME_RATIO = 16; - // NOTE: The following code calculates the _performanceThrottlingRatio based on how much the avatar-mixer was // able to sleep. This will eventually be used to ask for an additional avatar-mixer to help out. Currently the value // is unused as it is assumed this should not be hit before the avatar-mixer hits the desired bandwidth limit per client. @@ -272,6 +272,35 @@ void AvatarMixer::broadcastAvatarData() { ++framesSinceCutoffEvent; } + avatarLoops(); + + _lastFrameTimestamp = p_high_resolution_clock::now(); + +#ifdef WANT_DEBUG + auto sinceLastDebug = p_high_resolution_clock::now() - _lastDebugMessage; + auto sinceLastDebugUsecs = std::chrono::duration_cast(sinceLastDebug).count(); + quint64 DEBUG_INTERVAL = USECS_PER_SECOND * 5; + + if (sinceLastDebugUsecs > DEBUG_INTERVAL) { + qDebug() << "broadcast rate:" << _broadcastRate.rate() << "hz"; + _lastDebugMessage = p_high_resolution_clock::now(); + } +#endif + + quint64 endBroadcastAvatarData = usecTimestampNow(); + _broadcastAvatarDataElapsedTime += (endBroadcastAvatarData - startBroadcastAvatarData); +} + +void avatarLoopsInner(NodeList::const_iterator cbegin, NodeList::const_iterator cend); + +void avatarLoops() { + auto nodeList = DependencyManager::get(); + nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { + avatarLoopsInner(cbegin, cend); + }); +} + +void avatarLoopsInner(NodeList::const_iterator cbegin, NodeList::const_iterator cend) { auto nodeList = DependencyManager::get(); // setup for distributed random floating point values @@ -279,26 +308,19 @@ void AvatarMixer::broadcastAvatarData() { std::mt19937 generator(randomDevice()); std::uniform_real_distribution distribution; - nodeList->eachMatchingNode( - [&](const SharedNodePointer& node)->bool { - if (!node->getLinkedData()) { - return false; - } - if (node->getType() != NodeType::Agent) { - return false; - } - if (!node->getActiveSocket()) { - return false; - } - return true; - }, - [&](const SharedNodePointer& node) { + std::for_each(cbegin, cend, [&](const SharedNodePointer& node) { + if (node->getLinkedData() && (node->getType() == NodeType::Agent) && node->getActiveSocket()) { AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); MutexTryLocker lock(nodeData->getMutex()); + + // FIXME???? if (!lock.isLocked()) { return; } - ++_sumListeners; + + // FIXME -- mixer data + // ++_sumListeners; + nodeData->resetInViewStats(); const AvatarData& avatar = nodeData->getAvatar(); @@ -349,6 +371,8 @@ void AvatarMixer::broadcastAvatarData() { // get the current full rate distance so we can work with it float currentFullRateDistance = nodeData->getFullRateDistance(); + // FIXME -- mixer data + float _maxKbpsPerNode = 5000.0f; if (avatarDataRateLastSecond > _maxKbpsPerNode) { // is the FRD greater than the farthest avatar? @@ -379,82 +403,81 @@ void AvatarMixer::broadcastAvatarData() { // this is an AGENT we have received head data from // send back a packet with other active node data to this node - nodeList->eachMatchingNode( - [&](const SharedNodePointer& otherNode)->bool { + std::for_each(cbegin, cend, [&](const SharedNodePointer& otherNode) { + bool shouldConsider = false; + quint64 startIgnoreCalculation = usecTimestampNow(); - bool shouldConsider = false; - quint64 startIgnoreCalculation = usecTimestampNow(); + // make sure we have data for this avatar, that it isn't the same node, + // and isn't an avatar that the viewing node has ignored + // or that has ignored the viewing node + if (!otherNode->getLinkedData() + || otherNode->getUUID() == node->getUUID() + || (node->isIgnoringNodeWithID(otherNode->getUUID()) && !getsIgnoredByMe) + || (otherNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { - // make sure we have data for this avatar, that it isn't the same node, - // and isn't an avatar that the viewing node has ignored - // or that has ignored the viewing node - if (!otherNode->getLinkedData() - || otherNode->getUUID() == node->getUUID() - || (node->isIgnoringNodeWithID(otherNode->getUUID()) && !getsIgnoredByMe) - || (otherNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { + shouldConsider = false; - shouldConsider = false; + } else { + AvatarMixerClientData* otherData = reinterpret_cast(otherNode->getLinkedData()); + AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); + // Check to see if the space bubble is enabled + if (node->isIgnoreRadiusEnabled() || otherNode->isIgnoreRadiusEnabled()) { + // Define the minimum bubble size + static const glm::vec3 minBubbleSize = glm::vec3(0.3f, 1.3f, 0.3f); + // Define the scale of the box for the current node + glm::vec3 nodeBoxScale = (nodeData->getPosition() - nodeData->getGlobalBoundingBoxCorner()) * 2.0f; + // Define the scale of the box for the current other node + glm::vec3 otherNodeBoxScale = (otherData->getPosition() - otherData->getGlobalBoundingBoxCorner()) * 2.0f; - } else { - AvatarMixerClientData* otherData = reinterpret_cast(otherNode->getLinkedData()); - AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); - // Check to see if the space bubble is enabled - if (node->isIgnoreRadiusEnabled() || otherNode->isIgnoreRadiusEnabled()) { - // Define the minimum bubble size - static const glm::vec3 minBubbleSize = glm::vec3(0.3f, 1.3f, 0.3f); - // Define the scale of the box for the current node - glm::vec3 nodeBoxScale = (nodeData->getPosition() - nodeData->getGlobalBoundingBoxCorner()) * 2.0f; - // Define the scale of the box for the current other node - glm::vec3 otherNodeBoxScale = (otherData->getPosition() - otherData->getGlobalBoundingBoxCorner()) * 2.0f; - - // Set up the bounding box for the current node - AABox nodeBox(nodeData->getGlobalBoundingBoxCorner(), nodeBoxScale); - // Clamp the size of the bounding box to a minimum scale - if (glm::any(glm::lessThan(nodeBoxScale, minBubbleSize))) { - nodeBox.setScaleStayCentered(minBubbleSize); - } - // Set up the bounding box for the current other node - AABox otherNodeBox(otherData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - // Clamp the size of the bounding box to a minimum scale - if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { - otherNodeBox.setScaleStayCentered(minBubbleSize); - } - // Quadruple the scale of both bounding boxes - nodeBox.embiggen(4.0f); - otherNodeBox.embiggen(4.0f); - - // Perform the collision check between the two bounding boxes - if (nodeBox.touches(otherNodeBox)) { - nodeData->ignoreOther(node, otherNode); - shouldConsider = getsAnyIgnored; - } + // Set up the bounding box for the current node + AABox nodeBox(nodeData->getGlobalBoundingBoxCorner(), nodeBoxScale); + // Clamp the size of the bounding box to a minimum scale + if (glm::any(glm::lessThan(nodeBoxScale, minBubbleSize))) { + nodeBox.setScaleStayCentered(minBubbleSize); } - // Not close enough to ignore - if (shouldConsider) { - nodeData->removeFromRadiusIgnoringSet(node, otherNode->getUUID()); + // Set up the bounding box for the current other node + AABox otherNodeBox(otherData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + // Clamp the size of the bounding box to a minimum scale + if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { + otherNodeBox.setScaleStayCentered(minBubbleSize); } + // Quadruple the scale of both bounding boxes + nodeBox.embiggen(4.0f); + otherNodeBox.embiggen(4.0f); - shouldConsider = true; + // Perform the collision check between the two bounding boxes + if (nodeBox.touches(otherNodeBox)) { + nodeData->ignoreOther(node, otherNode); + shouldConsider = getsAnyIgnored; + } + } + // Not close enough to ignore + if (shouldConsider) { + nodeData->removeFromRadiusIgnoringSet(node, otherNode->getUUID()); } + shouldConsider = true; + quint64 endIgnoreCalculation = usecTimestampNow(); - _ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); - - return shouldConsider; - }, - [&](const SharedNodePointer& otherNode) { + // FIXME -- mixer data + //_ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); + } + if (shouldConsider) { quint64 startAvatarDataPacking = usecTimestampNow(); ++numOtherAvatars; AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); MutexTryLocker lock(otherNodeData->getMutex()); + + // FIXME -- might want to track this lock failed... if (!lock.isLocked()) { - // FIXME -- might want to track this lock failed... quint64 endAvatarDataPacking = usecTimestampNow(); - _avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + + // FIXME - mixer data + //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); return; } @@ -463,9 +486,17 @@ void AvatarMixer::broadcastAvatarData() { if (otherNodeData->getIdentityChangeTimestamp().time_since_epoch().count() > 0 && (forceSend - || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp + //|| otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp // FIXME - mixer data || distribution(generator) < IDENTITY_SEND_PROBABILITY)) { - sendIdentityPacket(otherNodeData, node); + + // FIXME --- used to be.../ mixer data dependency + //sendIdentityPacket(otherNodeData, node); + + QByteArray individualData = otherNodeData->getAvatar().identityByteArray(); + auto identityPacket = NLPacket::create(PacketType::AvatarIdentity, individualData.size()); + individualData.replace(0, NUM_BYTES_RFC4122_UUID, otherNodeData->getNodeID().toRfc4122()); + identityPacket->write(individualData); + DependencyManager::get()->sendPacket(std::move(identityPacket), *node); } const AvatarData& otherAvatar = otherNodeData->getAvatar(); @@ -484,7 +515,8 @@ void AvatarMixer::broadcastAvatarData() { && distribution(generator) > (nodeData->getFullRateDistance() / distanceToAvatar)) { quint64 endAvatarDataPacking = usecTimestampNow(); - _avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + // FIXME - mixer data + //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); return; } @@ -502,7 +534,8 @@ void AvatarMixer::broadcastAvatarData() { ++numAvatarsHeldBack; quint64 endAvatarDataPacking = usecTimestampNow(); - _avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + // FIXME - mixer data + //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); return; } else if (lastSeqFromSender - lastSeqToReceiver > 1) { // this is a skip - we still send the packet but capture the presence of the skip so we see it happening @@ -526,7 +559,8 @@ void AvatarMixer::broadcastAvatarData() { // this throttles the extra data to only be sent every Nth message if (!isInView && getsOutOfView && (lastSeqToReceiver % EXTRA_AVATAR_DATA_FRAME_RATIO > 0)) { quint64 endAvatarDataPacking = usecTimestampNow(); - _avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + // FIXME - mixer data + //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); return; } @@ -554,7 +588,9 @@ void AvatarMixer::broadcastAvatarData() { avatarPacketList->endSegment(); quint64 endAvatarDataPacking = usecTimestampNow(); - _avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + // FIXME - mixer data + //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + } }); quint64 startPacketSending = usecTimestampNow(); @@ -580,30 +616,19 @@ void AvatarMixer::broadcastAvatarData() { } quint64 endPacketSending = usecTimestampNow(); - _packetSendingElapsedTime += (endPacketSending - startPacketSending); + // FIXME - mixer data + //_packetSendingElapsedTime += (endPacketSending - startPacketSending); } - ); + }); // We're done encoding this version of the otherAvatars. Update their "lastSent" joint-states so // that we can notice differences, next time around. // // FIXME - this seems suspicious, the code seems to consider all avatars, but not all avatars will // have had their joints sent, so actually we should consider the time since they actually were sent???? - nodeList->eachMatchingNode( - [&](const SharedNodePointer& otherNode)->bool { - if (!otherNode->getLinkedData()) { - return false; - } - if (otherNode->getType() != NodeType::Agent) { - return false; - } - if (!otherNode->getActiveSocket()) { - return false; - } - return true; - }, - [&](const SharedNodePointer& otherNode) { + std::for_each(cbegin, cend, [&](const SharedNodePointer& otherNode) { + if (otherNode->getLinkedData() && (otherNode->getType() == NodeType::Agent) && otherNode->getActiveSocket()) { AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); MutexTryLocker lock(otherNodeData->getMutex()); if (!lock.isLocked()) { @@ -611,24 +636,8 @@ void AvatarMixer::broadcastAvatarData() { } AvatarData& otherAvatar = otherNodeData->getAvatar(); otherAvatar.doneEncoding(false); - }); - - _lastFrameTimestamp = p_high_resolution_clock::now(); - -#ifdef WANT_DEBUG - auto sinceLastDebug = p_high_resolution_clock::now() - _lastDebugMessage; - auto sinceLastDebugUsecs = std::chrono::duration_cast(sinceLastDebug).count(); - quint64 DEBUG_INTERVAL = USECS_PER_SECOND * 5; - - if (sinceLastDebugUsecs > DEBUG_INTERVAL) { - qDebug() << "broadcast rate:" << _broadcastRate.rate() << "hz"; - _lastDebugMessage = p_high_resolution_clock::now(); - } -#endif - - quint64 endBroadcastAvatarData = usecTimestampNow(); - _broadcastAvatarDataElapsedTime += (endBroadcastAvatarData - startBroadcastAvatarData); - + } + }); } void AvatarMixer::nodeKilled(SharedNodePointer killedNode) { @@ -894,18 +903,6 @@ void AvatarMixer::run() { ThreadedAssignment::commonInit(AVATAR_MIXER_LOGGING_NAME, NodeType::AvatarMixer); - // setup the timer that will be fired on the broadcast thread - _broadcastTimer = new QTimer; - _broadcastTimer->setTimerType(Qt::PreciseTimer); - _broadcastTimer->setInterval(AVATAR_DATA_SEND_INTERVAL_MSECS); - _broadcastTimer->moveToThread(&_broadcastThread); - - // connect appropriate signals and slots - connect(_broadcastTimer, &QTimer::timeout, this, &AvatarMixer::broadcastAvatarData, Qt::DirectConnection); - connect(&_broadcastThread, SIGNAL(started()), _broadcastTimer, SLOT(start())); - - // start our tight loop... - start(); } AvatarMixerClientData* AvatarMixer::getOrCreateClientData(SharedNodePointer node) { @@ -929,8 +926,8 @@ void AvatarMixer::domainSettingsRequestComplete() { // parse the settings to pull out the values we need parseDomainServerSettings(nodeList->getDomainHandler().getSettingsObject()); - // start the broadcastThread - _broadcastThread.start(); + // start our tight loop... + start(); } void AvatarMixer::handlePacketVersionMismatch(PacketType type, const HifiSockAddr& senderSockAddr, const QUuid& senderUUID) { diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index e0b03804a9..628c44abaf 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -62,8 +62,6 @@ private: void manageDisplayName(const SharedNodePointer& node); - QThread _broadcastThread; - p_high_resolution_clock::time_point _lastFrameTimestamp; float _trailingSleepRatio { 1.0f }; @@ -79,8 +77,6 @@ private: float _domainMinimumScale { MIN_AVATAR_SCALE }; float _domainMaximumScale { MAX_AVATAR_SCALE }; - QTimer* _broadcastTimer = nullptr; - RateCounter<> _broadcastRate; p_high_resolution_clock::time_point _lastDebugMessage; QHash> _sessionDisplayNames; From 23790b93e3e1f36fc5ce85ad8cf587bc83239989 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 16 Feb 2017 01:07:21 -0800 Subject: [PATCH 036/104] first cut at slaves doing broadcast --- assignment-client/src/avatars/AvatarMixer.cpp | 19 +- .../src/avatars/AvatarMixerClientData.h | 6 +- .../src/avatars/AvatarMixerSlave.cpp | 349 ++++++++++++++++++ .../src/avatars/AvatarMixerSlavePool.cpp | 4 +- 4 files changed, 362 insertions(+), 16 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 7aa0a61de7..626c001218 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -129,17 +129,14 @@ void AvatarMixer::start() { // this is where we need to put the real work... { // for now, call the single threaded version - broadcastAvatarData(); + //broadcastAvatarData(); - - /* auto start = usecTimestampNow(); nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { - _slavePool.processIncomingPackets(cbegin, cend); + _slavePool.anotherJob(cbegin, cend); }); auto end = usecTimestampNow(); - _processQueuedAvatarDataPacketsElapsedTime += (end - start); - */ + _broadcastAvatarDataElapsedTime += (end - start); } @@ -194,7 +191,7 @@ void AvatarMixer::manageDisplayName(const SharedNodePointer& node) { } } -void avatarLoops(); +static void avatarLoops(); // only send extra avatar data (avatars out of view, ignored) every Nth AvatarData frame // Extra avatar data will be sent (AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND/EXTRA_AVATAR_DATA_FRAME_RATIO) times @@ -291,16 +288,16 @@ void AvatarMixer::broadcastAvatarData() { _broadcastAvatarDataElapsedTime += (endBroadcastAvatarData - startBroadcastAvatarData); } -void avatarLoopsInner(NodeList::const_iterator cbegin, NodeList::const_iterator cend); +static void avatarLoopsInner(NodeList::const_iterator cbegin, NodeList::const_iterator cend); -void avatarLoops() { +static void avatarLoops() { auto nodeList = DependencyManager::get(); nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { avatarLoopsInner(cbegin, cend); }); } -void avatarLoopsInner(NodeList::const_iterator cbegin, NodeList::const_iterator cend) { +static void avatarLoopsInner(NodeList::const_iterator cbegin, NodeList::const_iterator cend) { auto nodeList = DependencyManager::get(); // setup for distributed random floating point values @@ -789,7 +786,6 @@ void AvatarMixer::sendStatsPacket() { statsObject["timing_average_b_ignoreCalculation"] = (float)_ignoreCalculationElapsedTime / (float)_numStatFrames; statsObject["timing_average_c_avatarDataPacking"] = (float)_avatarDataPackingElapsedTime / (float)_numStatFrames; statsObject["timing_average_d_packetSending"] = (float)_packetSendingElapsedTime / (float)_numStatFrames; - statsObject["timing_average_e_total_broadcastAvatarData"] = (float)_broadcastAvatarDataElapsedTime / (float)_numStatFrames; // this things all occur on the frequency of the tight loop int tightLoopFrames = _numTightLoopFrames; @@ -799,6 +795,7 @@ void AvatarMixer::sendStatsPacket() { statsObject["timing_average_y_processEvents"] = TIGHT_LOOP_STAT(_processEventsElapsedTime); statsObject["timing_average_y_queueIncomingPacket"] = TIGHT_LOOP_STAT(_queueIncomingPacketElapsedTime); + statsObject["timing_average_z_broadcastAvatarData"] = TIGHT_LOOP_STAT(_broadcastAvatarDataElapsedTime); statsObject["timing_average_z_displayNameManagement"] = TIGHT_LOOP_STAT(_displayNameManagementElapsedTime); statsObject["timing_average_z_handleAvatarDataPacket"] = TIGHT_LOOP_STAT(_handleAvatarDataPacketElapsedTime); statsObject["timing_average_z_handleAvatarIdentityPacket"] = TIGHT_LOOP_STAT(_handleAvatarIdentityPacketElapsedTime); diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index c69599cc26..8cd72050f7 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -86,9 +86,9 @@ public: void loadJSONStats(QJsonObject& jsonObject) const; - glm::vec3 getPosition() { return _avatar ? _avatar->getPosition() : glm::vec3(0); } - glm::vec3 getGlobalBoundingBoxCorner() { return _avatar ? _avatar->getGlobalBoundingBoxCorner() : glm::vec3(0); } - bool isRadiusIgnoring(const QUuid& other) { return _radiusIgnoredOthers.find(other) != _radiusIgnoredOthers.end(); } + glm::vec3 getPosition() const { return _avatar ? _avatar->getPosition() : glm::vec3(0); } + glm::vec3 getGlobalBoundingBoxCorner() const { return _avatar ? _avatar->getGlobalBoundingBoxCorner() : glm::vec3(0); } + bool isRadiusIgnoring(const QUuid& other) const { return _radiusIgnoredOthers.find(other) != _radiusIgnoredOthers.end(); } void addToRadiusIgnoringSet(const QUuid& other) { _radiusIgnoredOthers.insert(other); } void removeFromRadiusIgnoringSet(SharedNodePointer self, const QUuid& other); void ignoreOther(SharedNodePointer self, SharedNodePointer other); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 784b10ebfe..37fd258bfa 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -55,5 +55,354 @@ void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { _processIncomingPacketsElapsedTime += (end - start); } +#include +#include + +static const int AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND = 45; +static const unsigned int AVATAR_DATA_SEND_INTERVAL_MSECS = (1.0f / (float)AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND) * 1000; + +// only send extra avatar data (avatars out of view, ignored) every Nth AvatarData frame +// Extra avatar data will be sent (AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND/EXTRA_AVATAR_DATA_FRAME_RATIO) times +// per second. +// This value should be a power of two for performance purposes, as the mixer performs a modulo operation every frame +// to determine whether the extra data should be sent. +static const int EXTRA_AVATAR_DATA_FRAME_RATIO = 16; + +// An 80% chance of sending a identity packet within a 5 second interval. +// assuming 60 htz update rate. +const float IDENTITY_SEND_PROBABILITY = 1.0f / 187.0f; // FIXME... this is wrong for 45hz + void AvatarMixerSlave::anotherJob(const SharedNodePointer& node) { + //qDebug() << __FUNCTION__ << "node:" << node; + + auto nodeList = DependencyManager::get(); + + // setup for distributed random floating point values + std::random_device randomDevice; + std::mt19937 generator(randomDevice()); + std::uniform_real_distribution distribution; + + if (node->getLinkedData() && (node->getType() == NodeType::Agent) && node->getActiveSocket()) { + AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); + MutexTryLocker lock(nodeData->getMutex()); + + // FIXME???? + if (!lock.isLocked()) { + //qDebug() << __FUNCTION__ << "unable to lock... node:" << node << " would BAIL???... line:" << __LINE__; + //return; + } + + // FIXME -- mixer data + // ++_sumListeners; + + nodeData->resetInViewStats(); + + const AvatarData& avatar = nodeData->getAvatar(); + glm::vec3 myPosition = avatar.getClientGlobalPosition(); + + // reset the internal state for correct random number distribution + distribution.reset(); + + // reset the max distance for this frame + float maxAvatarDistanceThisFrame = 0.0f; + + // reset the number of sent avatars + nodeData->resetNumAvatarsSentLastFrame(); + + // keep a counter of the number of considered avatars + int numOtherAvatars = 0; + + // keep track of outbound data rate specifically for avatar data + int numAvatarDataBytes = 0; + + // keep track of the number of other avatars held back in this frame + int numAvatarsHeldBack = 0; + + // keep track of the number of other avatar frames skipped + int numAvatarsWithSkippedFrames = 0; + + // use the data rate specifically for avatar data for FRD adjustment checks + float avatarDataRateLastSecond = nodeData->getOutboundAvatarDataKbps(); + + // When this is true, the AvatarMixer will send Avatar data to a client about avatars that are not in the view frustrum + bool getsOutOfView = nodeData->getRequestsDomainListData(); + + // When this is true, the AvatarMixer will send Avatar data to a client about avatars that they've ignored + bool getsIgnoredByMe = getsOutOfView; + + // When this is true, the AvatarMixer will send Avatar data to a client about avatars that have ignored them + bool getsAnyIgnored = getsIgnoredByMe && node->getCanKick(); + + // Check if it is time to adjust what we send this client based on the observed + // bandwidth to this node. We do this once a second, which is also the window for + // the bandwidth reported by node->getOutboundBandwidth(); + if (nodeData->getNumFramesSinceFRDAdjustment() > AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND) { + + const float FRD_ADJUSTMENT_ACCEPTABLE_RATIO = 0.8f; + const float HYSTERISIS_GAP = (1 - FRD_ADJUSTMENT_ACCEPTABLE_RATIO); + const float HYSTERISIS_MIDDLE_PERCENTAGE = (1 - (HYSTERISIS_GAP * 0.5f)); + + // get the current full rate distance so we can work with it + float currentFullRateDistance = nodeData->getFullRateDistance(); + + // FIXME -- mixer data + float _maxKbpsPerNode = 5000.0f; + if (avatarDataRateLastSecond > _maxKbpsPerNode) { + + // is the FRD greater than the farthest avatar? + // if so, before we calculate anything, set it to that distance + currentFullRateDistance = std::min(currentFullRateDistance, nodeData->getMaxAvatarDistance()); + + // we're adjusting the full rate distance to target a bandwidth in the middle + // of the hysterisis gap + currentFullRateDistance *= (_maxKbpsPerNode * HYSTERISIS_MIDDLE_PERCENTAGE) / avatarDataRateLastSecond; + + nodeData->setFullRateDistance(currentFullRateDistance); + nodeData->resetNumFramesSinceFRDAdjustment(); + } else if (currentFullRateDistance < nodeData->getMaxAvatarDistance() + && avatarDataRateLastSecond < _maxKbpsPerNode * FRD_ADJUSTMENT_ACCEPTABLE_RATIO) { + // we are constrained AND we've recovered to below the acceptable ratio + // lets adjust the full rate distance to target a bandwidth in the middle of the hyterisis gap + currentFullRateDistance *= (_maxKbpsPerNode * HYSTERISIS_MIDDLE_PERCENTAGE) / avatarDataRateLastSecond; + + nodeData->setFullRateDistance(currentFullRateDistance); + nodeData->resetNumFramesSinceFRDAdjustment(); + } + } else { + nodeData->incrementNumFramesSinceFRDAdjustment(); + } + + // setup a PacketList for the avatarPackets + auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); + + // this is an AGENT we have received head data from + // send back a packet with other active node data to this node + std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) { + //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode; + + bool shouldConsider = false; + quint64 startIgnoreCalculation = usecTimestampNow(); + + // make sure we have data for this avatar, that it isn't the same node, + // and isn't an avatar that the viewing node has ignored + // or that has ignored the viewing node + if (!otherNode->getLinkedData() + || otherNode->getUUID() == node->getUUID() + || (node->isIgnoringNodeWithID(otherNode->getUUID()) && !getsIgnoredByMe) + || (otherNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { + + shouldConsider = false; + + } else { + const AvatarMixerClientData* otherData = reinterpret_cast(otherNode->getLinkedData()); + //AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); + // Check to see if the space bubble is enabled + if (node->isIgnoreRadiusEnabled() || otherNode->isIgnoreRadiusEnabled()) { + // Define the minimum bubble size + static const glm::vec3 minBubbleSize = glm::vec3(0.3f, 1.3f, 0.3f); + // Define the scale of the box for the current node + glm::vec3 nodeBoxScale = (nodeData->getPosition() - nodeData->getGlobalBoundingBoxCorner()) * 2.0f; + // Define the scale of the box for the current other node + glm::vec3 otherNodeBoxScale = (otherData->getPosition() - otherData->getGlobalBoundingBoxCorner()) * 2.0f; + + // Set up the bounding box for the current node + AABox nodeBox(nodeData->getGlobalBoundingBoxCorner(), nodeBoxScale); + // Clamp the size of the bounding box to a minimum scale + if (glm::any(glm::lessThan(nodeBoxScale, minBubbleSize))) { + nodeBox.setScaleStayCentered(minBubbleSize); + } + // Set up the bounding box for the current other node + AABox otherNodeBox(otherData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + // Clamp the size of the bounding box to a minimum scale + if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { + otherNodeBox.setScaleStayCentered(minBubbleSize); + } + // Quadruple the scale of both bounding boxes + nodeBox.embiggen(4.0f); + otherNodeBox.embiggen(4.0f); + + // Perform the collision check between the two bounding boxes + if (nodeBox.touches(otherNodeBox)) { + nodeData->ignoreOther(node, otherNode); + shouldConsider = getsAnyIgnored; + } + } + // Not close enough to ignore + if (shouldConsider) { + nodeData->removeFromRadiusIgnoringSet(node, otherNode->getUUID()); + } + + shouldConsider = true; + + quint64 endIgnoreCalculation = usecTimestampNow(); + // FIXME -- mixer data + //_ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); + } + + //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << "shouldConsider:" << shouldConsider; + + + if (shouldConsider) { + quint64 startAvatarDataPacking = usecTimestampNow(); + + ++numOtherAvatars; + + AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); + MutexTryLocker lock(otherNodeData->getMutex()); + + // FIXME -- might want to track this lock failed... + if (!lock.isLocked()) { + + quint64 endAvatarDataPacking = usecTimestampNow(); + + // FIXME - mixer data + //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << " failed to lock... would BAIL??... line:" << __LINE__; + //return; + } + + // make sure we send out identity packets to and from new arrivals. + bool forceSend = !otherNodeData->checkAndSetHasReceivedFirstPacketsFrom(node->getUUID()); + + if (otherNodeData->getIdentityChangeTimestamp().time_since_epoch().count() > 0 + && (forceSend + //|| otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp // FIXME - mixer data + || distribution(generator) < IDENTITY_SEND_PROBABILITY)) { + + // FIXME --- used to be.../ mixer data dependency + //sendIdentityPacket(otherNodeData, node); + + QByteArray individualData = otherNodeData->getAvatar().identityByteArray(); + auto identityPacket = NLPacket::create(PacketType::AvatarIdentity, individualData.size()); + individualData.replace(0, NUM_BYTES_RFC4122_UUID, otherNodeData->getNodeID().toRfc4122()); + identityPacket->write(individualData); + DependencyManager::get()->sendPacket(std::move(identityPacket), *node); + //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << " sending itentity packet for otherNode to node..."; + } + + const AvatarData& otherAvatar = otherNodeData->getAvatar(); + // Decide whether to send this avatar's data based on it's distance from us + + // The full rate distance is the distance at which EVERY update will be sent for this avatar + // at twice the full rate distance, there will be a 50% chance of sending this avatar's update + glm::vec3 otherPosition = otherAvatar.getClientGlobalPosition(); + float distanceToAvatar = glm::length(myPosition - otherPosition); + + // potentially update the max full rate distance for this frame + maxAvatarDistanceThisFrame = std::max(maxAvatarDistanceThisFrame, distanceToAvatar); + + if (distanceToAvatar != 0.0f + && !getsOutOfView + && distribution(generator) > (nodeData->getFullRateDistance() / distanceToAvatar)) { + + quint64 endAvatarDataPacking = usecTimestampNow(); + // FIXME - mixer data + //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << " distance/getsOutOfView... BAILING... line:" << __LINE__; + return; + } + + AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(otherNode->getUUID()); + AvatarDataSequenceNumber lastSeqFromSender = otherNodeData->getLastReceivedSequenceNumber(); + + if (lastSeqToReceiver > lastSeqFromSender && lastSeqToReceiver != UINT16_MAX) { + // we got out out of order packets from the sender, track it + otherNodeData->incrementNumOutOfOrderSends(); + } + + // make sure we haven't already sent this data from this sender to this receiver + // or that somehow we haven't sent + if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { + ++numAvatarsHeldBack; + + quint64 endAvatarDataPacking = usecTimestampNow(); + // FIXME - mixer data + //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << " lastSeqToReceiver... BAILING... line:" << __LINE__; + return; + } else if (lastSeqFromSender - lastSeqToReceiver > 1) { + // this is a skip - we still send the packet but capture the presence of the skip so we see it happening + ++numAvatarsWithSkippedFrames; + } + + // we're going to send this avatar + + // increment the number of avatars sent to this reciever + nodeData->incrementNumAvatarsSentLastFrame(); + + // set the last sent sequence number for this sender on the receiver + nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), + otherNodeData->getLastReceivedSequenceNumber()); + + // determine if avatar is in view, to determine how much data to include... + glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; + AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + bool isInView = nodeData->otherAvatarInView(otherNodeBox); + + // this throttles the extra data to only be sent every Nth message + if (!isInView && getsOutOfView && (lastSeqToReceiver % EXTRA_AVATAR_DATA_FRAME_RATIO > 0)) { + quint64 endAvatarDataPacking = usecTimestampNow(); + // FIXME - mixer data + //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << " isInView && getsOutOfView && (lastSeqToReceiver % EXTRA_AVATAR_DATA_FRAME_RATIO > 0)... BAILING... line:" << __LINE__; + return; + } + + // start a new segment in the PacketList for this avatar + avatarPacketList->startSegment(); + + AvatarData::AvatarDataDetail detail; + if (!isInView && !getsOutOfView) { + detail = AvatarData::MinimumData; + nodeData->incrementAvatarOutOfView(); + } else { + detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO + ? AvatarData::SendAllData : AvatarData::CullSmallData; + nodeData->incrementAvatarInView(); + } + + numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); + auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); + QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); + bool distanceAdjust = true; + glm::vec3 viewerPosition = myPosition; + QByteArray bytes = otherAvatar.toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, distanceAdjust, viewerPosition, &lastSentJointsForOther); + numAvatarDataBytes += avatarPacketList->write(bytes); + + avatarPacketList->endSegment(); + + quint64 endAvatarDataPacking = usecTimestampNow(); + // FIXME - mixer data + //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + } + }); + + quint64 startPacketSending = usecTimestampNow(); + + // close the current packet so that we're always sending something + avatarPacketList->closeCurrentPacket(true); + + // send the avatar data PacketList + //qDebug() << "about to call nodeList->sendPacketList() for node:" << node; + nodeList->sendPacketList(std::move(avatarPacketList), *node); + + // record the bytes sent for other avatar data in the AvatarMixerClientData + nodeData->recordSentAvatarData(numAvatarDataBytes); + + // record the number of avatars held back this frame + nodeData->recordNumOtherAvatarStarves(numAvatarsHeldBack); + nodeData->recordNumOtherAvatarSkips(numAvatarsWithSkippedFrames); + + if (numOtherAvatars == 0) { + // update the full rate distance to FLOAT_MAX since we didn't have any other avatars to send + nodeData->setMaxAvatarDistance(FLT_MAX); + } else { + nodeData->setMaxAvatarDistance(maxAvatarDistanceThisFrame); + } + + quint64 endPacketSending = usecTimestampNow(); + // FIXME - mixer data + //_packetSendingElapsedTime += (endPacketSending - startPacketSending); + } } + diff --git a/assignment-client/src/avatars/AvatarMixerSlavePool.cpp b/assignment-client/src/avatars/AvatarMixerSlavePool.cpp index 1b335f8383..28a3dce0e3 100644 --- a/assignment-client/src/avatars/AvatarMixerSlavePool.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlavePool.cpp @@ -69,13 +69,13 @@ static AvatarMixerSlave slave; void AvatarMixerSlavePool::processIncomingPackets(ConstIter begin, ConstIter end) { _function = &AvatarMixerSlave::processIncomingPackets; - _configure = [](AvatarMixerSlave& slave) {}; + _configure = [&](AvatarMixerSlave& slave) { slave.configure(begin, end); }; run(begin, end); } void AvatarMixerSlavePool::anotherJob(ConstIter begin, ConstIter end) { _function = &AvatarMixerSlave::anotherJob; - _configure = [](AvatarMixerSlave& slave) {}; + _configure = [&](AvatarMixerSlave& slave) { slave.configure(begin, end); }; run(begin, end); } From 755c690030c2bb0949da51716ff52e72829a078e Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 16 Feb 2017 10:07:49 -0800 Subject: [PATCH 037/104] some cleanup, better sleep management --- assignment-client/src/avatars/AvatarMixer.cpp | 432 +++--------------- assignment-client/src/avatars/AvatarMixer.h | 2 + .../src/avatars/AvatarMixerSlave.cpp | 177 ++++--- .../src/avatars/AvatarMixerSlave.h | 37 +- 4 files changed, 182 insertions(+), 466 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 626c001218..73b0514161 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -88,17 +88,51 @@ void AvatarMixer::sendIdentityPacket(AvatarMixerClientData* nodeData, const Shar #include #include +std::chrono::microseconds AvatarMixer::timeFrame(p_high_resolution_clock::time_point& timestamp) { + // advance the next frame + auto nextTimestamp = timestamp + std::chrono::microseconds((int)((float)USECS_PER_SECOND / (float)AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND)); + auto now = p_high_resolution_clock::now(); + + // compute how long the last frame took + auto duration = std::chrono::duration_cast(now - timestamp); + + // set the new frame timestamp + timestamp = std::max(now, nextTimestamp); + + // sleep until the next frame should start + // WIN32 sleep_until is broken until VS2015 Update 2 + // instead, std::max (above) guarantees that timestamp >= now, so we can sleep_for + std::this_thread::sleep_for(timestamp - now); + + return duration; +} + + void AvatarMixer::start() { auto nodeList = DependencyManager::get(); + auto frameTimestamp = p_high_resolution_clock::now(); + while (!_isFinished) { _numTightLoopFrames++; _loopRate.increment(); - // FIXME - we really should sleep for the remainder of what we haven't spent time processing - auto sleepAmount = std::chrono::milliseconds(AVATAR_DATA_SEND_INTERVAL_MSECS); - std::this_thread::sleep_for(sleepAmount); + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // DO THIS FIRST!!!!!!!! + // + // DONE --- 1) only sleep for remainder + // 2) clean up stats, add slave stats + // 3) delete dead code from mixer (now that it's in slave) + // 4) audit the locking and side-effects to node, otherNode, and nodeData + // 5) throttling?? + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + // calculates last frame duration and sleeps for the remainder of the target amount + auto frameDuration = timeFrame(frameTimestamp); // Allow nodes to process any pending/queued packets across our worker threads { @@ -201,9 +235,9 @@ static void avatarLoops(); static const int EXTRA_AVATAR_DATA_FRAME_RATIO = 16; -// NOTE: some additional optimizations to consider. -// 1) use the view frustum to cull those avatars that are out of view. Since avatar data doesn't need to be present -// if the avatar is not in view or in the keyhole. +// FIXME -- this is dead code... it needs to be removed... +// this "throttle" logic is the old approach. need to consider some +// reasonable throttle approach in new multi-core design void AvatarMixer::broadcastAvatarData() { quint64 startBroadcastAvatarData = usecTimestampNow(); _broadcastRate.increment(); @@ -269,7 +303,7 @@ void AvatarMixer::broadcastAvatarData() { ++framesSinceCutoffEvent; } - avatarLoops(); + //avatarLoops(); _lastFrameTimestamp = p_high_resolution_clock::now(); @@ -288,355 +322,6 @@ void AvatarMixer::broadcastAvatarData() { _broadcastAvatarDataElapsedTime += (endBroadcastAvatarData - startBroadcastAvatarData); } -static void avatarLoopsInner(NodeList::const_iterator cbegin, NodeList::const_iterator cend); - -static void avatarLoops() { - auto nodeList = DependencyManager::get(); - nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { - avatarLoopsInner(cbegin, cend); - }); -} - -static void avatarLoopsInner(NodeList::const_iterator cbegin, NodeList::const_iterator cend) { - auto nodeList = DependencyManager::get(); - - // setup for distributed random floating point values - std::random_device randomDevice; - std::mt19937 generator(randomDevice()); - std::uniform_real_distribution distribution; - - std::for_each(cbegin, cend, [&](const SharedNodePointer& node) { - if (node->getLinkedData() && (node->getType() == NodeType::Agent) && node->getActiveSocket()) { - AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); - MutexTryLocker lock(nodeData->getMutex()); - - // FIXME???? - if (!lock.isLocked()) { - return; - } - - // FIXME -- mixer data - // ++_sumListeners; - - nodeData->resetInViewStats(); - - const AvatarData& avatar = nodeData->getAvatar(); - glm::vec3 myPosition = avatar.getClientGlobalPosition(); - - // reset the internal state for correct random number distribution - distribution.reset(); - - // reset the max distance for this frame - float maxAvatarDistanceThisFrame = 0.0f; - - // reset the number of sent avatars - nodeData->resetNumAvatarsSentLastFrame(); - - // keep a counter of the number of considered avatars - int numOtherAvatars = 0; - - // keep track of outbound data rate specifically for avatar data - int numAvatarDataBytes = 0; - - // keep track of the number of other avatars held back in this frame - int numAvatarsHeldBack = 0; - - // keep track of the number of other avatar frames skipped - int numAvatarsWithSkippedFrames = 0; - - // use the data rate specifically for avatar data for FRD adjustment checks - float avatarDataRateLastSecond = nodeData->getOutboundAvatarDataKbps(); - - // When this is true, the AvatarMixer will send Avatar data to a client about avatars that are not in the view frustrum - bool getsOutOfView = nodeData->getRequestsDomainListData(); - - // When this is true, the AvatarMixer will send Avatar data to a client about avatars that they've ignored - bool getsIgnoredByMe = getsOutOfView; - - // When this is true, the AvatarMixer will send Avatar data to a client about avatars that have ignored them - bool getsAnyIgnored = getsIgnoredByMe && node->getCanKick(); - - // Check if it is time to adjust what we send this client based on the observed - // bandwidth to this node. We do this once a second, which is also the window for - // the bandwidth reported by node->getOutboundBandwidth(); - if (nodeData->getNumFramesSinceFRDAdjustment() > AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND) { - - const float FRD_ADJUSTMENT_ACCEPTABLE_RATIO = 0.8f; - const float HYSTERISIS_GAP = (1 - FRD_ADJUSTMENT_ACCEPTABLE_RATIO); - const float HYSTERISIS_MIDDLE_PERCENTAGE = (1 - (HYSTERISIS_GAP * 0.5f)); - - // get the current full rate distance so we can work with it - float currentFullRateDistance = nodeData->getFullRateDistance(); - - // FIXME -- mixer data - float _maxKbpsPerNode = 5000.0f; - if (avatarDataRateLastSecond > _maxKbpsPerNode) { - - // is the FRD greater than the farthest avatar? - // if so, before we calculate anything, set it to that distance - currentFullRateDistance = std::min(currentFullRateDistance, nodeData->getMaxAvatarDistance()); - - // we're adjusting the full rate distance to target a bandwidth in the middle - // of the hysterisis gap - currentFullRateDistance *= (_maxKbpsPerNode * HYSTERISIS_MIDDLE_PERCENTAGE) / avatarDataRateLastSecond; - - nodeData->setFullRateDistance(currentFullRateDistance); - nodeData->resetNumFramesSinceFRDAdjustment(); - } else if (currentFullRateDistance < nodeData->getMaxAvatarDistance() - && avatarDataRateLastSecond < _maxKbpsPerNode * FRD_ADJUSTMENT_ACCEPTABLE_RATIO) { - // we are constrained AND we've recovered to below the acceptable ratio - // lets adjust the full rate distance to target a bandwidth in the middle of the hyterisis gap - currentFullRateDistance *= (_maxKbpsPerNode * HYSTERISIS_MIDDLE_PERCENTAGE) / avatarDataRateLastSecond; - - nodeData->setFullRateDistance(currentFullRateDistance); - nodeData->resetNumFramesSinceFRDAdjustment(); - } - } else { - nodeData->incrementNumFramesSinceFRDAdjustment(); - } - - // setup a PacketList for the avatarPackets - auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); - - // this is an AGENT we have received head data from - // send back a packet with other active node data to this node - std::for_each(cbegin, cend, [&](const SharedNodePointer& otherNode) { - bool shouldConsider = false; - quint64 startIgnoreCalculation = usecTimestampNow(); - - // make sure we have data for this avatar, that it isn't the same node, - // and isn't an avatar that the viewing node has ignored - // or that has ignored the viewing node - if (!otherNode->getLinkedData() - || otherNode->getUUID() == node->getUUID() - || (node->isIgnoringNodeWithID(otherNode->getUUID()) && !getsIgnoredByMe) - || (otherNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { - - shouldConsider = false; - - } else { - AvatarMixerClientData* otherData = reinterpret_cast(otherNode->getLinkedData()); - AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); - // Check to see if the space bubble is enabled - if (node->isIgnoreRadiusEnabled() || otherNode->isIgnoreRadiusEnabled()) { - // Define the minimum bubble size - static const glm::vec3 minBubbleSize = glm::vec3(0.3f, 1.3f, 0.3f); - // Define the scale of the box for the current node - glm::vec3 nodeBoxScale = (nodeData->getPosition() - nodeData->getGlobalBoundingBoxCorner()) * 2.0f; - // Define the scale of the box for the current other node - glm::vec3 otherNodeBoxScale = (otherData->getPosition() - otherData->getGlobalBoundingBoxCorner()) * 2.0f; - - // Set up the bounding box for the current node - AABox nodeBox(nodeData->getGlobalBoundingBoxCorner(), nodeBoxScale); - // Clamp the size of the bounding box to a minimum scale - if (glm::any(glm::lessThan(nodeBoxScale, minBubbleSize))) { - nodeBox.setScaleStayCentered(minBubbleSize); - } - // Set up the bounding box for the current other node - AABox otherNodeBox(otherData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - // Clamp the size of the bounding box to a minimum scale - if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { - otherNodeBox.setScaleStayCentered(minBubbleSize); - } - // Quadruple the scale of both bounding boxes - nodeBox.embiggen(4.0f); - otherNodeBox.embiggen(4.0f); - - // Perform the collision check between the two bounding boxes - if (nodeBox.touches(otherNodeBox)) { - nodeData->ignoreOther(node, otherNode); - shouldConsider = getsAnyIgnored; - } - } - // Not close enough to ignore - if (shouldConsider) { - nodeData->removeFromRadiusIgnoringSet(node, otherNode->getUUID()); - } - - shouldConsider = true; - - quint64 endIgnoreCalculation = usecTimestampNow(); - // FIXME -- mixer data - //_ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); - } - - if (shouldConsider) { - quint64 startAvatarDataPacking = usecTimestampNow(); - - ++numOtherAvatars; - - AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); - MutexTryLocker lock(otherNodeData->getMutex()); - - // FIXME -- might want to track this lock failed... - if (!lock.isLocked()) { - - quint64 endAvatarDataPacking = usecTimestampNow(); - - // FIXME - mixer data - //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - return; - } - - // make sure we send out identity packets to and from new arrivals. - bool forceSend = !otherNodeData->checkAndSetHasReceivedFirstPacketsFrom(node->getUUID()); - - if (otherNodeData->getIdentityChangeTimestamp().time_since_epoch().count() > 0 - && (forceSend - //|| otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp // FIXME - mixer data - || distribution(generator) < IDENTITY_SEND_PROBABILITY)) { - - // FIXME --- used to be.../ mixer data dependency - //sendIdentityPacket(otherNodeData, node); - - QByteArray individualData = otherNodeData->getAvatar().identityByteArray(); - auto identityPacket = NLPacket::create(PacketType::AvatarIdentity, individualData.size()); - individualData.replace(0, NUM_BYTES_RFC4122_UUID, otherNodeData->getNodeID().toRfc4122()); - identityPacket->write(individualData); - DependencyManager::get()->sendPacket(std::move(identityPacket), *node); - } - - const AvatarData& otherAvatar = otherNodeData->getAvatar(); - // Decide whether to send this avatar's data based on it's distance from us - - // The full rate distance is the distance at which EVERY update will be sent for this avatar - // at twice the full rate distance, there will be a 50% chance of sending this avatar's update - glm::vec3 otherPosition = otherAvatar.getClientGlobalPosition(); - float distanceToAvatar = glm::length(myPosition - otherPosition); - - // potentially update the max full rate distance for this frame - maxAvatarDistanceThisFrame = std::max(maxAvatarDistanceThisFrame, distanceToAvatar); - - if (distanceToAvatar != 0.0f - && !getsOutOfView - && distribution(generator) > (nodeData->getFullRateDistance() / distanceToAvatar)) { - - quint64 endAvatarDataPacking = usecTimestampNow(); - // FIXME - mixer data - //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - return; - } - - AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(otherNode->getUUID()); - AvatarDataSequenceNumber lastSeqFromSender = otherNodeData->getLastReceivedSequenceNumber(); - - if (lastSeqToReceiver > lastSeqFromSender && lastSeqToReceiver != UINT16_MAX) { - // we got out out of order packets from the sender, track it - otherNodeData->incrementNumOutOfOrderSends(); - } - - // make sure we haven't already sent this data from this sender to this receiver - // or that somehow we haven't sent - if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { - ++numAvatarsHeldBack; - - quint64 endAvatarDataPacking = usecTimestampNow(); - // FIXME - mixer data - //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - return; - } else if (lastSeqFromSender - lastSeqToReceiver > 1) { - // this is a skip - we still send the packet but capture the presence of the skip so we see it happening - ++numAvatarsWithSkippedFrames; - } - - // we're going to send this avatar - - // increment the number of avatars sent to this reciever - nodeData->incrementNumAvatarsSentLastFrame(); - - // set the last sent sequence number for this sender on the receiver - nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), - otherNodeData->getLastReceivedSequenceNumber()); - - // determine if avatar is in view, to determine how much data to include... - glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; - AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - bool isInView = nodeData->otherAvatarInView(otherNodeBox); - - // this throttles the extra data to only be sent every Nth message - if (!isInView && getsOutOfView && (lastSeqToReceiver % EXTRA_AVATAR_DATA_FRAME_RATIO > 0)) { - quint64 endAvatarDataPacking = usecTimestampNow(); - // FIXME - mixer data - //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - return; - } - - // start a new segment in the PacketList for this avatar - avatarPacketList->startSegment(); - - AvatarData::AvatarDataDetail detail; - if (!isInView && !getsOutOfView) { - detail = AvatarData::MinimumData; - nodeData->incrementAvatarOutOfView(); - } else { - detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO - ? AvatarData::SendAllData : AvatarData::CullSmallData; - nodeData->incrementAvatarInView(); - } - - numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); - auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); - QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); - bool distanceAdjust = true; - glm::vec3 viewerPosition = myPosition; - QByteArray bytes = otherAvatar.toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, distanceAdjust, viewerPosition, &lastSentJointsForOther); - numAvatarDataBytes += avatarPacketList->write(bytes); - - avatarPacketList->endSegment(); - - quint64 endAvatarDataPacking = usecTimestampNow(); - // FIXME - mixer data - //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - } - }); - - quint64 startPacketSending = usecTimestampNow(); - - // close the current packet so that we're always sending something - avatarPacketList->closeCurrentPacket(true); - - // send the avatar data PacketList - nodeList->sendPacketList(std::move(avatarPacketList), *node); - - // record the bytes sent for other avatar data in the AvatarMixerClientData - nodeData->recordSentAvatarData(numAvatarDataBytes); - - // record the number of avatars held back this frame - nodeData->recordNumOtherAvatarStarves(numAvatarsHeldBack); - nodeData->recordNumOtherAvatarSkips(numAvatarsWithSkippedFrames); - - if (numOtherAvatars == 0) { - // update the full rate distance to FLOAT_MAX since we didn't have any other avatars to send - nodeData->setMaxAvatarDistance(FLT_MAX); - } else { - nodeData->setMaxAvatarDistance(maxAvatarDistanceThisFrame); - } - - quint64 endPacketSending = usecTimestampNow(); - // FIXME - mixer data - //_packetSendingElapsedTime += (endPacketSending - startPacketSending); - - } - }); - - // We're done encoding this version of the otherAvatars. Update their "lastSent" joint-states so - // that we can notice differences, next time around. - // - // FIXME - this seems suspicious, the code seems to consider all avatars, but not all avatars will - // have had their joints sent, so actually we should consider the time since they actually were sent???? - std::for_each(cbegin, cend, [&](const SharedNodePointer& otherNode) { - if (otherNode->getLinkedData() && (otherNode->getType() == NodeType::Agent) && otherNode->getActiveSocket()) { - AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); - MutexTryLocker lock(otherNodeData->getMutex()); - if (!lock.isLocked()) { - return; - } - AvatarData& otherAvatar = otherNodeData->getAvatar(); - otherAvatar.doneEncoding(false); - } - }); -} - void AvatarMixer::nodeKilled(SharedNodePointer killedNode) { if (killedNode->getType() == NodeType::Agent && killedNode->getLinkedData()) { @@ -774,18 +459,11 @@ void AvatarMixer::sendStatsPacket() { statsObject["threads"] = _slavePool.numThreads(); statsObject["average_listeners_last_second"] = (float) _sumListeners / (float) _numStatFrames; - statsObject["average_identity_packets_per_frame"] = (float) _sumIdentityPackets / (float) _numStatFrames; statsObject["trailing_sleep_percentage"] = _trailingSleepRatio * 100; statsObject["performance_throttling_ratio"] = _performanceThrottlingRatio; - statsObject["broadcast_loop_rate"] = _broadcastRate.rate(); - statsObject["tight_loop_rate"] = _loopRate.rate(); - - // broadcastAvatarDataElapsed timing details... - statsObject["timing_average_b_ignoreCalculation"] = (float)_ignoreCalculationElapsedTime / (float)_numStatFrames; - statsObject["timing_average_c_avatarDataPacking"] = (float)_avatarDataPackingElapsedTime / (float)_numStatFrames; - statsObject["timing_average_d_packetSending"] = (float)_packetSendingElapsedTime / (float)_numStatFrames; + statsObject["broadcast_loop_rate"] = _loopRate.rate(); // this things all occur on the frequency of the tight loop int tightLoopFrames = _numTightLoopFrames; @@ -810,23 +488,37 @@ void AvatarMixer::sendStatsPacket() { statsObject["timing_sendStats"] = (float)_sendStatsElapsedTime; + AvatarMixerSlaveStats aggregateStats; + QJsonObject slavesObject; // gather stats int slaveNumber = 1; _slavePool.each([&](AvatarMixerSlave& slave) { QJsonObject slaveObject; - int nodesProcessed, packetsProcessed; - quint64 processIncomingPacketsElapsedTime; - slave.harvestStats(nodesProcessed, packetsProcessed, processIncomingPacketsElapsedTime); - slaveObject["nodesProcessed"] = TIGHT_LOOP_STAT(nodesProcessed); - slaveObject["packetsProcessed"] = TIGHT_LOOP_STAT(packetsProcessed); - slaveObject["timing_average_processIncomingPackets"] = TIGHT_LOOP_STAT(processIncomingPacketsElapsedTime); + AvatarMixerSlaveStats stats; + slave.harvestStats(stats); + slaveObject["nodesProcessed"] = TIGHT_LOOP_STAT(stats.nodesProcessed); + slaveObject["packetsProcessed"] = TIGHT_LOOP_STAT(stats.packetsProcessed); + slaveObject["processIncomingPackets"] = TIGHT_LOOP_STAT(stats.processIncomingPacketsElapsedTime); + slaveObject["ignoreCalculation"] = TIGHT_LOOP_STAT(stats.ignoreCalculationElapsedTime); + slaveObject["avatarDataPacking"] = TIGHT_LOOP_STAT(stats.avatarDataPackingElapsedTime); + slaveObject["packetSending"] = TIGHT_LOOP_STAT(stats.packetSendingElapsedTime); slavesObject[QString::number(slaveNumber)] = slaveObject; slaveNumber++; + + aggregateStats += stats; }); statsObject["timing_slaves"] = slavesObject; + // broadcastAvatarDataElapsed timing details... + statsObject["timing_aggregate_nodesProcessed"] = TIGHT_LOOP_STAT(aggregateStats.nodesProcessed); + statsObject["timing_aggregate_packetsProcessed"] = TIGHT_LOOP_STAT(aggregateStats.packetsProcessed); + statsObject["timing_aggregate_processIncomingPackets"] = TIGHT_LOOP_STAT(aggregateStats.processIncomingPacketsElapsedTime); + statsObject["timing_aggregate_ignoreCalculation"] = TIGHT_LOOP_STAT(aggregateStats.ignoreCalculationElapsedTime); + statsObject["timing_aggregate_avatarDataPacking"] = TIGHT_LOOP_STAT(aggregateStats.avatarDataPackingElapsedTime); + statsObject["timing_aggregate_packetSending"] = TIGHT_LOOP_STAT(aggregateStats.packetSendingElapsedTime); + _handleViewFrustumPacketElapsedTime = 0; _handleAvatarDataPacketElapsedTime = 0; _handleAvatarIdentityPacketElapsedTime = 0; diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index 628c44abaf..f0a52c4157 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -55,6 +55,8 @@ private slots: private: AvatarMixerClientData* getOrCreateClientData(SharedNodePointer node); + std::chrono::microseconds AvatarMixer::timeFrame(p_high_resolution_clock::time_point& timestamp); + void broadcastAvatarData(); void parseDomainServerSettings(const QJsonObject& domainSettings); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 37fd258bfa..676dd55858 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -35,12 +35,9 @@ void AvatarMixerSlave::configure(ConstIter begin, ConstIter end) { _end = end; } -void AvatarMixerSlave::harvestStats(int& nodesProcessed, int& packetsProcessed, quint64& processIncomingPacketsElapsedTime) { - nodesProcessed = _nodesProcessed; - packetsProcessed = _packetsProcessed; - processIncomingPacketsElapsedTime = _processIncomingPacketsElapsedTime; - _packetsProcessed = _nodesProcessed = 0; - _processIncomingPacketsElapsedTime = 0; +void AvatarMixerSlave::harvestStats(AvatarMixerSlaveStats& stats) { + stats = _stats; + _stats.reset(); } @@ -48,11 +45,11 @@ void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { auto start = usecTimestampNow(); auto nodeData = dynamic_cast(node->getLinkedData()); if (nodeData) { - _nodesProcessed++; - _packetsProcessed += nodeData->processPackets(); + _stats.nodesProcessed++; + _stats.packetsProcessed += nodeData->processPackets(); } auto end = usecTimestampNow(); - _processIncomingPacketsElapsedTime += (end - start); + _stats.processIncomingPacketsElapsedTime += (end - start); } #include @@ -235,8 +232,7 @@ void AvatarMixerSlave::anotherJob(const SharedNodePointer& node) { shouldConsider = true; quint64 endIgnoreCalculation = usecTimestampNow(); - // FIXME -- mixer data - //_ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); + _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); } //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << "shouldConsider:" << shouldConsider; @@ -252,11 +248,6 @@ void AvatarMixerSlave::anotherJob(const SharedNodePointer& node) { // FIXME -- might want to track this lock failed... if (!lock.isLocked()) { - - quint64 endAvatarDataPacking = usecTimestampNow(); - - // FIXME - mixer data - //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << " failed to lock... would BAIL??... line:" << __LINE__; //return; } @@ -291,89 +282,92 @@ void AvatarMixerSlave::anotherJob(const SharedNodePointer& node) { // potentially update the max full rate distance for this frame maxAvatarDistanceThisFrame = std::max(maxAvatarDistanceThisFrame, distanceToAvatar); + // FIXME-- understand this code... WHAT IS IT DOING!!! if (distanceToAvatar != 0.0f && !getsOutOfView && distribution(generator) > (nodeData->getFullRateDistance() / distanceToAvatar)) { quint64 endAvatarDataPacking = usecTimestampNow(); - // FIXME - mixer data - //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << " distance/getsOutOfView... BAILING... line:" << __LINE__; - return; + _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + qDebug() << __FUNCTION__ << "inner loop, node:" << node->getUUID() << "otherNode:" << otherNode->getUUID() << " BAILING... line:" << __LINE__; + shouldConsider = false; } - AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(otherNode->getUUID()); - AvatarDataSequenceNumber lastSeqFromSender = otherNodeData->getLastReceivedSequenceNumber(); + if (shouldConsider) { + AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(otherNode->getUUID()); + AvatarDataSequenceNumber lastSeqFromSender = otherNodeData->getLastReceivedSequenceNumber(); - if (lastSeqToReceiver > lastSeqFromSender && lastSeqToReceiver != UINT16_MAX) { - // we got out out of order packets from the sender, track it - otherNodeData->incrementNumOutOfOrderSends(); + if (lastSeqToReceiver > lastSeqFromSender && lastSeqToReceiver != UINT16_MAX) { + // we got out out of order packets from the sender, track it + otherNodeData->incrementNumOutOfOrderSends(); + } + + // make sure we haven't already sent this data from this sender to this receiver + // or that somehow we haven't sent + if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { + ++numAvatarsHeldBack; + + quint64 endAvatarDataPacking = usecTimestampNow(); + _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + qDebug() << __FUNCTION__ << "inner loop, node:" << node->getUUID() << "otherNode:" << otherNode->getUUID() << " BAILING... line:" << __LINE__; + shouldConsider = false; + } else if (lastSeqFromSender - lastSeqToReceiver > 1) { + // this is a skip - we still send the packet but capture the presence of the skip so we see it happening + ++numAvatarsWithSkippedFrames; + } + + // we're going to send this avatar + if (shouldConsider) { + // increment the number of avatars sent to this reciever + nodeData->incrementNumAvatarsSentLastFrame(); // FIXME - this seems weird... + + // set the last sent sequence number for this sender on the receiver + nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), + otherNodeData->getLastReceivedSequenceNumber()); + + // determine if avatar is in view, to determine how much data to include... + glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; + AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + bool isInView = nodeData->otherAvatarInView(otherNodeBox); + + // this throttles the extra data to only be sent every Nth message + if (!isInView && getsOutOfView && (lastSeqToReceiver % EXTRA_AVATAR_DATA_FRAME_RATIO > 0)) { + quint64 endAvatarDataPacking = usecTimestampNow(); + + _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + qDebug() << __FUNCTION__ << "inner loop, node:" << node->getUUID() << "otherNode:" << otherNode->getUUID() << " BAILING... line:" << __LINE__; + shouldConsider = false; + } + + if (shouldConsider) { + // start a new segment in the PacketList for this avatar + avatarPacketList->startSegment(); + + AvatarData::AvatarDataDetail detail; + if (!isInView && !getsOutOfView) { + detail = AvatarData::MinimumData; + nodeData->incrementAvatarOutOfView(); + } else { + detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO + ? AvatarData::SendAllData : AvatarData::CullSmallData; + nodeData->incrementAvatarInView(); + } + + numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); + auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); + QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); + bool distanceAdjust = true; + glm::vec3 viewerPosition = myPosition; + QByteArray bytes = otherAvatar.toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, distanceAdjust, viewerPosition, &lastSentJointsForOther); + numAvatarDataBytes += avatarPacketList->write(bytes); + + avatarPacketList->endSegment(); + + quint64 endAvatarDataPacking = usecTimestampNow(); + _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + } + } } - - // make sure we haven't already sent this data from this sender to this receiver - // or that somehow we haven't sent - if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { - ++numAvatarsHeldBack; - - quint64 endAvatarDataPacking = usecTimestampNow(); - // FIXME - mixer data - //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << " lastSeqToReceiver... BAILING... line:" << __LINE__; - return; - } else if (lastSeqFromSender - lastSeqToReceiver > 1) { - // this is a skip - we still send the packet but capture the presence of the skip so we see it happening - ++numAvatarsWithSkippedFrames; - } - - // we're going to send this avatar - - // increment the number of avatars sent to this reciever - nodeData->incrementNumAvatarsSentLastFrame(); - - // set the last sent sequence number for this sender on the receiver - nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), - otherNodeData->getLastReceivedSequenceNumber()); - - // determine if avatar is in view, to determine how much data to include... - glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; - AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - bool isInView = nodeData->otherAvatarInView(otherNodeBox); - - // this throttles the extra data to only be sent every Nth message - if (!isInView && getsOutOfView && (lastSeqToReceiver % EXTRA_AVATAR_DATA_FRAME_RATIO > 0)) { - quint64 endAvatarDataPacking = usecTimestampNow(); - // FIXME - mixer data - //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << " isInView && getsOutOfView && (lastSeqToReceiver % EXTRA_AVATAR_DATA_FRAME_RATIO > 0)... BAILING... line:" << __LINE__; - return; - } - - // start a new segment in the PacketList for this avatar - avatarPacketList->startSegment(); - - AvatarData::AvatarDataDetail detail; - if (!isInView && !getsOutOfView) { - detail = AvatarData::MinimumData; - nodeData->incrementAvatarOutOfView(); - } else { - detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO - ? AvatarData::SendAllData : AvatarData::CullSmallData; - nodeData->incrementAvatarInView(); - } - - numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); - auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); - QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); - bool distanceAdjust = true; - glm::vec3 viewerPosition = myPosition; - QByteArray bytes = otherAvatar.toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, distanceAdjust, viewerPosition, &lastSentJointsForOther); - numAvatarDataBytes += avatarPacketList->write(bytes); - - avatarPacketList->endSegment(); - - quint64 endAvatarDataPacking = usecTimestampNow(); - // FIXME - mixer data - //_avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); } }); @@ -401,8 +395,7 @@ void AvatarMixerSlave::anotherJob(const SharedNodePointer& node) { } quint64 endPacketSending = usecTimestampNow(); - // FIXME - mixer data - //_packetSendingElapsedTime += (endPacketSending - startPacketSending); + _stats.packetSendingElapsedTime += (endPacketSending - startPacketSending); } } diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index d75c6ae396..2e5d3df513 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -21,6 +21,37 @@ #include */ +class AvatarMixerSlaveStats { +public: + int nodesProcessed { 0 }; + int packetsProcessed { 0 }; + quint64 processIncomingPacketsElapsedTime { 0 }; + + quint64 ignoreCalculationElapsedTime { 0 }; + quint64 avatarDataPackingElapsedTime { 0 }; + quint64 packetSendingElapsedTime { 0 }; + + void reset() { + nodesProcessed = 0; + packetsProcessed = 0; + processIncomingPacketsElapsedTime = 0; + ignoreCalculationElapsedTime = 0; + avatarDataPackingElapsedTime = 0; + packetSendingElapsedTime = 0; + } + + AvatarMixerSlaveStats& operator+=(const AvatarMixerSlaveStats& rhs) { + nodesProcessed += rhs.nodesProcessed; + packetsProcessed += rhs.packetsProcessed; + processIncomingPacketsElapsedTime += rhs.processIncomingPacketsElapsedTime; + ignoreCalculationElapsedTime += rhs.ignoreCalculationElapsedTime; + avatarDataPackingElapsedTime += rhs.avatarDataPackingElapsedTime; + packetSendingElapsedTime += rhs.packetSendingElapsedTime; + return *this; + } + +}; + class AvatarMixerSlave { public: using ConstIter = NodeList::const_iterator; @@ -30,16 +61,14 @@ public: void processIncomingPackets(const SharedNodePointer& node); void anotherJob(const SharedNodePointer& node); - void harvestStats(int& nodesProcessed, int& packetsProcessed, quint64& processIncomingPacketsElapsedTime); + void harvestStats(AvatarMixerSlaveStats& stats); private: // frame state ConstIter _begin; ConstIter _end; - int _nodesProcessed { 0 }; - int _packetsProcessed { 0 }; - quint64 _processIncomingPacketsElapsedTime { 0 }; + AvatarMixerSlaveStats _stats; }; #endif // hifi_AvatarMixerSlave_h From 0a48ea75a78d4bb6774ca7c24db9d00df9c8d8ca Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 16 Feb 2017 10:45:58 -0800 Subject: [PATCH 038/104] fix unix build error --- assignment-client/src/avatars/AvatarMixer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index f0a52c4157..e932601483 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -55,7 +55,7 @@ private slots: private: AvatarMixerClientData* getOrCreateClientData(SharedNodePointer node); - std::chrono::microseconds AvatarMixer::timeFrame(p_high_resolution_clock::time_point& timestamp); + std::chrono::microseconds timeFrame(p_high_resolution_clock::time_point& timestamp); void broadcastAvatarData(); From d532b3a4b8dcb46da7a114bf6d2f6911c5e6cf72 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 16 Feb 2017 13:25:51 -0800 Subject: [PATCH 039/104] more work on multi-core --- assignment-client/src/avatars/AvatarMixer.cpp | 62 +++++++++++++------ assignment-client/src/avatars/AvatarMixer.h | 8 ++- .../src/avatars/AvatarMixerSlave.cpp | 23 ++++--- .../src/avatars/AvatarMixerSlave.h | 3 + libraries/networking/src/LimitedNodeList.h | 32 +++++++--- 5 files changed, 91 insertions(+), 37 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 73b0514161..cc6100f282 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -134,17 +134,22 @@ void AvatarMixer::start() { // calculates last frame duration and sleeps for the remainder of the target amount auto frameDuration = timeFrame(frameTimestamp); + int lockWait, nodeTransform, functor; + // Allow nodes to process any pending/queued packets across our worker threads { auto start = usecTimestampNow(); + nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { auto end = usecTimestampNow(); _processQueuedAvatarDataPacketsLockWaitElapsedTime += (end - start); _slavePool.processIncomingPackets(cbegin, cend); - }); + }, &lockWait, &nodeTransform, &functor); auto end = usecTimestampNow(); _processQueuedAvatarDataPacketsElapsedTime += (end - start); + + //qDebug() << "PROCESS PACKETS... " << "lockWait:" << lockWait << "nodeTransform:" << nodeTransform << "functor:" << functor; } // process pending display names... this doesn't currently run on multiple threads, because it @@ -155,9 +160,11 @@ void AvatarMixer::start() { std::for_each(cbegin, cend, [&](const SharedNodePointer& node) { manageDisplayName(node); }); - }); + }, &lockWait, &nodeTransform, &functor); auto end = usecTimestampNow(); _displayNameManagementElapsedTime += (end - start); + + //qDebug() << "PROCESS PACKETS... " << "lockWait:" << lockWait << "nodeTransform:" << nodeTransform << "functor:" << functor; } // this is where we need to put the real work... @@ -167,10 +174,17 @@ void AvatarMixer::start() { auto start = usecTimestampNow(); nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { + auto start = usecTimestampNow(); _slavePool.anotherJob(cbegin, cend); - }); + auto end = usecTimestampNow(); + _broadcastAvatarDataInner += (end - start); + }, &lockWait, &nodeTransform, &functor); auto end = usecTimestampNow(); _broadcastAvatarDataElapsedTime += (end - start); + + _broadcastAvatarDataLockWait += lockWait; + _broadcastAvatarDataNodeTransform += nodeTransform; + _broadcastAvatarDataNodeFunctor += functor; } @@ -239,9 +253,6 @@ static const int EXTRA_AVATAR_DATA_FRAME_RATIO = 16; // this "throttle" logic is the old approach. need to consider some // reasonable throttle approach in new multi-core design void AvatarMixer::broadcastAvatarData() { - quint64 startBroadcastAvatarData = usecTimestampNow(); - _broadcastRate.increment(); - int idleTime = AVATAR_DATA_SEND_INTERVAL_MSECS; if (_lastFrameTimestamp.time_since_epoch().count() > 0) { @@ -317,9 +328,6 @@ void AvatarMixer::broadcastAvatarData() { _lastDebugMessage = p_high_resolution_clock::now(); } #endif - - quint64 endBroadcastAvatarData = usecTimestampNow(); - _broadcastAvatarDataElapsedTime += (endBroadcastAvatarData - startBroadcastAvatarData); } void AvatarMixer::nodeKilled(SharedNodePointer killedNode) { @@ -473,7 +481,12 @@ void AvatarMixer::sendStatsPacket() { statsObject["timing_average_y_processEvents"] = TIGHT_LOOP_STAT(_processEventsElapsedTime); statsObject["timing_average_y_queueIncomingPacket"] = TIGHT_LOOP_STAT(_queueIncomingPacketElapsedTime); - statsObject["timing_average_z_broadcastAvatarData"] = TIGHT_LOOP_STAT(_broadcastAvatarDataElapsedTime); + statsObject["timing_average_a1_broadcastAvatarData"] = TIGHT_LOOP_STAT(_broadcastAvatarDataElapsedTime); + statsObject["timing_average_a2_innnerBroadcastAvatarData"] = TIGHT_LOOP_STAT(_broadcastAvatarDataInner); + statsObject["timing_average_a3_broadcastAvatarDataLockWait"] = TIGHT_LOOP_STAT(_broadcastAvatarDataLockWait); + statsObject["timing_average_a4_broadcastAvatarDataNodeTransform"] = TIGHT_LOOP_STAT(_broadcastAvatarDataNodeTransform); + statsObject["timing_average_a5_broadcastAvatarDataNodeFunctor"] = TIGHT_LOOP_STAT(_broadcastAvatarDataNodeFunctor); + statsObject["timing_average_z_displayNameManagement"] = TIGHT_LOOP_STAT(_displayNameManagementElapsedTime); statsObject["timing_average_z_handleAvatarDataPacket"] = TIGHT_LOOP_STAT(_handleAvatarDataPacketElapsedTime); statsObject["timing_average_z_handleAvatarIdentityPacket"] = TIGHT_LOOP_STAT(_handleAvatarIdentityPacketElapsedTime); @@ -499,10 +512,13 @@ void AvatarMixer::sendStatsPacket() { slave.harvestStats(stats); slaveObject["nodesProcessed"] = TIGHT_LOOP_STAT(stats.nodesProcessed); slaveObject["packetsProcessed"] = TIGHT_LOOP_STAT(stats.packetsProcessed); - slaveObject["processIncomingPackets"] = TIGHT_LOOP_STAT(stats.processIncomingPacketsElapsedTime); - slaveObject["ignoreCalculation"] = TIGHT_LOOP_STAT(stats.ignoreCalculationElapsedTime); - slaveObject["avatarDataPacking"] = TIGHT_LOOP_STAT(stats.avatarDataPackingElapsedTime); - slaveObject["packetSending"] = TIGHT_LOOP_STAT(stats.packetSendingElapsedTime); + slaveObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT(stats.processIncomingPacketsElapsedTime); + slaveObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT(stats.ignoreCalculationElapsedTime); + slaveObject["timing_3_avatarDataPacking"] = TIGHT_LOOP_STAT(stats.avatarDataPackingElapsedTime); + slaveObject["timing_4_packetSending"] = TIGHT_LOOP_STAT(stats.packetSendingElapsedTime); + slaveObject["timing_5_jobElapsedTime"] = TIGHT_LOOP_STAT(stats.jobElapsedTime); + + slavesObject[QString::number(slaveNumber)] = slaveObject; slaveNumber++; @@ -512,12 +528,13 @@ void AvatarMixer::sendStatsPacket() { statsObject["timing_slaves"] = slavesObject; // broadcastAvatarDataElapsed timing details... - statsObject["timing_aggregate_nodesProcessed"] = TIGHT_LOOP_STAT(aggregateStats.nodesProcessed); - statsObject["timing_aggregate_packetsProcessed"] = TIGHT_LOOP_STAT(aggregateStats.packetsProcessed); - statsObject["timing_aggregate_processIncomingPackets"] = TIGHT_LOOP_STAT(aggregateStats.processIncomingPacketsElapsedTime); - statsObject["timing_aggregate_ignoreCalculation"] = TIGHT_LOOP_STAT(aggregateStats.ignoreCalculationElapsedTime); - statsObject["timing_aggregate_avatarDataPacking"] = TIGHT_LOOP_STAT(aggregateStats.avatarDataPackingElapsedTime); - statsObject["timing_aggregate_packetSending"] = TIGHT_LOOP_STAT(aggregateStats.packetSendingElapsedTime); + statsObject["aggregate_nodesProcessed"] = TIGHT_LOOP_STAT(aggregateStats.nodesProcessed); + statsObject["aggregate_packetsProcessed"] = TIGHT_LOOP_STAT(aggregateStats.packetsProcessed); + statsObject["timing_aggregate_1_processIncomingPackets"] = TIGHT_LOOP_STAT(aggregateStats.processIncomingPacketsElapsedTime); + statsObject["timing_aggregate_2_ignoreCalculation"] = TIGHT_LOOP_STAT(aggregateStats.ignoreCalculationElapsedTime); + statsObject["timing_aggregate_3_avatarDataPacking"] = TIGHT_LOOP_STAT(aggregateStats.avatarDataPackingElapsedTime); + statsObject["timing_aggregate_4_packetSending"] = TIGHT_LOOP_STAT(aggregateStats.packetSendingElapsedTime); + statsObject["timing_aggregate_4_jobElapsedTime"] = TIGHT_LOOP_STAT(aggregateStats.jobElapsedTime); _handleViewFrustumPacketElapsedTime = 0; _handleAvatarDataPacketElapsedTime = 0; @@ -571,6 +588,11 @@ void AvatarMixer::sendStatsPacket() { _numTightLoopFrames = 0; _broadcastAvatarDataElapsedTime = 0; + _broadcastAvatarDataInner = 0; + _broadcastAvatarDataLockWait = 0; + _broadcastAvatarDataNodeTransform = 0; + _broadcastAvatarDataNodeFunctor = 0; + _displayNameManagementElapsedTime = 0; _ignoreCalculationElapsedTime = 0; _avatarDataPackingElapsedTime = 0; diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index e932601483..1e9e3bfb95 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -83,12 +83,18 @@ private: p_high_resolution_clock::time_point _lastDebugMessage; QHash> _sessionDisplayNames; - quint64 _broadcastAvatarDataElapsedTime { 0 }; // total time spent in broadcastAvatarData since last stats window quint64 _displayNameManagementElapsedTime { 0 }; // total time spent in broadcastAvatarData/display name management... since last stats window quint64 _ignoreCalculationElapsedTime { 0 }; quint64 _avatarDataPackingElapsedTime { 0 }; quint64 _packetSendingElapsedTime { 0 }; + quint64 _broadcastAvatarDataElapsedTime { 0 }; // total time spent in broadcastAvatarData since last stats window + quint64 _broadcastAvatarDataInner { 0 }; + quint64 _broadcastAvatarDataLockWait { 0 }; + quint64 _broadcastAvatarDataNodeTransform { 0 }; + quint64 _broadcastAvatarDataNodeFunctor { 0 }; + + quint64 _handleViewFrustumPacketElapsedTime { 0 }; quint64 _handleAvatarDataPacketElapsedTime { 0 }; diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 676dd55858..036eba5b36 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -70,6 +70,8 @@ static const int EXTRA_AVATAR_DATA_FRAME_RATIO = 16; const float IDENTITY_SEND_PROBABILITY = 1.0f / 187.0f; // FIXME... this is wrong for 45hz void AvatarMixerSlave::anotherJob(const SharedNodePointer& node) { + quint64 start = usecTimestampNow(); + //qDebug() << __FUNCTION__ << "node:" << node; auto nodeList = DependencyManager::get(); @@ -81,13 +83,13 @@ void AvatarMixerSlave::anotherJob(const SharedNodePointer& node) { if (node->getLinkedData() && (node->getType() == NodeType::Agent) && node->getActiveSocket()) { AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); - MutexTryLocker lock(nodeData->getMutex()); + //MutexTryLocker lock(nodeData->getMutex()); // FIXME???? - if (!lock.isLocked()) { + //if (!lock.isLocked()) { //qDebug() << __FUNCTION__ << "unable to lock... node:" << node << " would BAIL???... line:" << __LINE__; //return; - } + //} // FIXME -- mixer data // ++_sumListeners; @@ -244,13 +246,13 @@ void AvatarMixerSlave::anotherJob(const SharedNodePointer& node) { ++numOtherAvatars; AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); - MutexTryLocker lock(otherNodeData->getMutex()); + //MutexTryLocker lock(otherNodeData->getMutex()); // FIXME -- might want to track this lock failed... - if (!lock.isLocked()) { + //if (!lock.isLocked()) { //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << " failed to lock... would BAIL??... line:" << __LINE__; //return; - } + //} // make sure we send out identity packets to and from new arrivals. bool forceSend = !otherNodeData->checkAndSetHasReceivedFirstPacketsFrom(node->getUUID()); @@ -289,7 +291,7 @@ void AvatarMixerSlave::anotherJob(const SharedNodePointer& node) { quint64 endAvatarDataPacking = usecTimestampNow(); _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - qDebug() << __FUNCTION__ << "inner loop, node:" << node->getUUID() << "otherNode:" << otherNode->getUUID() << " BAILING... line:" << __LINE__; + //qDebug() << __FUNCTION__ << "inner loop, node:" << node->getUUID() << "otherNode:" << otherNode->getUUID() << " BAILING... line:" << __LINE__; shouldConsider = false; } @@ -309,7 +311,7 @@ void AvatarMixerSlave::anotherJob(const SharedNodePointer& node) { quint64 endAvatarDataPacking = usecTimestampNow(); _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - qDebug() << __FUNCTION__ << "inner loop, node:" << node->getUUID() << "otherNode:" << otherNode->getUUID() << " BAILING... line:" << __LINE__; + //qDebug() << __FUNCTION__ << "inner loop, node:" << node->getUUID() << "otherNode:" << otherNode->getUUID() << " BAILING... line:" << __LINE__; shouldConsider = false; } else if (lastSeqFromSender - lastSeqToReceiver > 1) { // this is a skip - we still send the packet but capture the presence of the skip so we see it happening @@ -335,7 +337,7 @@ void AvatarMixerSlave::anotherJob(const SharedNodePointer& node) { quint64 endAvatarDataPacking = usecTimestampNow(); _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - qDebug() << __FUNCTION__ << "inner loop, node:" << node->getUUID() << "otherNode:" << otherNode->getUUID() << " BAILING... line:" << __LINE__; + //qDebug() << __FUNCTION__ << "inner loop, node:" << node->getUUID() << "otherNode:" << otherNode->getUUID() << " BAILING... line:" << __LINE__; shouldConsider = false; } @@ -397,5 +399,8 @@ void AvatarMixerSlave::anotherJob(const SharedNodePointer& node) { quint64 endPacketSending = usecTimestampNow(); _stats.packetSendingElapsedTime += (endPacketSending - startPacketSending); } + + quint64 end = usecTimestampNow(); + _stats.jobElapsedTime += (end - start); } diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index 2e5d3df513..ac204ec0f6 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -30,6 +30,7 @@ public: quint64 ignoreCalculationElapsedTime { 0 }; quint64 avatarDataPackingElapsedTime { 0 }; quint64 packetSendingElapsedTime { 0 }; + quint64 jobElapsedTime { 0 }; void reset() { nodesProcessed = 0; @@ -38,6 +39,7 @@ public: ignoreCalculationElapsedTime = 0; avatarDataPackingElapsedTime = 0; packetSendingElapsedTime = 0; + jobElapsedTime = 0; } AvatarMixerSlaveStats& operator+=(const AvatarMixerSlaveStats& rhs) { @@ -47,6 +49,7 @@ public: ignoreCalculationElapsedTime += rhs.ignoreCalculationElapsedTime; avatarDataPackingElapsedTime += rhs.avatarDataPackingElapsedTime; packetSendingElapsedTime += rhs.packetSendingElapsedTime; + jobElapsedTime += rhs.jobElapsedTime; return *this; } diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 5256e55397..cbca64bcd8 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -182,15 +182,33 @@ public: // This allows multiple threads (i.e. a thread pool) to share a lock // without deadlocking when a dying node attempts to acquire a write lock template - void nestedEach(NestedNodeLambda functor) { - QReadLocker readLock(&_nodeMutex); + void nestedEach(NestedNodeLambda functor, + int* lockWaitOut = nullptr, + int* nodeTransformOut = nullptr, + int* functorOut = nullptr) { + auto start = usecTimestampNow(); + { + QReadLocker readLock(&_nodeMutex); + auto endLock = usecTimestampNow(); + if (lockWaitOut) { + *lockWaitOut = (endLock - start); + } - std::vector nodes(_nodeHash.size()); - std::transform(_nodeHash.cbegin(), _nodeHash.cend(), nodes.begin(), [](const NodeHash::value_type& it) { - return it.second; - }); + std::vector nodes(_nodeHash.size()); + std::transform(_nodeHash.cbegin(), _nodeHash.cend(), nodes.begin(), [](const NodeHash::value_type& it) { + return it.second; + }); + auto endTransform = usecTimestampNow(); + if (nodeTransformOut) { + *nodeTransformOut = (endTransform - endLock); + } - functor(nodes.cbegin(), nodes.cend()); + functor(nodes.cbegin(), nodes.cend()); + auto endFunctor = usecTimestampNow(); + if (functorOut) { + *functorOut = (endFunctor - endTransform); + } + } } template From d49c83cac3485792a9a4f2f091a7edb725a337dd Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 16 Feb 2017 14:19:14 -0800 Subject: [PATCH 040/104] fix build buster, some tweaks --- assignment-client/src/avatars/AvatarMixer.cpp | 16 +++++++------- .../src/avatars/AvatarMixerClientData.cpp | 2 +- .../src/avatars/AvatarMixerSlave.cpp | 21 ++++++++++++------- .../src/avatars/AvatarMixerSlave.h | 3 +++ libraries/networking/src/LimitedNodeList.h | 1 + 5 files changed, 27 insertions(+), 16 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index cc6100f282..363fb41ee3 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -514,11 +514,10 @@ void AvatarMixer::sendStatsPacket() { slaveObject["packetsProcessed"] = TIGHT_LOOP_STAT(stats.packetsProcessed); slaveObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT(stats.processIncomingPacketsElapsedTime); slaveObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT(stats.ignoreCalculationElapsedTime); - slaveObject["timing_3_avatarDataPacking"] = TIGHT_LOOP_STAT(stats.avatarDataPackingElapsedTime); - slaveObject["timing_4_packetSending"] = TIGHT_LOOP_STAT(stats.packetSendingElapsedTime); - slaveObject["timing_5_jobElapsedTime"] = TIGHT_LOOP_STAT(stats.jobElapsedTime); - - + slaveObject["timing_3_toByteArray"] = TIGHT_LOOP_STAT(stats.toByteArrayElapsedTime); + slaveObject["timing_4_avatarDataPacking"] = TIGHT_LOOP_STAT(stats.avatarDataPackingElapsedTime); + slaveObject["timing_5_packetSending"] = TIGHT_LOOP_STAT(stats.packetSendingElapsedTime); + slaveObject["timing_6_jobElapsedTime"] = TIGHT_LOOP_STAT(stats.jobElapsedTime); slavesObject[QString::number(slaveNumber)] = slaveObject; slaveNumber++; @@ -532,9 +531,10 @@ void AvatarMixer::sendStatsPacket() { statsObject["aggregate_packetsProcessed"] = TIGHT_LOOP_STAT(aggregateStats.packetsProcessed); statsObject["timing_aggregate_1_processIncomingPackets"] = TIGHT_LOOP_STAT(aggregateStats.processIncomingPacketsElapsedTime); statsObject["timing_aggregate_2_ignoreCalculation"] = TIGHT_LOOP_STAT(aggregateStats.ignoreCalculationElapsedTime); - statsObject["timing_aggregate_3_avatarDataPacking"] = TIGHT_LOOP_STAT(aggregateStats.avatarDataPackingElapsedTime); - statsObject["timing_aggregate_4_packetSending"] = TIGHT_LOOP_STAT(aggregateStats.packetSendingElapsedTime); - statsObject["timing_aggregate_4_jobElapsedTime"] = TIGHT_LOOP_STAT(aggregateStats.jobElapsedTime); + statsObject["timing_aggregate_3_toByteArray"] = TIGHT_LOOP_STAT(aggregateStats.toByteArrayElapsedTime); + statsObject["timing_aggregate_4_avatarDataPacking"] = TIGHT_LOOP_STAT(aggregateStats.avatarDataPackingElapsedTime); + statsObject["timing_aggregate_5_packetSending"] = TIGHT_LOOP_STAT(aggregateStats.packetSendingElapsedTime); + statsObject["timing_aggregate_6_jobElapsedTime"] = TIGHT_LOOP_STAT(aggregateStats.jobElapsedTime); _handleViewFrustumPacketElapsedTime = 0; _handleAvatarDataPacketElapsedTime = 0; diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index 09e3a6a87a..a598495269 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -41,7 +41,7 @@ int AvatarMixerClientData::processPackets() { _packetQueue.node.clear(); while (!_packetQueue.empty()) { - auto& packet = _packetQueue.back(); + auto& packet = _packetQueue.front(); packetsProcessed++; diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 036eba5b36..89218c6ea6 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -355,13 +355,20 @@ void AvatarMixerSlave::anotherJob(const SharedNodePointer& node) { nodeData->incrementAvatarInView(); } - numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); - auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); - QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); - bool distanceAdjust = true; - glm::vec3 viewerPosition = myPosition; - QByteArray bytes = otherAvatar.toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, distanceAdjust, viewerPosition, &lastSentJointsForOther); - numAvatarDataBytes += avatarPacketList->write(bytes); + { + numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); + auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); + QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); + bool distanceAdjust = true; + glm::vec3 viewerPosition = myPosition; + + quint64 start = usecTimestampNow(); + QByteArray bytes = otherAvatar.toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, distanceAdjust, viewerPosition, &lastSentJointsForOther); + quint64 end = usecTimestampNow(); + _stats.toByteArrayElapsedTime += (end - start); + + numAvatarDataBytes += avatarPacketList->write(bytes); + } avatarPacketList->endSegment(); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index ac204ec0f6..cbdfc4a08c 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -30,6 +30,7 @@ public: quint64 ignoreCalculationElapsedTime { 0 }; quint64 avatarDataPackingElapsedTime { 0 }; quint64 packetSendingElapsedTime { 0 }; + quint64 toByteArrayElapsedTime { 0 }; quint64 jobElapsedTime { 0 }; void reset() { @@ -39,6 +40,7 @@ public: ignoreCalculationElapsedTime = 0; avatarDataPackingElapsedTime = 0; packetSendingElapsedTime = 0; + toByteArrayElapsedTime = 0; jobElapsedTime = 0; } @@ -49,6 +51,7 @@ public: ignoreCalculationElapsedTime += rhs.ignoreCalculationElapsedTime; avatarDataPackingElapsedTime += rhs.avatarDataPackingElapsedTime; packetSendingElapsedTime += rhs.packetSendingElapsedTime; + toByteArrayElapsedTime += rhs.toByteArrayElapsedTime; jobElapsedTime += rhs.jobElapsedTime; return *this; } diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index cbca64bcd8..3eb898463a 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -34,6 +34,7 @@ #include #include +#include #include "DomainHandler.h" #include "Node.h" From df54762da91c2526d9f83ff9cd9760dd38a60d2f Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 16 Feb 2017 15:56:02 -0800 Subject: [PATCH 041/104] add some more stats --- assignment-client/src/avatars/AvatarMixer.cpp | 9 +++++++-- assignment-client/src/avatars/AvatarMixerSlave.cpp | 2 ++ assignment-client/src/avatars/AvatarMixerSlave.h | 8 ++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 363fb41ee3..c60b2c8fbc 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -511,7 +511,9 @@ void AvatarMixer::sendStatsPacket() { AvatarMixerSlaveStats stats; slave.harvestStats(stats); slaveObject["nodesProcessed"] = TIGHT_LOOP_STAT(stats.nodesProcessed); - slaveObject["packetsProcessed"] = TIGHT_LOOP_STAT(stats.packetsProcessed); + slaveObject["numPacketsReceived"] = TIGHT_LOOP_STAT(stats.packetsProcessed); + statsObject["numPacketsSent"] = TIGHT_LOOP_STAT(stats.numPacketsSent); + slaveObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT(stats.processIncomingPacketsElapsedTime); slaveObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT(stats.ignoreCalculationElapsedTime); slaveObject["timing_3_toByteArray"] = TIGHT_LOOP_STAT(stats.toByteArrayElapsedTime); @@ -528,7 +530,10 @@ void AvatarMixer::sendStatsPacket() { // broadcastAvatarDataElapsed timing details... statsObject["aggregate_nodesProcessed"] = TIGHT_LOOP_STAT(aggregateStats.nodesProcessed); - statsObject["aggregate_packetsProcessed"] = TIGHT_LOOP_STAT(aggregateStats.packetsProcessed); + statsObject["aggregate_numPacketsReceived"] = TIGHT_LOOP_STAT(aggregateStats.packetsProcessed); + statsObject["aggregate_numPacketsSent"] = TIGHT_LOOP_STAT(aggregateStats.numPacketsSent); + + statsObject["timing_aggregate_1_processIncomingPackets"] = TIGHT_LOOP_STAT(aggregateStats.processIncomingPacketsElapsedTime); statsObject["timing_aggregate_2_ignoreCalculation"] = TIGHT_LOOP_STAT(aggregateStats.ignoreCalculationElapsedTime); statsObject["timing_aggregate_3_toByteArray"] = TIGHT_LOOP_STAT(aggregateStats.toByteArrayElapsedTime); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 89218c6ea6..ca1d84ecfb 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -385,6 +385,8 @@ void AvatarMixerSlave::anotherJob(const SharedNodePointer& node) { // close the current packet so that we're always sending something avatarPacketList->closeCurrentPacket(true); + _stats.numPacketsSent += (int)avatarPacketList->getNumPackets(); + // send the avatar data PacketList //qDebug() << "about to call nodeList->sendPacketList() for node:" << node; nodeList->sendPacketList(std::move(avatarPacketList), *node); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index cbdfc4a08c..939fa39116 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -27,6 +27,7 @@ public: int packetsProcessed { 0 }; quint64 processIncomingPacketsElapsedTime { 0 }; + int numPacketsSent { 0 }; quint64 ignoreCalculationElapsedTime { 0 }; quint64 avatarDataPackingElapsedTime { 0 }; quint64 packetSendingElapsedTime { 0 }; @@ -34,9 +35,14 @@ public: quint64 jobElapsedTime { 0 }; void reset() { + // receiving job stats nodesProcessed = 0; packetsProcessed = 0; + numPacketsSent = 0; processIncomingPacketsElapsedTime = 0; + + // sending job stats + numPacketsSent = 0; ignoreCalculationElapsedTime = 0; avatarDataPackingElapsedTime = 0; packetSendingElapsedTime = 0; @@ -48,6 +54,8 @@ public: nodesProcessed += rhs.nodesProcessed; packetsProcessed += rhs.packetsProcessed; processIncomingPacketsElapsedTime += rhs.processIncomingPacketsElapsedTime; + + numPacketsSent += rhs.numPacketsSent; ignoreCalculationElapsedTime += rhs.ignoreCalculationElapsedTime; avatarDataPackingElapsedTime += rhs.avatarDataPackingElapsedTime; packetSendingElapsedTime += rhs.packetSendingElapsedTime; From ece32a3c684e5d5f2178e69a1227046749ce88d3 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 16 Feb 2017 16:19:40 -0800 Subject: [PATCH 042/104] cleanup --- assignment-client/src/avatars/AvatarMixer.cpp | 15 ++++++--------- assignment-client/src/avatars/AvatarMixer.h | 8 +------- .../src/avatars/AvatarMixerClientData.cpp | 9 --------- .../src/avatars/AvatarMixerSlave.cpp | 2 +- assignment-client/src/avatars/AvatarMixerSlave.h | 11 +---------- .../src/avatars/AvatarMixerSlavePool.cpp | 4 ++-- .../src/avatars/AvatarMixerSlavePool.h | 2 +- 7 files changed, 12 insertions(+), 39 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index c60b2c8fbc..8249daed9d 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -124,10 +124,12 @@ void AvatarMixer::start() { // DO THIS FIRST!!!!!!!! // // DONE --- 1) only sleep for remainder - // 2) clean up stats, add slave stats + // DONE --- 2) clean up stats, add slave stats // 3) delete dead code from mixer (now that it's in slave) // 4) audit the locking and side-effects to node, otherNode, and nodeData // 5) throttling?? + // 6) Error in PacketList::writeData - attempted to write a segment to an unordered packet that is larger than the payload size. + // 7) fix two different versions of toByteArray() ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -175,7 +177,7 @@ void AvatarMixer::start() { auto start = usecTimestampNow(); nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { auto start = usecTimestampNow(); - _slavePool.anotherJob(cbegin, cend); + _slavePool.broadcastAvatarData(cbegin, cend); auto end = usecTimestampNow(); _broadcastAvatarDataInner += (end - start); }, &lockWait, &nodeTransform, &functor); @@ -260,8 +262,6 @@ void AvatarMixer::broadcastAvatarData() { idleTime = std::chrono::duration_cast(idleDuration).count(); } - ++_numStatFrames; - const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; @@ -466,8 +466,8 @@ void AvatarMixer::sendStatsPacket() { QJsonObject statsObject; statsObject["threads"] = _slavePool.numThreads(); - statsObject["average_listeners_last_second"] = (float) _sumListeners / (float) _numStatFrames; - statsObject["average_identity_packets_per_frame"] = (float) _sumIdentityPackets / (float) _numStatFrames; + //statsObject["average_listeners_last_second"] = (float) _sumListeners / (float) _numStatFrames; + //statsObject["average_identity_packets_per_frame"] = (float) _sumIdentityPackets / (float) _numStatFrames; statsObject["trailing_sleep_percentage"] = _trailingSleepRatio * 100; statsObject["performance_throttling_ratio"] = _performanceThrottlingRatio; @@ -488,7 +488,6 @@ void AvatarMixer::sendStatsPacket() { statsObject["timing_average_a5_broadcastAvatarDataNodeFunctor"] = TIGHT_LOOP_STAT(_broadcastAvatarDataNodeFunctor); statsObject["timing_average_z_displayNameManagement"] = TIGHT_LOOP_STAT(_displayNameManagementElapsedTime); - statsObject["timing_average_z_handleAvatarDataPacket"] = TIGHT_LOOP_STAT(_handleAvatarDataPacketElapsedTime); statsObject["timing_average_z_handleAvatarIdentityPacket"] = TIGHT_LOOP_STAT(_handleAvatarIdentityPacketElapsedTime); statsObject["timing_average_z_handleKillAvatarPacket"] = TIGHT_LOOP_STAT(_handleKillAvatarPacketElapsedTime); statsObject["timing_average_z_handleNodeIgnoreRequestPacket"] = TIGHT_LOOP_STAT(_handleNodeIgnoreRequestPacketElapsedTime); @@ -542,7 +541,6 @@ void AvatarMixer::sendStatsPacket() { statsObject["timing_aggregate_6_jobElapsedTime"] = TIGHT_LOOP_STAT(aggregateStats.jobElapsedTime); _handleViewFrustumPacketElapsedTime = 0; - _handleAvatarDataPacketElapsedTime = 0; _handleAvatarIdentityPacketElapsedTime = 0; _handleKillAvatarPacketElapsedTime = 0; _handleNodeIgnoreRequestPacketElapsedTime = 0; @@ -589,7 +587,6 @@ void AvatarMixer::sendStatsPacket() { _sumListeners = 0; _sumIdentityPackets = 0; - _numStatFrames = 0; _numTightLoopFrames = 0; _broadcastAvatarDataElapsedTime = 0; diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index 1e9e3bfb95..7a79ec1d57 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -15,8 +15,6 @@ #ifndef hifi_AvatarMixer_h #define hifi_AvatarMixer_h -#include - #include #include @@ -42,7 +40,6 @@ public slots: private slots: void queueIncomingPacket(QSharedPointer message, SharedNodePointer node); void handleViewFrustumPacket(QSharedPointer message, SharedNodePointer senderNode); - //void handleAvatarDataPacket(QSharedPointer message, SharedNodePointer senderNode); void handleAvatarIdentityPacket(QSharedPointer message, SharedNodePointer senderNode); void handleKillAvatarPacket(QSharedPointer message); void handleNodeIgnoreRequestPacket(QSharedPointer message, SharedNodePointer senderNode); @@ -71,7 +68,7 @@ private: int _sumListeners { 0 }; int _numStatFrames { 0 }; - std::atomic _numTightLoopFrames; + int _numTightLoopFrames { 0 }; int _sumIdentityPackets { 0 }; float _maxKbpsPerNode = 0.0f; @@ -94,10 +91,7 @@ private: quint64 _broadcastAvatarDataNodeTransform { 0 }; quint64 _broadcastAvatarDataNodeFunctor { 0 }; - - quint64 _handleViewFrustumPacketElapsedTime { 0 }; - quint64 _handleAvatarDataPacketElapsedTime { 0 }; quint64 _handleAvatarIdentityPacketElapsedTime { 0 }; quint64 _handleKillAvatarPacketElapsedTime { 0 }; quint64 _handleNodeIgnoreRequestPacketElapsedTime { 0 }; diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index a598495269..b5d4e390bb 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -25,15 +25,6 @@ void AvatarMixerClientData::queuePacket(QSharedPointer message, _packetQueue.push(message); } -// -// packetReceiver.registerListener(PacketType::ViewFrustum, this, "handleViewFrustumPacket"); -// packetReceiver.registerListener(PacketType::AvatarData, this, "handleAvatarDataPacket"); -// packetReceiver.registerListener(PacketType::AvatarIdentity, this, "handleAvatarIdentityPacket"); -// packetReceiver.registerListener(PacketType::KillAvatar, this, "handleKillAvatarPacket"); -// packetReceiver.registerListener(PacketType::NodeIgnoreRequest, this, "handleNodeIgnoreRequestPacket"); -// packetReceiver.registerListener(PacketType::RadiusIgnoreRequest, this, "handleRadiusIgnoreRequestPacket"); -// packetReceiver.registerListener(PacketType::RequestsDomainListData, this, "handleRequestsDomainListDataPacket"); - int AvatarMixerClientData::processPackets() { int packetsProcessed = 0; SharedNodePointer node = _packetQueue.node; diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index ca1d84ecfb..a7d0dc9c52 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -69,7 +69,7 @@ static const int EXTRA_AVATAR_DATA_FRAME_RATIO = 16; // assuming 60 htz update rate. const float IDENTITY_SEND_PROBABILITY = 1.0f / 187.0f; // FIXME... this is wrong for 45hz -void AvatarMixerSlave::anotherJob(const SharedNodePointer& node) { +void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { quint64 start = usecTimestampNow(); //qDebug() << __FUNCTION__ << "node:" << node; diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index 939fa39116..b693356349 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -12,15 +12,6 @@ #ifndef hifi_AvatarMixerSlave_h #define hifi_AvatarMixerSlave_h -/* -#include -#include -#include -#include -#include -#include -*/ - class AvatarMixerSlaveStats { public: int nodesProcessed { 0 }; @@ -73,7 +64,7 @@ public: void configure(ConstIter begin, ConstIter end); void processIncomingPackets(const SharedNodePointer& node); - void anotherJob(const SharedNodePointer& node); + void broadcastAvatarData(const SharedNodePointer& node); void harvestStats(AvatarMixerSlaveStats& stats); diff --git a/assignment-client/src/avatars/AvatarMixerSlavePool.cpp b/assignment-client/src/avatars/AvatarMixerSlavePool.cpp index 28a3dce0e3..f410bb566d 100644 --- a/assignment-client/src/avatars/AvatarMixerSlavePool.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlavePool.cpp @@ -73,8 +73,8 @@ void AvatarMixerSlavePool::processIncomingPackets(ConstIter begin, ConstIter end run(begin, end); } -void AvatarMixerSlavePool::anotherJob(ConstIter begin, ConstIter end) { - _function = &AvatarMixerSlave::anotherJob; +void AvatarMixerSlavePool::broadcastAvatarData(ConstIter begin, ConstIter end) { + _function = &AvatarMixerSlave::broadcastAvatarData; _configure = [&](AvatarMixerSlave& slave) { slave.configure(begin, end); }; run(begin, end); } diff --git a/assignment-client/src/avatars/AvatarMixerSlavePool.h b/assignment-client/src/avatars/AvatarMixerSlavePool.h index 8f13477a93..0a689b35ed 100644 --- a/assignment-client/src/avatars/AvatarMixerSlavePool.h +++ b/assignment-client/src/avatars/AvatarMixerSlavePool.h @@ -65,7 +65,7 @@ public: // Jobs the slave pool can do... void processIncomingPackets(ConstIter begin, ConstIter end); - void anotherJob(ConstIter begin, ConstIter end); + void broadcastAvatarData(ConstIter begin, ConstIter end); // iterate over all slaves void each(std::function functor); From 0da81efd93eaed495464ca08112922c103dd6ea3 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 15 Feb 2017 13:59:42 -0800 Subject: [PATCH 043/104] Add includeDescendants and includeParents to ESS entity query --- assignment-client/src/scripts/EntityScriptServer.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/assignment-client/src/scripts/EntityScriptServer.cpp b/assignment-client/src/scripts/EntityScriptServer.cpp index 930b3946c6..3a03cde5d2 100644 --- a/assignment-client/src/scripts/EntityScriptServer.cpp +++ b/assignment-client/src/scripts/EntityScriptServer.cpp @@ -266,6 +266,14 @@ void EntityScriptServer::run() { QJsonObject queryJSONParameters; static const QString SERVER_SCRIPTS_PROPERTY = "serverScripts"; queryJSONParameters[SERVER_SCRIPTS_PROPERTY] = EntityQueryFilterSymbol::NonDefault; + + QJsonObject queryFlags; + static const QString INCLUDE_DESCENDANTS_PROPERTY = "includeDescendants"; + static const QString INCLUDE_PARENTS_PROPERTY = "includeParents"; + queryFlags[INCLUDE_DESCENDANTS_PROPERTY] = true; + queryFlags[INCLUDE_PARENTS_PROPERTY] = true; + + queryJSONParameters["flags"] = queryFlags; // setup the JSON parameters so that OctreeQuery does not use a frustum and uses our JSON filter _entityViewer.getOctreeQuery().setUsesFrustum(false); From a3883a746cfe788b07df188fcda0a2b80b43f9dd Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 15 Feb 2017 18:55:24 -0800 Subject: [PATCH 044/104] add the basic structure for inclusion of ancestors/descendants in ESS queries --- .../src/entities/EntityServer.cpp | 9 +- assignment-client/src/entities/EntityServer.h | 1 + .../src/entities/EntityTreeSendThread.cpp | 107 ++++++++++++++++++ .../src/entities/EntityTreeSendThread.h | 34 ++++++ .../src/octree/OctreeSendThread.cpp | 3 + .../src/octree/OctreeSendThread.h | 12 +- assignment-client/src/octree/OctreeServer.cpp | 6 +- assignment-client/src/octree/OctreeServer.h | 1 + .../src/scripts/EntityScriptServer.cpp | 13 +-- libraries/entities/src/EntityNodeData.cpp | 25 ++++ libraries/entities/src/EntityNodeData.h | 24 +++- libraries/entities/src/EntityTreeElement.cpp | 21 ++-- 12 files changed, 228 insertions(+), 28 deletions(-) create mode 100644 assignment-client/src/entities/EntityTreeSendThread.cpp create mode 100644 assignment-client/src/entities/EntityTreeSendThread.h create mode 100644 libraries/entities/src/EntityNodeData.cpp diff --git a/assignment-client/src/entities/EntityServer.cpp b/assignment-client/src/entities/EntityServer.cpp index 425bea2c38..dc0a2add3a 100644 --- a/assignment-client/src/entities/EntityServer.cpp +++ b/assignment-client/src/entities/EntityServer.cpp @@ -17,10 +17,11 @@ #include #include +#include "AssignmentParentFinder.h" +#include "EntityNodeData.h" #include "EntityServer.h" #include "EntityServerConsts.h" -#include "EntityNodeData.h" -#include "AssignmentParentFinder.h" +#include "EntityTreeSendThread.h" const char* MODEL_SERVER_NAME = "Entity"; const char* MODEL_SERVER_LOGGING_TARGET_NAME = "entity-server"; @@ -77,6 +78,10 @@ OctreePointer EntityServer::createTree() { return tree; } +OctreeServer::UniqueSendThread EntityServer::newSendThread(const SharedNodePointer& node) { + return std::unique_ptr(new EntityTreeSendThread(this, node)); +} + void EntityServer::beforeRun() { _pruneDeletedEntitiesTimer = new QTimer(); connect(_pruneDeletedEntitiesTimer, SIGNAL(timeout()), this, SLOT(pruneDeletedEntities())); diff --git a/assignment-client/src/entities/EntityServer.h b/assignment-client/src/entities/EntityServer.h index 325435fe7e..40676e79bd 100644 --- a/assignment-client/src/entities/EntityServer.h +++ b/assignment-client/src/entities/EntityServer.h @@ -67,6 +67,7 @@ public slots: protected: virtual OctreePointer createTree() override; + virtual UniqueSendThread newSendThread(const SharedNodePointer& node) override; private slots: void handleEntityPacket(QSharedPointer message, SharedNodePointer senderNode); diff --git a/assignment-client/src/entities/EntityTreeSendThread.cpp b/assignment-client/src/entities/EntityTreeSendThread.cpp new file mode 100644 index 0000000000..11a7ddf038 --- /dev/null +++ b/assignment-client/src/entities/EntityTreeSendThread.cpp @@ -0,0 +1,107 @@ +// +// EntityTreeSendThread.cpp +// assignment-client/src/entities +// +// Created by Stephen Birarda on 2/15/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "EntityTreeSendThread.h" + +#include +#include + +#include "EntityServer.h" + +void EntityTreeSendThread::preDistributionProcessing() { + auto node = _node.toStrongRef(); + auto nodeData = static_cast(node->getLinkedData()); + + if (nodeData) { + auto jsonQuery = nodeData->getJSONParameters(); + + // check if we have a JSON query with flags + auto flags = jsonQuery[EntityJSONQueryProperties::FLAGS_PROPERTY].toObject(); + if (!flags.isEmpty()) { + // check the flags object for specific flags that require special pre-processing + + bool includeAncestors = flags[EntityJSONQueryProperties::INCLUDE_ANCESTORS_PROPERTY].toBool(); + bool includeDescendants = flags[EntityJSONQueryProperties::INCLUDE_DESCENDANTS_PROPERTY].toBool(); + + if (includeAncestors || includeDescendants) { + // we need to either include the ancestors, descendants, or both for entities matching the filter + // included in the JSON query + + auto entityTree = std::static_pointer_cast(_myServer->getOctree()); + + // enumerate the set of entity IDs we know currently match the filter + foreach(const QUuid& entityID, nodeData->getSentFilteredEntities()) { + if (includeAncestors) { + // we need to include ancestors - recurse up to reach them all and add their IDs + // to the set of extra entities to include for this node + entityTree->withReadLock([&]{ + auto filteredEntity = entityTree->findEntityByID(entityID); + if (filteredEntity) { + addAncestorsToExtraFlaggedEntities(entityID, *filteredEntity, *nodeData); + } + }); + } + + if (includeDescendants) { + // we need to include descendants - recurse down to reach them all and add their IDs + // to the set of extra entities to include for this node + entityTree->withReadLock([&]{ + auto filteredEntity = entityTree->findEntityByID(entityID); + if (filteredEntity) { + addDescendantsToExtraFlaggedEntities(entityID, *filteredEntity, *nodeData); + } + }); + } + } + } + + } + } +} + +void EntityTreeSendThread::addAncestorsToExtraFlaggedEntities(const QUuid& filteredEntityID, + EntityItem& entityItem, EntityNodeData& nodeData) { + // check if this entity has a parent that is also an entity + bool success = false; + auto entityParent = entityItem.getParentPointer(success); + + if (success && entityParent && entityParent->getNestableType() == NestableType::Entity) { + // we found a parent that is an entity item + + // first add it to the extra list of things we need to send + nodeData.insertFlaggedExtraEntity(filteredEntityID, entityParent->getID()); + +// qDebug() << "Adding" << entityParent->getID() << "which is an ancestor of" << filteredEntityID; + + // now recursively call ourselves to get its ancestors added too + auto parentEntityItem = std::static_pointer_cast(entityParent); + addAncestorsToExtraFlaggedEntities(filteredEntityID, *parentEntityItem, nodeData); + } +} + +void EntityTreeSendThread::addDescendantsToExtraFlaggedEntities(const QUuid& filteredEntityID, + EntityItem& entityItem, EntityNodeData& nodeData) { + // enumerate the immediate children of this entity + foreach (SpatiallyNestablePointer child, entityItem.getChildren()) { + if (child && child->getNestableType() == NestableType::Entity) { + // this is a child that is an entity + + // first add it to the extra list of things we need to send + nodeData.insertFlaggedExtraEntity(filteredEntityID, child->getID()); + +// qDebug() << "Adding" << child->getID() << "which is a descendant of" << filteredEntityID; + + // now recursively call ourselves to get its descendants added too + auto childEntityItem = std::static_pointer_cast(child); + addDescendantsToExtraFlaggedEntities(filteredEntityID, *childEntityItem, nodeData); + } + } +} diff --git a/assignment-client/src/entities/EntityTreeSendThread.h b/assignment-client/src/entities/EntityTreeSendThread.h new file mode 100644 index 0000000000..8273b0379f --- /dev/null +++ b/assignment-client/src/entities/EntityTreeSendThread.h @@ -0,0 +1,34 @@ +// +// EntityTreeSendThread.h +// assignment-client/src/entities +// +// Created by Stephen Birarda on 2/15/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_EntityTreeSendThread_h +#define hifi_EntityTreeSendThread_h + +#include "../octree/OctreeSendThread.h" + +class EntityNodeData; +class EntityItem; + +class EntityTreeSendThread : public OctreeSendThread { + +public: + EntityTreeSendThread(OctreeServer* myServer, const SharedNodePointer& node) : OctreeSendThread(myServer, node) {}; + +protected: + virtual void preDistributionProcessing() override; + +private: + void addAncestorsToExtraFlaggedEntities(const QUuid& filteredEntityID, EntityItem& entityItem, EntityNodeData& nodeData); + void addDescendantsToExtraFlaggedEntities(const QUuid& filteredEntityID, EntityItem& entityItem, EntityNodeData& nodeData); + +}; + +#endif // hifi_EntityTreeSendThread_h diff --git a/assignment-client/src/octree/OctreeSendThread.cpp b/assignment-client/src/octree/OctreeSendThread.cpp index afc17d71aa..c3bdb3752d 100644 --- a/assignment-client/src/octree/OctreeSendThread.cpp +++ b/assignment-client/src/octree/OctreeSendThread.cpp @@ -309,6 +309,9 @@ int OctreeSendThread::packetDistributor(SharedNodePointer node, OctreeQueryNode* return 0; } + // give our pre-distribution processing a chance to do what it needs + preDistributionProcessing(); + // calculate max number of packets that can be sent during this interval int clientMaxPacketsPerInterval = std::max(1, (nodeData->getMaxQueryPacketsPerSecond() / INTERVALS_PER_SECOND)); int maxPacketsPerInterval = std::min(clientMaxPacketsPerInterval, _myServer->getPacketsPerClientPerInterval()); diff --git a/assignment-client/src/octree/OctreeSendThread.h b/assignment-client/src/octree/OctreeSendThread.h index 7efe5b3a86..06c9b5f1d6 100644 --- a/assignment-client/src/octree/OctreeSendThread.h +++ b/assignment-client/src/octree/OctreeSendThread.h @@ -17,6 +17,8 @@ #include #include +#include +#include class OctreeQueryNode; class OctreeServer; @@ -49,13 +51,17 @@ protected: /// Implements generic processing behavior for this thread. virtual bool process() override; + /// Called before a packetDistributor pass to allow for pre-distribution processing + virtual void preDistributionProcessing() {}; + + OctreeServer* _myServer { nullptr }; + QWeakPointer _node; + private: int handlePacketSend(SharedNodePointer node, OctreeQueryNode* nodeData, int& trueBytesSent, int& truePacketsSent, bool dontSuppressDuplicate = false); int packetDistributor(SharedNodePointer node, OctreeQueryNode* nodeData, bool viewFrustumChanged); - - OctreeServer* _myServer { nullptr }; - QWeakPointer _node; + QUuid _nodeUuid; OctreePacketData _packetData; diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp index c36a9be050..2eee2ee229 100644 --- a/assignment-client/src/octree/OctreeServer.cpp +++ b/assignment-client/src/octree/OctreeServer.cpp @@ -872,8 +872,12 @@ void OctreeServer::parsePayload() { } } +OctreeServer::UniqueSendThread OctreeServer::newSendThread(const SharedNodePointer& node) { + return std::unique_ptr(new OctreeSendThread(this, node)); +} + OctreeServer::UniqueSendThread OctreeServer::createSendThread(const SharedNodePointer& node) { - auto sendThread = std::unique_ptr(new OctreeSendThread(this, node)); + auto sendThread = newSendThread(node); // we want to be notified when the thread finishes connect(sendThread.get(), &GenericThread::finished, this, &OctreeServer::removeSendThread); diff --git a/assignment-client/src/octree/OctreeServer.h b/assignment-client/src/octree/OctreeServer.h index 2bcf36628d..3bb327eb06 100644 --- a/assignment-client/src/octree/OctreeServer.h +++ b/assignment-client/src/octree/OctreeServer.h @@ -158,6 +158,7 @@ protected: QString getStatusLink(); UniqueSendThread createSendThread(const SharedNodePointer& node); + virtual UniqueSendThread newSendThread(const SharedNodePointer& node); int _argc; const char** _argv; diff --git a/assignment-client/src/scripts/EntityScriptServer.cpp b/assignment-client/src/scripts/EntityScriptServer.cpp index 3a03cde5d2..06df255968 100644 --- a/assignment-client/src/scripts/EntityScriptServer.cpp +++ b/assignment-client/src/scripts/EntityScriptServer.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -264,16 +265,14 @@ void EntityScriptServer::run() { // setup the JSON filter that asks for entities with a non-default serverScripts property QJsonObject queryJSONParameters; - static const QString SERVER_SCRIPTS_PROPERTY = "serverScripts"; - queryJSONParameters[SERVER_SCRIPTS_PROPERTY] = EntityQueryFilterSymbol::NonDefault; + queryJSONParameters[EntityJSONQueryProperties::SERVER_SCRIPTS_PROPERTY] = EntityQueryFilterSymbol::NonDefault; QJsonObject queryFlags; - static const QString INCLUDE_DESCENDANTS_PROPERTY = "includeDescendants"; - static const QString INCLUDE_PARENTS_PROPERTY = "includeParents"; - queryFlags[INCLUDE_DESCENDANTS_PROPERTY] = true; - queryFlags[INCLUDE_PARENTS_PROPERTY] = true; - queryJSONParameters["flags"] = queryFlags; + queryFlags[EntityJSONQueryProperties::INCLUDE_ANCESTORS_PROPERTY] = true; + queryFlags[EntityJSONQueryProperties::INCLUDE_DESCENDANTS_PROPERTY] = true; + + queryJSONParameters[EntityJSONQueryProperties::FLAGS_PROPERTY] = queryFlags; // setup the JSON parameters so that OctreeQuery does not use a frustum and uses our JSON filter _entityViewer.getOctreeQuery().setUsesFrustum(false); diff --git a/libraries/entities/src/EntityNodeData.cpp b/libraries/entities/src/EntityNodeData.cpp new file mode 100644 index 0000000000..3e28bbcce4 --- /dev/null +++ b/libraries/entities/src/EntityNodeData.cpp @@ -0,0 +1,25 @@ +// +// EntityNodeData.cpp +// libraries/entities/src +// +// Created by Stephen Birarda on 2/15/17 +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "EntityNodeData.h" + +bool EntityNodeData::isEntityFlaggedAsExtra(const QUuid& entityID) const { + + // enumerate each of the sets for the entities that matched our filter + // and immediately return true if any of them contain this entity ID + foreach(QSet entitySet, _flaggedExtraEntities) { + if (entitySet.contains(entityID)) { + return true; + } + } + + return false; +} diff --git a/libraries/entities/src/EntityNodeData.h b/libraries/entities/src/EntityNodeData.h index b3a576b1ad..681d72f2d4 100644 --- a/libraries/entities/src/EntityNodeData.h +++ b/libraries/entities/src/EntityNodeData.h @@ -1,6 +1,6 @@ // // EntityNodeData.h -// assignment-client/src/entities +// libraries/entities/src // // Created by Brad Hefta-Gaub on 4/29/14 // Copyright 2014 High Fidelity, Inc. @@ -16,6 +16,13 @@ #include +namespace EntityJSONQueryProperties { + static const QString SERVER_SCRIPTS_PROPERTY = "serverScripts"; + static const QString FLAGS_PROPERTY = "flags"; + static const QString INCLUDE_ANCESTORS_PROPERTY = "includeAncestors"; + static const QString INCLUDE_DESCENDANTS_PROPERTY = "includeDescendants"; +} + class EntityNodeData : public OctreeQueryNode { public: virtual PacketType getMyPacketType() const override { return PacketType::EntityData; } @@ -24,13 +31,20 @@ public: void setLastDeletedEntitiesSentAt(quint64 sentAt) { _lastDeletedEntitiesSentAt = sentAt; } // these can only be called from the OctreeSendThread for the given Node - void insertEntitySentLastFrame(const QUuid& entityID) { _entitiesSentLastFrame.insert(entityID); } - void removeEntitySentLastFrame(const QUuid& entityID) { _entitiesSentLastFrame.remove(entityID); } - bool sentEntityLastFrame(const QUuid& entityID) { return _entitiesSentLastFrame.contains(entityID); } + void insertSentFilteredEntity(const QUuid& entityID) { _sentFilteredEntities.insert(entityID); } + void removeSentFilteredEntity(const QUuid& entityID) { _sentFilteredEntities.remove(entityID); } + bool sentFilteredEntity(const QUuid& entityID) { return _sentFilteredEntities.contains(entityID); } + QSet getSentFilteredEntities() { return _sentFilteredEntities; } + + // these can only be called from the OctreeSendThread for the given Node + void insertFlaggedExtraEntity(const QUuid& filteredEntityID, const QUuid& extraEntityID) + { _flaggedExtraEntities[filteredEntityID].insert(extraEntityID); } + bool isEntityFlaggedAsExtra(const QUuid& entityID) const; private: quint64 _lastDeletedEntitiesSentAt { usecTimestampNow() }; - QSet _entitiesSentLastFrame; + QSet _sentFilteredEntities; + QHash> _flaggedExtraEntities; }; #endif // hifi_EntityNodeData_h diff --git a/libraries/entities/src/EntityTreeElement.cpp b/libraries/entities/src/EntityTreeElement.cpp index 525c1ec65f..755c19e625 100644 --- a/libraries/entities/src/EntityTreeElement.cpp +++ b/libraries/entities/src/EntityTreeElement.cpp @@ -310,16 +310,17 @@ OctreeElement::AppendState EntityTreeElement::appendElementData(OctreePacketData if (entityMatchesFilters) { // make sure this entity is in the set of entities sent last frame - entityNodeData->insertEntitySentLastFrame(entity->getID()); - - } else { - // we might include this entity if it matched in the previous frame - if (entityNodeData->sentEntityLastFrame(entity->getID())) { - - entityNodeData->removeEntitySentLastFrame(entity->getID()); - } else { - includeThisEntity = false; - } + entityNodeData->insertSentFilteredEntity(entity->getID()); + } else if (entityNodeData->sentFilteredEntity(entity->getID())) { + // this entity matched in the previous frame - we send it still so the client realizes it just + // fell outside of their filter + entityNodeData->removeSentFilteredEntity(entity->getID()); + } else if (!entityNodeData->isEntityFlaggedAsExtra(entity->getID())) { + // we don't send this entity because + // (1) it didn't match our filter + // (2) it didn't match our filter last frame + // (3) it isn't one the JSON query flags told us we should still include + includeThisEntity = false; } } From 9d336a84ab3214d5774c22e59a7d9834ab47accb Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 16 Feb 2017 13:05:33 -0800 Subject: [PATCH 045/104] force a full scene send for new ancestors/descendants --- .../src/entities/EntityTreeSendThread.cpp | 47 +++++++++++++++---- .../src/entities/EntityTreeSendThread.h | 5 +- .../src/octree/OctreeSendThread.cpp | 21 +++++++-- libraries/entities/src/EntityNodeData.cpp | 5 ++ libraries/entities/src/EntityNodeData.h | 10 ++-- libraries/octree/src/OctreeQueryNode.h | 5 ++ 6 files changed, 75 insertions(+), 18 deletions(-) diff --git a/assignment-client/src/entities/EntityTreeSendThread.cpp b/assignment-client/src/entities/EntityTreeSendThread.cpp index 11a7ddf038..76c56bdf37 100644 --- a/assignment-client/src/entities/EntityTreeSendThread.cpp +++ b/assignment-client/src/entities/EntityTreeSendThread.cpp @@ -35,8 +35,13 @@ void EntityTreeSendThread::preDistributionProcessing() { // we need to either include the ancestors, descendants, or both for entities matching the filter // included in the JSON query + // first reset our flagged extra entities so we start with an empty set + nodeData->resetFlaggedExtraEntities(); + auto entityTree = std::static_pointer_cast(_myServer->getOctree()); + bool requiresFullScene = false; + // enumerate the set of entity IDs we know currently match the filter foreach(const QUuid& entityID, nodeData->getSentFilteredEntities()) { if (includeAncestors) { @@ -45,7 +50,7 @@ void EntityTreeSendThread::preDistributionProcessing() { entityTree->withReadLock([&]{ auto filteredEntity = entityTree->findEntityByID(entityID); if (filteredEntity) { - addAncestorsToExtraFlaggedEntities(entityID, *filteredEntity, *nodeData); + requiresFullScene |= addAncestorsToExtraFlaggedEntities(entityID, *filteredEntity, *nodeData); } }); } @@ -56,18 +61,27 @@ void EntityTreeSendThread::preDistributionProcessing() { entityTree->withReadLock([&]{ auto filteredEntity = entityTree->findEntityByID(entityID); if (filteredEntity) { - addDescendantsToExtraFlaggedEntities(entityID, *filteredEntity, *nodeData); + requiresFullScene |= addDescendantsToExtraFlaggedEntities(entityID, *filteredEntity, *nodeData); } }); } } + + if (requiresFullScene) { + // for one or more of the entities matching our filter we found new extra entities to include + + // because it is possible that one of these entities hasn't changed since our last send + // and therefore would not be recursed to, we need to force a full traversal for this pass + // of the tree to allow it to grab all of the extra entities we're asking it to include + nodeData->setShouldForceFullScene(requiresFullScene); + } } } } } -void EntityTreeSendThread::addAncestorsToExtraFlaggedEntities(const QUuid& filteredEntityID, +bool EntityTreeSendThread::addAncestorsToExtraFlaggedEntities(const QUuid& filteredEntityID, EntityItem& entityItem, EntityNodeData& nodeData) { // check if this entity has a parent that is also an entity bool success = false; @@ -77,31 +91,48 @@ void EntityTreeSendThread::addAncestorsToExtraFlaggedEntities(const QUuid& filte // we found a parent that is an entity item // first add it to the extra list of things we need to send - nodeData.insertFlaggedExtraEntity(filteredEntityID, entityParent->getID()); + bool parentWasNew = nodeData.insertFlaggedExtraEntity(filteredEntityID, entityParent->getID()); // qDebug() << "Adding" << entityParent->getID() << "which is an ancestor of" << filteredEntityID; // now recursively call ourselves to get its ancestors added too auto parentEntityItem = std::static_pointer_cast(entityParent); - addAncestorsToExtraFlaggedEntities(filteredEntityID, *parentEntityItem, nodeData); + bool ancestorsWereNew = addAncestorsToExtraFlaggedEntities(filteredEntityID, *parentEntityItem, nodeData); + + // return boolean if our parent or any of our ancestors were new additions (via insertFlaggedExtraEntity) + return parentWasNew || ancestorsWereNew; } + + // since we didn't have a parent niether of our parents or ancestors could be new additions + return false; } -void EntityTreeSendThread::addDescendantsToExtraFlaggedEntities(const QUuid& filteredEntityID, +bool EntityTreeSendThread::addDescendantsToExtraFlaggedEntities(const QUuid& filteredEntityID, EntityItem& entityItem, EntityNodeData& nodeData) { + bool hasNewChild = false; + bool hasNewDescendants = false; + // enumerate the immediate children of this entity foreach (SpatiallyNestablePointer child, entityItem.getChildren()) { + + if (child && child->getNestableType() == NestableType::Entity) { // this is a child that is an entity // first add it to the extra list of things we need to send - nodeData.insertFlaggedExtraEntity(filteredEntityID, child->getID()); + hasNewChild |= nodeData.insertFlaggedExtraEntity(filteredEntityID, child->getID()); // qDebug() << "Adding" << child->getID() << "which is a descendant of" << filteredEntityID; // now recursively call ourselves to get its descendants added too auto childEntityItem = std::static_pointer_cast(child); - addDescendantsToExtraFlaggedEntities(filteredEntityID, *childEntityItem, nodeData); + hasNewDescendants |= addDescendantsToExtraFlaggedEntities(filteredEntityID, *childEntityItem, nodeData); } } + + // return our boolean indicating if we added new children or descendants as extra entities to send + // (via insertFlaggedExtraEntity) + return hasNewChild || hasNewDescendants; } + + diff --git a/assignment-client/src/entities/EntityTreeSendThread.h b/assignment-client/src/entities/EntityTreeSendThread.h index 8273b0379f..bfb4c743f1 100644 --- a/assignment-client/src/entities/EntityTreeSendThread.h +++ b/assignment-client/src/entities/EntityTreeSendThread.h @@ -26,8 +26,9 @@ protected: virtual void preDistributionProcessing() override; private: - void addAncestorsToExtraFlaggedEntities(const QUuid& filteredEntityID, EntityItem& entityItem, EntityNodeData& nodeData); - void addDescendantsToExtraFlaggedEntities(const QUuid& filteredEntityID, EntityItem& entityItem, EntityNodeData& nodeData); + // the following two methods return booleans to indicate if any extra flagged entities were new additions to set + bool addAncestorsToExtraFlaggedEntities(const QUuid& filteredEntityID, EntityItem& entityItem, EntityNodeData& nodeData); + bool addDescendantsToExtraFlaggedEntities(const QUuid& filteredEntityID, EntityItem& entityItem, EntityNodeData& nodeData); }; diff --git a/assignment-client/src/octree/OctreeSendThread.cpp b/assignment-client/src/octree/OctreeSendThread.cpp index c3bdb3752d..7922da8af4 100644 --- a/assignment-client/src/octree/OctreeSendThread.cpp +++ b/assignment-client/src/octree/OctreeSendThread.cpp @@ -309,8 +309,11 @@ int OctreeSendThread::packetDistributor(SharedNodePointer node, OctreeQueryNode* return 0; } - // give our pre-distribution processing a chance to do what it needs - preDistributionProcessing(); + if (nodeData->elementBag.isEmpty()) { + // if we're about to do a fresh pass, + // give our pre-distribution processing a chance to do what it needs + preDistributionProcessing(); + } // calculate max number of packets that can be sent during this interval int clientMaxPacketsPerInterval = std::max(1, (nodeData->getMaxQueryPacketsPerSecond() / INTERVALS_PER_SECOND)); @@ -319,9 +322,17 @@ int OctreeSendThread::packetDistributor(SharedNodePointer node, OctreeQueryNode* int truePacketsSent = 0; int trueBytesSent = 0; int packetsSentThisInterval = 0; - bool isFullScene = nodeData->haveJSONParametersChanged() || - (nodeData->getUsesFrustum() - && ((!viewFrustumChanged && nodeData->getViewFrustumJustStoppedChanging()) || nodeData->hasLodChanged())); + + bool isFullScene = nodeData->shouldForceFullScene(); + if (isFullScene) { + // we're forcing a full scene, clear the force in OctreeQueryNode so we don't force it next time again + nodeData->setShouldForceFullScene(false); + } else { + // we aren't forcing a full scene, check if something else suggests we should + isFullScene = nodeData->haveJSONParametersChanged() || + (nodeData->getUsesFrustum() + && ((!viewFrustumChanged && nodeData->getViewFrustumJustStoppedChanging()) || nodeData->hasLodChanged())); + } bool somethingToSend = true; // assume we have something diff --git a/libraries/entities/src/EntityNodeData.cpp b/libraries/entities/src/EntityNodeData.cpp index 3e28bbcce4..2f28f69565 100644 --- a/libraries/entities/src/EntityNodeData.cpp +++ b/libraries/entities/src/EntityNodeData.cpp @@ -11,6 +11,11 @@ #include "EntityNodeData.h" +bool EntityNodeData::insertFlaggedExtraEntity(const QUuid& filteredEntityID, const QUuid& extraEntityID) { + _flaggedExtraEntities[filteredEntityID].insert(extraEntityID); + return !_previousFlaggedExtraEntities[filteredEntityID].contains(extraEntityID); +} + bool EntityNodeData::isEntityFlaggedAsExtra(const QUuid& entityID) const { // enumerate each of the sets for the entities that matched our filter diff --git a/libraries/entities/src/EntityNodeData.h b/libraries/entities/src/EntityNodeData.h index 681d72f2d4..eb5a1610cc 100644 --- a/libraries/entities/src/EntityNodeData.h +++ b/libraries/entities/src/EntityNodeData.h @@ -36,15 +36,19 @@ public: bool sentFilteredEntity(const QUuid& entityID) { return _sentFilteredEntities.contains(entityID); } QSet getSentFilteredEntities() { return _sentFilteredEntities; } - // these can only be called from the OctreeSendThread for the given Node - void insertFlaggedExtraEntity(const QUuid& filteredEntityID, const QUuid& extraEntityID) - { _flaggedExtraEntities[filteredEntityID].insert(extraEntityID); } + // the following flagged extra entity methods can only be called from the OctreeSendThread for the given Node + + // inserts the extra entity and returns a boolean indicating wether the extraEntityID was a new addition + bool insertFlaggedExtraEntity(const QUuid& filteredEntityID, const QUuid& extraEntityID); + bool isEntityFlaggedAsExtra(const QUuid& entityID) const; + void resetFlaggedExtraEntities() { _previousFlaggedExtraEntities = _flaggedExtraEntities; _flaggedExtraEntities.clear(); } private: quint64 _lastDeletedEntitiesSentAt { usecTimestampNow() }; QSet _sentFilteredEntities; QHash> _flaggedExtraEntities; + QHash> _previousFlaggedExtraEntities; }; #endif // hifi_EntityNodeData_h diff --git a/libraries/octree/src/OctreeQueryNode.h b/libraries/octree/src/OctreeQueryNode.h index 10c5598b30..305f43d0f8 100644 --- a/libraries/octree/src/OctreeQueryNode.h +++ b/libraries/octree/src/OctreeQueryNode.h @@ -103,6 +103,9 @@ public: // call only from OctreeSendThread for the given node bool haveJSONParametersChanged(); + bool shouldForceFullScene() const { return _shouldForceFullScene; } + bool setShouldForceFullScene(bool shouldForceFullScene) { _shouldForceFullScene = shouldForceFullScene; } + private: OctreeQueryNode(const OctreeQueryNode &); OctreeQueryNode& operator= (const OctreeQueryNode&); @@ -148,6 +151,8 @@ private: std::array _lastOctreePayload; QJsonObject _lastCheckJSONParameters; + + bool _shouldForceFullScene { false }; }; #endif // hifi_OctreeQueryNode_h From fe11a945ef4deb04150602e77f6ac6310fe69b29 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 16 Feb 2017 13:38:09 -0800 Subject: [PATCH 046/104] remove debug from EntityTreeSendThread --- assignment-client/src/entities/EntityTreeSendThread.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/assignment-client/src/entities/EntityTreeSendThread.cpp b/assignment-client/src/entities/EntityTreeSendThread.cpp index 76c56bdf37..e049647428 100644 --- a/assignment-client/src/entities/EntityTreeSendThread.cpp +++ b/assignment-client/src/entities/EntityTreeSendThread.cpp @@ -93,8 +93,6 @@ bool EntityTreeSendThread::addAncestorsToExtraFlaggedEntities(const QUuid& filte // first add it to the extra list of things we need to send bool parentWasNew = nodeData.insertFlaggedExtraEntity(filteredEntityID, entityParent->getID()); -// qDebug() << "Adding" << entityParent->getID() << "which is an ancestor of" << filteredEntityID; - // now recursively call ourselves to get its ancestors added too auto parentEntityItem = std::static_pointer_cast(entityParent); bool ancestorsWereNew = addAncestorsToExtraFlaggedEntities(filteredEntityID, *parentEntityItem, nodeData); @@ -122,8 +120,6 @@ bool EntityTreeSendThread::addDescendantsToExtraFlaggedEntities(const QUuid& fil // first add it to the extra list of things we need to send hasNewChild |= nodeData.insertFlaggedExtraEntity(filteredEntityID, child->getID()); -// qDebug() << "Adding" << child->getID() << "which is a descendant of" << filteredEntityID; - // now recursively call ourselves to get its descendants added too auto childEntityItem = std::static_pointer_cast(child); hasNewDescendants |= addDescendantsToExtraFlaggedEntities(filteredEntityID, *childEntityItem, nodeData); From 4ece56b6739d805e29507c9fe21811869746d0fe Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 16 Feb 2017 13:39:21 -0800 Subject: [PATCH 047/104] spacing cleanup in EntityTreeSendThread --- assignment-client/src/entities/EntityTreeSendThread.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/assignment-client/src/entities/EntityTreeSendThread.cpp b/assignment-client/src/entities/EntityTreeSendThread.cpp index e049647428..7febdc67e1 100644 --- a/assignment-client/src/entities/EntityTreeSendThread.cpp +++ b/assignment-client/src/entities/EntityTreeSendThread.cpp @@ -76,7 +76,6 @@ void EntityTreeSendThread::preDistributionProcessing() { nodeData->setShouldForceFullScene(requiresFullScene); } } - } } } @@ -113,7 +112,6 @@ bool EntityTreeSendThread::addDescendantsToExtraFlaggedEntities(const QUuid& fil // enumerate the immediate children of this entity foreach (SpatiallyNestablePointer child, entityItem.getChildren()) { - if (child && child->getNestableType() == NestableType::Entity) { // this is a child that is an entity From d4119d5676958e46526d13f976cff978febac83c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 16 Feb 2017 13:41:04 -0800 Subject: [PATCH 048/104] fix for non-void set of should force scene --- libraries/octree/src/OctreeQueryNode.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/octree/src/OctreeQueryNode.h b/libraries/octree/src/OctreeQueryNode.h index 305f43d0f8..fd89a89949 100644 --- a/libraries/octree/src/OctreeQueryNode.h +++ b/libraries/octree/src/OctreeQueryNode.h @@ -104,7 +104,7 @@ public: bool haveJSONParametersChanged(); bool shouldForceFullScene() const { return _shouldForceFullScene; } - bool setShouldForceFullScene(bool shouldForceFullScene) { _shouldForceFullScene = shouldForceFullScene; } + void setShouldForceFullScene(bool shouldForceFullScene) { _shouldForceFullScene = shouldForceFullScene; } private: OctreeQueryNode(const OctreeQueryNode &); From 9f9fc03751b5a5eab37d58f45a5d0185536315c6 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 16 Feb 2017 15:08:42 -0800 Subject: [PATCH 049/104] call EntityTree update when ScriptEngine update occurs --- assignment-client/src/scripts/EntityScriptServer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/assignment-client/src/scripts/EntityScriptServer.cpp b/assignment-client/src/scripts/EntityScriptServer.cpp index 06df255968..f8f728f834 100644 --- a/assignment-client/src/scripts/EntityScriptServer.cpp +++ b/assignment-client/src/scripts/EntityScriptServer.cpp @@ -419,6 +419,7 @@ void EntityScriptServer::resetEntitiesScriptEngine() { connect(newEngine.data(), &ScriptEngine::update, this, [this] { _entityViewer.queryOctree(); + _entityViewer.getTree()->update(); }); From 78301fd9477f0363767ab9d8d40849ac9b6308c5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 16 Feb 2017 15:59:45 -0800 Subject: [PATCH 050/104] call getParentPointer when parent ID changes to update children --- libraries/shared/src/SpatiallyNestable.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/shared/src/SpatiallyNestable.cpp b/libraries/shared/src/SpatiallyNestable.cpp index ddc3f416e0..75574967e4 100644 --- a/libraries/shared/src/SpatiallyNestable.cpp +++ b/libraries/shared/src/SpatiallyNestable.cpp @@ -70,6 +70,9 @@ void SpatiallyNestable::setParentID(const QUuid& parentID) { _parentKnowsMe = false; } }); + + bool success = false; + getParentPointer(success); } Transform SpatiallyNestable::getParentTransform(bool& success, int depth) const { From be12a8ffa4636c4e0910da44b42f5e2ef1736cd9 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 16 Feb 2017 16:58:06 -0800 Subject: [PATCH 051/104] stats cleanup --- assignment-client/src/avatars/AvatarMixer.cpp | 89 ++++++++++++------- 1 file changed, 55 insertions(+), 34 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 8249daed9d..5394b6efec 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -464,44 +464,63 @@ void AvatarMixer::sendStatsPacket() { QJsonObject statsObject; - statsObject["threads"] = _slavePool.numThreads(); //statsObject["average_listeners_last_second"] = (float) _sumListeners / (float) _numStatFrames; //statsObject["average_identity_packets_per_frame"] = (float) _sumIdentityPackets / (float) _numStatFrames; - statsObject["trailing_sleep_percentage"] = _trailingSleepRatio * 100; - statsObject["performance_throttling_ratio"] = _performanceThrottlingRatio; statsObject["broadcast_loop_rate"] = _loopRate.rate(); + statsObject["threads"] = _slavePool.numThreads(); + statsObject["throttling_1_trailing_sleep_percentage"] = _trailingSleepRatio * 100; + statsObject["throttling_2_performance_ratio"] = _performanceThrottlingRatio; + // this things all occur on the frequency of the tight loop int tightLoopFrames = _numTightLoopFrames; int tenTimesPerFrame = tightLoopFrames * 10; #define TIGHT_LOOP_STAT(x) (x > tenTimesPerFrame) ? x / tightLoopFrames : ((float)x / (float)tightLoopFrames); - statsObject["timing_average_y_processEvents"] = TIGHT_LOOP_STAT(_processEventsElapsedTime); - statsObject["timing_average_y_queueIncomingPacket"] = TIGHT_LOOP_STAT(_queueIncomingPacketElapsedTime); + QJsonObject singleCoreTasks; + singleCoreTasks["processEvents"] = TIGHT_LOOP_STAT(_processEventsElapsedTime); + singleCoreTasks["queueIncomingPacket"] = TIGHT_LOOP_STAT(_queueIncomingPacketElapsedTime); - statsObject["timing_average_a1_broadcastAvatarData"] = TIGHT_LOOP_STAT(_broadcastAvatarDataElapsedTime); - statsObject["timing_average_a2_innnerBroadcastAvatarData"] = TIGHT_LOOP_STAT(_broadcastAvatarDataInner); - statsObject["timing_average_a3_broadcastAvatarDataLockWait"] = TIGHT_LOOP_STAT(_broadcastAvatarDataLockWait); - statsObject["timing_average_a4_broadcastAvatarDataNodeTransform"] = TIGHT_LOOP_STAT(_broadcastAvatarDataNodeTransform); - statsObject["timing_average_a5_broadcastAvatarDataNodeFunctor"] = TIGHT_LOOP_STAT(_broadcastAvatarDataNodeFunctor); + QJsonObject incomingPacketStats; + incomingPacketStats["handleAvatarIdentityPacket"] = TIGHT_LOOP_STAT(_handleAvatarIdentityPacketElapsedTime); + incomingPacketStats["handleKillAvatarPacket"] = TIGHT_LOOP_STAT(_handleKillAvatarPacketElapsedTime); + incomingPacketStats["handleNodeIgnoreRequestPacket"] = TIGHT_LOOP_STAT(_handleNodeIgnoreRequestPacketElapsedTime); + incomingPacketStats["handleRadiusIgnoreRequestPacket"] = TIGHT_LOOP_STAT(_handleRadiusIgnoreRequestPacketElapsedTime); + incomingPacketStats["handleRequestsDomainListDataPacket"] = TIGHT_LOOP_STAT(_handleRequestsDomainListDataPacketElapsedTime); + incomingPacketStats["handleViewFrustumPacket"] = TIGHT_LOOP_STAT(_handleViewFrustumPacketElapsedTime); - statsObject["timing_average_z_displayNameManagement"] = TIGHT_LOOP_STAT(_displayNameManagementElapsedTime); - statsObject["timing_average_z_handleAvatarIdentityPacket"] = TIGHT_LOOP_STAT(_handleAvatarIdentityPacketElapsedTime); - statsObject["timing_average_z_handleKillAvatarPacket"] = TIGHT_LOOP_STAT(_handleKillAvatarPacketElapsedTime); - statsObject["timing_average_z_handleNodeIgnoreRequestPacket"] = TIGHT_LOOP_STAT(_handleNodeIgnoreRequestPacketElapsedTime); - statsObject["timing_average_z_handleRadiusIgnoreRequestPacket"] = TIGHT_LOOP_STAT(_handleRadiusIgnoreRequestPacketElapsedTime); - statsObject["timing_average_z_handleRequestsDomainListDataPacket"] = TIGHT_LOOP_STAT(_handleRequestsDomainListDataPacketElapsedTime); - statsObject["timing_average_z_handleViewFrustumPacket"] = TIGHT_LOOP_STAT(_handleViewFrustumPacketElapsedTime); - statsObject["timing_average_z_processQueuedAvatarDataPackets"] = TIGHT_LOOP_STAT(_processQueuedAvatarDataPacketsElapsedTime); - statsObject["timing_average_z_processQueuedAvatarDataPacketsLockWait"] = TIGHT_LOOP_STAT(_processQueuedAvatarDataPacketsLockWaitElapsedTime); + singleCoreTasks["incoming_packets"] = incomingPacketStats; + singleCoreTasks["sendStats"] = (float)_sendStatsElapsedTime; - statsObject["timing_sendStats"] = (float)_sendStatsElapsedTime; + statsObject["singleCoreTasks"] = singleCoreTasks; + + QJsonObject parallelTasks; + + QJsonObject processQueuedAvatarDataPacketsStats; + processQueuedAvatarDataPacketsStats["1_total"] = TIGHT_LOOP_STAT(_processQueuedAvatarDataPacketsElapsedTime); + processQueuedAvatarDataPacketsStats["2_lockWait"] = TIGHT_LOOP_STAT(_processQueuedAvatarDataPacketsLockWaitElapsedTime); + parallelTasks["processQueuedAvatarDataPackets"] = processQueuedAvatarDataPacketsStats; + + QJsonObject broadcastAvatarDataStats; + + broadcastAvatarDataStats["1_total"] = TIGHT_LOOP_STAT(_broadcastAvatarDataElapsedTime); + broadcastAvatarDataStats["2_innner"] = TIGHT_LOOP_STAT(_broadcastAvatarDataInner); + broadcastAvatarDataStats["3_lockWait"] = TIGHT_LOOP_STAT(_broadcastAvatarDataLockWait); + broadcastAvatarDataStats["4_NodeTransform"] = TIGHT_LOOP_STAT(_broadcastAvatarDataNodeTransform); + broadcastAvatarDataStats["5_Functor"] = TIGHT_LOOP_STAT(_broadcastAvatarDataNodeFunctor); + + parallelTasks["broadcastAvatarData"] = broadcastAvatarDataStats; + + QJsonObject displayNameManagementStats; + displayNameManagementStats["1_total"] = TIGHT_LOOP_STAT(_displayNameManagementElapsedTime); + parallelTasks["displayNameManagement"] = displayNameManagementStats; + + statsObject["parallelTasks"] = parallelTasks; AvatarMixerSlaveStats aggregateStats; - QJsonObject slavesObject; // gather stats int slaveNumber = 1; @@ -511,7 +530,7 @@ void AvatarMixer::sendStatsPacket() { slave.harvestStats(stats); slaveObject["nodesProcessed"] = TIGHT_LOOP_STAT(stats.nodesProcessed); slaveObject["numPacketsReceived"] = TIGHT_LOOP_STAT(stats.packetsProcessed); - statsObject["numPacketsSent"] = TIGHT_LOOP_STAT(stats.numPacketsSent); + slaveObject["numPacketsSent"] = TIGHT_LOOP_STAT(stats.numPacketsSent); slaveObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT(stats.processIncomingPacketsElapsedTime); slaveObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT(stats.ignoreCalculationElapsedTime); @@ -525,20 +544,22 @@ void AvatarMixer::sendStatsPacket() { aggregateStats += stats; }); - statsObject["timing_slaves"] = slavesObject; - // broadcastAvatarDataElapsed timing details... - statsObject["aggregate_nodesProcessed"] = TIGHT_LOOP_STAT(aggregateStats.nodesProcessed); - statsObject["aggregate_numPacketsReceived"] = TIGHT_LOOP_STAT(aggregateStats.packetsProcessed); - statsObject["aggregate_numPacketsSent"] = TIGHT_LOOP_STAT(aggregateStats.numPacketsSent); - + QJsonObject slavesAggregatObject; - statsObject["timing_aggregate_1_processIncomingPackets"] = TIGHT_LOOP_STAT(aggregateStats.processIncomingPacketsElapsedTime); - statsObject["timing_aggregate_2_ignoreCalculation"] = TIGHT_LOOP_STAT(aggregateStats.ignoreCalculationElapsedTime); - statsObject["timing_aggregate_3_toByteArray"] = TIGHT_LOOP_STAT(aggregateStats.toByteArrayElapsedTime); - statsObject["timing_aggregate_4_avatarDataPacking"] = TIGHT_LOOP_STAT(aggregateStats.avatarDataPackingElapsedTime); - statsObject["timing_aggregate_5_packetSending"] = TIGHT_LOOP_STAT(aggregateStats.packetSendingElapsedTime); - statsObject["timing_aggregate_6_jobElapsedTime"] = TIGHT_LOOP_STAT(aggregateStats.jobElapsedTime); + slavesAggregatObject["nodesProcessed"] = TIGHT_LOOP_STAT(aggregateStats.nodesProcessed); + slavesAggregatObject["numPacketsReceived"] = TIGHT_LOOP_STAT(aggregateStats.packetsProcessed); + slavesAggregatObject["numPacketsSent"] = TIGHT_LOOP_STAT(aggregateStats.numPacketsSent); + + slavesAggregatObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT(aggregateStats.processIncomingPacketsElapsedTime); + slavesAggregatObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT(aggregateStats.ignoreCalculationElapsedTime); + slavesAggregatObject["timing_3_toByteArray"] = TIGHT_LOOP_STAT(aggregateStats.toByteArrayElapsedTime); + slavesAggregatObject["timing_4_avatarDataPacking"] = TIGHT_LOOP_STAT(aggregateStats.avatarDataPackingElapsedTime); + slavesAggregatObject["timing_5_packetSending"] = TIGHT_LOOP_STAT(aggregateStats.packetSendingElapsedTime); + slavesAggregatObject["timing_6_jobElapsedTime"] = TIGHT_LOOP_STAT(aggregateStats.jobElapsedTime); + + statsObject["slaves_aggregate"] = slavesAggregatObject; + statsObject["slaves_individual"] = slavesObject; _handleViewFrustumPacketElapsedTime = 0; _handleAvatarIdentityPacketElapsedTime = 0; From c1e41311fc3a32fe6ae38b0ed3068091724771df Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Thu, 16 Feb 2017 17:18:09 -0800 Subject: [PATCH 052/104] working toward using uuids for overlay IDs --- interface/src/Application.cpp | 19 ++-- interface/src/Application.h | 6 +- interface/src/devices/DdeFaceTracker.h | 2 +- interface/src/ui/overlays/Overlay.cpp | 22 +++++ interface/src/ui/overlays/Overlay.h | 30 +++++- interface/src/ui/overlays/OverlayPanel.cpp | 6 +- interface/src/ui/overlays/OverlayPanel.h | 11 ++- interface/src/ui/overlays/Overlays.cpp | 88 +++++++++-------- interface/src/ui/overlays/Overlays.h | 108 +++++++++++---------- interface/src/ui/overlays/Web3DOverlay.cpp | 4 +- 10 files changed, 181 insertions(+), 115 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 488e97b5e6..a00cfb37d2 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -170,6 +170,7 @@ #include "ui/StandAloneJSConsole.h" #include "ui/Stats.h" #include "ui/UpdateDialog.h" +#include "ui/overlays/Overlays.h" #include "Util.h" #include "InterfaceParentFinder.h" @@ -528,7 +529,7 @@ bool setupEssentials(int& argc, char** argv) { // to take care of highlighting keyboard focused items, rather than // continuing to overburden Application.cpp std::shared_ptr _keyboardFocusHighlight{ nullptr }; -int _keyboardFocusHighlightID{ -1 }; +OverlayID _keyboardFocusHighlightID{ -1 }; // FIXME hack access to the internal share context for the Chromium helper @@ -1227,12 +1228,12 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo // Keyboard focus handling for Web overlays. auto overlays = &(qApp->getOverlays()); - connect(overlays, &Overlays::mousePressOnOverlay, [=](unsigned int overlayID, const PointerEvent& event) { + connect(overlays, &Overlays::mousePressOnOverlay, [=](OverlayID overlayID, const PointerEvent& event) { setKeyboardFocusEntity(UNKNOWN_ENTITY_ID); setKeyboardFocusOverlay(overlayID); }); - connect(overlays, &Overlays::overlayDeleted, [=](unsigned int overlayID) { + connect(overlays, &Overlays::overlayDeleted, [=](OverlayID overlayID) { if (overlayID == _keyboardFocusedOverlay.get()) { setKeyboardFocusOverlay(UNKNOWN_OVERLAY_ID); } @@ -1682,9 +1683,9 @@ void Application::cleanupBeforeQuit() { _applicationStateDevice.reset(); { - if (_keyboardFocusHighlightID > 0) { + if (_keyboardFocusHighlightID) { getOverlays().deleteOverlay(_keyboardFocusHighlightID); - _keyboardFocusHighlightID = -1; + _keyboardFocusHighlightID = UNKNOWN_OVERLAY_ID; } _keyboardFocusHighlight = nullptr; } @@ -4090,7 +4091,7 @@ void Application::rotationModeChanged() const { void Application::setKeyboardFocusHighlight(const glm::vec3& position, const glm::quat& rotation, const glm::vec3& dimensions) { // Create focus - if (_keyboardFocusHighlightID < 0 || !getOverlays().isAddedOverlay(_keyboardFocusHighlightID)) { + if (_keyboardFocusHighlightID == UNKNOWN_OVERLAY_ID || !getOverlays().isAddedOverlay(_keyboardFocusHighlightID)) { _keyboardFocusHighlight = std::make_shared(); _keyboardFocusHighlight->setAlpha(1.0f); _keyboardFocusHighlight->setBorderSize(1.0f); @@ -4152,11 +4153,11 @@ void Application::setKeyboardFocusEntity(EntityItemID entityItemID) { } } -unsigned int Application::getKeyboardFocusOverlay() { +OverlayID Application::getKeyboardFocusOverlay() { return _keyboardFocusedOverlay.get(); } -void Application::setKeyboardFocusOverlay(unsigned int overlayID) { +void Application::setKeyboardFocusOverlay(OverlayID overlayID) { if (overlayID != _keyboardFocusedOverlay.get()) { _keyboardFocusedOverlay.set(overlayID); @@ -5527,6 +5528,8 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEngine* scri auto entityScriptServerLog = DependencyManager::get(); scriptEngine->registerGlobalObject("EntityScriptServerLog", entityScriptServerLog.data()); + qScriptRegisterMetaType(scriptEngine, OverlayIDtoScriptValue, OverlayIDfromScriptValue); + // connect this script engines printedMessage signal to the global ScriptEngines these various messages connect(scriptEngine, &ScriptEngine::printedMessage, DependencyManager::get().data(), &ScriptEngines::onPrintedMessage); connect(scriptEngine, &ScriptEngine::errorMessage, DependencyManager::get().data(), &ScriptEngines::onErrorMessage); diff --git a/interface/src/Application.h b/interface/src/Application.h index cab830ec88..7dcb35babc 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -381,8 +381,8 @@ public slots: void setKeyboardFocusEntity(QUuid id); void setKeyboardFocusEntity(EntityItemID entityItemID); - unsigned int getKeyboardFocusOverlay(); - void setKeyboardFocusOverlay(unsigned int overlayID); + OverlayID getKeyboardFocusOverlay(); + void setKeyboardFocusOverlay(OverlayID overlayID); void addAssetToWorldMessageClose(); @@ -610,7 +610,7 @@ private: DialogsManagerScriptingInterface* _dialogsManagerScriptingInterface = new DialogsManagerScriptingInterface(); ThreadSafeValueCache _keyboardFocusedEntity; - ThreadSafeValueCache _keyboardFocusedOverlay; + ThreadSafeValueCache _keyboardFocusedOverlay; quint64 _lastAcceptedKeyPress = 0; bool _isForeground = true; // starts out assumed to be in foreground bool _inPaint = false; diff --git a/interface/src/devices/DdeFaceTracker.h b/interface/src/devices/DdeFaceTracker.h index 931ab099e9..973c3b224e 100644 --- a/interface/src/devices/DdeFaceTracker.h +++ b/interface/src/devices/DdeFaceTracker.h @@ -149,7 +149,7 @@ private: int _calibrationCount; QVector _calibrationValues; TextOverlay* _calibrationBillboard; - int _calibrationBillboardID; + OverlayID _calibrationBillboardID; QString _calibrationMessage; bool _isCalibrated; void addCalibrationDatum(); diff --git a/interface/src/ui/overlays/Overlay.cpp b/interface/src/ui/overlays/Overlay.cpp index 82b90d228c..8ce4b7e97a 100644 --- a/interface/src/ui/overlays/Overlay.cpp +++ b/interface/src/ui/overlays/Overlay.cpp @@ -204,3 +204,25 @@ void Overlay::removeFromScene(Overlay::Pointer overlay, std::shared_ptr qVectorOverlayIDFromScriptValue(const QScriptValue& array) { + if (!array.isArray()) { + return QVector(); + } + QVector newVector; + int length = array.property("length").toInteger(); + newVector.reserve(length); + for (int i = 0; i < length; i++) { + OverlayID id; + OverlayIDfromScriptValue(array.property(i), id); + newVector << id; + } + return newVector; +} diff --git a/interface/src/ui/overlays/Overlay.h b/interface/src/ui/overlays/Overlay.h index 51792b24b3..d55b72a189 100644 --- a/interface/src/ui/overlays/Overlay.h +++ b/interface/src/ui/overlays/Overlay.h @@ -15,6 +15,24 @@ #include // for xColor #include +class OverlayID { +public: + OverlayID() {} + OverlayID(int value) { id = value; } + + OverlayID& operator=(const OverlayID& other) { id = other.id; return *this; } + OverlayID& operator++() { id++; return *this; } + + // OverlayID& operator=(unsigned int value) { id = value; return *this; } + bool operator==(const OverlayID& other) const { return id == other.id; } + bool operator!=(const OverlayID& other) const { return id != other.id; } + // bool operator<(const OverlayID& other) const { return id < other.id; } + // bool operator>(const OverlayID& other) const { return id > other.id; } + operator bool() const { return id != 0; } + + unsigned int id; +}; + class Overlay : public QObject { Q_OBJECT @@ -32,8 +50,8 @@ public: Overlay(const Overlay* overlay); ~Overlay(); - unsigned int getOverlayID() { return _overlayID; } - void setOverlayID(unsigned int overlayID) { _overlayID = overlayID; } + OverlayID getOverlayID() { return _overlayID; } + void setOverlayID(OverlayID overlayID) { _overlayID = overlayID; } virtual void update(float deltatime) {} virtual void render(RenderArgs* args) = 0; @@ -89,7 +107,7 @@ protected: render::ItemID _renderItemID{ render::Item::INVALID_ITEM_ID }; - unsigned int _overlayID; // what Overlays.cpp knows this instance as + OverlayID _overlayID; // what Overlays.cpp knows this instance as bool _isLoaded; float _alpha; @@ -117,5 +135,11 @@ namespace render { template <> const ShapeKey shapeGetShapeKey(const Overlay::Pointer& overlay); } +Q_DECLARE_METATYPE(OverlayID); +Q_DECLARE_METATYPE(QVector); +QScriptValue OverlayIDtoScriptValue(QScriptEngine* engine, const OverlayID& id); +void OverlayIDfromScriptValue(const QScriptValue &object, OverlayID& id); +QVector qVectorOverlayIDFromScriptValue(const QScriptValue& array); + #endif // hifi_Overlay_h diff --git a/interface/src/ui/overlays/OverlayPanel.cpp b/interface/src/ui/overlays/OverlayPanel.cpp index cb57c6ec6b..484e1da216 100644 --- a/interface/src/ui/overlays/OverlayPanel.cpp +++ b/interface/src/ui/overlays/OverlayPanel.cpp @@ -51,13 +51,13 @@ void propertyBindingFromVariant(const QVariant& objectVar, PropertyBinding& valu } -void OverlayPanel::addChild(unsigned int childId) { +void OverlayPanel::addChild(OverlayID childId) { if (!_children.contains(childId)) { _children.append(childId); } } -void OverlayPanel::removeChild(unsigned int childId) { +void OverlayPanel::removeChild(OverlayID childId) { if (_children.contains(childId)) { _children.removeOne(childId); } @@ -89,7 +89,7 @@ QVariant OverlayPanel::getProperty(const QString &property) { if (property == "children") { QVariantList array; for (int i = 0; i < _children.length(); i++) { - array.append(_children[i]); + array.append(OverlayIDtoScriptValue(nullptr, _children[i])); } return array; } diff --git a/interface/src/ui/overlays/OverlayPanel.h b/interface/src/ui/overlays/OverlayPanel.h index b0b8cdb989..5bffe3851e 100644 --- a/interface/src/ui/overlays/OverlayPanel.h +++ b/interface/src/ui/overlays/OverlayPanel.h @@ -20,6 +20,7 @@ #include "PanelAttachable.h" #include "Billboardable.h" +#include "Overlay.h" class PropertyBinding { public: @@ -54,10 +55,10 @@ public: void setAnchorScale(const glm::vec3& scale) { _anchorTransform.setScale(scale); } void setVisible(bool visible) { _visible = visible; } - const QList& getChildren() { return _children; } - void addChild(unsigned int childId); - void removeChild(unsigned int childId); - unsigned int popLastChild() { return _children.takeLast(); } + const QList& getChildren() { return _children; } + void addChild(OverlayID childId); + void removeChild(OverlayID childId); + OverlayID popLastChild() { return _children.takeLast(); } void setProperties(const QVariantMap& properties); QVariant getProperty(const QString& property); @@ -74,7 +75,7 @@ private: QUuid _anchorRotationBindEntity; bool _visible = true; - QList _children; + QList _children; QScriptEngine* _scriptEngine; }; diff --git a/interface/src/ui/overlays/Overlays.cpp b/interface/src/ui/overlays/Overlays.cpp index e81e48f2bc..42e6d2c679 100644 --- a/interface/src/ui/overlays/Overlays.cpp +++ b/interface/src/ui/overlays/Overlays.cpp @@ -139,7 +139,7 @@ void Overlays::enable() { _enabled = true; } -Overlay::Pointer Overlays::getOverlay(unsigned int id) const { +Overlay::Pointer Overlays::getOverlay(OverlayID id) const { if (_overlaysHUD.contains(id)) { return _overlaysHUD[id]; } @@ -149,7 +149,7 @@ Overlay::Pointer Overlays::getOverlay(unsigned int id) const { return nullptr; } -unsigned int Overlays::addOverlay(const QString& type, const QVariant& properties) { +OverlayID Overlays::addOverlay(const QString& type, const QVariant& properties) { Overlay::Pointer thisOverlay = nullptr; if (type == ImageOverlay::TYPE) { @@ -191,11 +191,11 @@ unsigned int Overlays::addOverlay(const QString& type, const QVariant& propertie return 0; } -unsigned int Overlays::addOverlay(Overlay::Pointer overlay) { +OverlayID Overlays::addOverlay(Overlay::Pointer overlay) { QWriteLocker lock(&_lock); - unsigned int thisID = _nextOverlayID; + OverlayID thisID = _nextOverlayID; overlay->setOverlayID(thisID); - _nextOverlayID++; + ++_nextOverlayID; if (overlay->is3D()) { _overlaysWorld[thisID] = overlay; @@ -210,11 +210,11 @@ unsigned int Overlays::addOverlay(Overlay::Pointer overlay) { return thisID; } -unsigned int Overlays::cloneOverlay(unsigned int id) { +OverlayID Overlays::cloneOverlay(OverlayID id) { Overlay::Pointer thisOverlay = getOverlay(id); if (thisOverlay) { - unsigned int cloneId = addOverlay(Overlay::Pointer(thisOverlay->createClone())); + OverlayID cloneId = addOverlay(Overlay::Pointer(thisOverlay->createClone())); auto attachable = std::dynamic_pointer_cast(thisOverlay); if (attachable && attachable->getParentPanel()) { attachable->getParentPanel()->addChild(cloneId); @@ -225,7 +225,7 @@ unsigned int Overlays::cloneOverlay(unsigned int id) { return 0; // Not found } -bool Overlays::editOverlay(unsigned int id, const QVariant& properties) { +bool Overlays::editOverlay(OverlayID id, const QVariant& properties) { QWriteLocker lock(&_lock); Overlay::Pointer thisOverlay = getOverlay(id); @@ -243,7 +243,7 @@ bool Overlays::editOverlays(const QVariant& propertiesById) { QWriteLocker lock(&_lock); for (const auto& key : map.keys()) { bool convertSuccess; - unsigned int id = key.toUInt(&convertSuccess); + OverlayID id = key.toUInt(&convertSuccess); if (!convertSuccess) { success = false; continue; @@ -260,7 +260,7 @@ bool Overlays::editOverlays(const QVariant& propertiesById) { return success; } -void Overlays::deleteOverlay(unsigned int id) { +void Overlays::deleteOverlay(OverlayID id) { Overlay::Pointer overlayToDelete; { @@ -286,7 +286,7 @@ void Overlays::deleteOverlay(unsigned int id) { emit overlayDeleted(id); } -QString Overlays::getOverlayType(unsigned int overlayId) const { +QString Overlays::getOverlayType(OverlayID overlayId) const { Overlay::Pointer overlay = getOverlay(overlayId); if (overlay) { return overlay->getType(); @@ -294,7 +294,7 @@ QString Overlays::getOverlayType(unsigned int overlayId) const { return ""; } -QObject* Overlays::getOverlayObject(unsigned int id) { +QObject* Overlays::getOverlayObject(OverlayID id) { Overlay::Pointer thisOverlay = getOverlay(id); if (thisOverlay) { return qobject_cast(&(*thisOverlay)); @@ -302,7 +302,7 @@ QObject* Overlays::getOverlayObject(unsigned int id) { return nullptr; } -unsigned int Overlays::getParentPanel(unsigned int childId) const { +OverlayID Overlays::getParentPanel(OverlayID childId) const { Overlay::Pointer overlay = getOverlay(childId); auto attachable = std::dynamic_pointer_cast(overlay); if (attachable) { @@ -313,7 +313,7 @@ unsigned int Overlays::getParentPanel(unsigned int childId) const { return 0; } -void Overlays::setParentPanel(unsigned int childId, unsigned int panelId) { +void Overlays::setParentPanel(OverlayID childId, OverlayID panelId) { auto attachable = std::dynamic_pointer_cast(getOverlay(childId)); if (attachable) { if (_panels.contains(panelId)) { @@ -343,13 +343,13 @@ void Overlays::setParentPanel(unsigned int childId, unsigned int panelId) { } } -unsigned int Overlays::getOverlayAtPoint(const glm::vec2& point) { +OverlayID Overlays::getOverlayAtPoint(const glm::vec2& point) { glm::vec2 pointCopy = point; QReadLocker lock(&_lock); if (!_enabled) { return 0; } - QMapIterator i(_overlaysHUD); + QMapIterator i(_overlaysHUD); i.toBack(); const float LARGE_NEGATIVE_FLOAT = -9999999; @@ -361,7 +361,7 @@ unsigned int Overlays::getOverlayAtPoint(const glm::vec2& point) { while (i.hasPrevious()) { i.previous(); - unsigned int thisID = i.key(); + OverlayID thisID = i.key(); if (i.value()->is3D()) { auto thisOverlay = std::dynamic_pointer_cast(i.value()); if (thisOverlay && !thisOverlay->getIgnoreRayIntersection()) { @@ -381,7 +381,7 @@ unsigned int Overlays::getOverlayAtPoint(const glm::vec2& point) { return 0; // not found } -OverlayPropertyResult Overlays::getProperty(unsigned int id, const QString& property) { +OverlayPropertyResult Overlays::getProperty(OverlayID id, const QString& property) { OverlayPropertyResult result; Overlay::Pointer thisOverlay = getOverlay(id); QReadLocker lock(&_lock); @@ -406,15 +406,21 @@ void OverlayPropertyResultFromScriptValue(const QScriptValue& object, OverlayPro } -RayToOverlayIntersectionResult Overlays::findRayIntersection(const PickRay& ray) { +RayToOverlayIntersectionResult Overlays::findRayIntersection(const PickRay& ray, bool precisionPicking, + const QScriptValue& overlayIDsToInclude, + const QScriptValue& overlayIDsToDiscard, + bool visibleOnly, bool collidableOnly) { float bestDistance = std::numeric_limits::max(); bool bestIsFront = false; + const QVector overlaysToInclude = qVectorOverlayIDFromScriptValue(overlayIDsToInclude); + const QVector overlaysToDiscard = qVectorOverlayIDFromScriptValue(overlayIDsToDiscard); + RayToOverlayIntersectionResult result; - QMapIterator i(_overlaysWorld); + QMapIterator i(_overlaysWorld); i.toBack(); while (i.hasPrevious()) { i.previous(); - unsigned int thisID = i.key(); + OverlayID thisID = i.key(); auto thisOverlay = std::dynamic_pointer_cast(i.value()); if (thisOverlay && thisOverlay->getVisible() && !thisOverlay->getIgnoreRayIntersection() && thisOverlay->isLoaded()) { float thisDistance; @@ -454,7 +460,7 @@ RayToOverlayIntersectionResult::RayToOverlayIntersectionResult() : QScriptValue RayToOverlayIntersectionResultToScriptValue(QScriptEngine* engine, const RayToOverlayIntersectionResult& value) { auto obj = engine->newObject(); obj.setProperty("intersects", value.intersects); - obj.setProperty("overlayID", value.overlayID); + obj.setProperty("overlayID", OverlayIDtoScriptValue(engine, value.overlayID)); obj.setProperty("distance", value.distance); QString faceName = ""; @@ -523,7 +529,7 @@ void RayToOverlayIntersectionResultFromScriptValue(const QScriptValue& objectVar value.extraInfo = object["extraInfo"].toString(); } -bool Overlays::isLoaded(unsigned int id) { +bool Overlays::isLoaded(OverlayID id) { QReadLocker lock(&_lock); Overlay::Pointer thisOverlay = getOverlay(id); if (!thisOverlay) { @@ -532,7 +538,7 @@ bool Overlays::isLoaded(unsigned int id) { return thisOverlay->isLoaded(); } -QSizeF Overlays::textSize(unsigned int id, const QString& text) const { +QSizeF Overlays::textSize(OverlayID id, const QString& text) const { Overlay::Pointer thisOverlay = _overlaysHUD[id]; if (thisOverlay) { if (auto textOverlay = std::dynamic_pointer_cast(thisOverlay)) { @@ -547,30 +553,30 @@ QSizeF Overlays::textSize(unsigned int id, const QString& text) const { return QSizeF(0.0f, 0.0f); } -unsigned int Overlays::addPanel(OverlayPanel::Pointer panel) { +OverlayID Overlays::addPanel(OverlayPanel::Pointer panel) { QWriteLocker lock(&_lock); - unsigned int thisID = _nextOverlayID; - _nextOverlayID++; + OverlayID thisID = _nextOverlayID; + ++_nextOverlayID; _panels[thisID] = panel; return thisID; } -unsigned int Overlays::addPanel(const QVariant& properties) { +OverlayID Overlays::addPanel(const QVariant& properties) { OverlayPanel::Pointer panel = std::make_shared(); panel->init(_scriptEngine); panel->setProperties(properties.toMap()); return addPanel(panel); } -void Overlays::editPanel(unsigned int panelId, const QVariant& properties) { +void Overlays::editPanel(OverlayID panelId, const QVariant& properties) { if (_panels.contains(panelId)) { _panels[panelId]->setProperties(properties.toMap()); } } -OverlayPropertyResult Overlays::getPanelProperty(unsigned int panelId, const QString& property) { +OverlayPropertyResult Overlays::getPanelProperty(OverlayID panelId, const QString& property) { OverlayPropertyResult result; if (_panels.contains(panelId)) { OverlayPanel::Pointer thisPanel = getPanel(panelId); @@ -581,7 +587,7 @@ OverlayPropertyResult Overlays::getPanelProperty(unsigned int panelId, const QSt } -void Overlays::deletePanel(unsigned int panelId) { +void Overlays::deletePanel(OverlayID panelId) { OverlayPanel::Pointer panelToDelete; { @@ -594,7 +600,7 @@ void Overlays::deletePanel(unsigned int panelId) { } while (!panelToDelete->getChildren().isEmpty()) { - unsigned int childId = panelToDelete->popLastChild(); + OverlayID childId = panelToDelete->popLastChild(); deleteOverlay(childId); deletePanel(childId); } @@ -602,39 +608,39 @@ void Overlays::deletePanel(unsigned int panelId) { emit panelDeleted(panelId); } -bool Overlays::isAddedOverlay(unsigned int id) { +bool Overlays::isAddedOverlay(OverlayID id) { return _overlaysHUD.contains(id) || _overlaysWorld.contains(id); } -void Overlays::sendMousePressOnOverlay(unsigned int overlayID, const PointerEvent& event) { +void Overlays::sendMousePressOnOverlay(OverlayID overlayID, const PointerEvent& event) { emit mousePressOnOverlay(overlayID, event); } -void Overlays::sendMouseReleaseOnOverlay(unsigned int overlayID, const PointerEvent& event) { +void Overlays::sendMouseReleaseOnOverlay(OverlayID overlayID, const PointerEvent& event) { emit mouseReleaseOnOverlay(overlayID, event); } -void Overlays::sendMouseMoveOnOverlay(unsigned int overlayID, const PointerEvent& event) { +void Overlays::sendMouseMoveOnOverlay(OverlayID overlayID, const PointerEvent& event) { emit mouseMoveOnOverlay(overlayID, event); } -void Overlays::sendHoverEnterOverlay(unsigned int id, PointerEvent event) { +void Overlays::sendHoverEnterOverlay(OverlayID id, PointerEvent event) { emit hoverEnterOverlay(id, event); } -void Overlays::sendHoverOverOverlay(unsigned int id, PointerEvent event) { +void Overlays::sendHoverOverOverlay(OverlayID id, PointerEvent event) { emit hoverOverOverlay(id, event); } -void Overlays::sendHoverLeaveOverlay(unsigned int id, PointerEvent event) { +void Overlays::sendHoverLeaveOverlay(OverlayID id, PointerEvent event) { emit hoverLeaveOverlay(id, event); } -unsigned int Overlays::getKeyboardFocusOverlay() const { +OverlayID Overlays::getKeyboardFocusOverlay() const { return qApp->getKeyboardFocusOverlay(); } -void Overlays::setKeyboardFocusOverlay(unsigned int id) { +void Overlays::setKeyboardFocusOverlay(OverlayID id) { qApp->setKeyboardFocusOverlay(id); } diff --git a/interface/src/ui/overlays/Overlays.h b/interface/src/ui/overlays/Overlays.h index 90644206ee..5c784739cb 100644 --- a/interface/src/ui/overlays/Overlays.h +++ b/interface/src/ui/overlays/Overlays.h @@ -53,7 +53,7 @@ class RayToOverlayIntersectionResult { public: RayToOverlayIntersectionResult(); bool intersects; - unsigned int overlayID; + OverlayID overlayID; float distance; BoxFace face; glm::vec3 surfaceNormal; @@ -77,12 +77,12 @@ void RayToOverlayIntersectionResultFromScriptValue(const QScriptValue& object, R * @namespace Overlays */ -const unsigned int UNKNOWN_OVERLAY_ID = 0; +const OverlayID UNKNOWN_OVERLAY_ID = 0; class Overlays : public QObject { Q_OBJECT - Q_PROPERTY(unsigned int keyboardFocusOverlay READ getKeyboardFocusOverlay WRITE setKeyboardFocusOverlay) + Q_PROPERTY(OverlayID keyboardFocusOverlay READ getKeyboardFocusOverlay WRITE setKeyboardFocusOverlay) public: Overlays(); @@ -93,12 +93,12 @@ public: void disable(); void enable(); - Overlay::Pointer getOverlay(unsigned int id) const; - OverlayPanel::Pointer getPanel(unsigned int id) const { return _panels[id]; } + Overlay::Pointer getOverlay(OverlayID id) const; + OverlayPanel::Pointer getPanel(OverlayID id) const { return _panels[id]; } /// adds an overlay that's already been created - unsigned int addOverlay(Overlay* overlay) { return addOverlay(Overlay::Pointer(overlay)); } - unsigned int addOverlay(Overlay::Pointer overlay); + OverlayID addOverlay(Overlay* overlay) { return addOverlay(Overlay::Pointer(overlay)); } + OverlayID addOverlay(Overlay::Pointer overlay); void mousePressEvent(QMouseEvent* event); void mouseReleaseEvent(QMouseEvent* event); @@ -116,7 +116,7 @@ public slots: * @param {Overlays.OverlayProperties} The properties of the overlay that you want to add. * @return {Overlays.OverlayID} The ID of the newly created overlay. */ - unsigned int addOverlay(const QString& type, const QVariant& properties); + OverlayID addOverlay(const QString& type, const QVariant& properties); /**jsdoc * Create a clone of an existing overlay. @@ -125,7 +125,7 @@ public slots: * @param {Overlays.OverlayID} overlayID The ID of the overlay to clone. * @return {Overlays.OverlayID} The ID of the new overlay. */ - unsigned int cloneOverlay(unsigned int id); + OverlayID cloneOverlay(OverlayID id); /**jsdoc * Edit an overlay's properties. @@ -134,7 +134,7 @@ public slots: * @param {Overlays.OverlayID} overlayID The ID of the overlay to edit. * @return {bool} `true` if the overlay was found and edited, otherwise false. */ - bool editOverlay(unsigned int id, const QVariant& properties); + bool editOverlay(OverlayID id, const QVariant& properties); /// edits an overlay updating only the included properties, will return the identified OverlayID in case of /// successful edit, if the input id is for an unknown overlay this function will have no effect @@ -146,7 +146,7 @@ public slots: * @function Overlays.deleteOverlay * @param {Overlays.OverlayID} overlayID The ID of the overlay to delete. */ - void deleteOverlay(unsigned int id); + void deleteOverlay(OverlayID id); /**jsdoc * Get the type of an overlay. @@ -155,7 +155,7 @@ public slots: * @param {Overlays.OverlayID} overlayID The ID of the overlay to get the type of. * @return {string} The type of the overlay if found, otherwise the empty string. */ - QString getOverlayType(unsigned int overlayId) const; + QString getOverlayType(OverlayID overlayId) const; /**jsdoc * Get the overlay Script object. @@ -164,7 +164,7 @@ public slots: * @param {Overlays.OverlayID} overlayID The ID of the overlay to get the script object of. * @return {Object} The script object for the overlay if found. */ - QObject* getOverlayObject(unsigned int id); + QObject* getOverlayObject(OverlayID id); /**jsdoc * Get the ID of the overlay at a particular point on the HUD/screen. @@ -174,7 +174,7 @@ public slots: * @return {Overlays.OverlayID} The ID of the overlay at the point specified. * If no overlay is found, `0` will be returned. */ - unsigned int getOverlayAtPoint(const glm::vec2& point); + OverlayID getOverlayAtPoint(const glm::vec2& point); /**jsdoc * Get the value of a an overlay's property. @@ -185,16 +185,26 @@ public slots: * @return {Object} The value of the property. If the overlay or the property could * not be found, `undefined` will be returned. */ - OverlayPropertyResult getProperty(unsigned int id, const QString& property); + OverlayPropertyResult getProperty(OverlayID id, const QString& property); /*jsdoc * Find the closest 3D overlay hit by a pick ray. * * @function Overlays.findRayIntersection * @param {PickRay} The PickRay to use for finding overlays. + * @param {bool} Unused; Exists to match Entity interface + * @param {List of Overlays.OverlayID} Whitelist for intersection test. + * @param {List of Overlays.OverlayID} Blacklist for intersection test. + * @param {bool} Unused; Exists to match Entity interface + * @param {bool} Unused; Exists to match Entity interface * @return {Overlays.RayToOverlayIntersectionResult} The result of the ray cast. */ - RayToOverlayIntersectionResult findRayIntersection(const PickRay& ray); + RayToOverlayIntersectionResult findRayIntersection(const PickRay& ray, + bool precisionPicking = false, + const QScriptValue& overlayIDsToInclude = QScriptValue(), + const QScriptValue& overlayIDsToDiscard = QScriptValue(), + bool visibleOnly = false, + bool collidableOnly = false); /**jsdoc * Check whether an overlay's assets have been loaded. For example, if the @@ -204,7 +214,7 @@ public slots: * @param {Overlays.OverlayID} The ID of the overlay to check. * @return {bool} `true` if the overlay's assets have been loaded, otherwise `false`. */ - bool isLoaded(unsigned int id); + bool isLoaded(OverlayID id); /**jsdoc * Calculates the size of the given text in the specified overlay if it is a text overlay. @@ -216,7 +226,7 @@ public slots: * @param {string} The string to measure. * @return {Vec2} The size of the text. */ - QSizeF textSize(unsigned int id, const QString& text) const; + QSizeF textSize(OverlayID id, const QString& text) const; /**jsdoc * Get the width of the virtual 2D HUD. @@ -235,39 +245,39 @@ public slots: float height() const; /// return true if there is an overlay with that id else false - bool isAddedOverlay(unsigned int id); + bool isAddedOverlay(OverlayID id); - unsigned int getParentPanel(unsigned int childId) const; - void setParentPanel(unsigned int childId, unsigned int panelId); + OverlayID getParentPanel(OverlayID childId) const; + void setParentPanel(OverlayID childId, OverlayID panelId); /// adds a panel that has already been created - unsigned int addPanel(OverlayPanel::Pointer panel); + OverlayID addPanel(OverlayPanel::Pointer panel); /// creates and adds a panel based on a set of properties - unsigned int addPanel(const QVariant& properties); + OverlayID addPanel(const QVariant& properties); /// edit the properties of a panel - void editPanel(unsigned int panelId, const QVariant& properties); + void editPanel(OverlayID panelId, const QVariant& properties); /// get a property of a panel - OverlayPropertyResult getPanelProperty(unsigned int panelId, const QString& property); + OverlayPropertyResult getPanelProperty(OverlayID panelId, const QString& property); /// deletes a panel and all child overlays - void deletePanel(unsigned int panelId); + void deletePanel(OverlayID panelId); /// return true if there is a panel with that id else false - bool isAddedPanel(unsigned int id) { return _panels.contains(id); } + bool isAddedPanel(OverlayID id) { return _panels.contains(id); } - void sendMousePressOnOverlay(unsigned int overlayID, const PointerEvent& event); - void sendMouseReleaseOnOverlay(unsigned int overlayID, const PointerEvent& event); - void sendMouseMoveOnOverlay(unsigned int overlayID, const PointerEvent& event); + void sendMousePressOnOverlay(OverlayID overlayID, const PointerEvent& event); + void sendMouseReleaseOnOverlay(OverlayID overlayID, const PointerEvent& event); + void sendMouseMoveOnOverlay(OverlayID overlayID, const PointerEvent& event); - void sendHoverEnterOverlay(unsigned int id, PointerEvent event); - void sendHoverOverOverlay(unsigned int id, PointerEvent event); - void sendHoverLeaveOverlay(unsigned int id, PointerEvent event); + void sendHoverEnterOverlay(OverlayID id, PointerEvent event); + void sendHoverOverOverlay(OverlayID id, PointerEvent event); + void sendHoverLeaveOverlay(OverlayID id, PointerEvent event); - unsigned int getKeyboardFocusOverlay() const; - void setKeyboardFocusOverlay(unsigned int id); + OverlayID getKeyboardFocusOverlay() const; + void setKeyboardFocusOverlay(OverlayID id); signals: /**jsdoc @@ -276,26 +286,26 @@ signals: * @function Overlays.overlayDeleted * @param {OverlayID} The ID of the overlay that was deleted. */ - void overlayDeleted(unsigned int id); - void panelDeleted(unsigned int id); + void overlayDeleted(OverlayID id); + void panelDeleted(OverlayID id); - void mousePressOnOverlay(unsigned int overlayID, const PointerEvent& event); - void mouseReleaseOnOverlay(unsigned int overlayID, const PointerEvent& event); - void mouseMoveOnOverlay(unsigned int overlayID, const PointerEvent& event); + void mousePressOnOverlay(OverlayID overlayID, const PointerEvent& event); + void mouseReleaseOnOverlay(OverlayID overlayID, const PointerEvent& event); + void mouseMoveOnOverlay(OverlayID overlayID, const PointerEvent& event); void mousePressOffOverlay(); - void hoverEnterOverlay(unsigned int overlayID, const PointerEvent& event); - void hoverOverOverlay(unsigned int overlayID, const PointerEvent& event); - void hoverLeaveOverlay(unsigned int overlayID, const PointerEvent& event); + void hoverEnterOverlay(OverlayID overlayID, const PointerEvent& event); + void hoverOverOverlay(OverlayID overlayID, const PointerEvent& event); + void hoverLeaveOverlay(OverlayID overlayID, const PointerEvent& event); private: void cleanupOverlaysToDelete(); - QMap _overlaysHUD; - QMap _overlaysWorld; - QMap _panels; + QMap _overlaysHUD; + QMap _overlaysWorld; + QMap _panels; QList _overlaysToDelete; - unsigned int _nextOverlayID; + OverlayID _nextOverlayID; QReadWriteLock _lock; QReadWriteLock _deleteLock; @@ -305,8 +315,8 @@ private: PointerEvent calculatePointerEvent(Overlay::Pointer overlay, PickRay ray, RayToOverlayIntersectionResult rayPickResult, QMouseEvent* event, PointerEvent::EventType eventType); - unsigned int _currentClickingOnOverlayID { UNKNOWN_OVERLAY_ID }; - unsigned int _currentHoverOverOverlayID { UNKNOWN_OVERLAY_ID }; + OverlayID _currentClickingOnOverlayID { UNKNOWN_OVERLAY_ID }; + OverlayID _currentHoverOverOverlayID { UNKNOWN_OVERLAY_ID }; }; #endif // hifi_Overlays_h diff --git a/interface/src/ui/overlays/Web3DOverlay.cpp b/interface/src/ui/overlays/Web3DOverlay.cpp index cb649e8766..bfc37ccf60 100644 --- a/interface/src/ui/overlays/Web3DOverlay.cpp +++ b/interface/src/ui/overlays/Web3DOverlay.cpp @@ -198,7 +198,7 @@ void Web3DOverlay::render(RenderArgs* args) { _webSurface->getRootItem()->setProperty("scriptURL", _scriptURL); currentContext->makeCurrent(currentSurface); - auto forwardPointerEvent = [=](unsigned int overlayID, const PointerEvent& event) { + auto forwardPointerEvent = [=](OverlayID overlayID, const PointerEvent& event) { if (overlayID == getOverlayID()) { handlePointerEvent(event); } @@ -208,7 +208,7 @@ void Web3DOverlay::render(RenderArgs* args) { _mouseReleaseConnection = connect(&(qApp->getOverlays()), &Overlays::mouseReleaseOnOverlay, forwardPointerEvent); _mouseMoveConnection = connect(&(qApp->getOverlays()), &Overlays::mouseMoveOnOverlay, forwardPointerEvent); _hoverLeaveConnection = connect(&(qApp->getOverlays()), &Overlays::hoverLeaveOverlay, - [=](unsigned int overlayID, const PointerEvent& event) { + [=](OverlayID overlayID, const PointerEvent& event) { if (this->_pressed && this->getOverlayID() == overlayID) { // If the user mouses off the overlay while the button is down, simulate a touch end. QTouchEvent::TouchPoint point; From 05c135b9d8d86df89511ecb596b1ccbc189c7496 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Thu, 16 Feb 2017 21:22:23 -0800 Subject: [PATCH 053/104] uuid keys for overlays appears to work --- interface/src/Application.cpp | 10 ++--- interface/src/devices/DdeFaceTracker.cpp | 2 +- interface/src/ui/overlays/Base3DOverlay.h | 3 ++ interface/src/ui/overlays/Overlay.cpp | 10 ++--- interface/src/ui/overlays/Overlay.h | 32 ++++++---------- interface/src/ui/overlays/OverlayPanel.cpp | 2 +- interface/src/ui/overlays/Overlays.cpp | 43 ++++++++-------------- interface/src/ui/overlays/Overlays.h | 5 +-- 8 files changed, 43 insertions(+), 64 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index a00cfb37d2..5f26fda674 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -529,7 +529,7 @@ bool setupEssentials(int& argc, char** argv) { // to take care of highlighting keyboard focused items, rather than // continuing to overburden Application.cpp std::shared_ptr _keyboardFocusHighlight{ nullptr }; -OverlayID _keyboardFocusHighlightID{ -1 }; +OverlayID _keyboardFocusHighlightID{ UNKNOWN_OVERLAY_ID }; // FIXME hack access to the internal share context for the Chromium helper @@ -1683,9 +1683,9 @@ void Application::cleanupBeforeQuit() { _applicationStateDevice.reset(); { - if (_keyboardFocusHighlightID) { + if (_keyboardFocusHighlightID != UNKNOWN_OVERLAY_ID) { getOverlays().deleteOverlay(_keyboardFocusHighlightID); - _keyboardFocusHighlightID = UNKNOWN_OVERLAY_ID; + _keyboardFocusHighlightID = UNKNOWN_OVERLAY_ID; } _keyboardFocusHighlight = nullptr; } @@ -3072,7 +3072,7 @@ void Application::mouseMoveEvent(QMouseEvent* event) { buttons, event->modifiers()); if (compositor.getReticleVisible() || !isHMDMode() || !compositor.getReticleOverDesktop() || - getOverlays().getOverlayAtPoint(glm::vec2(transformedPos.x(), transformedPos.y()))) { + getOverlays().getOverlayAtPoint(glm::vec2(transformedPos.x(), transformedPos.y())) != UNKNOWN_OVERLAY_ID) { getOverlays().mouseMoveEvent(&mappedEvent); getEntities()->mouseMoveEvent(&mappedEvent); } @@ -4091,7 +4091,7 @@ void Application::rotationModeChanged() const { void Application::setKeyboardFocusHighlight(const glm::vec3& position, const glm::quat& rotation, const glm::vec3& dimensions) { // Create focus - if (_keyboardFocusHighlightID == UNKNOWN_OVERLAY_ID || !getOverlays().isAddedOverlay(_keyboardFocusHighlightID)) { + if (_keyboardFocusHighlightID == UNKNOWN_OVERLAY_ID || !getOverlays().isAddedOverlay(_keyboardFocusHighlightID)) { _keyboardFocusHighlight = std::make_shared(); _keyboardFocusHighlight->setAlpha(1.0f); _keyboardFocusHighlight->setBorderSize(1.0f); diff --git a/interface/src/devices/DdeFaceTracker.cpp b/interface/src/devices/DdeFaceTracker.cpp index 2ddd8d9d04..fa7b2c173e 100644 --- a/interface/src/devices/DdeFaceTracker.cpp +++ b/interface/src/devices/DdeFaceTracker.cpp @@ -193,7 +193,7 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui _calibrationCount(0), _calibrationValues(), _calibrationBillboard(NULL), - _calibrationBillboardID(0), + _calibrationBillboardID(UNKNOWN_OVERLAY_ID), _calibrationMessage(QString()), _isCalibrated(false) { diff --git a/interface/src/ui/overlays/Base3DOverlay.h b/interface/src/ui/overlays/Base3DOverlay.h index 18936df504..7906b9d6c0 100644 --- a/interface/src/ui/overlays/Base3DOverlay.h +++ b/interface/src/ui/overlays/Base3DOverlay.h @@ -23,6 +23,9 @@ public: Base3DOverlay(); Base3DOverlay(const Base3DOverlay* base3DOverlay); + virtual OverlayID getOverlayID() const override { return OverlayID(getID().toString()); } + void setOverlayID(OverlayID overlayID) override { setID(overlayID); } + // getters virtual bool is3D() const override { return true; } diff --git a/interface/src/ui/overlays/Overlay.cpp b/interface/src/ui/overlays/Overlay.cpp index 8ce4b7e97a..0ad2c94241 100644 --- a/interface/src/ui/overlays/Overlay.cpp +++ b/interface/src/ui/overlays/Overlay.cpp @@ -189,7 +189,7 @@ float Overlay::updatePulse() { _pulseDirection *= -1.0f; } _pulse += pulseDelta; - + return _pulse; } @@ -205,11 +205,11 @@ void Overlay::removeFromScene(Overlay::Pointer overlay, std::shared_ptr qVectorOverlayIDFromScriptValue(const QScriptValue& array) { @@ -220,9 +220,7 @@ QVector qVectorOverlayIDFromScriptValue(const QScriptValue& array) { int length = array.property("length").toInteger(); newVector.reserve(length); for (int i = 0; i < length; i++) { - OverlayID id; - OverlayIDfromScriptValue(array.property(i), id); - newVector << id; + newVector << OverlayID(array.property(i).toString()); } return newVector; } diff --git a/interface/src/ui/overlays/Overlay.h b/interface/src/ui/overlays/Overlay.h index d55b72a189..320cd532c4 100644 --- a/interface/src/ui/overlays/Overlay.h +++ b/interface/src/ui/overlays/Overlay.h @@ -15,22 +15,11 @@ #include // for xColor #include -class OverlayID { +class OverlayID : public QUuid { public: - OverlayID() {} - OverlayID(int value) { id = value; } - - OverlayID& operator=(const OverlayID& other) { id = other.id; return *this; } - OverlayID& operator++() { id++; return *this; } - - // OverlayID& operator=(unsigned int value) { id = value; return *this; } - bool operator==(const OverlayID& other) const { return id == other.id; } - bool operator!=(const OverlayID& other) const { return id != other.id; } - // bool operator<(const OverlayID& other) const { return id < other.id; } - // bool operator>(const OverlayID& other) const { return id > other.id; } - operator bool() const { return id != 0; } - - unsigned int id; + OverlayID() : QUuid() {} + OverlayID(QString v) : QUuid(v) {} + OverlayID(QUuid v) : QUuid(v) {} }; class Overlay : public QObject { @@ -50,8 +39,8 @@ public: Overlay(const Overlay* overlay); ~Overlay(); - OverlayID getOverlayID() { return _overlayID; } - void setOverlayID(OverlayID overlayID) { _overlayID = overlayID; } + virtual OverlayID getOverlayID() const { return _overlayID; } + virtual void setOverlayID(OverlayID overlayID) { _overlayID = overlayID; } virtual void update(float deltatime) {} virtual void render(RenderArgs* args) = 0; @@ -107,8 +96,6 @@ protected: render::ItemID _renderItemID{ render::Item::INVALID_ITEM_ID }; - OverlayID _overlayID; // what Overlays.cpp knows this instance as - bool _isLoaded; float _alpha; @@ -125,10 +112,13 @@ protected: xColor _color; bool _visible; // should the overlay be drawn at all Anchor _anchor; + +private: + OverlayID _overlayID; // only used for non-3d overlays }; namespace render { - template <> const ItemKey payloadGetKey(const Overlay::Pointer& overlay); + template <> const ItemKey payloadGetKey(const Overlay::Pointer& overlay); template <> const Item::Bound payloadGetBound(const Overlay::Pointer& overlay); template <> int payloadGetLayer(const Overlay::Pointer& overlay); template <> void payloadRender(const Overlay::Pointer& overlay, RenderArgs* args); @@ -141,5 +131,5 @@ QScriptValue OverlayIDtoScriptValue(QScriptEngine* engine, const OverlayID& id); void OverlayIDfromScriptValue(const QScriptValue &object, OverlayID& id); QVector qVectorOverlayIDFromScriptValue(const QScriptValue& array); - + #endif // hifi_Overlay_h diff --git a/interface/src/ui/overlays/OverlayPanel.cpp b/interface/src/ui/overlays/OverlayPanel.cpp index 484e1da216..df2b91c4ef 100644 --- a/interface/src/ui/overlays/OverlayPanel.cpp +++ b/interface/src/ui/overlays/OverlayPanel.cpp @@ -89,7 +89,7 @@ QVariant OverlayPanel::getProperty(const QString &property) { if (property == "children") { QVariantList array; for (int i = 0; i < _children.length(); i++) { - array.append(OverlayIDtoScriptValue(nullptr, _children[i])); + array.append(OverlayIDtoScriptValue(nullptr, _children[i]).toVariant()); } return array; } diff --git a/interface/src/ui/overlays/Overlays.cpp b/interface/src/ui/overlays/Overlays.cpp index 42e6d2c679..98590e2166 100644 --- a/interface/src/ui/overlays/Overlays.cpp +++ b/interface/src/ui/overlays/Overlays.cpp @@ -39,9 +39,6 @@ Q_LOGGING_CATEGORY(trace_render_overlays, "trace.render.overlays") -Overlays::Overlays() : - _nextOverlayID(1) {} - void Overlays::cleanupAllOverlays() { { QWriteLocker lock(&_lock); @@ -188,14 +185,13 @@ OverlayID Overlays::addOverlay(const QString& type, const QVariant& properties) thisOverlay->setProperties(properties.toMap()); return addOverlay(thisOverlay); } - return 0; + return UNKNOWN_OVERLAY_ID; } OverlayID Overlays::addOverlay(Overlay::Pointer overlay) { QWriteLocker lock(&_lock); - OverlayID thisID = _nextOverlayID; + OverlayID thisID = OverlayID(QUuid::createUuid()); overlay->setOverlayID(thisID); - ++_nextOverlayID; if (overlay->is3D()) { _overlaysWorld[thisID] = overlay; @@ -220,9 +216,9 @@ OverlayID Overlays::cloneOverlay(OverlayID id) { attachable->getParentPanel()->addChild(cloneId); } return cloneId; - } - - return 0; // Not found + } + + return UNKNOWN_OVERLAY_ID; // Not found } bool Overlays::editOverlay(OverlayID id, const QVariant& properties) { @@ -242,13 +238,7 @@ bool Overlays::editOverlays(const QVariant& propertiesById) { bool success = true; QWriteLocker lock(&_lock); for (const auto& key : map.keys()) { - bool convertSuccess; - OverlayID id = key.toUInt(&convertSuccess); - if (!convertSuccess) { - success = false; - continue; - } - + OverlayID id = OverlayID(key); Overlay::Pointer thisOverlay = getOverlay(id); if (!thisOverlay) { success = false; @@ -310,7 +300,7 @@ OverlayID Overlays::getParentPanel(OverlayID childId) const { } else if (_panels.contains(childId)) { return _panels.key(getPanel(childId)->getParentPanel()); } - return 0; + return UNKNOWN_OVERLAY_ID; } void Overlays::setParentPanel(OverlayID childId, OverlayID panelId) { @@ -347,7 +337,7 @@ OverlayID Overlays::getOverlayAtPoint(const glm::vec2& point) { glm::vec2 pointCopy = point; QReadLocker lock(&_lock); if (!_enabled) { - return 0; + return UNKNOWN_OVERLAY_ID; } QMapIterator i(_overlaysHUD); i.toBack(); @@ -378,7 +368,7 @@ OverlayID Overlays::getOverlayAtPoint(const glm::vec2& point) { } } - return 0; // not found + return UNKNOWN_OVERLAY_ID; // not found } OverlayPropertyResult Overlays::getProperty(OverlayID id, const QString& property) { @@ -447,14 +437,14 @@ RayToOverlayIntersectionResult Overlays::findRayIntersection(const PickRay& ray, return result; } -RayToOverlayIntersectionResult::RayToOverlayIntersectionResult() : - intersects(false), - overlayID(-1), +RayToOverlayIntersectionResult::RayToOverlayIntersectionResult() : + intersects(false), + overlayID(UNKNOWN_OVERLAY_ID), distance(0), face(), intersection(), extraInfo() -{ +{ } QScriptValue RayToOverlayIntersectionResultToScriptValue(QScriptEngine* engine, const RayToOverlayIntersectionResult& value) { @@ -463,7 +453,7 @@ QScriptValue RayToOverlayIntersectionResultToScriptValue(QScriptEngine* engine, obj.setProperty("overlayID", OverlayIDtoScriptValue(engine, value.overlayID)); obj.setProperty("distance", value.distance); - QString faceName = ""; + QString faceName = ""; // handle BoxFace switch (value.face) { case MIN_X_FACE: @@ -499,7 +489,7 @@ QScriptValue RayToOverlayIntersectionResultToScriptValue(QScriptEngine* engine, void RayToOverlayIntersectionResultFromScriptValue(const QScriptValue& objectVar, RayToOverlayIntersectionResult& value) { QVariantMap object = objectVar.toVariant().toMap(); value.intersects = object["intersects"].toBool(); - value.overlayID = object["overlayID"].toInt(); + value.overlayID = OverlayID(QUuid(object["overlayID"].toString())); value.distance = object["distance"].toFloat(); QString faceName = object["face"].toString(); @@ -556,8 +546,7 @@ QSizeF Overlays::textSize(OverlayID id, const QString& text) const { OverlayID Overlays::addPanel(OverlayPanel::Pointer panel) { QWriteLocker lock(&_lock); - OverlayID thisID = _nextOverlayID; - ++_nextOverlayID; + OverlayID thisID = QUuid::createUuid(); _panels[thisID] = panel; return thisID; diff --git a/interface/src/ui/overlays/Overlays.h b/interface/src/ui/overlays/Overlays.h index 5c784739cb..4a4d6f9466 100644 --- a/interface/src/ui/overlays/Overlays.h +++ b/interface/src/ui/overlays/Overlays.h @@ -77,7 +77,7 @@ void RayToOverlayIntersectionResultFromScriptValue(const QScriptValue& object, R * @namespace Overlays */ -const OverlayID UNKNOWN_OVERLAY_ID = 0; +const OverlayID UNKNOWN_OVERLAY_ID = OverlayID(); class Overlays : public QObject { Q_OBJECT @@ -85,7 +85,7 @@ class Overlays : public QObject { Q_PROPERTY(OverlayID keyboardFocusOverlay READ getKeyboardFocusOverlay WRITE setKeyboardFocusOverlay) public: - Overlays(); + Overlays() {}; void init(); void update(float deltatime); @@ -305,7 +305,6 @@ private: QMap _overlaysWorld; QMap _panels; QList _overlaysToDelete; - OverlayID _nextOverlayID; QReadWriteLock _lock; QReadWriteLock _deleteLock; From 78147c27ad45186765bcbfcb606a1c13debaa6b0 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 17 Feb 2017 08:33:27 -0800 Subject: [PATCH 054/104] log too large byteArray instead of crashing --- assignment-client/src/avatars/AvatarMixer.cpp | 9 +++++---- assignment-client/src/avatars/AvatarMixerSlave.cpp | 6 +++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 3587b675b1..db158c861c 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -125,11 +125,12 @@ void AvatarMixer::start() { // // DONE --- 1) only sleep for remainder // DONE --- 2) clean up stats, add slave stats - // 3) delete dead code from mixer (now that it's in slave) - // 4) audit the locking and side-effects to node, otherNode, and nodeData + // 3a) out of view??? is it broken? + // 3) Error in PacketList::writeData - attempted to write a segment to an unordered packet that is larger than the payload size. + // 4) fix two different versions of toByteArray() // 5) throttling?? - // 6) Error in PacketList::writeData - attempted to write a segment to an unordered packet that is larger than the payload size. - // 7) fix two different versions of toByteArray() + // 6) audit the locking and side-effects to node, otherNode, and nodeData + // 7) delete dead code from mixer (now that it's in slave) ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index af76b52547..bff34d9751 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -367,7 +367,11 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { quint64 end = usecTimestampNow(); _stats.toByteArrayElapsedTime += (end - start); - numAvatarDataBytes += avatarPacketList->write(bytes); + if (bytes.size() > 1400) { + qDebug() << "WARNING: otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size(); + } else { + numAvatarDataBytes += avatarPacketList->write(bytes); + } } avatarPacketList->endSegment(); From 81f63440236169d882002fcc74c10023cec56e16 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Fri, 17 Feb 2017 09:36:00 -0800 Subject: [PATCH 055/104] allow overlays to be parents --- interface/src/InterfaceParentFinder.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/interface/src/InterfaceParentFinder.cpp b/interface/src/InterfaceParentFinder.cpp index 824e81b6d8..14088bc716 100644 --- a/interface/src/InterfaceParentFinder.cpp +++ b/interface/src/InterfaceParentFinder.cpp @@ -45,12 +45,20 @@ SpatiallyNestableWeakPointer InterfaceParentFinder::find(QUuid parentID, bool& s success = true; return parent; } - if (parentID == AVATAR_SELF_ID) { success = true; return avatarManager->getMyAvatar(); } + // search overlays + auto& overlays = qApp->getOverlays(); + auto overlay = overlays.getOverlay(parentID); + parent = std::dynamic_pointer_cast(overlay); // this will return nullptr for non-3d overlays + if (!parent.expired()) { + success = true; + return parent; + } + success = false; return parent; } From 2b02998fae4cb9ecfb05128c5a4962698d9b05dd Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Fri, 17 Feb 2017 09:47:40 -0800 Subject: [PATCH 056/104] hook up overlaysToDiscard and overlaysToInclude in Overlays::findRayIntersection --- interface/src/ui/overlays/Overlays.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/interface/src/ui/overlays/Overlays.cpp b/interface/src/ui/overlays/Overlays.cpp index 98590e2166..6b120cab56 100644 --- a/interface/src/ui/overlays/Overlays.cpp +++ b/interface/src/ui/overlays/Overlays.cpp @@ -412,13 +412,19 @@ RayToOverlayIntersectionResult Overlays::findRayIntersection(const PickRay& ray, i.previous(); OverlayID thisID = i.key(); auto thisOverlay = std::dynamic_pointer_cast(i.value()); + + if ((overlaysToDiscard.size() > 0 && overlaysToDiscard.contains(thisID)) || + (overlaysToInclude.size() > 0 && !overlaysToInclude.contains(thisID))) { + continue; + } + if (thisOverlay && thisOverlay->getVisible() && !thisOverlay->getIgnoreRayIntersection() && thisOverlay->isLoaded()) { float thisDistance; BoxFace thisFace; glm::vec3 thisSurfaceNormal; QString thisExtraInfo; - if (thisOverlay->findRayIntersectionExtraInfo(ray.origin, ray.direction, thisDistance, - thisFace, thisSurfaceNormal, thisExtraInfo)) { + if (thisOverlay->findRayIntersectionExtraInfo(ray.origin, ray.direction, thisDistance, + thisFace, thisSurfaceNormal, thisExtraInfo)) { bool isDrawInFront = thisOverlay->getDrawInFront(); if (thisDistance < bestDistance && (!bestIsFront || isDrawInFront)) { bestIsFront = isDrawInFront; From 92ca7de0bf1064574772ccf1a637658df2422539 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 17 Feb 2017 10:12:35 -0800 Subject: [PATCH 057/104] some tweaks to support too large avatar data --- assignment-client/src/avatars/AvatarMixerSlave.cpp | 5 ++++- libraries/avatars/src/AvatarData.cpp | 6 +++--- libraries/avatars/src/AvatarData.h | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index bff34d9751..ca2db55cdd 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -361,9 +361,12 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); bool distanceAdjust = true; glm::vec3 viewerPosition = myPosition; + AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray + bool dropFaceTracking = true; // this is a hack for now... always drop face tracking quint64 start = usecTimestampNow(); - QByteArray bytes = otherAvatar.toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, distanceAdjust, viewerPosition, &lastSentJointsForOther); + QByteArray bytes = otherAvatar.toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); quint64 end = usecTimestampNow(); _stats.toByteArrayElapsedTime += (end - start); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index cbd5d42263..66d1fb39bf 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -169,7 +169,7 @@ float AvatarData::getDistanceBasedMinTranslationDistance(glm::vec3 viewerPositio QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) { + bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) { // if no timestamp was included, then assume the avatarData is single instance // and is tracking its own last encoding time. @@ -575,7 +575,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) const { + AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) const { bool cullSmallChanges = (dataDetail == CullSmallData); bool sendAll = (dataDetail == SendAllData); @@ -625,7 +625,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent tranlationChangedSince(lastSentTime) || parentInfoChangedSince(lastSentTime)); - bool hasFaceTrackerInfo = hasFaceTracker() && (sendAll || faceTrackerInfoChangedSince(lastSentTime)); + bool hasFaceTrackerInfo = !dropFaceTracking && hasFaceTracker() && (sendAll || faceTrackerInfoChangedSince(lastSentTime)); bool hasJointData = sendAll || !sendMinimum; // Leading flags, to indicate how much data is actually included in the packet... diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 45e62d4045..64c0e11577 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -356,7 +356,7 @@ public: // FIXME virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) const; + AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) const; virtual void doneEncoding(bool cullSmallChanges); From 47cdade1e9a3ba3d123678a46c3a9552beb386f1 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Fri, 17 Feb 2017 11:47:37 -0800 Subject: [PATCH 058/104] fix home-button on tablet --- interface/src/scripting/HMDScriptingInterface.h | 8 ++++---- scripts/system/libraries/WebTablet.js | 7 +------ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/interface/src/scripting/HMDScriptingInterface.h b/interface/src/scripting/HMDScriptingInterface.h index f5744bb8d1..643fac02fc 100644 --- a/interface/src/scripting/HMDScriptingInterface.h +++ b/interface/src/scripting/HMDScriptingInterface.h @@ -30,7 +30,7 @@ class HMDScriptingInterface : public AbstractHMDScriptingInterface, public Depen Q_PROPERTY(bool mounted READ isMounted) Q_PROPERTY(bool showTablet READ getShouldShowTablet) Q_PROPERTY(QUuid tabletID READ getCurrentTableUIID WRITE setCurrentTabletUIID) - Q_PROPERTY(unsigned int homeButtonID READ getCurrentHomeButtonUUID WRITE setCurrentHomeButtonUUID) + Q_PROPERTY(QUuid homeButtonID READ getCurrentHomeButtonUUID WRITE setCurrentHomeButtonUUID) public: @@ -93,13 +93,13 @@ public: void setCurrentTabletUIID(QUuid tabletID) { _tabletUIID = tabletID; } QUuid getCurrentTableUIID() const { return _tabletUIID; } - void setCurrentHomeButtonUUID(unsigned int homeButtonID) { _homeButtonID = homeButtonID; } - unsigned int getCurrentHomeButtonUUID() const { return _homeButtonID; } + void setCurrentHomeButtonUUID(QUuid homeButtonID) { _homeButtonID = homeButtonID; } + QUuid getCurrentHomeButtonUUID() const { return _homeButtonID; } private: bool _showTablet { false }; QUuid _tabletUIID; // this is the entityID of the WebEntity which is part of (a child of) the tablet-ui. - unsigned int _homeButtonID; + QUuid _homeButtonID; QUuid _tabletEntityID; // Get the position of the HMD diff --git a/scripts/system/libraries/WebTablet.js b/scripts/system/libraries/WebTablet.js index 74bbd788be..0990440801 100644 --- a/scripts/system/libraries/WebTablet.js +++ b/scripts/system/libraries/WebTablet.js @@ -159,7 +159,7 @@ WebTablet = function (url, width, dpi, hand, clientOnly) { }); this.receive = function (channel, senderID, senderUUID, localOnly) { - if (_this.homeButtonEntity === parseInt(senderID)) { + if (_this.homeButtonEntity == senderID) { var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system"); var onHomeScreen = tablet.onHomeScreen(); if (onHomeScreen) { @@ -219,7 +219,6 @@ WebTablet = function (url, width, dpi, hand, clientOnly) { }; WebTablet.prototype.setHomeButtonTexture = function() { - print(this.homeButtonEntity); Entities.editEntity(this.tabletEntityID, {textures: JSON.stringify({"tex.close": HOME_BUTTON_TEXTURE})}); }; @@ -385,14 +384,10 @@ WebTablet.prototype.register = function() { WebTablet.prototype.cleanUpOldTabletsOnJoint = function(jointIndex) { var children = Entities.getChildrenIDsOfJoint(MyAvatar.sessionUUID, jointIndex); children = children.concat(Entities.getChildrenIDsOfJoint(AVATAR_SELF_ID, jointIndex)); - print("cleanup " + children); children.forEach(function(childID) { var props = Entities.getEntityProperties(childID, ["name"]); if (props.name === "WebTablet Tablet") { - print("cleaning up " + props.name); Entities.deleteEntity(childID); - } else { - print("not cleaning up " + props.name); } }); }; From d7bb6f105bcee1df7e21a9f29d7b3905fa26073e Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Fri, 17 Feb 2017 15:43:20 -0800 Subject: [PATCH 059/104] make tablet part IDs available through qApp --- interface/src/Application.cpp | 10 ++++++++++ interface/src/Application.h | 3 +++ interface/src/avatar/MyAvatar.cpp | 2 +- interface/src/scripting/HMDScriptingInterface.h | 12 ++++++++---- interface/src/ui/overlays/Overlay.h | 3 +-- 5 files changed, 23 insertions(+), 7 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 5f26fda674..40431737a2 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -6861,3 +6861,13 @@ void Application::toggleMuteAudio() { auto menu = Menu::getInstance(); menu->setIsOptionChecked(MenuOption::MuteAudio, !menu->isOptionChecked(MenuOption::MuteAudio)); } + +OverlayID Application::getTabletScreenID() { + auto HMD = DependencyManager::get(); + return HMD->getCurrentTabletScreenID(); +} + +OverlayID Application::getTabletHomeButtonID() { + auto HMD = DependencyManager::get(); + return HMD->getCurrentHomeButtonUUID(); +} diff --git a/interface/src/Application.h b/interface/src/Application.h index 7dcb35babc..cc3e603a82 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -293,6 +293,9 @@ public: Q_INVOKABLE void sendHoverOverEntity(QUuid id, PointerEvent event); Q_INVOKABLE void sendHoverLeaveEntity(QUuid id, PointerEvent event); + OverlayID getTabletScreenID(); + OverlayID getTabletHomeButtonID(); + signals: void svoImportRequested(const QString& url); diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 29f41c89fd..e39b7e1e50 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -808,7 +808,7 @@ void MyAvatar::saveData() { auto hmdInterface = DependencyManager::get(); _avatarEntitiesLock.withReadLock([&] { for (auto entityID : _avatarEntityData.keys()) { - if (hmdInterface->getCurrentTableUIID() == entityID) { + if (hmdInterface->getCurrentTabletUIID() == entityID) { // don't persist the tablet between domains / sessions continue; } diff --git a/interface/src/scripting/HMDScriptingInterface.h b/interface/src/scripting/HMDScriptingInterface.h index 643fac02fc..463a21ded8 100644 --- a/interface/src/scripting/HMDScriptingInterface.h +++ b/interface/src/scripting/HMDScriptingInterface.h @@ -29,9 +29,9 @@ class HMDScriptingInterface : public AbstractHMDScriptingInterface, public Depen Q_PROPERTY(glm::quat orientation READ getOrientation) Q_PROPERTY(bool mounted READ isMounted) Q_PROPERTY(bool showTablet READ getShouldShowTablet) - Q_PROPERTY(QUuid tabletID READ getCurrentTableUIID WRITE setCurrentTabletUIID) + Q_PROPERTY(QUuid tabletID READ getCurrentTabletUIID WRITE setCurrentTabletUIID) Q_PROPERTY(QUuid homeButtonID READ getCurrentHomeButtonUUID WRITE setCurrentHomeButtonUUID) - + Q_PROPERTY(QUuid tabletScreenID READ getCurrentTabletScreenID WRITE setCurrentTabletScreenID) public: Q_INVOKABLE glm::vec3 calculateRayUICollisionPoint(const glm::vec3& position, const glm::vec3& direction) const; @@ -91,14 +91,18 @@ public: bool getShouldShowTablet() const { return _showTablet; } void setCurrentTabletUIID(QUuid tabletID) { _tabletUIID = tabletID; } - QUuid getCurrentTableUIID() const { return _tabletUIID; } + QUuid getCurrentTabletUIID() const { return _tabletUIID; } void setCurrentHomeButtonUUID(QUuid homeButtonID) { _homeButtonID = homeButtonID; } QUuid getCurrentHomeButtonUUID() const { return _homeButtonID; } + void setCurrentTabletScreenID(QUuid tabletID) { _tabletScreenID = tabletID; } + QUuid getCurrentTabletScreenID() const { return _tabletScreenID; } + private: bool _showTablet { false }; - QUuid _tabletUIID; // this is the entityID of the WebEntity which is part of (a child of) the tablet-ui. + QUuid _tabletUIID; // this is the entityID of the tablet frame + QUuid _tabletScreenID; // this is the overlayID which is part of (a child of) the tablet-ui. QUuid _homeButtonID; QUuid _tabletEntityID; diff --git a/interface/src/ui/overlays/Overlay.h b/interface/src/ui/overlays/Overlay.h index 320cd532c4..0c96af1a99 100644 --- a/interface/src/ui/overlays/Overlay.h +++ b/interface/src/ui/overlays/Overlay.h @@ -128,8 +128,7 @@ namespace render { Q_DECLARE_METATYPE(OverlayID); Q_DECLARE_METATYPE(QVector); QScriptValue OverlayIDtoScriptValue(QScriptEngine* engine, const OverlayID& id); -void OverlayIDfromScriptValue(const QScriptValue &object, OverlayID& id); +void OverlayIDfromScriptValue(const QScriptValue& object, OverlayID& id); QVector qVectorOverlayIDFromScriptValue(const QScriptValue& array); - #endif // hifi_Overlay_h From 856a38af0cbe516280ef3f701e687421fb9bece8 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Fri, 17 Feb 2017 17:08:23 -0800 Subject: [PATCH 060/104] Fix for audio, toolbar and tablet on macos. --- libraries/audio-client/src/AudioClient.cpp | 6 +++--- libraries/audio-client/src/AudioClient.h | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 2e532d67bf..081d74a5aa 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1120,7 +1120,7 @@ void AudioClient::prepareLocalAudioInjectors() { while (samplesNeeded > 0) { // lock for every write to avoid locking out the device callback // this lock is intentional - the buffer is only lock-free in its use in the device callback - Lock lock(_localAudioMutex); + RecursiveLock lock(_localAudioMutex); samplesNeeded = bufferCapacity - _localSamplesAvailable.load(std::memory_order_relaxed); if (samplesNeeded <= 0) { @@ -1457,7 +1457,7 @@ void AudioClient::outputNotify() { bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo) { bool supportedFormat = false; - Lock lock(_localAudioMutex); + RecursiveLock lock(_localAudioMutex); _localSamplesAvailable.exchange(0, std::memory_order_release); // cleanup any previously initialized device @@ -1671,7 +1671,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { int injectorSamplesPopped = 0; { - Lock lock(_audio->_localAudioMutex); + RecursiveLock lock(_audio->_localAudioMutex); bool append = networkSamplesPopped > 0; samplesRequested = std::min(samplesRequested, _audio->_localSamplesAvailable.load(std::memory_order_acquire)); if ((injectorSamplesPopped = _localInjectorsStream.appendSamples(mixBuffer, samplesRequested, append)) > 0) { diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 699ba71ef7..453444ccdd 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -94,6 +94,8 @@ public: using AudioPositionGetter = std::function; using AudioOrientationGetter = std::function; + using RecursiveMutex = std::recursive_mutex; + using RecursiveLock = std::unique_lock; using Mutex = std::mutex; using Lock = std::unique_lock; @@ -328,7 +330,7 @@ private: int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; float* _localOutputMixBuffer { NULL }; AudioInjectorsThread _localAudioThread; - Mutex _localAudioMutex; + RecursiveMutex _localAudioMutex; // for output audio (used by this thread) int _outputPeriod { 0 }; From 291b823cfa7ff28e82c371fe5dba0abcad5688e7 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 17 Feb 2017 18:28:07 -0800 Subject: [PATCH 061/104] const cleanup and fix crash --- assignment-client/src/Agent.cpp | 2 +- assignment-client/src/avatars/AvatarMixer.cpp | 18 +- .../src/avatars/AvatarMixerClientData.cpp | 1 + .../src/avatars/AvatarMixerClientData.h | 1 + .../src/avatars/AvatarMixerSlave.cpp | 24 +- .../src/avatars/ScriptableAvatar.cpp | 5 +- .../src/avatars/ScriptableAvatar.h | 3 +- interface/src/avatar/MyAvatar.cpp | 7 +- interface/src/avatar/MyAvatar.h | 3 +- libraries/avatars/src/AvatarData.cpp | 499 +++--------------- libraries/avatars/src/AvatarData.h | 40 +- 11 files changed, 117 insertions(+), 486 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 806608cd5f..a458719346 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -546,7 +546,7 @@ void Agent::processAgentAvatar() { auto scriptedAvatar = DependencyManager::get(); AvatarData::AvatarDataDetail dataDetail = (randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO) ? AvatarData::SendAllData : AvatarData::CullSmallData; - QByteArray avatarByteArray = scriptedAvatar->toByteArray(dataDetail, 0, scriptedAvatar->getLastSentJointData()); + QByteArray avatarByteArray = scriptedAvatar->toByteArray(dataDetail); scriptedAvatar->doneEncoding(true); static AvatarDataSequenceNumber sequenceNumber = 0; diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index db158c861c..51f6067e6a 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -125,12 +125,18 @@ void AvatarMixer::start() { // // DONE --- 1) only sleep for remainder // DONE --- 2) clean up stats, add slave stats - // 3a) out of view??? is it broken? - // 3) Error in PacketList::writeData - attempted to write a segment to an unordered packet that is larger than the payload size. - // 4) fix two different versions of toByteArray() - // 5) throttling?? - // 6) audit the locking and side-effects to node, otherNode, and nodeData - // 7) delete dead code from mixer (now that it's in slave) + // DONE --- 3) out of view??? is it broken? - verified - it's working + // 4) Error in PacketList::writeData - attempted to write a segment to an unordered packet that is larger than the payload size. + // DONE --- 4a) hack to not send face data mostly seems to work... + // 4b) some kind of a better approach to handling otherAvatar.toByteArray() for content that is larger than MTU + // 5) fix two different versions of toByteArray() + // 6) throttling?? + // 7) audit the locking and side-effects to node, otherNode, and nodeData + // 8) delete dead code from mixer (now that it's in slave) + // 9) better stats in the nodes: + // how many avatars are actually "in view" for the avtar in question (even if they are over bandwidth budget) + // + // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index b5d4e390bb..6fdd2fd23e 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -58,6 +58,7 @@ int AvatarMixerClientData::parseData(ReceivedMessage& message) { return _avatar->parseDataFromBuffer(message.readWithoutCopy(message.getBytesLeftToRead())); } +// FIXME -- this needs a mutex in new model. bool AvatarMixerClientData::checkAndSetHasReceivedFirstPacketsFrom(const QUuid& uuid) { if (_hasReceivedFirstPacketsFrom.find(uuid) == _hasReceivedFirstPacketsFrom.end()) { _hasReceivedFirstPacketsFrom.insert(uuid); diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index 8cd72050f7..87f7bb9cc9 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -42,6 +42,7 @@ public: int parseData(ReceivedMessage& message) override; AvatarData& getAvatar() { return *_avatar; } + const AvatarData* getConstAvatarData() const { return _avatar.get(); } bool checkAndSetHasReceivedFirstPacketsFrom(const QUuid& uuid); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index ca2db55cdd..2a7f9a4d8d 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -245,17 +245,12 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { ++numOtherAvatars; - AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); - //MutexTryLocker lock(otherNodeData->getMutex()); - - // FIXME -- might want to track this lock failed... - //if (!lock.isLocked()) { - //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << " failed to lock... would BAIL??... line:" << __LINE__; - //return; - //} + const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); // make sure we send out identity packets to and from new arrivals. - bool forceSend = !otherNodeData->checkAndSetHasReceivedFirstPacketsFrom(node->getUUID()); + // FIXME this is where our crash was on friday!! + // this is getting called on multiple threads... needs mutex of better solution + bool forceSend = !nodeData->checkAndSetHasReceivedFirstPacketsFrom(otherNode->getUUID()); if (otherNodeData->getIdentityChangeTimestamp().time_since_epoch().count() > 0 && (forceSend @@ -265,7 +260,7 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // FIXME --- used to be.../ mixer data dependency //sendIdentityPacket(otherNodeData, node); - QByteArray individualData = otherNodeData->getAvatar().identityByteArray(); + QByteArray individualData = otherNodeData->getConstAvatarData()->identityByteArray(); auto identityPacket = NLPacket::create(PacketType::AvatarIdentity, individualData.size()); individualData.replace(0, NUM_BYTES_RFC4122_UUID, otherNodeData->getNodeID().toRfc4122()); identityPacket->write(individualData); @@ -273,12 +268,12 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << " sending itentity packet for otherNode to node..."; } - const AvatarData& otherAvatar = otherNodeData->getAvatar(); + const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); // Decide whether to send this avatar's data based on it's distance from us // The full rate distance is the distance at which EVERY update will be sent for this avatar // at twice the full rate distance, there will be a 50% chance of sending this avatar's update - glm::vec3 otherPosition = otherAvatar.getClientGlobalPosition(); + glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition(); float distanceToAvatar = glm::length(myPosition - otherPosition); // potentially update the max full rate distance for this frame @@ -299,10 +294,13 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(otherNode->getUUID()); AvatarDataSequenceNumber lastSeqFromSender = otherNodeData->getLastReceivedSequenceNumber(); + /*** + // FIXME this does not belong here... it should be in the packet processing if (lastSeqToReceiver > lastSeqFromSender && lastSeqToReceiver != UINT16_MAX) { // we got out out of order packets from the sender, track it otherNodeData->incrementNumOutOfOrderSends(); } + ***/ // make sure we haven't already sent this data from this sender to this receiver // or that somehow we haven't sent @@ -365,7 +363,7 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { bool dropFaceTracking = true; // this is a hack for now... always drop face tracking quint64 start = usecTimestampNow(); - QByteArray bytes = otherAvatar.toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, + QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); quint64 end = usecTimestampNow(); _stats.toByteArrayElapsedTime += (end - start); diff --git a/assignment-client/src/avatars/ScriptableAvatar.cpp b/assignment-client/src/avatars/ScriptableAvatar.cpp index 95bcbb587e..cdda070da2 100644 --- a/assignment-client/src/avatars/ScriptableAvatar.cpp +++ b/assignment-client/src/avatars/ScriptableAvatar.cpp @@ -14,10 +14,9 @@ #include #include "ScriptableAvatar.h" -QByteArray ScriptableAvatar::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) { +QByteArray ScriptableAvatar::toByteArray(AvatarDataDetail dataDetail) { _globalPosition = getPosition(); - return AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut); + return AvatarData::toByteArray(dataDetail); } diff --git a/assignment-client/src/avatars/ScriptableAvatar.h b/assignment-client/src/avatars/ScriptableAvatar.h index be7a90adf9..b6804da43d 100644 --- a/assignment-client/src/avatars/ScriptableAvatar.h +++ b/assignment-client/src/avatars/ScriptableAvatar.h @@ -28,8 +28,7 @@ public: Q_INVOKABLE AnimationDetails getAnimationDetails(); virtual void setSkeletonModelURL(const QUrl& skeletonModelURL) override; - virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector* sentJointDataOut = nullptr) override; + virtual QByteArray toByteArray(AvatarDataDetail dataDetail) override; private slots: diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 29f41c89fd..21b75e4e71 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -227,8 +227,7 @@ void MyAvatar::simulateAttachments(float deltaTime) { // don't update attachments here, do it in harvestResultsFromPhysicsSimulation() } -QByteArray MyAvatar::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) { +QByteArray MyAvatar::toByteArray(AvatarDataDetail dataDetail) { CameraMode mode = qApp->getCamera()->getMode(); _globalPosition = getPosition(); _globalBoundingBoxDimensions.x = _characterController.getCapsuleRadius(); @@ -239,12 +238,12 @@ QByteArray MyAvatar::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTi // fake the avatar position that is sent up to the AvatarMixer glm::vec3 oldPosition = getPosition(); setPosition(getSkeletonPosition()); - QByteArray array = AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut); + QByteArray array = AvatarData::toByteArray(dataDetail); // copy the correct position back setPosition(oldPosition); return array; } - return AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut); + return AvatarData::toByteArray(dataDetail); } void MyAvatar::centerBody() { diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index c4fe86356d..982f0505ff 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -338,8 +338,7 @@ private: glm::quat getWorldBodyOrientation() const; - virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector* sentJointDataOut = nullptr) override; + virtual QByteArray toByteArray(AvatarDataDetail dataDetail) override; void simulate(float deltaTime); void updateFromTrackers(float deltaTime); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 66d1fb39bf..f48b31881a 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -168,414 +168,19 @@ float AvatarData::getDistanceBasedMinTranslationDistance(glm::vec3 viewerPositio } -QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) { - - // if no timestamp was included, then assume the avatarData is single instance - // and is tracking its own last encoding time. - if (lastSentTime == 0) { - lastSentTime = _lastToByteArray; - _lastToByteArray = usecTimestampNow(); - } - - bool cullSmallChanges = (dataDetail == CullSmallData); - bool sendAll = (dataDetail == SendAllData); - bool sendMinimum = (dataDetail == MinimumData); - - lazyInitHeadData(); - - QByteArray avatarDataByteArray(udt::MAX_PACKET_SIZE, 0); - unsigned char* destinationBuffer = reinterpret_cast(avatarDataByteArray.data()); - unsigned char* startPosition = destinationBuffer; - - // FIXME - - // - // BUG -- if you enter a space bubble, and then back away, the avatar has wrong orientation until "send all" happens... - // this is an iFrame issue... what to do about that? - // - // BUG -- Resizing avatar seems to "take too long"... the avatar doesn't redraw at smaller size right away - // - // TODO consider these additional optimizations in the future - // 1) SensorToWorld - should we only send this for avatars with attachments?? - 20 bytes - 7.20 kbps - // 2) GUIID for the session change to 2byte index (savings) - 14 bytes - 5.04 kbps - // 3) Improve Joints -- currently we use rotational tolerances, but if we had skeleton/bone length data - // we could do a better job of determining if the change in joints actually translates to visible - // changes at distance. - // - // Potential savings: - // 63 rotations * 6 bytes = 136kbps - // 3 translations * 6 bytes = 6.48kbps - // - - auto parentID = getParentID(); - - bool hasAvatarGlobalPosition = true; // always include global position - bool hasAvatarOrientation = sendAll || rotationChangedSince(lastSentTime); - bool hasAvatarBoundingBox = sendAll || avatarBoundingBoxChangedSince(lastSentTime); - bool hasAvatarScale = sendAll || avatarScaleChangedSince(lastSentTime); - bool hasLookAtPosition = sendAll || lookAtPositionChangedSince(lastSentTime); - bool hasAudioLoudness = sendAll || audioLoudnessChangedSince(lastSentTime); - bool hasSensorToWorldMatrix = sendAll || sensorToWorldMatrixChangedSince(lastSentTime); - bool hasAdditionalFlags = sendAll || additionalFlagsChangedSince(lastSentTime); - - // local position, and parent info only apply to avatars that are parented. The local position - // and the parent info can change independently though, so we track their "changed since" - // separately - bool hasParentInfo = sendAll || parentInfoChangedSince(lastSentTime); - bool hasAvatarLocalPosition = hasParent() && (sendAll || - tranlationChangedSince(lastSentTime) || - parentInfoChangedSince(lastSentTime)); - - bool hasFaceTrackerInfo = hasFaceTracker() && (sendAll || faceTrackerInfoChangedSince(lastSentTime)); - bool hasJointData = sendAll || !sendMinimum; - - // Leading flags, to indicate how much data is actually included in the packet... - AvatarDataPacket::HasFlags packetStateFlags = - (hasAvatarGlobalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_GLOBAL_POSITION : 0) - | (hasAvatarBoundingBox ? AvatarDataPacket::PACKET_HAS_AVATAR_BOUNDING_BOX : 0) - | (hasAvatarOrientation ? AvatarDataPacket::PACKET_HAS_AVATAR_ORIENTATION : 0) - | (hasAvatarScale ? AvatarDataPacket::PACKET_HAS_AVATAR_SCALE : 0) - | (hasLookAtPosition ? AvatarDataPacket::PACKET_HAS_LOOK_AT_POSITION : 0) - | (hasAudioLoudness ? AvatarDataPacket::PACKET_HAS_AUDIO_LOUDNESS : 0) - | (hasSensorToWorldMatrix ? AvatarDataPacket::PACKET_HAS_SENSOR_TO_WORLD_MATRIX : 0) - | (hasAdditionalFlags ? AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS : 0) - | (hasParentInfo ? AvatarDataPacket::PACKET_HAS_PARENT_INFO : 0) - | (hasAvatarLocalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_LOCAL_POSITION : 0) - | (hasFaceTrackerInfo ? AvatarDataPacket::PACKET_HAS_FACE_TRACKER_INFO : 0) - | (hasJointData ? AvatarDataPacket::PACKET_HAS_JOINT_DATA : 0); - - memcpy(destinationBuffer, &packetStateFlags, sizeof(packetStateFlags)); - destinationBuffer += sizeof(packetStateFlags); - - if (hasAvatarGlobalPosition) { - auto startSection = destinationBuffer; - auto data = reinterpret_cast(destinationBuffer); - data->globalPosition[0] = _globalPosition.x; - data->globalPosition[1] = _globalPosition.y; - data->globalPosition[2] = _globalPosition.z; - destinationBuffer += sizeof(AvatarDataPacket::AvatarGlobalPosition); - - int numBytes = destinationBuffer - startSection; - - _globalPositionRateOutbound.increment(numBytes); - } - - if (hasAvatarBoundingBox) { - auto startSection = destinationBuffer; - auto data = reinterpret_cast(destinationBuffer); - - data->avatarDimensions[0] = _globalBoundingBoxDimensions.x; - data->avatarDimensions[1] = _globalBoundingBoxDimensions.y; - data->avatarDimensions[2] = _globalBoundingBoxDimensions.z; - - data->boundOriginOffset[0] = _globalBoundingBoxOffset.x; - data->boundOriginOffset[1] = _globalBoundingBoxOffset.y; - data->boundOriginOffset[2] = _globalBoundingBoxOffset.z; - - destinationBuffer += sizeof(AvatarDataPacket::AvatarBoundingBox); - - int numBytes = destinationBuffer - startSection; - _avatarBoundingBoxRateOutbound.increment(numBytes); - } - - if (hasAvatarOrientation) { - auto startSection = destinationBuffer; - auto localOrientation = getLocalOrientation(); - destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, localOrientation); - - int numBytes = destinationBuffer - startSection; - _avatarOrientationRateOutbound.increment(numBytes); - } - - if (hasAvatarScale) { - auto startSection = destinationBuffer; - auto data = reinterpret_cast(destinationBuffer); - auto scale = getDomainLimitedScale(); - packFloatRatioToTwoByte((uint8_t*)(&data->scale), scale); - destinationBuffer += sizeof(AvatarDataPacket::AvatarScale); - - int numBytes = destinationBuffer - startSection; - _avatarScaleRateOutbound.increment(numBytes); - } - - if (hasLookAtPosition) { - auto startSection = destinationBuffer; - auto data = reinterpret_cast(destinationBuffer); - auto lookAt = _headData->getLookAtPosition(); - data->lookAtPosition[0] = lookAt.x; - data->lookAtPosition[1] = lookAt.y; - data->lookAtPosition[2] = lookAt.z; - destinationBuffer += sizeof(AvatarDataPacket::LookAtPosition); - - int numBytes = destinationBuffer - startSection; - _lookAtPositionRateOutbound.increment(numBytes); - } - - if (hasAudioLoudness) { - auto startSection = destinationBuffer; - auto data = reinterpret_cast(destinationBuffer); - data->audioLoudness = packFloatGainToByte(_headData->getAudioLoudness() / AUDIO_LOUDNESS_SCALE); - destinationBuffer += sizeof(AvatarDataPacket::AudioLoudness); - - int numBytes = destinationBuffer - startSection; - _audioLoudnessRateOutbound.increment(numBytes); - } - - if (hasSensorToWorldMatrix) { - auto startSection = destinationBuffer; - auto data = reinterpret_cast(destinationBuffer); - glm::mat4 sensorToWorldMatrix = getSensorToWorldMatrix(); - packOrientationQuatToSixBytes(data->sensorToWorldQuat, glmExtractRotation(sensorToWorldMatrix)); - glm::vec3 scale = extractScale(sensorToWorldMatrix); - packFloatScalarToSignedTwoByteFixed((uint8_t*)&data->sensorToWorldScale, scale.x, SENSOR_TO_WORLD_SCALE_RADIX); - data->sensorToWorldTrans[0] = sensorToWorldMatrix[3][0]; - data->sensorToWorldTrans[1] = sensorToWorldMatrix[3][1]; - data->sensorToWorldTrans[2] = sensorToWorldMatrix[3][2]; - destinationBuffer += sizeof(AvatarDataPacket::SensorToWorldMatrix); - - int numBytes = destinationBuffer - startSection; - _sensorToWorldRateOutbound.increment(numBytes); - } - - if (hasAdditionalFlags) { - auto startSection = destinationBuffer; - auto data = reinterpret_cast(destinationBuffer); - - uint8_t flags { 0 }; - - setSemiNibbleAt(flags, KEY_STATE_START_BIT, _keyState); - - // hand state - bool isFingerPointing = _handState & IS_FINGER_POINTING_FLAG; - setSemiNibbleAt(flags, HAND_STATE_START_BIT, _handState & ~IS_FINGER_POINTING_FLAG); - if (isFingerPointing) { - setAtBit(flags, HAND_STATE_FINGER_POINTING_BIT); - } - // faceshift state - if (_headData->_isFaceTrackerConnected) { - setAtBit(flags, IS_FACESHIFT_CONNECTED); - } - // eye tracker state - if (_headData->_isEyeTrackerConnected) { - setAtBit(flags, IS_EYE_TRACKER_CONNECTED); - } - // referential state - if (!parentID.isNull()) { - setAtBit(flags, HAS_REFERENTIAL); - } - data->flags = flags; - destinationBuffer += sizeof(AvatarDataPacket::AdditionalFlags); - - int numBytes = destinationBuffer - startSection; - _additionalFlagsRateOutbound.increment(numBytes); - } - - if (hasParentInfo) { - auto startSection = destinationBuffer; - auto parentInfo = reinterpret_cast(destinationBuffer); - QByteArray referentialAsBytes = parentID.toRfc4122(); - memcpy(parentInfo->parentUUID, referentialAsBytes.data(), referentialAsBytes.size()); - parentInfo->parentJointIndex = getParentJointIndex(); - destinationBuffer += sizeof(AvatarDataPacket::ParentInfo); - - int numBytes = destinationBuffer - startSection; - _parentInfoRateOutbound.increment(numBytes); - } - - if (hasAvatarLocalPosition) { - auto startSection = destinationBuffer; - auto data = reinterpret_cast(destinationBuffer); - auto localPosition = getLocalPosition(); - data->localPosition[0] = localPosition.x; - data->localPosition[1] = localPosition.y; - data->localPosition[2] = localPosition.z; - destinationBuffer += sizeof(AvatarDataPacket::AvatarLocalPosition); - - int numBytes = destinationBuffer - startSection; - _localPositionRateOutbound.increment(numBytes); - } - - // If it is connected, pack up the data - if (hasFaceTrackerInfo) { - auto startSection = destinationBuffer; - auto faceTrackerInfo = reinterpret_cast(destinationBuffer); - - faceTrackerInfo->leftEyeBlink = _headData->_leftEyeBlink; - faceTrackerInfo->rightEyeBlink = _headData->_rightEyeBlink; - faceTrackerInfo->averageLoudness = _headData->_averageLoudness; - faceTrackerInfo->browAudioLift = _headData->_browAudioLift; - faceTrackerInfo->numBlendshapeCoefficients = _headData->_blendshapeCoefficients.size(); - destinationBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo); - - // followed by a variable number of float coefficients - memcpy(destinationBuffer, _headData->_blendshapeCoefficients.data(), _headData->_blendshapeCoefficients.size() * sizeof(float)); - destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); - - int numBytes = destinationBuffer - startSection; - _faceTrackerRateOutbound.increment(numBytes); - } - - // If it is connected, pack up the data - if (hasJointData) { - auto startSection = destinationBuffer; - QReadLocker readLock(&_jointDataLock); - - // joint rotation data - int numJoints = _jointData.size(); - *destinationBuffer++ = (uint8_t)numJoints; - - unsigned char* validityPosition = destinationBuffer; - unsigned char validity = 0; - int validityBit = 0; - -#ifdef WANT_DEBUG - int rotationSentCount = 0; - unsigned char* beforeRotations = destinationBuffer; -#endif - - if (sentJointDataOut) { - sentJointDataOut->resize(_jointData.size()); // Make sure the destination is resized before using it - } - float minRotationDOT = !distanceAdjust ? AVATAR_MIN_ROTATION_DOT : getDistanceBasedMinRotationDOT(viewerPosition); - - for (int i = 0; i < _jointData.size(); i++) { - const JointData& data = _jointData[i]; - - // The dot product for smaller rotations is a smaller number. - // So if the dot() is less than the value, then the rotation is a larger angle of rotation - bool largeEnoughRotation = fabsf(glm::dot(data.rotation, lastSentJointData[i].rotation)) < minRotationDOT; - - if (sendAll || lastSentJointData[i].rotation != data.rotation) { - if (sendAll || !cullSmallChanges || largeEnoughRotation) { - if (data.rotationSet) { - validity |= (1 << validityBit); -#ifdef WANT_DEBUG - rotationSentCount++; -#endif - if (sentJointDataOut) { - auto jointDataOut = *sentJointDataOut; - jointDataOut[i].rotation = data.rotation; - } - - } - } - } - if (++validityBit == BITS_IN_BYTE) { - *destinationBuffer++ = validity; - validityBit = validity = 0; - } - } - if (validityBit != 0) { - *destinationBuffer++ = validity; - } - - validityBit = 0; - validity = *validityPosition++; - for (int i = 0; i < _jointData.size(); i++) { - const JointData& data = _jointData[i]; - if (validity & (1 << validityBit)) { - destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, data.rotation); - } - if (++validityBit == BITS_IN_BYTE) { - validityBit = 0; - validity = *validityPosition++; - } - } - - - // joint translation data - validityPosition = destinationBuffer; - validity = 0; - validityBit = 0; - -#ifdef WANT_DEBUG - int translationSentCount = 0; - unsigned char* beforeTranslations = destinationBuffer; -#endif - - float minTranslation = !distanceAdjust ? AVATAR_MIN_TRANSLATION : getDistanceBasedMinTranslationDistance(viewerPosition); - - float maxTranslationDimension = 0.0; - for (int i = 0; i < _jointData.size(); i++) { - const JointData& data = _jointData[i]; - if (sendAll || lastSentJointData[i].translation != data.translation) { - if (sendAll || - !cullSmallChanges || - glm::distance(data.translation, lastSentJointData[i].translation) > minTranslation) { - if (data.translationSet) { - validity |= (1 << validityBit); - #ifdef WANT_DEBUG - translationSentCount++; - #endif - maxTranslationDimension = glm::max(fabsf(data.translation.x), maxTranslationDimension); - maxTranslationDimension = glm::max(fabsf(data.translation.y), maxTranslationDimension); - maxTranslationDimension = glm::max(fabsf(data.translation.z), maxTranslationDimension); - - if (sentJointDataOut) { - auto jointDataOut = *sentJointDataOut; - jointDataOut[i].translation = data.translation; - } - - } - } - } - if (++validityBit == BITS_IN_BYTE) { - *destinationBuffer++ = validity; - validityBit = validity = 0; - } - } - - if (validityBit != 0) { - *destinationBuffer++ = validity; - } - - validityBit = 0; - validity = *validityPosition++; - for (int i = 0; i < _jointData.size(); i++) { - const JointData& data = _jointData[i]; - if (validity & (1 << validityBit)) { - destinationBuffer += - packFloatVec3ToSignedTwoByteFixed(destinationBuffer, data.translation, TRANSLATION_COMPRESSION_RADIX); - } - if (++validityBit == BITS_IN_BYTE) { - validityBit = 0; - validity = *validityPosition++; - } - } - - // faux joints - Transform controllerLeftHandTransform = Transform(getControllerLeftHandMatrix()); - destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, controllerLeftHandTransform.getRotation()); - destinationBuffer += packFloatVec3ToSignedTwoByteFixed(destinationBuffer, controllerLeftHandTransform.getTranslation(), - TRANSLATION_COMPRESSION_RADIX); - Transform controllerRightHandTransform = Transform(getControllerRightHandMatrix()); - destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, controllerRightHandTransform.getRotation()); - destinationBuffer += packFloatVec3ToSignedTwoByteFixed(destinationBuffer, controllerRightHandTransform.getTranslation(), - TRANSLATION_COMPRESSION_RADIX); - -#ifdef WANT_DEBUG - if (sendAll) { - qCDebug(avatars) << "AvatarData::toByteArray" << cullSmallChanges << sendAll - << "rotations:" << rotationSentCount << "translations:" << translationSentCount - << "largest:" << maxTranslationDimension - << "size:" - << (beforeRotations - startPosition) << "+" - << (beforeTranslations - beforeRotations) << "+" - << (destinationBuffer - beforeTranslations) << "=" - << (destinationBuffer - startPosition); - } -#endif - - int numBytes = destinationBuffer - startSection; - _jointDataRateOutbound.increment(numBytes); - } - - int avatarDataSize = destinationBuffer - startPosition; - return avatarDataByteArray.left(avatarDataSize); +// we want to track outbound data in this case... +QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail) { + AvatarDataPacket::HasFlags hasFlagsOut; + auto lastSentTime = _lastToByteArray; + _lastToByteArray = usecTimestampNow(); + return AvatarData::toByteArray(dataDetail, lastSentTime, getLastSentJointData(), + hasFlagsOut, false, false, glm::vec3(0), nullptr, + &_outboundDataRate); } QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) const { + AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, + glm::vec3 viewerPosition, QVector* sentJointDataOut, AvatarDataRate* outboundDataRateOut) const { bool cullSmallChanges = (dataDetail == CullSmallData); bool sendAll = (dataDetail == SendAllData); @@ -656,7 +261,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent int numBytes = destinationBuffer - startSection; - //_globalPositionRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->globalPositionRate.increment(numBytes); + } } if (hasAvatarBoundingBox) { @@ -674,7 +281,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += sizeof(AvatarDataPacket::AvatarBoundingBox); int numBytes = destinationBuffer - startSection; - //_avatarBoundingBoxRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->avatarBoundingBoxRate.increment(numBytes); + } } if (hasAvatarOrientation) { @@ -683,7 +292,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, localOrientation); int numBytes = destinationBuffer - startSection; - //_avatarOrientationRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->avatarOrientationRate.increment(numBytes); + } } if (hasAvatarScale) { @@ -694,7 +305,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += sizeof(AvatarDataPacket::AvatarScale); int numBytes = destinationBuffer - startSection; - //_avatarScaleRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->avatarScaleRate.increment(numBytes); + } } if (hasLookAtPosition) { @@ -707,7 +320,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += sizeof(AvatarDataPacket::LookAtPosition); int numBytes = destinationBuffer - startSection; - //_lookAtPositionRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->lookAtPositionRate.increment(numBytes); + } } if (hasAudioLoudness) { @@ -717,7 +332,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += sizeof(AvatarDataPacket::AudioLoudness); int numBytes = destinationBuffer - startSection; - //_audioLoudnessRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->audioLoudnessRate.increment(numBytes); + } } if (hasSensorToWorldMatrix) { @@ -733,7 +350,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += sizeof(AvatarDataPacket::SensorToWorldMatrix); int numBytes = destinationBuffer - startSection; - //_sensorToWorldRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->sensorToWorldRate.increment(numBytes); + } } if (hasAdditionalFlags) { @@ -766,7 +385,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += sizeof(AvatarDataPacket::AdditionalFlags); int numBytes = destinationBuffer - startSection; - //_additionalFlagsRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->additionalFlagsRate.increment(numBytes); + } } if (hasParentInfo) { @@ -778,7 +399,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += sizeof(AvatarDataPacket::ParentInfo); int numBytes = destinationBuffer - startSection; - //_parentInfoRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->parentInfoRate.increment(numBytes); + } } if (hasAvatarLocalPosition) { @@ -791,7 +414,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += sizeof(AvatarDataPacket::AvatarLocalPosition); int numBytes = destinationBuffer - startSection; - //_localPositionRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->localPositionRate.increment(numBytes); + } } // If it is connected, pack up the data @@ -811,7 +436,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); int numBytes = destinationBuffer - startSection; - //_faceTrackerRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->faceTrackerRate.increment(numBytes); + } } // If it is connected, pack up the data @@ -966,7 +593,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent #endif int numBytes = destinationBuffer - startSection; - //_jointDataRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->jointDataRate.increment(numBytes); + } } int avatarDataSize = destinationBuffer - startPosition; @@ -1451,29 +1080,29 @@ float AvatarData::getDataRate(const QString& rateName) const { } else if (rateName == "jointData") { return _jointDataRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "globalPositionOutbound") { - return _globalPositionRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.globalPositionRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "localPositionOutbound") { - return _localPositionRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.localPositionRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "avatarBoundingBoxOutbound") { - return _avatarBoundingBoxRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.avatarBoundingBoxRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "avatarOrientationOutbound") { - return _avatarOrientationRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.avatarOrientationRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "avatarScaleOutbound") { - return _avatarScaleRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.avatarScaleRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "lookAtPositionOutbound") { - return _lookAtPositionRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.lookAtPositionRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "audioLoudnessOutbound") { - return _audioLoudnessRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.audioLoudnessRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "sensorToWorkMatrixOutbound") { - return _sensorToWorldRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.sensorToWorldRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "additionalFlagsOutbound") { - return _additionalFlagsRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.additionalFlagsRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "parentInfoOutbound") { - return _parentInfoRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.parentInfoRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "faceTrackerOutbound") { - return _faceTrackerRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.faceTrackerRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "jointDataOutbound") { - return _jointDataRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.jointDataRate.rate() / BYTES_PER_KILOBIT; } return 0.0f; } @@ -1807,7 +1436,7 @@ void AvatarData::parseAvatarIdentityPacket(const QByteArray& data, Identity& ide } static const QUrl emptyURL(""); -const QUrl& AvatarData::cannonicalSkeletonModelURL(const QUrl& emptyURL) { +QUrl AvatarData::cannonicalSkeletonModelURL(const QUrl& emptyURL) const { // We don't put file urls on the wire, but instead convert to empty. return _skeletonModelURL.scheme() == "file" ? emptyURL : _skeletonModelURL; } @@ -1845,7 +1474,7 @@ void AvatarData::processAvatarIdentity(const Identity& identity, bool& identityC } } -QByteArray AvatarData::identityByteArray() { +QByteArray AvatarData::identityByteArray() const { QByteArray identityData; QDataStream identityStream(&identityData, QIODevice::Append); const QUrl& urlToSend = cannonicalSkeletonModelURL(emptyURL); @@ -2008,13 +1637,7 @@ void AvatarData::sendAvatarDataPacket() { bool cullSmallData = (randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO); auto dataDetail = cullSmallData ? SendAllData : CullSmallData; - QVector lastSentJointData; - { - QReadLocker readLock(&_jointDataLock); - _lastSentJointData.resize(_jointData.size()); - lastSentJointData = _lastSentJointData; - } - QByteArray avatarByteArray = toByteArray(dataDetail, 0, lastSentJointData); + QByteArray avatarByteArray = toByteArray(dataDetail); doneEncoding(cullSmallData); static AvatarDataSequenceNumber sequenceNumber = 0; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 64c0e11577..14348b0622 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -288,6 +288,23 @@ class AttachmentData; class Transform; using TransformPointer = std::shared_ptr; +class AvatarDataRate { +public: + RateCounter<> globalPositionRate; + RateCounter<> localPositionRate; + RateCounter<> avatarBoundingBoxRate; + RateCounter<> avatarOrientationRate; + RateCounter<> avatarScaleRate; + RateCounter<> lookAtPositionRate; + RateCounter<> audioLoudnessRate; + RateCounter<> sensorToWorldRate; + RateCounter<> additionalFlagsRate; + RateCounter<> parentInfoRate; + RateCounter<> faceTrackerRate; + RateCounter<> jointDataRate; +}; + + class AvatarData : public QObject, public SpatiallyNestable { Q_OBJECT @@ -351,12 +368,12 @@ public: SendAllData } AvatarDataDetail; - virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector* sentJointDataOut = nullptr); + virtual QByteArray toByteArray(AvatarDataDetail dataDetail); // FIXME virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) const; + AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, + QVector* sentJointDataOut, AvatarDataRate* outboundDataRateOut = nullptr) const; virtual void doneEncoding(bool cullSmallChanges); @@ -497,7 +514,7 @@ public: // displayNameChanged returns true if displayName has changed, false otherwise. void processAvatarIdentity(const Identity& identity, bool& identityChanged, bool& displayNameChanged); - QByteArray identityByteArray(); + QByteArray identityByteArray() const; const QUrl& getSkeletonModelURL() const { return _skeletonModelURL; } const QString& getDisplayName() const { return _displayName; } @@ -626,7 +643,7 @@ protected: QVector _attachmentData; QString _displayName; QString _sessionDisplayName { }; - const QUrl& cannonicalSkeletonModelURL(const QUrl& empty); + QUrl cannonicalSkeletonModelURL(const QUrl& empty) const; float _displayNameTargetAlpha; float _displayNameAlpha; @@ -697,18 +714,7 @@ protected: RateCounter<> _jointDataUpdateRate; // Some rate data for outgoing data - RateCounter<> _globalPositionRateOutbound; - RateCounter<> _localPositionRateOutbound; - RateCounter<> _avatarBoundingBoxRateOutbound; - RateCounter<> _avatarOrientationRateOutbound; - RateCounter<> _avatarScaleRateOutbound; - RateCounter<> _lookAtPositionRateOutbound; - RateCounter<> _audioLoudnessRateOutbound; - RateCounter<> _sensorToWorldRateOutbound; - RateCounter<> _additionalFlagsRateOutbound; - RateCounter<> _parentInfoRateOutbound; - RateCounter<> _faceTrackerRateOutbound; - RateCounter<> _jointDataRateOutbound; + AvatarDataRate _outboundDataRate; glm::vec3 _globalBoundingBoxDimensions; glm::vec3 _globalBoundingBoxOffset; From c2c843d841581d6cbc3af66ed2fa01e0d29eb12f Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 17 Feb 2017 21:05:02 -0800 Subject: [PATCH 062/104] fix mac warnings --- assignment-client/src/avatars/AvatarMixer.cpp | 69 ++++++++----------- .../src/avatars/AvatarMixerSlave.cpp | 1 - 2 files changed, 30 insertions(+), 40 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 51f6067e6a..636ef3237f 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -69,7 +69,6 @@ AvatarMixer::~AvatarMixer() { // An 80% chance of sending a identity packet within a 5 second interval. // assuming 60 htz update rate. -const float IDENTITY_SEND_PROBABILITY = 1.0f / 187.0f; void AvatarMixer::sendIdentityPacket(AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { QByteArray individualData = nodeData->getAvatar().identityByteArray(); @@ -142,6 +141,7 @@ void AvatarMixer::start() { // calculates last frame duration and sleeps for the remainder of the target amount auto frameDuration = timeFrame(frameTimestamp); + Q_UNUSED(frameDuration); int lockWait, nodeTransform, functor; @@ -248,16 +248,6 @@ void AvatarMixer::manageDisplayName(const SharedNodePointer& node) { } } -static void avatarLoops(); - -// only send extra avatar data (avatars out of view, ignored) every Nth AvatarData frame -// Extra avatar data will be sent (AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND/EXTRA_AVATAR_DATA_FRAME_RATIO) times -// per second. -// This value should be a power of two for performance purposes, as the mixer performs a modulo operation every frame -// to determine whether the extra data should be sent. -static const int EXTRA_AVATAR_DATA_FRAME_RATIO = 16; - - // FIXME -- this is dead code... it needs to be removed... // this "throttle" logic is the old approach. need to consider some // reasonable throttle approach in new multi-core design @@ -484,18 +474,19 @@ void AvatarMixer::sendStatsPacket() { int tightLoopFrames = _numTightLoopFrames; int tenTimesPerFrame = tightLoopFrames * 10; #define TIGHT_LOOP_STAT(x) (x > tenTimesPerFrame) ? x / tightLoopFrames : ((float)x / (float)tightLoopFrames); + #define TIGHT_LOOP_STAT_UINT64(x) (x > (quint64)tenTimesPerFrame) ? x / tightLoopFrames : ((float)x / (float)tightLoopFrames); QJsonObject singleCoreTasks; - singleCoreTasks["processEvents"] = TIGHT_LOOP_STAT(_processEventsElapsedTime); - singleCoreTasks["queueIncomingPacket"] = TIGHT_LOOP_STAT(_queueIncomingPacketElapsedTime); + singleCoreTasks["processEvents"] = TIGHT_LOOP_STAT_UINT64(_processEventsElapsedTime); + singleCoreTasks["queueIncomingPacket"] = TIGHT_LOOP_STAT_UINT64(_queueIncomingPacketElapsedTime); QJsonObject incomingPacketStats; - incomingPacketStats["handleAvatarIdentityPacket"] = TIGHT_LOOP_STAT(_handleAvatarIdentityPacketElapsedTime); - incomingPacketStats["handleKillAvatarPacket"] = TIGHT_LOOP_STAT(_handleKillAvatarPacketElapsedTime); - incomingPacketStats["handleNodeIgnoreRequestPacket"] = TIGHT_LOOP_STAT(_handleNodeIgnoreRequestPacketElapsedTime); - incomingPacketStats["handleRadiusIgnoreRequestPacket"] = TIGHT_LOOP_STAT(_handleRadiusIgnoreRequestPacketElapsedTime); - incomingPacketStats["handleRequestsDomainListDataPacket"] = TIGHT_LOOP_STAT(_handleRequestsDomainListDataPacketElapsedTime); - incomingPacketStats["handleViewFrustumPacket"] = TIGHT_LOOP_STAT(_handleViewFrustumPacketElapsedTime); + incomingPacketStats["handleAvatarIdentityPacket"] = TIGHT_LOOP_STAT_UINT64(_handleAvatarIdentityPacketElapsedTime); + incomingPacketStats["handleKillAvatarPacket"] = TIGHT_LOOP_STAT_UINT64(_handleKillAvatarPacketElapsedTime); + incomingPacketStats["handleNodeIgnoreRequestPacket"] = TIGHT_LOOP_STAT_UINT64(_handleNodeIgnoreRequestPacketElapsedTime); + incomingPacketStats["handleRadiusIgnoreRequestPacket"] = TIGHT_LOOP_STAT_UINT64(_handleRadiusIgnoreRequestPacketElapsedTime); + incomingPacketStats["handleRequestsDomainListDataPacket"] = TIGHT_LOOP_STAT_UINT64(_handleRequestsDomainListDataPacketElapsedTime); + incomingPacketStats["handleViewFrustumPacket"] = TIGHT_LOOP_STAT_UINT64(_handleViewFrustumPacketElapsedTime); singleCoreTasks["incoming_packets"] = incomingPacketStats; singleCoreTasks["sendStats"] = (float)_sendStatsElapsedTime; @@ -505,22 +496,22 @@ void AvatarMixer::sendStatsPacket() { QJsonObject parallelTasks; QJsonObject processQueuedAvatarDataPacketsStats; - processQueuedAvatarDataPacketsStats["1_total"] = TIGHT_LOOP_STAT(_processQueuedAvatarDataPacketsElapsedTime); - processQueuedAvatarDataPacketsStats["2_lockWait"] = TIGHT_LOOP_STAT(_processQueuedAvatarDataPacketsLockWaitElapsedTime); + processQueuedAvatarDataPacketsStats["1_total"] = TIGHT_LOOP_STAT_UINT64(_processQueuedAvatarDataPacketsElapsedTime); + processQueuedAvatarDataPacketsStats["2_lockWait"] = TIGHT_LOOP_STAT_UINT64(_processQueuedAvatarDataPacketsLockWaitElapsedTime); parallelTasks["processQueuedAvatarDataPackets"] = processQueuedAvatarDataPacketsStats; QJsonObject broadcastAvatarDataStats; - broadcastAvatarDataStats["1_total"] = TIGHT_LOOP_STAT(_broadcastAvatarDataElapsedTime); - broadcastAvatarDataStats["2_innner"] = TIGHT_LOOP_STAT(_broadcastAvatarDataInner); - broadcastAvatarDataStats["3_lockWait"] = TIGHT_LOOP_STAT(_broadcastAvatarDataLockWait); - broadcastAvatarDataStats["4_NodeTransform"] = TIGHT_LOOP_STAT(_broadcastAvatarDataNodeTransform); - broadcastAvatarDataStats["5_Functor"] = TIGHT_LOOP_STAT(_broadcastAvatarDataNodeFunctor); + broadcastAvatarDataStats["1_total"] = TIGHT_LOOP_STAT_UINT64(_broadcastAvatarDataElapsedTime); + broadcastAvatarDataStats["2_innner"] = TIGHT_LOOP_STAT_UINT64(_broadcastAvatarDataInner); + broadcastAvatarDataStats["3_lockWait"] = TIGHT_LOOP_STAT_UINT64(_broadcastAvatarDataLockWait); + broadcastAvatarDataStats["4_NodeTransform"] = TIGHT_LOOP_STAT_UINT64(_broadcastAvatarDataNodeTransform); + broadcastAvatarDataStats["5_Functor"] = TIGHT_LOOP_STAT_UINT64(_broadcastAvatarDataNodeFunctor); parallelTasks["broadcastAvatarData"] = broadcastAvatarDataStats; QJsonObject displayNameManagementStats; - displayNameManagementStats["1_total"] = TIGHT_LOOP_STAT(_displayNameManagementElapsedTime); + displayNameManagementStats["1_total"] = TIGHT_LOOP_STAT_UINT64(_displayNameManagementElapsedTime); parallelTasks["displayNameManagement"] = displayNameManagementStats; statsObject["parallelTasks"] = parallelTasks; @@ -538,12 +529,12 @@ void AvatarMixer::sendStatsPacket() { slaveObject["numPacketsReceived"] = TIGHT_LOOP_STAT(stats.packetsProcessed); slaveObject["numPacketsSent"] = TIGHT_LOOP_STAT(stats.numPacketsSent); - slaveObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT(stats.processIncomingPacketsElapsedTime); - slaveObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT(stats.ignoreCalculationElapsedTime); - slaveObject["timing_3_toByteArray"] = TIGHT_LOOP_STAT(stats.toByteArrayElapsedTime); - slaveObject["timing_4_avatarDataPacking"] = TIGHT_LOOP_STAT(stats.avatarDataPackingElapsedTime); - slaveObject["timing_5_packetSending"] = TIGHT_LOOP_STAT(stats.packetSendingElapsedTime); - slaveObject["timing_6_jobElapsedTime"] = TIGHT_LOOP_STAT(stats.jobElapsedTime); + slaveObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT_UINT64(stats.processIncomingPacketsElapsedTime); + slaveObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT_UINT64(stats.ignoreCalculationElapsedTime); + slaveObject["timing_3_toByteArray"] = TIGHT_LOOP_STAT_UINT64(stats.toByteArrayElapsedTime); + slaveObject["timing_4_avatarDataPacking"] = TIGHT_LOOP_STAT_UINT64(stats.avatarDataPackingElapsedTime); + slaveObject["timing_5_packetSending"] = TIGHT_LOOP_STAT_UINT64(stats.packetSendingElapsedTime); + slaveObject["timing_6_jobElapsedTime"] = TIGHT_LOOP_STAT_UINT64(stats.jobElapsedTime); slavesObject[QString::number(slaveNumber)] = slaveObject; slaveNumber++; @@ -557,12 +548,12 @@ void AvatarMixer::sendStatsPacket() { slavesAggregatObject["numPacketsReceived"] = TIGHT_LOOP_STAT(aggregateStats.packetsProcessed); slavesAggregatObject["numPacketsSent"] = TIGHT_LOOP_STAT(aggregateStats.numPacketsSent); - slavesAggregatObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT(aggregateStats.processIncomingPacketsElapsedTime); - slavesAggregatObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT(aggregateStats.ignoreCalculationElapsedTime); - slavesAggregatObject["timing_3_toByteArray"] = TIGHT_LOOP_STAT(aggregateStats.toByteArrayElapsedTime); - slavesAggregatObject["timing_4_avatarDataPacking"] = TIGHT_LOOP_STAT(aggregateStats.avatarDataPackingElapsedTime); - slavesAggregatObject["timing_5_packetSending"] = TIGHT_LOOP_STAT(aggregateStats.packetSendingElapsedTime); - slavesAggregatObject["timing_6_jobElapsedTime"] = TIGHT_LOOP_STAT(aggregateStats.jobElapsedTime); + slavesAggregatObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.processIncomingPacketsElapsedTime); + slavesAggregatObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.ignoreCalculationElapsedTime); + slavesAggregatObject["timing_3_toByteArray"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.toByteArrayElapsedTime); + slavesAggregatObject["timing_4_avatarDataPacking"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.avatarDataPackingElapsedTime); + slavesAggregatObject["timing_5_packetSending"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.packetSendingElapsedTime); + slavesAggregatObject["timing_6_jobElapsedTime"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.jobElapsedTime); statsObject["slaves_aggregate"] = slavesAggregatObject; statsObject["slaves_individual"] = slavesObject; diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 2a7f9a4d8d..1ddefd8813 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -56,7 +56,6 @@ void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { #include static const int AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND = 45; -static const unsigned int AVATAR_DATA_SEND_INTERVAL_MSECS = (1.0f / (float)AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND) * 1000; // only send extra avatar data (avatars out of view, ignored) every Nth AvatarData frame // Extra avatar data will be sent (AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND/EXTRA_AVATAR_DATA_FRAME_RATIO) times From 4570b3439d4ad0c601c74bf2b162754a44d7246a Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 17 Feb 2017 21:38:30 -0800 Subject: [PATCH 063/104] some cleanup --- assignment-client/src/avatars/AvatarMixer.cpp | 33 +++++++++-------- .../src/avatars/AvatarMixerSlave.cpp | 36 ++++++------------- .../src/avatars/AvatarMixerSlave.h | 4 +++ .../src/avatars/AvatarMixerSlavePool.cpp | 10 ++++-- .../src/avatars/AvatarMixerSlavePool.h | 3 +- 5 files changed, 42 insertions(+), 44 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 636ef3237f..6b5f84960b 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -120,20 +120,25 @@ void AvatarMixer::start() { ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // DO THIS FIRST!!!!!!!! + // WORK ITEMS... // // DONE --- 1) only sleep for remainder // DONE --- 2) clean up stats, add slave stats // DONE --- 3) out of view??? is it broken? - verified - it's working - // 4) Error in PacketList::writeData - attempted to write a segment to an unordered packet that is larger than the payload size. // DONE --- 4a) hack to not send face data mostly seems to work... + // DONE --- 5) fix two different versions of toByteArray() + // DONE --- 7) audit the locking and side-effects to node, otherNode, and nodeData + // DONE --- 8) delete dead code from mixer (now that it's in slave) + // DONE --- 10) FIXME on sending identity packets + // DONE --- 12) FIXME _maxKbpsPerNode + // DONE --- 11) FIXME ++_sumListeners; + // + // 4) Error in PacketList::writeData - attempted to write a segment to an unordered packet that is larger than the payload size. // 4b) some kind of a better approach to handling otherAvatar.toByteArray() for content that is larger than MTU - // 5) fix two different versions of toByteArray() - // 6) throttling?? - // 7) audit the locking and side-effects to node, otherNode, and nodeData - // 8) delete dead code from mixer (now that it's in slave) + // 6) CPU throttling?? // 9) better stats in the nodes: // how many avatars are actually "in view" for the avtar in question (even if they are over bandwidth budget) + // 13) FIXME -- otherNodeData->incrementNumOutOfOrderSends(); // // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -157,8 +162,6 @@ void AvatarMixer::start() { }, &lockWait, &nodeTransform, &functor); auto end = usecTimestampNow(); _processQueuedAvatarDataPacketsElapsedTime += (end - start); - - //qDebug() << "PROCESS PACKETS... " << "lockWait:" << lockWait << "nodeTransform:" << nodeTransform << "functor:" << functor; } // process pending display names... this doesn't currently run on multiple threads, because it @@ -168,23 +171,19 @@ void AvatarMixer::start() { nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { std::for_each(cbegin, cend, [&](const SharedNodePointer& node) { manageDisplayName(node); + ++_sumListeners; }); }, &lockWait, &nodeTransform, &functor); auto end = usecTimestampNow(); _displayNameManagementElapsedTime += (end - start); - - //qDebug() << "PROCESS PACKETS... " << "lockWait:" << lockWait << "nodeTransform:" << nodeTransform << "functor:" << functor; } // this is where we need to put the real work... { - // for now, call the single threaded version - //broadcastAvatarData(); - auto start = usecTimestampNow(); nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { auto start = usecTimestampNow(); - _slavePool.broadcastAvatarData(cbegin, cend); + _slavePool.broadcastAvatarData(cbegin, cend, _lastFrameTimestamp, _maxKbpsPerNode); auto end = usecTimestampNow(); _broadcastAvatarDataInner += (end - start); }, &lockWait, &nodeTransform, &functor); @@ -210,6 +209,9 @@ void AvatarMixer::start() { auto end = usecTimestampNow(); _processEventsElapsedTime += (end - start); } + + _lastFrameTimestamp = frameTimestamp; + } } @@ -461,7 +463,6 @@ void AvatarMixer::sendStatsPacket() { QJsonObject statsObject; - //statsObject["average_listeners_last_second"] = (float) _sumListeners / (float) _numStatFrames; //statsObject["average_identity_packets_per_frame"] = (float) _sumIdentityPackets / (float) _numStatFrames; statsObject["broadcast_loop_rate"] = _loopRate.rate(); @@ -476,6 +477,8 @@ void AvatarMixer::sendStatsPacket() { #define TIGHT_LOOP_STAT(x) (x > tenTimesPerFrame) ? x / tightLoopFrames : ((float)x / (float)tightLoopFrames); #define TIGHT_LOOP_STAT_UINT64(x) (x > (quint64)tenTimesPerFrame) ? x / tightLoopFrames : ((float)x / (float)tightLoopFrames); + statsObject["average_listeners_last_second"] = TIGHT_LOOP_STAT(_sumListeners); + QJsonObject singleCoreTasks; singleCoreTasks["processEvents"] = TIGHT_LOOP_STAT_UINT64(_processEventsElapsedTime); singleCoreTasks["queueIncomingPacket"] = TIGHT_LOOP_STAT_UINT64(_queueIncomingPacketElapsedTime); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 1ddefd8813..fb67cbf64e 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -35,6 +35,15 @@ void AvatarMixerSlave::configure(ConstIter begin, ConstIter end) { _end = end; } +void AvatarMixerSlave::configureBroadcast(ConstIter begin, ConstIter end, + p_high_resolution_clock::time_point lastFrameTimestamp, + float maxKbpsPerNode) { + _begin = begin; + _end = end; + _lastFrameTimestamp = lastFrameTimestamp; + _maxKbpsPerNode = maxKbpsPerNode; +} + void AvatarMixerSlave::harvestStats(AvatarMixerSlaveStats& stats) { stats = _stats; _stats.reset(); @@ -71,8 +80,6 @@ const float IDENTITY_SEND_PROBABILITY = 1.0f / 187.0f; // FIXME... this is wrong void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { quint64 start = usecTimestampNow(); - //qDebug() << __FUNCTION__ << "node:" << node; - auto nodeList = DependencyManager::get(); // setup for distributed random floating point values @@ -82,13 +89,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { if (node->getLinkedData() && (node->getType() == NodeType::Agent) && node->getActiveSocket()) { AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); - //MutexTryLocker lock(nodeData->getMutex()); - - // FIXME???? - //if (!lock.isLocked()) { - //qDebug() << __FUNCTION__ << "unable to lock... node:" << node << " would BAIL???... line:" << __LINE__; - //return; - //} // FIXME -- mixer data // ++_sumListeners; @@ -143,8 +143,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // get the current full rate distance so we can work with it float currentFullRateDistance = nodeData->getFullRateDistance(); - // FIXME -- mixer data - float _maxKbpsPerNode = 5000.0f; if (avatarDataRateLastSecond > _maxKbpsPerNode) { // is the FRD greater than the farthest avatar? @@ -176,7 +174,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // this is an AGENT we have received head data from // send back a packet with other active node data to this node std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) { - //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode; bool shouldConsider = false; quint64 startIgnoreCalculation = usecTimestampNow(); @@ -236,9 +233,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); } - //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << "shouldConsider:" << shouldConsider; - - if (shouldConsider) { quint64 startAvatarDataPacking = usecTimestampNow(); @@ -247,24 +241,19 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); // make sure we send out identity packets to and from new arrivals. - // FIXME this is where our crash was on friday!! - // this is getting called on multiple threads... needs mutex of better solution bool forceSend = !nodeData->checkAndSetHasReceivedFirstPacketsFrom(otherNode->getUUID()); + // FIXME - this clause seems suspicious "... || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp ..." if (otherNodeData->getIdentityChangeTimestamp().time_since_epoch().count() > 0 && (forceSend - //|| otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp // FIXME - mixer data + || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp || distribution(generator) < IDENTITY_SEND_PROBABILITY)) { - // FIXME --- used to be.../ mixer data dependency - //sendIdentityPacket(otherNodeData, node); - QByteArray individualData = otherNodeData->getConstAvatarData()->identityByteArray(); auto identityPacket = NLPacket::create(PacketType::AvatarIdentity, individualData.size()); individualData.replace(0, NUM_BYTES_RFC4122_UUID, otherNodeData->getNodeID().toRfc4122()); identityPacket->write(individualData); DependencyManager::get()->sendPacket(std::move(identityPacket), *node); - //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << " sending itentity packet for otherNode to node..."; } const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); @@ -285,7 +274,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { quint64 endAvatarDataPacking = usecTimestampNow(); _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - //qDebug() << __FUNCTION__ << "inner loop, node:" << node->getUUID() << "otherNode:" << otherNode->getUUID() << " BAILING... line:" << __LINE__; shouldConsider = false; } @@ -308,7 +296,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { quint64 endAvatarDataPacking = usecTimestampNow(); _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - //qDebug() << __FUNCTION__ << "inner loop, node:" << node->getUUID() << "otherNode:" << otherNode->getUUID() << " BAILING... line:" << __LINE__; shouldConsider = false; } else if (lastSeqFromSender - lastSeqToReceiver > 1) { // this is a skip - we still send the packet but capture the presence of the skip so we see it happening @@ -334,7 +321,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { quint64 endAvatarDataPacking = usecTimestampNow(); _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - //qDebug() << __FUNCTION__ << "inner loop, node:" << node->getUUID() << "otherNode:" << otherNode->getUUID() << " BAILING... line:" << __LINE__; shouldConsider = false; } diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index b693356349..9382fcfeda 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -62,6 +62,7 @@ public: using ConstIter = NodeList::const_iterator; void configure(ConstIter begin, ConstIter end); + void configureBroadcast(ConstIter begin, ConstIter end, p_high_resolution_clock::time_point lastFrameTimestamp, float maxKbpsPerNode); void processIncomingPackets(const SharedNodePointer& node); void broadcastAvatarData(const SharedNodePointer& node); @@ -73,6 +74,9 @@ private: ConstIter _begin; ConstIter _end; + p_high_resolution_clock::time_point _lastFrameTimestamp; + float _maxKbpsPerNode { 0.0f }; + AvatarMixerSlaveStats _stats; }; diff --git a/assignment-client/src/avatars/AvatarMixerSlavePool.cpp b/assignment-client/src/avatars/AvatarMixerSlavePool.cpp index f410bb566d..c0dcf9cbba 100644 --- a/assignment-client/src/avatars/AvatarMixerSlavePool.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlavePool.cpp @@ -69,13 +69,17 @@ static AvatarMixerSlave slave; void AvatarMixerSlavePool::processIncomingPackets(ConstIter begin, ConstIter end) { _function = &AvatarMixerSlave::processIncomingPackets; - _configure = [&](AvatarMixerSlave& slave) { slave.configure(begin, end); }; + _configure = [&](AvatarMixerSlave& slave) { + slave.configure(begin, end); + }; run(begin, end); } -void AvatarMixerSlavePool::broadcastAvatarData(ConstIter begin, ConstIter end) { +void AvatarMixerSlavePool::broadcastAvatarData(ConstIter begin, ConstIter end, p_high_resolution_clock::time_point lastFrameTimestamp, float maxKbpsPerNode) { _function = &AvatarMixerSlave::broadcastAvatarData; - _configure = [&](AvatarMixerSlave& slave) { slave.configure(begin, end); }; + _configure = [&](AvatarMixerSlave& slave) { + slave.configureBroadcast(begin, end, lastFrameTimestamp, maxKbpsPerNode); + }; run(begin, end); } diff --git a/assignment-client/src/avatars/AvatarMixerSlavePool.h b/assignment-client/src/avatars/AvatarMixerSlavePool.h index 0a689b35ed..e54681401d 100644 --- a/assignment-client/src/avatars/AvatarMixerSlavePool.h +++ b/assignment-client/src/avatars/AvatarMixerSlavePool.h @@ -65,7 +65,8 @@ public: // Jobs the slave pool can do... void processIncomingPackets(ConstIter begin, ConstIter end); - void broadcastAvatarData(ConstIter begin, ConstIter end); + void broadcastAvatarData(ConstIter begin, ConstIter end, + p_high_resolution_clock::time_point lastFrameTimestamp, float maxKbpsPerNode); // iterate over all slaves void each(std::function functor); From d23343080926e04e402b56f0af70ba6d82083977 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 17 Feb 2017 21:58:35 -0800 Subject: [PATCH 064/104] fix warnings --- assignment-client/src/Agent.cpp | 2 +- assignment-client/src/avatars/AvatarMixer.cpp | 1 + assignment-client/src/avatars/ScriptableAvatar.cpp | 4 ++-- assignment-client/src/avatars/ScriptableAvatar.h | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index a458719346..355e47be46 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -546,7 +546,7 @@ void Agent::processAgentAvatar() { auto scriptedAvatar = DependencyManager::get(); AvatarData::AvatarDataDetail dataDetail = (randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO) ? AvatarData::SendAllData : AvatarData::CullSmallData; - QByteArray avatarByteArray = scriptedAvatar->toByteArray(dataDetail); + QByteArray avatarByteArray = scriptedAvatar->toByteArrayStateful(dataDetail); scriptedAvatar->doneEncoding(true); static AvatarDataSequenceNumber sequenceNumber = 0; diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 6b5f84960b..6c04ca8799 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -132,6 +132,7 @@ void AvatarMixer::start() { // DONE --- 10) FIXME on sending identity packets // DONE --- 12) FIXME _maxKbpsPerNode // DONE --- 11) FIXME ++_sumListeners; + // DONE --- 14) fix toByteArray() virtual hiding!!! // // 4) Error in PacketList::writeData - attempted to write a segment to an unordered packet that is larger than the payload size. // 4b) some kind of a better approach to handling otherAvatar.toByteArray() for content that is larger than MTU diff --git a/assignment-client/src/avatars/ScriptableAvatar.cpp b/assignment-client/src/avatars/ScriptableAvatar.cpp index cdda070da2..516bf7a1e3 100644 --- a/assignment-client/src/avatars/ScriptableAvatar.cpp +++ b/assignment-client/src/avatars/ScriptableAvatar.cpp @@ -14,9 +14,9 @@ #include #include "ScriptableAvatar.h" -QByteArray ScriptableAvatar::toByteArray(AvatarDataDetail dataDetail) { +QByteArray ScriptableAvatar::toByteArrayStateful(AvatarDataDetail dataDetail) { _globalPosition = getPosition(); - return AvatarData::toByteArray(dataDetail); + return AvatarData::toByteArrayStateful(dataDetail); } diff --git a/assignment-client/src/avatars/ScriptableAvatar.h b/assignment-client/src/avatars/ScriptableAvatar.h index b6804da43d..1028912e55 100644 --- a/assignment-client/src/avatars/ScriptableAvatar.h +++ b/assignment-client/src/avatars/ScriptableAvatar.h @@ -28,7 +28,7 @@ public: Q_INVOKABLE AnimationDetails getAnimationDetails(); virtual void setSkeletonModelURL(const QUrl& skeletonModelURL) override; - virtual QByteArray toByteArray(AvatarDataDetail dataDetail) override; + virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail) override; private slots: From 42d916a719b41def5b237de556bd00714e19a763 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 17 Feb 2017 21:58:48 -0800 Subject: [PATCH 065/104] fix warnings --- interface/src/avatar/MyAvatar.cpp | 6 +++--- interface/src/avatar/MyAvatar.h | 2 +- libraries/avatars/src/AvatarData.cpp | 4 ++-- libraries/avatars/src/AvatarData.h | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 21b75e4e71..fa3ddb6bbd 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -227,7 +227,7 @@ void MyAvatar::simulateAttachments(float deltaTime) { // don't update attachments here, do it in harvestResultsFromPhysicsSimulation() } -QByteArray MyAvatar::toByteArray(AvatarDataDetail dataDetail) { +QByteArray MyAvatar::toByteArrayStateful(AvatarDataDetail dataDetail) { CameraMode mode = qApp->getCamera()->getMode(); _globalPosition = getPosition(); _globalBoundingBoxDimensions.x = _characterController.getCapsuleRadius(); @@ -238,12 +238,12 @@ QByteArray MyAvatar::toByteArray(AvatarDataDetail dataDetail) { // fake the avatar position that is sent up to the AvatarMixer glm::vec3 oldPosition = getPosition(); setPosition(getSkeletonPosition()); - QByteArray array = AvatarData::toByteArray(dataDetail); + QByteArray array = AvatarData::toByteArrayStateful(dataDetail); // copy the correct position back setPosition(oldPosition); return array; } - return AvatarData::toByteArray(dataDetail); + return AvatarData::toByteArrayStateful(dataDetail); } void MyAvatar::centerBody() { diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 982f0505ff..d94d1088e2 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -338,7 +338,7 @@ private: glm::quat getWorldBodyOrientation() const; - virtual QByteArray toByteArray(AvatarDataDetail dataDetail) override; + virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail) override; void simulate(float deltaTime); void updateFromTrackers(float deltaTime); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index f48b31881a..2263964e8f 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -169,7 +169,7 @@ float AvatarData::getDistanceBasedMinTranslationDistance(glm::vec3 viewerPositio // we want to track outbound data in this case... -QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail) { +QByteArray AvatarData::toByteArrayStateful(AvatarDataDetail dataDetail) { AvatarDataPacket::HasFlags hasFlagsOut; auto lastSentTime = _lastToByteArray; _lastToByteArray = usecTimestampNow(); @@ -1637,7 +1637,7 @@ void AvatarData::sendAvatarDataPacket() { bool cullSmallData = (randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO); auto dataDetail = cullSmallData ? SendAllData : CullSmallData; - QByteArray avatarByteArray = toByteArray(dataDetail); + QByteArray avatarByteArray = toByteArrayStateful(dataDetail); doneEncoding(cullSmallData); static AvatarDataSequenceNumber sequenceNumber = 0; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 14348b0622..2e034073b3 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -368,7 +368,7 @@ public: SendAllData } AvatarDataDetail; - virtual QByteArray toByteArray(AvatarDataDetail dataDetail); + virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail); // FIXME virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, From 71af81851efad3e8dc80b79ab7421d18ec4e9c73 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 17 Feb 2017 22:20:32 -0800 Subject: [PATCH 066/104] migrate to new style throttling --- assignment-client/src/avatars/AvatarMixer.cpp | 148 ++++++++---------- assignment-client/src/avatars/AvatarMixer.h | 9 +- 2 files changed, 66 insertions(+), 91 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 6c04ca8799..da0289a08f 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -111,43 +111,43 @@ void AvatarMixer::start() { auto nodeList = DependencyManager::get(); + unsigned int frame = 1; auto frameTimestamp = p_high_resolution_clock::now(); while (!_isFinished) { - _numTightLoopFrames++; - _loopRate.increment(); ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // WORK ITEMS... // - // DONE --- 1) only sleep for remainder - // DONE --- 2) clean up stats, add slave stats - // DONE --- 3) out of view??? is it broken? - verified - it's working - // DONE --- 4a) hack to not send face data mostly seems to work... - // DONE --- 5) fix two different versions of toByteArray() - // DONE --- 7) audit the locking and side-effects to node, otherNode, and nodeData - // DONE --- 8) delete dead code from mixer (now that it's in slave) - // DONE --- 10) FIXME on sending identity packets - // DONE --- 12) FIXME _maxKbpsPerNode - // DONE --- 11) FIXME ++_sumListeners; - // DONE --- 14) fix toByteArray() virtual hiding!!! + // DONE --- only sleep for remainder + // DONE --- clean up stats, add slave stats + // DONE --- out of view??? is it broken? - verified - it's working + // DONE --- hack to not send face data mostly seems to work... + // DONE --- fix two different versions of toByteArray() + // DONE --- audit the locking and side-effects to node, otherNode, and nodeData + // DONE --- delete dead code from mixer (now that it's in slave) + // DONE --- FIXME on sending identity packets + // DONE --- FIXME _maxKbpsPerNode + // DONE --- FIXME ++_sumListeners; + // DONE --- fix toByteArray() virtual hiding!!! // - // 4) Error in PacketList::writeData - attempted to write a segment to an unordered packet that is larger than the payload size. - // 4b) some kind of a better approach to handling otherAvatar.toByteArray() for content that is larger than MTU - // 6) CPU throttling?? - // 9) better stats in the nodes: + // 1) CPU throttling - now we're calculating it (like audio mixer, how to use it???) + // + // 2) Error in PacketList::writeData - attempted to write a segment to an unordered packet that is larger than the payload size. + // 2b) some kind of a better approach to handling otherAvatar.toByteArray() for content that is larger than MTU + // 3) better stats in the nodes: // how many avatars are actually "in view" for the avtar in question (even if they are over bandwidth budget) - // 13) FIXME -- otherNodeData->incrementNumOutOfOrderSends(); - // + // 4) FIXME -- otherNodeData->incrementNumOutOfOrderSends(); + // 5) average_identity_packets_per_frame??? // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // calculates last frame duration and sleeps for the remainder of the target amount auto frameDuration = timeFrame(frameTimestamp); - Q_UNUSED(frameDuration); + throttle(frameDuration, frame); int lockWait, nodeTransform, functor; @@ -196,6 +196,9 @@ void AvatarMixer::start() { _broadcastAvatarDataNodeFunctor += functor; } + ++frame; + ++_numTightLoopFrames; + _loopRate.increment(); // play nice with qt event-looping { @@ -251,81 +254,52 @@ void AvatarMixer::manageDisplayName(const SharedNodePointer& node) { } } -// FIXME -- this is dead code... it needs to be removed... -// this "throttle" logic is the old approach. need to consider some -// reasonable throttle approach in new multi-core design -void AvatarMixer::broadcastAvatarData() { - int idleTime = AVATAR_DATA_SEND_INTERVAL_MSECS; +void AvatarMixer::throttle(std::chrono::microseconds duration, int frame) { + // throttle using a modified proportional-integral controller + const float FRAME_TIME = USECS_PER_SECOND / AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND; + float mixRatio = duration.count() / FRAME_TIME; - if (_lastFrameTimestamp.time_since_epoch().count() > 0) { - auto idleDuration = p_high_resolution_clock::now() - _lastFrameTimestamp; - idleTime = std::chrono::duration_cast(idleDuration).count(); - } + // constants are determined based on a "regular" 16-CPU EC2 server - const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; - const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; + // target different mix and backoff ratios (they also have different backoff rates) + // this is to prevent oscillation, and encourage throttling to find a steady state + const float TARGET = 0.9f; + // on a "regular" machine with 100 avatars, this is the largest value where + // - overthrottling can be recovered + // - oscillations will not occur after the recovery + const float BACKOFF_TARGET = 0.44f; - const float RATIO_BACK_OFF = 0.02f; + // the mixer is known to struggle at about 80 on a "regular" machine + // so throttle 2/80 the streams to ensure smooth audio (throttling is linear) + const float THROTTLE_RATE = 2 / 80.0f; + const float BACKOFF_RATE = THROTTLE_RATE / 4; - const int TRAILING_AVERAGE_FRAMES = 100; - int framesSinceCutoffEvent = TRAILING_AVERAGE_FRAMES; + // recovery should be bounded so that large changes in user count is a tolerable experience + // throttling is linear, so most cases will not need a full recovery + const int RECOVERY_TIME = 180; - const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; + // weight more recent frames to determine if throttling is necessary, + const int TRAILING_FRAMES = (int)(100 * RECOVERY_TIME * BACKOFF_RATE); + const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_FRAMES; const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; + _trailingMixRatio = PREVIOUS_FRAMES_RATIO * _trailingMixRatio + CURRENT_FRAME_RATIO * mixRatio; - // NOTE: The following code calculates the _performanceThrottlingRatio based on how much the avatar-mixer was - // able to sleep. This will eventually be used to ask for an additional avatar-mixer to help out. Currently the value - // is unused as it is assumed this should not be hit before the avatar-mixer hits the desired bandwidth limit per client. - // It is reported in the domain-server stats for the avatar-mixer. - - _trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio) - + (idleTime * CURRENT_FRAME_RATIO / (float) AVATAR_DATA_SEND_INTERVAL_MSECS); - - float lastCutoffRatio = _performanceThrottlingRatio; - bool hasRatioChanged = false; - - if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) { - if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) { - // we're struggling - change our performance throttling ratio - _performanceThrottlingRatio = _performanceThrottlingRatio + (0.5f * (1.0f - _performanceThrottlingRatio)); - - qDebug() << "Mixer is struggling, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" - << lastCutoffRatio << "and is now" << _performanceThrottlingRatio; - hasRatioChanged = true; - } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _performanceThrottlingRatio != 0) { - // we've recovered and can back off the performance throttling - _performanceThrottlingRatio = _performanceThrottlingRatio - RATIO_BACK_OFF; - - if (_performanceThrottlingRatio < 0) { - _performanceThrottlingRatio = 0; - } - - qDebug() << "Mixer is recovering, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" - << lastCutoffRatio << "and is now" << _performanceThrottlingRatio; - hasRatioChanged = true; + if (frame % TRAILING_FRAMES == 0) { + if (_trailingMixRatio > TARGET) { + int proportionalTerm = 1 + (_trailingMixRatio - TARGET) / 0.1f; + _throttlingRatio += THROTTLE_RATE * proportionalTerm; + _throttlingRatio = std::min(_throttlingRatio, 1.0f); + qDebug("avatar-mixer is struggling (%f mix/sleep) - throttling %f of streams", + (double)_trailingMixRatio, (double)_throttlingRatio); } - - if (hasRatioChanged) { - framesSinceCutoffEvent = 0; + else if (_throttlingRatio > 0.0f && _trailingMixRatio <= BACKOFF_TARGET) { + int proportionalTerm = 1 + (TARGET - _trailingMixRatio) / 0.2f; + _throttlingRatio -= BACKOFF_RATE * proportionalTerm; + _throttlingRatio = std::max(_throttlingRatio, 0.0f); + qDebug("avatar-mixer is recovering (%f mix/sleep) - throttling %f of streams", + (double)_trailingMixRatio, (double)_throttlingRatio); } } - - if (!hasRatioChanged) { - ++framesSinceCutoffEvent; - } - - _lastFrameTimestamp = p_high_resolution_clock::now(); - -#ifdef WANT_DEBUG - auto sinceLastDebug = p_high_resolution_clock::now() - _lastDebugMessage; - auto sinceLastDebugUsecs = std::chrono::duration_cast(sinceLastDebug).count(); - quint64 DEBUG_INTERVAL = USECS_PER_SECOND * 5; - - if (sinceLastDebugUsecs > DEBUG_INTERVAL) { - qDebug() << "broadcast rate:" << _broadcastRate.rate() << "hz"; - _lastDebugMessage = p_high_resolution_clock::now(); - } -#endif } void AvatarMixer::nodeKilled(SharedNodePointer killedNode) { @@ -469,8 +443,8 @@ void AvatarMixer::sendStatsPacket() { statsObject["broadcast_loop_rate"] = _loopRate.rate(); statsObject["threads"] = _slavePool.numThreads(); - statsObject["throttling_1_trailing_sleep_percentage"] = _trailingSleepRatio * 100; - statsObject["throttling_2_performance_ratio"] = _performanceThrottlingRatio; + statsObject["trailing_mix_ratio"] = _trailingMixRatio; + statsObject["throttling_ratio"] = _throttlingRatio; // this things all occur on the frequency of the tight loop int tightLoopFrames = _numTightLoopFrames; diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index 7a79ec1d57..f03a47dbd8 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -53,9 +53,8 @@ private slots: private: AvatarMixerClientData* getOrCreateClientData(SharedNodePointer node); std::chrono::microseconds timeFrame(p_high_resolution_clock::time_point& timestamp); + void throttle(std::chrono::microseconds duration, int frame); - - void broadcastAvatarData(); void parseDomainServerSettings(const QJsonObject& domainSettings); void sendIdentityPacket(AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode); @@ -63,8 +62,10 @@ private: p_high_resolution_clock::time_point _lastFrameTimestamp; - float _trailingSleepRatio { 1.0f }; - float _performanceThrottlingRatio { 0.0f }; + // FIXME - new throttling - use these values somehow + float _trailingMixRatio { 0.0f }; + float _throttlingRatio { 0.0f }; + int _sumListeners { 0 }; int _numStatFrames { 0 }; From 66a6666b5298738c2d1b6e6c1af8301725db1c58 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Sat, 18 Feb 2017 12:29:01 -0800 Subject: [PATCH 067/104] more cleanup, improved stats, port throttling --- assignment-client/src/avatars/AvatarMixer.cpp | 83 ++++++++++--------- assignment-client/src/avatars/AvatarMixer.h | 1 + .../src/avatars/AvatarMixerClientData.cpp | 1 - .../src/avatars/AvatarMixerSlave.cpp | 57 +++++++++---- .../src/avatars/AvatarMixerSlave.h | 25 +++++- .../src/avatars/AvatarMixerSlavePool.cpp | 6 +- .../src/avatars/AvatarMixerSlavePool.h | 2 +- libraries/avatars/src/AvatarData.h | 1 - 8 files changed, 114 insertions(+), 62 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index da0289a08f..8ba7181e9c 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -9,6 +9,15 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// WORK ITEMS... +// +// 1) FIXME in AvatarMixerSlave.cpp -- otherNodeData->incrementNumOutOfOrderSends(); +// This code appears to be determining if a node sent out of order packets, that logic should not be in +// the broadcast method, but would make more sense in the incoming packet processing section +// +////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + #include #include #include @@ -116,38 +125,8 @@ void AvatarMixer::start() { while (!_isFinished) { - ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // WORK ITEMS... - // - // DONE --- only sleep for remainder - // DONE --- clean up stats, add slave stats - // DONE --- out of view??? is it broken? - verified - it's working - // DONE --- hack to not send face data mostly seems to work... - // DONE --- fix two different versions of toByteArray() - // DONE --- audit the locking and side-effects to node, otherNode, and nodeData - // DONE --- delete dead code from mixer (now that it's in slave) - // DONE --- FIXME on sending identity packets - // DONE --- FIXME _maxKbpsPerNode - // DONE --- FIXME ++_sumListeners; - // DONE --- fix toByteArray() virtual hiding!!! - // - // 1) CPU throttling - now we're calculating it (like audio mixer, how to use it???) - // - // 2) Error in PacketList::writeData - attempted to write a segment to an unordered packet that is larger than the payload size. - // 2b) some kind of a better approach to handling otherAvatar.toByteArray() for content that is larger than MTU - // 3) better stats in the nodes: - // how many avatars are actually "in view" for the avtar in question (even if they are over bandwidth budget) - // 4) FIXME -- otherNodeData->incrementNumOutOfOrderSends(); - // 5) average_identity_packets_per_frame??? - // - ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - - // calculates last frame duration and sleeps for the remainder of the target amount - auto frameDuration = timeFrame(frameTimestamp); - throttle(frameDuration, frame); + auto frameDuration = timeFrame(frameTimestamp); // calculates last frame duration and sleeps remainder of target amount + throttle(frameDuration, frame); // determines _throttlingRatio for upcoming mix frame int lockWait, nodeTransform, functor; @@ -184,7 +163,7 @@ void AvatarMixer::start() { auto start = usecTimestampNow(); nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { auto start = usecTimestampNow(); - _slavePool.broadcastAvatarData(cbegin, cend, _lastFrameTimestamp, _maxKbpsPerNode); + _slavePool.broadcastAvatarData(cbegin, cend, _lastFrameTimestamp, _maxKbpsPerNode, _throttlingRatio); auto end = usecTimestampNow(); _broadcastAvatarDataInner += (end - start); }, &lockWait, &nodeTransform, &functor); @@ -497,15 +476,28 @@ void AvatarMixer::sendStatsPacket() { AvatarMixerSlaveStats aggregateStats; QJsonObject slavesObject; + + float secondsSinceLastStats = (float)(start - _lastStatsTime) / (float)USECS_PER_SECOND; // gather stats int slaveNumber = 1; _slavePool.each([&](AvatarMixerSlave& slave) { QJsonObject slaveObject; AvatarMixerSlaveStats stats; slave.harvestStats(stats); - slaveObject["nodesProcessed"] = TIGHT_LOOP_STAT(stats.nodesProcessed); - slaveObject["numPacketsReceived"] = TIGHT_LOOP_STAT(stats.packetsProcessed); - slaveObject["numPacketsSent"] = TIGHT_LOOP_STAT(stats.numPacketsSent); + slaveObject["recevied_1_nodesProcessed"] = TIGHT_LOOP_STAT(stats.nodesProcessed); + slaveObject["received_2_numPacketsReceived"] = TIGHT_LOOP_STAT(stats.packetsProcessed); + + slaveObject["sent_1_nodesBroadcastedTo"] = TIGHT_LOOP_STAT(stats.nodesBroadcastedTo); + slaveObject["sent_2_numBytesSent"] = TIGHT_LOOP_STAT(stats.numBytesSent); + slaveObject["sent_3_numPacketsSent"] = TIGHT_LOOP_STAT(stats.numPacketsSent); + slaveObject["sent_4_numIdentityPackets"] = TIGHT_LOOP_STAT(stats.numIdentityPackets); + + float averageNodes = ((float)stats.nodesBroadcastedTo / (float)tightLoopFrames); + float averageOutboundAvatarKbps = averageNodes ? ((stats.numBytesSent / secondsSinceLastStats) / BYTES_PER_KILOBIT) / averageNodes : 0.0f; + slaveObject["sent_5_averageOutboundAvatarKbps"] = averageOutboundAvatarKbps; + + float averageOthersIncluded = averageNodes ? stats.numOthersIncluded / averageNodes : 0.0f; + slaveObject["sent_6_averageOthersIncluded"] = TIGHT_LOOP_STAT(averageOthersIncluded); slaveObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT_UINT64(stats.processIncomingPacketsElapsedTime); slaveObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT_UINT64(stats.ignoreCalculationElapsedTime); @@ -522,10 +514,21 @@ void AvatarMixer::sendStatsPacket() { QJsonObject slavesAggregatObject; - slavesAggregatObject["nodesProcessed"] = TIGHT_LOOP_STAT(aggregateStats.nodesProcessed); - slavesAggregatObject["numPacketsReceived"] = TIGHT_LOOP_STAT(aggregateStats.packetsProcessed); - slavesAggregatObject["numPacketsSent"] = TIGHT_LOOP_STAT(aggregateStats.numPacketsSent); + slavesAggregatObject["recevied_1_nodesProcessed"] = TIGHT_LOOP_STAT(aggregateStats.nodesProcessed); + slavesAggregatObject["received_2_numPacketsReceived"] = TIGHT_LOOP_STAT(aggregateStats.packetsProcessed); + slavesAggregatObject["sent_1_nodesBroadcastedTo"] = TIGHT_LOOP_STAT(aggregateStats.nodesBroadcastedTo); + slavesAggregatObject["sent_2_numBytesSent"] = TIGHT_LOOP_STAT(aggregateStats.numBytesSent); + slavesAggregatObject["sent_3_numPacketsSent"] = TIGHT_LOOP_STAT(aggregateStats.numPacketsSent); + slavesAggregatObject["sent_4_numIdentityPackets"] = TIGHT_LOOP_STAT(aggregateStats.numIdentityPackets); + + float averageNodes = ((float)aggregateStats.nodesBroadcastedTo / (float)tightLoopFrames); + float averageOutboundAvatarKbps = averageNodes ? ((aggregateStats.numBytesSent / secondsSinceLastStats) / BYTES_PER_KILOBIT) / averageNodes : 0.0f; + slavesAggregatObject["sent_5_averageOutboundAvatarKbps"] = averageOutboundAvatarKbps; + + float averageOthersIncluded = averageNodes ? aggregateStats.numOthersIncluded / averageNodes : 0.0f; + slavesAggregatObject["sent_6_averageOthersIncluded"] = TIGHT_LOOP_STAT(averageOthersIncluded); + slavesAggregatObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.processIncomingPacketsElapsedTime); slavesAggregatObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.ignoreCalculationElapsedTime); slavesAggregatObject["timing_3_toByteArray"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.toByteArrayElapsedTime); @@ -600,6 +603,8 @@ void AvatarMixer::sendStatsPacket() { auto end = usecTimestampNow(); _sendStatsElapsedTime = (end - start); + _lastStatsTime = start; + } void AvatarMixer::run() { diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index f03a47dbd8..32b0ffed69 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -104,6 +104,7 @@ private: quint64 _processEventsElapsedTime { 0 }; quint64 _sendStatsElapsedTime { 0 }; quint64 _queueIncomingPacketElapsedTime { 0 }; + quint64 _lastStatsTime { usecTimestampNow() }; RateCounter<> _loopRate; // this is the rate that the main thread tight loop runs diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index 6fdd2fd23e..b5d4e390bb 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -58,7 +58,6 @@ int AvatarMixerClientData::parseData(ReceivedMessage& message) { return _avatar->parseDataFromBuffer(message.readWithoutCopy(message.getBytesLeftToRead())); } -// FIXME -- this needs a mutex in new model. bool AvatarMixerClientData::checkAndSetHasReceivedFirstPacketsFrom(const QUuid& uuid) { if (_hasReceivedFirstPacketsFrom.find(uuid) == _hasReceivedFirstPacketsFrom.end()) { _hasReceivedFirstPacketsFrom.insert(uuid); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index fb67cbf64e..3ef46eef9c 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -10,6 +10,7 @@ // #include +#include #include #include @@ -37,11 +38,12 @@ void AvatarMixerSlave::configure(ConstIter begin, ConstIter end) { void AvatarMixerSlave::configureBroadcast(ConstIter begin, ConstIter end, p_high_resolution_clock::time_point lastFrameTimestamp, - float maxKbpsPerNode) { + float maxKbpsPerNode, float throttlingRatio) { _begin = begin; _end = end; _lastFrameTimestamp = lastFrameTimestamp; _maxKbpsPerNode = maxKbpsPerNode; + _throttlingRatio = throttlingRatio; } void AvatarMixerSlave::harvestStats(AvatarMixerSlaveStats& stats) { @@ -61,8 +63,15 @@ void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { _stats.processIncomingPacketsElapsedTime += (end - start); } -#include -#include + +void AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { + QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); + auto identityPacket = NLPacket::create(PacketType::AvatarIdentity, individualData.size()); + individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); + identityPacket->write(individualData); + DependencyManager::get()->sendPacket(std::move(identityPacket), *destinationNode); + _stats.numIdentityPackets++; +} static const int AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND = 45; @@ -88,10 +97,9 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { std::uniform_real_distribution distribution; if (node->getLinkedData() && (node->getType() == NodeType::Agent) && node->getActiveSocket()) { - AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); + _stats.nodesBroadcastedTo++; - // FIXME -- mixer data - // ++_sumListeners; + AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); nodeData->resetInViewStats(); @@ -249,11 +257,7 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp || distribution(generator) < IDENTITY_SEND_PROBABILITY)) { - QByteArray individualData = otherNodeData->getConstAvatarData()->identityByteArray(); - auto identityPacket = NLPacket::create(PacketType::AvatarIdentity, individualData.size()); - individualData.replace(0, NUM_BYTES_RFC4122_UUID, otherNodeData->getNodeID().toRfc4122()); - identityPacket->write(individualData); - DependencyManager::get()->sendPacket(std::move(identityPacket), *node); + sendIdentityPacket(otherNodeData, node); } const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); @@ -339,13 +343,13 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { } { - numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); + bool includeThisAvatar = true; auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); bool distanceAdjust = true; glm::vec3 viewerPosition = myPosition; AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray - bool dropFaceTracking = true; // this is a hack for now... always drop face tracking + bool dropFaceTracking = false; quint64 start = usecTimestampNow(); QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, @@ -353,10 +357,30 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { quint64 end = usecTimestampNow(); _stats.toByteArrayElapsedTime += (end - start); - if (bytes.size() > 1400) { - qDebug() << "WARNING: otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size(); - } else { + static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID); + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qDebug() << "WARNING: otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data"; + + dropFaceTracking = true; // first try dropping the facial data + bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qDebug() << "WARNING: otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData"; + bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther, + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + } + + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qDebug() << "WARNING: otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!"; + includeThisAvatar = false; + } + } + + if (includeThisAvatar) { + numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); numAvatarDataBytes += avatarPacketList->write(bytes); + _stats.numOthersIncluded++; } } @@ -376,6 +400,7 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { avatarPacketList->closeCurrentPacket(true); _stats.numPacketsSent += (int)avatarPacketList->getNumPackets(); + _stats.numBytesSent += numAvatarDataBytes; // send the avatar data PacketList //qDebug() << "about to call nodeList->sendPacketList() for node:" << node; diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index 9382fcfeda..00948746ec 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -12,13 +12,19 @@ #ifndef hifi_AvatarMixerSlave_h #define hifi_AvatarMixerSlave_h +class AvatarMixerClientData; + class AvatarMixerSlaveStats { public: int nodesProcessed { 0 }; int packetsProcessed { 0 }; quint64 processIncomingPacketsElapsedTime { 0 }; + int nodesBroadcastedTo { 0 }; int numPacketsSent { 0 }; + int numBytesSent { 0 }; + int numIdentityPackets { 0 }; + int numOthersIncluded { 0 }; quint64 ignoreCalculationElapsedTime { 0 }; quint64 avatarDataPackingElapsedTime { 0 }; quint64 packetSendingElapsedTime { 0 }; @@ -29,16 +35,22 @@ public: // receiving job stats nodesProcessed = 0; packetsProcessed = 0; - numPacketsSent = 0; processIncomingPacketsElapsedTime = 0; // sending job stats + nodesBroadcastedTo = 0; numPacketsSent = 0; + numBytesSent = 0; + numIdentityPackets = 0; + numOthersIncluded = 0; ignoreCalculationElapsedTime = 0; avatarDataPackingElapsedTime = 0; packetSendingElapsedTime = 0; toByteArrayElapsedTime = 0; jobElapsedTime = 0; + + //qDebug() << "reset!!! " << "_stats.numBytesSent:" << numBytesSent << "_stats.nodesBroadcastedTo:" << nodesBroadcastedTo; + } AvatarMixerSlaveStats& operator+=(const AvatarMixerSlaveStats& rhs) { @@ -46,7 +58,11 @@ public: packetsProcessed += rhs.packetsProcessed; processIncomingPacketsElapsedTime += rhs.processIncomingPacketsElapsedTime; + nodesBroadcastedTo += rhs.nodesBroadcastedTo; numPacketsSent += rhs.numPacketsSent; + numBytesSent += rhs.numBytesSent; + numIdentityPackets += rhs.numIdentityPackets; + numOthersIncluded += rhs.numOthersIncluded; ignoreCalculationElapsedTime += rhs.ignoreCalculationElapsedTime; avatarDataPackingElapsedTime += rhs.avatarDataPackingElapsedTime; packetSendingElapsedTime += rhs.packetSendingElapsedTime; @@ -62,7 +78,9 @@ public: using ConstIter = NodeList::const_iterator; void configure(ConstIter begin, ConstIter end); - void configureBroadcast(ConstIter begin, ConstIter end, p_high_resolution_clock::time_point lastFrameTimestamp, float maxKbpsPerNode); + void configureBroadcast(ConstIter begin, ConstIter end, + p_high_resolution_clock::time_point lastFrameTimestamp, + float maxKbpsPerNode, float throttlingRatio); void processIncomingPackets(const SharedNodePointer& node); void broadcastAvatarData(const SharedNodePointer& node); @@ -70,12 +88,15 @@ public: void harvestStats(AvatarMixerSlaveStats& stats); private: + void sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode); + // frame state ConstIter _begin; ConstIter _end; p_high_resolution_clock::time_point _lastFrameTimestamp; float _maxKbpsPerNode { 0.0f }; + float _throttlingRatio { 0.0f }; AvatarMixerSlaveStats _stats; }; diff --git a/assignment-client/src/avatars/AvatarMixerSlavePool.cpp b/assignment-client/src/avatars/AvatarMixerSlavePool.cpp index c0dcf9cbba..07d4fa8851 100644 --- a/assignment-client/src/avatars/AvatarMixerSlavePool.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlavePool.cpp @@ -75,10 +75,12 @@ void AvatarMixerSlavePool::processIncomingPackets(ConstIter begin, ConstIter end run(begin, end); } -void AvatarMixerSlavePool::broadcastAvatarData(ConstIter begin, ConstIter end, p_high_resolution_clock::time_point lastFrameTimestamp, float maxKbpsPerNode) { +void AvatarMixerSlavePool::broadcastAvatarData(ConstIter begin, ConstIter end, + p_high_resolution_clock::time_point lastFrameTimestamp, + float maxKbpsPerNode, float throttlingRatio) { _function = &AvatarMixerSlave::broadcastAvatarData; _configure = [&](AvatarMixerSlave& slave) { - slave.configureBroadcast(begin, end, lastFrameTimestamp, maxKbpsPerNode); + slave.configureBroadcast(begin, end, lastFrameTimestamp, maxKbpsPerNode, throttlingRatio); }; run(begin, end); } diff --git a/assignment-client/src/avatars/AvatarMixerSlavePool.h b/assignment-client/src/avatars/AvatarMixerSlavePool.h index e54681401d..6bef0515bb 100644 --- a/assignment-client/src/avatars/AvatarMixerSlavePool.h +++ b/assignment-client/src/avatars/AvatarMixerSlavePool.h @@ -66,7 +66,7 @@ public: // Jobs the slave pool can do... void processIncomingPackets(ConstIter begin, ConstIter end); void broadcastAvatarData(ConstIter begin, ConstIter end, - p_high_resolution_clock::time_point lastFrameTimestamp, float maxKbpsPerNode); + p_high_resolution_clock::time_point lastFrameTimestamp, float maxKbpsPerNode, float throttlingRatio); // iterate over all slaves void each(std::function functor); diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 2e034073b3..264da75de2 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -370,7 +370,6 @@ public: virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail); - // FIXME virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut, AvatarDataRate* outboundDataRateOut = nullptr) const; From e95e7f663ce3933414a65fcfb222a6bd92c5c532 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Sat, 18 Feb 2017 13:04:00 -0800 Subject: [PATCH 068/104] more cleanup --- assignment-client/src/avatars/AvatarMixer.cpp | 28 ++++--------- .../src/avatars/AvatarMixerClientData.cpp | 10 ++++- .../src/avatars/AvatarMixerSlave.cpp | 42 +++++++++++-------- 3 files changed, 41 insertions(+), 39 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 8ba7181e9c..3978768907 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -9,18 +9,11 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// WORK ITEMS... -// -// 1) FIXME in AvatarMixerSlave.cpp -- otherNodeData->incrementNumOutOfOrderSends(); -// This code appears to be determining if a node sent out of order packets, that logic should not be in -// the broadcast method, but would make more sense in the incoming packet processing section -// -////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - #include -#include +#include #include +#include +#include #include #include @@ -93,9 +86,6 @@ void AvatarMixer::sendIdentityPacket(AvatarMixerClientData* nodeData, const Shar ++_sumIdentityPackets; } -#include -#include - std::chrono::microseconds AvatarMixer::timeFrame(p_high_resolution_clock::time_point& timestamp) { // advance the next frame auto nextTimestamp = timestamp + std::chrono::microseconds((int)((float)USECS_PER_SECOND / (float)AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND)); @@ -199,7 +189,7 @@ void AvatarMixer::start() { } -// NOTE: nodeData->getAvatar() might be side effected, most be called when access to node/nodeData +// NOTE: nodeData->getAvatar() might be side effected, must be called when access to node/nodeData // is guarenteed to not be accessed by other thread void AvatarMixer::manageDisplayName(const SharedNodePointer& node) { AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); @@ -248,9 +238,10 @@ void AvatarMixer::throttle(std::chrono::microseconds duration, int frame) { // - oscillations will not occur after the recovery const float BACKOFF_TARGET = 0.44f; - // the mixer is known to struggle at about 80 on a "regular" machine - // so throttle 2/80 the streams to ensure smooth audio (throttling is linear) - const float THROTTLE_RATE = 2 / 80.0f; + // the mixer is known to struggle at about 150 on a "regular" machine + // so throttle 2/150 the streams to ensure smooth mixing (throttling is linear) + const float STRUGGLES_AT = 150.0f; + const float THROTTLE_RATE = 2 / STRUGGLES_AT; const float BACKOFF_RATE = THROTTLE_RATE / 4; // recovery should be bounded so that large changes in user count is a tolerable experience @@ -417,10 +408,7 @@ void AvatarMixer::sendStatsPacket() { QJsonObject statsObject; - //statsObject["average_identity_packets_per_frame"] = (float) _sumIdentityPackets / (float) _numStatFrames; - statsObject["broadcast_loop_rate"] = _loopRate.rate(); - statsObject["threads"] = _slavePool.numThreads(); statsObject["trailing_mix_ratio"] = _trailingMixRatio; statsObject["throttling_ratio"] = _throttlingRatio; diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index b5d4e390bb..df6a64e874 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -51,9 +51,17 @@ int AvatarMixerClientData::processPackets() { } int AvatarMixerClientData::parseData(ReceivedMessage& message) { + // pull the sequence number from the data first - message.readPrimitive(&_lastReceivedSequenceNumber); + uint16_t sequenceNumber; + + message.readPrimitive(&sequenceNumber); + if (sequenceNumber < _lastReceivedSequenceNumber && _lastReceivedSequenceNumber != UINT16_MAX) { + incrementNumOutOfOrderSends(); + } + _lastReceivedSequenceNumber = sequenceNumber; + // compute the offset to the data payload return _avatar->parseDataFromBuffer(message.readWithoutCopy(message.getBytesLeftToRead())); } diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 3ef46eef9c..bc5ed355f9 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -198,7 +198,9 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { } else { const AvatarMixerClientData* otherData = reinterpret_cast(otherNode->getLinkedData()); - //AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); + + shouldConsider = true; // assume we will consider... + // Check to see if the space bubble is enabled if (node->isIgnoreRadiusEnabled() || otherNode->isIgnoreRadiusEnabled()) { // Define the minimum bubble size @@ -235,8 +237,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { nodeData->removeFromRadiusIgnoringSet(node, otherNode->getUUID()); } - shouldConsider = true; - quint64 endIgnoreCalculation = usecTimestampNow(); _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); } @@ -271,7 +271,11 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // potentially update the max full rate distance for this frame maxAvatarDistanceThisFrame = std::max(maxAvatarDistanceThisFrame, distanceToAvatar); - // FIXME-- understand this code... WHAT IS IT DOING!!! + // This code handles the random dropping of avatar data based on the ratio of + // "getFullRateDistance" to actual distance. + // + // NOTE: If the recieving node is in "PAL mode" then it's asked to get things even that + // are out of view, this also appears to disable this random distribution. if (distanceToAvatar != 0.0f && !getsOutOfView && distribution(generator) > (nodeData->getFullRateDistance() / distanceToAvatar)) { @@ -285,14 +289,14 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(otherNode->getUUID()); AvatarDataSequenceNumber lastSeqFromSender = otherNodeData->getLastReceivedSequenceNumber(); - /*** - // FIXME this does not belong here... it should be in the packet processing - if (lastSeqToReceiver > lastSeqFromSender && lastSeqToReceiver != UINT16_MAX) { - // we got out out of order packets from the sender, track it - otherNodeData->incrementNumOutOfOrderSends(); - } - ***/ - + // FIXME - This code does appear to be working. But it seems brittle. + // It supports determining if the frame of data for this "other" + // avatar has already been sent to the reciever. This has been + // verified to work on a desktop display that renders at 60hz and + // therefore sends to mixer at 30hz. Each second you'd expect to + // have 15 (45hz-30hz) duplicate frames. In this case, the stat + // avg_other_av_skips_per_second does report 15. + // // make sure we haven't already sent this data from this sender to this receiver // or that somehow we haven't sent if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { @@ -308,12 +312,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // we're going to send this avatar if (shouldConsider) { - // increment the number of avatars sent to this reciever - nodeData->incrementNumAvatarsSentLastFrame(); // FIXME - this seems weird... - - // set the last sent sequence number for this sender on the receiver - nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), - otherNodeData->getLastReceivedSequenceNumber()); // determine if avatar is in view, to determine how much data to include... glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; @@ -381,6 +379,14 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); numAvatarDataBytes += avatarPacketList->write(bytes); _stats.numOthersIncluded++; + + // increment the number of avatars sent to this reciever + nodeData->incrementNumAvatarsSentLastFrame(); + + // set the last sent sequence number for this sender on the receiver + nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), + otherNodeData->getLastReceivedSequenceNumber()); + } } From 2b79602220344811de6cf530f3f571460a6af052 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Sat, 18 Feb 2017 13:14:14 -0800 Subject: [PATCH 069/104] debug cleanup --- assignment-client/src/avatars/AvatarMixer.cpp | 21 ++++++++++--------- .../src/avatars/AvatarMixerSlave.cpp | 9 ++++---- .../src/avatars/AvatarMixerSlave.h | 3 --- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 3978768907..93ec0d0e8b 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -219,7 +220,7 @@ void AvatarMixer::manageDisplayName(const SharedNodePointer& node) { nodeData->flagIdentityChange(); nodeData->setAvatarSessionDisplayNameMustChange(false); sendIdentityPacket(nodeData, node); // Tell node whose name changed about its new session display name. Others will find out below. - qDebug() << "Giving session display name" << sessionDisplayName << "to node with ID" << node->getUUID(); + qCDebug(avatars) << "Giving session display name" << sessionDisplayName << "to node with ID" << node->getUUID(); } } @@ -345,7 +346,7 @@ void AvatarMixer::handleRequestsDomainListDataPacket(QSharedPointerreadPrimitive(&isRequesting); nodeData->setRequestsDomainListData(isRequesting); - qDebug() << "node" << nodeData->getNodeID() << "requestsDomainListData" << isRequesting; + qCDebug(avatars) << "node" << nodeData->getNodeID() << "requestsDomainListData" << isRequesting; } } auto end = usecTimestampNow(); @@ -596,7 +597,7 @@ void AvatarMixer::sendStatsPacket() { } void AvatarMixer::run() { - qDebug() << "Waiting for connection to domain to request settings from domain-server."; + qCDebug(avatars) << "Waiting for connection to domain to request settings from domain-server."; // wait until we have the domain-server settings, otherwise we bail DomainHandler& domainHandler = DependencyManager::get()->getDomainHandler(); @@ -656,11 +657,11 @@ void AvatarMixer::parseDomainServerSettings(const QJsonObject& domainSettings) { const float DEFAULT_NODE_SEND_BANDWIDTH = 5.0f; QJsonValue nodeBandwidthValue = avatarMixerGroupObject[NODE_SEND_BANDWIDTH_KEY]; if (!nodeBandwidthValue.isDouble()) { - qDebug() << NODE_SEND_BANDWIDTH_KEY << "is not a double - will continue with default value"; + qCDebug(avatars) << NODE_SEND_BANDWIDTH_KEY << "is not a double - will continue with default value"; } _maxKbpsPerNode = nodeBandwidthValue.toDouble(DEFAULT_NODE_SEND_BANDWIDTH) * KILO_PER_MEGA; - qDebug() << "The maximum send bandwidth per node is" << _maxKbpsPerNode << "kbps."; + qCDebug(avatars) << "The maximum send bandwidth per node is" << _maxKbpsPerNode << "kbps."; const QString AUTO_THREADS = "auto_threads"; bool autoThreads = avatarMixerGroupObject[AUTO_THREADS].toBool(); @@ -669,13 +670,13 @@ void AvatarMixer::parseDomainServerSettings(const QJsonObject& domainSettings) { const QString NUM_THREADS = "num_threads"; int numThreads = avatarMixerGroupObject[NUM_THREADS].toString().toInt(&ok); if (!ok) { - qWarning() << "Avatar mixer: Error reading thread count. Using 1 thread."; + qCWarning(avatars) << "Avatar mixer: Error reading thread count. Using 1 thread."; numThreads = 1; } - qDebug() << "Avatar mixer will use specified number of threads:" << numThreads; + qCDebug(avatars) << "Avatar mixer will use specified number of threads:" << numThreads; _slavePool.setNumThreads(numThreads); } else { - qDebug() << "Avatar mixer will automatically determine number of threads to use. Using:" << _slavePool.numThreads() << "threads."; + qCDebug(avatars) << "Avatar mixer will automatically determine number of threads to use. Using:" << _slavePool.numThreads() << "threads."; } const QString AVATARS_SETTINGS_KEY = "avatars"; @@ -693,7 +694,7 @@ void AvatarMixer::parseDomainServerSettings(const QJsonObject& domainSettings) { std::swap(_domainMinimumScale, _domainMaximumScale); } - qDebug() << "This domain requires a minimum avatar scale of" << _domainMinimumScale - << "and a maximum avatar scale of" << _domainMaximumScale; + qCDebug(avatars) << "This domain requires a minimum avatar scale of" << _domainMinimumScale + << "and a maximum avatar scale of" << _domainMaximumScale; } diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index bc5ed355f9..9c58b1be55 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -26,6 +27,7 @@ #include #include + #include "AvatarMixer.h" #include "AvatarMixerClientData.h" #include "AvatarMixerSlave.h" @@ -357,20 +359,20 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID); if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qDebug() << "WARNING: otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data"; + qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data"; dropFaceTracking = true; // first try dropping the facial data bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qDebug() << "WARNING: otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData"; + qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData"; bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther, hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); } if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qDebug() << "WARNING: otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!"; + qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!"; includeThisAvatar = false; } } @@ -409,7 +411,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { _stats.numBytesSent += numAvatarDataBytes; // send the avatar data PacketList - //qDebug() << "about to call nodeList->sendPacketList() for node:" << node; nodeList->sendPacketList(std::move(avatarPacketList), *node); // record the bytes sent for other avatar data in the AvatarMixerClientData diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index 00948746ec..b1a5b56c4f 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -48,9 +48,6 @@ public: packetSendingElapsedTime = 0; toByteArrayElapsedTime = 0; jobElapsedTime = 0; - - //qDebug() << "reset!!! " << "_stats.numBytesSent:" << numBytesSent << "_stats.nodesBroadcastedTo:" << nodesBroadcastedTo; - } AvatarMixerSlaveStats& operator+=(const AvatarMixerSlaveStats& rhs) { From bc858f82fcd29808b42ad67424d9b3f63b59ed22 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Sat, 18 Feb 2017 14:07:20 -0800 Subject: [PATCH 070/104] fixed a FIXME comment --- assignment-client/src/avatars/AvatarMixerSlave.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 9c58b1be55..46133a221f 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -84,9 +84,15 @@ static const int AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND = 45; // to determine whether the extra data should be sent. static const int EXTRA_AVATAR_DATA_FRAME_RATIO = 16; -// An 80% chance of sending a identity packet within a 5 second interval. -// assuming 60 htz update rate. -const float IDENTITY_SEND_PROBABILITY = 1.0f / 187.0f; // FIXME... this is wrong for 45hz +// FIXME - There is some old logic (unchanged as of 2/17/17) that randomly decides to send an identity +// packet. That logic had the following comment about the constants it uses... +// +// An 80% chance of sending a identity packet within a 5 second interval. +// assuming 60 htz update rate. +// +// Assuming the calculation of the constant is in fact correct for 80% and 60hz and 5 seconds (an assumption +// that I have not verified) then the constant is definitely wrong now, since we send at 45hz. +const float IDENTITY_SEND_PROBABILITY = 1.0f / 187.0f; void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { quint64 start = usecTimestampNow(); From 73d64120dfd48c45fd8ebf05720567fc4ae14f8f Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Sat, 18 Feb 2017 14:08:03 -0800 Subject: [PATCH 071/104] removed dead comment --- assignment-client/src/avatars/AvatarMixer.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 93ec0d0e8b..3b46c96406 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -70,9 +70,6 @@ void AvatarMixer::queueIncomingPacket(QSharedPointer message, S AvatarMixer::~AvatarMixer() { } -// An 80% chance of sending a identity packet within a 5 second interval. -// assuming 60 htz update rate. - void AvatarMixer::sendIdentityPacket(AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { QByteArray individualData = nodeData->getAvatar().identityByteArray(); From 942c5689fcc0975edd24ab4550fbe2582aba3265 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Sat, 18 Feb 2017 14:09:47 -0800 Subject: [PATCH 072/104] more comment cleanup --- assignment-client/src/avatars/AvatarMixer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 3b46c96406..3a8aa0182c 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -216,7 +216,7 @@ void AvatarMixer::manageDisplayName(const SharedNodePointer& node) { soFar.second++; // refcount nodeData->flagIdentityChange(); nodeData->setAvatarSessionDisplayNameMustChange(false); - sendIdentityPacket(nodeData, node); // Tell node whose name changed about its new session display name. Others will find out below. + sendIdentityPacket(nodeData, node); // Tell node whose name changed about its new session display name. qCDebug(avatars) << "Giving session display name" << sessionDisplayName << "to node with ID" << node->getUUID(); } } From 4a650ff2bb81efcdea2c8ff33a19396085266816 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Sun, 19 Feb 2017 18:37:35 -0800 Subject: [PATCH 073/104] Overlays::getOverlayAtPoint was depending on the order in which the overlays were added --- interface/src/ui/overlays/Overlay.h | 5 +++++ interface/src/ui/overlays/Overlays.cpp | 10 ++++++++-- interface/src/ui/overlays/Overlays.h | 1 + .../script-engine/src/TabletScriptingInterface.cpp | 2 +- scripts/tutorials/createDice.js | 4 ++-- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/interface/src/ui/overlays/Overlay.h b/interface/src/ui/overlays/Overlay.h index 0c96af1a99..9ad4f0ba70 100644 --- a/interface/src/ui/overlays/Overlay.h +++ b/interface/src/ui/overlays/Overlay.h @@ -91,6 +91,9 @@ public: render::ItemID getRenderItemID() const { return _renderItemID; } void setRenderItemID(render::ItemID renderItemID) { _renderItemID = renderItemID; } + unsigned int getStackOrder() const { return _stackOrder; } + void setStackOrder(unsigned int stackOrder) { _stackOrder = stackOrder; } + protected: float updatePulse(); @@ -113,6 +116,8 @@ protected: bool _visible; // should the overlay be drawn at all Anchor _anchor; + unsigned int _stackOrder { 0 }; + private: OverlayID _overlayID; // only used for non-3d overlays }; diff --git a/interface/src/ui/overlays/Overlays.cpp b/interface/src/ui/overlays/Overlays.cpp index 6b120cab56..f3a78c9350 100644 --- a/interface/src/ui/overlays/Overlays.cpp +++ b/interface/src/ui/overlays/Overlays.cpp @@ -192,6 +192,7 @@ OverlayID Overlays::addOverlay(Overlay::Pointer overlay) { QWriteLocker lock(&_lock); OverlayID thisID = OverlayID(QUuid::createUuid()); overlay->setOverlayID(thisID); + overlay->setStackOrder(_stackOrder++); if (overlay->is3D()) { _overlaysWorld[thisID] = overlay; @@ -348,6 +349,8 @@ OverlayID Overlays::getOverlayAtPoint(const glm::vec2& point) { BoxFace thisFace; glm::vec3 thisSurfaceNormal; float distance; + unsigned int bestStackOrder = 0; + OverlayID bestOverlayID = UNKNOWN_OVERLAY_ID; while (i.hasPrevious()) { i.previous(); @@ -363,12 +366,15 @@ OverlayID Overlays::getOverlayAtPoint(const glm::vec2& point) { auto thisOverlay = std::dynamic_pointer_cast(i.value()); if (thisOverlay && thisOverlay->getVisible() && thisOverlay->isLoaded() && thisOverlay->getBoundingRect().contains(pointCopy.x, pointCopy.y, false)) { - return thisID; + if (thisOverlay->getStackOrder() > bestStackOrder) { + bestOverlayID = thisID; + bestStackOrder = thisOverlay->getStackOrder(); + } } } } - return UNKNOWN_OVERLAY_ID; // not found + return bestOverlayID; } OverlayPropertyResult Overlays::getProperty(OverlayID id, const QString& property) { diff --git a/interface/src/ui/overlays/Overlays.h b/interface/src/ui/overlays/Overlays.h index 4a4d6f9466..7c6ba34f58 100644 --- a/interface/src/ui/overlays/Overlays.h +++ b/interface/src/ui/overlays/Overlays.h @@ -305,6 +305,7 @@ private: QMap _overlaysWorld; QMap _panels; QList _overlaysToDelete; + unsigned int _stackOrder { 1 }; QReadWriteLock _lock; QReadWriteLock _deleteLock; diff --git a/libraries/script-engine/src/TabletScriptingInterface.cpp b/libraries/script-engine/src/TabletScriptingInterface.cpp index c78ce251c8..d66bb4d2f6 100644 --- a/libraries/script-engine/src/TabletScriptingInterface.cpp +++ b/libraries/script-engine/src/TabletScriptingInterface.cpp @@ -172,7 +172,7 @@ static const char* WEB_VIEW_SOURCE_URL = "TabletWebView.qml"; static const char* VRMENU_SOURCE_URL = "TabletMenu.qml"; class TabletRootWindow : public QmlWindowClass { - virtual QString qmlSource() const { return "hifi/tablet/WindowRoot.qml"; } + virtual QString qmlSource() const override { return "hifi/tablet/WindowRoot.qml"; } }; TabletProxy::TabletProxy(QString name) : _name(name) { diff --git a/scripts/tutorials/createDice.js b/scripts/tutorials/createDice.js index 8975578c66..0d39d11d48 100644 --- a/scripts/tutorials/createDice.js +++ b/scripts/tutorials/createDice.js @@ -33,7 +33,7 @@ var BUTTON_SIZE = 32; var PADDING = 3; var BOTTOM_PADDING = 50; //a helper library for creating toolbars -Script.include("http://hifi-production.s3.amazonaws.com/tutorials/dice/toolBars.js"); +Script.include("/~/system/libraries/toolBars.js"); var toolBar = new ToolBar(0, 0, ToolBar.HORIZONTAL, "highfidelity.toolbars-dice", function(screenSize) { return { @@ -139,4 +139,4 @@ function scriptEnding() { } Controller.mousePressEvent.connect(mousePressEvent); -Script.scriptEnding.connect(scriptEnding); \ No newline at end of file +Script.scriptEnding.connect(scriptEnding); From 3b996f2b7fdcab5e968e9651e41db26cc63372b6 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 21 Feb 2017 09:29:07 -0800 Subject: [PATCH 074/104] bump packet version for new JSON query options --- libraries/networking/src/udt/PacketHeaders.cpp | 2 +- libraries/networking/src/udt/PacketHeaders.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index ddbc30d020..a6718402a3 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -51,7 +51,7 @@ PacketVersion versionForPacketType(PacketType packetType) { case PacketType::EntityPhysics: return VERSION_ENTITIES_ZONE_FILTERS; case PacketType::EntityQuery: - return static_cast(EntityQueryPacketVersion::JsonFilter); + return static_cast(EntityQueryPacketVersion::JSONFilterWithFamilyTree); case PacketType::AvatarIdentity: case PacketType::AvatarData: case PacketType::BulkAvatarData: diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index de3d0369b5..050e3088f8 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -207,7 +207,8 @@ const PacketVersion VERSION_ENTITIES_PHYSICS_PACKET = 67; const PacketVersion VERSION_ENTITIES_ZONE_FILTERS = 68; enum class EntityQueryPacketVersion: PacketVersion { - JsonFilter = 18 + JSONFilter = 18, + JSONFilterWithFamilyTree = 19 }; enum class AssetServerPacketVersion: PacketVersion { From 13b2b6086f224fbcef336ddb6c153d2bb565f054 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Tue, 21 Feb 2017 19:47:26 +0100 Subject: [PATCH 075/104] Fix recorder.js playback in interface --- interface/src/avatar/MyAvatar.cpp | 24 +++++-- interface/src/avatar/MyAvatar.h | 1 + libraries/animation/src/Rig.cpp | 6 +- libraries/animation/src/Rig.h | 2 + libraries/audio-client/src/AudioClient.cpp | 62 +++++++++---------- libraries/audio-client/src/AudioClient.h | 13 ++-- libraries/audio/src/AudioInjectorOptions.cpp | 2 +- .../developer/utilities/record/recorder.js | 10 ++- 8 files changed, 71 insertions(+), 49 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 29f41c89fd..b9f673287b 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -148,13 +148,22 @@ MyAvatar::MyAvatar(RigPointer rig) : auto player = DependencyManager::get(); auto recorder = DependencyManager::get(); connect(player.data(), &Deck::playbackStateChanged, [=] { - if (player->isPlaying()) { + bool isPlaying = player->isPlaying(); + if (isPlaying) { auto recordingInterface = DependencyManager::get(); if (recordingInterface->getPlayFromCurrentLocation()) { setRecordingBasis(); } } else { clearRecordingBasis(); + useFullAvatarURL(_fullAvatarURLFromPreferences, _fullAvatarModelName); + } + + auto audioIO = DependencyManager::get(); + audioIO->setPlayingBackRecording(isPlaying); + + if (_rig) { + _rig->setEnableAnimations(!isPlaying); } }); @@ -180,8 +189,8 @@ MyAvatar::MyAvatar(RigPointer rig) : if (recordingInterface->getPlayerUseSkeletonModel() && dummyAvatar.getSkeletonModelURL().isValid() && (dummyAvatar.getSkeletonModelURL() != getSkeletonModelURL())) { - // FIXME - //myAvatar->useFullAvatarURL() + + setSkeletonModelURL(dummyAvatar.getSkeletonModelURL()); } if (recordingInterface->getPlayerUseDisplayName() && dummyAvatar.getDisplayName() != getDisplayName()) { @@ -204,6 +213,11 @@ MyAvatar::MyAvatar(RigPointer rig) : // head orientation _headData->setLookAtPosition(headData->getLookAtPosition()); } + + auto jointData = dummyAvatar.getRawJointData(); + if (jointData.length() > 0 && _rig) { + _rig->copyJointsFromJointData(jointData); + } }); connect(rig.get(), SIGNAL(onLoadComplete()), this, SIGNAL(onLoadComplete())); @@ -472,7 +486,9 @@ void MyAvatar::simulate(float deltaTime) { { PerformanceTimer perfTimer("joints"); // copy out the skeleton joints from the model - _rig->copyJointsIntoJointData(_jointData); + if (_rigEnabled) { + _rig->copyJointsIntoJointData(_jointData); + } } { diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index c4fe86356d..5c2c0f2765 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -486,6 +486,7 @@ private: std::unordered_set _headBoneSet; RigPointer _rig; bool _prevShouldDrawHead; + bool _rigEnabled { true }; bool _enableDebugDrawDefaultPose { false }; bool _enableDebugDrawAnimPose { false }; diff --git a/libraries/animation/src/Rig.cpp b/libraries/animation/src/Rig.cpp index ac16b16c1d..07462e9878 100644 --- a/libraries/animation/src/Rig.cpp +++ b/libraries/animation/src/Rig.cpp @@ -483,6 +483,10 @@ void Rig::setEnableInverseKinematics(bool enable) { _enableInverseKinematics = enable; } +void Rig::setEnableAnimations(bool enable) { + _enabledAnimations = enable; +} + AnimPose Rig::getAbsoluteDefaultPose(int index) const { if (_animSkeleton && index >= 0 && index < _animSkeleton->getNumJoints()) { return _absoluteDefaultPoses[index]; @@ -907,7 +911,7 @@ void Rig::updateAnimations(float deltaTime, glm::mat4 rootTransform) { setModelOffset(rootTransform); - if (_animNode) { + if (_animNode && _enabledAnimations) { PerformanceTimer perfTimer("handleTriggers"); updateAnimationStateHandlers(); diff --git a/libraries/animation/src/Rig.h b/libraries/animation/src/Rig.h index aa091fe10c..78a669b249 100644 --- a/libraries/animation/src/Rig.h +++ b/libraries/animation/src/Rig.h @@ -210,6 +210,7 @@ public: void computeAvatarBoundingCapsule(const FBXGeometry& geometry, float& radiusOut, float& heightOut, glm::vec3& offsetOut) const; void setEnableInverseKinematics(bool enable); + void setEnableAnimations(bool enable); const glm::mat4& getGeometryToRigTransform() const { return _geometryToRigTransform; } @@ -314,6 +315,7 @@ protected: int32_t _numOverrides { 0 }; bool _lastEnableInverseKinematics { true }; bool _enableInverseKinematics { true }; + bool _enabledAnimations { true }; mutable uint32_t _jointNameWarningCount { 0 }; diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 2e532d67bf..0e2991ae30 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -39,13 +39,10 @@ #include #include #include -#include #include #include -#include #include -#include "PositionalAudioStream.h" #include "AudioClientLogging.h" #include "AudioLogging.h" @@ -151,17 +148,17 @@ static inline float convertToFloat(int16_t sample) { AudioClient::AudioClient() : AbstractAudioInterface(), _gate(this), - _audioInput(NULL), + _audioInput(nullptr), _desiredInputFormat(), _inputFormat(), _numInputCallbackBytes(0), - _audioOutput(NULL), + _audioOutput(nullptr), _desiredOutputFormat(), _outputFormat(), _outputFrameSize(0), _numOutputCallbackBytes(0), - _loopbackAudioOutput(NULL), - _loopbackOutputDevice(NULL), + _loopbackAudioOutput(nullptr), + _loopbackOutputDevice(nullptr), _inputRingBuffer(0), _localInjectorsStream(0), _receivedAudioStream(RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES), @@ -179,9 +176,9 @@ AudioClient::AudioClient() : _isNoiseGateEnabled(true), _reverb(false), _reverbOptions(&_scriptReverbOptions), - _inputToNetworkResampler(NULL), - _networkToOutputResampler(NULL), - _localToOutputResampler(NULL), + _inputToNetworkResampler(nullptr), + _networkToOutputResampler(nullptr), + _localToOutputResampler(nullptr), _localAudioThread(this), _audioLimiter(AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT), _outgoingAvatarAudioSequenceNumber(0), @@ -294,12 +291,12 @@ QString friendlyNameForAudioDevice(IMMDevice* pEndpoint) { IPropertyStore* pPropertyStore; pEndpoint->OpenPropertyStore(STGM_READ, &pPropertyStore); pEndpoint->Release(); - pEndpoint = NULL; + pEndpoint = nullptr; PROPVARIANT pv; PropVariantInit(&pv); HRESULT hr = pPropertyStore->GetValue(PKEY_Device_FriendlyName, &pv); pPropertyStore->Release(); - pPropertyStore = NULL; + pPropertyStore = nullptr; deviceName = QString::fromWCharArray((wchar_t*)pv.pwszVal); if (!IsWindows8OrGreater()) { // Windows 7 provides only the 31 first characters of the device name. @@ -313,9 +310,9 @@ QString friendlyNameForAudioDevice(IMMDevice* pEndpoint) { QString AudioClient::friendlyNameForAudioDevice(wchar_t* guid) { QString deviceName; HRESULT hr = S_OK; - CoInitialize(NULL); - IMMDeviceEnumerator* pMMDeviceEnumerator = NULL; - CoCreateInstance(__uuidof(MMDeviceEnumerator), NULL, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), (void**)&pMMDeviceEnumerator); + CoInitialize(nullptr); + IMMDeviceEnumerator* pMMDeviceEnumerator = nullptr; + CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), (void**)&pMMDeviceEnumerator); IMMDevice* pEndpoint; hr = pMMDeviceEnumerator->GetDevice(guid, &pEndpoint); if (hr == E_NOTFOUND) { @@ -325,7 +322,7 @@ QString AudioClient::friendlyNameForAudioDevice(wchar_t* guid) { deviceName = ::friendlyNameForAudioDevice(pEndpoint); } pMMDeviceEnumerator->Release(); - pMMDeviceEnumerator = NULL; + pMMDeviceEnumerator = nullptr; CoUninitialize(); return deviceName; } @@ -396,9 +393,9 @@ QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { } } else { HRESULT hr = S_OK; - CoInitialize(NULL); - IMMDeviceEnumerator* pMMDeviceEnumerator = NULL; - CoCreateInstance(__uuidof(MMDeviceEnumerator), NULL, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), (void**)&pMMDeviceEnumerator); + CoInitialize(nullptr); + IMMDeviceEnumerator* pMMDeviceEnumerator = nullptr; + CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), (void**)&pMMDeviceEnumerator); IMMDevice* pEndpoint; hr = pMMDeviceEnumerator->GetDefaultAudioEndpoint(mode == QAudio::AudioOutput ? eRender : eCapture, eMultimedia, &pEndpoint); if (hr == E_NOTFOUND) { @@ -408,7 +405,7 @@ QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { deviceName = friendlyNameForAudioDevice(pEndpoint); } pMMDeviceEnumerator->Release(); - pMMDeviceEnumerator = NULL; + pMMDeviceEnumerator = nullptr; CoUninitialize(); } @@ -968,8 +965,7 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { } void AudioClient::handleAudioInput() { - - if (!_inputDevice) { + if (!_inputDevice || _playingBackRecording) { return; } @@ -1358,10 +1354,10 @@ bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceIn // That in turn causes it to be disconnected (see for example // http://stackoverflow.com/questions/9264750/qt-signals-and-slots-object-disconnect). _audioInput->stop(); - _inputDevice = NULL; + _inputDevice = nullptr; delete _audioInput; - _audioInput = NULL; + _audioInput = nullptr; _numInputCallbackBytes = 0; _inputAudioDeviceName = ""; @@ -1370,7 +1366,7 @@ bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceIn if (_inputToNetworkResampler) { // if we were using an input to network resampler, delete it here delete _inputToNetworkResampler; - _inputToNetworkResampler = NULL; + _inputToNetworkResampler = nullptr; } if (!inputDeviceInfo.isNull()) { @@ -1465,29 +1461,29 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice _audioOutput->stop(); delete _audioOutput; - _audioOutput = NULL; + _audioOutput = nullptr; - _loopbackOutputDevice = NULL; + _loopbackOutputDevice = nullptr; delete _loopbackAudioOutput; - _loopbackAudioOutput = NULL; + _loopbackAudioOutput = nullptr; delete[] _outputMixBuffer; - _outputMixBuffer = NULL; + _outputMixBuffer = nullptr; delete[] _outputScratchBuffer; - _outputScratchBuffer = NULL; + _outputScratchBuffer = nullptr; delete[] _localOutputMixBuffer; - _localOutputMixBuffer = NULL; + _localOutputMixBuffer = nullptr; } if (_networkToOutputResampler) { // if we were using an input to network resampler, delete it here delete _networkToOutputResampler; - _networkToOutputResampler = NULL; + _networkToOutputResampler = nullptr; delete _localToOutputResampler; - _localToOutputResampler = NULL; + _localToOutputResampler = nullptr; } if (!outputDeviceInfo.isNull()) { diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 699ba71ef7..9dc86c1ecf 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -145,6 +145,8 @@ public: void setPositionGetter(AudioPositionGetter positionGetter) { _positionGetter = positionGetter; } void setOrientationGetter(AudioOrientationGetter orientationGetter) { _orientationGetter = orientationGetter; } + void setPlayingBackRecording(bool playingBackRecording) { _playingBackRecording = playingBackRecording; } + Q_INVOKABLE void setAvatarBoundingBoxParameters(glm::vec3 corner, glm::vec3 scale); void checkDevices(); @@ -326,14 +328,14 @@ private: // for local audio (used by audio injectors thread) float _localMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; - float* _localOutputMixBuffer { NULL }; + float* _localOutputMixBuffer { nullptr }; AudioInjectorsThread _localAudioThread; Mutex _localAudioMutex; // for output audio (used by this thread) int _outputPeriod { 0 }; - float* _outputMixBuffer { NULL }; - int16_t* _outputScratchBuffer { NULL }; + float* _outputMixBuffer { nullptr }; + int16_t* _outputScratchBuffer { nullptr }; AudioLimiter _audioLimiter; @@ -367,13 +369,16 @@ private: QVector _inputDevices; QVector _outputDevices; - bool _hasReceivedFirstPacket = false; + bool _hasReceivedFirstPacket { false }; QVector _activeLocalAudioInjectors; + bool _playingBackRecording { false }; + CodecPluginPointer _codec; QString _selectedCodecName; Encoder* _encoder { nullptr }; // for outbound mic stream + Encoder* _playbackEncoder { nullptr }; QThread* _checkDevicesThread { nullptr }; }; diff --git a/libraries/audio/src/AudioInjectorOptions.cpp b/libraries/audio/src/AudioInjectorOptions.cpp index 1a92697828..0af74a796c 100644 --- a/libraries/audio/src/AudioInjectorOptions.cpp +++ b/libraries/audio/src/AudioInjectorOptions.cpp @@ -91,4 +91,4 @@ void injectorOptionsFromScriptValue(const QScriptValue& object, AudioInjectorOpt qCWarning(audio) << "Unknown audio injector option:" << it.name(); } } - } \ No newline at end of file +} diff --git a/scripts/developer/utilities/record/recorder.js b/scripts/developer/utilities/record/recorder.js index 083037461d..ea0a4505cc 100644 --- a/scripts/developer/utilities/record/recorder.js +++ b/scripts/developer/utilities/record/recorder.js @@ -1,3 +1,4 @@ +debugger; // // Recorder.js // examples @@ -12,14 +13,14 @@ HIFI_PUBLIC_BUCKET = "http://s3.amazonaws.com/hifi-public/"; Script.include("/~/system/libraries/toolBars.js"); -var recordingFile = "recording.rec"; +var recordingFile = "recording.hfr"; function setPlayerOptions() { Recording.setPlayFromCurrentLocation(true); Recording.setPlayerUseDisplayName(false); Recording.setPlayerUseAttachments(false); Recording.setPlayerUseHeadModel(false); - Recording.setPlayerUseSkeletonModel(false); + Recording.setPlayerUseSkeletonModel(true); } var windowDimensions = Controller.getViewportDimensions(); @@ -142,7 +143,6 @@ function setupTimer() { backgroundAlpha: 1.0, visible: true }); - } function updateTimer() { @@ -272,7 +272,7 @@ function mousePressEvent(event) { } } else if (loadIcon === toolBar.clicked(clickedOverlay)) { if (!Recording.isRecording() && !Recording.isPlaying()) { - recordingFile = Window.browse("Load recorcding from file", ".", "Recordings (*.hfr *.rec *.HFR *.REC)"); + recordingFile = Window.browse("Load recording from file", ".", "Recordings (*.hfr *.rec *.HFR *.REC)"); if (!(recordingFile === "null" || recordingFile === null || recordingFile === "")) { Recording.loadRecording(recordingFile); } @@ -345,5 +345,3 @@ Script.scriptEnding.connect(scriptEnding); // Should be called last to put everything into position moveUI(); - - From 7564b6f0f5690a223ccb0a8fd6bb6d3125ea1de0 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 21 Feb 2017 10:59:36 -0800 Subject: [PATCH 076/104] More consistent highlights/toggle for toolbar buttons When the "app" for a button is visible the button should become active. Also clicking on a highlighted/active button should close the associated "app". This should be noticeable on the following tablet apps. * audio * menu * people * market * users --- scripts/system/audio.js | 48 ++++++++++++++++--- scripts/system/marketplaces/marketplaces.js | 35 ++++++++++---- scripts/system/menu.js | 28 +++++++++-- scripts/system/pal.js | 52 +++++++++++++-------- scripts/system/tablet-goto.js | 27 +++++++++-- scripts/system/tablet-users.js | 31 +++++++++--- 6 files changed, 170 insertions(+), 51 deletions(-) diff --git a/scripts/system/audio.js b/scripts/system/audio.js index c0fdb43b40..6e7e95d659 100644 --- a/scripts/system/audio.js +++ b/scripts/system/audio.js @@ -16,19 +16,51 @@ var TABLET_BUTTON_NAME = "AUDIO"; var HOME_BUTTON_TEXTURE = "http://hifi-content.s3.amazonaws.com/alan/dev/tablet-with-home-button.fbx/tablet-with-home-button.fbm/button-root.png"; +var MUTE_ICONS = { + icon: "icons/tablet-icons/mic-mute-i.svg", + activeIcon: "icons/tablet-icons/mic-mute-a.svg" +}; + +var UNMUTE_ICONS = { + icon: "icons/tablet-icons/mic-unmute-i.svg", + activeIcon: "icons/tablet-icons/mic-unmute-a.svg" +}; + function onMuteToggled() { - button.editProperties({ isActive: AudioDevice.getMuted() }); + if (AudioDevice.getMuted()) { + button.editProperties(MUTE_ICONS); + } else { + button.editProperties(UNMUTE_ICONS); + } } -function onClicked(){ - var entity = HMD.tabletID; - Entities.editEntity(entity, { textures: JSON.stringify({ "tex.close": HOME_BUTTON_TEXTURE }) }); - tablet.gotoMenuScreen("Audio"); + +var shouldActivateButton = false; +var onAudioScreen = false; + +function onClicked() { + if (onAudioScreen) { + // for toolbar-mode: go back to home screen, this will close the window. + tablet.gotoHomeScreen(); + } else { + var entity = HMD.tabletID; + Entities.editEntity(entity, { textures: JSON.stringify({ "tex.close": HOME_BUTTON_TEXTURE }) }); + shouldActivateButton = true; + tablet.gotoMenuScreen("Audio"); + onAudioScreen = true; + } +} + +function onScreenChanged(type, url) { + // for toolbar mode: change button to active when window is first openend, false otherwise. + button.editProperties({isActive: shouldActivateButton}); + shouldActivateButton = false; + onAudioScreen = false; } var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system"); var button = tablet.addButton({ - icon: "icons/tablet-icons/mic-unmute-i.svg", - activeIcon: "icons/tablet-icons/mic-mute-a.svg", + icon: AudioDevice.getMuted() ? MUTE_ICONS.icon : UNMUTE_ICONS.icon, + activeIcon: AudioDevice.getMuted() ? MUTE_ICONS.activeIcon : UNMUTE_ICONS.activeIcon, text: TABLET_BUTTON_NAME, sortOrder: 1 }); @@ -36,10 +68,12 @@ var button = tablet.addButton({ onMuteToggled(); button.clicked.connect(onClicked); +tablet.screenChanged.connect(onScreenChanged); AudioDevice.muteToggled.connect(onMuteToggled); Script.scriptEnding.connect(function () { button.clicked.disconnect(onClicked); + tablet.screenChanged.disconnect(onScreenChanged); AudioDevice.muteToggled.disconnect(onMuteToggled); tablet.removeButton(button); }); diff --git a/scripts/system/marketplaces/marketplaces.js b/scripts/system/marketplaces/marketplaces.js index c5ce5a634b..68da7696be 100644 --- a/scripts/system/marketplaces/marketplaces.js +++ b/scripts/system/marketplaces/marketplaces.js @@ -52,10 +52,16 @@ function onMessageBoxClosed(id, button) { Window.messageBoxClosed.connect(onMessageBoxClosed); +var shouldActivateButton = false; +var onMarketplaceScreen = false; + function showMarketplace() { UserActivityLogger.openedMarketplace(); + shouldActivateButton = true; tablet.gotoWebScreen(MARKETPLACE_URL_INITIAL, MARKETPLACES_INJECT_SCRIPT_URL); + onMarketplaceScreen = true; + tablet.webEventReceived.connect(function (message) { if (message === GOTO_DIRECTORY) { @@ -98,15 +104,10 @@ function showMarketplace() { }); } -function toggleMarketplace() { - var entity = HMD.tabletID; - Entities.editEntity(entity, {textures: JSON.stringify({"tex.close": HOME_BUTTON_TEXTURE})}); - showMarketplace(); -} - var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system"); var marketplaceButton = tablet.addButton({ icon: "icons/tablet-icons/market-i.svg", + activeIcon: "icons/tablet-icons/market-a.svg", text: "MARKET", sortOrder: 9 }); @@ -117,16 +118,30 @@ function onCanWriteAssetsChanged() { } function onClick() { - toggleMarketplace(); + if (onMarketplaceScreen) { + // for toolbar-mode: go back to home screen, this will close the window. + tablet.gotoHomeScreen(); + } else { + var entity = HMD.tabletID; + Entities.editEntity(entity, {textures: JSON.stringify({"tex.close": HOME_BUTTON_TEXTURE})}); + showMarketplace(); + } +} + +function onScreenChanged(type, url) { + // for toolbar mode: change button to active when window is first openend, false otherwise. + marketplaceButton.editProperties({isActive: shouldActivateButton}); + shouldActivateButton = false; + onMarketplaceScreen = false; } marketplaceButton.clicked.connect(onClick); +tablet.screenChanged.connect(onScreenChanged); Entities.canWriteAssetsChanged.connect(onCanWriteAssetsChanged); Script.scriptEnding.connect(function () { - if (tablet) { - tablet.removeButton(marketplaceButton); - } + tablet.removeButton(marketplaceButton); + tablet.screenChanged.disconnect(onScreenChanged); Entities.canWriteAssetsChanged.disconnect(onCanWriteAssetsChanged); }); diff --git a/scripts/system/menu.js b/scripts/system/menu.js index 13c6ce1e0d..1d5f8bccd6 100644 --- a/scripts/system/menu.js +++ b/scripts/system/menu.js @@ -10,26 +10,46 @@ // var HOME_BUTTON_TEXTURE = "http://hifi-content.s3.amazonaws.com/alan/dev/tablet-with-home-button.fbx/tablet-with-home-button.fbm/button-root.png"; -//var HOME_BUTTON_TEXTURE = Script.resourcesPath() + "meshes/tablet-with-home-button.fbx/tablet-with-home-button.fbm/button-root.png"; +// var HOME_BUTTON_TEXTURE = Script.resourcesPath() + "meshes/tablet-with-home-button.fbx/tablet-with-home-button.fbm/button-root.png"; (function() { var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system"); var button = tablet.addButton({ icon: "icons/tablet-icons/menu-i.svg", + activeIcon: "icons/tablet-icons/menu-a.svg", text: "MENU", sortOrder: 3 }); + var shouldActivateButton = false; + var onMenuScreen = false; + function onClicked() { - var entity = HMD.tabletID; - Entities.editEntity(entity, {textures: JSON.stringify({"tex.close": HOME_BUTTON_TEXTURE})}); - tablet.gotoMenuScreen(); + if (onMenuScreen) { + // for toolbar-mode: go back to home screen, this will close the window. + tablet.gotoHomeScreen(); + } else { + var entity = HMD.tabletID; + Entities.editEntity(entity, {textures: JSON.stringify({"tex.close": HOME_BUTTON_TEXTURE})}); + shouldActivateButton = true; + tablet.gotoMenuScreen(); + onMenuScreen = true; + } + } + + function onScreenChanged(type, url) { + // for toolbar mode: change button to active when window is first openend, false otherwise. + button.editProperties({isActive: shouldActivateButton}); + shouldActivateButton = false; + onMenuScreen = false; } button.clicked.connect(onClicked); + tablet.screenChanged.connect(onScreenChanged); Script.scriptEnding.connect(function () { button.clicked.disconnect(onClicked); tablet.removeButton(button); + tablet.screenChanged.disconnect(onScreenChanged); }); }()); diff --git a/scripts/system/pal.js b/scripts/system/pal.js index d47544e0f0..36ecc1f084 100644 --- a/scripts/system/pal.js +++ b/scripts/system/pal.js @@ -477,23 +477,17 @@ var button; var buttonName = "PEOPLE"; var tablet = null; -function onTabletScreenChanged(type, url) { - if (type !== "QML" || url !== "../Pal.qml") { - off(); - } -} - function startup() { tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system"); button = tablet.addButton({ text: buttonName, icon: "icons/tablet-icons/people-i.svg", + activeIcon: "icons/tablet-icons/people-a.svg", sortOrder: 7 }); tablet.fromQml.connect(fromQml); button.clicked.connect(onTabletButtonClicked); tablet.screenChanged.connect(onTabletScreenChanged); - Users.usernameFromIDReply.connect(usernameFromIDReply); Window.domainChanged.connect(clearLocalQMLDataAndClosePAL); Window.domainConnectionRefused.connect(clearLocalQMLDataAndClosePAL); @@ -524,17 +518,39 @@ function off() { Users.requestsDomainListData = false; } +var onPalScreen = false; +var shouldActivateButton = false; + function onTabletButtonClicked() { - tablet.loadQMLSource("../Pal.qml"); - Users.requestsDomainListData = true; - populateUserList(); - isWired = true; - Script.update.connect(updateOverlays); - Controller.mousePressEvent.connect(handleMouseEvent); - Controller.mouseMoveEvent.connect(handleMouseMoveEvent); - triggerMapping.enable(); - triggerPressMapping.enable(); - audioTimer = createAudioInterval(conserveResources ? AUDIO_LEVEL_CONSERVED_UPDATE_INTERVAL_MS : AUDIO_LEVEL_UPDATE_INTERVAL_MS); + if (onPalScreen) { + // for toolbar-mode: go back to home screen, this will close the window. + tablet.gotoHomeScreen(); + } else { + shouldActivateButton = true; + tablet.loadQMLSource("../Pal.qml"); + onPalScreen = true; + Users.requestsDomainListData = true; + populateUserList(); + isWired = true; + Script.update.connect(updateOverlays); + Controller.mousePressEvent.connect(handleMouseEvent); + Controller.mouseMoveEvent.connect(handleMouseMoveEvent); + triggerMapping.enable(); + triggerPressMapping.enable(); + audioTimer = createAudioInterval(conserveResources ? AUDIO_LEVEL_CONSERVED_UPDATE_INTERVAL_MS : AUDIO_LEVEL_UPDATE_INTERVAL_MS); + } +} + +function onTabletScreenChanged(type, url) { + // for toolbar mode: change button to active when window is first openend, false otherwise. + button.editProperties({isActive: shouldActivateButton}); + shouldActivateButton = false; + onPalScreen = false; + + // disable sphere overlays when not on pal screen. + if (type !== "QML" || url !== "../Pal.qml") { + off(); + } } // @@ -621,14 +637,12 @@ function shutdown() { button.clicked.disconnect(onTabletButtonClicked); tablet.removeButton(button); tablet.screenChanged.disconnect(onTabletScreenChanged); - Users.usernameFromIDReply.disconnect(usernameFromIDReply); Window.domainChanged.disconnect(clearLocalQMLDataAndClosePAL); Window.domainConnectionRefused.disconnect(clearLocalQMLDataAndClosePAL); Messages.subscribe(CHANNEL); Messages.messageReceived.disconnect(receiveMessage); Users.avatarDisconnected.disconnect(avatarDisconnected); - off(); } diff --git a/scripts/system/tablet-goto.js b/scripts/system/tablet-goto.js index 6c3e12cd9b..eb95d9d8a3 100644 --- a/scripts/system/tablet-goto.js +++ b/scripts/system/tablet-goto.js @@ -14,10 +14,27 @@ (function() { // BEGIN LOCAL_SCOPE var gotoQmlSource = "TabletAddressDialog.qml"; var buttonName = "GOTO"; + var onGotoScreen = false; + var shouldActivateButton = false; - function onClicked(){ - tablet.loadQMLSource(gotoQmlSource); + function onClicked() { + if (onGotoScreen) { + // for toolbar-mode: go back to home screen, this will close the window. + tablet.gotoHomeScreen(); + } else { + shouldActivateButton = true; + tablet.loadQMLSource(gotoQmlSource); + onGotoScreen = true; + } } + + function onScreenChanged(type, url) { + // for toolbar mode: change button to active when window is first openend, false otherwise. + button.editProperties({isActive: shouldActivateButton}); + shouldActivateButton = false; + onGotoScreen = false; + } + var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system"); var button = tablet.addButton({ icon: "icons/tablet-icons/goto-i.svg", @@ -27,12 +44,12 @@ }); button.clicked.connect(onClicked); + tablet.screenChanged.connect(onScreenChanged); Script.scriptEnding.connect(function () { button.clicked.disconnect(onClicked); - if (tablet) { - tablet.removeButton(button); - } + tablet.removeButton(button); + tablet.screenChanged.disconnect(onScreenChanged); }); }()); // END LOCAL_SCOPE diff --git a/scripts/system/tablet-users.js b/scripts/system/tablet-users.js index ce50c4686d..8e89ac74b7 100644 --- a/scripts/system/tablet-users.js +++ b/scripts/system/tablet-users.js @@ -1,7 +1,7 @@ "use strict"; // -// users.js +// tablet-users.js // // Created by Faye Li on 18 Jan 2017. // Copyright 2017 High Fidelity, Inc. @@ -36,16 +36,34 @@ var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system"); var button = tablet.addButton({ icon: "icons/tablet-icons/users-i.svg", + activeIcon: "icons/tablet-icons/users-a.svg", text: "USERS", sortOrder: 11 }); + var onUsersScreen = false; + var shouldActivateButton = false; + function onClicked() { - var tabletEntity = HMD.tabletID; - if (tabletEntity) { - Entities.editEntity(tabletEntity, {textures: JSON.stringify({"tex.close" : HOME_BUTTON_TEXTURE})}); + if (onUsersScreen) { + // for toolbar-mode: go back to home screen, this will close the window. + tablet.gotoHomeScreen(); + } else { + var tabletEntity = HMD.tabletID; + if (tabletEntity) { + Entities.editEntity(tabletEntity, {textures: JSON.stringify({"tex.close" : HOME_BUTTON_TEXTURE})}); + } + shouldActivateButton = true; + tablet.gotoWebScreen(USERS_URL); + onUsersScreen = true; } - tablet.gotoWebScreen(USERS_URL); + } + + function onScreenChanged() { + // for toolbar mode: change button to active when window is first openend, false otherwise. + button.editProperties({isActive: shouldActivateButton}); + shouldActivateButton = false; + onUsersScreen = false; } function onWebEventReceived(event) { @@ -88,12 +106,13 @@ // update your visibility (all, friends, or none) myVisibility = event.data.visibility; GlobalServices.findableBy = myVisibility; - } + } } } button.clicked.connect(onClicked); tablet.webEventReceived.connect(onWebEventReceived); + tablet.screenChanged.connect(onScreenChanged); function cleanup() { button.clicked.disconnect(onClicked); From 15518e9aa17c970187ba368dc851d7ac2ecc6798 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Tue, 21 Feb 2017 20:20:23 +0100 Subject: [PATCH 077/104] remove debugger line --- scripts/developer/utilities/record/recorder.js | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/developer/utilities/record/recorder.js b/scripts/developer/utilities/record/recorder.js index ea0a4505cc..0e335116d5 100644 --- a/scripts/developer/utilities/record/recorder.js +++ b/scripts/developer/utilities/record/recorder.js @@ -1,4 +1,3 @@ -debugger; // // Recorder.js // examples From 72eb797910733125b6e844de7350260f17258791 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Tue, 21 Feb 2017 20:22:29 +0100 Subject: [PATCH 078/104] remove unused encoder (was for testing) --- libraries/audio-client/src/AudioClient.h | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 9dc86c1ecf..739c4b459c 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -378,7 +378,6 @@ private: CodecPluginPointer _codec; QString _selectedCodecName; Encoder* _encoder { nullptr }; // for outbound mic stream - Encoder* _playbackEncoder { nullptr }; QThread* _checkDevicesThread { nullptr }; }; From 27dd82b517c31bb3b65b6e2d07791688e4017a9e Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Tue, 21 Feb 2017 13:34:07 -0800 Subject: [PATCH 079/104] fix sorting algorithm for avatar render updates --- interface/src/avatar/AvatarManager.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/interface/src/avatar/AvatarManager.cpp b/interface/src/avatar/AvatarManager.cpp index c3fc974365..80b9c95cee 100644 --- a/interface/src/avatar/AvatarManager.cpp +++ b/interface/src/avatar/AvatarManager.cpp @@ -154,8 +154,7 @@ public: AvatarPriority(AvatarSharedPointer a, float p) : avatar(a), priority(p) {} AvatarSharedPointer avatar; float priority; - // NOTE: we invert the less-than operator to sort high priorities to front - bool operator<(const AvatarPriority& other) const { return priority > other.priority; } + bool operator<(const AvatarPriority& other) const { return priority < other.priority; } }; void AvatarManager::updateOtherAvatars(float deltaTime) { @@ -206,9 +205,8 @@ void AvatarManager::updateOtherAvatars(float deltaTime) { float radius = avatar->getBoundingRadius(); const glm::vec3& forward = cameraView.getDirection(); float apparentSize = radius / distance; - float cosineAngle = glm::length(offset - glm::dot(offset, forward) * forward) / distance; - const float TIME_PENALTY = 0.080f; // seconds - float age = (float)(startTime - avatar->getLastRenderUpdateTime()) / (float)(USECS_PER_SECOND) - TIME_PENALTY; + float cosineAngle = glm::length(glm::dot(offset, forward) * forward) / distance; + float age = (float)(startTime - avatar->getLastRenderUpdateTime()) / (float)(USECS_PER_SECOND); // NOTE: we are adding values of different units to get a single measure of "priority". // Thus we multiply each component by a conversion "weight" that scales its units // relative to the others. These weights are pure magic tuning and are hard coded in the From 3b181eb80378db12d1917efa7cc20040fa15cc60 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Tue, 21 Feb 2017 13:56:28 -0800 Subject: [PATCH 080/104] code review --- interface/src/Application.cpp | 8 ++++---- interface/src/Application.h | 4 ++-- interface/src/ui/overlays/Overlays.cpp | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index e9f6e3d7f5..3de7906f56 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -6895,12 +6895,12 @@ void Application::toggleMuteAudio() { menu->setIsOptionChecked(MenuOption::MuteAudio, !menu->isOptionChecked(MenuOption::MuteAudio)); } -OverlayID Application::getTabletScreenID() { +OverlayID Application::getTabletScreenID() const { auto HMD = DependencyManager::get(); - return HMD->getCurrentTabletScreenID(); + return HMD->getCurrentTabletScreenID(); } -OverlayID Application::getTabletHomeButtonID() { +OverlayID Application::getTabletHomeButtonID() const { auto HMD = DependencyManager::get(); - return HMD->getCurrentHomeButtonUUID(); + return HMD->getCurrentHomeButtonUUID(); } diff --git a/interface/src/Application.h b/interface/src/Application.h index 26264af09c..662523ce1d 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -298,8 +298,8 @@ public: Q_INVOKABLE void sendHoverOverEntity(QUuid id, PointerEvent event); Q_INVOKABLE void sendHoverLeaveEntity(QUuid id, PointerEvent event); - OverlayID getTabletScreenID(); - OverlayID getTabletHomeButtonID(); + OverlayID getTabletScreenID() const; + OverlayID getTabletHomeButtonID() const; signals: void svoImportRequested(const QString& url); diff --git a/interface/src/ui/overlays/Overlays.cpp b/interface/src/ui/overlays/Overlays.cpp index f3a78c9350..c18d9ddaef 100644 --- a/interface/src/ui/overlays/Overlays.cpp +++ b/interface/src/ui/overlays/Overlays.cpp @@ -462,7 +462,7 @@ RayToOverlayIntersectionResult::RayToOverlayIntersectionResult() : QScriptValue RayToOverlayIntersectionResultToScriptValue(QScriptEngine* engine, const RayToOverlayIntersectionResult& value) { auto obj = engine->newObject(); obj.setProperty("intersects", value.intersects); - obj.setProperty("overlayID", OverlayIDtoScriptValue(engine, value.overlayID)); + obj.setProperty("overlayID", OverlayIDtoScriptValue(engine, value.overlayID)); obj.setProperty("distance", value.distance); QString faceName = ""; From 2017ea4491a91027cc6d3c498b346bc9d36e6c8e Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 21 Feb 2017 14:18:00 -0800 Subject: [PATCH 081/104] hacking on using sorted avatars for bandwidth budget --- .../src/avatars/AvatarMixerClientData.cpp | 12 ++- .../src/avatars/AvatarMixerClientData.h | 11 +++ .../src/avatars/AvatarMixerSlave.cpp | 76 ++++++++++++++++++- interface/src/avatar/Avatar.cpp | 5 -- interface/src/avatar/Avatar.h | 1 - interface/src/avatar/AvatarManager.cpp | 66 ++++------------ libraries/avatars/src/AvatarData.cpp | 63 +++++++++++++++ libraries/avatars/src/AvatarData.h | 27 ++++++- libraries/avatars/src/AvatarHashMap.cpp | 1 + libraries/avatars/src/AvatarHashMap.h | 1 - 10 files changed, 199 insertions(+), 64 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index df6a64e874..1df6c24a60 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -74,14 +74,22 @@ bool AvatarMixerClientData::checkAndSetHasReceivedFirstPacketsFrom(const QUuid& return true; } +uint64_t AvatarMixerClientData::getLastBroadcastTime(const QUuid& nodeUUID) const { + // return the matching PacketSequenceNumber, or the default if we don't have it + auto nodeMatch = _lastBroadcastTimes.find(nodeUUID); + if (nodeMatch != _lastBroadcastTimes.end()) { + return nodeMatch->second; + } + return 0; +} + uint16_t AvatarMixerClientData::getLastBroadcastSequenceNumber(const QUuid& nodeUUID) const { // return the matching PacketSequenceNumber, or the default if we don't have it auto nodeMatch = _lastBroadcastSequenceNumbers.find(nodeUUID); if (nodeMatch != _lastBroadcastSequenceNumbers.end()) { return nodeMatch->second; - } else { - return 0; } + return 0; } void AvatarMixerClientData::ignoreOther(SharedNodePointer self, SharedNodePointer other) { diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index 87f7bb9cc9..7eea31c9ab 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -43,6 +43,7 @@ public: int parseData(ReceivedMessage& message) override; AvatarData& getAvatar() { return *_avatar; } const AvatarData* getConstAvatarData() const { return _avatar.get(); } + AvatarSharedPointer getAvatarSharedPointer() const { return _avatar; } bool checkAndSetHasReceivedFirstPacketsFrom(const QUuid& uuid); @@ -51,6 +52,12 @@ public: { _lastBroadcastSequenceNumbers[nodeUUID] = sequenceNumber; } Q_INVOKABLE void removeLastBroadcastSequenceNumber(const QUuid& nodeUUID) { _lastBroadcastSequenceNumbers.erase(nodeUUID); } + uint64_t getLastBroadcastTime(const QUuid& nodeUUID) const; + void setLastBroadcastTime(const QUuid& nodeUUID, uint64_t broadcastTime) { + _lastBroadcastTimes[nodeUUID] = broadcastTime; + } + Q_INVOKABLE void removeLastBroadcastTime(const QUuid& nodeUUID) { _lastBroadcastTimes.erase(nodeUUID); } + uint16_t getLastReceivedSequenceNumber() const { return _lastReceivedSequenceNumber; } HRCTime getIdentityChangeTimestamp() const { return _identityChangeTimestamp; } @@ -106,6 +113,9 @@ public: bool getRequestsDomainListData() { return _requestsDomainListData; } void setRequestsDomainListData(bool requesting) { _requestsDomainListData = requesting; } + ViewFrustum getViewFrustom() const { return _currentViewFrustum; } + + quint64 getLastOtherAvatarEncodeTime(QUuid otherAvatar) { quint64 result = 0; if (_lastOtherAvatarEncodeTime.find(otherAvatar) != _lastOtherAvatarEncodeTime.end()) { @@ -134,6 +144,7 @@ private: uint16_t _lastReceivedSequenceNumber { 0 }; std::unordered_map _lastBroadcastSequenceNumbers; std::unordered_set _hasReceivedFirstPacketsFrom; + std::unordered_map _lastBroadcastTimes; // this is a map of the last time we encoded an "other" avatar for // sending to "this" node diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 46133a221f..4c95c96483 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -187,9 +187,75 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // setup a PacketList for the avatarPackets auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); + + QList avatarList; + std::unordered_map avatarDataToNodes; + + //qDebug() << "------------------------------"; + int listItem = 0; + std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) { + const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); + if (otherNodeData) { + listItem++; + AvatarSharedPointer otherAvatar = otherNodeData->getAvatarSharedPointer(); + avatarList << otherAvatar; + avatarDataToNodes[otherAvatar] = otherNode; + /* + qDebug() << "listItem [" << listItem << "] " + << "otherNode:" << otherNode.data() + << "otherNode->getUUID():" << otherNode->getUUID() + << "otherNodeData:" << otherNodeData + << "otherAvatar:" << otherAvatar.get(); + + qDebug() << "avatarDataToNodes[" << otherAvatar.get() << "]=" << otherNode.data(); + */ + } + + }); + /* + qDebug() << "------------------------------"; + qDebug() << "avatarList.size:" << avatarList.size(); + qDebug() << "avatarDataToNodes.size:" << avatarDataToNodes.size(); + */ + + ViewFrustum cameraView = nodeData->getViewFrustom(); + std::priority_queue sortedAvatars = AvatarData::sortAvatars( + avatarList, cameraView, + + [&](AvatarSharedPointer avatar)->uint64_t{ + auto avatarNode = avatarDataToNodes[avatar]; + if (avatarNode) { + return nodeData->getLastBroadcastTime(avatarNode->getUUID()); + } + return 0; // ??? + }, + + [this](AvatarSharedPointer avatar)->bool{ + // FIXME -- when to ignore this node + return false; + }); + + // this is an AGENT we have received head data from // send back a packet with other active node data to this node - std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) { + //std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) { + + int avatarRank = 0; + while (!sortedAvatars.empty()) { + AvatarPriority sortData = sortedAvatars.top(); + sortedAvatars.pop(); + const auto& avatarData = sortData.avatar; + avatarRank++; + + auto otherNode = avatarDataToNodes[avatarData]; + + //qDebug() << "otherNode (" << otherNode.data() << ")= avatarDataToNodes[" << avatarData.get() << "]"; + + if (!otherNode) { + //qDebug() << "For viewer:" << node->getUUID() << "... process other avatar [" << avatarRank << ":" << avatarData.get() << "... otherNode unknown!!"; + continue; + } + //qDebug() << "For viewer:" << node->getUUID() << "... process other avatar [" << avatarRank << "] avatarData: " << avatarData.get() << " otherNode:" << otherNode->getUUID() << " ... "; bool shouldConsider = false; quint64 startIgnoreCalculation = usecTimestampNow(); @@ -286,7 +352,8 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // are out of view, this also appears to disable this random distribution. if (distanceToAvatar != 0.0f && !getsOutOfView - && distribution(generator) > (nodeData->getFullRateDistance() / distanceToAvatar)) { + // -- && distribution(generator) > (nodeData->getFullRateDistance() / distanceToAvatar) /// FIX ME... no longer doing random + ) { quint64 endAvatarDataPacking = usecTimestampNow(); _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); @@ -395,6 +462,9 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), otherNodeData->getLastReceivedSequenceNumber()); + // remember the last time we sent details about this other node to the receiver + nodeData->setLastBroadcastTime(otherNode->getUUID(), start); + } } @@ -406,7 +476,7 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { } } } - }); + }; quint64 startPacketSending = usecTimestampNow(); diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index ed8f083a41..64ecb1dc29 100644 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -334,11 +334,6 @@ void Avatar::updateAvatarEntities() { setAvatarEntityDataChanged(false); } -bool Avatar::shouldDie() const { - const qint64 AVATAR_SILENCE_THRESHOLD_USECS = 5 * USECS_PER_SECOND; - return _owningAvatarMixer.isNull() || getUsecsSinceLastUpdate() > AVATAR_SILENCE_THRESHOLD_USECS; -} - void Avatar::simulate(float deltaTime, bool inView) { PROFILE_RANGE(simulation, "simulate"); diff --git a/interface/src/avatar/Avatar.h b/interface/src/avatar/Avatar.h index 80d387fd33..53f0562bdd 100644 --- a/interface/src/avatar/Avatar.h +++ b/interface/src/avatar/Avatar.h @@ -178,7 +178,6 @@ public: uint64_t getLastRenderUpdateTime() const { return _lastRenderUpdateTime; } void setLastRenderUpdateTime(uint64_t time) { _lastRenderUpdateTime = time; } - bool shouldDie() const; void animateScaleChanges(float deltaTime); void setTargetScale(float targetScale) override; diff --git a/interface/src/avatar/AvatarManager.cpp b/interface/src/avatar/AvatarManager.cpp index c3fc974365..1a9866aae4 100644 --- a/interface/src/avatar/AvatarManager.cpp +++ b/interface/src/avatar/AvatarManager.cpp @@ -148,16 +148,6 @@ float AvatarManager::getAvatarSimulationRate(const QUuid& sessionID, const QStri } - -class AvatarPriority { -public: - AvatarPriority(AvatarSharedPointer a, float p) : avatar(a), priority(p) {} - AvatarSharedPointer avatar; - float priority; - // NOTE: we invert the less-than operator to sort high priorities to front - bool operator<(const AvatarPriority& other) const { return priority > other.priority; } -}; - void AvatarManager::updateOtherAvatars(float deltaTime) { // lock the hash for read to check the size QReadLocker lock(&_hashLock); @@ -173,57 +163,31 @@ void AvatarManager::updateOtherAvatars(float deltaTime) { QList avatarList = avatarMap.values(); ViewFrustum cameraView; qApp->copyDisplayViewFrustum(cameraView); - glm::vec3 frustumCenter = cameraView.getPosition(); - const float OUT_OF_VIEW_PENALTY = -10.0; + std::priority_queue sortedAvatars = AvatarData::sortAvatars( + avatarList, cameraView, - std::priority_queue sortedAvatars; - { - PROFILE_RANGE(simulation, "sort"); - for (int32_t i = 0; i < avatarList.size(); ++i) { - const auto& avatar = std::static_pointer_cast(avatarList.at(i)); - if (avatar == _myAvatar || !avatar->isInitialized()) { + [](AvatarSharedPointer avatar)->uint64_t{ + return std::static_pointer_cast(avatar)->getLastRenderUpdateTime(); + }, + + [this](AvatarSharedPointer avatar)->bool{ + const auto& castedAvatar = std::static_pointer_cast(avatar); + if (castedAvatar == _myAvatar || !castedAvatar->isInitialized()) { // DO NOT update _myAvatar! Its update has already been done earlier in the main loop. // DO NOT update or fade out uninitialized Avatars - continue; + return true; // ignore it } if (avatar->shouldDie()) { removeAvatar(avatar->getID()); - continue; + return true; // ignore it } if (avatar->isDead()) { - continue; + return true; // ignore it } - // priority = weighted linear combination of: - // (a) apparentSize - // (b) proximity to center of view - // (c) time since last update - // (d) TIME_PENALTY to help recently updated entries sort toward back - glm::vec3 avatarPosition = avatar->getPosition(); - glm::vec3 offset = avatarPosition - frustumCenter; - float distance = glm::length(offset) + 0.001f; // add 1mm to avoid divide by zero - float radius = avatar->getBoundingRadius(); - const glm::vec3& forward = cameraView.getDirection(); - float apparentSize = radius / distance; - float cosineAngle = glm::length(offset - glm::dot(offset, forward) * forward) / distance; - const float TIME_PENALTY = 0.080f; // seconds - float age = (float)(startTime - avatar->getLastRenderUpdateTime()) / (float)(USECS_PER_SECOND) - TIME_PENALTY; - // NOTE: we are adding values of different units to get a single measure of "priority". - // Thus we multiply each component by a conversion "weight" that scales its units - // relative to the others. These weights are pure magic tuning and are hard coded in the - // relation below: (hint: unitary weights are not explicityly shown) - float priority = apparentSize + 0.25f * cosineAngle + age; - - // decrement priority of avatars outside keyhole - if (distance > cameraView.getCenterRadius()) { - if (!cameraView.sphereIntersectsFrustum(avatarPosition, radius)) { - priority += OUT_OF_VIEW_PENALTY; - } - } - sortedAvatars.push(AvatarPriority(avatar, priority)); - } - } + return false; + }); render::PendingChanges pendingChanges; const uint64_t RENDER_UPDATE_BUDGET = 1500; // usec @@ -256,7 +220,7 @@ void AvatarManager::updateOtherAvatars(float deltaTime) { uint64_t now = usecTimestampNow(); if (now < renderExpiry) { // we're within budget - const float OUT_OF_VIEW_THRESHOLD = 0.5f * OUT_OF_VIEW_PENALTY; + const float OUT_OF_VIEW_THRESHOLD = 0.5f * AvatarData::OUT_OF_VIEW_PENALTY; bool inView = sortData.priority > OUT_OF_VIEW_THRESHOLD; avatar->simulate(deltaTime, inView); avatar->updateRenderItem(pendingChanges); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 2263964e8f..dffdc8c1ac 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include "AvatarLogging.h" @@ -2309,3 +2310,65 @@ void RayToAvatarIntersectionResultFromScriptValue(const QScriptValue& object, Ra vec3FromScriptValue(intersection, value.intersection); } } + +const float AvatarData::OUT_OF_VIEW_PENALTY = -10.0f; + +std::priority_queue AvatarData::sortAvatars( + QList avatarList, + const ViewFrustum& cameraView, + std::function lastUpdated, + std::function shouldIgnore) { + + uint64_t startTime = usecTimestampNow(); + + glm::vec3 frustumCenter = cameraView.getPosition(); + + std::priority_queue sortedAvatars; + { + PROFILE_RANGE(simulation, "sort"); + for (int32_t i = 0; i < avatarList.size(); ++i) { + const auto& avatar = avatarList.at(i); + + // FIXME - probably some lambda that allows the caller to reject some avatars + + if (shouldIgnore(avatar)) { + continue; + } + + // priority = weighted linear combination of: + // (a) apparentSize + // (b) proximity to center of view + // (c) time since last update + // (d) TIME_PENALTY to help recently updated entries sort toward back + glm::vec3 avatarPosition = avatar->getPosition(); + glm::vec3 offset = avatarPosition - frustumCenter; + float distance = glm::length(offset) + 0.001f; // add 1mm to avoid divide by zero + + // FIXME - AvatarData has something equivolent to this + float radius = 1.0f; // avatar->getBoundingRadius(); + + const glm::vec3& forward = cameraView.getDirection(); + float apparentSize = radius / distance; + float cosineAngle = glm::length(glm::dot(offset, forward) * forward) / distance; + + // FIXME - probably some lambda that allows the caller to specify the "updated since" + uint64_t lastUpdatedTime = lastUpdated(avatar); // avatar->getLastRenderUpdateTime() + float age = (float)(startTime - lastUpdatedTime) / (float)(USECS_PER_SECOND); + + // NOTE: we are adding values of different units to get a single measure of "priority". + // Thus we multiply each component by a conversion "weight" that scales its units + // relative to the others. These weights are pure magic tuning and are hard coded in the + // relation below: (hint: unitary weights are not explicityly shown) + float priority = apparentSize + 0.25f * cosineAngle + age; + + // decrement priority of avatars outside keyhole + if (distance > cameraView.getCenterRadius()) { + if (!cameraView.sphereIntersectsFrustum(avatarPosition, radius)) { + priority += OUT_OF_VIEW_PENALTY; + } + } + sortedAvatars.push(AvatarPriority(avatar, priority)); + } + } + return sortedAvatars; +} diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 264da75de2..71ade436ff 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -14,6 +14,8 @@ #include #include +#include + /* VS2010 defines stdint.h, but not inttypes.h */ #if defined(_MSC_VER) typedef signed char int8_t; @@ -57,6 +59,7 @@ typedef unsigned long long quint64; #include #include #include +#include #include "AABox.h" #include "HeadData.h" @@ -304,6 +307,14 @@ public: RateCounter<> jointDataRate; }; +class AvatarPriority { +public: + AvatarPriority(AvatarSharedPointer a, float p) : avatar(a), priority(p) {} + AvatarSharedPointer avatar; + float priority; + // NOTE: we invert the less-than operator to sort high priorities to front + bool operator<(const AvatarPriority& other) const { return priority < other.priority; } +}; class AvatarData : public QObject, public SpatiallyNestable { Q_OBJECT @@ -539,7 +550,7 @@ public: void setOwningAvatarMixer(const QWeakPointer& owningAvatarMixer) { _owningAvatarMixer = owningAvatarMixer; } - const AABox& getLocalAABox() const { return _localAABox; } + //const AABox& getLocalAABox() const { return _localAABox; } int getUsecsSinceLastUpdate() const { return _averageBytesReceived.getUsecsSinceLastEvent(); } int getAverageBytesReceivedPerSecond() const; @@ -578,6 +589,20 @@ public: } + bool shouldDie() const { + const qint64 AVATAR_SILENCE_THRESHOLD_USECS = 5 * USECS_PER_SECOND; + return _owningAvatarMixer.isNull() || getUsecsSinceLastUpdate() > AVATAR_SILENCE_THRESHOLD_USECS; + } + + static const float OUT_OF_VIEW_PENALTY; + + static std::priority_queue sortAvatars( + QList avatarList, + const ViewFrustum& cameraView, + std::function lastUpdated, + std::function shouldIgnore); + + public slots: void sendAvatarDataPacket(); void sendIdentityPacket(); diff --git a/libraries/avatars/src/AvatarHashMap.cpp b/libraries/avatars/src/AvatarHashMap.cpp index 00c515a635..cdbf5f2a85 100644 --- a/libraries/avatars/src/AvatarHashMap.cpp +++ b/libraries/avatars/src/AvatarHashMap.cpp @@ -190,3 +190,4 @@ void AvatarHashMap::sessionUUIDChanged(const QUuid& sessionUUID, const QUuid& ol _lastOwnerSessionUUID = oldUUID; emit avatarSessionChangedEvent(sessionUUID, oldUUID); } + diff --git a/libraries/avatars/src/AvatarHashMap.h b/libraries/avatars/src/AvatarHashMap.h index dd097aef22..104ac83261 100644 --- a/libraries/avatars/src/AvatarHashMap.h +++ b/libraries/avatars/src/AvatarHashMap.h @@ -27,7 +27,6 @@ #include "AvatarData.h" - class AvatarHashMap : public QObject, public Dependency { Q_OBJECT SINGLETON_DEPENDENCY From dd447689e0cefb603814fc019e0dd87e213fcb88 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Tue, 21 Feb 2017 15:09:44 -0800 Subject: [PATCH 082/104] expose avatar sort coefficients for fine tuning --- interface/src/avatar/AvatarManager.cpp | 39 ++++++++++++++++++++++---- interface/src/avatar/AvatarManager.h | 9 ++++++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/interface/src/avatar/AvatarManager.cpp b/interface/src/avatar/AvatarManager.cpp index 80b9c95cee..d2c2ea36a3 100644 --- a/interface/src/avatar/AvatarManager.cpp +++ b/interface/src/avatar/AvatarManager.cpp @@ -204,14 +204,17 @@ void AvatarManager::updateOtherAvatars(float deltaTime) { float distance = glm::length(offset) + 0.001f; // add 1mm to avoid divide by zero float radius = avatar->getBoundingRadius(); const glm::vec3& forward = cameraView.getDirection(); - float apparentSize = radius / distance; + float apparentSize = 2.0f * radius / distance; float cosineAngle = glm::length(glm::dot(offset, forward) * forward) / distance; float age = (float)(startTime - avatar->getLastRenderUpdateTime()) / (float)(USECS_PER_SECOND); + // NOTE: we are adding values of different units to get a single measure of "priority". - // Thus we multiply each component by a conversion "weight" that scales its units - // relative to the others. These weights are pure magic tuning and are hard coded in the - // relation below: (hint: unitary weights are not explicityly shown) - float priority = apparentSize + 0.25f * cosineAngle + age; + // Thus we multiply each component by a conversion "weight" that scales its units relative to the others. + // These weights are pure magic tuning and should be hard coded in the relation below, + // but are currently exposed for anyone who would like to explore fine tuning: + float priority = _avatarSortCoefficientSize * apparentSize + + _avatarSortCoefficientCenter * cosineAngle + + _avatarSortCoefficientAge * age; // decrement priority of avatars outside keyhole if (distance > cameraView.getCenterRadius()) { @@ -591,3 +594,29 @@ RayToAvatarIntersectionResult AvatarManager::findRayIntersection(const PickRay& return result; } + +// HACK +float AvatarManager::getAvatarSortCoefficient(const QString& name) { + if (name == "size") { + return _avatarSortCoefficientSize; + } else if (name == "center") { + return _avatarSortCoefficientCenter; + } else if (name == "age") { + return _avatarSortCoefficientAge; + } + return 0.0f; +} + +// HACK +void AvatarManager::setAvatarSortCoefficient(const QString& name, const QScriptValue& value) { + if (value.isNumber()) { + float numericalValue = (float)value.toNumber(); + if (name == "size") { + _avatarSortCoefficientSize = numericalValue; + } else if (name == "center") { + _avatarSortCoefficientCenter = numericalValue; + } else if (name == "age") { + _avatarSortCoefficientAge = numericalValue; + } + } +} diff --git a/interface/src/avatar/AvatarManager.h b/interface/src/avatar/AvatarManager.h index 4d503842b9..b4e1a700cb 100644 --- a/interface/src/avatar/AvatarManager.h +++ b/interface/src/avatar/AvatarManager.h @@ -81,6 +81,10 @@ public: const QScriptValue& avatarIdsToInclude = QScriptValue(), const QScriptValue& avatarIdsToDiscard = QScriptValue()); + // TODO: remove this HACK once we settle on optimal default sort coefficients + Q_INVOKABLE float getAvatarSortCoefficient(const QString& name); + Q_INVOKABLE void setAvatarSortCoefficient(const QString& name, const QScriptValue& value); + float getMyAvatarSendRate() const { return _myAvatarSendRate.rate(); } public slots: @@ -120,6 +124,11 @@ private: int _partiallySimulatedAvatars { 0 }; float _avatarSimulationTime { 0.0f }; + // TODO: remove this HACK once we settle on optimal sort coefficients + // These coefficients exposed for fine tuning the sort priority for transfering new _jointData to the render pipeline. + float _avatarSortCoefficientSize { 0.5f }; + float _avatarSortCoefficientCenter { 0.25 }; + float _avatarSortCoefficientAge { 1.0f }; }; Q_DECLARE_METATYPE(AvatarManager::LocalLight) From 4866be1f522eb56c93fbbcb705da6a864097f588 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 21 Feb 2017 16:01:46 -0800 Subject: [PATCH 083/104] More consistent tablet angle when first creating the tablet. The tablet should more constantly be placed above your hand, while facing your head. It fixes the issue where the tablet would appear almost horizontal when your hand was close to your HMD and at eye level. --- scripts/system/libraries/WebTablet.js | 28 ++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/scripts/system/libraries/WebTablet.js b/scripts/system/libraries/WebTablet.js index 74bbd788be..618b70ff20 100644 --- a/scripts/system/libraries/WebTablet.js +++ b/scripts/system/libraries/WebTablet.js @@ -49,24 +49,30 @@ function calcSpawnInfo(hand, height) { var handController = getControllerWorldLocation(hand, true); var controllerPosition = handController.position; - // compute the angle of the chord with length (height / 2) - var theta = Math.asin(height / (2 * Vec3.distance(headPos, controllerPosition))); + // base of the tablet is slightly above controller position + var TABLET_BASE_DISPLACEMENT = {x: 0, y: 0.1, z: 0}; + var tabletBase = Vec3.sum(controllerPosition, TABLET_BASE_DISPLACEMENT); - // then we can use this angle to rotate the vector between the HMD position and the center of the tablet. - // this vector, u, will become our new look at direction. - var d = Vec3.normalize(Vec3.subtract(headPos, controllerPosition)); + var d = Vec3.subtract(headPos, tabletBase); + var theta = Math.acos(d.y / Vec3.length(d)); + d.y = 0; + if (Vec3.length(d) < 0.0001) { + d = {x: 1, y: 0, z: 0}; + } else { + d = Vec3.normalize(d); + } var w = Vec3.normalize(Vec3.cross(Y_AXIS, d)); - var q = Quat.angleAxis(theta * (180 / Math.PI), w); + var ANGLE_OFFSET = 25; + var q = Quat.angleAxis(theta * (180 / Math.PI) - (90 - ANGLE_OFFSET), w); var u = Vec3.multiplyQbyV(q, d); // use u to compute a full lookAt quaternion. - var lookAtRot = Quat.lookAt(controllerPosition, Vec3.sum(controllerPosition, u), Y_AXIS); - - // adjust the tablet position by a small amount. - var yDisplacement = (height / 2) + 0.1; + var lookAtRot = Quat.lookAt(tabletBase, Vec3.sum(tabletBase, u), Y_AXIS); + var yDisplacement = (height / 2); var zDisplacement = 0.05; var tabletOffset = Vec3.multiplyQbyV(lookAtRot, {x: 0, y: yDisplacement, z: zDisplacement}); - finalPosition = Vec3.sum(controllerPosition, tabletOffset); + finalPosition = Vec3.sum(tabletBase, tabletOffset); + return { position: finalPosition, rotation: lookAtRot From 8cc0b383c4c28d27de675a50465f17d8328ab307 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 21 Feb 2017 16:22:22 -0800 Subject: [PATCH 084/104] wire up bandwidth buget to sorting --- .../src/avatars/AvatarMixerSlave.cpp | 23 ++++++++++++++----- libraries/avatars/src/AvatarData.cpp | 2 ++ .../developer/debugging/debugAvatarMixer.js | 10 +++++--- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 4c95c96483..5c71b88af9 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -213,11 +213,15 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { }); /* - qDebug() << "------------------------------"; qDebug() << "avatarList.size:" << avatarList.size(); qDebug() << "avatarDataToNodes.size:" << avatarDataToNodes.size(); */ + //qDebug() << "------------------------------"; + AvatarSharedPointer thisAvatar = nodeData->getAvatarSharedPointer(); + + //qDebug() << "thisAvatar:" << thisAvatar.get(); + ViewFrustum cameraView = nodeData->getViewFrustom(); std::priority_queue sortedAvatars = AvatarData::sortAvatars( avatarList, cameraView, @@ -230,10 +234,10 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { return 0; // ??? }, - [this](AvatarSharedPointer avatar)->bool{ - // FIXME -- when to ignore this node - return false; + [thisAvatar](AvatarSharedPointer avatar)->bool{ + return (avatar == thisAvatar); // ignore ourselves... }); + //qDebug() << "------------------------------"; // this is an AGENT we have received head data from @@ -269,6 +273,7 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { || (otherNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { shouldConsider = false; + //qDebug() << "shouldConsider = false ... line:" << __LINE__; } else { const AvatarMixerClientData* otherData = reinterpret_cast(otherNode->getLinkedData()); @@ -304,6 +309,8 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { if (nodeBox.touches(otherNodeBox)) { nodeData->ignoreOther(node, otherNode); shouldConsider = getsAnyIgnored; + //qDebug() << "shouldConsider = getsAnyIgnored " << getsAnyIgnored << " ... line:" << __LINE__; + } } // Not close enough to ignore @@ -352,12 +359,13 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // are out of view, this also appears to disable this random distribution. if (distanceToAvatar != 0.0f && !getsOutOfView - // -- && distribution(generator) > (nodeData->getFullRateDistance() / distanceToAvatar) /// FIX ME... no longer doing random + && distribution(generator) > (nodeData->getFullRateDistance() / distanceToAvatar) /// FIX ME... we don't want to do this random stuff ) { quint64 endAvatarDataPacking = usecTimestampNow(); _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); shouldConsider = false; + //qDebug() << "shouldConsider = false ... line:" << __LINE__; } if (shouldConsider) { @@ -380,7 +388,9 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { quint64 endAvatarDataPacking = usecTimestampNow(); _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); shouldConsider = false; - } else if (lastSeqFromSender - lastSeqToReceiver > 1) { + //qDebug() << "shouldConsider = false ... line:" << __LINE__; + } + else if (lastSeqFromSender - lastSeqToReceiver > 1) { // this is a skip - we still send the packet but capture the presence of the skip so we see it happening ++numAvatarsWithSkippedFrames; } @@ -399,6 +409,7 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); shouldConsider = false; + //qDebug() << "shouldConsider = false ... line:" << __LINE__; } if (shouldConsider) { diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index dffdc8c1ac..9416216c5a 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -2367,6 +2367,8 @@ std::priority_queue AvatarData::sortAvatars( priority += OUT_OF_VIEW_PENALTY; } } + + //qDebug() << "avatar:" << avatar.get() << "priority:" << priority << "apparentSize:" << apparentSize << "cosineAngle:" << cosineAngle << "age:" << age; sortedAvatars.push(AvatarPriority(avatar, priority)); } } diff --git a/scripts/developer/debugging/debugAvatarMixer.js b/scripts/developer/debugging/debugAvatarMixer.js index ebd43fc2f0..90f2de13a9 100644 --- a/scripts/developer/debugging/debugAvatarMixer.js +++ b/scripts/developer/debugging/debugAvatarMixer.js @@ -58,11 +58,14 @@ function updateOverlays() { // setup a position for the overlay that is just above this avatar's head var overlayPosition = avatar.getJointPosition("Head"); - overlayPosition.y += 1.05; + overlayPosition.y += 1.15; + + var rows = 8; var text = avatarID + "\n" +"--- Data from Mixer ---\n" +"All: " + AvatarManager.getAvatarDataRate(avatarID).toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID).toFixed(2) + "hz)" + "\n" + /* +" GP: " + AvatarManager.getAvatarDataRate(avatarID,"globalPosition").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"globalPosition").toFixed(2) + "hz)" + "\n" +" LP: " + AvatarManager.getAvatarDataRate(avatarID,"localPosition").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"localPosition").toFixed(2) + "hz)" + "\n" +" BB: " + AvatarManager.getAvatarDataRate(avatarID,"avatarBoundingBox").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"avatarBoundingBox").toFixed(2) + "hz)" + "\n" @@ -74,11 +77,12 @@ function updateOverlays() { +" AF: " + AvatarManager.getAvatarDataRate(avatarID,"additionalFlags").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"additionalFlags").toFixed(2) + "hz)" + "\n" +" PI: " + AvatarManager.getAvatarDataRate(avatarID,"parentInfo").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"parentInfo").toFixed(2) + "hz)" + "\n" +" FT: " + AvatarManager.getAvatarDataRate(avatarID,"faceTracker").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"faceTracker").toFixed(2) + "hz)" + "\n" + */ +" JD: " + AvatarManager.getAvatarDataRate(avatarID,"jointData").toFixed(2) + "kbps (" + AvatarManager.getAvatarUpdateRate(avatarID,"jointData").toFixed(2) + "hz)" + "\n" +"--- Simulation ---\n" +"All: " + AvatarManager.getAvatarSimulationRate(avatarID,"avatar").toFixed(2) + "hz \n" +" inView: " + AvatarManager.getAvatarSimulationRate(avatarID,"avatarInView").toFixed(2) + "hz \n" - +" SM: " + AvatarManager.getAvatarSimulationRate(avatarID,"skeletonModel").toFixed(2) + "hz \n" + //+" SM: " + AvatarManager.getAvatarSimulationRate(avatarID,"skeletonModel").toFixed(2) + "hz \n" +" JD: " + AvatarManager.getAvatarSimulationRate(avatarID,"jointData").toFixed(2) + "hz \n" if (avatarID in debugOverlays) { @@ -93,7 +97,7 @@ function updateOverlays() { position: overlayPosition, dimensions: { x: 1.25, - y: 19 * 0.13 + y: rows * 0.13 }, lineHeight: 0.1, font:{size:0.1}, From 379b7bc56dcc43e7699356c9f220e711a0b2012b Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 21 Feb 2017 17:24:22 -0800 Subject: [PATCH 085/104] Fix for system toolbar hiding when scripts are reloaded --- interface/resources/qml/hifi/toolbars/Toolbar.qml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/interface/resources/qml/hifi/toolbars/Toolbar.qml b/interface/resources/qml/hifi/toolbars/Toolbar.qml index 0080e49815..c0d984e822 100644 --- a/interface/resources/qml/hifi/toolbars/Toolbar.qml +++ b/interface/resources/qml/hifi/toolbars/Toolbar.qml @@ -120,6 +120,8 @@ Window { function addButton(properties) { properties = properties || {} + unpinnedAlpha = 1; + // If a name is specified, then check if there's an existing button with that name // and return it if so. This will allow multiple clients to listen to a single button, // and allow scripts to be idempotent so they don't duplicate buttons if they're reloaded @@ -154,7 +156,7 @@ Window { updatePinned(); if (buttons.length === 0) { - visible = false; + unpinnedAlpha = 0; } } From 4c42e95607852cec20d23ffe716290aba8157c16 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 22 Feb 2017 08:13:37 -0800 Subject: [PATCH 086/104] add some debugging --- .../src/avatars/AvatarMixerSlave.cpp | 23 +++++++++++++------ libraries/avatars/src/AvatarData.cpp | 11 ++++++--- libraries/avatars/src/AvatarData.h | 3 ++- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 5c71b88af9..360c6e26fd 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -217,11 +217,19 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { qDebug() << "avatarDataToNodes.size:" << avatarDataToNodes.size(); */ - //qDebug() << "------------------------------"; AvatarSharedPointer thisAvatar = nodeData->getAvatarSharedPointer(); //qDebug() << "thisAvatar:" << thisAvatar.get(); +#ifdef WANT_DEBUG + bool printDebug = nodeData->getAvatarSharedPointer()->getDisplayName() == "ZappoMan"; + + if (printDebug) { + qDebug() << "------------------------------"; + } +#else + bool printDebug = false; +#endif ViewFrustum cameraView = nodeData->getViewFrustom(); std::priority_queue sortedAvatars = AvatarData::sortAvatars( avatarList, cameraView, @@ -236,14 +244,15 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { [thisAvatar](AvatarSharedPointer avatar)->bool{ return (avatar == thisAvatar); // ignore ourselves... - }); - //qDebug() << "------------------------------"; + }, printDebug); +#ifdef WANT_DEBUG + if (printDebug) { + qDebug() << "------------------------------"; + } +#endif - // this is an AGENT we have received head data from - // send back a packet with other active node data to this node - //std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) { - + // loop through our sorted avatars and allocate our bandwidth to them accordingly int avatarRank = 0; while (!sortedAvatars.empty()) { AvatarPriority sortData = sortedAvatars.top(); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 9416216c5a..fd21670853 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -2317,7 +2317,8 @@ std::priority_queue AvatarData::sortAvatars( QList avatarList, const ViewFrustum& cameraView, std::function lastUpdated, - std::function shouldIgnore) { + std::function shouldIgnore, + bool printDebug) { uint64_t startTime = usecTimestampNow(); @@ -2359,16 +2360,20 @@ std::priority_queue AvatarData::sortAvatars( // Thus we multiply each component by a conversion "weight" that scales its units // relative to the others. These weights are pure magic tuning and are hard coded in the // relation below: (hint: unitary weights are not explicityly shown) - float priority = apparentSize + 0.25f * cosineAngle + age; + float priority = apparentSize + 10.0f * (0.25f * cosineAngle) + age; // decrement priority of avatars outside keyhole + bool outOfView = false; if (distance > cameraView.getCenterRadius()) { if (!cameraView.sphereIntersectsFrustum(avatarPosition, radius)) { priority += OUT_OF_VIEW_PENALTY; + outOfView = true; } } - //qDebug() << "avatar:" << avatar.get() << "priority:" << priority << "apparentSize:" << apparentSize << "cosineAngle:" << cosineAngle << "age:" << age; + if (printDebug) { + qDebug() << "avatar:" << avatar.get() << "priority:" << priority << "apparentSize:" << apparentSize << "cosineAngle:" << cosineAngle << "age:" << age << "outOfView:" << outOfView; + } sortedAvatars.push(AvatarPriority(avatar, priority)); } } diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 71ade436ff..a75e81ed3d 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -600,7 +600,8 @@ public: QList avatarList, const ViewFrustum& cameraView, std::function lastUpdated, - std::function shouldIgnore); + std::function shouldIgnore, + bool printDebug = false); public slots: From 05995163cf610405f0519511d8009b220697a613 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 22 Feb 2017 09:49:47 -0800 Subject: [PATCH 087/104] add support for tuning sorting --- assignment-client/src/avatars/AvatarMixer.cpp | 20 ++++++++++++++++++- assignment-client/src/avatars/AvatarMixer.h | 2 ++ interface/src/avatar/AvatarManager.cpp | 15 ++++++++++++++ libraries/avatars/src/AvatarData.h | 6 +++--- libraries/networking/src/udt/PacketHeaders.h | 3 ++- 5 files changed, 41 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 3a8aa0182c..59896432e7 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -47,7 +47,7 @@ AvatarMixer::AvatarMixer(ReceivedMessage& message) : auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); packetReceiver.registerListener(PacketType::AvatarData, this, "queueIncomingPacket"); - + packetReceiver.registerListener(PacketType::AdjustAvatarSorting, this, "handleAdjustAvatarSorting"); packetReceiver.registerListener(PacketType::ViewFrustum, this, "handleViewFrustumPacket"); packetReceiver.registerListener(PacketType::AvatarIdentity, this, "handleAvatarIdentityPacket"); packetReceiver.registerListener(PacketType::KillAvatar, this, "handleKillAvatarPacket"); @@ -317,6 +317,24 @@ void AvatarMixer::nodeKilled(SharedNodePointer killedNode) { } } + +void AvatarMixer::handleAdjustAvatarSorting(QSharedPointer message, SharedNodePointer senderNode) { + auto start = usecTimestampNow(); + + message->readPrimitive(&AvatarData::_avatarSortCoefficientSize); + message->readPrimitive(&AvatarData::_avatarSortCoefficientCenter); + message->readPrimitive(&AvatarData::_avatarSortCoefficientAge); + + qCDebug(avatars) << "New avatar sorting... " + << "size:" << AvatarData::_avatarSortCoefficientSize + << "center:" << AvatarData::_avatarSortCoefficientCenter + << "age:" << AvatarData::_avatarSortCoefficientAge; + + auto end = usecTimestampNow(); + _handleAdjustAvatarSortingElapsedTime += (end - start); +} + + void AvatarMixer::handleViewFrustumPacket(QSharedPointer message, SharedNodePointer senderNode) { auto start = usecTimestampNow(); getOrCreateClientData(senderNode); diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index 32b0ffed69..1925ec1ebd 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -39,6 +39,7 @@ public slots: private slots: void queueIncomingPacket(QSharedPointer message, SharedNodePointer node); + void handleAdjustAvatarSorting(QSharedPointer message, SharedNodePointer senderNode); void handleViewFrustumPacket(QSharedPointer message, SharedNodePointer senderNode); void handleAvatarIdentityPacket(QSharedPointer message, SharedNodePointer senderNode); void handleKillAvatarPacket(QSharedPointer message); @@ -92,6 +93,7 @@ private: quint64 _broadcastAvatarDataNodeTransform { 0 }; quint64 _broadcastAvatarDataNodeFunctor { 0 }; + quint64 _handleAdjustAvatarSortingElapsedTime { 0 }; quint64 _handleViewFrustumPacketElapsedTime { 0 }; quint64 _handleAvatarIdentityPacketElapsedTime { 0 }; quint64 _handleKillAvatarPacketElapsedTime { 0 }; diff --git a/interface/src/avatar/AvatarManager.cpp b/interface/src/avatar/AvatarManager.cpp index 9dc73d9239..4cdf735e96 100644 --- a/interface/src/avatar/AvatarManager.cpp +++ b/interface/src/avatar/AvatarManager.cpp @@ -572,14 +572,29 @@ float AvatarManager::getAvatarSortCoefficient(const QString& name) { // HACK void AvatarManager::setAvatarSortCoefficient(const QString& name, const QScriptValue& value) { + bool somethingChanged = false; if (value.isNumber()) { float numericalValue = (float)value.toNumber(); if (name == "size") { AvatarData::_avatarSortCoefficientSize = numericalValue; + somethingChanged = true; } else if (name == "center") { AvatarData::_avatarSortCoefficientCenter = numericalValue; + somethingChanged = true; } else if (name == "age") { AvatarData::_avatarSortCoefficientAge = numericalValue; + somethingChanged = true; } } + if (somethingChanged) { + size_t packetSize = sizeof(AvatarData::_avatarSortCoefficientSize) + + sizeof(AvatarData::_avatarSortCoefficientCenter) + + sizeof(AvatarData::_avatarSortCoefficientAge); + + auto packet = NLPacket::create(PacketType::AdjustAvatarSorting, packetSize); + packet->writePrimitive(AvatarData::_avatarSortCoefficientSize); + packet->writePrimitive(AvatarData::_avatarSortCoefficientCenter); + packet->writePrimitive(AvatarData::_avatarSortCoefficientAge); + DependencyManager::get()->broadcastToNodes(std::move(packet), NodeSet() << NodeType::AvatarMixer); + } } diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 9d5bc507e0..daa496f6e1 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -605,9 +605,9 @@ public: // TODO: remove this HACK once we settle on optimal sort coefficients // These coefficients exposed for fine tuning the sort priority for transfering new _jointData to the render pipeline. - static float _avatarSortCoefficientSize { 0.5f }; - static float _avatarSortCoefficientCenter { 0.25 }; - static float _avatarSortCoefficientAge { 1.0f }; + static float _avatarSortCoefficientSize; + static float _avatarSortCoefficientCenter; + static float _avatarSortCoefficientAge; diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 050e3088f8..01973a6786 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -112,7 +112,8 @@ public: ReloadEntityServerScript, EntityPhysics, EntityServerScriptLog, - LAST_PACKET_TYPE = EntityServerScriptLog + AdjustAvatarSorting, + LAST_PACKET_TYPE = AdjustAvatarSorting }; }; From 71b569a697d6fec7c73df2114c515cd55d2d0235 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Wed, 22 Feb 2017 10:22:12 -0800 Subject: [PATCH 088/104] Bug-fix for crash on tablet when going from web -> home. --- libraries/script-engine/src/TabletScriptingInterface.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/script-engine/src/TabletScriptingInterface.cpp b/libraries/script-engine/src/TabletScriptingInterface.cpp index c78ce251c8..0d4934c35a 100644 --- a/libraries/script-engine/src/TabletScriptingInterface.cpp +++ b/libraries/script-engine/src/TabletScriptingInterface.cpp @@ -366,6 +366,7 @@ void TabletProxy::gotoWebScreen(const QString& url, const QString& injectedJavaS } if (root) { + removeButtonsFromHomeScreen(); QMetaObject::invokeMethod(root, "loadSource", Q_ARG(const QVariant&, QVariant(WEB_VIEW_SOURCE_URL))); QMetaObject::invokeMethod(root, "setShown", Q_ARG(const QVariant&, QVariant(true))); QMetaObject::invokeMethod(root, "loadWebUrl", Q_ARG(const QVariant&, QVariant(url)), Q_ARG(const QVariant&, QVariant(injectedJavaScriptUrl))); From 618f37eb0622abfdbc5e540956535ef20121a9f7 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 22 Feb 2017 11:14:22 -0800 Subject: [PATCH 089/104] fix crash in recording playback when clip loader fails --- libraries/script-engine/src/RecordingScriptingInterface.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libraries/script-engine/src/RecordingScriptingInterface.cpp b/libraries/script-engine/src/RecordingScriptingInterface.cpp index 710f342322..41bb780b47 100644 --- a/libraries/script-engine/src/RecordingScriptingInterface.cpp +++ b/libraries/script-engine/src/RecordingScriptingInterface.cpp @@ -56,6 +56,11 @@ bool RecordingScriptingInterface::loadRecording(const QString& url) { using namespace recording; auto loader = ClipCache::instance().getClipLoader(url); + if (!loader) { + qWarning() << "Clip failed to load from " << url; + return false; + } + if (!loader->isLoaded()) { QEventLoop loop; QObject::connect(loader.data(), &Resource::loaded, &loop, &QEventLoop::quit); From aed1d693776047ba0f6fe001cd9700e7df8aed9d Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Wed, 22 Feb 2017 14:14:10 -0800 Subject: [PATCH 090/104] don't do a haptic pulse when av's hand goes near grabbable tablet --- scripts/system/controllers/handControllerGrab.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js index 95c05c2717..ea76490b7b 100644 --- a/scripts/system/controllers/handControllerGrab.js +++ b/scripts/system/controllers/handControllerGrab.js @@ -1233,7 +1233,13 @@ function MyController(hand) { }); if (grabbableEntities.length > 0) { if (!this.grabPointIntersectsEntity) { - Controller.triggerHapticPulse(1, 20, this.hand); + // don't do haptic pulse for tablet + var nonTabletEntities = grabbableEntities.filter(function(entityID) { + return entityID != HMD.tabletID && entityID != HMD.homeButtonID; + }); + if (nonTabletEntities.length > 0) { + Controller.triggerHapticPulse(1, 20, this.hand); + } this.grabPointIntersectsEntity = true; this.grabPointSphereOn(); } From 6ac33dbf61382605d77b5a76b2d086a3e4636632 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Wed, 22 Feb 2017 17:23:36 -0800 Subject: [PATCH 091/104] count avatar joint updates and non-updates --- interface/resources/qml/Stats.qml | 4 +-- interface/src/avatar/Avatar.h | 2 ++ interface/src/avatar/AvatarManager.cpp | 42 ++++++++++++++++++++------ interface/src/avatar/AvatarManager.h | 8 ++--- interface/src/ui/Stats.cpp | 4 +-- interface/src/ui/Stats.h | 8 ++--- 6 files changed, 47 insertions(+), 21 deletions(-) diff --git a/interface/resources/qml/Stats.qml b/interface/resources/qml/Stats.qml index faf37d5366..58d589b667 100644 --- a/interface/resources/qml/Stats.qml +++ b/interface/resources/qml/Stats.qml @@ -107,11 +107,11 @@ Item { } StatText { visible: root.expanded - text: "Fully Simulated Avatars: " + root.fullySimulatedAvatarCount + text: "Avatars Updated: " + root.updatedAvatarCount } StatText { visible: root.expanded - text: "Partially Simulated Avatars: " + root.partiallySimulatedAvatarCount + text: "Avatars NOT Updated: " + root.notUpdatedAvatarCount } } } diff --git a/interface/src/avatar/Avatar.h b/interface/src/avatar/Avatar.h index 53f0562bdd..ca4dbd2af8 100644 --- a/interface/src/avatar/Avatar.h +++ b/interface/src/avatar/Avatar.h @@ -183,6 +183,8 @@ public: Q_INVOKABLE float getSimulationRate(const QString& rateName = QString("")) const; + bool hasNewJointData() const { return _hasNewJointData; } + public slots: // FIXME - these should be migrated to use Pose data instead diff --git a/interface/src/avatar/AvatarManager.cpp b/interface/src/avatar/AvatarManager.cpp index 4cdf735e96..af9f915fec 100644 --- a/interface/src/avatar/AvatarManager.cpp +++ b/interface/src/avatar/AvatarManager.cpp @@ -195,8 +195,8 @@ void AvatarManager::updateOtherAvatars(float deltaTime) { uint64_t renderExpiry = startTime + RENDER_UPDATE_BUDGET; uint64_t maxExpiry = startTime + MAX_UPDATE_BUDGET; - int fullySimulatedAvatars = 0; - int partiallySimulatedAvatars = 0; + int numAvatarsUpdated = 0; + int numAVatarsNotUpdated = 0; while (!sortedAvatars.empty()) { const AvatarPriority& sortData = sortedAvatars.top(); const auto& avatar = std::static_pointer_cast(sortData.avatar); @@ -217,33 +217,57 @@ void AvatarManager::updateOtherAvatars(float deltaTime) { } avatar->animateScaleChanges(deltaTime); + const float OUT_OF_VIEW_THRESHOLD = 0.5f * AvatarData::OUT_OF_VIEW_PENALTY; uint64_t now = usecTimestampNow(); if (now < renderExpiry) { // we're within budget - const float OUT_OF_VIEW_THRESHOLD = 0.5f * AvatarData::OUT_OF_VIEW_PENALTY; bool inView = sortData.priority > OUT_OF_VIEW_THRESHOLD; + if (inView && avatar->hasNewJointData()) { + numAvatarsUpdated++; + } avatar->simulate(deltaTime, inView); avatar->updateRenderItem(pendingChanges); avatar->setLastRenderUpdateTime(startTime); - fullySimulatedAvatars++; } else if (now < maxExpiry) { // we've spent most of our time budget, but we still simulate() the avatar as it if were out of view // --> some avatars may freeze until their priority trickles up - const bool inView = false; - avatar->simulate(deltaTime, inView); - partiallySimulatedAvatars++; + bool inView = sortData.priority > OUT_OF_VIEW_THRESHOLD; + if (inView && avatar->hasNewJointData()) { + numAVatarsNotUpdated++; + } + avatar->simulate(deltaTime, false); } else { // we've spent ALL of our time budget --> bail on the rest of the avatar updates + // --> more avatars may freeze until their priority trickles up // --> some scale or fade animations may glitch // --> some avatar velocity measurements may be a little off + + // HACK: no time simulate, but we will take the time to count how many were tragically missed + bool inView = sortData.priority > OUT_OF_VIEW_THRESHOLD; + if (!inView) { + break; + } + if (inView && avatar->hasNewJointData()) { + numAVatarsNotUpdated++; + } + sortedAvatars.pop(); + while (inView && !sortedAvatars.empty()) { + const AvatarPriority& newSortData = sortedAvatars.top(); + const auto& newAvatar = std::static_pointer_cast(newSortData.avatar); + inView = newSortData.priority > OUT_OF_VIEW_THRESHOLD; + if (inView && newAvatar->hasNewJointData()) { + numAVatarsNotUpdated++; + } + sortedAvatars.pop(); + } break; } sortedAvatars.pop(); } _avatarSimulationTime = (float)(usecTimestampNow() - startTime) / (float)USECS_PER_MSEC; - _fullySimulatedAvatars = fullySimulatedAvatars; - _partiallySimulatedAvatars = partiallySimulatedAvatars; + _numAvatarsUpdated = numAvatarsUpdated; + _numAvatarsNotUpdated = numAVatarsNotUpdated; qApp->getMain3DScene()->enqueuePendingChanges(pendingChanges); simulateAvatarFades(deltaTime); diff --git a/interface/src/avatar/AvatarManager.h b/interface/src/avatar/AvatarManager.h index 8302bf4ed6..e1f5a3b411 100644 --- a/interface/src/avatar/AvatarManager.h +++ b/interface/src/avatar/AvatarManager.h @@ -43,8 +43,8 @@ public: std::shared_ptr getMyAvatar() { return _myAvatar; } AvatarSharedPointer getAvatarBySessionID(const QUuid& sessionID) const override; - int getFullySimulatedAvatars() const { return _fullySimulatedAvatars; } - int getPartiallySimulatedAvatars() const { return _partiallySimulatedAvatars; } + int getNumAvatarsUpdated() const { return _numAvatarsUpdated; } + int getNumAvatarsNotUpdated() const { return _numAvatarsNotUpdated; } float getAvatarSimulationTime() const { return _avatarSimulationTime; } void updateMyAvatar(float deltaTime); @@ -120,8 +120,8 @@ private: VectorOfMotionStates _motionStatesToRemoveFromPhysics; RateCounter<> _myAvatarSendRate; - int _fullySimulatedAvatars { 0 }; - int _partiallySimulatedAvatars { 0 }; + int _numAvatarsUpdated { 0 }; + int _numAvatarsNotUpdated { 0 }; float _avatarSimulationTime { 0.0f }; }; diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index e82f99bed2..1075bbdaa4 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -121,8 +121,8 @@ void Stats::updateStats(bool force) { auto avatarManager = DependencyManager::get(); // we need to take one avatar out so we don't include ourselves STAT_UPDATE(avatarCount, avatarManager->size() - 1); - STAT_UPDATE(fullySimulatedAvatarCount, avatarManager->getFullySimulatedAvatars()); - STAT_UPDATE(partiallySimulatedAvatarCount, avatarManager->getPartiallySimulatedAvatars()); + STAT_UPDATE(updatedAvatarCount, avatarManager->getNumAvatarsUpdated()); + STAT_UPDATE(notUpdatedAvatarCount, avatarManager->getNumAvatarsNotUpdated()); STAT_UPDATE(serverCount, (int)nodeList->size()); STAT_UPDATE(framerate, qApp->getFps()); if (qApp->getActiveDisplayPlugin()) { diff --git a/interface/src/ui/Stats.h b/interface/src/ui/Stats.h index f501f4b09a..6be084100c 100644 --- a/interface/src/ui/Stats.h +++ b/interface/src/ui/Stats.h @@ -49,8 +49,8 @@ class Stats : public QQuickItem { STATS_PROPERTY(int, simrate, 0) STATS_PROPERTY(int, avatarSimrate, 0) STATS_PROPERTY(int, avatarCount, 0) - STATS_PROPERTY(int, fullySimulatedAvatarCount, 0) - STATS_PROPERTY(int, partiallySimulatedAvatarCount, 0) + STATS_PROPERTY(int, updatedAvatarCount, 0) + STATS_PROPERTY(int, notUpdatedAvatarCount, 0) STATS_PROPERTY(int, packetInCount, 0) STATS_PROPERTY(int, packetOutCount, 0) STATS_PROPERTY(float, mbpsIn, 0) @@ -159,8 +159,8 @@ signals: void simrateChanged(); void avatarSimrateChanged(); void avatarCountChanged(); - void fullySimulatedAvatarCountChanged(); - void partiallySimulatedAvatarCountChanged(); + void updatedAvatarCountChanged(); + void notUpdatedAvatarCountChanged(); void packetInCountChanged(); void packetOutCountChanged(); void mbpsInChanged(); From 4c4506b1f7c12f23a3c3401c9b15cf1641806220 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 22 Feb 2017 21:47:40 -0800 Subject: [PATCH 092/104] some cleanup --- assignment-client/src/avatars/AvatarMixer.cpp | 20 ++++---- .../src/avatars/AvatarMixerClientData.h | 10 ++-- .../src/avatars/AvatarMixerSlave.cpp | 46 ++----------------- 3 files changed, 21 insertions(+), 55 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 59896432e7..50363f62e3 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -309,7 +309,8 @@ void AvatarMixer::nodeKilled(SharedNodePointer killedNode) { }, [&](const SharedNodePointer& node) { QMetaObject::invokeMethod(node->getLinkedData(), - "removeLastBroadcastSequenceNumber", + //"removeLastBroadcastSequenceNumber", + "cleanupKilledNode", Qt::AutoConnection, Q_ARG(const QUuid&, QUuid(killedNode->getUUID()))); } @@ -321,14 +322,17 @@ void AvatarMixer::nodeKilled(SharedNodePointer killedNode) { void AvatarMixer::handleAdjustAvatarSorting(QSharedPointer message, SharedNodePointer senderNode) { auto start = usecTimestampNow(); - message->readPrimitive(&AvatarData::_avatarSortCoefficientSize); - message->readPrimitive(&AvatarData::_avatarSortCoefficientCenter); - message->readPrimitive(&AvatarData::_avatarSortCoefficientAge); + // only allow admins with kick rights to change this value... + if (senderNode->getCanKick()) { + message->readPrimitive(&AvatarData::_avatarSortCoefficientSize); + message->readPrimitive(&AvatarData::_avatarSortCoefficientCenter); + message->readPrimitive(&AvatarData::_avatarSortCoefficientAge); - qCDebug(avatars) << "New avatar sorting... " - << "size:" << AvatarData::_avatarSortCoefficientSize - << "center:" << AvatarData::_avatarSortCoefficientCenter - << "age:" << AvatarData::_avatarSortCoefficientAge; + qCDebug(avatars) << "New avatar sorting... " + << "size:" << AvatarData::_avatarSortCoefficientSize + << "center:" << AvatarData::_avatarSortCoefficientCenter + << "age:" << AvatarData::_avatarSortCoefficientAge; + } auto end = usecTimestampNow(); _handleAdjustAvatarSortingElapsedTime += (end - start); diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index 7eea31c9ab..9cf73a94f4 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -53,11 +53,14 @@ public: Q_INVOKABLE void removeLastBroadcastSequenceNumber(const QUuid& nodeUUID) { _lastBroadcastSequenceNumbers.erase(nodeUUID); } uint64_t getLastBroadcastTime(const QUuid& nodeUUID) const; - void setLastBroadcastTime(const QUuid& nodeUUID, uint64_t broadcastTime) { - _lastBroadcastTimes[nodeUUID] = broadcastTime; - } + void setLastBroadcastTime(const QUuid& nodeUUID, uint64_t broadcastTime) { _lastBroadcastTimes[nodeUUID] = broadcastTime; } Q_INVOKABLE void removeLastBroadcastTime(const QUuid& nodeUUID) { _lastBroadcastTimes.erase(nodeUUID); } + Q_INVOKABLE void cleanupKilledNode(const QUuid& nodeUUID) { + removeLastBroadcastSequenceNumber(nodeUUID); + removeLastBroadcastTime(nodeUUID); + } + uint16_t getLastReceivedSequenceNumber() const { return _lastReceivedSequenceNumber; } HRCTime getIdentityChangeTimestamp() const { return _identityChangeTimestamp; } @@ -115,7 +118,6 @@ public: ViewFrustum getViewFrustom() const { return _currentViewFrustum; } - quint64 getLastOtherAvatarEncodeTime(QUuid otherAvatar) { quint64 result = 0; if (_lastOtherAvatarEncodeTime.find(otherAvatar) != _lastOtherAvatarEncodeTime.end()) { diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 360c6e26fd..5ad4561aa9 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -191,7 +191,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { QList avatarList; std::unordered_map avatarDataToNodes; - //qDebug() << "------------------------------"; int listItem = 0; std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) { const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); @@ -200,36 +199,14 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { AvatarSharedPointer otherAvatar = otherNodeData->getAvatarSharedPointer(); avatarList << otherAvatar; avatarDataToNodes[otherAvatar] = otherNode; - /* - qDebug() << "listItem [" << listItem << "] " - << "otherNode:" << otherNode.data() - << "otherNode->getUUID():" << otherNode->getUUID() - << "otherNodeData:" << otherNodeData - << "otherAvatar:" << otherAvatar.get(); - - qDebug() << "avatarDataToNodes[" << otherAvatar.get() << "]=" << otherNode.data(); - */ } }); - /* - qDebug() << "avatarList.size:" << avatarList.size(); - qDebug() << "avatarDataToNodes.size:" << avatarDataToNodes.size(); - */ AvatarSharedPointer thisAvatar = nodeData->getAvatarSharedPointer(); //qDebug() << "thisAvatar:" << thisAvatar.get(); -#ifdef WANT_DEBUG - bool printDebug = nodeData->getAvatarSharedPointer()->getDisplayName() == "ZappoMan"; - - if (printDebug) { - qDebug() << "------------------------------"; - } -#else - bool printDebug = false; -#endif ViewFrustum cameraView = nodeData->getViewFrustom(); std::priority_queue sortedAvatars = AvatarData::sortAvatars( avatarList, cameraView, @@ -239,18 +216,12 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { if (avatarNode) { return nodeData->getLastBroadcastTime(avatarNode->getUUID()); } - return 0; // ??? + return 0; }, [thisAvatar](AvatarSharedPointer avatar)->bool{ return (avatar == thisAvatar); // ignore ourselves... - }, printDebug); - -#ifdef WANT_DEBUG - if (printDebug) { - qDebug() << "------------------------------"; - } -#endif + }); // loop through our sorted avatars and allocate our bandwidth to them accordingly int avatarRank = 0; @@ -262,13 +233,9 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { auto otherNode = avatarDataToNodes[avatarData]; - //qDebug() << "otherNode (" << otherNode.data() << ")= avatarDataToNodes[" << avatarData.get() << "]"; - if (!otherNode) { - //qDebug() << "For viewer:" << node->getUUID() << "... process other avatar [" << avatarRank << ":" << avatarData.get() << "... otherNode unknown!!"; continue; } - //qDebug() << "For viewer:" << node->getUUID() << "... process other avatar [" << avatarRank << "] avatarData: " << avatarData.get() << " otherNode:" << otherNode->getUUID() << " ... "; bool shouldConsider = false; quint64 startIgnoreCalculation = usecTimestampNow(); @@ -282,8 +249,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { || (otherNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { shouldConsider = false; - //qDebug() << "shouldConsider = false ... line:" << __LINE__; - } else { const AvatarMixerClientData* otherData = reinterpret_cast(otherNode->getLinkedData()); @@ -318,8 +283,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { if (nodeBox.touches(otherNodeBox)) { nodeData->ignoreOther(node, otherNode); shouldConsider = getsAnyIgnored; - //qDebug() << "shouldConsider = getsAnyIgnored " << getsAnyIgnored << " ... line:" << __LINE__; - } } // Not close enough to ignore @@ -397,9 +360,7 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { quint64 endAvatarDataPacking = usecTimestampNow(); _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); shouldConsider = false; - //qDebug() << "shouldConsider = false ... line:" << __LINE__; - } - else if (lastSeqFromSender - lastSeqToReceiver > 1) { + } else if (lastSeqFromSender - lastSeqToReceiver > 1) { // this is a skip - we still send the packet but capture the presence of the skip so we see it happening ++numAvatarsWithSkippedFrames; } @@ -418,7 +379,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); shouldConsider = false; - //qDebug() << "shouldConsider = false ... line:" << __LINE__; } if (shouldConsider) { From edf7c016a12cd51ac5892d495a518142c44233e7 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 22 Feb 2017 22:06:28 -0800 Subject: [PATCH 093/104] more cleanup --- assignment-client/src/avatars/AvatarMixer.cpp | 1 - .../src/avatars/AvatarMixerSlave.cpp | 6 ++---- libraries/avatars/src/AvatarData.cpp | 17 +++-------------- libraries/avatars/src/AvatarData.h | 7 +------ 4 files changed, 6 insertions(+), 25 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 50363f62e3..a347b5ef47 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -309,7 +309,6 @@ void AvatarMixer::nodeKilled(SharedNodePointer killedNode) { }, [&](const SharedNodePointer& node) { QMetaObject::invokeMethod(node->getLinkedData(), - //"removeLastBroadcastSequenceNumber", "cleanupKilledNode", Qt::AutoConnection, Q_ARG(const QUuid&, QUuid(killedNode->getUUID()))); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 5ad4561aa9..9b501a5e31 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -187,7 +187,8 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // setup a PacketList for the avatarPackets auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); - + // setup list of AvatarData as well as maps to map betweeen the AvatarData and the original nodes + // for calling the AvatarData::sortAvatars() function and getting our sorted list of client nodes QList avatarList; std::unordered_map avatarDataToNodes; @@ -204,9 +205,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { }); AvatarSharedPointer thisAvatar = nodeData->getAvatarSharedPointer(); - - //qDebug() << "thisAvatar:" << thisAvatar.get(); - ViewFrustum cameraView = nodeData->getViewFrustom(); std::priority_queue sortedAvatars = AvatarData::sortAvatars( avatarList, cameraView, diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 7c1b1baf26..e2574683a2 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -69,8 +69,7 @@ AvatarData::AvatarData() : _displayNameAlpha(1.0f), _errorLogExpiry(0), _owningAvatarMixer(), - _targetVelocity(0.0f), - _localAABox(DEFAULT_LOCAL_AABOX_CORNER, DEFAULT_LOCAL_AABOX_SCALE) + _targetVelocity(0.0f) { setBodyPitch(0.0f); setBodyYaw(-90.0f); @@ -2321,8 +2320,7 @@ std::priority_queue AvatarData::sortAvatars( QList avatarList, const ViewFrustum& cameraView, std::function lastUpdated, - std::function shouldIgnore, - bool printDebug) { + std::function shouldIgnore) { uint64_t startTime = usecTimestampNow(); @@ -2335,8 +2333,6 @@ std::priority_queue AvatarData::sortAvatars( const auto& avatar = avatarList.at(i); bool outOfView = false; - // FIXME - probably some lambda that allows the caller to reject some avatars - if (shouldIgnore(avatar)) { continue; } @@ -2352,13 +2348,10 @@ std::priority_queue AvatarData::sortAvatars( // FIXME - AvatarData has something equivolent to this float radius = 1.0f; // avatar->getBoundingRadius(); - const glm::vec3& forward = cameraView.getDirection(); float apparentSize = 2.0f * radius / distance; float cosineAngle = glm::length(glm::dot(offset, forward) * forward) / distance; - - uint64_t lastUpdatedTime = lastUpdated(avatar); // avatar->getLastRenderUpdateTime() - float age = (float)(startTime - lastUpdatedTime) / (float)(USECS_PER_SECOND); + float age = (float)(startTime - lastUpdated(avatar)) / (float)(USECS_PER_SECOND); // NOTE: we are adding values of different units to get a single measure of "priority". // Thus we multiply each component by a conversion "weight" that scales its units relative to the others. @@ -2375,10 +2368,6 @@ std::priority_queue AvatarData::sortAvatars( outOfView = true; } } - - if (printDebug) { - qDebug() << "avatar:" << avatar.get() << "priority:" << priority << "apparentSize:" << apparentSize << "cosineAngle:" << cosineAngle << "age:" << age << "outOfView:" << outOfView; - } sortedAvatars.push(AvatarPriority(avatar, priority)); } } diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index daa496f6e1..8608137b20 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -550,8 +550,6 @@ public: void setOwningAvatarMixer(const QWeakPointer& owningAvatarMixer) { _owningAvatarMixer = owningAvatarMixer; } - //const AABox& getLocalAABox() const { return _localAABox; } - int getUsecsSinceLastUpdate() const { return _averageBytesReceived.getUsecsSinceLastEvent(); } int getAverageBytesReceivedPerSecond() const; int getReceiveRate() const; @@ -600,8 +598,7 @@ public: QList avatarList, const ViewFrustum& cameraView, std::function lastUpdated, - std::function shouldIgnore, - bool printDebug = false); + std::function shouldIgnore); // TODO: remove this HACK once we settle on optimal sort coefficients // These coefficients exposed for fine tuning the sort priority for transfering new _jointData to the render pipeline. @@ -692,8 +689,6 @@ protected: glm::vec3 _targetVelocity; - AABox _localAABox; - SimpleMovingAverage _averageBytesReceived; // During recording, this holds the starting position, orientation & scale of the recorded avatar From 06f0087459a56431f40cd6ec72f7501303b9c36d Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 22 Feb 2017 22:36:37 -0800 Subject: [PATCH 094/104] wire up radius properly --- assignment-client/src/avatars/AvatarMixerSlave.cpp | 5 +++++ interface/src/avatar/AvatarManager.cpp | 4 ++++ libraries/avatars/src/AvatarData.cpp | 7 ++++--- libraries/avatars/src/AvatarData.h | 3 ++- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 9b501a5e31..0004afa97c 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -217,6 +217,11 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { return 0; }, + [&](AvatarSharedPointer avatar)->float{ + glm::vec3 nodeBoxHalfScale = (avatar->getPosition() - avatar->getGlobalBoundingBoxCorner()); + return glm::max(nodeBoxHalfScale.x, glm::max(nodeBoxHalfScale.y, nodeBoxHalfScale.z)); + }, + [thisAvatar](AvatarSharedPointer avatar)->bool{ return (avatar == thisAvatar); // ignore ourselves... }); diff --git a/interface/src/avatar/AvatarManager.cpp b/interface/src/avatar/AvatarManager.cpp index af9f915fec..b1a09b46ce 100644 --- a/interface/src/avatar/AvatarManager.cpp +++ b/interface/src/avatar/AvatarManager.cpp @@ -171,6 +171,10 @@ void AvatarManager::updateOtherAvatars(float deltaTime) { return std::static_pointer_cast(avatar)->getLastRenderUpdateTime(); }, + [](AvatarSharedPointer avatar)->float{ + return std::static_pointer_cast(avatar)->getBoundingRadius(); + }, + [this](AvatarSharedPointer avatar)->bool{ const auto& castedAvatar = std::static_pointer_cast(avatar); if (castedAvatar == _myAvatar || !castedAvatar->isInitialized()) { diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index e2574683a2..0c145a6a30 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -2319,7 +2319,8 @@ float AvatarData::_avatarSortCoefficientAge { 1.0f }; std::priority_queue AvatarData::sortAvatars( QList avatarList, const ViewFrustum& cameraView, - std::function lastUpdated, + std::function getLastUpdated, + std::function getBoundingRadius, std::function shouldIgnore) { uint64_t startTime = usecTimestampNow(); @@ -2346,12 +2347,12 @@ std::priority_queue AvatarData::sortAvatars( float distance = glm::length(offset) + 0.001f; // add 1mm to avoid divide by zero // FIXME - AvatarData has something equivolent to this - float radius = 1.0f; // avatar->getBoundingRadius(); + float radius = getBoundingRadius(avatar); const glm::vec3& forward = cameraView.getDirection(); float apparentSize = 2.0f * radius / distance; float cosineAngle = glm::length(glm::dot(offset, forward) * forward) / distance; - float age = (float)(startTime - lastUpdated(avatar)) / (float)(USECS_PER_SECOND); + float age = (float)(startTime - getLastUpdated(avatar)) / (float)(USECS_PER_SECOND); // NOTE: we are adding values of different units to get a single measure of "priority". // Thus we multiply each component by a conversion "weight" that scales its units relative to the others. diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 8608137b20..0eddc29cda 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -597,7 +597,8 @@ public: static std::priority_queue sortAvatars( QList avatarList, const ViewFrustum& cameraView, - std::function lastUpdated, + std::function getLastUpdated, + std::function getBoundingRadius, std::function shouldIgnore); // TODO: remove this HACK once we settle on optimal sort coefficients From 40037bee55d063ae1f81c3811e9c7623370f37c5 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 22 Feb 2017 23:28:40 -0800 Subject: [PATCH 095/104] added some stats for the random drops --- assignment-client/src/avatars/AvatarMixer.cpp | 8 +++++++- assignment-client/src/avatars/AvatarMixerSlave.cpp | 10 +++++++--- assignment-client/src/avatars/AvatarMixerSlave.h | 6 ++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index a347b5ef47..a90fdb100c 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -506,6 +506,9 @@ void AvatarMixer::sendStatsPacket() { float averageOthersIncluded = averageNodes ? stats.numOthersIncluded / averageNodes : 0.0f; slaveObject["sent_6_averageOthersIncluded"] = TIGHT_LOOP_STAT(averageOthersIncluded); + float averageRandomDrops = averageNodes ? stats.randomDrops / averageNodes : 0.0f; + slaveObject["sent_7_averageRandomDrops"] = TIGHT_LOOP_STAT(averageRandomDrops); + slaveObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT_UINT64(stats.processIncomingPacketsElapsedTime); slaveObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT_UINT64(stats.ignoreCalculationElapsedTime); slaveObject["timing_3_toByteArray"] = TIGHT_LOOP_STAT_UINT64(stats.toByteArrayElapsedTime); @@ -533,9 +536,12 @@ void AvatarMixer::sendStatsPacket() { float averageOutboundAvatarKbps = averageNodes ? ((aggregateStats.numBytesSent / secondsSinceLastStats) / BYTES_PER_KILOBIT) / averageNodes : 0.0f; slavesAggregatObject["sent_5_averageOutboundAvatarKbps"] = averageOutboundAvatarKbps; + float averageRandomDrops = averageNodes ? aggregateStats.randomDrops / averageNodes : 0.0f; + slavesAggregatObject["sent_7_averageRandomDrops"] = TIGHT_LOOP_STAT(averageRandomDrops); + float averageOthersIncluded = averageNodes ? aggregateStats.numOthersIncluded / averageNodes : 0.0f; slavesAggregatObject["sent_6_averageOthersIncluded"] = TIGHT_LOOP_STAT(averageOthersIncluded); - + slavesAggregatObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.processIncomingPacketsElapsedTime); slavesAggregatObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.ignoreCalculationElapsedTime); slavesAggregatObject["timing_3_toByteArray"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.toByteArrayElapsedTime); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 0004afa97c..5073aab934 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -332,15 +332,19 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // // NOTE: If the recieving node is in "PAL mode" then it's asked to get things even that // are out of view, this also appears to disable this random distribution. + // + // FIXME - This approach for managing the outbound bandwidth is less than ideal, + // it would be better to more directly budget the number of bytes to send + // per frame and simply exit the sorted avatar list once that budget is + // surpassed. We will work on that next. [BHG 2/22/17] if (distanceToAvatar != 0.0f && !getsOutOfView - && distribution(generator) > (nodeData->getFullRateDistance() / distanceToAvatar) /// FIX ME... we don't want to do this random stuff + && distribution(generator) > (nodeData->getFullRateDistance() / distanceToAvatar) ) { - quint64 endAvatarDataPacking = usecTimestampNow(); _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); shouldConsider = false; - //qDebug() << "shouldConsider = false ... line:" << __LINE__; + _stats.randomDrops++; } if (shouldConsider) { diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index b1a5b56c4f..50ae4570b5 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -25,6 +25,8 @@ public: int numBytesSent { 0 }; int numIdentityPackets { 0 }; int numOthersIncluded { 0 }; + int randomDrops { 0 }; + quint64 ignoreCalculationElapsedTime { 0 }; quint64 avatarDataPackingElapsedTime { 0 }; quint64 packetSendingElapsedTime { 0 }; @@ -43,6 +45,8 @@ public: numBytesSent = 0; numIdentityPackets = 0; numOthersIncluded = 0; + randomDrops = 0; + ignoreCalculationElapsedTime = 0; avatarDataPackingElapsedTime = 0; packetSendingElapsedTime = 0; @@ -60,6 +64,8 @@ public: numBytesSent += rhs.numBytesSent; numIdentityPackets += rhs.numIdentityPackets; numOthersIncluded += rhs.numOthersIncluded; + randomDrops += rhs.randomDrops; + ignoreCalculationElapsedTime += rhs.ignoreCalculationElapsedTime; avatarDataPackingElapsedTime += rhs.avatarDataPackingElapsedTime; packetSendingElapsedTime += rhs.packetSendingElapsedTime; From d4adee8b388d6c2cb482f8e0792316892594a68c Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 23 Feb 2017 00:15:07 -0800 Subject: [PATCH 096/104] add over bandwidth exit to sorted avatar loop --- assignment-client/src/avatars/AvatarMixer.cpp | 10 ++++++-- .../src/avatars/AvatarMixerSlave.cpp | 23 +++++++++++++++---- .../src/avatars/AvatarMixerSlave.h | 3 +++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index a90fdb100c..6db9d05234 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -508,6 +508,9 @@ void AvatarMixer::sendStatsPacket() { float averageRandomDrops = averageNodes ? stats.randomDrops / averageNodes : 0.0f; slaveObject["sent_7_averageRandomDrops"] = TIGHT_LOOP_STAT(averageRandomDrops); + + float averageOverBudgetAvatars = averageNodes ? stats.overBudgetAvatars / averageNodes : 0.0f; + slaveObject["sent_8_averageOverBudgetAvatars"] = TIGHT_LOOP_STAT(averageOverBudgetAvatars); slaveObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT_UINT64(stats.processIncomingPacketsElapsedTime); slaveObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT_UINT64(stats.ignoreCalculationElapsedTime); @@ -536,11 +539,14 @@ void AvatarMixer::sendStatsPacket() { float averageOutboundAvatarKbps = averageNodes ? ((aggregateStats.numBytesSent / secondsSinceLastStats) / BYTES_PER_KILOBIT) / averageNodes : 0.0f; slavesAggregatObject["sent_5_averageOutboundAvatarKbps"] = averageOutboundAvatarKbps; + float averageOthersIncluded = averageNodes ? aggregateStats.numOthersIncluded / averageNodes : 0.0f; + slavesAggregatObject["sent_6_averageOthersIncluded"] = TIGHT_LOOP_STAT(averageOthersIncluded); + float averageRandomDrops = averageNodes ? aggregateStats.randomDrops / averageNodes : 0.0f; slavesAggregatObject["sent_7_averageRandomDrops"] = TIGHT_LOOP_STAT(averageRandomDrops); - float averageOthersIncluded = averageNodes ? aggregateStats.numOthersIncluded / averageNodes : 0.0f; - slavesAggregatObject["sent_6_averageOthersIncluded"] = TIGHT_LOOP_STAT(averageOthersIncluded); + float averageOverBudgetAvatars = averageNodes ? aggregateStats.overBudgetAvatars / averageNodes : 0.0f; + slavesAggregatObject["sent_8_averageOverBudgetAvatars"] = TIGHT_LOOP_STAT(averageOverBudgetAvatars); slavesAggregatObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.processIncomingPacketsElapsedTime); slavesAggregatObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.ignoreCalculationElapsedTime); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 5073aab934..21eaebc176 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -129,6 +129,11 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // keep track of outbound data rate specifically for avatar data int numAvatarDataBytes = 0; + // max number of avatarBytes per frame + auto maxAvatarBytesPerFrame = (_maxKbpsPerNode * BYTES_PER_KILOBIT) / AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND; + + int overBudgetAvatars = 0; + // keep track of the number of other avatars held back in this frame int numAvatarsHeldBack = 0; @@ -333,10 +338,10 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // NOTE: If the recieving node is in "PAL mode" then it's asked to get things even that // are out of view, this also appears to disable this random distribution. // - // FIXME - This approach for managing the outbound bandwidth is less than ideal, - // it would be better to more directly budget the number of bytes to send - // per frame and simply exit the sorted avatar list once that budget is - // surpassed. We will work on that next. [BHG 2/22/17] + // FIXME - This is the old approach for managing the outbound bandwidth. I've left it + // in for now, even though we're also directly managing budget by calculating the + // number of bytes to send per frame and simply exiting the sorted avatar list + // once that budget is surpassed. I need to remove this old logic next. [BHG 2/22/17] if (distanceToAvatar != 0.0f && !getsOutOfView && distribution(generator) > (nodeData->getFullRateDistance() / distanceToAvatar) @@ -372,6 +377,16 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { ++numAvatarsWithSkippedFrames; } + // NOTE: Here's where we determine if we are over budget and stop considering avatars after this. + // + // FIXME - revisit this code, remove the random drop logic, and move this code higher up into + // the loop so we don't bother with all these other calculations once over budget. [BHG 2/22/17] + if (numAvatarDataBytes > maxAvatarBytesPerFrame) { + overBudgetAvatars++; + _stats.overBudgetAvatars++; + shouldConsider = false; + } + // we're going to send this avatar if (shouldConsider) { diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index 50ae4570b5..1bbd367b2b 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -26,6 +26,7 @@ public: int numIdentityPackets { 0 }; int numOthersIncluded { 0 }; int randomDrops { 0 }; + int overBudgetAvatars { 0 }; quint64 ignoreCalculationElapsedTime { 0 }; quint64 avatarDataPackingElapsedTime { 0 }; @@ -46,6 +47,7 @@ public: numIdentityPackets = 0; numOthersIncluded = 0; randomDrops = 0; + overBudgetAvatars = 0; ignoreCalculationElapsedTime = 0; avatarDataPackingElapsedTime = 0; @@ -65,6 +67,7 @@ public: numIdentityPackets += rhs.numIdentityPackets; numOthersIncluded += rhs.numOthersIncluded; randomDrops += rhs.randomDrops; + overBudgetAvatars += rhs.overBudgetAvatars; ignoreCalculationElapsedTime += rhs.ignoreCalculationElapsedTime; avatarDataPackingElapsedTime += rhs.avatarDataPackingElapsedTime; From 4f17498daf5c22d1e67e302d6b77ae248a5d8e49 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 23 Feb 2017 00:19:06 -0800 Subject: [PATCH 097/104] fix warning --- libraries/avatars/src/AvatarData.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 0c145a6a30..4993c0ac5d 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -2332,7 +2332,6 @@ std::priority_queue AvatarData::sortAvatars( PROFILE_RANGE(simulation, "sort"); for (int32_t i = 0; i < avatarList.size(); ++i) { const auto& avatar = avatarList.at(i); - bool outOfView = false; if (shouldIgnore(avatar)) { continue; @@ -2366,7 +2365,6 @@ std::priority_queue AvatarData::sortAvatars( if (distance > cameraView.getCenterRadius()) { if (!cameraView.sphereIntersectsFrustum(avatarPosition, radius)) { priority += OUT_OF_VIEW_PENALTY; - outOfView = true; } } sortedAvatars.push(AvatarPriority(avatar, priority)); From 5033e7be11bfd59654416b92662bd0774a56af8b Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Thu, 23 Feb 2017 18:22:36 +0100 Subject: [PATCH 098/104] CR fixes --- interface/src/avatar/MyAvatar.cpp | 2 +- libraries/audio-client/src/AudioClient.cpp | 46 +++++++++++----------- libraries/audio-client/src/AudioClient.h | 10 ++--- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index b9f673287b..19d403a5c5 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -160,7 +160,7 @@ MyAvatar::MyAvatar(RigPointer rig) : } auto audioIO = DependencyManager::get(); - audioIO->setPlayingBackRecording(isPlaying); + audioIO->setIsPlayingBackRecording(isPlaying); if (_rig) { _rig->setEnableAnimations(!isPlaying); diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 0e2991ae30..0e06cf8661 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -148,17 +148,17 @@ static inline float convertToFloat(int16_t sample) { AudioClient::AudioClient() : AbstractAudioInterface(), _gate(this), - _audioInput(nullptr), + _audioInput(NULL), _desiredInputFormat(), _inputFormat(), _numInputCallbackBytes(0), - _audioOutput(nullptr), + _audioOutput(NULL), _desiredOutputFormat(), _outputFormat(), _outputFrameSize(0), _numOutputCallbackBytes(0), - _loopbackAudioOutput(nullptr), - _loopbackOutputDevice(nullptr), + _loopbackAudioOutput(NULL), + _loopbackOutputDevice(NULL), _inputRingBuffer(0), _localInjectorsStream(0), _receivedAudioStream(RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES), @@ -176,9 +176,9 @@ AudioClient::AudioClient() : _isNoiseGateEnabled(true), _reverb(false), _reverbOptions(&_scriptReverbOptions), - _inputToNetworkResampler(nullptr), - _networkToOutputResampler(nullptr), - _localToOutputResampler(nullptr), + _inputToNetworkResampler(NULL), + _networkToOutputResampler(NULL), + _localToOutputResampler(NULL), _localAudioThread(this), _audioLimiter(AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT), _outgoingAvatarAudioSequenceNumber(0), @@ -393,9 +393,9 @@ QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { } } else { HRESULT hr = S_OK; - CoInitialize(nullptr); - IMMDeviceEnumerator* pMMDeviceEnumerator = nullptr; - CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), (void**)&pMMDeviceEnumerator); + CoInitialize(NULL); + IMMDeviceEnumerator* pMMDeviceEnumerator = NULL; + CoCreateInstance(__uuidof(MMDeviceEnumerator), NULL, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), (void**)&pMMDeviceEnumerator); IMMDevice* pEndpoint; hr = pMMDeviceEnumerator->GetDefaultAudioEndpoint(mode == QAudio::AudioOutput ? eRender : eCapture, eMultimedia, &pEndpoint); if (hr == E_NOTFOUND) { @@ -405,7 +405,7 @@ QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { deviceName = friendlyNameForAudioDevice(pEndpoint); } pMMDeviceEnumerator->Release(); - pMMDeviceEnumerator = nullptr; + pMMDeviceEnumerator = NULL; CoUninitialize(); } @@ -965,7 +965,7 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { } void AudioClient::handleAudioInput() { - if (!_inputDevice || _playingBackRecording) { + if (!_inputDevice || _isPlayingBackRecording) { return; } @@ -1354,10 +1354,10 @@ bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceIn // That in turn causes it to be disconnected (see for example // http://stackoverflow.com/questions/9264750/qt-signals-and-slots-object-disconnect). _audioInput->stop(); - _inputDevice = nullptr; + _inputDevice = NULL; delete _audioInput; - _audioInput = nullptr; + _audioInput = NULL; _numInputCallbackBytes = 0; _inputAudioDeviceName = ""; @@ -1366,7 +1366,7 @@ bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceIn if (_inputToNetworkResampler) { // if we were using an input to network resampler, delete it here delete _inputToNetworkResampler; - _inputToNetworkResampler = nullptr; + _inputToNetworkResampler = NULL; } if (!inputDeviceInfo.isNull()) { @@ -1461,29 +1461,29 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice _audioOutput->stop(); delete _audioOutput; - _audioOutput = nullptr; + _audioOutput = NULL; - _loopbackOutputDevice = nullptr; + _loopbackOutputDevice = NULL; delete _loopbackAudioOutput; - _loopbackAudioOutput = nullptr; + _loopbackAudioOutput = NULL; delete[] _outputMixBuffer; - _outputMixBuffer = nullptr; + _outputMixBuffer = NULL; delete[] _outputScratchBuffer; - _outputScratchBuffer = nullptr; + _outputScratchBuffer = NULL; delete[] _localOutputMixBuffer; - _localOutputMixBuffer = nullptr; + _localOutputMixBuffer = NULL; } if (_networkToOutputResampler) { // if we were using an input to network resampler, delete it here delete _networkToOutputResampler; - _networkToOutputResampler = nullptr; + _networkToOutputResampler = NULL; delete _localToOutputResampler; - _localToOutputResampler = nullptr; + _localToOutputResampler = NULL; } if (!outputDeviceInfo.isNull()) { diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 739c4b459c..072990e0c5 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -145,7 +145,7 @@ public: void setPositionGetter(AudioPositionGetter positionGetter) { _positionGetter = positionGetter; } void setOrientationGetter(AudioOrientationGetter orientationGetter) { _orientationGetter = orientationGetter; } - void setPlayingBackRecording(bool playingBackRecording) { _playingBackRecording = playingBackRecording; } + void setIsPlayingBackRecording(bool isPlayingBackRecording) { _isPlayingBackRecording = isPlayingBackRecording; } Q_INVOKABLE void setAvatarBoundingBoxParameters(glm::vec3 corner, glm::vec3 scale); @@ -328,14 +328,14 @@ private: // for local audio (used by audio injectors thread) float _localMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; - float* _localOutputMixBuffer { nullptr }; + float* _localOutputMixBuffer { NULL }; AudioInjectorsThread _localAudioThread; Mutex _localAudioMutex; // for output audio (used by this thread) int _outputPeriod { 0 }; - float* _outputMixBuffer { nullptr }; - int16_t* _outputScratchBuffer { nullptr }; + float* _outputMixBuffer { NULL }; + int16_t* _outputScratchBuffer { NULL }; AudioLimiter _audioLimiter; @@ -373,7 +373,7 @@ private: QVector _activeLocalAudioInjectors; - bool _playingBackRecording { false }; + bool _isPlayingBackRecording { false }; CodecPluginPointer _codec; QString _selectedCodecName; From dde9640c66a9be73e8c65c8d87a3b12d22f6a787 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 23 Feb 2017 10:29:17 -0800 Subject: [PATCH 099/104] remove old full rate distance code --- assignment-client/src/avatars/AvatarMixer.cpp | 10 +- .../src/avatars/AvatarMixerClientData.cpp | 2 - .../src/avatars/AvatarMixerClientData.h | 9 -- .../src/avatars/AvatarMixerSlave.cpp | 131 ++++++------------ .../src/avatars/AvatarMixerSlave.h | 5 +- libraries/avatars/src/AvatarData.cpp | 7 + libraries/avatars/src/AvatarData.h | 2 + 7 files changed, 53 insertions(+), 113 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 6db9d05234..0f6863f9ae 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -506,11 +506,8 @@ void AvatarMixer::sendStatsPacket() { float averageOthersIncluded = averageNodes ? stats.numOthersIncluded / averageNodes : 0.0f; slaveObject["sent_6_averageOthersIncluded"] = TIGHT_LOOP_STAT(averageOthersIncluded); - float averageRandomDrops = averageNodes ? stats.randomDrops / averageNodes : 0.0f; - slaveObject["sent_7_averageRandomDrops"] = TIGHT_LOOP_STAT(averageRandomDrops); - float averageOverBudgetAvatars = averageNodes ? stats.overBudgetAvatars / averageNodes : 0.0f; - slaveObject["sent_8_averageOverBudgetAvatars"] = TIGHT_LOOP_STAT(averageOverBudgetAvatars); + slaveObject["sent_7_averageOverBudgetAvatars"] = TIGHT_LOOP_STAT(averageOverBudgetAvatars); slaveObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT_UINT64(stats.processIncomingPacketsElapsedTime); slaveObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT_UINT64(stats.ignoreCalculationElapsedTime); @@ -542,11 +539,8 @@ void AvatarMixer::sendStatsPacket() { float averageOthersIncluded = averageNodes ? aggregateStats.numOthersIncluded / averageNodes : 0.0f; slavesAggregatObject["sent_6_averageOthersIncluded"] = TIGHT_LOOP_STAT(averageOthersIncluded); - float averageRandomDrops = averageNodes ? aggregateStats.randomDrops / averageNodes : 0.0f; - slavesAggregatObject["sent_7_averageRandomDrops"] = TIGHT_LOOP_STAT(averageRandomDrops); - float averageOverBudgetAvatars = averageNodes ? aggregateStats.overBudgetAvatars / averageNodes : 0.0f; - slavesAggregatObject["sent_8_averageOverBudgetAvatars"] = TIGHT_LOOP_STAT(averageOverBudgetAvatars); + slavesAggregatObject["sent_7_averageOverBudgetAvatars"] = TIGHT_LOOP_STAT(averageOverBudgetAvatars); slavesAggregatObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.processIncomingPacketsElapsedTime); slavesAggregatObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.ignoreCalculationElapsedTime); diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index 1df6c24a60..717e14535f 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -126,8 +126,6 @@ bool AvatarMixerClientData::otherAvatarInView(const AABox& otherAvatarBox) { void AvatarMixerClientData::loadJSONStats(QJsonObject& jsonObject) const { jsonObject["display_name"] = _avatar->getDisplayName(); - jsonObject["full_rate_distance"] = _fullRateDistance; - jsonObject["max_av_distance"] = _maxAvatarDistance; jsonObject["num_avs_sent_last_frame"] = _numAvatarsSentLastFrame; jsonObject["avg_other_av_starves_per_second"] = getAvgNumOtherAvatarStarvesPerSecond(); jsonObject["avg_other_av_skips_per_second"] = getAvgNumOtherAvatarSkipsPerSecond(); diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index 9cf73a94f4..51b8d692e2 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -68,12 +68,6 @@ public: bool getAvatarSessionDisplayNameMustChange() const { return _avatarSessionDisplayNameMustChange; } void setAvatarSessionDisplayNameMustChange(bool set = true) { _avatarSessionDisplayNameMustChange = set; } - void setFullRateDistance(float fullRateDistance) { _fullRateDistance = fullRateDistance; } - float getFullRateDistance() const { return _fullRateDistance; } - - void setMaxAvatarDistance(float maxAvatarDistance) { _maxAvatarDistance = maxAvatarDistance; } - float getMaxAvatarDistance() const { return _maxAvatarDistance; } - void resetNumAvatarsSentLastFrame() { _numAvatarsSentLastFrame = 0; } void incrementNumAvatarsSentLastFrame() { ++_numAvatarsSentLastFrame; } int getNumAvatarsSentLastFrame() const { return _numAvatarsSentLastFrame; } @@ -156,9 +150,6 @@ private: HRCTime _identityChangeTimestamp; bool _avatarSessionDisplayNameMustChange{ false }; - float _fullRateDistance = FLT_MAX; - float _maxAvatarDistance = FLT_MAX; - int _numAvatarsSentLastFrame = 0; int _numFramesSinceAdjustment = 0; diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 21eaebc176..1f9fbde89a 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -66,13 +66,16 @@ void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { } -void AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { +int AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { + int bytesSent = 0; QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); auto identityPacket = NLPacket::create(PacketType::AvatarIdentity, individualData.size()); - individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); + individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious + bytesSent += individualData.size(); identityPacket->write(individualData); DependencyManager::get()->sendPacket(std::move(identityPacket), *destinationNode); _stats.numIdentityPackets++; + return bytesSent; } static const int AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND = 45; @@ -117,9 +120,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // reset the internal state for correct random number distribution distribution.reset(); - // reset the max distance for this frame - float maxAvatarDistanceThisFrame = 0.0f; - // reset the number of sent avatars nodeData->resetNumAvatarsSentLastFrame(); @@ -128,10 +128,14 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // keep track of outbound data rate specifically for avatar data int numAvatarDataBytes = 0; + int identityBytesSent = 0; // max number of avatarBytes per frame auto maxAvatarBytesPerFrame = (_maxKbpsPerNode * BYTES_PER_KILOBIT) / AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND; + // FIXME - find a way to not send the sessionID for every avatar + int minimumBytesPerAvatar = AvatarDataPacket::AVATAR_HAS_FLAGS_SIZE + NUM_BYTES_RFC4122_UUID; + int overBudgetAvatars = 0; // keep track of the number of other avatars held back in this frame @@ -140,9 +144,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // keep track of the number of other avatar frames skipped int numAvatarsWithSkippedFrames = 0; - // use the data rate specifically for avatar data for FRD adjustment checks - float avatarDataRateLastSecond = nodeData->getOutboundAvatarDataKbps(); - // When this is true, the AvatarMixer will send Avatar data to a client about avatars that are not in the view frustrum bool getsOutOfView = nodeData->getRequestsDomainListData(); @@ -152,43 +153,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // When this is true, the AvatarMixer will send Avatar data to a client about avatars that have ignored them bool getsAnyIgnored = getsIgnoredByMe && node->getCanKick(); - // Check if it is time to adjust what we send this client based on the observed - // bandwidth to this node. We do this once a second, which is also the window for - // the bandwidth reported by node->getOutboundBandwidth(); - if (nodeData->getNumFramesSinceFRDAdjustment() > AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND) { - - const float FRD_ADJUSTMENT_ACCEPTABLE_RATIO = 0.8f; - const float HYSTERISIS_GAP = (1 - FRD_ADJUSTMENT_ACCEPTABLE_RATIO); - const float HYSTERISIS_MIDDLE_PERCENTAGE = (1 - (HYSTERISIS_GAP * 0.5f)); - - // get the current full rate distance so we can work with it - float currentFullRateDistance = nodeData->getFullRateDistance(); - - if (avatarDataRateLastSecond > _maxKbpsPerNode) { - - // is the FRD greater than the farthest avatar? - // if so, before we calculate anything, set it to that distance - currentFullRateDistance = std::min(currentFullRateDistance, nodeData->getMaxAvatarDistance()); - - // we're adjusting the full rate distance to target a bandwidth in the middle - // of the hysterisis gap - currentFullRateDistance *= (_maxKbpsPerNode * HYSTERISIS_MIDDLE_PERCENTAGE) / avatarDataRateLastSecond; - - nodeData->setFullRateDistance(currentFullRateDistance); - nodeData->resetNumFramesSinceFRDAdjustment(); - } else if (currentFullRateDistance < nodeData->getMaxAvatarDistance() - && avatarDataRateLastSecond < _maxKbpsPerNode * FRD_ADJUSTMENT_ACCEPTABLE_RATIO) { - // we are constrained AND we've recovered to below the acceptable ratio - // lets adjust the full rate distance to target a bandwidth in the middle of the hyterisis gap - currentFullRateDistance *= (_maxKbpsPerNode * HYSTERISIS_MIDDLE_PERCENTAGE) / avatarDataRateLastSecond; - - nodeData->setFullRateDistance(currentFullRateDistance); - nodeData->resetNumFramesSinceFRDAdjustment(); - } - } else { - nodeData->incrementNumFramesSinceFRDAdjustment(); - } - // setup a PacketList for the avatarPackets auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); @@ -233,18 +197,39 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // loop through our sorted avatars and allocate our bandwidth to them accordingly int avatarRank = 0; + + // this is overly conservative, because it includes some avatars we might not consider + // FIXME - move the ignore logic up into the sorting list so we get a better estimate + int remainingAvatars = (int)sortedAvatars.size(); + while (!sortedAvatars.empty()) { AvatarPriority sortData = sortedAvatars.top(); sortedAvatars.pop(); const auto& avatarData = sortData.avatar; avatarRank++; + remainingAvatars--; auto otherNode = avatarDataToNodes[avatarData]; + // FIXME - should't this be an assert? if (!otherNode) { continue; } + // NOTE: Here's where we determine if we are over budget and drop to bare minimum data + int minimRemainingAvatarBytes = minimumBytesPerAvatar * remainingAvatars; + bool overBudget = (identityBytesSent + numAvatarDataBytes + minimRemainingAvatarBytes) > maxAvatarBytesPerFrame; + /** + qDebug() << "Budget debugging:" + << "numAvatarDataBytes:" << numAvatarDataBytes << "\n" + << "identityBytesSent:" << identityBytesSent << "\n" + << "remainingAvatars:" << remainingAvatars << "\n" + << "minimumBytesPerAvatar:" << minimumBytesPerAvatar << "\n" + << "minimRemainingAvatarBytes:" << minimRemainingAvatarBytes << "\n" + << "overBudget:" << overBudget << "\n" + << "overBudgetAvatars:" << overBudgetAvatars; + **/ + bool shouldConsider = false; quint64 startIgnoreCalculation = usecTimestampNow(); @@ -313,45 +298,21 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { bool forceSend = !nodeData->checkAndSetHasReceivedFirstPacketsFrom(otherNode->getUUID()); // FIXME - this clause seems suspicious "... || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp ..." - if (otherNodeData->getIdentityChangeTimestamp().time_since_epoch().count() > 0 + if (!overBudget + && otherNodeData->getIdentityChangeTimestamp().time_since_epoch().count() > 0 && (forceSend || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp || distribution(generator) < IDENTITY_SEND_PROBABILITY)) { - sendIdentityPacket(otherNodeData, node); + identityBytesSent += sendIdentityPacket(otherNodeData, node); } const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); // Decide whether to send this avatar's data based on it's distance from us - // The full rate distance is the distance at which EVERY update will be sent for this avatar - // at twice the full rate distance, there will be a 50% chance of sending this avatar's update glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition(); float distanceToAvatar = glm::length(myPosition - otherPosition); - // potentially update the max full rate distance for this frame - maxAvatarDistanceThisFrame = std::max(maxAvatarDistanceThisFrame, distanceToAvatar); - - // This code handles the random dropping of avatar data based on the ratio of - // "getFullRateDistance" to actual distance. - // - // NOTE: If the recieving node is in "PAL mode" then it's asked to get things even that - // are out of view, this also appears to disable this random distribution. - // - // FIXME - This is the old approach for managing the outbound bandwidth. I've left it - // in for now, even though we're also directly managing budget by calculating the - // number of bytes to send per frame and simply exiting the sorted avatar list - // once that budget is surpassed. I need to remove this old logic next. [BHG 2/22/17] - if (distanceToAvatar != 0.0f - && !getsOutOfView - && distribution(generator) > (nodeData->getFullRateDistance() / distanceToAvatar) - ) { - quint64 endAvatarDataPacking = usecTimestampNow(); - _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - shouldConsider = false; - _stats.randomDrops++; - } - if (shouldConsider) { AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(otherNode->getUUID()); AvatarDataSequenceNumber lastSeqFromSender = otherNodeData->getLastReceivedSequenceNumber(); @@ -377,16 +338,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { ++numAvatarsWithSkippedFrames; } - // NOTE: Here's where we determine if we are over budget and stop considering avatars after this. - // - // FIXME - revisit this code, remove the random drop logic, and move this code higher up into - // the loop so we don't bother with all these other calculations once over budget. [BHG 2/22/17] - if (numAvatarDataBytes > maxAvatarBytesPerFrame) { - overBudgetAvatars++; - _stats.overBudgetAvatars++; - shouldConsider = false; - } - // we're going to send this avatar if (shouldConsider) { @@ -396,19 +347,26 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { bool isInView = nodeData->otherAvatarInView(otherNodeBox); // this throttles the extra data to only be sent every Nth message + /** if (!isInView && !getsOutOfView && (lastSeqToReceiver % EXTRA_AVATAR_DATA_FRAME_RATIO > 0)) { quint64 endAvatarDataPacking = usecTimestampNow(); _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); shouldConsider = false; } + ***/ if (shouldConsider) { // start a new segment in the PacketList for this avatar avatarPacketList->startSegment(); AvatarData::AvatarDataDetail detail; - if (!isInView && !getsOutOfView) { + + if (overBudget) { + overBudgetAvatars++; + _stats.overBudgetAvatars++; + detail = AvatarData::NoData; + } else if (!isInView && !getsOutOfView) { detail = AvatarData::MinimumData; nodeData->incrementAvatarOutOfView(); } else { @@ -498,13 +456,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { nodeData->recordNumOtherAvatarStarves(numAvatarsHeldBack); nodeData->recordNumOtherAvatarSkips(numAvatarsWithSkippedFrames); - if (numOtherAvatars == 0) { - // update the full rate distance to FLOAT_MAX since we didn't have any other avatars to send - nodeData->setMaxAvatarDistance(FLT_MAX); - } else { - nodeData->setMaxAvatarDistance(maxAvatarDistanceThisFrame); - } - quint64 endPacketSending = usecTimestampNow(); _stats.packetSendingElapsedTime += (endPacketSending - startPacketSending); } diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index 1bbd367b2b..04141d9d72 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -25,7 +25,6 @@ public: int numBytesSent { 0 }; int numIdentityPackets { 0 }; int numOthersIncluded { 0 }; - int randomDrops { 0 }; int overBudgetAvatars { 0 }; quint64 ignoreCalculationElapsedTime { 0 }; @@ -46,7 +45,6 @@ public: numBytesSent = 0; numIdentityPackets = 0; numOthersIncluded = 0; - randomDrops = 0; overBudgetAvatars = 0; ignoreCalculationElapsedTime = 0; @@ -66,7 +64,6 @@ public: numBytesSent += rhs.numBytesSent; numIdentityPackets += rhs.numIdentityPackets; numOthersIncluded += rhs.numOthersIncluded; - randomDrops += rhs.randomDrops; overBudgetAvatars += rhs.overBudgetAvatars; ignoreCalculationElapsedTime += rhs.ignoreCalculationElapsedTime; @@ -94,7 +91,7 @@ public: void harvestStats(AvatarMixerSlaveStats& stats); private: - void sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode); + int sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode); // frame state ConstIter _begin; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 4993c0ac5d..6a01c2930c 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -192,6 +192,13 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent unsigned char* destinationBuffer = reinterpret_cast(avatarDataByteArray.data()); unsigned char* startPosition = destinationBuffer; + // special case, if we were asked for no data, then just include the flags all set to nothing + if (dataDetail == NoData) { + AvatarDataPacket::HasFlags packetStateFlags = 0; + memcpy(destinationBuffer, &packetStateFlags, sizeof(packetStateFlags)); + return avatarDataByteArray.left(sizeof(packetStateFlags)); + } + // FIXME - // // BUG -- if you enter a space bubble, and then back away, the avatar has wrong orientation until "send all" happens... diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 0eddc29cda..3797132274 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -136,6 +136,7 @@ namespace AvatarDataPacket { const HasFlags PACKET_HAS_AVATAR_LOCAL_POSITION = 1U << 9; const HasFlags PACKET_HAS_FACE_TRACKER_INFO = 1U << 10; const HasFlags PACKET_HAS_JOINT_DATA = 1U << 11; + const size_t AVATAR_HAS_FLAGS_SIZE = 2; // NOTE: AvatarDataPackets start with a uint16_t sequence number that is not reflected in the Header structure. @@ -373,6 +374,7 @@ public: void setHandPosition(const glm::vec3& handPosition); typedef enum { + NoData, MinimumData, CullSmallData, IncludeSmallData, From 99e51e88ae3e8265611428efb9de1fba4ce26aed Mon Sep 17 00:00:00 2001 From: kunalgosar Date: Fri, 3 Feb 2017 15:13:11 -0800 Subject: [PATCH 100/104] Add QMetaObject for AvatarEntityMap --- libraries/avatars/src/AvatarData.cpp | 19 +++++++++++++++++++ libraries/avatars/src/AvatarData.h | 6 ++++++ libraries/script-engine/src/ScriptEngine.cpp | 1 + 3 files changed, 26 insertions(+) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 2263964e8f..bbd3e05994 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -2309,3 +2309,22 @@ void RayToAvatarIntersectionResultFromScriptValue(const QScriptValue& object, Ra vec3FromScriptValue(intersection, value.intersection); } } + +QScriptValue AvatarEntityMapToScriptValue(QScriptEngine* engine, const AvatarEntityMap& value) { + QScriptValue obj = engine->newObject(); + for (auto entityID : value.keys()) { + QScriptValue EntityProperties = engine->newVariant(QVariant::fromValue(value.value(entityID))); + QString key = entityID.toString(); + obj.setProperty(key, EntityProperties); + } + return obj; +} + +void AvatarEntityMapFromScriptValue(const QScriptValue& object, AvatarEntityMap& value) { + QScriptValueIterator itr(object); + while (itr.hasNext()) { + itr.next(); + QUuid EntityID = QUuid(itr.name()); + value[EntityID] = qvariant_cast(itr.value().data().toVariant()); + } +} diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 264da75de2..502ae06425 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -44,6 +44,7 @@ typedef unsigned long long quint64; #include #include #include +#include #include #include @@ -816,6 +817,11 @@ Q_DECLARE_METATYPE(RayToAvatarIntersectionResult) QScriptValue RayToAvatarIntersectionResultToScriptValue(QScriptEngine* engine, const RayToAvatarIntersectionResult& results); void RayToAvatarIntersectionResultFromScriptValue(const QScriptValue& object, RayToAvatarIntersectionResult& results); +Q_DECLARE_METATYPE(AvatarEntityMap) + +QScriptValue AvatarEntityMapToScriptValue(QScriptEngine* engine, const AvatarEntityMap& value); +void AvatarEntityMapFromScriptValue(const QScriptValue& object, AvatarEntityMap& value); + // faux joint indexes (-1 means invalid) const int SENSOR_TO_WORLD_MATRIX_INDEX = 65534; // -2 const int CONTROLLER_RIGHTHAND_INDEX = 65533; // -3 diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index f1ff4c4686..83f2f5ccc0 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -576,6 +576,7 @@ void ScriptEngine::init() { qScriptRegisterMetaType(this, EntityItemIDtoScriptValue, EntityItemIDfromScriptValue); qScriptRegisterMetaType(this, RayToEntityIntersectionResultToScriptValue, RayToEntityIntersectionResultFromScriptValue); qScriptRegisterMetaType(this, RayToAvatarIntersectionResultToScriptValue, RayToAvatarIntersectionResultFromScriptValue); + qScriptRegisterMetaType(this, AvatarEntityMapToScriptValue, AvatarEntityMapFromScriptValue); qScriptRegisterSequenceMetaType>(this); qScriptRegisterSequenceMetaType>(this); From 3af05ece7de2b59e5f4540f33606957a1940cf80 Mon Sep 17 00:00:00 2001 From: kunalgosar Date: Tue, 7 Feb 2017 13:28:09 -0800 Subject: [PATCH 101/104] Made QScript Object more JS friendly --- libraries/avatars/src/AvatarData.cpp | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index bbd3e05994..4ea3d8bb3e 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include "AvatarLogging.h" @@ -2313,9 +2314,18 @@ void RayToAvatarIntersectionResultFromScriptValue(const QScriptValue& object, Ra QScriptValue AvatarEntityMapToScriptValue(QScriptEngine* engine, const AvatarEntityMap& value) { QScriptValue obj = engine->newObject(); for (auto entityID : value.keys()) { - QScriptValue EntityProperties = engine->newVariant(QVariant::fromValue(value.value(entityID))); + QByteArray entityProperties = value.value(entityID); + QJsonDocument jsonEntityProperties = QJsonDocument::fromBinaryData(entityProperties); + if (!jsonEntityProperties.isObject()) { + qCDebug(avatars) << "bad AvatarEntityData in AvatarEntityMap" << QString(entityProperties.toHex()); + } + + QVariant variantEntityProperties = jsonEntityProperties.toVariant(); + QVariantMap entityPropertiesMap = variantEntityProperties.toMap(); + QScriptValue scriptEntityProperties = variantMapToScriptValue(entityPropertiesMap, *engine); + QString key = entityID.toString(); - obj.setProperty(key, EntityProperties); + obj.setProperty(key, scriptEntityProperties); } return obj; } @@ -2325,6 +2335,12 @@ void AvatarEntityMapFromScriptValue(const QScriptValue& object, AvatarEntityMap& while (itr.hasNext()) { itr.next(); QUuid EntityID = QUuid(itr.name()); - value[EntityID] = qvariant_cast(itr.value().data().toVariant()); + + QScriptValue scriptEntityProperties = itr.value(); + QVariant variantEntityProperties = scriptEntityProperties.toVariant(); + QJsonDocument jsonEntityProperties = QJsonDocument::fromVariant(variantEntityProperties); + QByteArray binaryEntityProperties = jsonEntityProperties.toBinaryData(); + + value[EntityID] = binaryEntityProperties; } } From f61e16fccfe901116ffd75e56608c0be4ac7a6b5 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 23 Feb 2017 11:01:55 -0800 Subject: [PATCH 102/104] more aggressive out of view data culling --- assignment-client/src/avatars/AvatarMixerSlave.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 1f9fbde89a..dfc91b90c1 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -367,7 +367,7 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { _stats.overBudgetAvatars++; detail = AvatarData::NoData; } else if (!isInView && !getsOutOfView) { - detail = AvatarData::MinimumData; + detail = AvatarData::NoData; nodeData->incrementAvatarOutOfView(); } else { detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO From 8cb8d686ecf33798645243e11b7c7e1c9cc68993 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 23 Feb 2017 13:09:39 -0800 Subject: [PATCH 103/104] more cleanup and some CR feedback fixes --- .../src/avatars/AvatarMixerSlave.cpp | 395 ++++++++---------- 1 file changed, 182 insertions(+), 213 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index dfc91b90c1..a839784cd1 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -80,13 +80,6 @@ int AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, static const int AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND = 45; -// only send extra avatar data (avatars out of view, ignored) every Nth AvatarData frame -// Extra avatar data will be sent (AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND/EXTRA_AVATAR_DATA_FRAME_RATIO) times -// per second. -// This value should be a power of two for performance purposes, as the mixer performs a modulo operation every frame -// to determine whether the extra data should be sent. -static const int EXTRA_AVATAR_DATA_FRAME_RATIO = 16; - // FIXME - There is some old logic (unchanged as of 2/17/17) that randomly decides to send an identity // packet. That logic had the following comment about the constants it uses... // @@ -156,6 +149,20 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // setup a PacketList for the avatarPackets auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); + // Define the minimum bubble size + static const glm::vec3 minBubbleSize = glm::vec3(0.3f, 1.3f, 0.3f); + // Define the scale of the box for the current node + glm::vec3 nodeBoxScale = (nodeData->getPosition() - nodeData->getGlobalBoundingBoxCorner()) * 2.0f; + // Set up the bounding box for the current node + AABox nodeBox(nodeData->getGlobalBoundingBoxCorner(), nodeBoxScale); + // Clamp the size of the bounding box to a minimum scale + if (glm::any(glm::lessThan(nodeBoxScale, minBubbleSize))) { + nodeBox.setScaleStayCentered(minBubbleSize); + } + // Quadruple the scale of both bounding boxes + nodeBox.embiggen(4.0f); + + // setup list of AvatarData as well as maps to map betweeen the AvatarData and the original nodes // for calling the AvatarData::sortAvatars() function and getting our sorted list of client nodes QList avatarList; @@ -170,7 +177,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { avatarList << otherAvatar; avatarDataToNodes[otherAvatar] = otherNode; } - }); AvatarSharedPointer thisAvatar = nodeData->getAvatarSharedPointer(); @@ -191,8 +197,87 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { return glm::max(nodeBoxHalfScale.x, glm::max(nodeBoxHalfScale.y, nodeBoxHalfScale.z)); }, - [thisAvatar](AvatarSharedPointer avatar)->bool{ - return (avatar == thisAvatar); // ignore ourselves... + [&](AvatarSharedPointer avatar)->bool{ + if (avatar == thisAvatar) { + return true; // ignore ourselves... + } + + bool shouldIgnore = false; + + // We will also ignore other nodes for a couple of different reasons: + // 1) ignore bubbles and ignore specific node + // 2) the node hasn't really updated it's frame data recently, this can + // happen if for example the avatar is connected on a desktop and sending + // updates at ~30hz. So every 3 frames we skip a frame. + auto avatarNode = avatarDataToNodes[avatar]; + if (avatarNode) { + const AvatarMixerClientData* avatarNodeData = reinterpret_cast(avatarNode->getLinkedData()); + if (avatarNodeData) { + quint64 startIgnoreCalculation = usecTimestampNow(); + + // make sure we have data for this avatar, that it isn't the same node, + // and isn't an avatar that the viewing node has ignored + // or that has ignored the viewing node + if (!avatarNode->getLinkedData() + || avatarNode->getUUID() == node->getUUID() + || (node->isIgnoringNodeWithID(avatarNode->getUUID()) && !getsIgnoredByMe) + || (avatarNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { + shouldIgnore = true; + } else { + + // Check to see if the space bubble is enabled + if (node->isIgnoreRadiusEnabled() || avatarNode->isIgnoreRadiusEnabled()) { + + // Define the scale of the box for the current other node + glm::vec3 otherNodeBoxScale = (avatarNodeData->getPosition() - avatarNodeData->getGlobalBoundingBoxCorner()) * 2.0f; + // Set up the bounding box for the current other node + AABox otherNodeBox(avatarNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + // Clamp the size of the bounding box to a minimum scale + if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { + otherNodeBox.setScaleStayCentered(minBubbleSize); + } + // Quadruple the scale of both bounding boxes + otherNodeBox.embiggen(4.0f); + + // Perform the collision check between the two bounding boxes + if (nodeBox.touches(otherNodeBox)) { + nodeData->ignoreOther(node, avatarNode); + shouldIgnore = !getsAnyIgnored; + } + } + // Not close enough to ignore + if (!shouldIgnore) { + nodeData->removeFromRadiusIgnoringSet(node, avatarNode->getUUID()); + } + } + quint64 endIgnoreCalculation = usecTimestampNow(); + _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); + + if (!shouldIgnore) { + AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getUUID()); + AvatarDataSequenceNumber lastSeqFromSender = avatarNodeData->getLastReceivedSequenceNumber(); + + // FIXME - This code does appear to be working. But it seems brittle. + // It supports determining if the frame of data for this "other" + // avatar has already been sent to the reciever. This has been + // verified to work on a desktop display that renders at 60hz and + // therefore sends to mixer at 30hz. Each second you'd expect to + // have 15 (45hz-30hz) duplicate frames. In this case, the stat + // avg_other_av_skips_per_second does report 15. + // + // make sure we haven't already sent this data from this sender to this receiver + // or that somehow we haven't sent + if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { + ++numAvatarsHeldBack; + shouldIgnore = true; + } else if (lastSeqFromSender - lastSeqToReceiver > 1) { + // this is a skip - we still send the packet but capture the presence of the skip so we see it happening + ++numAvatarsWithSkippedFrames; + } + } + } + } + return shouldIgnore; }); // loop through our sorted avatars and allocate our bandwidth to them accordingly @@ -219,223 +304,107 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // NOTE: Here's where we determine if we are over budget and drop to bare minimum data int minimRemainingAvatarBytes = minimumBytesPerAvatar * remainingAvatars; bool overBudget = (identityBytesSent + numAvatarDataBytes + minimRemainingAvatarBytes) > maxAvatarBytesPerFrame; - /** - qDebug() << "Budget debugging:" - << "numAvatarDataBytes:" << numAvatarDataBytes << "\n" - << "identityBytesSent:" << identityBytesSent << "\n" - << "remainingAvatars:" << remainingAvatars << "\n" - << "minimumBytesPerAvatar:" << minimumBytesPerAvatar << "\n" - << "minimRemainingAvatarBytes:" << minimRemainingAvatarBytes << "\n" - << "overBudget:" << overBudget << "\n" - << "overBudgetAvatars:" << overBudgetAvatars; - **/ - bool shouldConsider = false; - quint64 startIgnoreCalculation = usecTimestampNow(); + quint64 startAvatarDataPacking = usecTimestampNow(); - // make sure we have data for this avatar, that it isn't the same node, - // and isn't an avatar that the viewing node has ignored - // or that has ignored the viewing node - if (!otherNode->getLinkedData() - || otherNode->getUUID() == node->getUUID() - || (node->isIgnoringNodeWithID(otherNode->getUUID()) && !getsIgnoredByMe) - || (otherNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { + ++numOtherAvatars; - shouldConsider = false; + const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); + + // make sure we send out identity packets to and from new arrivals. + bool forceSend = !nodeData->checkAndSetHasReceivedFirstPacketsFrom(otherNode->getUUID()); + + // FIXME - this clause seems suspicious "... || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp ..." + if (!overBudget + && otherNodeData->getIdentityChangeTimestamp().time_since_epoch().count() > 0 + && (forceSend + || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp + || distribution(generator) < IDENTITY_SEND_PROBABILITY)) { + + identityBytesSent += sendIdentityPacket(otherNodeData, node); + } + + const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); + glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition(); + + // determine if avatar is in view, to determine how much data to include... + glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; + AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + bool isInView = nodeData->otherAvatarInView(otherNodeBox); + + // start a new segment in the PacketList for this avatar + avatarPacketList->startSegment(); + + AvatarData::AvatarDataDetail detail; + + if (overBudget) { + overBudgetAvatars++; + _stats.overBudgetAvatars++; + detail = AvatarData::NoData; + } else if (!isInView && !getsOutOfView) { + detail = AvatarData::NoData; + nodeData->incrementAvatarOutOfView(); } else { - const AvatarMixerClientData* otherData = reinterpret_cast(otherNode->getLinkedData()); - - shouldConsider = true; // assume we will consider... - - // Check to see if the space bubble is enabled - if (node->isIgnoreRadiusEnabled() || otherNode->isIgnoreRadiusEnabled()) { - // Define the minimum bubble size - static const glm::vec3 minBubbleSize = glm::vec3(0.3f, 1.3f, 0.3f); - // Define the scale of the box for the current node - glm::vec3 nodeBoxScale = (nodeData->getPosition() - nodeData->getGlobalBoundingBoxCorner()) * 2.0f; - // Define the scale of the box for the current other node - glm::vec3 otherNodeBoxScale = (otherData->getPosition() - otherData->getGlobalBoundingBoxCorner()) * 2.0f; - - // Set up the bounding box for the current node - AABox nodeBox(nodeData->getGlobalBoundingBoxCorner(), nodeBoxScale); - // Clamp the size of the bounding box to a minimum scale - if (glm::any(glm::lessThan(nodeBoxScale, minBubbleSize))) { - nodeBox.setScaleStayCentered(minBubbleSize); - } - // Set up the bounding box for the current other node - AABox otherNodeBox(otherData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - // Clamp the size of the bounding box to a minimum scale - if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { - otherNodeBox.setScaleStayCentered(minBubbleSize); - } - // Quadruple the scale of both bounding boxes - nodeBox.embiggen(4.0f); - otherNodeBox.embiggen(4.0f); - - // Perform the collision check between the two bounding boxes - if (nodeBox.touches(otherNodeBox)) { - nodeData->ignoreOther(node, otherNode); - shouldConsider = getsAnyIgnored; - } - } - // Not close enough to ignore - if (shouldConsider) { - nodeData->removeFromRadiusIgnoringSet(node, otherNode->getUUID()); - } - - quint64 endIgnoreCalculation = usecTimestampNow(); - _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); + detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO + ? AvatarData::SendAllData : AvatarData::CullSmallData; + nodeData->incrementAvatarInView(); } - if (shouldConsider) { - quint64 startAvatarDataPacking = usecTimestampNow(); + bool includeThisAvatar = true; + auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); + QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); + bool distanceAdjust = true; + glm::vec3 viewerPosition = myPosition; + AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray + bool dropFaceTracking = false; - ++numOtherAvatars; - - const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); - - // make sure we send out identity packets to and from new arrivals. - bool forceSend = !nodeData->checkAndSetHasReceivedFirstPacketsFrom(otherNode->getUUID()); - - // FIXME - this clause seems suspicious "... || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp ..." - if (!overBudget - && otherNodeData->getIdentityChangeTimestamp().time_since_epoch().count() > 0 - && (forceSend - || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp - || distribution(generator) < IDENTITY_SEND_PROBABILITY)) { - - identityBytesSent += sendIdentityPacket(otherNodeData, node); - } - - const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); - // Decide whether to send this avatar's data based on it's distance from us - - glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition(); - float distanceToAvatar = glm::length(myPosition - otherPosition); - - if (shouldConsider) { - AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(otherNode->getUUID()); - AvatarDataSequenceNumber lastSeqFromSender = otherNodeData->getLastReceivedSequenceNumber(); - - // FIXME - This code does appear to be working. But it seems brittle. - // It supports determining if the frame of data for this "other" - // avatar has already been sent to the reciever. This has been - // verified to work on a desktop display that renders at 60hz and - // therefore sends to mixer at 30hz. Each second you'd expect to - // have 15 (45hz-30hz) duplicate frames. In this case, the stat - // avg_other_av_skips_per_second does report 15. - // - // make sure we haven't already sent this data from this sender to this receiver - // or that somehow we haven't sent - if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { - ++numAvatarsHeldBack; - - quint64 endAvatarDataPacking = usecTimestampNow(); - _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - shouldConsider = false; - } else if (lastSeqFromSender - lastSeqToReceiver > 1) { - // this is a skip - we still send the packet but capture the presence of the skip so we see it happening - ++numAvatarsWithSkippedFrames; - } - - // we're going to send this avatar - if (shouldConsider) { - - // determine if avatar is in view, to determine how much data to include... - glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; - AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - bool isInView = nodeData->otherAvatarInView(otherNodeBox); - - // this throttles the extra data to only be sent every Nth message - /** - if (!isInView && !getsOutOfView && (lastSeqToReceiver % EXTRA_AVATAR_DATA_FRAME_RATIO > 0)) { - quint64 endAvatarDataPacking = usecTimestampNow(); - - _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - shouldConsider = false; - } - ***/ - - if (shouldConsider) { - // start a new segment in the PacketList for this avatar - avatarPacketList->startSegment(); - - AvatarData::AvatarDataDetail detail; - - if (overBudget) { - overBudgetAvatars++; - _stats.overBudgetAvatars++; - detail = AvatarData::NoData; - } else if (!isInView && !getsOutOfView) { - detail = AvatarData::NoData; - nodeData->incrementAvatarOutOfView(); - } else { - detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO - ? AvatarData::SendAllData : AvatarData::CullSmallData; - nodeData->incrementAvatarInView(); - } - - { - bool includeThisAvatar = true; - auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); - QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); - bool distanceAdjust = true; - glm::vec3 viewerPosition = myPosition; - AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray - bool dropFaceTracking = false; - - quint64 start = usecTimestampNow(); - QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); - quint64 end = usecTimestampNow(); - _stats.toByteArrayElapsedTime += (end - start); - - static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID); - if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data"; - - dropFaceTracking = true; // first try dropping the facial data - bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); - - if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData"; - bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther, + quint64 start = usecTimestampNow(); + QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); - } + quint64 end = usecTimestampNow(); + _stats.toByteArrayElapsedTime += (end - start); - if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!"; - includeThisAvatar = false; - } - } + static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID); + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data"; - if (includeThisAvatar) { - numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); - numAvatarDataBytes += avatarPacketList->write(bytes); - _stats.numOthersIncluded++; + dropFaceTracking = true; // first try dropping the facial data + bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); - // increment the number of avatars sent to this reciever - nodeData->incrementNumAvatarsSentLastFrame(); + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData"; + bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther, + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + } - // set the last sent sequence number for this sender on the receiver - nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), - otherNodeData->getLastReceivedSequenceNumber()); - - // remember the last time we sent details about this other node to the receiver - nodeData->setLastBroadcastTime(otherNode->getUUID(), start); - - } - } - - avatarPacketList->endSegment(); - - quint64 endAvatarDataPacking = usecTimestampNow(); - _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - } - } + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!"; + includeThisAvatar = false; } } + + if (includeThisAvatar) { + numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); + numAvatarDataBytes += avatarPacketList->write(bytes); + _stats.numOthersIncluded++; + + // increment the number of avatars sent to this reciever + nodeData->incrementNumAvatarsSentLastFrame(); + + // set the last sent sequence number for this sender on the receiver + nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), + otherNodeData->getLastReceivedSequenceNumber()); + + // remember the last time we sent details about this other node to the receiver + nodeData->setLastBroadcastTime(otherNode->getUUID(), start); + + } + + avatarPacketList->endSegment(); + + quint64 endAvatarDataPacking = usecTimestampNow(); + _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); }; quint64 startPacketSending = usecTimestampNow(); From 1e91f74ce745929d6a502f7f7677640e64b617d6 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 23 Feb 2017 13:32:35 -0800 Subject: [PATCH 104/104] clean up some uneedded nullptr checks, make them asserts --- .../src/avatars/AvatarMixerSlave.cpp | 133 +++++++++--------- 1 file changed, 65 insertions(+), 68 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index a839784cd1..8d9a5e6951 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -171,6 +171,10 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { int listItem = 0; std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) { const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); + + // theoretically it's possible for a Node to be in the NodeList (and therefore end up here), + // but not have yet sent data that's linked to the node. Check for that case and don't + // consider those nodes. if (otherNodeData) { listItem++; AvatarSharedPointer otherAvatar = otherNodeData->getAvatarSharedPointer(); @@ -186,10 +190,8 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { [&](AvatarSharedPointer avatar)->uint64_t{ auto avatarNode = avatarDataToNodes[avatar]; - if (avatarNode) { - return nodeData->getLastBroadcastTime(avatarNode->getUUID()); - } - return 0; + assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map + return nodeData->getLastBroadcastTime(avatarNode->getUUID()); }, [&](AvatarSharedPointer avatar)->float{ @@ -210,72 +212,72 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // happen if for example the avatar is connected on a desktop and sending // updates at ~30hz. So every 3 frames we skip a frame. auto avatarNode = avatarDataToNodes[avatar]; - if (avatarNode) { - const AvatarMixerClientData* avatarNodeData = reinterpret_cast(avatarNode->getLinkedData()); - if (avatarNodeData) { - quint64 startIgnoreCalculation = usecTimestampNow(); - // make sure we have data for this avatar, that it isn't the same node, - // and isn't an avatar that the viewing node has ignored - // or that has ignored the viewing node - if (!avatarNode->getLinkedData() - || avatarNode->getUUID() == node->getUUID() - || (node->isIgnoringNodeWithID(avatarNode->getUUID()) && !getsIgnoredByMe) - || (avatarNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { - shouldIgnore = true; - } else { + assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map - // Check to see if the space bubble is enabled - if (node->isIgnoreRadiusEnabled() || avatarNode->isIgnoreRadiusEnabled()) { + const AvatarMixerClientData* avatarNodeData = reinterpret_cast(avatarNode->getLinkedData()); + assert(avatarNodeData); // we can't have gotten here without avatarNode having valid data + quint64 startIgnoreCalculation = usecTimestampNow(); - // Define the scale of the box for the current other node - glm::vec3 otherNodeBoxScale = (avatarNodeData->getPosition() - avatarNodeData->getGlobalBoundingBoxCorner()) * 2.0f; - // Set up the bounding box for the current other node - AABox otherNodeBox(avatarNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - // Clamp the size of the bounding box to a minimum scale - if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { - otherNodeBox.setScaleStayCentered(minBubbleSize); - } - // Quadruple the scale of both bounding boxes - otherNodeBox.embiggen(4.0f); + // make sure we have data for this avatar, that it isn't the same node, + // and isn't an avatar that the viewing node has ignored + // or that has ignored the viewing node + if (!avatarNode->getLinkedData() + || avatarNode->getUUID() == node->getUUID() + || (node->isIgnoringNodeWithID(avatarNode->getUUID()) && !getsIgnoredByMe) + || (avatarNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { + shouldIgnore = true; + } else { - // Perform the collision check between the two bounding boxes - if (nodeBox.touches(otherNodeBox)) { - nodeData->ignoreOther(node, avatarNode); - shouldIgnore = !getsAnyIgnored; - } - } - // Not close enough to ignore - if (!shouldIgnore) { - nodeData->removeFromRadiusIgnoringSet(node, avatarNode->getUUID()); - } + // Check to see if the space bubble is enabled + if (node->isIgnoreRadiusEnabled() || avatarNode->isIgnoreRadiusEnabled()) { + + // Define the scale of the box for the current other node + glm::vec3 otherNodeBoxScale = (avatarNodeData->getPosition() - avatarNodeData->getGlobalBoundingBoxCorner()) * 2.0f; + // Set up the bounding box for the current other node + AABox otherNodeBox(avatarNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + // Clamp the size of the bounding box to a minimum scale + if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { + otherNodeBox.setScaleStayCentered(minBubbleSize); } - quint64 endIgnoreCalculation = usecTimestampNow(); - _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); + // Quadruple the scale of both bounding boxes + otherNodeBox.embiggen(4.0f); - if (!shouldIgnore) { - AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getUUID()); - AvatarDataSequenceNumber lastSeqFromSender = avatarNodeData->getLastReceivedSequenceNumber(); - - // FIXME - This code does appear to be working. But it seems brittle. - // It supports determining if the frame of data for this "other" - // avatar has already been sent to the reciever. This has been - // verified to work on a desktop display that renders at 60hz and - // therefore sends to mixer at 30hz. Each second you'd expect to - // have 15 (45hz-30hz) duplicate frames. In this case, the stat - // avg_other_av_skips_per_second does report 15. - // - // make sure we haven't already sent this data from this sender to this receiver - // or that somehow we haven't sent - if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { - ++numAvatarsHeldBack; - shouldIgnore = true; - } else if (lastSeqFromSender - lastSeqToReceiver > 1) { - // this is a skip - we still send the packet but capture the presence of the skip so we see it happening - ++numAvatarsWithSkippedFrames; - } + // Perform the collision check between the two bounding boxes + if (nodeBox.touches(otherNodeBox)) { + nodeData->ignoreOther(node, avatarNode); + shouldIgnore = !getsAnyIgnored; } } + // Not close enough to ignore + if (!shouldIgnore) { + nodeData->removeFromRadiusIgnoringSet(node, avatarNode->getUUID()); + } + } + quint64 endIgnoreCalculation = usecTimestampNow(); + _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); + + if (!shouldIgnore) { + AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getUUID()); + AvatarDataSequenceNumber lastSeqFromSender = avatarNodeData->getLastReceivedSequenceNumber(); + + // FIXME - This code does appear to be working. But it seems brittle. + // It supports determining if the frame of data for this "other" + // avatar has already been sent to the reciever. This has been + // verified to work on a desktop display that renders at 60hz and + // therefore sends to mixer at 30hz. Each second you'd expect to + // have 15 (45hz-30hz) duplicate frames. In this case, the stat + // avg_other_av_skips_per_second does report 15. + // + // make sure we haven't already sent this data from this sender to this receiver + // or that somehow we haven't sent + if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { + ++numAvatarsHeldBack; + shouldIgnore = true; + } else if (lastSeqFromSender - lastSeqToReceiver > 1) { + // this is a skip - we still send the packet but capture the presence of the skip so we see it happening + ++numAvatarsWithSkippedFrames; + } } return shouldIgnore; }); @@ -284,7 +286,6 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { int avatarRank = 0; // this is overly conservative, because it includes some avatars we might not consider - // FIXME - move the ignore logic up into the sorting list so we get a better estimate int remainingAvatars = (int)sortedAvatars.size(); while (!sortedAvatars.empty()) { @@ -295,11 +296,7 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { remainingAvatars--; auto otherNode = avatarDataToNodes[avatarData]; - - // FIXME - should't this be an assert? - if (!otherNode) { - continue; - } + assert(otherNode); // we can't have gotten here without the avatarData being a valid key in the map // NOTE: Here's where we determine if we are over budget and drop to bare minimum data int minimRemainingAvatarBytes = minimumBytesPerAvatar * remainingAvatars;