From b6a626c92893753a628746aa3bce034f27e279cb Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 2 Jul 2014 17:18:12 -0700 Subject: [PATCH 01/70] Removed comment --- interface/src/ui/ApplicationOverlay.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 44a83e164a..c423a967f2 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -335,7 +335,6 @@ void ApplicationOverlay::displayOverlayTexture3DTV(Camera& whichCamera, float as glColor4f(1.0f, 1.0f, 1.0f, _alpha); //Render - // fov -= RADIANS_PER_DEGREE * 2.5f; //reduce by 5 degrees so it fits in the view const GLfloat distance = 1.0f; const GLfloat halfQuadHeight = distance * tan(fov); From bf2959675fd537ffcbfee9acfa19985baf9395a5 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 3 Jul 2014 16:09:54 -0700 Subject: [PATCH 02/70] First pass at laser oculus UI pointer --- examples/editModels.js | 3 +- examples/flockingBirds.js | 6 +- interface/src/Application.cpp | 5 + interface/src/devices/OculusManager.cpp | 28 +++++ interface/src/devices/OculusManager.h | 5 + interface/src/ui/ApplicationOverlay.cpp | 158 +++++++++++++++++------- 6 files changed, 153 insertions(+), 52 deletions(-) diff --git a/examples/editModels.js b/examples/editModels.js index 64c203534c..9e1581e205 100644 --- a/examples/editModels.js +++ b/examples/editModels.js @@ -31,8 +31,7 @@ var toolWidth = 50; var LASER_WIDTH = 4; var LASER_COLOR = { red: 255, green: 0, blue: 0 }; -var LASER_LENGTH_FACTOR = 500 -; +var LASER_LENGTH_FACTOR = 500; var MIN_ANGULAR_SIZE = 2; var MAX_ANGULAR_SIZE = 45; diff --git a/examples/flockingBirds.js b/examples/flockingBirds.js index 3fa8681abe..0eb348b7b9 100644 --- a/examples/flockingBirds.js +++ b/examples/flockingBirds.js @@ -31,7 +31,7 @@ var count=0; // iterations var enableFlyTowardPoints = true; // some birds have a point they want to fly to var enabledClustedFlyTowardPoints = true; // the flyToward points will be generally near each other -var flyToFrames = 10; // number of frames the bird would like to attempt to fly to it's flyTo point +var flyToFrames = 100; // number of frames the bird would like to attempt to fly to it's flyTo point var PROBABILITY_OF_FLY_TOWARD_CHANGE = 0.01; // chance the bird will decide to change its flyTo point var PROBABILITY_EACH_BIRD_WILL_FLY_TOWARD = 0.2; // chance the bird will decide to flyTo, otherwise it follows var flyingToCount = 0; // count of birds currently flying to someplace @@ -56,11 +56,11 @@ var PROBABILITY_TO_LEAD = 0.1; // probability a bird will choose to lead var birds = new Array(); // array of bird state data -var flockStartPosition = { x: 100, y: 10, z: 100}; +var flockStartPosition = MyAvatar.position; var flockStartVelocity = { x: 0, y: 0, z: 0}; var flockStartThrust = { x: 0, y: 0, z: 0}; // slightly upward against gravity var INITIAL_XY_VELOCITY_SCALE = 2; -var birdRadius = 0.0625; +var birdRadius = 0.0925; var baseBirdColor = { red: 0, green: 255, blue: 255 }; var glidingColor = { red: 255, green: 0, blue: 0 }; var thrustUpwardColor = { red: 0, green: 255, blue: 0 }; diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 7cfadc1ff2..c0f952d604 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2778,6 +2778,11 @@ void Application::displaySide(Camera& whichCamera, bool selfAvatarOnly) { bool mirrorMode = (whichCamera.getInterpolatedMode() == CAMERA_MODE_MIRROR); { PerformanceTimer perfTimer("paintGL/displaySide/renderAvatars"); + + if (1 || OculusManager::isConnected()) { + OculusManager::renderLaserPointer(); + } + _avatarManager.renderAvatars(mirrorMode ? Avatar::MIRROR_RENDER_MODE : Avatar::NORMAL_RENDER_MODE, selfAvatarOnly); } diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index e156e148ab..71073d2647 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -455,3 +455,31 @@ QSize OculusManager::getRenderTargetSize() { #endif } +void OculusManager::renderLaserPointer() { +#ifdef HAVE_LIBOVR + const float PALM_TIP_ROD_RADIUS = 0.009f; + + MyAvatar* myAvatar = Application::getInstance()->getAvatar(); + + //If the Oculus is enabled, we will draw a blue cursor ray + + // Draw the palm ball and disk + for (size_t i = 0; i < myAvatar->getHand()->getNumPalms(); ++i) { + PalmData& palm = myAvatar->getHand()->getPalms()[i]; + if (palm.isActive()) { + glColor4f(0, 1, 1, 1); + glm::vec3 tip = getLaserPointerTipPosition(&palm); + glm::vec3 root = palm.getPosition(); + Avatar::renderJointConnectingCone(root, tip, PALM_TIP_ROD_RADIUS, PALM_TIP_ROD_RADIUS); + } + } +#endif +} + +glm::vec3 OculusManager::getLaserPointerTipPosition(const PalmData* palm) { +#ifdef HAVE_LIBOVR + const float PALM_TIP_ROD_LENGTH_MULT = 2.0f; + return (palm->getTipPosition() - palm->getPosition()) * PALM_TIP_ROD_LENGTH_MULT; +#endif + return glm::vec3(0.0f); +} \ No newline at end of file diff --git a/interface/src/devices/OculusManager.h b/interface/src/devices/OculusManager.h index 7798875c2c..73ce6ef2d3 100644 --- a/interface/src/devices/OculusManager.h +++ b/interface/src/devices/OculusManager.h @@ -23,6 +23,7 @@ const float DEFAULT_OCULUS_UI_ANGULAR_SIZE = 72.0f; class Camera; +class PalmData; /// Handles interaction with the Oculus Rift. class OculusManager { @@ -41,6 +42,10 @@ public: /// param \roll[out] roll in radians static void getEulerAngles(float& yaw, float& pitch, float& roll); static QSize getRenderTargetSize(); + + /// Renders a laser pointer for UI picking + static void renderLaserPointer(); + static glm::vec3 getLaserPointerTipPosition(const PalmData* palm); private: #ifdef HAVE_LIBOVR diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index c423a967f2..f63a302cde 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -277,7 +277,7 @@ void ApplicationOverlay::displayOverlayTextureOculus(Camera& whichCamera) { glColor4f(1.0f, 1.0f, 1.0f, _alpha); renderTexturedHemisphere(); - + renderControllerPointersOculus(); glPopMatrix(); @@ -560,7 +560,26 @@ void ApplicationOverlay::renderControllerPointers() { } } +bool findSphereIntersection(const glm::vec3 &start, const glm::vec3 &end, const glm::vec3 &spos, const float r, glm::vec3 &result) { + double a = pow(end.x - start.x, 2) + pow(end.y - start.y, 2) + pow(end.z - start.z, 2); + double b = 2.0f * ((end.x - start.x) * (start.x - spos.x) + (end.y - start.x) * (start.y - spos.y) + (end.z - start.z) * (start.z - spos.z)); + double c = pow(start.x - spos.x, 2) + pow(start.y - spos.y, 2) + pow(start.z - spos.z, 2) - r*r; + + double delta = b * b - 4.0 * a * c; + printf("Intersection Delta %lf\n", delta); + + if (delta == 0) { + float d = -b / (2.0 * a); + result = start + d * (end - start); + } else { + return false; + } +} + void ApplicationOverlay::renderControllerPointersOculus() { + + const bool useLaser = true; + Application* application = Application::getInstance(); QGLWidget* glWidget = application->getGLWidget(); @@ -573,54 +592,99 @@ void ApplicationOverlay::renderControllerPointersOculus() { glDisable(GL_DEPTH_TEST); for (int i = 0; i < NUMBER_OF_MAGNIFIERS; i++) { + if (i > 0 && useLaser) { + MyAvatar* myAvatar = application->getAvatar(); + PalmData& palm = myAvatar->getHand()->getPalms()[i]; + if (palm.isActive()) { + + glm::vec3 tip = OculusManager::getLaserPointerTipPosition(&palm); + glm::vec3 result; - //Dont render the reticle if its inactive - if (!_reticleActive[i]) { - continue; + if (findSphereIntersection(myAvatar->getHead()->calculateAverageEyePosition(), + tip, glm::vec3(0, 0, 0), 1, result)){ + printf("Intersection Found: "); + printVector(result); + } + + + //float lX = sin((newULeft - 0.5f) * _textureFov); + //float rX = sin((newURight - 0.5f) * _textureFov); + //float bY = sin((newVBottom - 0.5f) * _textureFov); + //float tY = sin((newVTop - 0.5f) * _textureFov); + + //float dist; + ////Bottom Left + //dist = sqrt(lX * lX + bY * bY); + //float blZ = sqrt(1.0f - dist * dist); + ////Top Left + //dist = sqrt(lX * lX + tY * tY); + //float tlZ = sqrt(1.0f - dist * dist); + ////Bottom Right + //dist = sqrt(rX * rX + bY * bY); + //float brZ = sqrt(1.0f - dist * dist); + ////Top Right + //dist = sqrt(rX * rX + tY * tY); + //float trZ = sqrt(1.0f - dist * dist); + + //glBegin(GL_QUADS); + + //glColor4f(RETICLE_COLOR[0], RETICLE_COLOR[1], RETICLE_COLOR[2], _alpha); + + //glTexCoord2f(0.0f, 0.0f); glVertex3f(lX, tY, -tlZ); + //glTexCoord2f(1.0f, 0.0f); glVertex3f(rX, tY, -trZ); + //glTexCoord2f(1.0f, 1.0f); glVertex3f(rX, bY, -brZ); + //glTexCoord2f(0.0f, 1.0f); glVertex3f(lX, bY, -blZ); + + //glEnd(); + } + } else { + //Dont render the reticle if its inactive + if (!_reticleActive[i]) { + continue; + } + + float mouseX = (float)_mouseX[i]; + float mouseY = (float)_mouseY[i]; + mouseX -= reticleSize / 2; + mouseY += reticleSize / 2; + + //Get new UV coordinates from our magnification window + float newULeft = mouseX / widgetWidth; + float newURight = (mouseX + reticleSize) / widgetWidth; + float newVBottom = 1.0 - mouseY / widgetHeight; + float newVTop = 1.0 - (mouseY - reticleSize) / widgetHeight; + + // Project our position onto the hemisphere using the UV coordinates + float lX = sin((newULeft - 0.5f) * _textureFov); + float rX = sin((newURight - 0.5f) * _textureFov); + float bY = sin((newVBottom - 0.5f) * _textureFov); + float tY = sin((newVTop - 0.5f) * _textureFov); + + float dist; + //Bottom Left + dist = sqrt(lX * lX + bY * bY); + float blZ = sqrt(1.0f - dist * dist); + //Top Left + dist = sqrt(lX * lX + tY * tY); + float tlZ = sqrt(1.0f - dist * dist); + //Bottom Right + dist = sqrt(rX * rX + bY * bY); + float brZ = sqrt(1.0f - dist * dist); + //Top Right + dist = sqrt(rX * rX + tY * tY); + float trZ = sqrt(1.0f - dist * dist); + + glBegin(GL_QUADS); + + glColor4f(RETICLE_COLOR[0], RETICLE_COLOR[1], RETICLE_COLOR[2], _alpha); + + glTexCoord2f(0.0f, 0.0f); glVertex3f(lX, tY, -tlZ); + glTexCoord2f(1.0f, 0.0f); glVertex3f(rX, tY, -trZ); + glTexCoord2f(1.0f, 1.0f); glVertex3f(rX, bY, -brZ); + glTexCoord2f(0.0f, 1.0f); glVertex3f(lX, bY, -blZ); + + glEnd(); } - - float mouseX = (float)_mouseX[i]; - float mouseY = (float)_mouseY[i]; - mouseX -= reticleSize / 2; - mouseY += reticleSize / 2; - - //Get new UV coordinates from our magnification window - float newULeft = mouseX / widgetWidth; - float newURight = (mouseX + reticleSize) / widgetWidth; - float newVBottom = 1.0 - mouseY / widgetHeight; - float newVTop = 1.0 - (mouseY - reticleSize) / widgetHeight; - - // Project our position onto the hemisphere using the UV coordinates - float lX = sin((newULeft - 0.5f) * _textureFov); - float rX = sin((newURight - 0.5f) * _textureFov); - float bY = sin((newVBottom - 0.5f) * _textureFov); - float tY = sin((newVTop - 0.5f) * _textureFov); - - float dist; - //Bottom Left - dist = sqrt(lX * lX + bY * bY); - float blZ = sqrt(1.0f - dist * dist); - //Top Left - dist = sqrt(lX * lX + tY * tY); - float tlZ = sqrt(1.0f - dist * dist); - //Bottom Right - dist = sqrt(rX * rX + bY * bY); - float brZ = sqrt(1.0f - dist * dist); - //Top Right - dist = sqrt(rX * rX + tY * tY); - float trZ = sqrt(1.0f - dist * dist); - - glBegin(GL_QUADS); - - glColor4f(RETICLE_COLOR[0], RETICLE_COLOR[1], RETICLE_COLOR[2], _alpha); - - glTexCoord2f(0.0f, 0.0f); glVertex3f(lX, tY, -tlZ); - glTexCoord2f(1.0f, 0.0f); glVertex3f(rX, tY, -trZ); - glTexCoord2f(1.0f, 1.0f); glVertex3f(rX, bY, -brZ); - glTexCoord2f(0.0f, 1.0f); glVertex3f(lX, bY, -blZ); - - glEnd(); - } glEnable(GL_DEPTH_TEST); } From cc63f706a745dbc1c2c57aa9d405752526ec6b6f Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Tue, 8 Jul 2014 11:43:24 -0700 Subject: [PATCH 03/70] Functional sixense "laser" oculus selection mode. --- interface/src/devices/OculusManager.cpp | 14 +- interface/src/devices/SixenseManager.cpp | 28 ++- interface/src/ui/ApplicationOverlay.cpp | 238 ++++++++++++++++++----- interface/src/ui/ApplicationOverlay.h | 1 + 4 files changed, 215 insertions(+), 66 deletions(-) diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index befe48e42d..976bd5ba8e 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -443,7 +443,15 @@ void OculusManager::getEulerAngles(float& yaw, float& pitch, float& roll) { ovrPosef pose = ss.Predicted.Pose; Quatf orientation = Quatf(pose.Orientation); orientation.GetEulerAngles(&yaw, &pitch, &roll); + } else { + yaw = 0.0f; + pitch = 0.0f; + roll = 0.0f; } +#else + yaw = 0.0f; + pitch = 0.0f; + roll = 0.0f; #endif } @@ -461,7 +469,7 @@ QSize OculusManager::getRenderTargetSize() { void OculusManager::renderLaserPointer() { #ifdef HAVE_LIBOVR - const float PALM_TIP_ROD_RADIUS = 0.009f; + const float PALM_TIP_ROD_RADIUS = 0.002f; MyAvatar* myAvatar = Application::getInstance()->getAvatar(); @@ -482,8 +490,8 @@ void OculusManager::renderLaserPointer() { glm::vec3 OculusManager::getLaserPointerTipPosition(const PalmData* palm) { #ifdef HAVE_LIBOVR - const float PALM_TIP_ROD_LENGTH_MULT = 2.0f; - return (palm->getTipPosition() - palm->getPosition()) * PALM_TIP_ROD_LENGTH_MULT; + const float PALM_TIP_ROD_LENGTH_MULT = 11.0f; + return palm->getPosition() + (palm->getTipPosition() - palm->getPosition()) * PALM_TIP_ROD_LENGTH_MULT; #endif return glm::vec3(0.0f); } diff --git a/interface/src/devices/SixenseManager.cpp b/interface/src/devices/SixenseManager.cpp index c50fc887d6..ecf7fea5cb 100644 --- a/interface/src/devices/SixenseManager.cpp +++ b/interface/src/devices/SixenseManager.cpp @@ -13,6 +13,7 @@ #include "Application.h" #include "SixenseManager.h" +#include "devices/OculusManager.h" #include "UserActivityLogger.h" #ifdef HAVE_SIXENSE @@ -358,9 +359,7 @@ void SixenseManager::emulateMouse(PalmData* palm, int index) { MyAvatar* avatar = application->getAvatar(); QGLWidget* widget = application->getGLWidget(); QPoint pos; - // Get directon relative to avatar orientation - glm::vec3 direction = glm::inverse(avatar->getOrientation()) * palm->getFingerDirection(); - + Qt::MouseButton bumperButton; Qt::MouseButton triggerButton; @@ -372,15 +371,24 @@ void SixenseManager::emulateMouse(PalmData* palm, int index) { triggerButton = Qt::LeftButton; } - // Get the angles, scaled between (-0.5,0.5) - float xAngle = (atan2(direction.z, direction.x) + M_PI_2); - float yAngle = 0.5f - ((atan2(direction.z, direction.y) + M_PI_2)); + if (OculusManager::isConnected()) { + pos = application->getApplicationOverlay().getOculusPalmClickLocation(palm); + printf("CLICK: %d %d\n", pos.x(), pos.y()); + } else { + // Get directon relative to avatar orientation + glm::vec3 direction = glm::inverse(avatar->getOrientation()) * palm->getFingerDirection(); - // Get the pixel range over which the xAngle and yAngle are scaled - float cursorRange = widget->width() * getCursorPixelRangeMult(); + // Get the angles, scaled between (-0.5,0.5) + float xAngle = (atan2(direction.z, direction.x) + M_PI_2); + float yAngle = 0.5f - ((atan2(direction.z, direction.y) + M_PI_2)); - pos.setX(widget->width() / 2.0f + cursorRange * xAngle); - pos.setY(widget->height() / 2.0f + cursorRange * yAngle); + // Get the pixel range over which the xAngle and yAngle are scaled + float cursorRange = widget->width() * getCursorPixelRangeMult(); + + pos.setX(widget->width() / 2.0f + cursorRange * xAngle); + pos.setY(widget->height() / 2.0f + cursorRange * yAngle); + + } //If we are off screen then we should stop processing, and if a trigger or bumper is pressed, //we should unpress them. diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 42c4f36470..64aafbb49c 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -205,6 +205,103 @@ void ApplicationOverlay::getClickLocation(int &x, int &y) const { } } + +//Checks if the given ray intersects the sphere at the origin. result will store a multiplier that should +//be multiplied by dir and added to origin to get the location of the collision +bool raySphereIntersect(const glm::vec3 &dir, const glm::vec3 &origin, float r, float* result) +{ + //Source: http://wiki.cgsociety.org/index.php/Ray_Sphere_Intersection + + //Compute A, B and C coefficients + float a = glm::dot(dir, dir); + float b = 2 * glm::dot(dir, origin); + float c = glm::dot(origin, origin) - (r * r); + + //Find discriminant + float disc = b * b - 4 * a * c; + + // if discriminant is negative there are no real roots, so return + // false as ray misses sphere + if (disc < 0) { + return false; + } + + // compute q as described above + float distSqrt = sqrtf(disc); + float q; + if (b < 0) { + q = (-b - distSqrt) / 2.0; + } else { + q = (-b + distSqrt) / 2.0; + } + + // compute t0 and t1 + float t0 = q / a; + float t1 = c / q; + + // make sure t0 is smaller than t1 + if (t0 > t1) { + // if t0 is bigger than t1 swap them around + float temp = t0; + t0 = t1; + t1 = temp; + } + + // if t1 is less than zero, the object is in the ray's negative direction + // and consequently the ray misses the sphere + if (t1 < 0) { + return false; + } + + // if t0 is less than zero, the intersection point is at t1 + if (t0 < 0) { + *result = t1; + return true; + } else { // else the intersection point is at t0 + *result = t0; + return true; + } +} + + +QPoint ApplicationOverlay::getOculusPalmClickLocation(PalmData *palm) const { + + Application* application = Application::getInstance(); + QGLWidget* glWidget = application->getGLWidget(); + MyAvatar* myAvatar = application->getAvatar(); + + const int widgetWidth = glWidget->width(); + const int widgetHeight = glWidget->height(); + + + glm::vec3 tip = OculusManager::getLaserPointerTipPosition(palm); + glm::vec3 eyePos = myAvatar->getHead()->calculateAverageEyePosition(); + glm::quat orientation = glm::inverse(myAvatar->getOrientation()); + glm::vec3 dir = orientation * glm::normalize(eyePos - tip); //direction of ray goes towards camera + glm::vec3 tipPos = orientation * (tip - eyePos); + + QPoint rv; + + float t; + + //Find intersection of crosshair ray + if (raySphereIntersect(dir, tipPos, 1, &t)){ + glm::vec3 collisionPos = tipPos + dir * t; + + float u = asin(collisionPos.x) / (_textureFov)+0.5f; + float v = 1.0 - (asin(collisionPos.y) / (_textureFov)+0.5f); + + rv.setX(u * glWidget->width()); + rv.setY(v * glWidget->height()); + + } else { + //if they did not click on the overlay, just set the coords to INT_MAX + rv.setX(INT_MAX); + rv.setY(INT_MAX); + } + return rv; +} + // Draws the FBO texture for Oculus rift. void ApplicationOverlay::displayOverlayTextureOculus(Camera& whichCamera) { @@ -221,11 +318,14 @@ void ApplicationOverlay::displayOverlayTextureOculus(Camera& whichCamera) { glEnable(GL_BLEND); glBlendFuncSeparate(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA, GL_CONSTANT_ALPHA, GL_ONE); - glBindTexture(GL_TEXTURE_2D, getFramebufferObject()->texture()); glEnable(GL_DEPTH_TEST); glDisable(GL_LIGHTING); glEnable(GL_TEXTURE_2D); + renderControllerPointersOculus(); + + glBindTexture(GL_TEXTURE_2D, getFramebufferObject()->texture()); + glMatrixMode(GL_MODELVIEW); glPushMatrix(); @@ -277,8 +377,6 @@ void ApplicationOverlay::displayOverlayTextureOculus(Camera& whichCamera) { renderTexturedHemisphere(); - renderControllerPointersOculus(); - glPopMatrix(); glDepthMask(GL_TRUE); @@ -559,22 +657,6 @@ void ApplicationOverlay::renderControllerPointers() { } } -bool findSphereIntersection(const glm::vec3 &start, const glm::vec3 &end, const glm::vec3 &spos, const float r, glm::vec3 &result) { - double a = pow(end.x - start.x, 2) + pow(end.y - start.y, 2) + pow(end.z - start.z, 2); - double b = 2.0f * ((end.x - start.x) * (start.x - spos.x) + (end.y - start.x) * (start.y - spos.y) + (end.z - start.z) * (start.z - spos.z)); - double c = pow(start.x - spos.x, 2) + pow(start.y - spos.y, 2) + pow(start.z - spos.z, 2) - r*r; - - double delta = b * b - 4.0 * a * c; - printf("Intersection Delta %lf\n", delta); - - if (delta == 0) { - float d = -b / (2.0 * a); - result = start + d * (end - start); - } else { - return false; - } -} - void ApplicationOverlay::renderControllerPointersOculus() { const bool useLaser = true; @@ -589,54 +671,104 @@ void ApplicationOverlay::renderControllerPointersOculus() { glBindTexture(GL_TEXTURE_2D, _crosshairTexture); glDisable(GL_DEPTH_TEST); - - for (int i = 0; i < NUMBER_OF_MAGNIFIERS; i++) { - if (i > 0 && useLaser) { - MyAvatar* myAvatar = application->getAvatar(); + glMatrixMode(GL_MODELVIEW); + MyAvatar* myAvatar = application->getAvatar(); + + //Determine how much we need to iterate + const int ITERATIONS = max(myAvatar->getHand()->getNumPalms(), 3); + + for (int i = 0; i < ITERATIONS; i++) { + if (useLaser && i < myAvatar->getHand()->getNumPalms()) { + PalmData& palm = myAvatar->getHand()->getPalms()[i]; if (palm.isActive()) { - glm::vec3 tip = OculusManager::getLaserPointerTipPosition(&palm); glm::vec3 result; + glm::vec3 eyePos = myAvatar->getHead()->calculateAverageEyePosition(); + glm::quat orientation = glm::inverse(myAvatar->getOrientation()); + glm::vec3 dir = orientation * glm::normalize(eyePos - tip); //direction of ray goes towards camera + glm::vec3 tipPos = (tip - eyePos); + + float t; + float length = glm::length(eyePos - tip); + float size = 0.045f * length; - if (findSphereIntersection(myAvatar->getHead()->calculateAverageEyePosition(), - tip, glm::vec3(0, 0, 0), 1, result)){ - printf("Intersection Found: "); - printVector(result); + glm::vec3 up = glm::vec3(0.0, 1.0, 0.0) * size; + glm::vec3 right = glm::vec3(1.0, 0.0, 0.0) * size; + + glm::vec3 cursorVerts[4]; + cursorVerts[0] = -right + up; + cursorVerts[1] = right + up; + cursorVerts[2] = right - up; + cursorVerts[3] = -right - up; + + glPushMatrix(); + // glLoadIdentity(); + + // objToCamProj is the vector in world coordinates from the + // local origin to the camera projected in the XZ plane + glm::vec3 cursorToCameraXZ(-tipPos.x, 0, -tipPos.z); + cursorToCameraXZ = glm::normalize(cursorToCameraXZ); + + // This is the original lookAt vector for the object + // in world coordinates + glm::vec3 direction(0, 0, 1); + glTranslatef(tip.x, tip.y, tip.z); + + // easy fix to determine wether the angle is negative or positive + // for positive angles upAux will be a vector pointing in the + // positive y direction, otherwise upAux will point downwards + // effectively reversing the rotation. + glm::vec3 upAux = glm::cross(direction, cursorToCameraXZ); + + // compute the angle + float angleCosine = glm::dot(direction, cursorToCameraXZ); + + // perform the rotation. The if statement is used for stability reasons + // if the lookAt and objToCamProj vectors are too close together then + // |angleCosine| could be bigger than 1 due to lack of precision + if ((angleCosine < 0.999999) && (angleCosine > -0.999999)) { + glRotatef(acos(angleCosine) * DEGREES_PER_RADIAN, upAux[0], upAux[1], upAux[2]); } + // so far it is just like the cylindrical billboard. The code for the + // second rotation comes now + // The second part tilts the object so that it faces the camera - //float lX = sin((newULeft - 0.5f) * _textureFov); - //float rX = sin((newURight - 0.5f) * _textureFov); - //float bY = sin((newVBottom - 0.5f) * _textureFov); - //float tY = sin((newVTop - 0.5f) * _textureFov); + // objToCam is the vector in world coordinates from + // the local origin to the camera + glm::vec3 cursorToCamera = glm::normalize(-tipPos); - //float dist; - ////Bottom Left - //dist = sqrt(lX * lX + bY * bY); - //float blZ = sqrt(1.0f - dist * dist); - ////Top Left - //dist = sqrt(lX * lX + tY * tY); - //float tlZ = sqrt(1.0f - dist * dist); - ////Bottom Right - //dist = sqrt(rX * rX + bY * bY); - //float brZ = sqrt(1.0f - dist * dist); - ////Top Right - //dist = sqrt(rX * rX + tY * tY); - //float trZ = sqrt(1.0f - dist * dist); + // Compute the angle between objToCamProj and objToCam, + //i.e. compute the required angle for the lookup vector - //glBegin(GL_QUADS); + angleCosine = glm::dot(cursorToCameraXZ, cursorToCamera); - //glColor4f(RETICLE_COLOR[0], RETICLE_COLOR[1], RETICLE_COLOR[2], _alpha); + // Tilt the object. The test is done to prevent instability + // when objToCam and objToCamProj have a very small + // angle between them - //glTexCoord2f(0.0f, 0.0f); glVertex3f(lX, tY, -tlZ); - //glTexCoord2f(1.0f, 0.0f); glVertex3f(rX, tY, -trZ); - //glTexCoord2f(1.0f, 1.0f); glVertex3f(rX, bY, -brZ); - //glTexCoord2f(0.0f, 1.0f); glVertex3f(lX, bY, -blZ); + if ((angleCosine < 0.99990) && (angleCosine > -0.9999)) + if (cursorToCamera.y < 0) { + glRotatef(acos(angleCosine) * DEGREES_PER_RADIAN, 1, 0, 0); + } else { + glRotatef(acos(angleCosine) * DEGREES_PER_RADIAN, -1, 0, 0); + } - //glEnd(); + glBegin(GL_QUADS); + + glColor4f(RETICLE_COLOR[0], RETICLE_COLOR[1], RETICLE_COLOR[2], _alpha); + + glTexCoord2f(0.0f, 0.0f); glVertex3f(cursorVerts[0].x, cursorVerts[0].y, cursorVerts[0].z); + glTexCoord2f(1.0f, 0.0f); glVertex3f(cursorVerts[1].x, cursorVerts[1].y, cursorVerts[1].z); + glTexCoord2f(1.0f, 1.0f); glVertex3f(cursorVerts[2].x, cursorVerts[2].y, cursorVerts[2].z); + glTexCoord2f(0.0f, 1.0f); glVertex3f(cursorVerts[3].x, cursorVerts[3].y, cursorVerts[3].z); + + glEnd(); + + glPopMatrix(); } - } else { + } else if (i < NUMBER_OF_MAGNIFIERS) { //Dont render the reticle if its inactive if (!_reticleActive[i]) { continue; diff --git a/interface/src/ui/ApplicationOverlay.h b/interface/src/ui/ApplicationOverlay.h index 7c1f87d575..ebac5f3cfc 100644 --- a/interface/src/ui/ApplicationOverlay.h +++ b/interface/src/ui/ApplicationOverlay.h @@ -32,6 +32,7 @@ public: void displayOverlayTexture3DTV(Camera& whichCamera, float aspectRatio, float fov); void computeOculusPickRay(float x, float y, glm::vec3& direction) const; void getClickLocation(int &x, int &y) const; + QPoint getOculusPalmClickLocation(PalmData *palm) const; // Getters QOpenGLFramebufferObject* getFramebufferObject(); From 0ce7b178b081d76816623b901cb3fa0d00e1efdc Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Tue, 8 Jul 2014 12:47:41 -0700 Subject: [PATCH 04/70] Made Oculus UI framebuffer resize when window resizes --- interface/src/ui/ApplicationOverlay.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 64aafbb49c..dfa6b63fa5 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -95,6 +95,8 @@ void ApplicationOverlay::renderOverlay(bool renderToTexture) { glMatrixMode(GL_PROJECTION); glPushMatrix(); + printf("%d %d\n", glWidget->width(), glWidget->height()); + glLoadIdentity(); gluOrtho2D(0, glWidget->width(), glWidget->height(), 0); glDisable(GL_DEPTH_TEST); @@ -205,7 +207,6 @@ void ApplicationOverlay::getClickLocation(int &x, int &y) const { } } - //Checks if the given ray intersects the sphere at the origin. result will store a multiplier that should //be multiplied by dir and added to origin to get the location of the collision bool raySphereIntersect(const glm::vec3 &dir, const glm::vec3 &origin, float r, float* result) @@ -1194,8 +1195,12 @@ void ApplicationOverlay::renderTexturedHemisphere() { } QOpenGLFramebufferObject* ApplicationOverlay::getFramebufferObject() { - if (!_framebufferObject) { - _framebufferObject = new QOpenGLFramebufferObject(Application::getInstance()->getGLWidget()->size()); + QSize size = Application::getInstance()->getGLWidget()->size(); + if (!_framebufferObject || _framebufferObject->size() != size) { + if (_framebufferObject){ + delete _framebufferObject; + } + _framebufferObject = new QOpenGLFramebufferObject(size); glBindTexture(GL_TEXTURE_2D, _framebufferObject->texture()); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); From e732436783bac75e951270ff2bf20f5800f28db9 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 8 Jul 2014 14:13:42 -0700 Subject: [PATCH 05/70] added _starveCount, _silentFramesDropped --- libraries/audio/src/PositionalAudioRingBuffer.cpp | 8 +++++++- libraries/audio/src/PositionalAudioRingBuffer.h | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 6b3a1eb94f..8a03561140 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -100,7 +100,9 @@ PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer:: _desiredJitterBufferFrames(1), _currentJitterBufferFrames(-1), _dynamicJitterBuffers(dynamicJitterBuffers), - _consecutiveNotMixedCount(0) + _consecutiveNotMixedCount(0), + _starveCount(0), + _silentFramesDropped(0) { } @@ -143,9 +145,12 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { addSilentFrame(numSilentFramesToAdd * samplesPerFrame); _currentJitterBufferFrames = _desiredJitterBufferFrames; + _silentFramesDropped += numFramesToDropDesired; } else { // we need to drop all frames to get the jitter buffer close as possible to its desired length _currentJitterBufferFrames -= numSilentFrames; + + _silentFramesDropped += numSilentFrames; } } else { addSilentFrame(numSilentSamples); @@ -217,6 +222,7 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { } else if (samplesAvailable() < samplesPerFrame) { // if the buffer doesn't have a full frame of samples to take for mixing, it is starved _isStarved = true; + _starveCount++; // set to -1 to indicate the jitter buffer is starved _currentJitterBufferFrames = -1; diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 31b0524b3b..5922b27002 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -110,6 +110,8 @@ protected: // extra stats int _consecutiveNotMixedCount; + int _starveCount; + int _silentFramesDropped; }; #endif // hifi_PositionalAudioRingBuffer_h From 87ad87e10fb1666bca84d4b1e6ddb1dbef8c0819 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Tue, 8 Jul 2014 16:25:21 -0700 Subject: [PATCH 06/70] Actual stereoscopic 3D UI. Looks better now. --- interface/src/ui/ApplicationOverlay.cpp | 85 ++++++++++++++----------- 1 file changed, 49 insertions(+), 36 deletions(-) diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index dfa6b63fa5..b768c522ad 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -313,8 +313,7 @@ void ApplicationOverlay::displayOverlayTextureOculus(Camera& whichCamera) { Application* application = Application::getInstance(); MyAvatar* myAvatar = application->getAvatar(); - const glm::vec3& viewMatrixTranslation = application->getViewMatrixTranslation(); - + glActiveTexture(GL_TEXTURE0); glEnable(GL_BLEND); @@ -323,34 +322,25 @@ void ApplicationOverlay::displayOverlayTextureOculus(Camera& whichCamera) { glDisable(GL_LIGHTING); glEnable(GL_TEXTURE_2D); - renderControllerPointersOculus(); - glBindTexture(GL_TEXTURE_2D, getFramebufferObject()->texture()); glMatrixMode(GL_MODELVIEW); - glPushMatrix(); - glLoadIdentity(); - // Transform to world space - glm::quat rotation = whichCamera.getRotation(); - glm::vec3 axis2 = glm::axis(rotation); - glRotatef(-glm::degrees(glm::angle(rotation)), axis2.x, axis2.y, axis2.z); - glTranslatef(viewMatrixTranslation.x, viewMatrixTranslation.y, viewMatrixTranslation.z); - - // Translate to the front of the camera - glm::vec3 pos = whichCamera.getPosition(); - glm::quat rot = myAvatar->getOrientation(); - glm::vec3 axis = glm::axis(rot); - - glTranslatef(pos.x, pos.y, pos.z); - glRotatef(glm::degrees(glm::angle(rot)), axis.x, axis.y, axis.z); - glDepthMask(GL_TRUE); glEnable(GL_ALPHA_TEST); glAlphaFunc(GL_GREATER, 0.01f); //Update and draw the magnifiers + + glPushMatrix(); + const glm::quat& orientation = myAvatar->getOrientation(); + const glm::vec3& position = myAvatar->getHead()->calculateAverageEyePosition(); + + glm::mat4 rotation = glm::toMat4(orientation); + + glTranslatef(position.x, position.y, position.z); + glMultMatrixf(&rotation[0][0]); for (int i = 0; i < NUMBER_OF_MAGNIFIERS; i++) { if (_magActive[i]) { @@ -370,6 +360,7 @@ void ApplicationOverlay::displayOverlayTextureOculus(Camera& whichCamera) { renderMagnifier(_magX[i], _magY[i], _magSizeMult[i], i != MOUSE); } } + glPopMatrix(); glDepthMask(GL_FALSE); glDisable(GL_ALPHA_TEST); @@ -378,7 +369,7 @@ void ApplicationOverlay::displayOverlayTextureOculus(Camera& whichCamera) { renderTexturedHemisphere(); - glPopMatrix(); + renderControllerPointersOculus(); glDepthMask(GL_TRUE); glBindTexture(GL_TEXTURE_2D, 0); @@ -504,7 +495,6 @@ void ApplicationOverlay::renderPointers() { glActiveTexture(GL_TEXTURE0); glBindTexture(GL_TEXTURE_2D, _crosshairTexture); - if (OculusManager::isConnected() && application->getLastMouseMoveType() == QEvent::MouseMove) { //If we are in oculus, render reticle later _reticleActive[MOUSE] = true; @@ -513,7 +503,6 @@ void ApplicationOverlay::renderPointers() { _mouseY[MOUSE] = application->getMouseY(); _magX[MOUSE] = _mouseX[MOUSE]; _magY[MOUSE] = _mouseY[MOUSE]; - _reticleActive[LEFT_CONTROLLER] = false; _reticleActive[RIGHT_CONTROLLER] = false; @@ -664,6 +653,7 @@ void ApplicationOverlay::renderControllerPointersOculus() { Application* application = Application::getInstance(); QGLWidget* glWidget = application->getGLWidget(); + glm::vec3 cursorVerts[4]; const int widgetWidth = glWidget->width(); const int widgetHeight = glWidget->height(); @@ -678,14 +668,14 @@ void ApplicationOverlay::renderControllerPointersOculus() { //Determine how much we need to iterate const int ITERATIONS = max(myAvatar->getHand()->getNumPalms(), 3); + glm::vec3 eyePos = myAvatar->getHead()->calculateAverageEyePosition(); + for (int i = 0; i < ITERATIONS; i++) { if (useLaser && i < myAvatar->getHand()->getNumPalms()) { PalmData& palm = myAvatar->getHand()->getPalms()[i]; if (palm.isActive()) { glm::vec3 tip = OculusManager::getLaserPointerTipPosition(&palm); - glm::vec3 result; - glm::vec3 eyePos = myAvatar->getHead()->calculateAverageEyePosition(); glm::quat orientation = glm::inverse(myAvatar->getOrientation()); glm::vec3 dir = orientation * glm::normalize(eyePos - tip); //direction of ray goes towards camera glm::vec3 tipPos = (tip - eyePos); @@ -697,7 +687,6 @@ void ApplicationOverlay::renderControllerPointersOculus() { glm::vec3 up = glm::vec3(0.0, 1.0, 0.0) * size; glm::vec3 right = glm::vec3(1.0, 0.0, 0.0) * size; - glm::vec3 cursorVerts[4]; cursorVerts[0] = -right + up; cursorVerts[1] = right + up; cursorVerts[2] = right - up; @@ -749,12 +738,13 @@ void ApplicationOverlay::renderControllerPointersOculus() { // when objToCam and objToCamProj have a very small // angle between them - if ((angleCosine < 0.99990) && (angleCosine > -0.9999)) - if (cursorToCamera.y < 0) { - glRotatef(acos(angleCosine) * DEGREES_PER_RADIAN, 1, 0, 0); - } else { - glRotatef(acos(angleCosine) * DEGREES_PER_RADIAN, -1, 0, 0); - } + // if ((angleCosine < 0.9999) && (angleCosine > -0.99999)) { + if (cursorToCamera.y < 0) { + glRotatef(acos(angleCosine) * DEGREES_PER_RADIAN, 1, 0, 0); + } else { + glRotatef(acos(angleCosine) * DEGREES_PER_RADIAN, -1, 0, 0); + } + // } glBegin(GL_QUADS); @@ -780,6 +770,8 @@ void ApplicationOverlay::renderControllerPointersOculus() { mouseX -= reticleSize / 2; mouseY += reticleSize / 2; + printf("MOUSEPOS: %f %f\n", mouseX, mouseY); + //Get new UV coordinates from our magnification window float newULeft = mouseX / widgetWidth; float newURight = (mouseX + reticleSize) / widgetWidth; @@ -810,10 +802,17 @@ void ApplicationOverlay::renderControllerPointersOculus() { glColor4f(RETICLE_COLOR[0], RETICLE_COLOR[1], RETICLE_COLOR[2], _alpha); - glTexCoord2f(0.0f, 0.0f); glVertex3f(lX, tY, -tlZ); - glTexCoord2f(1.0f, 0.0f); glVertex3f(rX, tY, -trZ); - glTexCoord2f(1.0f, 1.0f); glVertex3f(rX, bY, -brZ); - glTexCoord2f(0.0f, 1.0f); glVertex3f(lX, bY, -blZ); + + const glm::quat& orientation = myAvatar->getOrientation(); + cursorVerts[0] = orientation * glm::vec3(lX, tY, -tlZ) + eyePos; + cursorVerts[1] = orientation * glm::vec3(rX, tY, -trZ) + eyePos; + cursorVerts[2] = orientation * glm::vec3(rX, bY, -brZ) + eyePos; + cursorVerts[3] = orientation * glm::vec3(lX, bY, -blZ) + eyePos; + + glTexCoord2f(0.0f, 0.0f); glVertex3f(cursorVerts[0].x, cursorVerts[0].y, cursorVerts[0].z); + glTexCoord2f(1.0f, 0.0f); glVertex3f(cursorVerts[1].x, cursorVerts[1].y, cursorVerts[1].z); + glTexCoord2f(1.0f, 1.0f); glVertex3f(cursorVerts[2].x, cursorVerts[2].y, cursorVerts[2].z); + glTexCoord2f(0.0f, 1.0f); glVertex3f(cursorVerts[3].x, cursorVerts[3].y, cursorVerts[3].z); glEnd(); } @@ -1183,9 +1182,23 @@ void ApplicationOverlay::renderTexturedHemisphere() { glVertexPointer(3, GL_FLOAT, sizeof(TextureVertex), (void*)0); glTexCoordPointer(2, GL_FLOAT, sizeof(TextureVertex), (void*)12); + + glPushMatrix(); + Application* application = Application::getInstance(); + MyAvatar* myAvatar = application->getAvatar(); + const glm::quat& orientation = myAvatar->getOrientation(); + const glm::vec3& position = myAvatar->getHead()->calculateAverageEyePosition(); + + glm::mat4 rotation = glm::toMat4(orientation); + + glTranslatef(position.x, position.y, position.z); + glMultMatrixf(&rotation[0][0]); + glDrawRangeElements(GL_TRIANGLES, 0, vertices - 1, indices, GL_UNSIGNED_SHORT, 0); + glPopMatrix(); + glDisableClientState(GL_VERTEX_ARRAY); glDisableClientState(GL_TEXTURE_COORD_ARRAY); From f8fdd2d11f3411a4f7a521cfcb88b4bfd154fde7 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Tue, 8 Jul 2014 17:15:11 -0700 Subject: [PATCH 07/70] Minor adjustments --- interface/src/devices/OculusManager.cpp | 2 +- interface/src/ui/ApplicationOverlay.cpp | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index 976bd5ba8e..804ebd1873 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -490,7 +490,7 @@ void OculusManager::renderLaserPointer() { glm::vec3 OculusManager::getLaserPointerTipPosition(const PalmData* palm) { #ifdef HAVE_LIBOVR - const float PALM_TIP_ROD_LENGTH_MULT = 11.0f; + const float PALM_TIP_ROD_LENGTH_MULT = 4.0f; return palm->getPosition() + (palm->getTipPosition() - palm->getPosition()) * PALM_TIP_ROD_LENGTH_MULT; #endif return glm::vec3(0.0f); diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index b768c522ad..bc30ab6b54 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -274,7 +274,6 @@ QPoint ApplicationOverlay::getOculusPalmClickLocation(PalmData *palm) const { const int widgetWidth = glWidget->width(); const int widgetHeight = glWidget->height(); - glm::vec3 tip = OculusManager::getLaserPointerTipPosition(palm); glm::vec3 eyePos = myAvatar->getHead()->calculateAverageEyePosition(); glm::quat orientation = glm::inverse(myAvatar->getOrientation()); @@ -285,9 +284,11 @@ QPoint ApplicationOverlay::getOculusPalmClickLocation(PalmData *palm) const { float t; - //Find intersection of crosshair ray - if (raySphereIntersect(dir, tipPos, 1, &t)){ - glm::vec3 collisionPos = tipPos + dir * t; + //We back the ray up by dir to ensure that it will not start inside the UI. + glm::vec3 adjustedPos = tipPos - dir; + //Find intersection of crosshair ray. + if (raySphereIntersect(dir, adjustedPos, 1, &t)){ + glm::vec3 collisionPos = adjustedPos + dir * t; float u = asin(collisionPos.x) / (_textureFov)+0.5f; float v = 1.0 - (asin(collisionPos.y) / (_textureFov)+0.5f); @@ -682,7 +683,7 @@ void ApplicationOverlay::renderControllerPointersOculus() { float t; float length = glm::length(eyePos - tip); - float size = 0.045f * length; + float size = 0.03f * length; glm::vec3 up = glm::vec3(0.0, 1.0, 0.0) * size; glm::vec3 right = glm::vec3(1.0, 0.0, 0.0) * size; From 801be7edee99278d3fccf53a8a7626c18ddd641c Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Wed, 9 Jul 2014 13:55:38 +0200 Subject: [PATCH 08/70] Stop QMenuBar to steal keyboard focus when pressing Alt key in Linux. --- interface/src/Application.cpp | 2 ++ interface/src/GLCanvas.cpp | 36 +++++++++++++++++++++++++++++++++++ interface/src/GLCanvas.h | 1 + 3 files changed, 39 insertions(+) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index ace265ad4f..95cfaab4cf 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -803,6 +803,7 @@ bool Application::event(QEvent* event) { } void Application::keyPressEvent(QKeyEvent* event) { + qDebug("Application::keyPressEvent(%x)", event->key()); _keysPressed.insert(event->key()); @@ -1053,6 +1054,7 @@ void Application::keyPressEvent(QKeyEvent* event) { } void Application::keyReleaseEvent(QKeyEvent* event) { + qDebug("Application::keyReleaseEvent(%x)", event->key()); _keysPressed.remove(event->key()); diff --git a/interface/src/GLCanvas.cpp b/interface/src/GLCanvas.cpp index 17026b5d5c..87c1016c5e 100644 --- a/interface/src/GLCanvas.cpp +++ b/interface/src/GLCanvas.cpp @@ -23,6 +23,11 @@ GLCanvas::GLCanvas() : QGLWidget(QGLFormat(QGL::NoDepthBuffer)), _throttleRendering(false), _idleRenderInterval(MSECS_PER_FRAME_WHEN_THROTTLED) { +#ifdef Q_OS_LINUX + // Cause GLCanvas::eventFilter to be called. + // It wouldn't hurt to do this on Mac and PC too; but apparently it's only needed on linux. + qApp->installEventFilter(this); +#endif } bool GLCanvas::isThrottleRendering() const { @@ -162,3 +167,34 @@ void GLCanvas::dragEnterEvent(QDragEnterEvent* event) { void GLCanvas::dropEvent(QDropEvent* event) { Application::getInstance()->dropEvent(event); } + +// Pressing Alt (and Meta) key alone activates the menubar because its style inherits the +// SHMenuBarAltKeyNavigation from QWindowsStyle. This makes it impossible for a scripts to +// receive keyPress events for the Alt (and Meta) key in a reliable manner. +// +// This filter catches events before QMenuBar can steal the keyboard focus. +// The idea was borrowed from +// http://www.archivum.info/qt-interest@trolltech.com/2006-09/00053/Re-(Qt4)-Alt-key-focus-QMenuBar-(solved).html + +bool GLCanvas::eventFilter(QObject*, QEvent* event) { + switch (event->type()) { + case QEvent::KeyPress: + case QEvent::KeyRelease: + case QEvent::ShortcutOverride: + { + QKeyEvent* keyEvent = static_cast(event); + if (keyEvent->key() == Qt::Key_Alt || keyEvent->key() == Qt::Key_Meta) { + if (event->type() == QEvent::KeyPress) + keyPressEvent(keyEvent); + else if (event->type() == QEvent::KeyRelease) + keyReleaseEvent(keyEvent); + else + QGLWidget::event(event); + return true; + } + } + default: + break; + } + return false; +} diff --git a/interface/src/GLCanvas.h b/interface/src/GLCanvas.h index 024cd615ae..773fcb5c27 100644 --- a/interface/src/GLCanvas.h +++ b/interface/src/GLCanvas.h @@ -50,6 +50,7 @@ protected: private slots: void activeChanged(Qt::ApplicationState state); void throttleRender(); + bool eventFilter(QObject*, QEvent* event); }; #endif // hifi_GLCanvas_h From 54e8ed5e1125505a1cd37b75d6a3d066cc145d06 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 9 Jul 2014 09:41:49 -0700 Subject: [PATCH 09/70] added MovingMinMaxAvg and unit test; added additional stats to AudioStreamStats struct --- .gitignore | 3 + libraries/audio/src/AudioStreamStats.h | 27 +++- libraries/shared/src/MovingMinMaxAvg.h | 175 ++++++++++++++++++++++++ tests/shared/src/MovingMinMaxAvgTests.h | 25 ++++ tests/shared/src/main.cpp | 3 + 5 files changed, 231 insertions(+), 2 deletions(-) create mode 100644 libraries/shared/src/MovingMinMaxAvg.h create mode 100644 tests/shared/src/MovingMinMaxAvgTests.h diff --git a/.gitignore b/.gitignore index 4176dcc652..d3af3c4535 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,6 @@ interface/external/rtmidi/* # Ignore interfaceCache for Linux users interface/interfaceCache/ +tests/shared/src/MovingMinMaxAvgTests.cpp +examples/dancer.js +examples/happyBirthday.js diff --git a/libraries/audio/src/AudioStreamStats.h b/libraries/audio/src/AudioStreamStats.h index 004d697fcf..69a8751af9 100644 --- a/libraries/audio/src/AudioStreamStats.h +++ b/libraries/audio/src/AudioStreamStats.h @@ -19,7 +19,18 @@ public: AudioStreamStats() : _streamType(PositionalAudioRingBuffer::Microphone), _streamIdentifier(), - _jitterBufferFrames(0), + _timeGapMin(0), + _timeGapMax(0), + _timeGapAverage(0.0f), + _timeGapMovingMin(0), + _timeGapMovingMax(0), + _timeGapMovingAverage(0.0f), + _ringBufferFramesAvailable(0), + _ringBufferCurrentJitterBufferFrames(0), + _ringBufferDesiredJitterBufferFrames(0), + _ringBufferStarveCount(0), + _ringBufferOverflowCount(0), + _ringBufferSilentFramesDropped(0), _packetsReceived(0), _packetsUnreasonable(0), _packetsEarly(0), @@ -32,7 +43,19 @@ public: PositionalAudioRingBuffer::Type _streamType; QUuid _streamIdentifier; - quint16 _jitterBufferFrames; + quint64 _timeGapMin; + quint64 _timeGapMax; + float _timeGapAverage; + quint64 _timeGapMovingMin; + quint64 _timeGapMovingMax; + float _timeGapMovingAverage; + + quint32 _ringBufferFramesAvailable; + quint16 _ringBufferCurrentJitterBufferFrames; + quint16 _ringBufferDesiredJitterBufferFrames; + quint32 _ringBufferStarveCount; + quint32 _ringBufferOverflowCount; + quint32 _ringBufferSilentFramesDropped; quint32 _packetsReceived; quint32 _packetsUnreasonable; diff --git a/libraries/shared/src/MovingMinMaxAvg.h b/libraries/shared/src/MovingMinMaxAvg.h new file mode 100644 index 0000000000..606bf0b481 --- /dev/null +++ b/libraries/shared/src/MovingMinMaxAvg.h @@ -0,0 +1,175 @@ +// +// MovingMinMaxAvg.h +// libraries/shared/src +// +// Created by Yixin Wang on 7/8/2014 +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_MovingMinMaxAvg_h +#define hifi_MovingMinMaxAvg_h + +template +class MovingMinMaxAvg { + +public: + // This class collects 3 stats (min, max, avg) over a moving window of samples. + // The moving window contains _windowIntervals * _intervalLength samples. + // Those stats are updated every _intervalLength samples collected. When that happens, _newStatsAvaialble is set + // to true and it's up to the user to clear that flag. + // For example, if you want a moving avg of the past 5000 samples updated every 100 samples, you would instantiate + // this class with MovingMinMaxAvg(100, 50). If you want a moving min of the past 100 samples updated on every + // new sample, instantiate this class with MovingMinMaxAvg(1, 100). + + MovingMinMaxAvg(int intervalLength, int windowIntervals) + : _min(std::numeric_limits::max()), + _max(std::numeric_limits::min()), + _average(0.0), + _samplesCollected(0), + _intervalLength(intervalLength), + _windowIntervals(windowIntervals), + _existingSamplesInCurrentInterval(0), + _existingIntervals(0), + _windowMin(std::numeric_limits::max()), + _windowMax(std::numeric_limits::min()), + _windowAverage(0.0), + _currentIntervalMin(std::numeric_limits::max()), + _currentIntervalMax(std::numeric_limits::min()), + _currentIntervalAverage(0.0), + _newestIntervalStatsAt(0), + _newStatsAvailable(false) + { + _intervalMins = new T[_windowIntervals]; + _intervalMaxes = new T[_windowIntervals]; + _intervalAverages = new double[_windowIntervals]; + } + + ~MovingMinMaxAvg() { + delete[] _intervalMins; + delete[] _intervalMaxes; + delete[] _intervalAverages; + } + + void reset() { + _min = std::numeric_limits::max(); + _max = std::numeric_limits::min(); + _average = 0.0; + _samplesCollected = 0; + _existingSamplesInCurrentInterval = 0; + _existingIntervals = 0; + _windowMin = std::numeric_limits::max(); + _windowMax = std::numeric_limits::min(); + _windowAverage = 0.0; + _currentIntervalMin = std::numeric_limits::max(); + _currentIntervalMax = std::numeric_limits::min(); + _currentIntervalAverage = 0.0; + _newStatsAvailableFlag = false; + } + + void update(T newSample) { + + // update overall stats + if (newSample < _min) { + _min = newSample; + } + if (newSample > _max) { + _max = newSample; + } + _average = (_average * _samplesCollected + newSample) / (_samplesCollected + 1); + _samplesCollected++; + + // update the current interval stats + if (newSample < _currentIntervalMin) { + _currentIntervalMin = newSample; + } + if (newSample > _currentIntervalMax) { + _currentIntervalMax = newSample; + } + _currentIntervalAverage = (_currentIntervalAverage * _existingSamplesInCurrentInterval + newSample) / (_existingSamplesInCurrentInterval + 1); + _existingSamplesInCurrentInterval++; + + // if the current interval of samples is now full, record its stats into our past intervals' stats + if (_existingSamplesInCurrentInterval == _intervalLength) { + + // increment index of the newest interval's stats cyclically + _newestIntervalStatsAt = _newestIntervalStatsAt == _windowIntervals - 1 ? 0 : _newestIntervalStatsAt + 1; + + // record current interval's stats, then reset them + _intervalMins[_newestIntervalStatsAt] = _currentIntervalMin; + _intervalMaxes[_newestIntervalStatsAt] = _currentIntervalMax; + _intervalAverages[_newestIntervalStatsAt] = _currentIntervalAverage; + _currentIntervalMin = std::numeric_limits::max(); + _currentIntervalMax = std::numeric_limits::min(); + _currentIntervalAverage = 0.0; + _existingSamplesInCurrentInterval = 0; + + if (_existingIntervals < _windowIntervals) { + _existingIntervals++; + } + + // update the window's stats + int k = _newestIntervalStatsAt; + _windowMin = _intervalMins[k]; + _windowMax = _intervalMaxes[k]; + double intervalAveragesSum = _intervalAverages[k]; + for (int i = 1; i < _existingIntervals; i++) { + k = k == 0 ? _windowIntervals - 1 : k - 1; + if (_intervalMins[k] < _windowMin) { + _windowMin = _intervalMins[k]; + } + if (_intervalMaxes[k] > _windowMax) { + _windowMax = _intervalMaxes[k]; + } + intervalAveragesSum += _intervalAverages[k]; + } + _windowAverage = intervalAveragesSum / _existingIntervals; + + _newStatsAvailable = true; + } + } + + + bool getNewStatsAvailableFlag() const { return _newStatsAvailable; } + void clearNewStatsAvailableFlag() { _newStatsAvailable = false; } + + T getMin() const { return _min; } + T getMax() const { return _max; } + double getAverage() const { return _average; } + T getWindowMin() const { return _windowMin; } + T getWindowMax() const { return _windowMax; } + double getWindowAverage() const { return _windowAverage; } + +private: + // these are min/max/avg stats for all samples collected. + T _min; + T _max; + double _average; + int _samplesCollected; + + int _intervalLength; + int _windowIntervals; + + int _existingSamplesInCurrentInterval; + int _existingIntervals; + + // these are the min/max/avg stats for the samples in the moving window + T _windowMin; + T _windowMax; + double _windowAverage; + + T _currentIntervalMin; + T _currentIntervalMax; + double _currentIntervalAverage; + + T* _intervalMins; + T* _intervalMaxes; + double* _intervalAverages; + int _newestIntervalStatsAt; + + bool _newStatsAvailable; +}; + +#endif // hifi_OctalCode_h diff --git a/tests/shared/src/MovingMinMaxAvgTests.h b/tests/shared/src/MovingMinMaxAvgTests.h new file mode 100644 index 0000000000..52a2edf0af --- /dev/null +++ b/tests/shared/src/MovingMinMaxAvgTests.h @@ -0,0 +1,25 @@ +// +// MovingMinMaxAvgTests.h +// tests/shared/src +// +// Created by Yixin Wang on 7/8/2014 +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_MovingMinMaxAvgTests_h +#define hifi_MovingMinMaxAvgTests_h + +#include "MovingMinMaxAvg.h" +#include "SharedUtil.h" + +namespace MovingMinMaxAvgTests { + + quint64 randQuint64(); + + void runAllTests(); +} + +#endif // hifi_MovingMinMaxAvgTests_h diff --git a/tests/shared/src/main.cpp b/tests/shared/src/main.cpp index 6215d394a8..d4251eef7a 100644 --- a/tests/shared/src/main.cpp +++ b/tests/shared/src/main.cpp @@ -10,9 +10,12 @@ #include "AngularConstraintTests.h" #include "MovingPercentileTests.h" +#include "MovingMinMaxAvgTests.h" int main(int argc, char** argv) { + MovingMinMaxAvgTests::runAllTests(); MovingPercentileTests::runAllTests(); AngularConstraintTests::runAllTests(); + getchar(); return 0; } From d044af140001d6eec01e56b352d301fb2a8770ef Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 9 Jul 2014 11:12:02 -0700 Subject: [PATCH 10/70] Made it so you can only select in a 180 degree radius with oculus --- interface/src/devices/SixenseManager.cpp | 3 +-- interface/src/ui/ApplicationOverlay.cpp | 14 ++++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/interface/src/devices/SixenseManager.cpp b/interface/src/devices/SixenseManager.cpp index ecf7fea5cb..33d485ea84 100644 --- a/interface/src/devices/SixenseManager.cpp +++ b/interface/src/devices/SixenseManager.cpp @@ -373,7 +373,6 @@ void SixenseManager::emulateMouse(PalmData* palm, int index) { if (OculusManager::isConnected()) { pos = application->getApplicationOverlay().getOculusPalmClickLocation(palm); - printf("CLICK: %d %d\n", pos.x(), pos.y()); } else { // Get directon relative to avatar orientation glm::vec3 direction = glm::inverse(avatar->getOrientation()) * palm->getFingerDirection(); @@ -392,7 +391,7 @@ void SixenseManager::emulateMouse(PalmData* palm, int index) { //If we are off screen then we should stop processing, and if a trigger or bumper is pressed, //we should unpress them. - if (pos.x() < 0 || pos.x() > widget->width() || pos.y() < 0 || pos.y() > widget->height()) { + if (pos.x() == INT_MAX) { if (_bumperPressed[index]) { QMouseEvent mouseEvent(QEvent::MouseButtonRelease, pos, bumperButton, bumperButton, 0); diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index bc30ab6b54..87ee46ca2b 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -290,12 +290,18 @@ QPoint ApplicationOverlay::getOculusPalmClickLocation(PalmData *palm) const { if (raySphereIntersect(dir, adjustedPos, 1, &t)){ glm::vec3 collisionPos = adjustedPos + dir * t; - float u = asin(collisionPos.x) / (_textureFov)+0.5f; - float v = 1.0 - (asin(collisionPos.y) / (_textureFov)+0.5f); + //If we hit the back hemisphere, mark it as not a collision + if (collisionPos.z > 0) { + rv.setX(INT_MAX); + rv.setY(INT_MAX); + } else { - rv.setX(u * glWidget->width()); - rv.setY(v * glWidget->height()); + float u = asin(collisionPos.x) / (_textureFov)+0.5f; + float v = 1.0 - (asin(collisionPos.y) / (_textureFov)+0.5f); + rv.setX(u * glWidget->width()); + rv.setY(v * glWidget->height()); + } } else { //if they did not click on the overlay, just set the coords to INT_MAX rv.setX(INT_MAX); From d03d3ef8176113abeaf53a81ad784b3f40c80581 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 9 Jul 2014 11:59:50 -0700 Subject: [PATCH 11/70] new auidostreamstats displayed in interface, domain page stats updated --- .../src/audio/AudioMixerClientData.cpp | 93 +++++--- .../src/audio/AvatarAudioRingBuffer.cpp | 2 +- interface/src/ui/Stats.cpp | 43 +++- libraries/audio/src/AudioRingBuffer.h | 4 +- libraries/audio/src/AudioStreamStats.h | 2 + .../audio/src/InjectedAudioRingBuffer.cpp | 2 +- .../audio/src/PositionalAudioRingBuffer.cpp | 87 ++----- .../audio/src/PositionalAudioRingBuffer.h | 42 ++-- libraries/networking/src/PacketHeaders.cpp | 2 + libraries/shared/src/MovingMinMaxAvg.h | 21 +- tests/shared/src/MovingMinMaxAvgTests.cpp | 218 ++++++++++++++++++ 11 files changed, 369 insertions(+), 147 deletions(-) create mode 100644 tests/shared/src/MovingMinMaxAvgTests.cpp diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index d3883501d6..9e27103bd1 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -18,6 +18,7 @@ #include "AudioMixer.h" #include "AudioMixerClientData.h" +#include "MovingMinMaxAvg.h" AudioMixerClientData::AudioMixerClientData() : _ringBuffers(), @@ -159,25 +160,41 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { } AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer) const { + AudioStreamStats streamStats; - SequenceNumberStats streamSequenceNumberStats; + const SequenceNumberStats* streamSequenceNumberStats; streamStats._streamType = ringBuffer->getType(); if (streamStats._streamType == PositionalAudioRingBuffer::Injector) { streamStats._streamIdentifier = ((InjectedAudioRingBuffer*)ringBuffer)->getStreamIdentifier(); - streamSequenceNumberStats = _incomingInjectedAudioSequenceNumberStatsMap.value(streamStats._streamIdentifier); + streamSequenceNumberStats = &_incomingInjectedAudioSequenceNumberStatsMap[streamStats._streamIdentifier]; } else { - streamSequenceNumberStats = _incomingAvatarAudioSequenceNumberStats; + streamSequenceNumberStats = &_incomingAvatarAudioSequenceNumberStats; } - streamStats._jitterBufferFrames = ringBuffer->getCurrentJitterBufferFrames(); - streamStats._packetsReceived = streamSequenceNumberStats.getNumReceived(); - streamStats._packetsUnreasonable = streamSequenceNumberStats.getNumUnreasonable(); - streamStats._packetsEarly = streamSequenceNumberStats.getNumEarly(); - streamStats._packetsLate = streamSequenceNumberStats.getNumLate(); - streamStats._packetsLost = streamSequenceNumberStats.getNumLost(); - streamStats._packetsRecovered = streamSequenceNumberStats.getNumRecovered(); - streamStats._packetsDuplicate = streamSequenceNumberStats.getNumDuplicate(); + const MovingMinMaxAvg& timeGapStats = ringBuffer->getInterframeTimeGapStatsForStatsPacket(); + streamStats._timeGapMin = timeGapStats.getMin(); + streamStats._timeGapMax = timeGapStats.getMax(); + streamStats._timeGapAverage = timeGapStats.getAverage(); + streamStats._timeGapMovingMin = timeGapStats.getWindowMin(); + streamStats._timeGapMovingMax = timeGapStats.getWindowMax(); + streamStats._timeGapMovingAverage = timeGapStats.getWindowAverage(); + + streamStats._ringBufferFramesAvailable = ringBuffer->framesAvailable(); + streamStats._ringBufferCurrentJitterBufferFrames = ringBuffer->getCurrentJitterBufferFrames(); + streamStats._ringBufferDesiredJitterBufferFrames = ringBuffer->getDesiredJitterBufferFrames(); + streamStats._ringBufferStarveCount = ringBuffer->getStarveCount(); + streamStats._ringBufferConsecutiveNotMixedCount = ringBuffer->getConsecutiveNotMixedCount(); + streamStats._ringBufferOverflowCount = ringBuffer->getOverflowCount(); + streamStats._ringBufferSilentFramesDropped = ringBuffer->getSilentFramesDropped(); + + streamStats._packetsReceived = streamSequenceNumberStats->getNumReceived(); + streamStats._packetsUnreasonable = streamSequenceNumberStats->getNumUnreasonable(); + streamStats._packetsEarly = streamSequenceNumberStats->getNumEarly(); + streamStats._packetsLate = streamSequenceNumberStats->getNumLate(); + streamStats._packetsLost = streamSequenceNumberStats->getNumLost(); + streamStats._packetsRecovered = streamSequenceNumberStats->getNumRecovered(); + streamStats._packetsDuplicate = streamSequenceNumberStats->getNumDuplicate(); return streamStats; } @@ -236,44 +253,46 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { QString result; AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); if (avatarRingBuffer) { - int desiredJitterBuffer = avatarRingBuffer->getDesiredJitterBufferFrames(); - int calculatedJitterBuffer = avatarRingBuffer->getCalculatedDesiredJitterBufferFrames(); - int currentJitterBuffer = avatarRingBuffer->getCurrentJitterBufferFrames(); - int overflowCount = avatarRingBuffer->getOverflowCount(); - int samplesAvailable = avatarRingBuffer->samplesAvailable(); - int framesAvailable = (samplesAvailable / avatarRingBuffer->getSamplesPerFrame()); AudioStreamStats streamStats = getAudioStreamStatsOfStream(avatarRingBuffer); - result += "mic.desired:" + QString::number(desiredJitterBuffer) - + " calculated:" + QString::number(calculatedJitterBuffer) - + " current:" + QString::number(currentJitterBuffer) - + " available:" + QString::number(framesAvailable) - + " samples:" + QString::number(samplesAvailable) - + " overflows:" + QString::number(overflowCount) + result += "mic.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) + + " current:" + QString::number(streamStats._ringBufferCurrentJitterBufferFrames) + + " available:" + QString::number(streamStats._ringBufferFramesAvailable) + + " starves:" + QString::number(streamStats._ringBufferStarveCount) + + " not mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + + " overflows:" + QString::number(streamStats._ringBufferOverflowCount) + + " silents dropped:" + QString::number(streamStats._ringBufferSilentFramesDropped) + " early:" + QString::number(streamStats._packetsEarly) + " late:" + QString::number(streamStats._packetsLate) - + " lost:" + QString::number(streamStats._packetsLost); + + " lost:" + QString::number(streamStats._packetsLost) + + " min gap:" + QString::number(streamStats._timeGapMin) + + " max gap:" + QString::number(streamStats._timeGapMax) + + " avg gap:" + QString::number(streamStats._timeGapAverage, 'g', 2) + + " min 30s gap:" + QString::number(streamStats._timeGapMovingMin) + + " max 30s gap:" + QString::number(streamStats._timeGapMovingMax) + + " avg 30s gap:" + QString::number(streamStats._timeGapMovingAverage, 'g', 2); } else { result = "mic unknown"; } for (int i = 0; i < _ringBuffers.size(); i++) { if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) { - int desiredJitterBuffer = _ringBuffers[i]->getDesiredJitterBufferFrames(); - int calculatedJitterBuffer = _ringBuffers[i]->getCalculatedDesiredJitterBufferFrames(); - int currentJitterBuffer = _ringBuffers[i]->getCurrentJitterBufferFrames(); - int overflowCount = _ringBuffers[i]->getOverflowCount(); - int samplesAvailable = _ringBuffers[i]->samplesAvailable(); - int framesAvailable = (samplesAvailable / _ringBuffers[i]->getSamplesPerFrame()); AudioStreamStats streamStats = getAudioStreamStatsOfStream(_ringBuffers[i]); - result += "| injected[" + QString::number(i) + "].desired:" + QString::number(desiredJitterBuffer) - + " calculated:" + QString::number(calculatedJitterBuffer) - + " current:" + QString::number(currentJitterBuffer) - + " available:" + QString::number(framesAvailable) - + " samples:" + QString::number(samplesAvailable) - + " overflows:" + QString::number(overflowCount) + result += "mic.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) + + " current:" + QString::number(streamStats._ringBufferCurrentJitterBufferFrames) + + " available:" + QString::number(streamStats._ringBufferFramesAvailable) + + " starves:" + QString::number(streamStats._ringBufferStarveCount) + + " not mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + + " overflows:" + QString::number(streamStats._ringBufferOverflowCount) + + " silents dropped:" + QString::number(streamStats._ringBufferSilentFramesDropped) + " early:" + QString::number(streamStats._packetsEarly) + " late:" + QString::number(streamStats._packetsLate) - + " lost:" + QString::number(streamStats._packetsLost); + + " lost:" + QString::number(streamStats._packetsLost) + + " min gap:" + QString::number(streamStats._timeGapMin) + + " max gap:" + QString::number(streamStats._timeGapMax) + + " avg gap:" + QString::number(streamStats._timeGapAverage, 'g', 2) + + " min 30s gap:" + QString::number(streamStats._timeGapMovingMin) + + " max 30s gap:" + QString::number(streamStats._timeGapMovingMax) + + " avg 30s gap:" + QString::number(streamStats._timeGapMovingAverage, 'g', 2); } } return result; diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp index 9c6cc32f57..0177bc48ea 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp @@ -19,7 +19,7 @@ AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo, bool dynamicJitterBu } int AvatarAudioRingBuffer::parseData(const QByteArray& packet) { - _interframeTimeGapStats.frameReceived(); + frameReceived(); updateDesiredJitterBufferFrames(); _shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho); diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 379dd35df7..0165f82591 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -291,7 +291,7 @@ void Stats::display( const AudioStreamStats& audioMixerAvatarStreamStats = audio->getAudioMixerAvatarStreamStats(); const QHash& audioMixerInjectedStreamStatsMap = audio->getAudioMixerInjectedStreamStatsMap(); - lines = _expanded ? 10 + audioMixerInjectedStreamStatsMap.size(): 3; + lines = _expanded ? 12 + audioMixerInjectedStreamStatsMap.size() * 3: 3; drawBackground(backgroundColor, horizontalOffset, 0, _pingStatsWidth, lines * STATS_PELS_PER_LINE + 10); horizontalOffset += 5; @@ -354,17 +354,48 @@ void Stats::display( drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamLabelString, color); char upstreamAudioStatsString[30]; - sprintf(upstreamAudioStatsString, " mic: %d/%d/%d, %d", audioMixerAvatarStreamStats._packetsEarly, + sprintf(upstreamAudioStatsString, " mic: %d/%d/%d, %d/%d/%d", audioMixerAvatarStreamStats._packetsEarly, audioMixerAvatarStreamStats._packetsLate, audioMixerAvatarStreamStats._packetsLost, - audioMixerAvatarStreamStats._jitterBufferFrames); + audioMixerAvatarStreamStats._ringBufferFramesAvailable, audioMixerAvatarStreamStats._ringBufferCurrentJitterBufferFrames, + audioMixerAvatarStreamStats._ringBufferDesiredJitterBufferFrames); + + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + + sprintf(upstreamAudioStatsString, " %llu/%llu/%.2f, %u/%u", audioMixerAvatarStreamStats._timeGapMin, + audioMixerAvatarStreamStats._timeGapMax, audioMixerAvatarStreamStats._timeGapAverage, + audioMixerAvatarStreamStats._ringBufferStarveCount, audioMixerAvatarStreamStats._ringBufferOverflowCount); + + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + + sprintf(upstreamAudioStatsString, " %llu/%llu/%.2f, %u/%u", audioMixerAvatarStreamStats._timeGapMovingMin, + audioMixerAvatarStreamStats._timeGapMovingMax, audioMixerAvatarStreamStats._timeGapMovingAverage, + audioMixerAvatarStreamStats._ringBufferConsecutiveNotMixedCount, audioMixerAvatarStreamStats._ringBufferSilentFramesDropped); verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); foreach(AudioStreamStats injectedStreamStats, audioMixerInjectedStreamStatsMap) { - sprintf(upstreamAudioStatsString, " inj: %d/%d/%d, %d", injectedStreamStats._packetsEarly, - injectedStreamStats._packetsLate, injectedStreamStats._packetsLost, injectedStreamStats._jitterBufferFrames); - + sprintf(upstreamAudioStatsString, " inj: %d/%d/%d, %d/%d/%d", injectedStreamStats._packetsEarly, + injectedStreamStats._packetsLate, injectedStreamStats._packetsLost, + injectedStreamStats._ringBufferFramesAvailable, injectedStreamStats._ringBufferCurrentJitterBufferFrames, + injectedStreamStats._ringBufferDesiredJitterBufferFrames); + + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + + sprintf(upstreamAudioStatsString, " %llu/%llu/%.2f, %u/%u", injectedStreamStats._timeGapMin, + injectedStreamStats._timeGapMax, injectedStreamStats._timeGapAverage, + injectedStreamStats._ringBufferStarveCount, injectedStreamStats._ringBufferOverflowCount); + + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + + sprintf(upstreamAudioStatsString, " %llu/%llu/%.2f, %u/%u", injectedStreamStats._timeGapMovingMin, + injectedStreamStats._timeGapMovingMax, injectedStreamStats._timeGapMovingAverage, + injectedStreamStats._ringBufferConsecutiveNotMixedCount, injectedStreamStats._ringBufferSilentFramesDropped); + verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); } diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 047db70693..38f1adec21 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -20,6 +20,7 @@ #include #include "NodeData.h" +#include "SharedUtil.h" const int SAMPLE_RATE = 24000; @@ -29,7 +30,7 @@ const int NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL = 512; const int NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL = NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL / sizeof(int16_t); const unsigned int BUFFER_SEND_INTERVAL_USECS = floorf((NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL - / (float) SAMPLE_RATE) * 1000 * 1000); + / (float) SAMPLE_RATE) * USECS_PER_SECOND); const int MAX_SAMPLE_VALUE = std::numeric_limits::max(); const int MIN_SAMPLE_VALUE = std::numeric_limits::min(); @@ -65,6 +66,7 @@ public: void shiftReadPosition(unsigned int numSamples); int samplesAvailable() const; + int framesAvailable() const { return samplesAvailable() / _numFrameSamples; } bool isNotStarvedOrHasMinimumSamples(int numRequiredSamples) const; diff --git a/libraries/audio/src/AudioStreamStats.h b/libraries/audio/src/AudioStreamStats.h index 69a8751af9..2c66187309 100644 --- a/libraries/audio/src/AudioStreamStats.h +++ b/libraries/audio/src/AudioStreamStats.h @@ -29,6 +29,7 @@ public: _ringBufferCurrentJitterBufferFrames(0), _ringBufferDesiredJitterBufferFrames(0), _ringBufferStarveCount(0), + _ringBufferConsecutiveNotMixedCount(0), _ringBufferOverflowCount(0), _ringBufferSilentFramesDropped(0), _packetsReceived(0), @@ -54,6 +55,7 @@ public: quint16 _ringBufferCurrentJitterBufferFrames; quint16 _ringBufferDesiredJitterBufferFrames; quint32 _ringBufferStarveCount; + quint32 _ringBufferConsecutiveNotMixedCount; quint32 _ringBufferOverflowCount; quint32 _ringBufferSilentFramesDropped; diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index c84fe173c9..d3d0cdfb8d 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -31,7 +31,7 @@ InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier, const uchar MAX_INJECTOR_VOLUME = 255; int InjectedAudioRingBuffer::parseData(const QByteArray& packet) { - _interframeTimeGapStats.frameReceived(); + frameReceived(); updateDesiredJitterBufferFrames(); // setup a data stream to read from this packet diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 8a03561140..666b89e568 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -21,70 +21,6 @@ #include "PositionalAudioRingBuffer.h" #include "SharedUtil.h" -InterframeTimeGapStats::InterframeTimeGapStats() - : _lastFrameReceivedTime(0), - _numSamplesInCurrentInterval(0), - _currentIntervalMaxGap(0), - _newestIntervalMaxGapAt(0), - _windowMaxGap(0), - _newWindowMaxGapAvailable(false) -{ - memset(_intervalMaxGaps, 0, TIME_GAP_NUM_INTERVALS_IN_WINDOW * sizeof(quint64)); -} - -void InterframeTimeGapStats::frameReceived() { - quint64 now = usecTimestampNow(); - - // make sure this isn't the first time frameReceived() is called so can actually calculate a gap. - if (_lastFrameReceivedTime != 0) { - quint64 gap = now - _lastFrameReceivedTime; - - // update the current interval max - if (gap > _currentIntervalMaxGap) { - _currentIntervalMaxGap = gap; - - // keep the window max gap at least as large as the current interval max - // this allows the window max gap to respond immediately to a sudden spike in gap times - // also, this prevents the window max gap from staying at 0 until the first interval of samples filled up - if (_currentIntervalMaxGap > _windowMaxGap) { - _windowMaxGap = _currentIntervalMaxGap; - _newWindowMaxGapAvailable = true; - } - } - _numSamplesInCurrentInterval++; - - // if the current interval of samples is now full, record it in our interval maxes - if (_numSamplesInCurrentInterval == TIME_GAP_NUM_SAMPLES_IN_INTERVAL) { - - // find location to insert this interval's max (increment index cyclically) - _newestIntervalMaxGapAt = _newestIntervalMaxGapAt == TIME_GAP_NUM_INTERVALS_IN_WINDOW - 1 ? 0 : _newestIntervalMaxGapAt + 1; - - // record the current interval's max gap as the newest - _intervalMaxGaps[_newestIntervalMaxGapAt] = _currentIntervalMaxGap; - - // update the window max gap, which is the max out of all the past intervals' max gaps - _windowMaxGap = 0; - for (int i = 0; i < TIME_GAP_NUM_INTERVALS_IN_WINDOW; i++) { - if (_intervalMaxGaps[i] > _windowMaxGap) { - _windowMaxGap = _intervalMaxGaps[i]; - } - } - _newWindowMaxGapAvailable = true; - - // reset the current interval - _numSamplesInCurrentInterval = 0; - _currentIntervalMaxGap = 0; - } - } - _lastFrameReceivedTime = now; -} - -quint64 InterframeTimeGapStats::getWindowMaxGap() { - _newWindowMaxGapAvailable = false; - return _windowMaxGap; -} - - PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo, bool dynamicJitterBuffers) : AudioRingBuffer(isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, @@ -97,6 +33,9 @@ PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer:: _shouldOutputStarveDebug(true), _isStereo(isStereo), _listenerUnattenuatedZone(NULL), + _lastFrameReceivedTime(0), + _interframeTimeGapStatsForJitterCalc(TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES, TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS), + _interframeTimeGapStatsForStatsPacket(TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES, TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS), _desiredJitterBufferFrames(1), _currentJitterBufferFrames(-1), _dynamicJitterBuffers(dynamicJitterBuffers), @@ -230,7 +169,7 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { // reset our _shouldOutputStarveDebug to true so the next is printed _shouldOutputStarveDebug = true; - _consecutiveNotMixedCount++; + _consecutiveNotMixedCount = 1; return false; } @@ -240,7 +179,6 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { // minus one (since a frame will be read immediately after this) is the length of the jitter buffer _currentJitterBufferFrames = samplesAvailable() / samplesPerFrame - 1; _isStarved = false; - _consecutiveNotMixedCount = 0; } // since we've read data from ring buffer at least once - we've started @@ -253,21 +191,31 @@ int PositionalAudioRingBuffer::getCalculatedDesiredJitterBufferFrames() const { int calculatedDesiredJitterBufferFrames = 1; const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; - calculatedDesiredJitterBufferFrames = ceilf((float)_interframeTimeGapStats.peekWindowMaxGap() / USECS_PER_FRAME); + calculatedDesiredJitterBufferFrames = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME); if (calculatedDesiredJitterBufferFrames < 1) { calculatedDesiredJitterBufferFrames = 1; } return calculatedDesiredJitterBufferFrames; } +void PositionalAudioRingBuffer::frameReceived() { + quint64 now = usecTimestampNow(); + if (_lastFrameReceivedTime != 0) { + quint64 gap = now - _lastFrameReceivedTime; + _interframeTimeGapStatsForJitterCalc.update(gap); + _interframeTimeGapStatsForStatsPacket.update(gap); + } + _lastFrameReceivedTime = now; +} + void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { - if (_interframeTimeGapStats.hasNewWindowMaxGapAvailable()) { + if (_interframeTimeGapStatsForJitterCalc.getNewStatsAvailableFlag()) { if (!_dynamicJitterBuffers) { _desiredJitterBufferFrames = 1; // HACK to see if this fixes the audio silence } else { const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; - _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStats.getWindowMaxGap() / USECS_PER_FRAME); + _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME); if (_desiredJitterBufferFrames < 1) { _desiredJitterBufferFrames = 1; } @@ -276,5 +224,6 @@ void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { _desiredJitterBufferFrames = maxDesired; } } + _interframeTimeGapStatsForJitterCalc.clearNewStatsAvailableFlag(); } } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 5922b27002..a3adec0117 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -17,31 +17,17 @@ #include #include "AudioRingBuffer.h" +#include "MovingMinMaxAvg.h" -// this means that every 500 samples, the max for the past 10*500 samples will be calculated -const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 500; -const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 10; +// the time gaps stats for _desiredJitterBufferFrames calculation +// will recalculate the max for the past 5000 samples every 500 samples +const int TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES = 500; +const int TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS = 10; -// class used to track time between incoming frames for the purpose of varying the jitter buffer length -class InterframeTimeGapStats { -public: - InterframeTimeGapStats(); - - void frameReceived(); - bool hasNewWindowMaxGapAvailable() const { return _newWindowMaxGapAvailable; } - quint64 peekWindowMaxGap() const { return _windowMaxGap; } - quint64 getWindowMaxGap(); - -private: - quint64 _lastFrameReceivedTime; - - int _numSamplesInCurrentInterval; - quint64 _currentIntervalMaxGap; - quint64 _intervalMaxGaps[TIME_GAP_NUM_INTERVALS_IN_WINDOW]; - int _newestIntervalMaxGapAt; - quint64 _windowMaxGap; - bool _newWindowMaxGapAvailable; -}; +// the time gap stats for constructing AudioStreamStats will +// recalculate min/max/avg every ~1 second for the past ~30 seconds of time gap data +const int TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS; +const int TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS = 30; const int AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY = 100; @@ -79,17 +65,22 @@ public: int getSamplesPerFrame() const { return _isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; } + const MovingMinMaxAvg& getInterframeTimeGapStatsForStatsPacket() const { return _interframeTimeGapStatsForStatsPacket; } + int getCalculatedDesiredJitterBufferFrames() const; /// returns what we would calculate our desired as if asked int getDesiredJitterBufferFrames() const { return _desiredJitterBufferFrames; } int getCurrentJitterBufferFrames() const { return _currentJitterBufferFrames; } int getConsecutiveNotMixedCount() const { return _consecutiveNotMixedCount; } + int getStarveCount() const { return _starveCount; } + int getSilentFramesDropped() const { return _silentFramesDropped; } protected: // disallow copying of PositionalAudioRingBuffer objects PositionalAudioRingBuffer(const PositionalAudioRingBuffer&); PositionalAudioRingBuffer& operator= (const PositionalAudioRingBuffer&); + void frameReceived(); void updateDesiredJitterBufferFrames(); PositionalAudioRingBuffer::Type _type; @@ -103,7 +94,10 @@ protected: float _nextOutputTrailingLoudness; AABox* _listenerUnattenuatedZone; - InterframeTimeGapStats _interframeTimeGapStats; + quint64 _lastFrameReceivedTime; + MovingMinMaxAvg _interframeTimeGapStatsForJitterCalc; + MovingMinMaxAvg _interframeTimeGapStatsForStatsPacket; + int _desiredJitterBufferFrames; int _currentJitterBufferFrames; bool _dynamicJitterBuffers; diff --git a/libraries/networking/src/PacketHeaders.cpp b/libraries/networking/src/PacketHeaders.cpp index a5c05a6ae9..f17715ddfe 100644 --- a/libraries/networking/src/PacketHeaders.cpp +++ b/libraries/networking/src/PacketHeaders.cpp @@ -78,6 +78,8 @@ PacketVersion versionForPacketType(PacketType type) { return 2; case PacketTypeModelErase: return 1; + case PacketTypeAudioStreamStats: + return 1; default: return 0; } diff --git a/libraries/shared/src/MovingMinMaxAvg.h b/libraries/shared/src/MovingMinMaxAvg.h index 606bf0b481..5263e2ab6c 100644 --- a/libraries/shared/src/MovingMinMaxAvg.h +++ b/libraries/shared/src/MovingMinMaxAvg.h @@ -78,8 +78,7 @@ public: if (newSample > _max) { _max = newSample; } - _average = (_average * _samplesCollected + newSample) / (_samplesCollected + 1); - _samplesCollected++; + updateAverage(_average, _samplesCollected, (double)newSample); // update the current interval stats if (newSample < _currentIntervalMin) { @@ -88,8 +87,7 @@ public: if (newSample > _currentIntervalMax) { _currentIntervalMax = newSample; } - _currentIntervalAverage = (_currentIntervalAverage * _existingSamplesInCurrentInterval + newSample) / (_existingSamplesInCurrentInterval + 1); - _existingSamplesInCurrentInterval++; + updateAverage(_currentIntervalAverage, _existingSamplesInCurrentInterval, (double)newSample); // if the current interval of samples is now full, record its stats into our past intervals' stats if (_existingSamplesInCurrentInterval == _intervalLength) { @@ -114,8 +112,9 @@ public: int k = _newestIntervalStatsAt; _windowMin = _intervalMins[k]; _windowMax = _intervalMaxes[k]; - double intervalAveragesSum = _intervalAverages[k]; - for (int i = 1; i < _existingIntervals; i++) { + _windowAverage = _intervalAverages[k]; + int intervalsIncludedInWindowStats = 1; + while (intervalsIncludedInWindowStats < _existingIntervals) { k = k == 0 ? _windowIntervals - 1 : k - 1; if (_intervalMins[k] < _windowMin) { _windowMin = _intervalMins[k]; @@ -123,9 +122,8 @@ public: if (_intervalMaxes[k] > _windowMax) { _windowMax = _intervalMaxes[k]; } - intervalAveragesSum += _intervalAverages[k]; + updateAverage(_windowAverage, intervalsIncludedInWindowStats, _intervalAverages[k]); } - _windowAverage = intervalAveragesSum / _existingIntervals; _newStatsAvailable = true; } @@ -142,6 +140,13 @@ public: T getWindowMax() const { return _windowMax; } double getWindowAverage() const { return _windowAverage; } +private: + void updateAverage(double& average, int& numSamples, double newSample) { + // update some running average without overflowing it + average = average * ((double)numSamples / (numSamples + 1)) + newSample / (numSamples + 1); + numSamples++; + } + private: // these are min/max/avg stats for all samples collected. T _min; diff --git a/tests/shared/src/MovingMinMaxAvgTests.cpp b/tests/shared/src/MovingMinMaxAvgTests.cpp new file mode 100644 index 0000000000..ae23248e5f --- /dev/null +++ b/tests/shared/src/MovingMinMaxAvgTests.cpp @@ -0,0 +1,218 @@ +// +// MovingMinMaxAvgTests.cpp +// tests/shared/src +// +// Created by Yixin Wang on 7/8/2014 +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "MovingMinMaxAvgTests.h" +#include + +quint64 MovingMinMaxAvgTests::randQuint64() { + quint64 ret = 0; + for (int i = 0; i < 32; i++) { + ret = (ret + rand() % 4); + ret *= 4; + } + return ret; +} + +void MovingMinMaxAvgTests::runAllTests() { + { + // quint64 test + + const int INTERVAL_LENGTH = 100; + const int WINDOW_INTERVALS = 50; + + MovingMinMaxAvg stats(INTERVAL_LENGTH, WINDOW_INTERVALS); + + quint64 min = std::numeric_limits::max(); + quint64 max = 0; + double average = 0.0; + int totalSamples = 0; + + quint64 windowMin; + quint64 windowMax; + double windowAverage; + + QQueue windowSamples; + // fill window samples + for (int i = 0; i < 100000; i++) { + + quint64 sample = randQuint64(); + + windowSamples.enqueue(sample); + if (windowSamples.size() > INTERVAL_LENGTH * WINDOW_INTERVALS) { + windowSamples.dequeue(); + } + + stats.update(sample); + + min = std::min(min, sample); + max = std::max(max, sample); + average = (average * totalSamples + sample) / (totalSamples + 1); + totalSamples++; + + assert(stats.getMin() == min); + assert(stats.getMax() == max); + assert(abs(stats.getAverage() / average - 1.0) < 0.000001); + + if ((i + 1) % INTERVAL_LENGTH == 0) { + + assert(stats.getNewStatsAvailableFlag()); + stats.clearNewStatsAvailableFlag(); + + windowMin = std::numeric_limits::max(); + windowMax = 0; + windowAverage = 0.0; + foreach(quint64 s, windowSamples) { + windowMin = std::min(windowMin, s); + windowMax = std::max(windowMax, s); + windowAverage += (double)s; + } + windowAverage /= (double)windowSamples.size(); + + assert(stats.getWindowMin() == windowMin); + assert(stats.getWindowMax() == windowMax); + assert(abs(stats.getAverage() / average - 1.0) < 0.000001); + + } else { + assert(!stats.getNewStatsAvailableFlag()); + } + } + } + + { + // int test + + const int INTERVAL_LENGTH = 1; + const int WINDOW_INTERVALS = 75; + + MovingMinMaxAvg stats(INTERVAL_LENGTH, WINDOW_INTERVALS); + + int min = std::numeric_limits::max(); + int max = 0; + double average = 0.0; + int totalSamples = 0; + + int windowMin; + int windowMax; + double windowAverage; + + QQueue windowSamples; + // fill window samples + for (int i = 0; i < 100000; i++) { + + int sample = rand(); + + windowSamples.enqueue(sample); + if (windowSamples.size() > INTERVAL_LENGTH * WINDOW_INTERVALS) { + windowSamples.dequeue(); + } + + stats.update(sample); + + min = std::min(min, sample); + max = std::max(max, sample); + average = (average * totalSamples + sample) / (totalSamples + 1); + totalSamples++; + + assert(stats.getMin() == min); + assert(stats.getMax() == max); + assert(abs(stats.getAverage() / average - 1.0) < 0.000001); + + if ((i + 1) % INTERVAL_LENGTH == 0) { + + assert(stats.getNewStatsAvailableFlag()); + stats.clearNewStatsAvailableFlag(); + + windowMin = std::numeric_limits::max(); + windowMax = 0; + windowAverage = 0.0; + foreach(int s, windowSamples) { + windowMin = std::min(windowMin, s); + windowMax = std::max(windowMax, s); + windowAverage += (double)s; + } + windowAverage /= (double)windowSamples.size(); + + assert(stats.getWindowMin() == windowMin); + assert(stats.getWindowMax() == windowMax); + assert(abs(stats.getAverage() / average - 1.0) < 0.000001); + + } else { + assert(!stats.getNewStatsAvailableFlag()); + } + } + } + + { + // float test + + const int INTERVAL_LENGTH = 57; + const int WINDOW_INTERVALS = 1; + + MovingMinMaxAvg stats(INTERVAL_LENGTH, WINDOW_INTERVALS); + + float min = std::numeric_limits::max(); + float max = 0; + double average = 0.0; + int totalSamples = 0; + + float windowMin; + float windowMax; + double windowAverage; + + QQueue windowSamples; + // fill window samples + for (int i = 0; i < 100000; i++) { + + float sample = randFloat(); + + windowSamples.enqueue(sample); + if (windowSamples.size() > INTERVAL_LENGTH * WINDOW_INTERVALS) { + windowSamples.dequeue(); + } + + stats.update(sample); + + min = std::min(min, sample); + max = std::max(max, sample); + average = (average * totalSamples + sample) / (totalSamples + 1); + totalSamples++; + + assert(stats.getMin() == min); + assert(stats.getMax() == max); + assert(abs(stats.getAverage() / average - 1.0) < 0.000001); + + if ((i + 1) % INTERVAL_LENGTH == 0) { + + assert(stats.getNewStatsAvailableFlag()); + stats.clearNewStatsAvailableFlag(); + + windowMin = std::numeric_limits::max(); + windowMax = 0; + windowAverage = 0.0; + foreach(float s, windowSamples) { + windowMin = std::min(windowMin, s); + windowMax = std::max(windowMax, s); + windowAverage += (double)s; + } + windowAverage /= (double)windowSamples.size(); + + assert(stats.getWindowMin() == windowMin); + assert(stats.getWindowMax() == windowMax); + assert(abs(stats.getAverage() / average - 1.0) < 0.000001); + + } else { + assert(!stats.getNewStatsAvailableFlag()); + } + } + } + printf("moving min/max/avg test passed!\n"); +} + From e64f34a257d44056e46e85ac368747e3fde4871f Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 9 Jul 2014 12:03:32 -0700 Subject: [PATCH 12/70] Separated mouse and controller pointers --- interface/src/ui/ApplicationOverlay.cpp | 125 ++++++++++++------------ interface/src/ui/ApplicationOverlay.h | 2 +- 2 files changed, 63 insertions(+), 64 deletions(-) diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 87ee46ca2b..b9b10f697e 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -376,7 +376,7 @@ void ApplicationOverlay::displayOverlayTextureOculus(Camera& whichCamera) { renderTexturedHemisphere(); - renderControllerPointersOculus(); + renderPointersOculus(); glDepthMask(GL_TRUE); glBindTexture(GL_TEXTURE_2D, 0); @@ -654,7 +654,7 @@ void ApplicationOverlay::renderControllerPointers() { } } -void ApplicationOverlay::renderControllerPointersOculus() { +void ApplicationOverlay::renderPointersOculus() { const bool useLaser = true; @@ -672,12 +672,10 @@ void ApplicationOverlay::renderControllerPointersOculus() { glMatrixMode(GL_MODELVIEW); MyAvatar* myAvatar = application->getAvatar(); - //Determine how much we need to iterate - const int ITERATIONS = max(myAvatar->getHand()->getNumPalms(), 3); - glm::vec3 eyePos = myAvatar->getHead()->calculateAverageEyePosition(); - for (int i = 0; i < ITERATIONS; i++) { + //Controller Pointers + for (int i = 0; i < myAvatar->getHand()->getNumPalms(); i++) { if (useLaser && i < myAvatar->getHand()->getNumPalms()) { PalmData& palm = myAvatar->getHand()->getPalms()[i]; @@ -766,64 +764,65 @@ void ApplicationOverlay::renderControllerPointersOculus() { glPopMatrix(); } - } else if (i < NUMBER_OF_MAGNIFIERS) { - //Dont render the reticle if its inactive - if (!_reticleActive[i]) { - continue; - } - - float mouseX = (float)_mouseX[i]; - float mouseY = (float)_mouseY[i]; - mouseX -= reticleSize / 2; - mouseY += reticleSize / 2; - - printf("MOUSEPOS: %f %f\n", mouseX, mouseY); - - //Get new UV coordinates from our magnification window - float newULeft = mouseX / widgetWidth; - float newURight = (mouseX + reticleSize) / widgetWidth; - float newVBottom = 1.0 - mouseY / widgetHeight; - float newVTop = 1.0 - (mouseY - reticleSize) / widgetHeight; - - // Project our position onto the hemisphere using the UV coordinates - float lX = sin((newULeft - 0.5f) * _textureFov); - float rX = sin((newURight - 0.5f) * _textureFov); - float bY = sin((newVBottom - 0.5f) * _textureFov); - float tY = sin((newVTop - 0.5f) * _textureFov); - - float dist; - //Bottom Left - dist = sqrt(lX * lX + bY * bY); - float blZ = sqrt(1.0f - dist * dist); - //Top Left - dist = sqrt(lX * lX + tY * tY); - float tlZ = sqrt(1.0f - dist * dist); - //Bottom Right - dist = sqrt(rX * rX + bY * bY); - float brZ = sqrt(1.0f - dist * dist); - //Top Right - dist = sqrt(rX * rX + tY * tY); - float trZ = sqrt(1.0f - dist * dist); - - glBegin(GL_QUADS); - - glColor4f(RETICLE_COLOR[0], RETICLE_COLOR[1], RETICLE_COLOR[2], _alpha); - - - const glm::quat& orientation = myAvatar->getOrientation(); - cursorVerts[0] = orientation * glm::vec3(lX, tY, -tlZ) + eyePos; - cursorVerts[1] = orientation * glm::vec3(rX, tY, -trZ) + eyePos; - cursorVerts[2] = orientation * glm::vec3(rX, bY, -brZ) + eyePos; - cursorVerts[3] = orientation * glm::vec3(lX, bY, -blZ) + eyePos; - - glTexCoord2f(0.0f, 0.0f); glVertex3f(cursorVerts[0].x, cursorVerts[0].y, cursorVerts[0].z); - glTexCoord2f(1.0f, 0.0f); glVertex3f(cursorVerts[1].x, cursorVerts[1].y, cursorVerts[1].z); - glTexCoord2f(1.0f, 1.0f); glVertex3f(cursorVerts[2].x, cursorVerts[2].y, cursorVerts[2].z); - glTexCoord2f(0.0f, 1.0f); glVertex3f(cursorVerts[3].x, cursorVerts[3].y, cursorVerts[3].z); - - glEnd(); - } + } } + + //Mouse Pointer + if (_reticleActive[MOUSE]) { + + float mouseX = (float)_mouseX[MOUSE]; + float mouseY = (float)_mouseY[MOUSE]; + mouseX -= reticleSize / 2; + mouseY += reticleSize / 2; + + printf("MOUSEPOS: %f %f\n", mouseX, mouseY); + + //Get new UV coordinates from our magnification window + float newULeft = mouseX / widgetWidth; + float newURight = (mouseX + reticleSize) / widgetWidth; + float newVBottom = 1.0 - mouseY / widgetHeight; + float newVTop = 1.0 - (mouseY - reticleSize) / widgetHeight; + + // Project our position onto the hemisphere using the UV coordinates + float lX = sin((newULeft - 0.5f) * _textureFov); + float rX = sin((newURight - 0.5f) * _textureFov); + float bY = sin((newVBottom - 0.5f) * _textureFov); + float tY = sin((newVTop - 0.5f) * _textureFov); + + float dist; + //Bottom Left + dist = sqrt(lX * lX + bY * bY); + float blZ = sqrt(1.0f - dist * dist); + //Top Left + dist = sqrt(lX * lX + tY * tY); + float tlZ = sqrt(1.0f - dist * dist); + //Bottom Right + dist = sqrt(rX * rX + bY * bY); + float brZ = sqrt(1.0f - dist * dist); + //Top Right + dist = sqrt(rX * rX + tY * tY); + float trZ = sqrt(1.0f - dist * dist); + + glBegin(GL_QUADS); + + glColor4f(RETICLE_COLOR[0], RETICLE_COLOR[1], RETICLE_COLOR[2], _alpha); + + + const glm::quat& orientation = myAvatar->getOrientation(); + cursorVerts[0] = orientation * glm::vec3(lX, tY, -tlZ) + eyePos; + cursorVerts[1] = orientation * glm::vec3(rX, tY, -trZ) + eyePos; + cursorVerts[2] = orientation * glm::vec3(rX, bY, -brZ) + eyePos; + cursorVerts[3] = orientation * glm::vec3(lX, bY, -blZ) + eyePos; + + glTexCoord2f(0.0f, 0.0f); glVertex3f(cursorVerts[0].x, cursorVerts[0].y, cursorVerts[0].z); + glTexCoord2f(1.0f, 0.0f); glVertex3f(cursorVerts[1].x, cursorVerts[1].y, cursorVerts[1].z); + glTexCoord2f(1.0f, 1.0f); glVertex3f(cursorVerts[2].x, cursorVerts[2].y, cursorVerts[2].z); + glTexCoord2f(0.0f, 1.0f); glVertex3f(cursorVerts[3].x, cursorVerts[3].y, cursorVerts[3].z); + + glEnd(); + } + + glEnable(GL_DEPTH_TEST); } diff --git a/interface/src/ui/ApplicationOverlay.h b/interface/src/ui/ApplicationOverlay.h index ebac5f3cfc..117dd02f3b 100644 --- a/interface/src/ui/ApplicationOverlay.h +++ b/interface/src/ui/ApplicationOverlay.h @@ -49,7 +49,7 @@ private: void renderPointers(); void renderControllerPointers(); - void renderControllerPointersOculus(); + void renderPointersOculus(); void renderMagnifier(int mouseX, int mouseY, float sizeMult, bool showBorder) const; void renderAudioMeter(); void renderStatsAndLogs(); From a8cf19925677ae7df0387df87a223123ba399f54 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 9 Jul 2014 12:08:41 -0700 Subject: [PATCH 13/70] added format labels in interface for new audio stats --- interface/src/ui/Stats.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 0165f82591..e86e656f2e 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -291,7 +291,7 @@ void Stats::display( const AudioStreamStats& audioMixerAvatarStreamStats = audio->getAudioMixerAvatarStreamStats(); const QHash& audioMixerInjectedStreamStatsMap = audio->getAudioMixerInjectedStreamStatsMap(); - lines = _expanded ? 12 + audioMixerInjectedStreamStatsMap.size() * 3: 3; + lines = _expanded ? 12 + (audioMixerInjectedStreamStatsMap.size() + 1) * 3: 3; drawBackground(backgroundColor, horizontalOffset, 0, _pingStatsWidth, lines * STATS_PELS_PER_LINE + 10); horizontalOffset += 5; @@ -328,12 +328,21 @@ void Stats::display( drawText(horizontalOffset, verticalOffset, scale, rotation, font, voxelMaxPing, color); char audioMixerStatsLabelString[] = "AudioMixer stats:"; - char streamStatsFormatLabelString[] = "early/late/lost, jframes"; + char streamStatsFormatLabelString[] = "early/late/lost"; + char streamStatsFormatLabelString2[] = "avail/currJ/desiredJ"; + char streamStatsFormatLabelString3[] = "gaps: min/max/avg, starv/ovfl"; + char streamStatsFormatLabelString4[] = "30s gaps: (same), notmix/sdrop"; verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerStatsLabelString, color); verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString, color); + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString2, color); + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString3, color); + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString4, color); char downstreamLabelString[] = " Downstream:"; From a2225da13d53dfba7ce9436f3896fc26438ed796 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 9 Jul 2014 12:11:06 -0700 Subject: [PATCH 14/70] Code cleanup --- interface/src/ui/ApplicationOverlay.cpp | 135 ++++++++++-------------- 1 file changed, 53 insertions(+), 82 deletions(-) diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index b9b10f697e..15dc18e74e 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -95,8 +95,6 @@ void ApplicationOverlay::renderOverlay(bool renderToTexture) { glMatrixMode(GL_PROJECTION); glPushMatrix(); - printf("%d %d\n", glWidget->width(), glWidget->height()); - glLoadIdentity(); gluOrtho2D(0, glWidget->width(), glWidget->height(), 0); glDisable(GL_DEPTH_TEST); @@ -656,8 +654,6 @@ void ApplicationOverlay::renderControllerPointers() { void ApplicationOverlay::renderPointersOculus() { - const bool useLaser = true; - Application* application = Application::getInstance(); QGLWidget* glWidget = application->getGLWidget(); glm::vec3 cursorVerts[4]; @@ -676,94 +672,72 @@ void ApplicationOverlay::renderPointersOculus() { //Controller Pointers for (int i = 0; i < myAvatar->getHand()->getNumPalms(); i++) { - if (useLaser && i < myAvatar->getHand()->getNumPalms()) { - - PalmData& palm = myAvatar->getHand()->getPalms()[i]; - if (palm.isActive()) { - glm::vec3 tip = OculusManager::getLaserPointerTipPosition(&palm); - glm::quat orientation = glm::inverse(myAvatar->getOrientation()); - glm::vec3 dir = orientation * glm::normalize(eyePos - tip); //direction of ray goes towards camera - glm::vec3 tipPos = (tip - eyePos); + + PalmData& palm = myAvatar->getHand()->getPalms()[i]; + if (palm.isActive()) { + glm::vec3 tip = OculusManager::getLaserPointerTipPosition(&palm); + glm::quat orientation = glm::inverse(myAvatar->getOrientation()); + glm::vec3 dir = orientation * glm::normalize(eyePos - tip); //direction of ray goes towards camera + glm::vec3 tipPos = (tip - eyePos); - float t; - float length = glm::length(eyePos - tip); - float size = 0.03f * length; + float length = glm::length(eyePos - tip); + float size = 0.03f * length; - glm::vec3 up = glm::vec3(0.0, 1.0, 0.0) * size; - glm::vec3 right = glm::vec3(1.0, 0.0, 0.0) * size; + glm::vec3 up = glm::vec3(0.0, 1.0, 0.0) * size; + glm::vec3 right = glm::vec3(1.0, 0.0, 0.0) * size; - cursorVerts[0] = -right + up; - cursorVerts[1] = right + up; - cursorVerts[2] = right - up; - cursorVerts[3] = -right - up; + cursorVerts[0] = -right + up; + cursorVerts[1] = right + up; + cursorVerts[2] = right - up; + cursorVerts[3] = -right - up; - glPushMatrix(); - // glLoadIdentity(); + glPushMatrix(); - // objToCamProj is the vector in world coordinates from the - // local origin to the camera projected in the XZ plane - glm::vec3 cursorToCameraXZ(-tipPos.x, 0, -tipPos.z); - cursorToCameraXZ = glm::normalize(cursorToCameraXZ); + // objToCamProj is the vector in world coordinates from the + // local origin to the camera projected in the XZ plane + glm::vec3 cursorToCameraXZ(-tipPos.x, 0, -tipPos.z); + cursorToCameraXZ = glm::normalize(cursorToCameraXZ); - // This is the original lookAt vector for the object - // in world coordinates - glm::vec3 direction(0, 0, 1); - glTranslatef(tip.x, tip.y, tip.z); + //Translate the cursor to the tip of the oculus ray + glTranslatef(tip.x, tip.y, tip.z); - // easy fix to determine wether the angle is negative or positive - // for positive angles upAux will be a vector pointing in the - // positive y direction, otherwise upAux will point downwards - // effectively reversing the rotation. - glm::vec3 upAux = glm::cross(direction, cursorToCameraXZ); + glm::vec3 direction(0, 0, 1); + // easy fix to determine wether the angle is negative or positive + // for positive angles upAux will be a vector pointing in the + // positive y direction, otherwise upAux will point downwards + // effectively reversing the rotation. + glm::vec3 upAux = glm::cross(direction, cursorToCameraXZ); - // compute the angle - float angleCosine = glm::dot(direction, cursorToCameraXZ); + // compute the angle + float angleCosine = glm::dot(direction, cursorToCameraXZ); - // perform the rotation. The if statement is used for stability reasons - // if the lookAt and objToCamProj vectors are too close together then - // |angleCosine| could be bigger than 1 due to lack of precision - if ((angleCosine < 0.999999) && (angleCosine > -0.999999)) { - glRotatef(acos(angleCosine) * DEGREES_PER_RADIAN, upAux[0], upAux[1], upAux[2]); - } + //Rotate in XZ direction + glRotatef(acos(angleCosine) * DEGREES_PER_RADIAN, upAux[0], upAux[1], upAux[2]); - // so far it is just like the cylindrical billboard. The code for the - // second rotation comes now - // The second part tilts the object so that it faces the camera + glm::vec3 cursorToCamera = glm::normalize(-tipPos); - // objToCam is the vector in world coordinates from - // the local origin to the camera - glm::vec3 cursorToCamera = glm::normalize(-tipPos); + // Compute the angle between cursorToCameraXZ and cursorToCamera, + angleCosine = glm::dot(cursorToCameraXZ, cursorToCamera); - // Compute the angle between objToCamProj and objToCam, - //i.e. compute the required angle for the lookup vector - - angleCosine = glm::dot(cursorToCameraXZ, cursorToCamera); - - // Tilt the object. The test is done to prevent instability - // when objToCam and objToCamProj have a very small - // angle between them - - // if ((angleCosine < 0.9999) && (angleCosine > -0.99999)) { - if (cursorToCamera.y < 0) { - glRotatef(acos(angleCosine) * DEGREES_PER_RADIAN, 1, 0, 0); - } else { - glRotatef(acos(angleCosine) * DEGREES_PER_RADIAN, -1, 0, 0); - } - // } - - glBegin(GL_QUADS); - - glColor4f(RETICLE_COLOR[0], RETICLE_COLOR[1], RETICLE_COLOR[2], _alpha); - - glTexCoord2f(0.0f, 0.0f); glVertex3f(cursorVerts[0].x, cursorVerts[0].y, cursorVerts[0].z); - glTexCoord2f(1.0f, 0.0f); glVertex3f(cursorVerts[1].x, cursorVerts[1].y, cursorVerts[1].z); - glTexCoord2f(1.0f, 1.0f); glVertex3f(cursorVerts[2].x, cursorVerts[2].y, cursorVerts[2].z); - glTexCoord2f(0.0f, 1.0f); glVertex3f(cursorVerts[3].x, cursorVerts[3].y, cursorVerts[3].z); - - glEnd(); - - glPopMatrix(); + //Rotate in Y direction + if (cursorToCamera.y < 0) { + glRotatef(acos(angleCosine) * DEGREES_PER_RADIAN, 1, 0, 0); + } else { + glRotatef(acos(angleCosine) * DEGREES_PER_RADIAN, -1, 0, 0); } + + glBegin(GL_QUADS); + + glColor4f(RETICLE_COLOR[0], RETICLE_COLOR[1], RETICLE_COLOR[2], _alpha); + + glTexCoord2f(0.0f, 0.0f); glVertex3f(cursorVerts[0].x, cursorVerts[0].y, cursorVerts[0].z); + glTexCoord2f(1.0f, 0.0f); glVertex3f(cursorVerts[1].x, cursorVerts[1].y, cursorVerts[1].z); + glTexCoord2f(1.0f, 1.0f); glVertex3f(cursorVerts[2].x, cursorVerts[2].y, cursorVerts[2].z); + glTexCoord2f(0.0f, 1.0f); glVertex3f(cursorVerts[3].x, cursorVerts[3].y, cursorVerts[3].z); + + glEnd(); + + glPopMatrix(); } } @@ -775,8 +749,6 @@ void ApplicationOverlay::renderPointersOculus() { mouseX -= reticleSize / 2; mouseY += reticleSize / 2; - printf("MOUSEPOS: %f %f\n", mouseX, mouseY); - //Get new UV coordinates from our magnification window float newULeft = mouseX / widgetWidth; float newURight = (mouseX + reticleSize) / widgetWidth; @@ -807,7 +779,6 @@ void ApplicationOverlay::renderPointersOculus() { glColor4f(RETICLE_COLOR[0], RETICLE_COLOR[1], RETICLE_COLOR[2], _alpha); - const glm::quat& orientation = myAvatar->getOrientation(); cursorVerts[0] = orientation * glm::vec3(lX, tY, -tlZ) + eyePos; cursorVerts[1] = orientation * glm::vec3(rX, tY, -trZ) + eyePos; From efe443e2ff5b930b10bc8c075152af8e7b286b96 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 9 Jul 2014 14:01:46 -0700 Subject: [PATCH 15/70] Very basic metavoxel save/load. --- .../src/metavoxels/MetavoxelServer.cpp | 59 +++++++++++++++++++ .../src/metavoxels/MetavoxelServer.h | 23 ++++++++ libraries/metavoxels/src/Bitstream.cpp | 6 ++ libraries/metavoxels/src/Bitstream.h | 5 ++ 4 files changed, 93 insertions(+) diff --git a/assignment-client/src/metavoxels/MetavoxelServer.cpp b/assignment-client/src/metavoxels/MetavoxelServer.cpp index 14765e2ddc..928a693527 100644 --- a/assignment-client/src/metavoxels/MetavoxelServer.cpp +++ b/assignment-client/src/metavoxels/MetavoxelServer.cpp @@ -10,6 +10,9 @@ // #include +#include +#include +#include #include @@ -44,6 +47,18 @@ void MetavoxelServer::run() { _lastSend = QDateTime::currentMSecsSinceEpoch(); _sendTimer.start(SEND_INTERVAL); + + // initialize Bitstream before using it in multiple threads + Bitstream::preThreadingInit(); + + // create the persister and start the it in its own thread + _persister = new MetavoxelPersister(this); + QThread* persistenceThread = new QThread(this); + _persister->moveToThread(persistenceThread); + persistenceThread->start(); + + // queue up the load + QMetaObject::invokeMethod(_persister, "load"); } void MetavoxelServer::readPendingDatagrams() { @@ -67,6 +82,12 @@ void MetavoxelServer::readPendingDatagrams() { } } +void MetavoxelServer::aboutToFinish() { + QMetaObject::invokeMethod(_persister, "save", Q_ARG(const MetavoxelData&, _data)); + _persister->thread()->quit(); + _persister->thread()->wait(); +} + void MetavoxelServer::maybeAttachSession(const SharedNodePointer& node) { if (node->getType() == NodeType::Agent) { QMutexLocker locker(&node->getMutex()); @@ -193,3 +214,41 @@ void MetavoxelSession::sendPacketGroup(int alreadySent) { _sequencer.endPacket(); } } + +MetavoxelPersister::MetavoxelPersister(MetavoxelServer* server) : + _server(server) { +} + +const char* SAVE_FILE = "metavoxels.dat"; + +void MetavoxelPersister::load() { + QFile file(SAVE_FILE); + if (!file.exists()) { + return; + } + QDebug debug = qDebug() << "Reading from" << SAVE_FILE << "..."; + file.open(QIODevice::ReadOnly); + QDataStream inStream(&file); + Bitstream in(inStream); + MetavoxelData data; + try { + in >> data; + } catch (const BitstreamException& e) { + debug << "failed, " << e.getDescription(); + return; + } + QMetaObject::invokeMethod(_server, "setData", Q_ARG(const MetavoxelData&, data)); + debug << "done."; +} + +void MetavoxelPersister::save(const MetavoxelData& data) { + QDebug debug = qDebug() << "Writing to" << SAVE_FILE << "..."; + QSaveFile file(SAVE_FILE); + file.open(QIODevice::WriteOnly); + QDataStream outStream(&file); + Bitstream out(outStream); + out << data; + out.flush(); + file.commit(); + debug << "done."; +} diff --git a/assignment-client/src/metavoxels/MetavoxelServer.h b/assignment-client/src/metavoxels/MetavoxelServer.h index 6df769227c..9ed765f65f 100644 --- a/assignment-client/src/metavoxels/MetavoxelServer.h +++ b/assignment-client/src/metavoxels/MetavoxelServer.h @@ -20,6 +20,7 @@ #include class MetavoxelEditMessage; +class MetavoxelPersister; class MetavoxelSession; /// Maintains a shared metavoxel system, accepting change requests and broadcasting updates. @@ -33,11 +34,15 @@ public: void applyEdit(const MetavoxelEditMessage& edit); const MetavoxelData& getData() const { return _data; } + + Q_INVOKABLE void setData(const MetavoxelData& data) { _data = data; } virtual void run(); virtual void readPendingDatagrams(); + virtual void aboutToFinish(); + private slots: void maybeAttachSession(const SharedNodePointer& node); @@ -45,6 +50,8 @@ private slots: private: + MetavoxelPersister* _persister; + QTimer _sendTimer; qint64 _lastSend; @@ -88,4 +95,20 @@ private: int _reliableDeltaID; }; +/// Handles persistence in a separate thread. +class MetavoxelPersister : public QObject { + Q_OBJECT + +public: + + MetavoxelPersister(MetavoxelServer* server); + + Q_INVOKABLE void load(); + Q_INVOKABLE void save(const MetavoxelData& data); + +private: + + MetavoxelServer* _server; +}; + #endif // hifi_MetavoxelServer_h diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index bc662aa890..0d4f8a52b4 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -87,6 +87,12 @@ IDStreamer& IDStreamer::operator>>(int& value) { return *this; } +void Bitstream::preThreadingInit() { + getObjectStreamers(); + getEnumStreamers(); + getEnumStreamersByName(); +} + int Bitstream::registerMetaObject(const char* className, const QMetaObject* metaObject) { getMetaObjects().insert(className, metaObject); diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 70fde94b79..dacfeb2ee9 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -290,6 +290,11 @@ public: QHash sharedObjectValues; }; + /// Performs all of the various lazily initializations (of object streamers, etc.) If multiple threads need to use + /// Bitstream instances, call this beforehand to prevent errors from occurring when multiple threads attempt lazy + /// initialization simultaneously. + static void preThreadingInit(); + /// Registers a metaobject under its name so that instances of it can be streamed. Consider using the REGISTER_META_OBJECT /// at the top level of the source file associated with the class rather than calling this function directly. /// \return zero; the function only returns a value so that it can be used in static initialization From f9036dbaecbff5d7a4eceb018d40e51902febb80 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 9 Jul 2014 14:22:33 -0700 Subject: [PATCH 16/70] Lock the attribute mapping, since it can be accessed from multiple threads. --- libraries/metavoxels/src/AttributeRegistry.cpp | 9 +++++++++ libraries/metavoxels/src/AttributeRegistry.h | 8 +++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/libraries/metavoxels/src/AttributeRegistry.cpp b/libraries/metavoxels/src/AttributeRegistry.cpp index 33ce298859..b43759fa4f 100644 --- a/libraries/metavoxels/src/AttributeRegistry.cpp +++ b/libraries/metavoxels/src/AttributeRegistry.cpp @@ -9,7 +9,9 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +#include #include +#include #include "AttributeRegistry.h" #include "MetavoxelData.h" @@ -69,6 +71,7 @@ AttributePointer AttributeRegistry::registerAttribute(AttributePointer attribute if (!attribute) { return attribute; } + QWriteLocker locker(&_attributesLock); AttributePointer& pointer = _attributes[attribute->getName()]; if (!pointer) { pointer = attribute; @@ -77,9 +80,15 @@ AttributePointer AttributeRegistry::registerAttribute(AttributePointer attribute } void AttributeRegistry::deregisterAttribute(const QString& name) { + QWriteLocker locker(&_attributesLock); _attributes.remove(name); } +AttributePointer AttributeRegistry::getAttribute(const QString& name) { + QReadLocker locker(&_attributesLock); + return _attributes.value(name); +} + QScriptValue AttributeRegistry::getAttribute(QScriptContext* context, QScriptEngine* engine) { return engine->newQObject(getInstance()->getAttribute(context->argument(0).toString()).data(), QScriptEngine::QtOwnership, QScriptEngine::PreferExistingWrapperObject); diff --git a/libraries/metavoxels/src/AttributeRegistry.h b/libraries/metavoxels/src/AttributeRegistry.h index 7dc2e110b8..23e3c1fa97 100644 --- a/libraries/metavoxels/src/AttributeRegistry.h +++ b/libraries/metavoxels/src/AttributeRegistry.h @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -61,11 +62,14 @@ public: void deregisterAttribute(const QString& name); /// Retrieves an attribute by name. - AttributePointer getAttribute(const QString& name) const { return _attributes.value(name); } + AttributePointer getAttribute(const QString& name); /// Returns a reference to the attribute hash. const QHash& getAttributes() const { return _attributes; } + /// Returns a reference to the attributes lock. + QReadWriteLock& getAttributesLock() { return _attributesLock; } + /// Returns a reference to the standard SharedObjectPointer "guide" attribute. const AttributePointer& getGuideAttribute() const { return _guideAttribute; } @@ -92,6 +96,8 @@ private: static QScriptValue getAttribute(QScriptContext* context, QScriptEngine* engine); QHash _attributes; + QReadWriteLock _attributesLock; + AttributePointer _guideAttribute; AttributePointer _spannersAttribute; AttributePointer _colorAttribute; From 81e168f6578246859a9501794616281b055c6ece Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 9 Jul 2014 15:38:03 -0700 Subject: [PATCH 17/70] added RingBufferHistory template class, used it in SentPacketHistory and MovingMinMaxAvg --- .../networking/src/SentPacketHistory.cpp | 23 +-- libraries/networking/src/SentPacketHistory.h | 6 +- libraries/shared/src/MovingMinMaxAvg.h | 190 ++++++++---------- libraries/shared/src/RingBufferHistory.h | 146 ++++++++++++++ tests/shared/src/MovingMinMaxAvgTests.cpp | 4 +- 5 files changed, 234 insertions(+), 135 deletions(-) create mode 100644 libraries/shared/src/RingBufferHistory.h diff --git a/libraries/networking/src/SentPacketHistory.cpp b/libraries/networking/src/SentPacketHistory.cpp index 841b5e909c..3cdb0af8c0 100644 --- a/libraries/networking/src/SentPacketHistory.cpp +++ b/libraries/networking/src/SentPacketHistory.cpp @@ -14,8 +14,6 @@ SentPacketHistory::SentPacketHistory(int size) : _sentPackets(size), - _newestPacketAt(0), - _numExistingPackets(0), _newestSequenceNumber(std::numeric_limits::max()) { } @@ -29,16 +27,8 @@ void SentPacketHistory::packetSent(uint16_t sequenceNumber, const QByteArray& pa qDebug() << "Unexpected sequence number passed to SentPacketHistory::packetSent()!" << "Expected:" << expectedSequenceNumber << "Actual:" << sequenceNumber; } - _newestSequenceNumber = sequenceNumber; - - // increment _newestPacketAt cyclically, insert new packet there. - // this will overwrite the oldest packet in the buffer - _newestPacketAt = (_newestPacketAt == _sentPackets.size() - 1) ? 0 : _newestPacketAt + 1; - _sentPackets[_newestPacketAt] = packet; - if (_numExistingPackets < _sentPackets.size()) { - _numExistingPackets++; - } + _sentPackets.insert(packet); } const QByteArray* SentPacketHistory::getPacket(uint16_t sequenceNumber) const { @@ -51,13 +41,6 @@ const QByteArray* SentPacketHistory::getPacket(uint16_t sequenceNumber) const { if (seqDiff < 0) { seqDiff += UINT16_RANGE; } - // if desired sequence number is too old to be found in the history, return null - if (seqDiff >= _numExistingPackets) { - return NULL; - } - int packetAt = _newestPacketAt - seqDiff; - if (packetAt < 0) { - packetAt += _sentPackets.size(); - } - return &_sentPackets.at(packetAt); + + return _sentPackets.get(seqDiff); } diff --git a/libraries/networking/src/SentPacketHistory.h b/libraries/networking/src/SentPacketHistory.h index 325f973f7e..96d10a63cf 100644 --- a/libraries/networking/src/SentPacketHistory.h +++ b/libraries/networking/src/SentPacketHistory.h @@ -13,7 +13,7 @@ #include #include -#include +#include "RingBufferHistory.h" #include "SequenceNumberStats.h" @@ -26,9 +26,7 @@ public: const QByteArray* getPacket(uint16_t sequenceNumber) const; private: - QVector _sentPackets; // circular buffer - int _newestPacketAt; - int _numExistingPackets; + RingBufferHistory _sentPackets; // circular buffer uint16_t _newestSequenceNumber; }; diff --git a/libraries/shared/src/MovingMinMaxAvg.h b/libraries/shared/src/MovingMinMaxAvg.h index 5263e2ab6c..abe43cdfac 100644 --- a/libraries/shared/src/MovingMinMaxAvg.h +++ b/libraries/shared/src/MovingMinMaxAvg.h @@ -12,9 +12,48 @@ #ifndef hifi_MovingMinMaxAvg_h #define hifi_MovingMinMaxAvg_h +#include "RingBufferHistory.h" + template class MovingMinMaxAvg { +private: + class Stats { + public: + Stats() + : _min(std::numeric_limits::max()), + _max(std::numeric_limits::min()), + _average(0.0) {} + + void updateWithSample(T sample, int& numSamplesInAverage) { + if (sample < _min) { + _min = sample; + } + if (sample > _max) { + _max = sample; + } + _average = _average * ((double)numSamplesInAverage / (numSamplesInAverage + 1)) + + (double)sample / (numSamplesInAverage + 1); + numSamplesInAverage++; + } + + void updateWithOtherStats(const Stats& other, int& numStatsInAverage) { + if (other._min < _min) { + _min = other._min; + } + if (other._max > _max) { + _max = other._max; + } + _average = _average * ((double)numStatsInAverage / (numStatsInAverage + 1)) + + other._average / (numStatsInAverage + 1); + numStatsInAverage++; + } + + T _min; + T _max; + double _average; + }; + public: // This class collects 3 stats (min, max, avg) over a moving window of samples. // The moving window contains _windowIntervals * _intervalLength samples. @@ -25,104 +64,51 @@ public: // new sample, instantiate this class with MovingMinMaxAvg(1, 100). MovingMinMaxAvg(int intervalLength, int windowIntervals) - : _min(std::numeric_limits::max()), - _max(std::numeric_limits::min()), - _average(0.0), - _samplesCollected(0), - _intervalLength(intervalLength), + : _intervalLength(intervalLength), _windowIntervals(windowIntervals), + _overallStats(), + _samplesCollected(0), + _windowStats(), _existingSamplesInCurrentInterval(0), - _existingIntervals(0), - _windowMin(std::numeric_limits::max()), - _windowMax(std::numeric_limits::min()), - _windowAverage(0.0), - _currentIntervalMin(std::numeric_limits::max()), - _currentIntervalMax(std::numeric_limits::min()), - _currentIntervalAverage(0.0), - _newestIntervalStatsAt(0), + _currentIntervalStats(), + _intervalStats(windowIntervals), _newStatsAvailable(false) - { - _intervalMins = new T[_windowIntervals]; - _intervalMaxes = new T[_windowIntervals]; - _intervalAverages = new double[_windowIntervals]; - } - - ~MovingMinMaxAvg() { - delete[] _intervalMins; - delete[] _intervalMaxes; - delete[] _intervalAverages; - } + {} void reset() { - _min = std::numeric_limits::max(); - _max = std::numeric_limits::min(); - _average = 0.0; + _overallStats = Stats(); _samplesCollected = 0; + _windowStats = Stats(); _existingSamplesInCurrentInterval = 0; - _existingIntervals = 0; - _windowMin = std::numeric_limits::max(); - _windowMax = std::numeric_limits::min(); - _windowAverage = 0.0; - _currentIntervalMin = std::numeric_limits::max(); - _currentIntervalMax = std::numeric_limits::min(); - _currentIntervalAverage = 0.0; - _newStatsAvailableFlag = false; + _currentIntervalStats = Stats(); + _intervalStats.clear(); + _newStatsAvailable = false; } void update(T newSample) { // update overall stats - if (newSample < _min) { - _min = newSample; - } - if (newSample > _max) { - _max = newSample; - } - updateAverage(_average, _samplesCollected, (double)newSample); + _overallStats.updateWithSample(newSample, _samplesCollected); // update the current interval stats - if (newSample < _currentIntervalMin) { - _currentIntervalMin = newSample; - } - if (newSample > _currentIntervalMax) { - _currentIntervalMax = newSample; - } - updateAverage(_currentIntervalAverage, _existingSamplesInCurrentInterval, (double)newSample); + _currentIntervalStats.updateWithSample(newSample, _existingSamplesInCurrentInterval); // if the current interval of samples is now full, record its stats into our past intervals' stats if (_existingSamplesInCurrentInterval == _intervalLength) { - // increment index of the newest interval's stats cyclically - _newestIntervalStatsAt = _newestIntervalStatsAt == _windowIntervals - 1 ? 0 : _newestIntervalStatsAt + 1; - // record current interval's stats, then reset them - _intervalMins[_newestIntervalStatsAt] = _currentIntervalMin; - _intervalMaxes[_newestIntervalStatsAt] = _currentIntervalMax; - _intervalAverages[_newestIntervalStatsAt] = _currentIntervalAverage; - _currentIntervalMin = std::numeric_limits::max(); - _currentIntervalMax = std::numeric_limits::min(); - _currentIntervalAverage = 0.0; + _intervalStats.insert(_currentIntervalStats); + _currentIntervalStats = Stats(); _existingSamplesInCurrentInterval = 0; - if (_existingIntervals < _windowIntervals) { - _existingIntervals++; - } - - // update the window's stats - int k = _newestIntervalStatsAt; - _windowMin = _intervalMins[k]; - _windowMax = _intervalMaxes[k]; - _windowAverage = _intervalAverages[k]; - int intervalsIncludedInWindowStats = 1; - while (intervalsIncludedInWindowStats < _existingIntervals) { - k = k == 0 ? _windowIntervals - 1 : k - 1; - if (_intervalMins[k] < _windowMin) { - _windowMin = _intervalMins[k]; - } - if (_intervalMaxes[k] > _windowMax) { - _windowMax = _intervalMaxes[k]; - } - updateAverage(_windowAverage, intervalsIncludedInWindowStats, _intervalAverages[k]); + // update the window's stats by combining the intervals' stats + RingBufferHistory::Iterator i = _intervalStats.begin(); + RingBufferHistory::Iterator end = _intervalStats.end(); + _windowStats = Stats(); + int intervalsIncludedInWindowStats = 0; + while (i != end) { + _windowStats.updateWithOtherStats(*i, intervalsIncludedInWindowStats); + i++; } _newStatsAvailable = true; @@ -133,48 +119,34 @@ public: bool getNewStatsAvailableFlag() const { return _newStatsAvailable; } void clearNewStatsAvailableFlag() { _newStatsAvailable = false; } - T getMin() const { return _min; } - T getMax() const { return _max; } - double getAverage() const { return _average; } - T getWindowMin() const { return _windowMin; } - T getWindowMax() const { return _windowMax; } - double getWindowAverage() const { return _windowAverage; } + T getMin() const { return _overallStats._min; } + T getMax() const { return _overallStats._max; } + double getAverage() const { return _overallStats._average; } + T getWindowMin() const { return _windowStats._min; } + T getWindowMax() const { return _windowStats._max; } + double getWindowAverage() const { return _windowStats._average; } + + private: - void updateAverage(double& average, int& numSamples, double newSample) { - // update some running average without overflowing it - average = average * ((double)numSamples / (numSamples + 1)) + newSample / (numSamples + 1); - numSamples++; - } - -private: - // these are min/max/avg stats for all samples collected. - T _min; - T _max; - double _average; - int _samplesCollected; - int _intervalLength; int _windowIntervals; - int _existingSamplesInCurrentInterval; - int _existingIntervals; + // these are min/max/avg stats for all samples collected. + Stats _overallStats; + int _samplesCollected; // these are the min/max/avg stats for the samples in the moving window - T _windowMin; - T _windowMax; - double _windowAverage; + Stats _windowStats; + int _existingSamplesInCurrentInterval; - T _currentIntervalMin; - T _currentIntervalMax; - double _currentIntervalAverage; + // these are the min/max/avg stats for the current interval + Stats _currentIntervalStats; - T* _intervalMins; - T* _intervalMaxes; - double* _intervalAverages; - int _newestIntervalStatsAt; + // these are stored stats for the past intervals in the window + RingBufferHistory _intervalStats; bool _newStatsAvailable; }; -#endif // hifi_OctalCode_h +#endif // hifi_MovingMinMaxAvg_h diff --git a/libraries/shared/src/RingBufferHistory.h b/libraries/shared/src/RingBufferHistory.h new file mode 100644 index 0000000000..094963ddc6 --- /dev/null +++ b/libraries/shared/src/RingBufferHistory.h @@ -0,0 +1,146 @@ +// +// RingBufferHistory.h +// libraries/shared/src +// +// Created by Yixin Wang on 7/9/2014 +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_RingBufferHistory_h +#define hifi_RingBufferHistory_h + +#include +#include + +template +class RingBufferHistory { + +public: + + RingBufferHistory(int capacity) + : _size(capacity + 1), + _capacity(capacity), + _newestEntryAt(0), + _numEntries(0) + { + _buffer = new T[_size]; + } + + RingBufferHistory(const RingBufferHistory& other) + : _size(other._size), + _capacity(other._capacity), + _newestEntryAt(other._newestEntryAt), + _numEntries(other._numEntries) + { + _buffer = new T[_size]; + memcpy(_buffer, other._buffer, _size*sizeof(T)); + } + + RingBufferHistory& operator= (const RingBufferHistory& rhs) { + _size = rhs._size; + _capacity = rhs._capacity; + _newestEntryAt = rhs._newestEntryAt; + _numEntries = rhs._numEntries; + delete[] _buffer; + _buffer = new T[_size]; + memcpy(_buffer, rhs._buffer, _size*sizeof(T)); + } + + ~RingBufferHistory() { + delete[] _buffer; + } + + void clear() { + _numEntries = 0; + } + + void insert(const T& entry) { + // increment newest entry index cyclically + _newestEntryAt = (_newestEntryAt == _size - 1) ? 0 : _newestEntryAt + 1; + + // insert new entry + _buffer[_newestEntryAt] = entry; + if (_numEntries < _capacity) { + _numEntries++; + } + } + + // 0 retrieves the most recent entry, _numEntries - 1 retrieves the oldest. + // returns NULL if entryAge not within [0, _numEntries-1] + const T* get(int entryAge) const { + if (!(entryAge >= 0 && entryAge < _numEntries)) { + return NULL; + } + int entryAt = _newestEntryAt - entryAge; + if (entryAt < 0) { + entryAt += _size; + } + return &_buffer[entryAt]; + } + + T* get(int entryAge) { + return const_cast((static_cast(this))->get(entryAge)); + } + + const T* getNewestEntry() const { + return &_buffer[_newestEntryAt]; + } + + T* getNewestEntry() { + return &_buffer[_newestEntryAt]; + } + + int getCapacity() const { return _capacity; } + int getNumEntries() const { return _numEntries; } + +private: + T* _buffer; + int _size; + int _capacity; + int _newestEntryAt; + int _numEntries; + + + +public: + class Iterator : public std::iterator < std::forward_iterator_tag, T > { + public: + Iterator(T* buffer, int size, T* at) : _buffer(buffer), _bufferEnd(buffer+size), _at(at) {} + + bool operator==(const Iterator& rhs) { return _at == rhs._at; } + bool operator!=(const Iterator& rhs) { return _at != rhs._at; } + T& operator*() { return *_at; } + T* operator->() { return _at; } + + Iterator& operator++() { + _at = (_at == _buffer) ? _bufferEnd - 1 : _at - 1; + return *this; + } + + Iterator& operator++(int) { + Iterator tmp(*this); + ++(*this); + return tmp; + } + + private: + T* const _buffer; + T* const _bufferEnd; + T* _at; + }; + + Iterator begin() { return Iterator(_buffer, _size, &_buffer[_newestEntryAt]); } + + Iterator end() { + int endAt = _newestEntryAt - _numEntries; + if (endAt < 0) { + endAt += _size; + } + return Iterator(_buffer, _size, &_buffer[endAt]); + } +}; + +#endif // hifi_RingBufferHistory_h diff --git a/tests/shared/src/MovingMinMaxAvgTests.cpp b/tests/shared/src/MovingMinMaxAvgTests.cpp index ae23248e5f..108db82e35 100644 --- a/tests/shared/src/MovingMinMaxAvgTests.cpp +++ b/tests/shared/src/MovingMinMaxAvgTests.cpp @@ -59,7 +59,7 @@ void MovingMinMaxAvgTests::runAllTests() { assert(stats.getMin() == min); assert(stats.getMax() == max); - assert(abs(stats.getAverage() / average - 1.0) < 0.000001); + assert(abs(stats.getAverage() / average - 1.0) < 0.000001 || abs(stats.getAverage() - average) < 0.000001); if ((i + 1) % INTERVAL_LENGTH == 0) { @@ -78,7 +78,7 @@ void MovingMinMaxAvgTests::runAllTests() { assert(stats.getWindowMin() == windowMin); assert(stats.getWindowMax() == windowMax); - assert(abs(stats.getAverage() / average - 1.0) < 0.000001); + assert(abs(stats.getAverage() / average - 1.0) < 0.000001 || abs(stats.getAverage() - average) < 0.000001); } else { assert(!stats.getNewStatsAvailableFlag()); From 3b7945fc14a83c9bf87a2484651be44d66175a50 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 9 Jul 2014 15:52:23 -0700 Subject: [PATCH 18/70] Gather and print out some stats after loading the data. --- .../src/metavoxels/MetavoxelServer.cpp | 25 ++++++++------- libraries/metavoxels/src/MetavoxelData.cpp | 31 +++++++++++++++++++ libraries/metavoxels/src/MetavoxelData.h | 4 +++ 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/assignment-client/src/metavoxels/MetavoxelServer.cpp b/assignment-client/src/metavoxels/MetavoxelServer.cpp index 928a693527..956458404a 100644 --- a/assignment-client/src/metavoxels/MetavoxelServer.cpp +++ b/assignment-client/src/metavoxels/MetavoxelServer.cpp @@ -226,19 +226,22 @@ void MetavoxelPersister::load() { if (!file.exists()) { return; } - QDebug debug = qDebug() << "Reading from" << SAVE_FILE << "..."; - file.open(QIODevice::ReadOnly); - QDataStream inStream(&file); - Bitstream in(inStream); MetavoxelData data; - try { - in >> data; - } catch (const BitstreamException& e) { - debug << "failed, " << e.getDescription(); - return; + { + QDebug debug = qDebug() << "Reading from" << SAVE_FILE << "..."; + file.open(QIODevice::ReadOnly); + QDataStream inStream(&file); + Bitstream in(inStream); + try { + in >> data; + } catch (const BitstreamException& e) { + debug << "failed, " << e.getDescription(); + return; + } + QMetaObject::invokeMethod(_server, "setData", Q_ARG(const MetavoxelData&, data)); + debug << "done."; } - QMetaObject::invokeMethod(_server, "setData", Q_ARG(const MetavoxelData&, data)); - debug << "done."; + data.dumpStats(); } void MetavoxelPersister::save(const MetavoxelData& data) { diff --git a/libraries/metavoxels/src/MetavoxelData.cpp b/libraries/metavoxels/src/MetavoxelData.cpp index 2d61ede796..4b142d0f5e 100644 --- a/libraries/metavoxels/src/MetavoxelData.cpp +++ b/libraries/metavoxels/src/MetavoxelData.cpp @@ -10,6 +10,7 @@ // #include +#include #include #include @@ -627,6 +628,25 @@ bool MetavoxelData::deepEquals(const MetavoxelData& other, const MetavoxelLOD& l return true; } +void MetavoxelData::dumpStats(QDebug debug) const { + QDebugStateSaver saver(debug); + debug.nospace() << "[size=" << _size << ", roots=["; + int totalInternal = 0, totalLeaves = 0; + for (QHash::const_iterator it = _roots.constBegin(); it != _roots.constEnd(); it++) { + if (it != _roots.constBegin()) { + debug << ", "; + } + debug << it.key()->getName() << " (" << it.key()->metaObject()->className() << "): "; + int internal = 0, leaves = 0; + it.value()->countDescendants(internal, leaves); + debug << internal << " internal, " << leaves << " leaves, " << (internal + leaves) << " total"; + totalInternal += internal; + totalLeaves += leaves; + } + debug << "], totalInternal=" << totalInternal << ", totalLeaves=" << totalLeaves << + ", grandTotal=" << (totalInternal + totalLeaves) << "]"; +} + bool MetavoxelData::operator==(const MetavoxelData& other) const { return _size == other._size && _roots == other._roots; } @@ -1061,6 +1081,17 @@ void MetavoxelNode::getSpanners(const AttributePointer& attribute, const glm::ve } } +void MetavoxelNode::countDescendants(int& internalNodes, int& leaves) const { + if (isLeaf()) { + leaves++; + return; + } + internalNodes++; + for (int i = 0; i < CHILD_COUNT; i++) { + _children[i]->countDescendants(internalNodes, leaves); + } +} + int MetavoxelVisitor::encodeOrder(int first, int second, int third, int fourth, int fifth, int sixth, int seventh, int eighth) { return first | (second << 3) | (third << 6) | (fourth << 9) | diff --git a/libraries/metavoxels/src/MetavoxelData.h b/libraries/metavoxels/src/MetavoxelData.h index 6a7ba33eb5..347d5a1f0b 100644 --- a/libraries/metavoxels/src/MetavoxelData.h +++ b/libraries/metavoxels/src/MetavoxelData.h @@ -125,6 +125,8 @@ public: /// shallow comparison). bool deepEquals(const MetavoxelData& other, const MetavoxelLOD& lod = MetavoxelLOD()) const; + void dumpStats(QDebug debug = QDebug(QtDebugMsg)) const; + bool operator==(const MetavoxelData& other) const; bool operator!=(const MetavoxelData& other) const; @@ -221,6 +223,8 @@ public: void getSpanners(const AttributePointer& attribute, const glm::vec3& minimum, float size, const MetavoxelLOD& lod, SharedObjectSet& results) const; + void countDescendants(int& internalNodes, int& leaves) const; + private: Q_DISABLE_COPY(MetavoxelNode) From 090f1fd8521389912d0cca030f5ce25b7cb895de Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 9 Jul 2014 16:09:06 -0700 Subject: [PATCH 19/70] Fixed bug with 3rd person UI ray picking --- interface/src/ui/ApplicationOverlay.cpp | 18 +++++++++++------- interface/src/ui/ApplicationOverlay.h | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 15dc18e74e..0d05de76c5 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -165,7 +165,8 @@ void ApplicationOverlay::displayOverlayTexture() { } void ApplicationOverlay::computeOculusPickRay(float x, float y, glm::vec3& direction) const { - glm::quat rot = Application::getInstance()->getAvatar()->getOrientation(); + MyAvatar* myAvatar = Application::getInstance()->getAvatar(); + glm::quat rot = myAvatar->getOrientation(); //invert y direction y = 1.0 - y; @@ -177,8 +178,11 @@ void ApplicationOverlay::computeOculusPickRay(float x, float y, glm::vec3& direc float dist = sqrt(x * x + y * y); float z = -sqrt(1.0f - dist * dist); + glm::vec3 relativePosition = myAvatar->getHead()->calculateAverageEyePosition() + + glm::normalize(rot * glm::vec3(x, y, z)); + //Rotate the UI pick ray by the avatar orientation - direction = glm::normalize(rot * glm::vec3(x, y, z)); + direction = glm::normalize(relativePosition - Application::getInstance()->getCamera()->getPosition()); } // Calculates the click location on the screen by taking into account any @@ -275,7 +279,7 @@ QPoint ApplicationOverlay::getOculusPalmClickLocation(PalmData *palm) const { glm::vec3 tip = OculusManager::getLaserPointerTipPosition(palm); glm::vec3 eyePos = myAvatar->getHead()->calculateAverageEyePosition(); glm::quat orientation = glm::inverse(myAvatar->getOrientation()); - glm::vec3 dir = orientation * glm::normalize(eyePos - tip); //direction of ray goes towards camera + glm::vec3 dir = orientation * glm::normalize(application->getCamera()->getPosition() - tip); //direction of ray goes towards camera glm::vec3 tipPos = orientation * (tip - eyePos); QPoint rv; @@ -299,6 +303,8 @@ QPoint ApplicationOverlay::getOculusPalmClickLocation(PalmData *palm) const { rv.setX(u * glWidget->width()); rv.setY(v * glWidget->height()); + + printf("%d %d\n", rv.x(), rv.y()); } } else { //if they did not click on the overlay, just set the coords to INT_MAX @@ -374,7 +380,7 @@ void ApplicationOverlay::displayOverlayTextureOculus(Camera& whichCamera) { renderTexturedHemisphere(); - renderPointersOculus(); + renderPointersOculus(whichCamera.getPosition()); glDepthMask(GL_TRUE); glBindTexture(GL_TEXTURE_2D, 0); @@ -652,7 +658,7 @@ void ApplicationOverlay::renderControllerPointers() { } } -void ApplicationOverlay::renderPointersOculus() { +void ApplicationOverlay::renderPointersOculus(const glm::vec3& eyePos) { Application* application = Application::getInstance(); QGLWidget* glWidget = application->getGLWidget(); @@ -668,8 +674,6 @@ void ApplicationOverlay::renderPointersOculus() { glMatrixMode(GL_MODELVIEW); MyAvatar* myAvatar = application->getAvatar(); - glm::vec3 eyePos = myAvatar->getHead()->calculateAverageEyePosition(); - //Controller Pointers for (int i = 0; i < myAvatar->getHand()->getNumPalms(); i++) { diff --git a/interface/src/ui/ApplicationOverlay.h b/interface/src/ui/ApplicationOverlay.h index 117dd02f3b..726d0a3769 100644 --- a/interface/src/ui/ApplicationOverlay.h +++ b/interface/src/ui/ApplicationOverlay.h @@ -49,7 +49,7 @@ private: void renderPointers(); void renderControllerPointers(); - void renderPointersOculus(); + void renderPointersOculus(const glm::vec3& eyePos); void renderMagnifier(int mouseX, int mouseY, float sizeMult, bool showBorder) const; void renderAudioMeter(); void renderStatsAndLogs(); From 8147f2cd91b5471e5565406bab94c2664ffbb5b0 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 9 Jul 2014 16:30:00 -0700 Subject: [PATCH 20/70] Fixed some magnification window bugs --- interface/src/ui/ApplicationOverlay.cpp | 44 ++++++++++++------------- interface/src/ui/ApplicationOverlay.h | 2 +- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 0d05de76c5..c064bb040a 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -191,7 +191,7 @@ void ApplicationOverlay::getClickLocation(int &x, int &y) const { int dx; int dy; const float xRange = MAGNIFY_WIDTH * MAGNIFY_MULT / 2.0f; - const float yRange = MAGNIFY_WIDTH * MAGNIFY_MULT / 2.0f; + const float yRange = MAGNIFY_HEIGHT * MAGNIFY_MULT / 2.0f; //Loop through all magnification windows for (int i = 0; i < NUMBER_OF_MAGNIFIERS; i++) { @@ -267,7 +267,7 @@ bool raySphereIntersect(const glm::vec3 &dir, const glm::vec3 &origin, float r, } -QPoint ApplicationOverlay::getOculusPalmClickLocation(PalmData *palm) const { +QPoint ApplicationOverlay::getOculusPalmClickLocation(const PalmData *palm) const { Application* application = Application::getInstance(); QGLWidget* glWidget = application->getGLWidget(); @@ -303,8 +303,6 @@ QPoint ApplicationOverlay::getOculusPalmClickLocation(PalmData *palm) const { rv.setX(u * glWidget->width()); rv.setY(v * glWidget->height()); - - printf("%d %d\n", rv.x(), rv.y()); } } else { //if they did not click on the overlay, just set the coords to INT_MAX @@ -600,13 +598,32 @@ void ApplicationOverlay::renderControllerPointers() { } else { bumperPressed[index] = false; } + + //if we have the oculus, we should make the cursor smaller since it will be + //magnified + if (OculusManager::isConnected()) { + QPoint point = getOculusPalmClickLocation(palmData); + + _mouseX[index] = point.x(); + _mouseY[index] = point.y(); + + //When button 2 is pressed we drag the mag window + if (isPressed[index]) { + _magActive[index] = true; + _magX[index] = point.x(); + _magY[index] = point.y(); + } + + // If oculus is enabled, we draw the crosshairs later + continue; + } // Get directon relative to avatar orientation glm::vec3 direction = glm::inverse(myAvatar->getOrientation()) * palmData->getFingerDirection(); // Get the angles, scaled between (-0.5,0.5) - float xAngle = (atan2(direction.z, direction.x) + M_PI_2) ; + float xAngle = (atan2(direction.z, direction.x) + M_PI_2); float yAngle = 0.5f - ((atan2(direction.z, direction.y) + M_PI_2)); // Get the pixel range over which the xAngle and yAngle are scaled @@ -621,24 +638,7 @@ void ApplicationOverlay::renderControllerPointers() { continue; } _reticleActive[index] = true; - - //if we have the oculus, we should make the cursor smaller since it will be - //magnified - if (OculusManager::isConnected()) { - - _mouseX[index] = mouseX; - _mouseY[index] = mouseY; - //When button 2 is pressed we drag the mag window - if (isPressed[index]) { - _magActive[index] = true; - _magX[index] = mouseX; - _magY[index] = mouseY; - } - - // If oculus is enabled, we draw the crosshairs later - continue; - } const float reticleSize = 40.0f; diff --git a/interface/src/ui/ApplicationOverlay.h b/interface/src/ui/ApplicationOverlay.h index 726d0a3769..fc282c2903 100644 --- a/interface/src/ui/ApplicationOverlay.h +++ b/interface/src/ui/ApplicationOverlay.h @@ -32,7 +32,7 @@ public: void displayOverlayTexture3DTV(Camera& whichCamera, float aspectRatio, float fov); void computeOculusPickRay(float x, float y, glm::vec3& direction) const; void getClickLocation(int &x, int &y) const; - QPoint getOculusPalmClickLocation(PalmData *palm) const; + QPoint getOculusPalmClickLocation(const PalmData *palm) const; // Getters QOpenGLFramebufferObject* getFramebufferObject(); From 56ff74dd56334b9300fba948c239843a6098b263 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 9 Jul 2014 16:39:05 -0700 Subject: [PATCH 21/70] Removed ApplicationOverlay::resize() --- interface/src/Application.cpp | 1 - interface/src/ui/ApplicationOverlay.cpp | 14 +++----------- interface/src/ui/ApplicationOverlay.h | 5 +---- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 42184bbdda..06ba3e442e 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -723,7 +723,6 @@ void Application::resizeGL(int width, int height) { resetCamerasOnResizeGL(_myCamera, width, height); glViewport(0, 0, width, height); // shouldn't this account for the menu??? - _applicationOverlay.resize(); updateProjectionMatrix(); glLoadIdentity(); diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index caf103c00c..9ec87483dc 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -1189,20 +1189,12 @@ void ApplicationOverlay::renderTexturedHemisphere() { } -void ApplicationOverlay::resize() { - if (_framebufferObject != NULL) { - delete _framebufferObject; - _framebufferObject = NULL; - } - // _framebufferObject is recreated at the correct size the next time it is accessed via getFramebufferObject(). -} - QOpenGLFramebufferObject* ApplicationOverlay::getFramebufferObject() { QSize size = Application::getInstance()->getGLWidget()->size(); if (!_framebufferObject || _framebufferObject->size() != size) { - if (_framebufferObject){ - delete _framebufferObject; - } + + delete _framebufferObject; + _framebufferObject = new QOpenGLFramebufferObject(size); glBindTexture(GL_TEXTURE_2D, _framebufferObject->texture()); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); diff --git a/interface/src/ui/ApplicationOverlay.h b/interface/src/ui/ApplicationOverlay.h index a956858051..a64e26ed67 100644 --- a/interface/src/ui/ApplicationOverlay.h +++ b/interface/src/ui/ApplicationOverlay.h @@ -32,11 +32,8 @@ public: void displayOverlayTexture3DTV(Camera& whichCamera, float aspectRatio, float fov); void computeOculusPickRay(float x, float y, glm::vec3& direction) const; void getClickLocation(int &x, int &y) const; -<<<<<<< HEAD QPoint getOculusPalmClickLocation(const PalmData *palm) const; -======= - void resize(); ->>>>>>> af6704a83cec1d69b3dc3e1145238919fcb82933 + // Getters QOpenGLFramebufferObject* getFramebufferObject(); From 004db004a6318d3762ffe8d394529ab65e0eb304 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 9 Jul 2014 17:03:09 -0700 Subject: [PATCH 22/70] Dynamically resize lasers so they always end on UI --- interface/src/devices/OculusManager.cpp | 12 ++++++++++-- interface/src/ui/ApplicationOverlay.cpp | 18 ++++++++++++++++++ interface/src/ui/ApplicationOverlay.h | 1 + 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index 2e6d4cb9a2..b3c478e5e9 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -490,8 +490,16 @@ void OculusManager::renderLaserPointer() { glm::vec3 OculusManager::getLaserPointerTipPosition(const PalmData* palm) { #ifdef HAVE_LIBOVR - const float PALM_TIP_ROD_LENGTH_MULT = 4.0f; - return palm->getPosition() + (palm->getTipPosition() - palm->getPosition()) * PALM_TIP_ROD_LENGTH_MULT; + const ApplicationOverlay& applicationOverlay = Application::getInstance()->getApplicationOverlay(); + const float PALM_TIP_ROD_LENGTH_MULT = 40.0f; + + glm::vec3 direction = glm::normalize(palm->getTipPosition() - palm->getPosition()); + glm::vec3 result; + if (applicationOverlay.calculateRayUICollisionPoint(palm->getPosition(), direction, result)) { + return result; + } + + return palm->getPosition(); #endif return glm::vec3(0.0f); } diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 9ec87483dc..6d8af6641f 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -311,6 +311,24 @@ QPoint ApplicationOverlay::getOculusPalmClickLocation(const PalmData *palm) cons return rv; } +bool ApplicationOverlay::calculateRayUICollisionPoint(const glm::vec3& position, const glm::vec3& direction, glm::vec3& result) const { + Application* application = Application::getInstance(); + MyAvatar* myAvatar = application->getAvatar(); + + glm::quat orientation = myAvatar->getOrientation(); + + glm::vec3 relativePosition = orientation * (position - myAvatar->getHead()->calculateAverageEyePosition()); + glm::vec3 relativeDirection = orientation * direction; + + float t; + if (raySphereIntersect(relativeDirection, relativePosition, 1, &t)){ + result = position + direction * t; + return true; + } + + return false; +} + // Draws the FBO texture for Oculus rift. void ApplicationOverlay::displayOverlayTextureOculus(Camera& whichCamera) { diff --git a/interface/src/ui/ApplicationOverlay.h b/interface/src/ui/ApplicationOverlay.h index a64e26ed67..e355665b95 100644 --- a/interface/src/ui/ApplicationOverlay.h +++ b/interface/src/ui/ApplicationOverlay.h @@ -33,6 +33,7 @@ public: void computeOculusPickRay(float x, float y, glm::vec3& direction) const; void getClickLocation(int &x, int &y) const; QPoint getOculusPalmClickLocation(const PalmData *palm) const; + bool calculateRayUICollisionPoint(const glm::vec3& position, const glm::vec3& direction, glm::vec3& result) const; // Getters From dbd7ec18b0355fb3c157e0e7c4812830c781e025 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 9 Jul 2014 17:15:05 -0700 Subject: [PATCH 23/70] Show some stats on the client (node counts under LOD, reliable delta download progress). --- interface/src/ui/Stats.cpp | 36 ++++++++++++++++++- .../metavoxels/src/DatagramSequencer.cpp | 11 ++++++ libraries/metavoxels/src/DatagramSequencer.h | 4 +++ .../metavoxels/src/MetavoxelClientManager.cpp | 4 +++ .../metavoxels/src/MetavoxelClientManager.h | 2 ++ libraries/metavoxels/src/MetavoxelData.cpp | 23 ++++++++---- libraries/metavoxels/src/MetavoxelData.h | 6 +++- 7 files changed, 77 insertions(+), 9 deletions(-) diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 3de21b449b..b0cbea14a9 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -377,7 +377,7 @@ void Stats::display( MyAvatar* myAvatar = Application::getInstance()->getAvatar(); glm::vec3 avatarPos = myAvatar->getPosition(); - lines = _expanded ? 5 : 3; + lines = _expanded ? 8 : 3; drawBackground(backgroundColor, horizontalOffset, 0, _geoStatsWidth, lines * STATS_PELS_PER_LINE + 10); horizontalOffset += 5; @@ -419,6 +419,40 @@ void Stats::display( verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, downloads.str().c_str(), color); + + + int internal = 0, leaves = 0; + int received = 0, total = 0; + foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { + if (node->getType() == NodeType::MetavoxelServer) { + QMutexLocker locker(&node->getMutex()); + MetavoxelClient* client = static_cast(node->getLinkedData()); + if (client) { + client->getData().countNodes(internal, leaves, Application::getInstance()->getMetavoxels()->getLOD()); + int clientReceived = 0, clientTotal = 0; + if (client->getReliableDeltaProgress(clientReceived, clientTotal)) { + received += clientReceived; + total += clientTotal; + } + } + } + } + stringstream nodes; + nodes << "Metavoxels: " << (internal + leaves); + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, nodes.str().c_str(), color); + + stringstream nodeTypes; + nodeTypes << "Internal: " << internal << " Leaves: " << leaves; + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, nodeTypes.str().c_str(), color); + + if (total > 0) { + stringstream reliableDelta; + reliableDelta << "Reliable Delta: " << (received * 100 / total) << "%"; + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, reliableDelta.str().c_str(), color); + } } verticalOffset = 0; diff --git a/libraries/metavoxels/src/DatagramSequencer.cpp b/libraries/metavoxels/src/DatagramSequencer.cpp index 536cfc9dfb..8c58233eb9 100644 --- a/libraries/metavoxels/src/DatagramSequencer.cpp +++ b/libraries/metavoxels/src/DatagramSequencer.cpp @@ -692,6 +692,17 @@ void ReliableChannel::sendMessage(const QVariant& message) { endMessage(); } +bool ReliableChannel::getMessageReceiveProgress(int& received, int& total) const { + if (!_messagesEnabled || _buffer.bytesAvailable() < (int)sizeof(quint32)) { + return false; + } + quint32 length; + _buffer.readBytes(_buffer.pos(), sizeof(quint32), (char*)&length); + total = length; + received = _buffer.bytesAvailable(); + return true; +} + void ReliableChannel::sendClearSharedObjectMessage(int id) { ClearSharedObjectMessage message = { id }; sendMessage(QVariant::fromValue(message)); diff --git a/libraries/metavoxels/src/DatagramSequencer.h b/libraries/metavoxels/src/DatagramSequencer.h index b6dce464f7..998e196a05 100644 --- a/libraries/metavoxels/src/DatagramSequencer.h +++ b/libraries/metavoxels/src/DatagramSequencer.h @@ -376,6 +376,10 @@ public: /// writes the message to the bitstream, then calls endMessage). void sendMessage(const QVariant& message); + /// Determines the number of bytes downloaded towards the currently pending message. + /// \return true if there is a message pending, in which case the received and total arguments will be set + bool getMessageReceiveProgress(int& received, int& total) const; + signals: /// Fired when a framed message has been received on this channel. diff --git a/libraries/metavoxels/src/MetavoxelClientManager.cpp b/libraries/metavoxels/src/MetavoxelClientManager.cpp index f3ea1ae8c5..94d9116794 100644 --- a/libraries/metavoxels/src/MetavoxelClientManager.cpp +++ b/libraries/metavoxels/src/MetavoxelClientManager.cpp @@ -94,6 +94,10 @@ MetavoxelClient::MetavoxelClient(const SharedNodePointer& node, MetavoxelClientM SIGNAL(receivedMessage(const QVariant&, Bitstream&)), SLOT(handleMessage(const QVariant&, Bitstream&))); } +bool MetavoxelClient::getReliableDeltaProgress(int& received, int& total) const { + return _reliableDeltaChannel && _reliableDeltaChannel->getMessageReceiveProgress(received, total); +} + void MetavoxelClient::guide(MetavoxelVisitor& visitor) { visitor.setLOD(_manager->getLOD()); _data.guide(visitor); diff --git a/libraries/metavoxels/src/MetavoxelClientManager.h b/libraries/metavoxels/src/MetavoxelClientManager.h index ad6c86c8fc..809718aa01 100644 --- a/libraries/metavoxels/src/MetavoxelClientManager.h +++ b/libraries/metavoxels/src/MetavoxelClientManager.h @@ -53,6 +53,8 @@ public: MetavoxelData& getData() { return _data; } + bool getReliableDeltaProgress(int& received, int& total) const; + void guide(MetavoxelVisitor& visitor); void applyEdit(const MetavoxelEditMessage& edit, bool reliable = false); diff --git a/libraries/metavoxels/src/MetavoxelData.cpp b/libraries/metavoxels/src/MetavoxelData.cpp index 4b142d0f5e..1362731a8a 100644 --- a/libraries/metavoxels/src/MetavoxelData.cpp +++ b/libraries/metavoxels/src/MetavoxelData.cpp @@ -628,17 +628,25 @@ bool MetavoxelData::deepEquals(const MetavoxelData& other, const MetavoxelLOD& l return true; } +void MetavoxelData::countNodes(int& internal, int& leaves, const MetavoxelLOD& lod) const { + glm::vec3 minimum = getMinimum(); + for (QHash::const_iterator it = _roots.constBegin(); it != _roots.constEnd(); it++) { + it.value()->countNodes(it.key(), minimum, _size, lod, internal, leaves); + } +} + void MetavoxelData::dumpStats(QDebug debug) const { QDebugStateSaver saver(debug); debug.nospace() << "[size=" << _size << ", roots=["; int totalInternal = 0, totalLeaves = 0; + glm::vec3 minimum = getMinimum(); for (QHash::const_iterator it = _roots.constBegin(); it != _roots.constEnd(); it++) { if (it != _roots.constBegin()) { debug << ", "; } debug << it.key()->getName() << " (" << it.key()->metaObject()->className() << "): "; int internal = 0, leaves = 0; - it.value()->countDescendants(internal, leaves); + it.value()->countNodes(it.key(), minimum, _size, MetavoxelLOD(), internal, leaves); debug << internal << " internal, " << leaves << " leaves, " << (internal + leaves) << " total"; totalInternal += internal; totalLeaves += leaves; @@ -1076,19 +1084,20 @@ void MetavoxelNode::getSpanners(const AttributePointer& attribute, const glm::ve } float nextSize = size * 0.5f; for (int i = 0; i < CHILD_COUNT; i++) { - glm::vec3 nextMinimum = getNextMinimum(minimum, nextSize, i); - _children[i]->getSpanners(attribute, nextMinimum, nextSize, lod, results); + _children[i]->getSpanners(attribute, getNextMinimum(minimum, nextSize, i), nextSize, lod, results); } } -void MetavoxelNode::countDescendants(int& internalNodes, int& leaves) const { - if (isLeaf()) { +void MetavoxelNode::countNodes(const AttributePointer& attribute, const glm::vec3& minimum, + float size, const MetavoxelLOD& lod, int& internal, int& leaves) const { + if (isLeaf() || !lod.shouldSubdivide(minimum, size, attribute->getLODThresholdMultiplier())) { leaves++; return; } - internalNodes++; + internal++; + float nextSize = size * 0.5f; for (int i = 0; i < CHILD_COUNT; i++) { - _children[i]->countDescendants(internalNodes, leaves); + _children[i]->countNodes(attribute, getNextMinimum(minimum, nextSize, i), nextSize, lod, internal, leaves); } } diff --git a/libraries/metavoxels/src/MetavoxelData.h b/libraries/metavoxels/src/MetavoxelData.h index 347d5a1f0b..f558bf8e80 100644 --- a/libraries/metavoxels/src/MetavoxelData.h +++ b/libraries/metavoxels/src/MetavoxelData.h @@ -125,6 +125,9 @@ public: /// shallow comparison). bool deepEquals(const MetavoxelData& other, const MetavoxelLOD& lod = MetavoxelLOD()) const; + /// Counts the nodes in the data. + void countNodes(int& internalNodes, int& leaves, const MetavoxelLOD& lod = MetavoxelLOD()) const; + void dumpStats(QDebug debug = QDebug(QtDebugMsg)) const; bool operator==(const MetavoxelData& other) const; @@ -223,7 +226,8 @@ public: void getSpanners(const AttributePointer& attribute, const glm::vec3& minimum, float size, const MetavoxelLOD& lod, SharedObjectSet& results) const; - void countDescendants(int& internalNodes, int& leaves) const; + void countNodes(const AttributePointer& attribute, const glm::vec3& minimum, + float size, const MetavoxelLOD& lod, int& internalNodes, int& leaves) const; private: Q_DISABLE_COPY(MetavoxelNode) From a785ceb3da8726236875df11ed10320845c06fee Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 9 Jul 2014 17:33:41 -0700 Subject: [PATCH 24/70] Restored UI fade animation --- interface/src/Application.cpp | 4 ---- interface/src/devices/OculusManager.cpp | 11 +++++------ interface/src/devices/OculusManager.h | 2 +- interface/src/ui/ApplicationOverlay.cpp | 3 +++ 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 06ba3e442e..9842144bff 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2786,10 +2786,6 @@ void Application::displaySide(Camera& whichCamera, bool selfAvatarOnly) { { PerformanceTimer perfTimer("avatars"); - if (OculusManager::isConnected()) { - OculusManager::renderLaserPointer(); - } - _avatarManager.renderAvatars(mirrorMode ? Avatar::MIRROR_RENDER_MODE : Avatar::NORMAL_RENDER_MODE, selfAvatarOnly); } diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index b3c478e5e9..574aa3c6f6 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -269,8 +269,7 @@ void OculusManager::display(const glm::quat &bodyOrientation, const glm::vec3 &p // We only need to render the overlays to a texture once, then we just render the texture on the hemisphere // PrioVR will only work if renderOverlay is called, calibration is connected to Application::renderingOverlay() applicationOverlay.renderOverlay(true); - const bool displayOverlays = Menu::getInstance()->isOptionChecked(MenuOption::UserInterface); - + //Bind our framebuffer object. If we are rendering the glow effect, we let the glow effect shader take care of it if (Menu::getInstance()->isOptionChecked(MenuOption::EnableGlowEffect)) { Application::getInstance()->getGlowEffect()->prepare(); @@ -325,9 +324,7 @@ void OculusManager::display(const glm::quat &bodyOrientation, const glm::vec3 &p Application::getInstance()->displaySide(*_camera); - if (displayOverlays) { - applicationOverlay.displayOverlayTextureOculus(*_camera); - } + applicationOverlay.displayOverlayTextureOculus(*_camera); } //Wait till time-warp to reduce latency @@ -467,7 +464,8 @@ QSize OculusManager::getRenderTargetSize() { #endif } -void OculusManager::renderLaserPointer() { +//Renders sixense laser pointers for UI selection in the oculus +void OculusManager::renderLaserPointers() { #ifdef HAVE_LIBOVR const float PALM_TIP_ROD_RADIUS = 0.002f; @@ -488,6 +486,7 @@ void OculusManager::renderLaserPointer() { #endif } +//Gets the tip position for the laser pointer glm::vec3 OculusManager::getLaserPointerTipPosition(const PalmData* palm) { #ifdef HAVE_LIBOVR const ApplicationOverlay& applicationOverlay = Application::getInstance()->getApplicationOverlay(); diff --git a/interface/src/devices/OculusManager.h b/interface/src/devices/OculusManager.h index 73ce6ef2d3..871e5d649a 100644 --- a/interface/src/devices/OculusManager.h +++ b/interface/src/devices/OculusManager.h @@ -44,7 +44,7 @@ public: static QSize getRenderTargetSize(); /// Renders a laser pointer for UI picking - static void renderLaserPointer(); + static void renderLaserPointers(); static glm::vec3 getLaserPointerTipPosition(const PalmData* palm); private: diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 6d8af6641f..6324ee03b0 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -340,6 +340,9 @@ void ApplicationOverlay::displayOverlayTextureOculus(Camera& whichCamera) { MyAvatar* myAvatar = application->getAvatar(); + //Render the sixense lasers + OculusManager::renderLaserPointers(); + glActiveTexture(GL_TEXTURE0); glEnable(GL_BLEND); From 01f10024aed637881a17e517181cfe58862b764c Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 9 Jul 2014 18:13:33 -0700 Subject: [PATCH 25/70] all stats added, needs testing; created PacketStreamStats struct --- .../src/audio/AudioMixerClientData.cpp | 26 +++--- interface/src/Audio.cpp | 76 +++++++++++++-- interface/src/Audio.h | 34 ++++++- interface/src/ui/Stats.cpp | 92 +++++++++++++------ libraries/audio/src/AudioRingBuffer.h | 2 + libraries/audio/src/AudioStreamStats.h | 17 +--- .../networking/src/SequenceNumberStats.cpp | 34 +++---- .../networking/src/SequenceNumberStats.h | 45 ++++++--- libraries/shared/src/RingBufferHistory.h | 5 +- 9 files changed, 223 insertions(+), 108 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 9e27103bd1..f8d97dc44e 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -188,13 +188,13 @@ AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const Positio streamStats._ringBufferOverflowCount = ringBuffer->getOverflowCount(); streamStats._ringBufferSilentFramesDropped = ringBuffer->getSilentFramesDropped(); - streamStats._packetsReceived = streamSequenceNumberStats->getNumReceived(); - streamStats._packetsUnreasonable = streamSequenceNumberStats->getNumUnreasonable(); - streamStats._packetsEarly = streamSequenceNumberStats->getNumEarly(); - streamStats._packetsLate = streamSequenceNumberStats->getNumLate(); - streamStats._packetsLost = streamSequenceNumberStats->getNumLost(); - streamStats._packetsRecovered = streamSequenceNumberStats->getNumRecovered(); - streamStats._packetsDuplicate = streamSequenceNumberStats->getNumDuplicate(); + streamStats._packetStreamStats._numReceived = streamSequenceNumberStats->getNumReceived(); + streamStats._packetStreamStats._numUnreasonable = streamSequenceNumberStats->getNumUnreasonable(); + streamStats._packetStreamStats._numEarly = streamSequenceNumberStats->getNumEarly(); + streamStats._packetStreamStats._numLate = streamSequenceNumberStats->getNumLate(); + streamStats._packetStreamStats._numLost = streamSequenceNumberStats->getNumLost(); + streamStats._packetStreamStats._numRecovered = streamSequenceNumberStats->getNumRecovered(); + streamStats._packetStreamStats._numDuplicate = streamSequenceNumberStats->getNumDuplicate(); return streamStats; } @@ -261,9 +261,9 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { + " not mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + " overflows:" + QString::number(streamStats._ringBufferOverflowCount) + " silents dropped:" + QString::number(streamStats._ringBufferSilentFramesDropped) - + " early:" + QString::number(streamStats._packetsEarly) - + " late:" + QString::number(streamStats._packetsLate) - + " lost:" + QString::number(streamStats._packetsLost) + + " early:" + QString::number(streamStats._packetStreamStats._numEarly) + + " late:" + QString::number(streamStats._packetStreamStats._numLate) + + " lost:" + QString::number(streamStats._packetStreamStats._numLost) + " min gap:" + QString::number(streamStats._timeGapMin) + " max gap:" + QString::number(streamStats._timeGapMax) + " avg gap:" + QString::number(streamStats._timeGapAverage, 'g', 2) @@ -284,9 +284,9 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { + " not mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + " overflows:" + QString::number(streamStats._ringBufferOverflowCount) + " silents dropped:" + QString::number(streamStats._ringBufferSilentFramesDropped) - + " early:" + QString::number(streamStats._packetsEarly) - + " late:" + QString::number(streamStats._packetsLate) - + " lost:" + QString::number(streamStats._packetsLost) + + " early:" + QString::number(streamStats._packetStreamStats._numEarly) + + " late:" + QString::number(streamStats._packetStreamStats._numLate) + + " lost:" + QString::number(streamStats._packetStreamStats._numLost) + " min gap:" + QString::number(streamStats._timeGapMin) + " max gap:" + QString::number(streamStats._timeGapMax) + " avg gap:" + QString::number(streamStats._timeGapAverage, 'g', 2) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index cc455c5544..2073659e89 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -48,6 +48,11 @@ static const float AUDIO_CALLBACK_MSECS = (float) NETWORK_BUFFER_LENGTH_SAMPLES_ static const int NUMBER_OF_NOISE_SAMPLE_FRAMES = 300; +static const int AUDIO_STREAM_STATS_HISTORY_SIZE = 30; + +const int TIME_GAPS_STATS_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS; +const int TIME_GAP_STATS_WINDOW_INTERVALS = 30; + // Mute icon configration static const int MUTE_ICON_SIZE = 24; @@ -103,8 +108,13 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _scopeInput(0), _scopeOutputLeft(0), _scopeOutputRight(0), - _audioMixerAvatarStreamStats(), - _outgoingAvatarAudioSequenceNumber(0) + _audioMixerAvatarStreamAudioStats(), + _audioMixerAvatarStreamPacketStatsHistory(AUDIO_STREAM_STATS_HISTORY_SIZE), + _outgoingAvatarAudioSequenceNumber(0), + _incomingStreamPacketStatsHistory(AUDIO_STREAM_STATS_HISTORY_SIZE), + _interframeTimeGapStats(TIME_GAPS_STATS_INTERVAL_SAMPLES, TIME_GAP_STATS_WINDOW_INTERVALS), + _starveCount(0), + _consecutiveNotMixedCount(0) { // clear the array of locally injected samples memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); @@ -120,9 +130,20 @@ void Audio::init(QGLWidget *parent) { void Audio::reset() { _ringBuffer.reset(); + + _starveCount = 0; + _consecutiveNotMixedCount = 0; + + _audioMixerAvatarStreamAudioStats = AudioStreamStats(); + _audioMixerInjectedStreamAudioStatsMap.clear(); + + _audioMixerAvatarStreamPacketStatsHistory.clear(); + _audioMixerInjectedStreamPacketStatsHistoryMap.clear(); + _outgoingAvatarAudioSequenceNumber = 0; - _audioMixerInjectedStreamStatsMap.clear(); _incomingMixedAudioSequenceNumberStats.reset(); + + _incomingStreamPacketStatsHistory.clear(); } QAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName) { @@ -672,7 +693,7 @@ void Audio::handleAudioInput() { // memcpy our orientation memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation)); currentPacketPtr += sizeof(headOrientation); - +if (randFloat() < 0.95f) nodeList->writeDatagram(audioDataPacket, numAudioBytes + leadingBytes, audioMixer); _outgoingAvatarAudioSequenceNumber++; @@ -689,7 +710,9 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { _totalPacketsReceived++; - double timeDiff = (double)_timeSinceLastReceived.nsecsElapsed() / 1000000.0; // ns to ms + double timeDiff = (double)_timeSinceLastReceived.nsecsElapsed() / 1000.0; // ns to us + _interframeTimeGapStats.update((quint64)timeDiff); + timeDiff /= USECS_PER_MSEC; // us to ms _timeSinceLastReceived.start(); // Discard first few received packets for computing jitter (often they pile up on start) @@ -726,7 +749,8 @@ void Audio::parseAudioStreamStatsPacket(const QByteArray& packet) { quint8 appendFlag = *(reinterpret_cast(dataAt)); dataAt += sizeof(quint8); if (!appendFlag) { - _audioMixerInjectedStreamStatsMap.clear(); + _audioMixerInjectedStreamAudioStatsMap.clear(); + _audioMixerInjectedStreamPacketStatsHistoryMap.clear(); } // parse the number of stream stats structs to follow @@ -740,11 +764,21 @@ void Audio::parseAudioStreamStatsPacket(const QByteArray& packet) { dataAt += sizeof(AudioStreamStats); if (streamStats._streamType == PositionalAudioRingBuffer::Microphone) { - _audioMixerAvatarStreamStats = streamStats; + _audioMixerAvatarStreamAudioStats = streamStats; + _audioMixerAvatarStreamPacketStatsHistory.insert(streamStats._packetStreamStats); } else { - _audioMixerInjectedStreamStatsMap[streamStats._streamIdentifier] = streamStats; + if (!_audioMixerInjectedStreamAudioStatsMap.contains(streamStats._streamIdentifier)) { + _audioMixerInjectedStreamPacketStatsHistoryMap.insert(streamStats._streamIdentifier, + RingBufferHistory(AUDIO_STREAM_STATS_HISTORY_SIZE)); + } + _audioMixerInjectedStreamAudioStatsMap[streamStats._streamIdentifier] = streamStats; + _audioMixerInjectedStreamPacketStatsHistoryMap[streamStats._streamIdentifier].insert(streamStats._packetStreamStats); } } + + // when an audio stream stats packet is received, also record the current packets received and lost + // in the packet loss stats history + _incomingStreamPacketStatsHistory.insert(_incomingMixedAudioSequenceNumberStats.getStats()); } // NOTE: numSamples is the total number of single channel samples, since callers will always call this with stereo @@ -867,6 +901,9 @@ void Audio::processReceivedAudio(const QByteArray& audioByteArray) { //qDebug() << "Audio output just starved."; _ringBuffer.setIsStarved(true); _numFramesDisplayStarve = 10; + + _starveCount++; + _consecutiveNotMixedCount = 0; } int numNetworkOutputSamples; @@ -886,6 +923,7 @@ void Audio::processReceivedAudio(const QByteArray& audioByteArray) { if (!_ringBuffer.isNotStarvedOrHasMinimumSamples(numSamplesNeededToStartPlayback)) { // We are still waiting for enough samples to begin playback // qDebug() << numNetworkOutputSamples << " samples so far, waiting for " << numSamplesNeededToStartPlayback; + _consecutiveNotMixedCount++; } else { int numDeviceOutputSamples = numNetworkOutputSamples / networkOutputToOutputRatio; @@ -1515,3 +1553,25 @@ int Audio::calculateNumberOfFrameSamples(int numBytes) { int frameSamples = (int)(numBytes * CALLBACK_ACCELERATOR_RATIO + 0.5f) / sizeof(int16_t); return frameSamples; } + +void Audio::calculatePacketLossRate(const RingBufferHistory& statsHistory, + float& overallLossRate, float& windowLossRate) const { + + int numHistoryEntries = statsHistory.getNumEntries(); + if (numHistoryEntries == 0) { + overallLossRate = 0.0f; + windowLossRate = 0.0f; + } else { + const PacketStreamStats& newestStats = *statsHistory.getNewestEntry(); + overallLossRate = (float)newestStats._numLost / newestStats._numReceived; + + if (numHistoryEntries == 1) { + windowLossRate = overallLossRate; + } else { + int age = std::min(numHistoryEntries-1, AUDIO_STREAM_STATS_HISTORY_SIZE-1); + const PacketStreamStats& oldestStats = *statsHistory.get(age); + windowLossRate = (float)(newestStats._numLost - oldestStats._numLost) + / (newestStats._numReceived - oldestStats._numReceived); + } + } +} diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 9f04e5cb03..e6e06838d3 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -17,6 +17,8 @@ #include "InterfaceConfig.h" #include "AudioStreamStats.h" +#include "RingBufferHistory.h" +#include "MovingMinMaxAvg.h" #include #include @@ -107,8 +109,22 @@ public slots: float getInputVolume() const { return (_audioInput) ? _audioInput->volume() : 0.0f; } void setInputVolume(float volume) { if (_audioInput) _audioInput->setVolume(volume); } - const AudioStreamStats& getAudioMixerAvatarStreamStats() const { return _audioMixerAvatarStreamStats; } - const QHash& getAudioMixerInjectedStreamStatsMap() const { return _audioMixerInjectedStreamStatsMap; } + const AudioRingBuffer& getDownstreamRingBuffer() const { return _ringBuffer; } + + int getDesiredJitterBufferFrames() const { return _jitterBufferSamples / _ringBuffer.getNumFrameSamples(); } + + int getStarveCount() const { return _starveCount; } + int getConsecutiveNotMixedCount() const { return _consecutiveNotMixedCount; } + + const AudioStreamStats& getAudioMixerAvatarStreamAudioStats() const { return _audioMixerAvatarStreamAudioStats; } + const QHash& getAudioMixerInjectedStreamAudioStatsMap() const { return _audioMixerInjectedStreamAudioStatsMap; } + const RingBufferHistory& getAudioMixerAvatarStreamPacketStatsHistory() const { return _audioMixerAvatarStreamPacketStatsHistory; } + const QHash >& getAudioMixerInjectedStreamPacketStatsHistoryMap() const {return _audioMixerInjectedStreamPacketStatsHistoryMap; } + const RingBufferHistory& getIncomingStreamPacketStatsHistory() const { return _incomingStreamPacketStatsHistory; } + const MovingMinMaxAvg& getInterframeTimeGapStats() const { return _interframeTimeGapStats; } + + void calculatePacketLossRate(const RingBufferHistory& statsHistory, + float& overallLossRate, float& windowLossRate) const; signals: bool muteToggled(); @@ -241,11 +257,21 @@ private: QByteArray* _scopeOutputLeft; QByteArray* _scopeOutputRight; - AudioStreamStats _audioMixerAvatarStreamStats; - QHash _audioMixerInjectedStreamStatsMap; + int _starveCount; + int _consecutiveNotMixedCount; + + AudioStreamStats _audioMixerAvatarStreamAudioStats; + QHash _audioMixerInjectedStreamAudioStatsMap; + + RingBufferHistory _audioMixerAvatarStreamPacketStatsHistory; + QHash > _audioMixerInjectedStreamPacketStatsHistoryMap; quint16 _outgoingAvatarAudioSequenceNumber; SequenceNumberStats _incomingMixedAudioSequenceNumberStats; + + RingBufferHistory _incomingStreamPacketStatsHistory; + + MovingMinMaxAvg _interframeTimeGapStats; }; diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index e86e656f2e..01032aebc8 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -288,15 +288,12 @@ void Stats::display( Audio* audio = Application::getInstance()->getAudio(); - const AudioStreamStats& audioMixerAvatarStreamStats = audio->getAudioMixerAvatarStreamStats(); - const QHash& audioMixerInjectedStreamStatsMap = audio->getAudioMixerInjectedStreamStatsMap(); + const QHash& audioMixerInjectedStreamAudioStatsMap = audio->getAudioMixerInjectedStreamAudioStatsMap(); - lines = _expanded ? 12 + (audioMixerInjectedStreamStatsMap.size() + 1) * 3: 3; + lines = _expanded ? 11 + (audioMixerInjectedStreamAudioStatsMap.size() + 2) * 3 : 3; drawBackground(backgroundColor, horizontalOffset, 0, _pingStatsWidth, lines * STATS_PELS_PER_LINE + 10); horizontalOffset += 5; - - char audioJitter[30]; sprintf(audioJitter, "Buffer msecs %.1f", @@ -328,7 +325,7 @@ void Stats::display( drawText(horizontalOffset, verticalOffset, scale, rotation, font, voxelMaxPing, color); char audioMixerStatsLabelString[] = "AudioMixer stats:"; - char streamStatsFormatLabelString[] = "early/late/lost"; + char streamStatsFormatLabelString[] = "lost%/30s_lost%"; char streamStatsFormatLabelString2[] = "avail/currJ/desiredJ"; char streamStatsFormatLabelString3[] = "gaps: min/max/avg, starv/ovfl"; char streamStatsFormatLabelString4[] = "30s gaps: (same), notmix/sdrop"; @@ -349,61 +346,98 @@ void Stats::display( verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamLabelString, color); - const SequenceNumberStats& downstreamAudioSequenceNumberStats = audio->getIncomingMixedAudioSequenceNumberStats(); - char downstreamAudioStatsString[30]; + /* const SequenceNumberStats& downstreamAudioSequenceNumberStats = audio->getIncomingMixedAudioSequenceNumberStats(); + sprintf(downstreamAudioStatsString, " mix: %d/%d/%d, %d", downstreamAudioSequenceNumberStats.getNumEarly(), downstreamAudioSequenceNumberStats.getNumLate(), downstreamAudioSequenceNumberStats.getNumLost(), audio->getJitterBufferSamples() / NETWORK_BUFFER_LENGTH_SAMPLES_STEREO); + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color);*/ + + float packetLossRate, packetLossRate30s; + + char downstreamAudioStatsString[30]; + + audio->calculatePacketLossRate(audio->getIncomingStreamPacketStatsHistory(), packetLossRate, packetLossRate30s); + + sprintf(downstreamAudioStatsString, " mix: %.1f%%/%.1f%%, %d/?/%d", packetLossRate*100.0f, packetLossRate30s*100.0f, + audio->getDownstreamRingBuffer().framesAvailable(), audio->getDesiredJitterBufferFrames()); + verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); + + const MovingMinMaxAvg& timeGapStats = audio->getInterframeTimeGapStats(); + + sprintf(downstreamAudioStatsString, " %llu/%llu/%.2f, %d/%d", timeGapStats.getMin(), timeGapStats.getMax(), + timeGapStats.getAverage(), audio->getStarveCount(), audio->getDownstreamRingBuffer().getOverflowCount()); + + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); + + sprintf(downstreamAudioStatsString, " %llu/%llu/%.2f, %d/?", timeGapStats.getWindowMin(), timeGapStats.getWindowMax(), + timeGapStats.getWindowAverage(), audio->getConsecutiveNotMixedCount()); + + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); + char upstreamLabelString[] = " Upstream:"; verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamLabelString, color); char upstreamAudioStatsString[30]; - sprintf(upstreamAudioStatsString, " mic: %d/%d/%d, %d/%d/%d", audioMixerAvatarStreamStats._packetsEarly, - audioMixerAvatarStreamStats._packetsLate, audioMixerAvatarStreamStats._packetsLost, - audioMixerAvatarStreamStats._ringBufferFramesAvailable, audioMixerAvatarStreamStats._ringBufferCurrentJitterBufferFrames, - audioMixerAvatarStreamStats._ringBufferDesiredJitterBufferFrames); + + const AudioStreamStats& audioMixerAvatarAudioStreamStats = audio->getAudioMixerAvatarStreamAudioStats(); + + audio->calculatePacketLossRate(audio->getAudioMixerAvatarStreamPacketStatsHistory(), packetLossRate, packetLossRate30s); + + sprintf(upstreamAudioStatsString, " mic: %.1f%%/%.1f%%, %u/%u/%u", packetLossRate*100.0f, packetLossRate30s*100.0f, + audioMixerAvatarAudioStreamStats._ringBufferFramesAvailable, audioMixerAvatarAudioStreamStats._ringBufferCurrentJitterBufferFrames, + audioMixerAvatarAudioStreamStats._ringBufferDesiredJitterBufferFrames); verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - sprintf(upstreamAudioStatsString, " %llu/%llu/%.2f, %u/%u", audioMixerAvatarStreamStats._timeGapMin, - audioMixerAvatarStreamStats._timeGapMax, audioMixerAvatarStreamStats._timeGapAverage, - audioMixerAvatarStreamStats._ringBufferStarveCount, audioMixerAvatarStreamStats._ringBufferOverflowCount); + sprintf(upstreamAudioStatsString, " %llu/%llu/%.2f, %u/%u", audioMixerAvatarAudioStreamStats._timeGapMin, + audioMixerAvatarAudioStreamStats._timeGapMax, audioMixerAvatarAudioStreamStats._timeGapAverage, + audioMixerAvatarAudioStreamStats._ringBufferStarveCount, audioMixerAvatarAudioStreamStats._ringBufferOverflowCount); verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - sprintf(upstreamAudioStatsString, " %llu/%llu/%.2f, %u/%u", audioMixerAvatarStreamStats._timeGapMovingMin, - audioMixerAvatarStreamStats._timeGapMovingMax, audioMixerAvatarStreamStats._timeGapMovingAverage, - audioMixerAvatarStreamStats._ringBufferConsecutiveNotMixedCount, audioMixerAvatarStreamStats._ringBufferSilentFramesDropped); + sprintf(upstreamAudioStatsString, " %llu/%llu/%.2f, %u/%u", audioMixerAvatarAudioStreamStats._timeGapMovingMin, + audioMixerAvatarAudioStreamStats._timeGapMovingMax, audioMixerAvatarAudioStreamStats._timeGapMovingAverage, + audioMixerAvatarAudioStreamStats._ringBufferConsecutiveNotMixedCount, audioMixerAvatarAudioStreamStats._ringBufferSilentFramesDropped); verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - foreach(AudioStreamStats injectedStreamStats, audioMixerInjectedStreamStatsMap) { - sprintf(upstreamAudioStatsString, " inj: %d/%d/%d, %d/%d/%d", injectedStreamStats._packetsEarly, - injectedStreamStats._packetsLate, injectedStreamStats._packetsLost, - injectedStreamStats._ringBufferFramesAvailable, injectedStreamStats._ringBufferCurrentJitterBufferFrames, - injectedStreamStats._ringBufferDesiredJitterBufferFrames); + QHash > audioMixerInjectedStreamPacketStatsHistoryMap + = audio->getAudioMixerInjectedStreamPacketStatsHistoryMap(); + + foreach(const AudioStreamStats& injectedStreamAudioStats, audioMixerInjectedStreamAudioStatsMap) { + + audio->calculatePacketLossRate(audioMixerInjectedStreamPacketStatsHistoryMap[injectedStreamAudioStats._streamIdentifier], + packetLossRate, packetLossRate30s); + + sprintf(upstreamAudioStatsString, " inj: %.1f%%/%.1f%%, %u/%u/%u", packetLossRate*100.0f, packetLossRate30s*100.0f, + injectedStreamAudioStats._ringBufferFramesAvailable, injectedStreamAudioStats._ringBufferCurrentJitterBufferFrames, + injectedStreamAudioStats._ringBufferDesiredJitterBufferFrames); verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - sprintf(upstreamAudioStatsString, " %llu/%llu/%.2f, %u/%u", injectedStreamStats._timeGapMin, - injectedStreamStats._timeGapMax, injectedStreamStats._timeGapAverage, - injectedStreamStats._ringBufferStarveCount, injectedStreamStats._ringBufferOverflowCount); + sprintf(upstreamAudioStatsString, " %llu/%llu/%.2f, %u/%u", injectedStreamAudioStats._timeGapMin, + injectedStreamAudioStats._timeGapMax, injectedStreamAudioStats._timeGapAverage, + injectedStreamAudioStats._ringBufferStarveCount, injectedStreamAudioStats._ringBufferOverflowCount); verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - sprintf(upstreamAudioStatsString, " %llu/%llu/%.2f, %u/%u", injectedStreamStats._timeGapMovingMin, - injectedStreamStats._timeGapMovingMax, injectedStreamStats._timeGapMovingAverage, - injectedStreamStats._ringBufferConsecutiveNotMixedCount, injectedStreamStats._ringBufferSilentFramesDropped); + sprintf(upstreamAudioStatsString, " %llu/%llu/%.2f, %u/%u", injectedStreamAudioStats._timeGapMovingMin, + injectedStreamAudioStats._timeGapMovingMax, injectedStreamAudioStats._timeGapMovingAverage, + injectedStreamAudioStats._ringBufferConsecutiveNotMixedCount, injectedStreamAudioStats._ringBufferSilentFramesDropped); verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 38f1adec21..9f049fc5e8 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -67,6 +67,8 @@ public: int samplesAvailable() const; int framesAvailable() const { return samplesAvailable() / _numFrameSamples; } + + int getNumFrameSamples() const { return _numFrameSamples; } bool isNotStarvedOrHasMinimumSamples(int numRequiredSamples) const; diff --git a/libraries/audio/src/AudioStreamStats.h b/libraries/audio/src/AudioStreamStats.h index 2c66187309..191e10ba8e 100644 --- a/libraries/audio/src/AudioStreamStats.h +++ b/libraries/audio/src/AudioStreamStats.h @@ -13,6 +13,7 @@ #define hifi_AudioStreamStats_h #include "PositionalAudioRingBuffer.h" +#include "SequenceNumberStats.h" class AudioStreamStats { public: @@ -32,13 +33,7 @@ public: _ringBufferConsecutiveNotMixedCount(0), _ringBufferOverflowCount(0), _ringBufferSilentFramesDropped(0), - _packetsReceived(0), - _packetsUnreasonable(0), - _packetsEarly(0), - _packetsLate(0), - _packetsLost(0), - _packetsRecovered(0), - _packetsDuplicate(0) + _packetStreamStats() {} PositionalAudioRingBuffer::Type _streamType; @@ -59,13 +54,7 @@ public: quint32 _ringBufferOverflowCount; quint32 _ringBufferSilentFramesDropped; - quint32 _packetsReceived; - quint32 _packetsUnreasonable; - quint32 _packetsEarly; - quint32 _packetsLate; - quint32 _packetsLost; - quint32 _packetsRecovered; - quint32 _packetsDuplicate; + PacketStreamStats _packetStreamStats; }; #endif // hifi_AudioStreamStats_h diff --git a/libraries/networking/src/SequenceNumberStats.cpp b/libraries/networking/src/SequenceNumberStats.cpp index 3f696a522b..6a26e613b6 100644 --- a/libraries/networking/src/SequenceNumberStats.cpp +++ b/libraries/networking/src/SequenceNumberStats.cpp @@ -16,26 +16,14 @@ SequenceNumberStats::SequenceNumberStats() : _lastReceived(std::numeric_limits::max()), _missingSet(), - _numReceived(0), - _numUnreasonable(0), - _numEarly(0), - _numLate(0), - _numLost(0), - _numRecovered(0), - _numDuplicate(0), + _stats(), _lastSenderUUID() { } void SequenceNumberStats::reset() { _missingSet.clear(); - _numReceived = 0; - _numUnreasonable = 0; - _numEarly = 0; - _numLate = 0; - _numLost = 0; - _numRecovered = 0; - _numDuplicate = 0; + _stats = PacketStreamStats(); } static const int UINT16_RANGE = std::numeric_limits::max() + 1; @@ -51,9 +39,9 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderU } // determine our expected sequence number... handle rollover appropriately - quint16 expected = _numReceived > 0 ? _lastReceived + (quint16)1 : incoming; + quint16 expected = _stats._numReceived > 0 ? _lastReceived + (quint16)1 : incoming; - _numReceived++; + _stats._numReceived++; if (incoming == expected) { // on time _lastReceived = incoming; @@ -80,7 +68,7 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderU // ignore packet if gap is unreasonable qDebug() << "ignoring unreasonable sequence number:" << incoming << "previous:" << _lastReceived; - _numUnreasonable++; + _stats._numUnreasonable++; return; } @@ -92,8 +80,8 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderU qDebug() << ">>>>>>>> missing gap=" << (incomingInt - expectedInt); } - _numEarly++; - _numLost += (incomingInt - expectedInt); + _stats._numEarly++; + _stats._numLost += (incomingInt - expectedInt); _lastReceived = incoming; // add all sequence numbers that were skipped to the missing sequence numbers list @@ -110,7 +98,7 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderU if (wantExtraDebugging) { qDebug() << "this packet is later than expected..."; } - _numLate++; + _stats._numLate++; // do not update _lastReceived; it shouldn't become smaller @@ -119,13 +107,13 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderU if (wantExtraDebugging) { qDebug() << "found it in _missingSet"; } - _numLost--; - _numRecovered++; + _stats._numLost--; + _stats._numRecovered++; } else { if (wantExtraDebugging) { qDebug() << "sequence:" << incoming << "was NOT found in _missingSet and is probably a duplicate"; } - _numDuplicate++; + _stats._numDuplicate++; } } } diff --git a/libraries/networking/src/SequenceNumberStats.h b/libraries/networking/src/SequenceNumberStats.h index 88c8748b03..b399c23d2b 100644 --- a/libraries/networking/src/SequenceNumberStats.h +++ b/libraries/networking/src/SequenceNumberStats.h @@ -17,6 +17,26 @@ const int MAX_REASONABLE_SEQUENCE_GAP = 1000; +class PacketStreamStats { +public: + PacketStreamStats() + : _numReceived(0), + _numUnreasonable(0), + _numEarly(0), + _numLate(0), + _numLost(0), + _numRecovered(0), + _numDuplicate(0) + {} + quint32 _numReceived; + quint32 _numUnreasonable; + quint32 _numEarly; + quint32 _numLate; + quint32 _numLost; + quint32 _numRecovered; + quint32 _numDuplicate; +}; + class SequenceNumberStats { public: SequenceNumberStats(); @@ -25,27 +45,22 @@ public: void sequenceNumberReceived(quint16 incoming, QUuid senderUUID = QUuid(), const bool wantExtraDebugging = false); void pruneMissingSet(const bool wantExtraDebugging = false); - quint32 getNumReceived() const { return _numReceived; } - quint32 getNumUnreasonable() const { return _numUnreasonable; } - quint32 getNumOutOfOrder() const { return _numEarly + _numLate; } - quint32 getNumEarly() const { return _numEarly; } - quint32 getNumLate() const { return _numLate; } - quint32 getNumLost() const { return _numLost; } - quint32 getNumRecovered() const { return _numRecovered; } - quint32 getNumDuplicate() const { return _numDuplicate; } + quint32 getNumReceived() const { return _stats._numReceived; } + quint32 getNumUnreasonable() const { return _stats._numUnreasonable; } + quint32 getNumOutOfOrder() const { return _stats._numEarly + _stats._numLate; } + quint32 getNumEarly() const { return _stats._numEarly; } + quint32 getNumLate() const { return _stats._numLate; } + quint32 getNumLost() const { return _stats._numLost; } + quint32 getNumRecovered() const { return _stats._numRecovered; } + quint32 getNumDuplicate() const { return _stats._numDuplicate; } + const PacketStreamStats& getStats() const { return _stats; } const QSet& getMissingSet() const { return _missingSet; } private: quint16 _lastReceived; QSet _missingSet; - quint32 _numReceived; - quint32 _numUnreasonable; - quint32 _numEarly; - quint32 _numLate; - quint32 _numLost; - quint32 _numRecovered; - quint32 _numDuplicate; + PacketStreamStats _stats; QUuid _lastSenderUUID; }; diff --git a/libraries/shared/src/RingBufferHistory.h b/libraries/shared/src/RingBufferHistory.h index 094963ddc6..e33723b2b1 100644 --- a/libraries/shared/src/RingBufferHistory.h +++ b/libraries/shared/src/RingBufferHistory.h @@ -20,7 +20,7 @@ class RingBufferHistory { public: - RingBufferHistory(int capacity) + RingBufferHistory(int capacity = 10) : _size(capacity + 1), _capacity(capacity), _newestEntryAt(0), @@ -47,6 +47,7 @@ public: delete[] _buffer; _buffer = new T[_size]; memcpy(_buffer, rhs._buffer, _size*sizeof(T)); + return *this; } ~RingBufferHistory() { @@ -120,7 +121,7 @@ public: return *this; } - Iterator& operator++(int) { + Iterator operator++(int) { Iterator tmp(*this); ++(*this); return tmp; From 7fcf149f708f4b6b2448b26db8e6914739f93fc2 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 9 Jul 2014 18:14:58 -0700 Subject: [PATCH 26/70] removed if randFloat --- interface/src/Audio.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 2073659e89..e81683b213 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -693,7 +693,7 @@ void Audio::handleAudioInput() { // memcpy our orientation memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation)); currentPacketPtr += sizeof(headOrientation); -if (randFloat() < 0.95f) + nodeList->writeDatagram(audioDataPacket, numAudioBytes + leadingBytes, audioMixer); _outgoingAvatarAudioSequenceNumber++; From 0d83b9a6f6aa02bd0c5d109fdf8dbf0ed05d0951 Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 10 Jul 2014 09:51:01 -0700 Subject: [PATCH 27/70] tidied up code --- .../src/audio/AudioMixerClientData.cpp | 18 ++++++------ .../src/audio/AvatarAudioRingBuffer.cpp | 2 +- interface/src/Audio.cpp | 5 ++-- interface/src/ui/Stats.cpp | 20 ++++--------- libraries/audio/src/AudioStreamStats.h | 12 ++++---- .../audio/src/InjectedAudioRingBuffer.cpp | 2 +- .../audio/src/PositionalAudioRingBuffer.cpp | 2 +- .../audio/src/PositionalAudioRingBuffer.h | 2 +- libraries/shared/src/MovingMinMaxAvg.h | 4 --- libraries/shared/src/RingBufferHistory.h | 28 +++++++++---------- 10 files changed, 41 insertions(+), 54 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index f8d97dc44e..c624c1dc01 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -176,9 +176,9 @@ AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const Positio streamStats._timeGapMin = timeGapStats.getMin(); streamStats._timeGapMax = timeGapStats.getMax(); streamStats._timeGapAverage = timeGapStats.getAverage(); - streamStats._timeGapMovingMin = timeGapStats.getWindowMin(); - streamStats._timeGapMovingMax = timeGapStats.getWindowMax(); - streamStats._timeGapMovingAverage = timeGapStats.getWindowAverage(); + streamStats._timeGapWindowMin = timeGapStats.getWindowMin(); + streamStats._timeGapWindowMax = timeGapStats.getWindowMax(); + streamStats._timeGapWindowAverage = timeGapStats.getWindowAverage(); streamStats._ringBufferFramesAvailable = ringBuffer->framesAvailable(); streamStats._ringBufferCurrentJitterBufferFrames = ringBuffer->getCurrentJitterBufferFrames(); @@ -267,9 +267,9 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { + " min gap:" + QString::number(streamStats._timeGapMin) + " max gap:" + QString::number(streamStats._timeGapMax) + " avg gap:" + QString::number(streamStats._timeGapAverage, 'g', 2) - + " min 30s gap:" + QString::number(streamStats._timeGapMovingMin) - + " max 30s gap:" + QString::number(streamStats._timeGapMovingMax) - + " avg 30s gap:" + QString::number(streamStats._timeGapMovingAverage, 'g', 2); + + " min 30s gap:" + QString::number(streamStats._timeGapWindowMin) + + " max 30s gap:" + QString::number(streamStats._timeGapWindowMax) + + " avg 30s gap:" + QString::number(streamStats._timeGapWindowAverage, 'g', 2); } else { result = "mic unknown"; } @@ -290,9 +290,9 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { + " min gap:" + QString::number(streamStats._timeGapMin) + " max gap:" + QString::number(streamStats._timeGapMax) + " avg gap:" + QString::number(streamStats._timeGapAverage, 'g', 2) - + " min 30s gap:" + QString::number(streamStats._timeGapMovingMin) - + " max 30s gap:" + QString::number(streamStats._timeGapMovingMax) - + " avg 30s gap:" + QString::number(streamStats._timeGapMovingAverage, 'g', 2); + + " min 30s gap:" + QString::number(streamStats._timeGapWindowMin) + + " max 30s gap:" + QString::number(streamStats._timeGapWindowMax) + + " avg 30s gap:" + QString::number(streamStats._timeGapWindowAverage, 'g', 2); } } return result; diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp index 0177bc48ea..3fa9f64cff 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp @@ -19,7 +19,7 @@ AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo, bool dynamicJitterBu } int AvatarAudioRingBuffer::parseData(const QByteArray& packet) { - frameReceived(); + timeGapStatsFrameReceived(); updateDesiredJitterBufferFrames(); _shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho); diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index e81683b213..17e9054568 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -50,6 +50,7 @@ static const int NUMBER_OF_NOISE_SAMPLE_FRAMES = 300; static const int AUDIO_STREAM_STATS_HISTORY_SIZE = 30; +// audio frames time gap stats (min/max/avg) for last ~30 seconds are recalculated every ~1 second const int TIME_GAPS_STATS_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS; const int TIME_GAP_STATS_WINDOW_INTERVALS = 30; @@ -776,8 +777,8 @@ void Audio::parseAudioStreamStatsPacket(const QByteArray& packet) { } } - // when an audio stream stats packet is received, also record the current packets received and lost - // in the packet loss stats history + // when an audio stream stats packet is received, also record the downstream packets stats in the history + // for calculating packet loss rates _incomingStreamPacketStatsHistory.insert(_incomingMixedAudioSequenceNumberStats.getStats()); } diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 01032aebc8..25d5ee44bc 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -341,22 +341,12 @@ void Stats::display( verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString4, color); + float packetLossRate, packetLossRate30s; char downstreamLabelString[] = " Downstream:"; verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamLabelString, color); - /* const SequenceNumberStats& downstreamAudioSequenceNumberStats = audio->getIncomingMixedAudioSequenceNumberStats(); - - sprintf(downstreamAudioStatsString, " mix: %d/%d/%d, %d", downstreamAudioSequenceNumberStats.getNumEarly(), - downstreamAudioSequenceNumberStats.getNumLate(), downstreamAudioSequenceNumberStats.getNumLost(), - audio->getJitterBufferSamples() / NETWORK_BUFFER_LENGTH_SAMPLES_STEREO); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color);*/ - - float packetLossRate, packetLossRate30s; - char downstreamAudioStatsString[30]; audio->calculatePacketLossRate(audio->getIncomingStreamPacketStatsHistory(), packetLossRate, packetLossRate30s); @@ -406,8 +396,8 @@ void Stats::display( verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - sprintf(upstreamAudioStatsString, " %llu/%llu/%.2f, %u/%u", audioMixerAvatarAudioStreamStats._timeGapMovingMin, - audioMixerAvatarAudioStreamStats._timeGapMovingMax, audioMixerAvatarAudioStreamStats._timeGapMovingAverage, + sprintf(upstreamAudioStatsString, " %llu/%llu/%.2f, %u/%u", audioMixerAvatarAudioStreamStats._timeGapWindowMin, + audioMixerAvatarAudioStreamStats._timeGapWindowMax, audioMixerAvatarAudioStreamStats._timeGapWindowAverage, audioMixerAvatarAudioStreamStats._ringBufferConsecutiveNotMixedCount, audioMixerAvatarAudioStreamStats._ringBufferSilentFramesDropped); verticalOffset += STATS_PELS_PER_LINE; @@ -435,8 +425,8 @@ void Stats::display( verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - sprintf(upstreamAudioStatsString, " %llu/%llu/%.2f, %u/%u", injectedStreamAudioStats._timeGapMovingMin, - injectedStreamAudioStats._timeGapMovingMax, injectedStreamAudioStats._timeGapMovingAverage, + sprintf(upstreamAudioStatsString, " %llu/%llu/%.2f, %u/%u", injectedStreamAudioStats._timeGapWindowMin, + injectedStreamAudioStats._timeGapWindowMax, injectedStreamAudioStats._timeGapWindowAverage, injectedStreamAudioStats._ringBufferConsecutiveNotMixedCount, injectedStreamAudioStats._ringBufferSilentFramesDropped); verticalOffset += STATS_PELS_PER_LINE; diff --git a/libraries/audio/src/AudioStreamStats.h b/libraries/audio/src/AudioStreamStats.h index 191e10ba8e..2cd0bca880 100644 --- a/libraries/audio/src/AudioStreamStats.h +++ b/libraries/audio/src/AudioStreamStats.h @@ -23,9 +23,9 @@ public: _timeGapMin(0), _timeGapMax(0), _timeGapAverage(0.0f), - _timeGapMovingMin(0), - _timeGapMovingMax(0), - _timeGapMovingAverage(0.0f), + _timeGapWindowMin(0), + _timeGapWindowMax(0), + _timeGapWindowAverage(0.0f), _ringBufferFramesAvailable(0), _ringBufferCurrentJitterBufferFrames(0), _ringBufferDesiredJitterBufferFrames(0), @@ -42,9 +42,9 @@ public: quint64 _timeGapMin; quint64 _timeGapMax; float _timeGapAverage; - quint64 _timeGapMovingMin; - quint64 _timeGapMovingMax; - float _timeGapMovingAverage; + quint64 _timeGapWindowMin; + quint64 _timeGapWindowMax; + float _timeGapWindowAverage; quint32 _ringBufferFramesAvailable; quint16 _ringBufferCurrentJitterBufferFrames; diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index d3d0cdfb8d..bdad7b2a7a 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -31,7 +31,7 @@ InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier, const uchar MAX_INJECTOR_VOLUME = 255; int InjectedAudioRingBuffer::parseData(const QByteArray& packet) { - frameReceived(); + timeGapStatsFrameReceived(); updateDesiredJitterBufferFrames(); // setup a data stream to read from this packet diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 666b89e568..411b02400d 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -198,7 +198,7 @@ int PositionalAudioRingBuffer::getCalculatedDesiredJitterBufferFrames() const { return calculatedDesiredJitterBufferFrames; } -void PositionalAudioRingBuffer::frameReceived() { +void PositionalAudioRingBuffer::timeGapStatsFrameReceived() { quint64 now = usecTimestampNow(); if (_lastFrameReceivedTime != 0) { quint64 gap = now - _lastFrameReceivedTime; diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index a3adec0117..3a9e7ed124 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -80,7 +80,7 @@ protected: PositionalAudioRingBuffer(const PositionalAudioRingBuffer&); PositionalAudioRingBuffer& operator= (const PositionalAudioRingBuffer&); - void frameReceived(); + void timeGapStatsFrameReceived(); void updateDesiredJitterBufferFrames(); PositionalAudioRingBuffer::Type _type; diff --git a/libraries/shared/src/MovingMinMaxAvg.h b/libraries/shared/src/MovingMinMaxAvg.h index abe43cdfac..00aa35dd2c 100644 --- a/libraries/shared/src/MovingMinMaxAvg.h +++ b/libraries/shared/src/MovingMinMaxAvg.h @@ -86,7 +86,6 @@ public: } void update(T newSample) { - // update overall stats _overallStats.updateWithSample(newSample, _samplesCollected); @@ -115,7 +114,6 @@ public: } } - bool getNewStatsAvailableFlag() const { return _newStatsAvailable; } void clearNewStatsAvailableFlag() { _newStatsAvailable = false; } @@ -126,8 +124,6 @@ public: T getWindowMax() const { return _windowStats._max; } double getWindowAverage() const { return _windowStats._average; } - - private: int _intervalLength; int _windowIntervals; diff --git a/libraries/shared/src/RingBufferHistory.h b/libraries/shared/src/RingBufferHistory.h index e33723b2b1..fbb34ed9be 100644 --- a/libraries/shared/src/RingBufferHistory.h +++ b/libraries/shared/src/RingBufferHistory.h @@ -23,7 +23,7 @@ public: RingBufferHistory(int capacity = 10) : _size(capacity + 1), _capacity(capacity), - _newestEntryAt(0), + _newestEntryAtIndex(0), _numEntries(0) { _buffer = new T[_size]; @@ -32,7 +32,7 @@ public: RingBufferHistory(const RingBufferHistory& other) : _size(other._size), _capacity(other._capacity), - _newestEntryAt(other._newestEntryAt), + _newestEntryAtIndex(other._newestEntryAtIndex), _numEntries(other._numEntries) { _buffer = new T[_size]; @@ -42,7 +42,7 @@ public: RingBufferHistory& operator= (const RingBufferHistory& rhs) { _size = rhs._size; _capacity = rhs._capacity; - _newestEntryAt = rhs._newestEntryAt; + _newestEntryAtIndex = rhs._newestEntryAtIndex; _numEntries = rhs._numEntries; delete[] _buffer; _buffer = new T[_size]; @@ -60,10 +60,10 @@ public: void insert(const T& entry) { // increment newest entry index cyclically - _newestEntryAt = (_newestEntryAt == _size - 1) ? 0 : _newestEntryAt + 1; + _newestEntryAtIndex = (_newestEntryAtIndex == _size - 1) ? 0 : _newestEntryAtIndex + 1; // insert new entry - _buffer[_newestEntryAt] = entry; + _buffer[_newestEntryAtIndex] = entry; if (_numEntries < _capacity) { _numEntries++; } @@ -75,7 +75,7 @@ public: if (!(entryAge >= 0 && entryAge < _numEntries)) { return NULL; } - int entryAt = _newestEntryAt - entryAge; + int entryAt = _newestEntryAtIndex - entryAge; if (entryAt < 0) { entryAt += _size; } @@ -87,11 +87,11 @@ public: } const T* getNewestEntry() const { - return &_buffer[_newestEntryAt]; + return &_buffer[_newestEntryAtIndex]; } T* getNewestEntry() { - return &_buffer[_newestEntryAt]; + return &_buffer[_newestEntryAtIndex]; } int getCapacity() const { return _capacity; } @@ -101,7 +101,7 @@ private: T* _buffer; int _size; int _capacity; - int _newestEntryAt; + int _newestEntryAtIndex; int _numEntries; @@ -133,14 +133,14 @@ public: T* _at; }; - Iterator begin() { return Iterator(_buffer, _size, &_buffer[_newestEntryAt]); } + Iterator begin() { return Iterator(_buffer, _size, &_buffer[_newestEntryAtIndex]); } Iterator end() { - int endAt = _newestEntryAt - _numEntries; - if (endAt < 0) { - endAt += _size; + int endAtIndex = _newestEntryAtIndex - _numEntries; + if (endAtIndex < 0) { + endAtIndex += _size; } - return Iterator(_buffer, _size, &_buffer[endAt]); + return Iterator(_buffer, _size, &_buffer[endAtIndex]); } }; From d1b96110409db6c8e1cf515de25b52b70975c876 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Thu, 10 Jul 2014 18:52:38 +0200 Subject: [PATCH 28/70] Remove debug output lines. --- interface/src/Application.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 95cfaab4cf..ace265ad4f 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -803,7 +803,6 @@ bool Application::event(QEvent* event) { } void Application::keyPressEvent(QKeyEvent* event) { - qDebug("Application::keyPressEvent(%x)", event->key()); _keysPressed.insert(event->key()); @@ -1054,7 +1053,6 @@ void Application::keyPressEvent(QKeyEvent* event) { } void Application::keyReleaseEvent(QKeyEvent* event) { - qDebug("Application::keyReleaseEvent(%x)", event->key()); _keysPressed.remove(event->key()); From 3389b6440cc5aeac1fb2dea661f0b70e2d297fb6 Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 10 Jul 2014 09:53:37 -0700 Subject: [PATCH 29/70] undo .gitignore changes --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index d3af3c4535..4176dcc652 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,3 @@ interface/external/rtmidi/* # Ignore interfaceCache for Linux users interface/interfaceCache/ -tests/shared/src/MovingMinMaxAvgTests.cpp -examples/dancer.js -examples/happyBirthday.js From 37b60a63b2908d2332b4af6f77256b7962d6af5b Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 10 Jul 2014 10:01:32 -0700 Subject: [PATCH 30/70] removed spaces --- libraries/shared/src/RingBufferHistory.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/shared/src/RingBufferHistory.h b/libraries/shared/src/RingBufferHistory.h index fbb34ed9be..03cbce80df 100644 --- a/libraries/shared/src/RingBufferHistory.h +++ b/libraries/shared/src/RingBufferHistory.h @@ -104,8 +104,6 @@ private: int _newestEntryAtIndex; int _numEntries; - - public: class Iterator : public std::iterator < std::forward_iterator_tag, T > { public: From 763cc26fa96dd50c7669260941f9afc651ba826c Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 10 Jul 2014 10:21:03 -0700 Subject: [PATCH 31/70] added operator= to RingBufferHistory::Iterator --- libraries/shared/src/RingBufferHistory.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libraries/shared/src/RingBufferHistory.h b/libraries/shared/src/RingBufferHistory.h index 03cbce80df..7ec5bbd6ff 100644 --- a/libraries/shared/src/RingBufferHistory.h +++ b/libraries/shared/src/RingBufferHistory.h @@ -109,6 +109,12 @@ public: public: Iterator(T* buffer, int size, T* at) : _buffer(buffer), _bufferEnd(buffer+size), _at(at) {} + Iterator& operator=(const Iterator& other) { + _buffer = other._buffer; + _bufferEnd = other._bufferEnd; + _at = other._at; + } + bool operator==(const Iterator& rhs) { return _at == rhs._at; } bool operator!=(const Iterator& rhs) { return _at != rhs._at; } T& operator*() { return *_at; } From d555017fb4c62690b5447a3033e3ca413835d015 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Thu, 10 Jul 2014 19:25:00 +0200 Subject: [PATCH 32/70] Coding style fix. --- interface/src/GLCanvas.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/interface/src/GLCanvas.cpp b/interface/src/GLCanvas.cpp index 87c1016c5e..cde1890e6d 100644 --- a/interface/src/GLCanvas.cpp +++ b/interface/src/GLCanvas.cpp @@ -184,12 +184,13 @@ bool GLCanvas::eventFilter(QObject*, QEvent* event) { { QKeyEvent* keyEvent = static_cast(event); if (keyEvent->key() == Qt::Key_Alt || keyEvent->key() == Qt::Key_Meta) { - if (event->type() == QEvent::KeyPress) + if (event->type() == QEvent::KeyPress) { keyPressEvent(keyEvent); - else if (event->type() == QEvent::KeyRelease) + } else if (event->type() == QEvent::KeyRelease) { keyReleaseEvent(keyEvent); - else + } else { QGLWidget::event(event); + } return true; } } From baf825789e3b51194d607af30cf063bde455ab39 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 10 Jul 2014 10:28:56 -0700 Subject: [PATCH 33/70] Made Oculus UI radius not a magic number --- interface/src/ui/ApplicationOverlay.cpp | 14 ++++++++------ interface/src/ui/ApplicationOverlay.h | 1 + 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 6324ee03b0..005858fd4d 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -40,7 +40,8 @@ ApplicationOverlay::ApplicationOverlay() : _framebufferObject(NULL), _textureFov(DEFAULT_OCULUS_UI_ANGULAR_SIZE * RADIANS_PER_DEGREE), _alpha(1.0f), - _crosshairTexture(0) { + _crosshairTexture(0), + _oculusuiRadius(1.0f) { memset(_reticleActive, 0, sizeof(_reticleActive)); memset(_magActive, 0, sizeof(_reticleActive)); @@ -288,9 +289,10 @@ QPoint ApplicationOverlay::getOculusPalmClickLocation(const PalmData *palm) cons //We back the ray up by dir to ensure that it will not start inside the UI. glm::vec3 adjustedPos = tipPos - dir; //Find intersection of crosshair ray. - if (raySphereIntersect(dir, adjustedPos, 1, &t)){ + if (raySphereIntersect(dir, adjustedPos, _oculusuiRadius, &t)){ glm::vec3 collisionPos = adjustedPos + dir * t; - + //Normalize it in case its not a radius of 1 + collisionPos = glm::normalize(collisionPos); //If we hit the back hemisphere, mark it as not a collision if (collisionPos.z > 0) { rv.setX(INT_MAX); @@ -321,7 +323,7 @@ bool ApplicationOverlay::calculateRayUICollisionPoint(const glm::vec3& position, glm::vec3 relativeDirection = orientation * direction; float t; - if (raySphereIntersect(relativeDirection, relativePosition, 1, &t)){ + if (raySphereIntersect(relativeDirection, relativePosition, _oculusuiRadius, &t)){ result = position + direction * t; return true; } @@ -1196,8 +1198,8 @@ void ApplicationOverlay::renderTexturedHemisphere() { glTranslatef(position.x, position.y, position.z); glMultMatrixf(&rotation[0][0]); - - + glScalef(_oculusuiRadius, _oculusuiRadius, _oculusuiRadius); + glDrawRangeElements(GL_TRIANGLES, 0, vertices - 1, indices, GL_UNSIGNED_SHORT, 0); glPopMatrix(); diff --git a/interface/src/ui/ApplicationOverlay.h b/interface/src/ui/ApplicationOverlay.h index e355665b95..33ba818f40 100644 --- a/interface/src/ui/ApplicationOverlay.h +++ b/interface/src/ui/ApplicationOverlay.h @@ -71,6 +71,7 @@ private: float _magSizeMult[NUMBER_OF_MAGNIFIERS]; float _alpha; + float _oculusuiRadius; GLuint _crosshairTexture; }; From 35e4b253c2be7623e015a74f87f83f57ac7d018c Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 10 Jul 2014 10:35:29 -0700 Subject: [PATCH 34/70] added typename keyword in MovingMinMaxAvg.h --- libraries/shared/src/MovingMinMaxAvg.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/shared/src/MovingMinMaxAvg.h b/libraries/shared/src/MovingMinMaxAvg.h index 00aa35dd2c..79f4b0e3cb 100644 --- a/libraries/shared/src/MovingMinMaxAvg.h +++ b/libraries/shared/src/MovingMinMaxAvg.h @@ -101,8 +101,8 @@ public: _existingSamplesInCurrentInterval = 0; // update the window's stats by combining the intervals' stats - RingBufferHistory::Iterator i = _intervalStats.begin(); - RingBufferHistory::Iterator end = _intervalStats.end(); + typename RingBufferHistory::Iterator i = _intervalStats.begin(); + typename RingBufferHistory::Iterator end = _intervalStats.end(); _windowStats = Stats(); int intervalsIncludedInWindowStats = 0; while (i != end) { From 0b213f96168ccaf7f1550cd30f8ab344c51f391e Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 10 Jul 2014 10:44:33 -0700 Subject: [PATCH 35/70] streamlined AudioStreamStats packing; added #include --- .../src/audio/AudioMixerClientData.cpp | 13 ++----------- libraries/shared/src/MovingMinMaxAvg.h | 2 ++ 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index c624c1dc01..ae4a0269cc 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -162,14 +162,13 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer) const { AudioStreamStats streamStats; - const SequenceNumberStats* streamSequenceNumberStats; streamStats._streamType = ringBuffer->getType(); if (streamStats._streamType == PositionalAudioRingBuffer::Injector) { streamStats._streamIdentifier = ((InjectedAudioRingBuffer*)ringBuffer)->getStreamIdentifier(); - streamSequenceNumberStats = &_incomingInjectedAudioSequenceNumberStatsMap[streamStats._streamIdentifier]; + streamStats._packetStreamStats = _incomingInjectedAudioSequenceNumberStatsMap[streamStats._streamIdentifier].getStats(); } else { - streamSequenceNumberStats = &_incomingAvatarAudioSequenceNumberStats; + streamStats._packetStreamStats = _incomingAvatarAudioSequenceNumberStats.getStats(); } const MovingMinMaxAvg& timeGapStats = ringBuffer->getInterframeTimeGapStatsForStatsPacket(); @@ -187,14 +186,6 @@ AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const Positio streamStats._ringBufferConsecutiveNotMixedCount = ringBuffer->getConsecutiveNotMixedCount(); streamStats._ringBufferOverflowCount = ringBuffer->getOverflowCount(); streamStats._ringBufferSilentFramesDropped = ringBuffer->getSilentFramesDropped(); - - streamStats._packetStreamStats._numReceived = streamSequenceNumberStats->getNumReceived(); - streamStats._packetStreamStats._numUnreasonable = streamSequenceNumberStats->getNumUnreasonable(); - streamStats._packetStreamStats._numEarly = streamSequenceNumberStats->getNumEarly(); - streamStats._packetStreamStats._numLate = streamSequenceNumberStats->getNumLate(); - streamStats._packetStreamStats._numLost = streamSequenceNumberStats->getNumLost(); - streamStats._packetStreamStats._numRecovered = streamSequenceNumberStats->getNumRecovered(); - streamStats._packetStreamStats._numDuplicate = streamSequenceNumberStats->getNumDuplicate(); return streamStats; } diff --git a/libraries/shared/src/MovingMinMaxAvg.h b/libraries/shared/src/MovingMinMaxAvg.h index 79f4b0e3cb..7c645dbd93 100644 --- a/libraries/shared/src/MovingMinMaxAvg.h +++ b/libraries/shared/src/MovingMinMaxAvg.h @@ -12,6 +12,8 @@ #ifndef hifi_MovingMinMaxAvg_h #define hifi_MovingMinMaxAvg_h +#include + #include "RingBufferHistory.h" template From 4d591de76276614bb3dd40b50f123081f4281640 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 10 Jul 2014 10:52:15 -0700 Subject: [PATCH 36/70] Applied low velocity filter to palm tip position --- interface/src/devices/SixenseManager.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/interface/src/devices/SixenseManager.cpp b/interface/src/devices/SixenseManager.cpp index b9a736887e..f145b8d5be 100644 --- a/interface/src/devices/SixenseManager.cpp +++ b/interface/src/devices/SixenseManager.cpp @@ -173,8 +173,10 @@ void SixenseManager::update(float deltaTime) { // Use a velocity sensitive filter to damp small motions and preserve large ones with // no latency. float velocityFilter = glm::clamp(1.0f - glm::length(rawVelocity), 0.0f, 1.0f); - palm->setRawPosition(palm->getRawPosition() * velocityFilter + position * (1.0f - velocityFilter)); - palm->setRawRotation(safeMix(palm->getRawRotation(), rotation, 1.0f - velocityFilter)); + position = palm->getRawPosition() * velocityFilter + position * (1.0f - velocityFilter); + rotation = safeMix(palm->getRawRotation(), rotation, 1.0f - velocityFilter); + palm->setRawPosition(position); + palm->setRawRotation(rotation); } else { palm->setRawPosition(position); palm->setRawRotation(rotation); From fdd9acb47768df14819175f9bb802e39995535f0 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Thu, 10 Jul 2014 10:55:41 -0700 Subject: [PATCH 37/70] bug fix: bad logic for early exit --- interface/src/avatar/AvatarManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/avatar/AvatarManager.cpp b/interface/src/avatar/AvatarManager.cpp index 86ec7c2680..debe6489ea 100644 --- a/interface/src/avatar/AvatarManager.cpp +++ b/interface/src/avatar/AvatarManager.cpp @@ -41,7 +41,7 @@ void AvatarManager::init() { } void AvatarManager::updateOtherAvatars(float deltaTime) { - if (_avatarHash.size() > 1) { + if (_avatarHash.size() < 2) { return; } bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings); From ad6a219a60b0b4abbaecca44663ea60c8da47c62 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Thu, 10 Jul 2014 11:08:15 -0700 Subject: [PATCH 38/70] fix bug: remove redundant _skeleton.simulate() --- interface/src/avatar/Avatar.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index 4efb03b3f5..db4f44c82c 100644 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -145,20 +145,20 @@ void Avatar::simulate(float deltaTime) { _skeletonModel.setLODDistance(getLODDistance()); if (!_shouldRenderBillboard && inViewFrustum) { - if (_hasNewJointRotations) { - PerformanceTimer perfTimer("skeleton"); - for (int i = 0; i < _jointData.size(); i++) { - const JointData& data = _jointData.at(i); - _skeletonModel.setJointState(i, data.valid, data.rotation); - } - _skeletonModel.simulate(deltaTime); - } { - PerformanceTimer perfTimer("head"); + PerformanceTimer perfTimer("skeleton"); + if (_hasNewJointRotations) { + for (int i = 0; i < _jointData.size(); i++) { + const JointData& data = _jointData.at(i); + _skeletonModel.setJointState(i, data.valid, data.rotation); + } + } _skeletonModel.simulate(deltaTime, _hasNewJointRotations); simulateAttachments(deltaTime); _hasNewJointRotations = false; - + } + { + PerformanceTimer perfTimer("head"); glm::vec3 headPosition = _position; _skeletonModel.getHeadPosition(headPosition); Head* head = getHead(); From 0be70063c5236bec901e6e6c63b80ed7a922cf0a Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Thu, 10 Jul 2014 11:39:49 -0700 Subject: [PATCH 39/70] only set hasNewJointRotations true for valid rotation --- libraries/avatars/src/AvatarData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 3f3e71c5e8..6cfa4ba488 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -490,11 +490,11 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { for (int i = 0; i < numJoints; i++) { JointData& data = _jointData[i]; if (data.valid) { + _hasNewJointRotations = true; sourceBuffer += unpackOrientationQuatFromBytes(sourceBuffer, data.rotation); } } } // numJoints * 8 bytes - _hasNewJointRotations = true; return sourceBuffer - startPosition; } From a955a17472999b32f5e29e84770554ee479aac6b Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 10 Jul 2014 11:55:19 -0700 Subject: [PATCH 40/70] changed RingBufferHistory to use qvector instead of raw array --- libraries/shared/src/RingBufferHistory.h | 51 +++++------------------- 1 file changed, 11 insertions(+), 40 deletions(-) diff --git a/libraries/shared/src/RingBufferHistory.h b/libraries/shared/src/RingBufferHistory.h index 7ec5bbd6ff..a9d24e44b6 100644 --- a/libraries/shared/src/RingBufferHistory.h +++ b/libraries/shared/src/RingBufferHistory.h @@ -15,6 +15,8 @@ #include #include +#include + template class RingBufferHistory { @@ -24,34 +26,9 @@ public: : _size(capacity + 1), _capacity(capacity), _newestEntryAtIndex(0), - _numEntries(0) + _numEntries(0), + _buffer(capacity + 1) { - _buffer = new T[_size]; - } - - RingBufferHistory(const RingBufferHistory& other) - : _size(other._size), - _capacity(other._capacity), - _newestEntryAtIndex(other._newestEntryAtIndex), - _numEntries(other._numEntries) - { - _buffer = new T[_size]; - memcpy(_buffer, other._buffer, _size*sizeof(T)); - } - - RingBufferHistory& operator= (const RingBufferHistory& rhs) { - _size = rhs._size; - _capacity = rhs._capacity; - _newestEntryAtIndex = rhs._newestEntryAtIndex; - _numEntries = rhs._numEntries; - delete[] _buffer; - _buffer = new T[_size]; - memcpy(_buffer, rhs._buffer, _size*sizeof(T)); - return *this; - } - - ~RingBufferHistory() { - delete[] _buffer; } void clear() { @@ -98,22 +75,16 @@ public: int getNumEntries() const { return _numEntries; } private: - T* _buffer; int _size; int _capacity; int _newestEntryAtIndex; int _numEntries; + QVector _buffer; public: class Iterator : public std::iterator < std::forward_iterator_tag, T > { public: - Iterator(T* buffer, int size, T* at) : _buffer(buffer), _bufferEnd(buffer+size), _at(at) {} - - Iterator& operator=(const Iterator& other) { - _buffer = other._buffer; - _bufferEnd = other._bufferEnd; - _at = other._at; - } + Iterator(T* bufferFirst, T* bufferLast, T* at) : _bufferFirst(bufferFirst), _bufferLast(bufferLast), _at(at) {} bool operator==(const Iterator& rhs) { return _at == rhs._at; } bool operator!=(const Iterator& rhs) { return _at != rhs._at; } @@ -121,7 +92,7 @@ public: T* operator->() { return _at; } Iterator& operator++() { - _at = (_at == _buffer) ? _bufferEnd - 1 : _at - 1; + _at = (_at == _bufferFirst) ? _bufferLast : _at - 1; return *this; } @@ -132,19 +103,19 @@ public: } private: - T* const _buffer; - T* const _bufferEnd; + T* const _bufferFirst; + T* const _bufferLast; T* _at; }; - Iterator begin() { return Iterator(_buffer, _size, &_buffer[_newestEntryAtIndex]); } + Iterator begin() { return Iterator(&_buffer.first(), &_buffer.last(), &_buffer[_newestEntryAtIndex]); } Iterator end() { int endAtIndex = _newestEntryAtIndex - _numEntries; if (endAtIndex < 0) { endAtIndex += _size; } - return Iterator(_buffer, _size, &_buffer[endAtIndex]); + return Iterator(&_buffer.first(), &_buffer.last(), &_buffer[endAtIndex]); } }; From 45421ffb0e435d0ad61b6d913dd024f0913eb497 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 10 Jul 2014 12:06:36 -0700 Subject: [PATCH 41/70] Made palm ball and disk properly use avatar scale --- interface/src/avatar/Hand.cpp | 16 ++++++++++++---- interface/src/devices/OculusManager.cpp | 1 - 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/interface/src/avatar/Hand.cpp b/interface/src/avatar/Hand.cpp index 94f734ba06..7faf1aaadc 100644 --- a/interface/src/avatar/Hand.cpp +++ b/interface/src/avatar/Hand.cpp @@ -130,6 +130,9 @@ void Hand::render(bool isMine, Model::RenderMode renderMode) { void Hand::renderHandTargets(bool isMine) { glPushMatrix(); + MyAvatar* myAvatar = Application::getInstance()->getAvatar(); + const float avatarScale = Application::getInstance()->getAvatar()->getScale(); + const float alpha = 1.0f; const glm::vec3 handColor(1.0, 0.0, 0.0); // Color the hand targets red to be different than skin @@ -153,10 +156,10 @@ void Hand::renderHandTargets(bool isMine) { } } - const float PALM_BALL_RADIUS = 0.03f; - const float PALM_DISK_RADIUS = 0.06f; - const float PALM_DISK_THICKNESS = 0.01f; - const float PALM_FINGER_ROD_RADIUS = 0.003f; + const float PALM_BALL_RADIUS = 0.03f * avatarScale; + const float PALM_DISK_RADIUS = 0.06f * avatarScale; + const float PALM_DISK_THICKNESS = 0.01f * avatarScale; + const float PALM_FINGER_ROD_RADIUS = 0.003f * avatarScale; // Draw the palm ball and disk for (size_t i = 0; i < getNumPalms(); ++i) { @@ -165,6 +168,11 @@ void Hand::renderHandTargets(bool isMine) { glColor4f(handColor.r, handColor.g, handColor.b, alpha); glm::vec3 tip = palm.getFingerTipPosition(); glm::vec3 root = palm.getPosition(); + + //Scale the positions based on avatar scale + tip = myAvatar->getPosition() + avatarScale * (tip - myAvatar->getPosition()); + root = myAvatar->getPosition() + avatarScale * (root - myAvatar->getPosition()); + Avatar::renderJointConnectingCone(root, tip, PALM_FINGER_ROD_RADIUS, PALM_FINGER_ROD_RADIUS); // Render sphere at palm/finger root glm::vec3 offsetFromPalm = root + palm.getNormal() * PALM_DISK_THICKNESS; diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index 574aa3c6f6..0ad606a320 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -473,7 +473,6 @@ void OculusManager::renderLaserPointers() { //If the Oculus is enabled, we will draw a blue cursor ray - // Draw the palm ball and disk for (size_t i = 0; i < myAvatar->getHand()->getNumPalms(); ++i) { PalmData& palm = myAvatar->getHand()->getPalms()[i]; if (palm.isActive()) { From b4c9e51011420ac7bfeb11e2facc0a83e2e2974a Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 10 Jul 2014 14:14:29 -0700 Subject: [PATCH 42/70] Audio now periodically sends downstream audio stats to audiomixer --- assignment-client/src/audio/AudioMixer.cpp | 3 +- .../src/audio/AudioMixerClientData.cpp | 11 +++- .../src/audio/AudioMixerClientData.h | 2 + interface/src/Application.cpp | 17 ++++-- interface/src/Application.h | 1 + interface/src/Audio.cpp | 54 +++++++++++++++++++ interface/src/Audio.h | 3 ++ interface/src/ui/Stats.cpp | 18 ++++--- 8 files changed, 96 insertions(+), 13 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 6cdcaef133..2ba3809729 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -405,7 +405,8 @@ void AudioMixer::readPendingDatagrams() { if (mixerPacketType == PacketTypeMicrophoneAudioNoEcho || mixerPacketType == PacketTypeMicrophoneAudioWithEcho || mixerPacketType == PacketTypeInjectAudio - || mixerPacketType == PacketTypeSilentAudioFrame) { + || mixerPacketType == PacketTypeSilentAudioFrame + || mixerPacketType == PacketTypeAudioStreamStats) { nodeList->findNodeAndUpdateWithDataFromPacket(receivedPacket); } else if (mixerPacketType == PacketTypeMuteEnvironment) { diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index ae4a0269cc..915199b443 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -83,7 +83,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { // ask the AvatarAudioRingBuffer instance to parse the data avatarRingBuffer->parseData(packet); - } else { + } else if (packetType == PacketTypeInjectAudio) { // this is injected audio // grab the stream identifier for this injected audio @@ -107,6 +107,15 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { } matchingInjectedRingBuffer->parseData(packet); + } else if (packetType == PacketTypeAudioStreamStats) { + + const char* dataAt = packet.data(); + + // skip over header, appendFlag, and num stats packed + dataAt += (numBytesPacketHeader + sizeof(quint8) + sizeof(quint16)); + + // read the downstream audio stream stats + memcpy(&_downstreamAudioStreamStats, dataAt, sizeof(AudioStreamStats)); } return 0; diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 65fd4b3da3..526071832e 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -46,6 +46,8 @@ private: quint16 _outgoingMixedAudioSequenceNumber; SequenceNumberStats _incomingAvatarAudioSequenceNumberStats; QHash _incomingInjectedAudioSequenceNumberStatsMap; + + AudioStreamStats _downstreamAudioStreamStats; }; #endif // hifi_AudioMixerClientData_h diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index d7464f57a1..8edc788833 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -172,7 +172,8 @@ Application::Application(int& argc, char** argv, QElapsedTimer &startup_time) : _runningScriptsWidget(NULL), _runningScriptsWidgetWasVisible(false), _trayIcon(new QSystemTrayIcon(_window)), - _lastNackTime(usecTimestampNow()) + _lastNackTime(usecTimestampNow()), + _lastSendDownstreamAudioStats(usecTimestampNow()) { // read the ApplicationInfo.ini file for Name/Version/Domain information QSettings applicationInfo(Application::resourcesPath() + "info/ApplicationInfo.ini", QSettings::IniFormat); @@ -2125,10 +2126,11 @@ void Application::updateMyAvatar(float deltaTime) { loadViewFrustum(_myCamera, _viewFrustum); } + quint64 now = usecTimestampNow(); + // Update my voxel servers with my current voxel query... { PerformanceTimer perfTimer("queryOctree"); - quint64 now = usecTimestampNow(); quint64 sinceLastQuery = now - _lastQueriedTime; const quint64 TOO_LONG_SINCE_LAST_QUERY = 3 * USECS_PER_SECOND; bool queryIsDue = sinceLastQuery > TOO_LONG_SINCE_LAST_QUERY; @@ -2146,7 +2148,6 @@ void Application::updateMyAvatar(float deltaTime) { // sent nack packets containing missing sequence numbers of received packets from nodes { - quint64 now = usecTimestampNow(); quint64 sinceLastNack = now - _lastNackTime; const quint64 TOO_LONG_SINCE_LAST_NACK = 1 * USECS_PER_SECOND; if (sinceLastNack > TOO_LONG_SINCE_LAST_NACK) { @@ -2154,6 +2155,16 @@ void Application::updateMyAvatar(float deltaTime) { sendNackPackets(); } } + + { + quint64 sinceLastNack = now - _lastSendDownstreamAudioStats; + const quint64 TOO_LONG_SINCE_LAST_SEND_DOWNSTREAM_AUDIO_STATS = 1 * USECS_PER_SECOND; + if (sinceLastNack > TOO_LONG_SINCE_LAST_SEND_DOWNSTREAM_AUDIO_STATS) { + _lastSendDownstreamAudioStats = now; + + QMetaObject::invokeMethod(&_audio, "sendDownstreamAudioStatsPacket", Qt::QueuedConnection); + } + } } int Application::sendNackPackets() { diff --git a/interface/src/Application.h b/interface/src/Application.h index 321a43d548..b55a830e3e 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -586,6 +586,7 @@ private: QSystemTrayIcon* _trayIcon; quint64 _lastNackTime; + quint64 _lastSendDownstreamAudioStats; }; #endif // hifi_Application_h diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 17e9054568..7a445bf816 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -782,6 +782,60 @@ void Audio::parseAudioStreamStatsPacket(const QByteArray& packet) { _incomingStreamPacketStatsHistory.insert(_incomingMixedAudioSequenceNumberStats.getStats()); } +AudioStreamStats Audio::getDownstreamAudioStreamStats() const { + + AudioStreamStats stats; + stats._streamType = PositionalAudioRingBuffer::Microphone; + + stats._timeGapMin = _interframeTimeGapStats.getMin(); + stats._timeGapMax = _interframeTimeGapStats.getMax(); + stats._timeGapAverage = _interframeTimeGapStats.getAverage(); + stats._timeGapWindowMin = _interframeTimeGapStats.getWindowMin(); + stats._timeGapWindowMax = _interframeTimeGapStats.getWindowMax(); + stats._timeGapWindowAverage = _interframeTimeGapStats.getWindowAverage(); + + stats._ringBufferFramesAvailable = _ringBuffer.framesAvailable(); + stats._ringBufferCurrentJitterBufferFrames = 0; + stats._ringBufferDesiredJitterBufferFrames = getDesiredJitterBufferFrames(); + stats._ringBufferStarveCount = _starveCount; + stats._ringBufferConsecutiveNotMixedCount = _consecutiveNotMixedCount; + stats._ringBufferOverflowCount = _ringBuffer.getOverflowCount(); + stats._ringBufferSilentFramesDropped = 0; + + stats._packetStreamStats = _incomingMixedAudioSequenceNumberStats.getStats(); + + return stats; +} + +void Audio::sendDownstreamAudioStatsPacket() { + + char packet[MAX_PACKET_SIZE]; + + // pack header + int numBytesPacketHeader = populatePacketHeader(packet, PacketTypeAudioStreamStats); + char* dataAt = packet + numBytesPacketHeader; + + // pack append flag + quint8 appendFlag = 0; + memcpy(dataAt, &appendFlag, sizeof(quint8)); + dataAt += sizeof(quint8); + + // pack number of stats packed + quint16 numStreamStatsToPack = 1; + memcpy(dataAt, &numStreamStatsToPack, sizeof(quint16)); + dataAt += sizeof(quint16); + + // pack downstream audio stream stats + AudioStreamStats stats = getDownstreamAudioStreamStats(); + memcpy(dataAt, &stats, sizeof(AudioStreamStats)); + dataAt += sizeof(AudioStreamStats); + + // send packet + NodeList* nodeList = NodeList::getInstance(); + SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); + nodeList->writeDatagram(packet, dataAt - packet, audioMixer); +} + // NOTE: numSamples is the total number of single channel samples, since callers will always call this with stereo // data we know that we will have 2x samples for each stereo time sample at the format's sample rate void Audio::addSpatialAudioToBuffer(unsigned int sampleTime, const QByteArray& spatialAudio, unsigned int numSamples) { diff --git a/interface/src/Audio.h b/interface/src/Audio.h index e6e06838d3..e8e92db1a0 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -99,6 +99,9 @@ public slots: virtual void handleAudioByteArray(const QByteArray& audioByteArray); + AudioStreamStats getDownstreamAudioStreamStats() const; + void sendDownstreamAudioStatsPacket(); + bool switchInputToAudioDevice(const QString& inputDeviceName); bool switchOutputToAudioDevice(const QString& outputDeviceName); QString getDeviceName(QAudio::Mode mode) const { return (mode == QAudio::AudioInput) ? diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 0cd9e3fa05..add0754fe1 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -349,24 +349,26 @@ void Stats::display( char downstreamAudioStatsString[30]; + AudioStreamStats downstreamAudioStreamStats = audio->getDownstreamAudioStreamStats(); + audio->calculatePacketLossRate(audio->getIncomingStreamPacketStatsHistory(), packetLossRate, packetLossRate30s); - sprintf(downstreamAudioStatsString, " mix: %.1f%%/%.1f%%, %d/?/%d", packetLossRate*100.0f, packetLossRate30s*100.0f, - audio->getDownstreamRingBuffer().framesAvailable(), audio->getDesiredJitterBufferFrames()); + sprintf(downstreamAudioStatsString, " mix: %.1f%%/%.1f%%, %u/?/%u", packetLossRate*100.0f, packetLossRate30s*100.0f, + downstreamAudioStreamStats._ringBufferFramesAvailable, downstreamAudioStreamStats._ringBufferDesiredJitterBufferFrames); verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); - const MovingMinMaxAvg& timeGapStats = audio->getInterframeTimeGapStats(); - - sprintf(downstreamAudioStatsString, " %llu/%llu/%.2f, %d/%d", timeGapStats.getMin(), timeGapStats.getMax(), - timeGapStats.getAverage(), audio->getStarveCount(), audio->getDownstreamRingBuffer().getOverflowCount()); + sprintf(downstreamAudioStatsString, " %llu/%llu/%.2f, %u/%u", downstreamAudioStreamStats._timeGapMin, + downstreamAudioStreamStats._timeGapMax, downstreamAudioStreamStats._timeGapAverage, + downstreamAudioStreamStats._ringBufferStarveCount, downstreamAudioStreamStats._ringBufferOverflowCount); verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); - sprintf(downstreamAudioStatsString, " %llu/%llu/%.2f, %d/?", timeGapStats.getWindowMin(), timeGapStats.getWindowMax(), - timeGapStats.getWindowAverage(), audio->getConsecutiveNotMixedCount()); + sprintf(downstreamAudioStatsString, " %llu/%llu/%.2f, %u/?", downstreamAudioStreamStats._timeGapWindowMin, + downstreamAudioStreamStats._timeGapWindowMax, downstreamAudioStreamStats._timeGapWindowAverage, + downstreamAudioStreamStats._ringBufferConsecutiveNotMixedCount); verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); From fc21f7bbacbcce8009df663aaed2a48287f72c8b Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 10 Jul 2014 14:21:17 -0700 Subject: [PATCH 43/70] Removed outdated and incorrect getFingerTipPosition() --- interface/src/BuckyBalls.cpp | 2 +- interface/src/avatar/Hand.cpp | 4 ++-- libraries/avatars/src/HandData.cpp | 6 ------ libraries/avatars/src/HandData.h | 1 - 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/interface/src/BuckyBalls.cpp b/interface/src/BuckyBalls.cpp index 68d1167071..1bc093283a 100644 --- a/interface/src/BuckyBalls.cpp +++ b/interface/src/BuckyBalls.cpp @@ -60,7 +60,7 @@ BuckyBalls::BuckyBalls() { void BuckyBalls::grab(PalmData& palm, float deltaTime) { float penetration; - glm::vec3 fingerTipPosition = palm.getFingerTipPosition(); + glm::vec3 fingerTipPosition = palm.getTipPosition(); if (palm.getControllerButtons() & BUTTON_FWD) { if (!_bballIsGrabbed[palm.getSixenseID()]) { diff --git a/interface/src/avatar/Hand.cpp b/interface/src/avatar/Hand.cpp index 7faf1aaadc..aae9cb0a29 100644 --- a/interface/src/avatar/Hand.cpp +++ b/interface/src/avatar/Hand.cpp @@ -145,7 +145,7 @@ void Hand::renderHandTargets(bool isMine) { if (!palm.isActive()) { continue; } - glm::vec3 targetPosition = palm.getFingerTipPosition(); + glm::vec3 targetPosition = palm.getTipPosition(); glPushMatrix(); glTranslatef(targetPosition.x, targetPosition.y, targetPosition.z); @@ -166,7 +166,7 @@ void Hand::renderHandTargets(bool isMine) { PalmData& palm = getPalms()[i]; if (palm.isActive()) { glColor4f(handColor.r, handColor.g, handColor.b, alpha); - glm::vec3 tip = palm.getFingerTipPosition(); + glm::vec3 tip = palm.getTipPosition(); glm::vec3 root = palm.getPosition(); //Scale the positions based on avatar scale diff --git a/libraries/avatars/src/HandData.cpp b/libraries/avatars/src/HandData.cpp index e6ef10ed3e..69f9669973 100644 --- a/libraries/avatars/src/HandData.cpp +++ b/libraries/avatars/src/HandData.cpp @@ -108,12 +108,6 @@ glm::quat HandData::getBaseOrientation() const { glm::vec3 HandData::getBasePosition() const { return _owningAvatarData->getPosition(); } - -glm::vec3 PalmData::getFingerTipPosition() const { - glm::vec3 fingerOffset(0.0f, 0.0f, 0.3f); - glm::vec3 palmOffset(0.0f, -0.08f, 0.0f); - return getPosition() + _owningHandData->localToWorldDirection(_rawRotation * (fingerOffset + palmOffset)); -} glm::vec3 PalmData::getFingerDirection() const { const glm::vec3 LOCAL_FINGER_DIRECTION(0.0f, 0.0f, 1.0f); diff --git a/libraries/avatars/src/HandData.h b/libraries/avatars/src/HandData.h index 404d00b0f3..770a44197d 100755 --- a/libraries/avatars/src/HandData.h +++ b/libraries/avatars/src/HandData.h @@ -140,7 +140,6 @@ public: void getBallHoldPosition(glm::vec3& position) const; // return world-frame: - glm::vec3 getFingerTipPosition() const; glm::vec3 getFingerDirection() const; glm::vec3 getNormal() const; From 8bc92144453e486b614662daea5df829aa8cf416 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 10 Jul 2014 14:37:04 -0700 Subject: [PATCH 44/70] Added helper scaling function to Avatar. --- interface/src/avatar/Avatar.cpp | 5 +++++ interface/src/avatar/Avatar.h | 4 ++++ interface/src/avatar/Hand.cpp | 4 ++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index 3221e75e56..2ebbd55c85 100644 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -970,6 +970,11 @@ glm::quat Avatar::getJointCombinedRotation(const QString& name) const { return rotation; } +void Avatar::scaleVectorRelativeToPosition(glm::vec3 &positionToScale) const { + //Scale a world space vector as if it was relative to the position + positionToScale = _position + _scale * (positionToScale - _position); +} + void Avatar::setFaceModelURL(const QUrl& faceModelURL) { AvatarData::setFaceModelURL(faceModelURL); const QUrl DEFAULT_FACE_MODEL_URL = QUrl::fromLocalFile(Application::resourcesPath() + "meshes/defaultAvatar_head.fst"); diff --git a/interface/src/avatar/Avatar.h b/interface/src/avatar/Avatar.h index 8c14fc9ed0..6fb7991c3c 100755 --- a/interface/src/avatar/Avatar.h +++ b/interface/src/avatar/Avatar.h @@ -152,6 +152,10 @@ public: glm::vec3 getAcceleration() const { return _acceleration; } glm::vec3 getAngularVelocity() const { return _angularVelocity; } + + /// Scales a world space position vector relative to the avatar position and scale + /// \param vector position to be scaled. Will store the result + void scaleVectorRelativeToPosition(glm::vec3 &positionToScale) const; public slots: void updateCollisionGroups(); diff --git a/interface/src/avatar/Hand.cpp b/interface/src/avatar/Hand.cpp index aae9cb0a29..08e1cf83df 100644 --- a/interface/src/avatar/Hand.cpp +++ b/interface/src/avatar/Hand.cpp @@ -170,8 +170,8 @@ void Hand::renderHandTargets(bool isMine) { glm::vec3 root = palm.getPosition(); //Scale the positions based on avatar scale - tip = myAvatar->getPosition() + avatarScale * (tip - myAvatar->getPosition()); - root = myAvatar->getPosition() + avatarScale * (root - myAvatar->getPosition()); + myAvatar->scaleVectorRelativeToPosition(tip); + myAvatar->scaleVectorRelativeToPosition(root); Avatar::renderJointConnectingCone(root, tip, PALM_FINGER_ROD_RADIUS, PALM_FINGER_ROD_RADIUS); // Render sphere at palm/finger root From ce0101aa2a95825ea0cbfced16d7bb72c254fdeb Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 10 Jul 2014 14:53:29 -0700 Subject: [PATCH 45/70] fixed octreeEditPacketSender seq numbers errors; untested --- .../octree/src/OctreeEditPacketSender.cpp | 24 ++++++++++--------- libraries/octree/src/OctreeEditPacketSender.h | 6 ++--- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/libraries/octree/src/OctreeEditPacketSender.cpp b/libraries/octree/src/OctreeEditPacketSender.cpp index d67306e8c7..2ed8f6c2a0 100644 --- a/libraries/octree/src/OctreeEditPacketSender.cpp +++ b/libraries/octree/src/OctreeEditPacketSender.cpp @@ -34,7 +34,6 @@ OctreeEditPacketSender::OctreeEditPacketSender() : _maxPendingMessages(DEFAULT_MAX_PENDING_MESSAGES), _releaseQueuedMessagesPending(false), _serverJurisdictions(NULL), - _sequenceNumber(0), _maxPacketSize(MAX_PACKET_SIZE) { } @@ -88,7 +87,7 @@ bool OctreeEditPacketSender::serversExist() const { // This method is called when the edit packet layer has determined that it has a fully formed packet destined for // a known nodeID. -void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, const unsigned char* buffer, ssize_t length) { +void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, unsigned char* buffer, ssize_t length) { NodeList* nodeList = NodeList::getInstance(); foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { @@ -96,13 +95,18 @@ void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, const unsi if (node->getType() == getMyNodeType() && ((node->getUUID() == nodeUUID) || (nodeUUID.isNull()))) { if (node->getActiveSocket()) { + + // pack sequence number + int numBytesPacketHeader = numBytesForPacketHeader(reinterpret_cast(buffer)); + unsigned char* sequenceAt = buffer + numBytesPacketHeader; + quint16 sequence = _outgoingSequenceNumbers[nodeUUID]++; + memcpy(sequenceAt, &sequence, sizeof(quint16)); + + // send packet QByteArray packet(reinterpret_cast(buffer), length); queuePacketForSending(node, packet); - // extract sequence number and add packet to history - int numBytesPacketHeader = numBytesForPacketHeader(packet); - const char* dataAt = reinterpret_cast(packet.data()) + numBytesPacketHeader; - unsigned short int sequence = *((unsigned short int*)dataAt); + // add packet to history _sentPacketHistories[nodeUUID].packetSent(sequence, packet); // debugging output... @@ -312,11 +316,8 @@ void OctreeEditPacketSender::releaseQueuedPacket(EditPacketBuffer& packetBuffer) void OctreeEditPacketSender::initializePacket(EditPacketBuffer& packetBuffer, PacketType type) { packetBuffer._currentSize = populatePacketHeader(reinterpret_cast(&packetBuffer._currentBuffer[0]), type); - // pack in sequence numbers - unsigned short int* sequenceAt = (unsigned short int*)&packetBuffer._currentBuffer[packetBuffer._currentSize]; - *sequenceAt = _sequenceNumber; - packetBuffer._currentSize += sizeof(unsigned short int); // nudge past sequence - _sequenceNumber++; + // skip over sequence number for now; will be packed when packet is ready to be sent out + packetBuffer._currentSize += sizeof(quint16); // pack in timestamp quint64 now = usecTimestampNow(); @@ -373,5 +374,6 @@ void OctreeEditPacketSender::nodeKilled(SharedNodePointer node) { // TODO: add locks QUuid nodeUUID = node->getUUID(); _pendingEditPackets.remove(nodeUUID); + _outgoingSequenceNumbers.remove(nodeUUID); _sentPacketHistories.remove(nodeUUID); } diff --git a/libraries/octree/src/OctreeEditPacketSender.h b/libraries/octree/src/OctreeEditPacketSender.h index c16c0a2d4b..cdcfc21d4a 100644 --- a/libraries/octree/src/OctreeEditPacketSender.h +++ b/libraries/octree/src/OctreeEditPacketSender.h @@ -21,7 +21,7 @@ /// Used for construction of edit packets class EditPacketBuffer { public: - EditPacketBuffer() : _nodeUUID(), _currentType(PacketTypeUnknown), _currentSize(0) { } + EditPacketBuffer() : _nodeUUID(), _currentType(PacketTypeUnknown), _currentSize(0) { } EditPacketBuffer(PacketType type, unsigned char* codeColorBuffer, ssize_t length, const QUuid nodeUUID = QUuid()); QUuid _nodeUUID; PacketType _currentType; @@ -100,7 +100,7 @@ public: protected: bool _shouldSend; - void queuePacketToNode(const QUuid& nodeID, const unsigned char* buffer, ssize_t length); + void queuePacketToNode(const QUuid& nodeID, unsigned char* buffer, ssize_t length); void queuePendingPacketToNodes(PacketType type, unsigned char* buffer, ssize_t length); void queuePacketToNodes(unsigned char* buffer, ssize_t length); void initializePacket(EditPacketBuffer& packetBuffer, PacketType type); @@ -120,12 +120,12 @@ protected: NodeToJurisdictionMap* _serverJurisdictions; - unsigned short int _sequenceNumber; int _maxPacketSize; QMutex _releaseQueuedPacketMutex; // TODO: add locks for this and _pendingEditPackets QHash _sentPacketHistories; + QHash _outgoingSequenceNumbers; }; #endif // hifi_OctreeEditPacketSender_h From 817b50c7b8fd9a068a9ef841cb752ff3840f29c7 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 10 Jul 2014 15:34:45 -0700 Subject: [PATCH 46/70] Show upload/download stats for all reliable messages, not just reliable delta downloads. --- interface/src/ui/Stats.cpp | 22 +++++++------ .../metavoxels/src/DatagramSequencer.cpp | 32 ++++++++++++++++++- libraries/metavoxels/src/DatagramSequencer.h | 9 ++++++ libraries/metavoxels/src/Endpoint.h | 2 ++ .../metavoxels/src/MetavoxelClientManager.cpp | 4 --- .../metavoxels/src/MetavoxelClientManager.h | 2 -- 6 files changed, 54 insertions(+), 17 deletions(-) diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index b0cbea14a9..0d2762f923 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -422,18 +422,15 @@ void Stats::display( int internal = 0, leaves = 0; - int received = 0, total = 0; + int sendProgress = 0, sendTotal = 0; + int receiveProgress = 0, receiveTotal = 0; foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { if (node->getType() == NodeType::MetavoxelServer) { QMutexLocker locker(&node->getMutex()); MetavoxelClient* client = static_cast(node->getLinkedData()); if (client) { client->getData().countNodes(internal, leaves, Application::getInstance()->getMetavoxels()->getLOD()); - int clientReceived = 0, clientTotal = 0; - if (client->getReliableDeltaProgress(clientReceived, clientTotal)) { - received += clientReceived; - total += clientTotal; - } + client->getSequencer().addReliableChannelStats(sendProgress, sendTotal, receiveProgress, receiveTotal); } } } @@ -447,11 +444,16 @@ void Stats::display( verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, nodeTypes.str().c_str(), color); - if (total > 0) { - stringstream reliableDelta; - reliableDelta << "Reliable Delta: " << (received * 100 / total) << "%"; + if (sendTotal > 0 || receiveTotal > 0) { + stringstream reliableStats; + if (sendTotal > 0) { + reliableStats << "Upload: " << (sendProgress * 100 / sendTotal) << "% "; + } + if (receiveTotal > 0) { + reliableStats << "Download: " << (receiveProgress * 100 / receiveTotal) << "%"; + } verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, reliableDelta.str().c_str(), color); + drawText(horizontalOffset, verticalOffset, scale, rotation, font, reliableStats.str().c_str(), color); } } diff --git a/libraries/metavoxels/src/DatagramSequencer.cpp b/libraries/metavoxels/src/DatagramSequencer.cpp index 8c58233eb9..c757e131bb 100644 --- a/libraries/metavoxels/src/DatagramSequencer.cpp +++ b/libraries/metavoxels/src/DatagramSequencer.cpp @@ -79,6 +79,24 @@ ReliableChannel* DatagramSequencer::getReliableInputChannel(int index) { return channel; } +void DatagramSequencer::addReliableChannelStats(int& sendProgress, int& sendTotal, + int& receiveProgress, int& receiveTotal) const { + foreach (ReliableChannel* channel, _reliableOutputChannels) { + int sent, total; + if (channel->getMessageSendProgress(sent, total)) { + sendProgress += sent; + sendTotal += total; + } + } + foreach (ReliableChannel* channel, _reliableInputChannels) { + int received, total; + if (channel->getMessageReceiveProgress(received, total)) { + receiveProgress += received; + receiveTotal += total; + } + } +} + int DatagramSequencer::notePacketGroup(int desiredPackets) { // figure out how much data we have enqueued and increase the number of packets desired int totalAvailable = 0; @@ -684,6 +702,8 @@ void ReliableChannel::endMessage() { quint32 length = _buffer.pos() - _messageLengthPlaceholder; _buffer.writeBytes(_messageLengthPlaceholder, sizeof(quint32), (const char*)&length); + _messageReceivedOffset = getBytesWritten(); + _messageSize = length; } void ReliableChannel::sendMessage(const QVariant& message) { @@ -692,6 +712,15 @@ void ReliableChannel::sendMessage(const QVariant& message) { endMessage(); } +bool ReliableChannel::getMessageSendProgress(int& sent, int& total) const { + if (!_messagesEnabled || _offset >= _messageReceivedOffset) { + return false; + } + sent = qMax(0, _messageSize - (_messageReceivedOffset - _offset)); + total = _messageSize; + return true; +} + bool ReliableChannel::getMessageReceiveProgress(int& received, int& total) const { if (!_messagesEnabled || _buffer.bytesAvailable() < (int)sizeof(quint32)) { return false; @@ -728,7 +757,8 @@ ReliableChannel::ReliableChannel(DatagramSequencer* sequencer, int index, bool o _offset(0), _writePosition(0), _writePositionResetPacketNumber(0), - _messagesEnabled(true) { + _messagesEnabled(true), + _messageReceivedOffset(0) { _buffer.open(output ? QIODevice::WriteOnly : QIODevice::ReadOnly); _dataStream.setByteOrder(QDataStream::LittleEndian); diff --git a/libraries/metavoxels/src/DatagramSequencer.h b/libraries/metavoxels/src/DatagramSequencer.h index 998e196a05..4a01679c68 100644 --- a/libraries/metavoxels/src/DatagramSequencer.h +++ b/libraries/metavoxels/src/DatagramSequencer.h @@ -108,6 +108,9 @@ public: /// Returns the intput channel at the specified index, creating it if necessary. ReliableChannel* getReliableInputChannel(int index = 0); + /// Adds stats for all reliable channels to the referenced variables. + void addReliableChannelStats(int& sendProgress, int& sendTotal, int& receiveProgress, int& receiveTotal) const; + /// Notes that we're sending a group of packets. /// \param desiredPackets the number of packets we'd like to write in the group /// \return the number of packets to write in the group @@ -376,6 +379,10 @@ public: /// writes the message to the bitstream, then calls endMessage). void sendMessage(const QVariant& message); + /// Determines the number of bytes uploaded towards the currently pending message. + /// \return true if there is a message pending, in which case the sent and total arguments will be set + bool getMessageSendProgress(int& sent, int& total) const; + /// Determines the number of bytes downloaded towards the currently pending message. /// \return true if there is a message pending, in which case the received and total arguments will be set bool getMessageReceiveProgress(int& received, int& total) const; @@ -420,6 +427,8 @@ private: SpanList _acknowledged; bool _messagesEnabled; int _messageLengthPlaceholder; + int _messageReceivedOffset; + int _messageSize; }; #endif // hifi_DatagramSequencer_h diff --git a/libraries/metavoxels/src/Endpoint.h b/libraries/metavoxels/src/Endpoint.h index 3c681a7b98..d6999196d8 100644 --- a/libraries/metavoxels/src/Endpoint.h +++ b/libraries/metavoxels/src/Endpoint.h @@ -32,6 +32,8 @@ public: PacketRecord* baselineReceiveRecord = NULL); virtual ~Endpoint(); + const DatagramSequencer& getSequencer() const { return _sequencer; } + virtual void update(); virtual int parseData(const QByteArray& packet); diff --git a/libraries/metavoxels/src/MetavoxelClientManager.cpp b/libraries/metavoxels/src/MetavoxelClientManager.cpp index 94d9116794..f3ea1ae8c5 100644 --- a/libraries/metavoxels/src/MetavoxelClientManager.cpp +++ b/libraries/metavoxels/src/MetavoxelClientManager.cpp @@ -94,10 +94,6 @@ MetavoxelClient::MetavoxelClient(const SharedNodePointer& node, MetavoxelClientM SIGNAL(receivedMessage(const QVariant&, Bitstream&)), SLOT(handleMessage(const QVariant&, Bitstream&))); } -bool MetavoxelClient::getReliableDeltaProgress(int& received, int& total) const { - return _reliableDeltaChannel && _reliableDeltaChannel->getMessageReceiveProgress(received, total); -} - void MetavoxelClient::guide(MetavoxelVisitor& visitor) { visitor.setLOD(_manager->getLOD()); _data.guide(visitor); diff --git a/libraries/metavoxels/src/MetavoxelClientManager.h b/libraries/metavoxels/src/MetavoxelClientManager.h index 809718aa01..ad6c86c8fc 100644 --- a/libraries/metavoxels/src/MetavoxelClientManager.h +++ b/libraries/metavoxels/src/MetavoxelClientManager.h @@ -53,8 +53,6 @@ public: MetavoxelData& getData() { return _data; } - bool getReliableDeltaProgress(int& received, int& total) const; - void guide(MetavoxelVisitor& visitor); void applyEdit(const MetavoxelEditMessage& edit, bool reliable = false); From d858e20af25b2a72d595c2b23c2eaddb7a826fbf Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 10 Jul 2014 16:02:53 -0700 Subject: [PATCH 47/70] For now, tie the metavoxel LOD threshold to the avatar LOD parameter. --- interface/src/MetavoxelSystem.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/interface/src/MetavoxelSystem.cpp b/interface/src/MetavoxelSystem.cpp index 7a5119a62d..66933643ae 100644 --- a/interface/src/MetavoxelSystem.cpp +++ b/interface/src/MetavoxelSystem.cpp @@ -48,8 +48,10 @@ void MetavoxelSystem::init() { } MetavoxelLOD MetavoxelSystem::getLOD() const { - const float FIXED_LOD_THRESHOLD = 0.01f; - return MetavoxelLOD(Application::getInstance()->getCamera()->getPosition(), FIXED_LOD_THRESHOLD); + // the LOD threshold is temporarily tied to the avatar LOD parameter + const float BASE_LOD_THRESHOLD = 0.01f; + return MetavoxelLOD(Application::getInstance()->getCamera()->getPosition(), + BASE_LOD_THRESHOLD * Menu::getInstance()->getAvatarLODDistanceMultiplier()); } void MetavoxelSystem::simulate(float deltaTime) { From 4bfd393d028674face241aca774bd379bb8f565a Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 10 Jul 2014 16:42:00 -0700 Subject: [PATCH 48/70] Proper scaling for oculus UI using avatar scale --- interface/src/devices/OculusManager.cpp | 12 +++++++++- interface/src/ui/ApplicationOverlay.cpp | 31 +++++++++++++++---------- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index 0ad606a320..beb18bfc72 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -479,6 +479,10 @@ void OculusManager::renderLaserPointers() { glColor4f(0, 1, 1, 1); glm::vec3 tip = getLaserPointerTipPosition(&palm); glm::vec3 root = palm.getPosition(); + + //Scale the root vector with the avatar scale + myAvatar->scaleVectorRelativeToPosition(root); + Avatar::renderJointConnectingCone(root, tip, PALM_TIP_ROD_RADIUS, PALM_TIP_ROD_RADIUS); } } @@ -492,8 +496,14 @@ glm::vec3 OculusManager::getLaserPointerTipPosition(const PalmData* palm) { const float PALM_TIP_ROD_LENGTH_MULT = 40.0f; glm::vec3 direction = glm::normalize(palm->getTipPosition() - palm->getPosition()); + + glm::vec3 position = palm->getPosition(); + //scale the position with the avatar + Application::getInstance()->getAvatar()->scaleVectorRelativeToPosition(position); + + glm::vec3 result; - if (applicationOverlay.calculateRayUICollisionPoint(palm->getPosition(), direction, result)) { + if (applicationOverlay.calculateRayUICollisionPoint(position, direction, result)) { return result; } diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 005858fd4d..6d1e45df20 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -266,7 +266,7 @@ bool raySphereIntersect(const glm::vec3 &dir, const glm::vec3 &origin, float r, } } - +//Caculate the click location using one of the sixense controllers. Scale is not applied QPoint ApplicationOverlay::getOculusPalmClickLocation(const PalmData *palm) const { Application* application = Application::getInstance(); @@ -289,7 +289,7 @@ QPoint ApplicationOverlay::getOculusPalmClickLocation(const PalmData *palm) cons //We back the ray up by dir to ensure that it will not start inside the UI. glm::vec3 adjustedPos = tipPos - dir; //Find intersection of crosshair ray. - if (raySphereIntersect(dir, adjustedPos, _oculusuiRadius, &t)){ + if (raySphereIntersect(dir, adjustedPos, _oculusuiRadius * myAvatar->getScale(), &t)){ glm::vec3 collisionPos = adjustedPos + dir * t; //Normalize it in case its not a radius of 1 collisionPos = glm::normalize(collisionPos); @@ -313,6 +313,7 @@ QPoint ApplicationOverlay::getOculusPalmClickLocation(const PalmData *palm) cons return rv; } +//Finds the collision point of a world space ray bool ApplicationOverlay::calculateRayUICollisionPoint(const glm::vec3& position, const glm::vec3& direction, glm::vec3& result) const { Application* application = Application::getInstance(); MyAvatar* myAvatar = application->getAvatar(); @@ -323,7 +324,7 @@ bool ApplicationOverlay::calculateRayUICollisionPoint(const glm::vec3& position, glm::vec3 relativeDirection = orientation * direction; float t; - if (raySphereIntersect(relativeDirection, relativePosition, _oculusuiRadius, &t)){ + if (raySphereIntersect(relativeDirection, relativePosition, _oculusuiRadius * myAvatar->getScale(), &t)){ result = position + direction * t; return true; } @@ -862,18 +863,22 @@ void ApplicationOverlay::renderMagnifier(int mouseX, int mouseY, float sizeMult, float newVTop = 1.0 - (newMouseY - newHeight) / widgetHeight; // Project our position onto the hemisphere using the UV coordinates - float lX = sin((newULeft - 0.5f) * _textureFov); - float rX = sin((newURight - 0.5f) * _textureFov); - float bY = sin((newVBottom - 0.5f) * _textureFov); - float tY = sin((newVTop - 0.5f) * _textureFov); + float radius = _oculusuiRadius * application->getAvatar()->getScale(); + float radius2 = radius * radius; + + float lX = radius * sin((newULeft - 0.5f) * _textureFov); + float rX = radius * sin((newURight - 0.5f) * _textureFov); + float bY = radius * sin((newVBottom - 0.5f) * _textureFov); + float tY = radius * sin((newVTop - 0.5f) * _textureFov); float blZ, tlZ, brZ, trZ; float dist; float discriminant; + //Bottom Left dist = sqrt(lX * lX + bY * bY); - discriminant = 1.0f - dist * dist; + discriminant = radius2 - dist * dist; if (discriminant > 0) { blZ = sqrt(discriminant); } else { @@ -881,7 +886,7 @@ void ApplicationOverlay::renderMagnifier(int mouseX, int mouseY, float sizeMult, } //Top Left dist = sqrt(lX * lX + tY * tY); - discriminant = 1.0f - dist * dist; + discriminant = radius2 - dist * dist; if (discriminant > 0) { tlZ = sqrt(discriminant); } else { @@ -889,7 +894,7 @@ void ApplicationOverlay::renderMagnifier(int mouseX, int mouseY, float sizeMult, } //Bottom Right dist = sqrt(rX * rX + bY * bY); - discriminant = 1.0f - dist * dist; + discriminant = radius2 - dist * dist; if (discriminant > 0) { brZ = sqrt(discriminant); } else { @@ -897,7 +902,7 @@ void ApplicationOverlay::renderMagnifier(int mouseX, int mouseY, float sizeMult, } //Top Right dist = sqrt(rX * rX + tY * tY); - discriminant = 1.0f - dist * dist; + discriminant = radius2 - dist * dist; if (discriminant > 0) { trZ = sqrt(discriminant); } else { @@ -1198,7 +1203,9 @@ void ApplicationOverlay::renderTexturedHemisphere() { glTranslatef(position.x, position.y, position.z); glMultMatrixf(&rotation[0][0]); - glScalef(_oculusuiRadius, _oculusuiRadius, _oculusuiRadius); + + const float scale = _oculusuiRadius * myAvatar->getScale(); + glScalef(scale, scale, scale); glDrawRangeElements(GL_TRIANGLES, 0, vertices - 1, indices, GL_UNSIGNED_SHORT, 0); From 25f4f63a1e24a14fa7292e18ea4c46edeff7f827 Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 10 Jul 2014 16:42:23 -0700 Subject: [PATCH 49/70] added window PacketStreamStats to AudioStreamStats --- assignment-client/src/audio/AudioMixer.cpp | 3 - assignment-client/src/audio/AudioMixer.h | 2 + .../src/audio/AudioMixerClientData.cpp | 56 +++++++++++++++---- .../src/audio/AudioMixerClientData.h | 5 +- libraries/audio/src/AudioStreamStats.h | 4 +- .../networking/src/SequenceNumberStats.cpp | 29 +++++++++- .../networking/src/SequenceNumberStats.h | 15 ++++- libraries/shared/src/RingBufferHistory.h | 4 +- 8 files changed, 97 insertions(+), 21 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 2ba3809729..459f8a4b59 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -641,9 +641,6 @@ void AudioMixer::run() { ++framesSinceCutoffEvent; } - - const quint64 TOO_LONG_SINCE_LAST_SEND_AUDIO_STREAM_STATS = 1 * USECS_PER_SECOND; - bool sendAudioStreamStats = false; quint64 now = usecTimestampNow(); if (now - _lastSendAudioStreamStatsTime > TOO_LONG_SINCE_LAST_SEND_AUDIO_STREAM_STATS) { diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 2c94f32edc..afab7d47dc 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -21,6 +21,8 @@ class AvatarAudioRingBuffer; const int SAMPLE_PHASE_DELAY_AT_90 = 20; +const quint64 TOO_LONG_SINCE_LAST_SEND_AUDIO_STREAM_STATS = 1 * USECS_PER_SECOND; + /// Handles assignments of type AudioMixer - mixing streams of audio and re-distributing to various clients. class AudioMixer : public ThreadedAssignment { Q_OBJECT diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 915199b443..d0b5104dbd 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -20,10 +20,13 @@ #include "AudioMixerClientData.h" #include "MovingMinMaxAvg.h" +const int INCOMING_SEQ_STATS_HISTORY_LENGTH = INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS / + (TOO_LONG_SINCE_LAST_SEND_AUDIO_STREAM_STATS / USECS_PER_SECOND); + AudioMixerClientData::AudioMixerClientData() : _ringBuffers(), _outgoingMixedAudioSequenceNumber(0), - _incomingAvatarAudioSequenceNumberStats() + _incomingAvatarAudioSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH) { } @@ -89,6 +92,9 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { // grab the stream identifier for this injected audio QUuid streamIdentifier = QUuid::fromRfc4122(packet.mid(numBytesForPacketHeader(packet) + sizeof(quint16), NUM_BYTES_RFC4122_UUID)); + if (!_incomingInjectedAudioSequenceNumberStatsMap.contains(streamIdentifier)) { + _incomingInjectedAudioSequenceNumberStatsMap.insert(streamIdentifier, SequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH)); + } _incomingInjectedAudioSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence); InjectedAudioRingBuffer* matchingInjectedRingBuffer = NULL; @@ -175,11 +181,14 @@ AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const Positio streamStats._streamType = ringBuffer->getType(); if (streamStats._streamType == PositionalAudioRingBuffer::Injector) { streamStats._streamIdentifier = ((InjectedAudioRingBuffer*)ringBuffer)->getStreamIdentifier(); - streamStats._packetStreamStats = _incomingInjectedAudioSequenceNumberStatsMap[streamStats._streamIdentifier].getStats(); + const SequenceNumberStats& sequenceNumberStats = _incomingInjectedAudioSequenceNumberStatsMap[streamStats._streamIdentifier]; + streamStats._packetStreamStats = sequenceNumberStats.getStats(); + streamStats._packetStreamWindowStats = sequenceNumberStats.getStatsForHistoryWindow(); } else { streamStats._packetStreamStats = _incomingAvatarAudioSequenceNumberStats.getStats(); + streamStats._packetStreamWindowStats = _incomingAvatarAudioSequenceNumberStats.getStatsForHistoryWindow(); } - + const MovingMinMaxAvg& timeGapStats = ringBuffer->getInterframeTimeGapStatsForStatsPacket(); streamStats._timeGapMin = timeGapStats.getMin(); streamStats._timeGapMax = timeGapStats.getMax(); @@ -199,8 +208,18 @@ AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const Positio return streamStats; } -void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode) const { +void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode) { + // have all the seq number stats of each audio stream push their current stats into their history, + // which moves that history window 1 second forward (since that's how long since the last stats were pushed into history) + _incomingAvatarAudioSequenceNumberStats.pushStatsToHistory(); + QHash::Iterator i = _incomingInjectedAudioSequenceNumberStatsMap.begin(); + QHash::Iterator end = _incomingInjectedAudioSequenceNumberStatsMap.end(); + while (i != end) { + i.value().pushStatsToHistory(); + i++; + } + char packet[MAX_PACKET_SIZE]; NodeList* nodeList = NodeList::getInstance(); @@ -251,6 +270,23 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& QString AudioMixerClientData::getAudioStreamStatsString() const { QString result; + AudioStreamStats streamStats = _downstreamAudioStreamStats; + result += "downstream.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) + + " current: ?" + + " available:" + QString::number(streamStats._ringBufferFramesAvailable) + + " starves:" + QString::number(streamStats._ringBufferStarveCount) + + " not mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + + " overflows:" + QString::number(streamStats._ringBufferOverflowCount) + + " silents dropped: ?" + + " lost %:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'g', 2) + + " lost % 30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'g', 2) + + " min gap:" + QString::number(streamStats._timeGapMin) + + " max gap:" + QString::number(streamStats._timeGapMax) + + " avg gap:" + QString::number(streamStats._timeGapAverage, 'g', 2) + + " min 30s gap:" + QString::number(streamStats._timeGapWindowMin) + + " max 30s gap:" + QString::number(streamStats._timeGapWindowMax) + + " avg 30s gap:" + QString::number(streamStats._timeGapWindowAverage, 'g', 2); + AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); if (avatarRingBuffer) { AudioStreamStats streamStats = getAudioStreamStatsOfStream(avatarRingBuffer); @@ -261,9 +297,8 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { + " not mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + " overflows:" + QString::number(streamStats._ringBufferOverflowCount) + " silents dropped:" + QString::number(streamStats._ringBufferSilentFramesDropped) - + " early:" + QString::number(streamStats._packetStreamStats._numEarly) - + " late:" + QString::number(streamStats._packetStreamStats._numLate) - + " lost:" + QString::number(streamStats._packetStreamStats._numLost) + + " lost %:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'g', 2) + + " lost % 30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'g', 2) + " min gap:" + QString::number(streamStats._timeGapMin) + " max gap:" + QString::number(streamStats._timeGapMax) + " avg gap:" + QString::number(streamStats._timeGapAverage, 'g', 2) @@ -277,16 +312,15 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { for (int i = 0; i < _ringBuffers.size(); i++) { if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) { AudioStreamStats streamStats = getAudioStreamStatsOfStream(_ringBuffers[i]); - result += "mic.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) + result += "inj.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) + " current:" + QString::number(streamStats._ringBufferCurrentJitterBufferFrames) + " available:" + QString::number(streamStats._ringBufferFramesAvailable) + " starves:" + QString::number(streamStats._ringBufferStarveCount) + " not mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + " overflows:" + QString::number(streamStats._ringBufferOverflowCount) + " silents dropped:" + QString::number(streamStats._ringBufferSilentFramesDropped) - + " early:" + QString::number(streamStats._packetStreamStats._numEarly) - + " late:" + QString::number(streamStats._packetStreamStats._numLate) - + " lost:" + QString::number(streamStats._packetStreamStats._numLost) + + " lost %:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'g', 2) + + " lost % 30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'g', 2) + " min gap:" + QString::number(streamStats._timeGapMin) + " max gap:" + QString::number(streamStats._timeGapMax) + " avg gap:" + QString::number(streamStats._timeGapAverage, 'g', 2) diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 526071832e..7475c0a60e 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -20,6 +20,9 @@ #include "AudioStreamStats.h" #include "SequenceNumberStats.h" + +const int INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS = 30; + class AudioMixerClientData : public NodeData { public: AudioMixerClientData(); @@ -35,7 +38,7 @@ public: AudioStreamStats getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer) const; QString getAudioStreamStatsString() const; - void sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode) const; + void sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode); void incrementOutgoingMixedAudioSequenceNumber() { _outgoingMixedAudioSequenceNumber++; } quint16 getOutgoingSequenceNumber() const { return _outgoingMixedAudioSequenceNumber; } diff --git a/libraries/audio/src/AudioStreamStats.h b/libraries/audio/src/AudioStreamStats.h index 2cd0bca880..f26a3847c7 100644 --- a/libraries/audio/src/AudioStreamStats.h +++ b/libraries/audio/src/AudioStreamStats.h @@ -33,7 +33,8 @@ public: _ringBufferConsecutiveNotMixedCount(0), _ringBufferOverflowCount(0), _ringBufferSilentFramesDropped(0), - _packetStreamStats() + _packetStreamStats(), + _packetStreamWindowStats() {} PositionalAudioRingBuffer::Type _streamType; @@ -55,6 +56,7 @@ public: quint32 _ringBufferSilentFramesDropped; PacketStreamStats _packetStreamStats; + PacketStreamStats _packetStreamWindowStats; }; #endif // hifi_AudioStreamStats_h diff --git a/libraries/networking/src/SequenceNumberStats.cpp b/libraries/networking/src/SequenceNumberStats.cpp index 6a26e613b6..28dea7de63 100644 --- a/libraries/networking/src/SequenceNumberStats.cpp +++ b/libraries/networking/src/SequenceNumberStats.cpp @@ -13,17 +13,19 @@ #include -SequenceNumberStats::SequenceNumberStats() +SequenceNumberStats::SequenceNumberStats(int statsHistoryLength) : _lastReceived(std::numeric_limits::max()), _missingSet(), _stats(), - _lastSenderUUID() + _lastSenderUUID(), + _statsHistory(statsHistoryLength) { } void SequenceNumberStats::reset() { _missingSet.clear(); _stats = PacketStreamStats(); + _statsHistory.clear(); } static const int UINT16_RANGE = std::numeric_limits::max() + 1; @@ -168,3 +170,26 @@ void SequenceNumberStats::pruneMissingSet(const bool wantExtraDebugging) { } } } + +PacketStreamStats SequenceNumberStats::getStatsForHistoryWindow() const { + + const PacketStreamStats* newestStats = _statsHistory.getNewestEntry(); + const PacketStreamStats* oldestStats = _statsHistory.get(_statsHistory.getNumEntries() - 1); + + // this catches cases where history is length 1 or 0 (both are NULL in case of 0) + if (newestStats == oldestStats) { + return PacketStreamStats(); + } + + // calculate difference between newest stats and oldest stats to get window stats + PacketStreamStats windowStats; + windowStats._numReceived = newestStats->_numReceived - oldestStats->_numReceived; + windowStats._numUnreasonable = newestStats->_numUnreasonable - oldestStats->_numUnreasonable; + windowStats._numEarly = newestStats->_numEarly - oldestStats->_numEarly; + windowStats._numLate = newestStats->_numLate - oldestStats->_numLate; + windowStats._numLost = newestStats->_numLost - oldestStats->_numLost; + windowStats._numRecovered = newestStats->_numRecovered - oldestStats->_numRecovered; + windowStats._numDuplicate = newestStats->_numDuplicate - oldestStats->_numDuplicate; + + return windowStats; +} diff --git a/libraries/networking/src/SequenceNumberStats.h b/libraries/networking/src/SequenceNumberStats.h index b399c23d2b..8c16345aaf 100644 --- a/libraries/networking/src/SequenceNumberStats.h +++ b/libraries/networking/src/SequenceNumberStats.h @@ -13,6 +13,7 @@ #define hifi_SequenceNumberStats_h #include "SharedUtil.h" +#include "RingBufferHistory.h" #include const int MAX_REASONABLE_SEQUENCE_GAP = 1000; @@ -28,6 +29,14 @@ public: _numRecovered(0), _numDuplicate(0) {} + + float getUnreasonableRate() const { return (float)_numUnreasonable / _numReceived; } + float getNumEaryRate() const { return (float)_numEarly / _numReceived; } + float getLateRate() const { return (float)_numLate / _numReceived; } + float getLostRate() const { return (float)_numLost / _numReceived; } + float getRecoveredRate() const { return (float)_numRecovered / _numReceived; } + float getDuplicateRate() const { return (float)_numDuplicate / _numReceived; } + quint32 _numReceived; quint32 _numUnreasonable; quint32 _numEarly; @@ -39,11 +48,12 @@ public: class SequenceNumberStats { public: - SequenceNumberStats(); + SequenceNumberStats(int statsHistoryLength = 0); void reset(); void sequenceNumberReceived(quint16 incoming, QUuid senderUUID = QUuid(), const bool wantExtraDebugging = false); void pruneMissingSet(const bool wantExtraDebugging = false); + void pushStatsToHistory() { _statsHistory.insert(_stats); } quint32 getNumReceived() const { return _stats._numReceived; } quint32 getNumUnreasonable() const { return _stats._numUnreasonable; } @@ -54,6 +64,7 @@ public: quint32 getNumRecovered() const { return _stats._numRecovered; } quint32 getNumDuplicate() const { return _stats._numDuplicate; } const PacketStreamStats& getStats() const { return _stats; } + PacketStreamStats getStatsForHistoryWindow() const; const QSet& getMissingSet() const { return _missingSet; } private: @@ -63,6 +74,8 @@ private: PacketStreamStats _stats; QUuid _lastSenderUUID; + + RingBufferHistory _statsHistory; }; #endif // hifi_SequenceNumberStats_h diff --git a/libraries/shared/src/RingBufferHistory.h b/libraries/shared/src/RingBufferHistory.h index a9d24e44b6..e9875ec38a 100644 --- a/libraries/shared/src/RingBufferHistory.h +++ b/libraries/shared/src/RingBufferHistory.h @@ -64,11 +64,11 @@ public: } const T* getNewestEntry() const { - return &_buffer[_newestEntryAtIndex]; + return _numEntries == 0 ? NULL : &_buffer[_newestEntryAtIndex]; } T* getNewestEntry() { - return &_buffer[_newestEntryAtIndex]; + return _numEntries == 0 ? NULL : &_buffer[_newestEntryAtIndex]; } int getCapacity() const { return _capacity; } From a7ef7647edebb4256816dcf824bd44c86a3fb724 Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 10 Jul 2014 16:54:26 -0700 Subject: [PATCH 50/70] application stats tracking and packing updated; --- interface/src/Application.cpp | 1 - interface/src/Application.h | 2 ++ interface/src/Audio.cpp | 57 ++++++++--------------------------- interface/src/Audio.h | 13 ++------ 4 files changed, 17 insertions(+), 56 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 8edc788833..689250b318 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2158,7 +2158,6 @@ void Application::updateMyAvatar(float deltaTime) { { quint64 sinceLastNack = now - _lastSendDownstreamAudioStats; - const quint64 TOO_LONG_SINCE_LAST_SEND_DOWNSTREAM_AUDIO_STATS = 1 * USECS_PER_SECOND; if (sinceLastNack > TOO_LONG_SINCE_LAST_SEND_DOWNSTREAM_AUDIO_STATS) { _lastSendDownstreamAudioStats = now; diff --git a/interface/src/Application.h b/interface/src/Application.h index b55a830e3e..d956a949ac 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -125,6 +125,8 @@ static const float MIRROR_REARVIEW_DISTANCE = 0.65f; static const float MIRROR_REARVIEW_BODY_DISTANCE = 2.3f; static const float MIRROR_FIELD_OF_VIEW = 30.0f; +static const quint64 TOO_LONG_SINCE_LAST_SEND_DOWNSTREAM_AUDIO_STATS = 1 * USECS_PER_SECOND; + class Application : public QApplication { Q_OBJECT diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 7a445bf816..3689ff0143 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -48,15 +48,18 @@ static const float AUDIO_CALLBACK_MSECS = (float) NETWORK_BUFFER_LENGTH_SAMPLES_ static const int NUMBER_OF_NOISE_SAMPLE_FRAMES = 300; -static const int AUDIO_STREAM_STATS_HISTORY_SIZE = 30; - // audio frames time gap stats (min/max/avg) for last ~30 seconds are recalculated every ~1 second -const int TIME_GAPS_STATS_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS; -const int TIME_GAP_STATS_WINDOW_INTERVALS = 30; +static const int TIME_GAPS_STATS_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS; +static const int TIME_GAP_STATS_WINDOW_INTERVALS = 30; + +// incoming sequence number stats history will cover last 30s +static const int INCOMING_SEQ_STATS_HISTORY_LENGTH = INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS / + (TOO_LONG_SINCE_LAST_SEND_DOWNSTREAM_AUDIO_STATS / USECS_PER_SECOND); // Mute icon configration static const int MUTE_ICON_SIZE = 24; + Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : AbstractAudioInterface(parent), _audioInput(NULL), @@ -110,9 +113,8 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _scopeOutputLeft(0), _scopeOutputRight(0), _audioMixerAvatarStreamAudioStats(), - _audioMixerAvatarStreamPacketStatsHistory(AUDIO_STREAM_STATS_HISTORY_SIZE), _outgoingAvatarAudioSequenceNumber(0), - _incomingStreamPacketStatsHistory(AUDIO_STREAM_STATS_HISTORY_SIZE), + _incomingMixedAudioSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH), _interframeTimeGapStats(TIME_GAPS_STATS_INTERVAL_SAMPLES, TIME_GAP_STATS_WINDOW_INTERVALS), _starveCount(0), _consecutiveNotMixedCount(0) @@ -138,13 +140,8 @@ void Audio::reset() { _audioMixerAvatarStreamAudioStats = AudioStreamStats(); _audioMixerInjectedStreamAudioStatsMap.clear(); - _audioMixerAvatarStreamPacketStatsHistory.clear(); - _audioMixerInjectedStreamPacketStatsHistoryMap.clear(); - _outgoingAvatarAudioSequenceNumber = 0; _incomingMixedAudioSequenceNumberStats.reset(); - - _incomingStreamPacketStatsHistory.clear(); } QAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName) { @@ -751,7 +748,6 @@ void Audio::parseAudioStreamStatsPacket(const QByteArray& packet) { dataAt += sizeof(quint8); if (!appendFlag) { _audioMixerInjectedStreamAudioStatsMap.clear(); - _audioMixerInjectedStreamPacketStatsHistoryMap.clear(); } // parse the number of stream stats structs to follow @@ -766,20 +762,10 @@ void Audio::parseAudioStreamStatsPacket(const QByteArray& packet) { if (streamStats._streamType == PositionalAudioRingBuffer::Microphone) { _audioMixerAvatarStreamAudioStats = streamStats; - _audioMixerAvatarStreamPacketStatsHistory.insert(streamStats._packetStreamStats); } else { - if (!_audioMixerInjectedStreamAudioStatsMap.contains(streamStats._streamIdentifier)) { - _audioMixerInjectedStreamPacketStatsHistoryMap.insert(streamStats._streamIdentifier, - RingBufferHistory(AUDIO_STREAM_STATS_HISTORY_SIZE)); - } _audioMixerInjectedStreamAudioStatsMap[streamStats._streamIdentifier] = streamStats; - _audioMixerInjectedStreamPacketStatsHistoryMap[streamStats._streamIdentifier].insert(streamStats._packetStreamStats); } } - - // when an audio stream stats packet is received, also record the downstream packets stats in the history - // for calculating packet loss rates - _incomingStreamPacketStatsHistory.insert(_incomingMixedAudioSequenceNumberStats.getStats()); } AudioStreamStats Audio::getDownstreamAudioStreamStats() const { @@ -803,12 +789,17 @@ AudioStreamStats Audio::getDownstreamAudioStreamStats() const { stats._ringBufferSilentFramesDropped = 0; stats._packetStreamStats = _incomingMixedAudioSequenceNumberStats.getStats(); + stats._packetStreamWindowStats = _incomingMixedAudioSequenceNumberStats.getStatsForHistoryWindow(); return stats; } void Audio::sendDownstreamAudioStatsPacket() { + // push the current seq number stats into history, which moves the history window forward 1s + // (since that's how often pushStatsToHistory() is called) + _incomingMixedAudioSequenceNumberStats.pushStatsToHistory(); + char packet[MAX_PACKET_SIZE]; // pack header @@ -1608,25 +1599,3 @@ int Audio::calculateNumberOfFrameSamples(int numBytes) { int frameSamples = (int)(numBytes * CALLBACK_ACCELERATOR_RATIO + 0.5f) / sizeof(int16_t); return frameSamples; } - -void Audio::calculatePacketLossRate(const RingBufferHistory& statsHistory, - float& overallLossRate, float& windowLossRate) const { - - int numHistoryEntries = statsHistory.getNumEntries(); - if (numHistoryEntries == 0) { - overallLossRate = 0.0f; - windowLossRate = 0.0f; - } else { - const PacketStreamStats& newestStats = *statsHistory.getNewestEntry(); - overallLossRate = (float)newestStats._numLost / newestStats._numReceived; - - if (numHistoryEntries == 1) { - windowLossRate = overallLossRate; - } else { - int age = std::min(numHistoryEntries-1, AUDIO_STREAM_STATS_HISTORY_SIZE-1); - const PacketStreamStats& oldestStats = *statsHistory.get(age); - windowLossRate = (float)(newestStats._numLost - oldestStats._numLost) - / (newestStats._numReceived - oldestStats._numReceived); - } - } -} diff --git a/interface/src/Audio.h b/interface/src/Audio.h index e8e92db1a0..b40355e714 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -36,6 +36,8 @@ static const int NUM_AUDIO_CHANNELS = 2; +static const int INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS = 30; + class QAudioInput; class QAudioOutput; class QIODevice; @@ -121,14 +123,8 @@ public slots: const AudioStreamStats& getAudioMixerAvatarStreamAudioStats() const { return _audioMixerAvatarStreamAudioStats; } const QHash& getAudioMixerInjectedStreamAudioStatsMap() const { return _audioMixerInjectedStreamAudioStatsMap; } - const RingBufferHistory& getAudioMixerAvatarStreamPacketStatsHistory() const { return _audioMixerAvatarStreamPacketStatsHistory; } - const QHash >& getAudioMixerInjectedStreamPacketStatsHistoryMap() const {return _audioMixerInjectedStreamPacketStatsHistoryMap; } - const RingBufferHistory& getIncomingStreamPacketStatsHistory() const { return _incomingStreamPacketStatsHistory; } const MovingMinMaxAvg& getInterframeTimeGapStats() const { return _interframeTimeGapStats; } - void calculatePacketLossRate(const RingBufferHistory& statsHistory, - float& overallLossRate, float& windowLossRate) const; - signals: bool muteToggled(); void preProcessOriginalInboundAudio(unsigned int sampleTime, QByteArray& samples, const QAudioFormat& format); @@ -266,14 +262,9 @@ private: AudioStreamStats _audioMixerAvatarStreamAudioStats; QHash _audioMixerInjectedStreamAudioStatsMap; - RingBufferHistory _audioMixerAvatarStreamPacketStatsHistory; - QHash > _audioMixerInjectedStreamPacketStatsHistoryMap; - quint16 _outgoingAvatarAudioSequenceNumber; SequenceNumberStats _incomingMixedAudioSequenceNumberStats; - RingBufferHistory _incomingStreamPacketStatsHistory; - MovingMinMaxAvg _interframeTimeGapStats; }; From 71634b4c8dd7fc9d7bb4498ca371805ccab5ddde Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 10 Jul 2014 16:56:04 -0700 Subject: [PATCH 51/70] Fixed windows compiler warning --- interface/src/ui/ApplicationOverlay.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 6d1e45df20..15cb10ec84 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -700,7 +700,7 @@ void ApplicationOverlay::renderPointersOculus(const glm::vec3& eyePos) { MyAvatar* myAvatar = application->getAvatar(); //Controller Pointers - for (int i = 0; i < myAvatar->getHand()->getNumPalms(); i++) { + for (size_t i = 0; i < myAvatar->getHand()->getNumPalms(); i++) { PalmData& palm = myAvatar->getHand()->getPalms()[i]; if (palm.isActive()) { From e658ce8fb939995d99579d3f016baa4982c313db Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 10 Jul 2014 16:57:46 -0700 Subject: [PATCH 52/70] Fixed ubuntu compiler warning --- interface/src/ui/ApplicationOverlay.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 15cb10ec84..3cc9cf4250 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -40,8 +40,8 @@ ApplicationOverlay::ApplicationOverlay() : _framebufferObject(NULL), _textureFov(DEFAULT_OCULUS_UI_ANGULAR_SIZE * RADIANS_PER_DEGREE), _alpha(1.0f), - _crosshairTexture(0), - _oculusuiRadius(1.0f) { + _oculusuiRadius(1.0f), + _crosshairTexture(0) { memset(_reticleActive, 0, sizeof(_reticleActive)); memset(_magActive, 0, sizeof(_reticleActive)); From 4bf74c1b65c5c7bc2bad97c8db7031ae13d5ee43 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 10 Jul 2014 16:59:57 -0700 Subject: [PATCH 53/70] Fixed more ubuntu compiler warnings --- interface/src/ui/ApplicationOverlay.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 3cc9cf4250..fad4a64835 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -272,9 +272,6 @@ QPoint ApplicationOverlay::getOculusPalmClickLocation(const PalmData *palm) cons Application* application = Application::getInstance(); QGLWidget* glWidget = application->getGLWidget(); MyAvatar* myAvatar = application->getAvatar(); - - const int widgetWidth = glWidget->width(); - const int widgetHeight = glWidget->height(); glm::vec3 tip = OculusManager::getLaserPointerTipPosition(palm); glm::vec3 eyePos = myAvatar->getHead()->calculateAverageEyePosition(); @@ -700,13 +697,12 @@ void ApplicationOverlay::renderPointersOculus(const glm::vec3& eyePos) { MyAvatar* myAvatar = application->getAvatar(); //Controller Pointers - for (size_t i = 0; i < myAvatar->getHand()->getNumPalms(); i++) { + for (int i = 0; i < (int)myAvatar->getHand()->getNumPalms(); i++) { PalmData& palm = myAvatar->getHand()->getPalms()[i]; if (palm.isActive()) { glm::vec3 tip = OculusManager::getLaserPointerTipPosition(&palm); glm::quat orientation = glm::inverse(myAvatar->getOrientation()); - glm::vec3 dir = orientation * glm::normalize(eyePos - tip); //direction of ray goes towards camera glm::vec3 tipPos = (tip - eyePos); float length = glm::length(eyePos - tip); From 6c85caaa54415bbb272f9398deb50ca1f9f66b9c Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 10 Jul 2014 17:03:53 -0700 Subject: [PATCH 54/70] updated Stats for new AudioStreamStats format --- .../src/audio/AudioMixerClientData.cpp | 28 +++++++++---------- interface/src/ui/Stats.cpp | 22 +++++---------- 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index d0b5104dbd..789e73eb86 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -278,33 +278,33 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { + " not mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + " overflows:" + QString::number(streamStats._ringBufferOverflowCount) + " silents dropped: ?" - + " lost %:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'g', 2) - + " lost % 30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'g', 2) + + " lost %:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'f', 2) + + " lost % 30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'f', 2) + " min gap:" + QString::number(streamStats._timeGapMin) + " max gap:" + QString::number(streamStats._timeGapMax) - + " avg gap:" + QString::number(streamStats._timeGapAverage, 'g', 2) + + " avg gap:" + QString::number(streamStats._timeGapAverage, 'f', 2) + " min 30s gap:" + QString::number(streamStats._timeGapWindowMin) + " max 30s gap:" + QString::number(streamStats._timeGapWindowMax) - + " avg 30s gap:" + QString::number(streamStats._timeGapWindowAverage, 'g', 2); + + " avg 30s gap:" + QString::number(streamStats._timeGapWindowAverage, 'f', 2); AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); if (avatarRingBuffer) { AudioStreamStats streamStats = getAudioStreamStatsOfStream(avatarRingBuffer); - result += "mic.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) + result += " mic.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) + " current:" + QString::number(streamStats._ringBufferCurrentJitterBufferFrames) + " available:" + QString::number(streamStats._ringBufferFramesAvailable) + " starves:" + QString::number(streamStats._ringBufferStarveCount) + " not mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + " overflows:" + QString::number(streamStats._ringBufferOverflowCount) + " silents dropped:" + QString::number(streamStats._ringBufferSilentFramesDropped) - + " lost %:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'g', 2) - + " lost % 30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'g', 2) + + " lost %:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'f', 2) + + " lost % 30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'f', 2) + " min gap:" + QString::number(streamStats._timeGapMin) + " max gap:" + QString::number(streamStats._timeGapMax) - + " avg gap:" + QString::number(streamStats._timeGapAverage, 'g', 2) + + " avg gap:" + QString::number(streamStats._timeGapAverage, 'f', 2) + " min 30s gap:" + QString::number(streamStats._timeGapWindowMin) + " max 30s gap:" + QString::number(streamStats._timeGapWindowMax) - + " avg 30s gap:" + QString::number(streamStats._timeGapWindowAverage, 'g', 2); + + " avg 30s gap:" + QString::number(streamStats._timeGapWindowAverage, 'f', 2); } else { result = "mic unknown"; } @@ -312,21 +312,21 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { for (int i = 0; i < _ringBuffers.size(); i++) { if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) { AudioStreamStats streamStats = getAudioStreamStatsOfStream(_ringBuffers[i]); - result += "inj.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) + result += " inj.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) + " current:" + QString::number(streamStats._ringBufferCurrentJitterBufferFrames) + " available:" + QString::number(streamStats._ringBufferFramesAvailable) + " starves:" + QString::number(streamStats._ringBufferStarveCount) + " not mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + " overflows:" + QString::number(streamStats._ringBufferOverflowCount) + " silents dropped:" + QString::number(streamStats._ringBufferSilentFramesDropped) - + " lost %:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'g', 2) - + " lost % 30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'g', 2) + + " lost %:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'f', 2) + + " lost % 30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'f', 2) + " min gap:" + QString::number(streamStats._timeGapMin) + " max gap:" + QString::number(streamStats._timeGapMax) - + " avg gap:" + QString::number(streamStats._timeGapAverage, 'g', 2) + + " avg gap:" + QString::number(streamStats._timeGapAverage, 'f', 2) + " min 30s gap:" + QString::number(streamStats._timeGapWindowMin) + " max 30s gap:" + QString::number(streamStats._timeGapWindowMax) - + " avg 30s gap:" + QString::number(streamStats._timeGapWindowAverage, 'g', 2); + + " avg 30s gap:" + QString::number(streamStats._timeGapWindowAverage, 'f', 2); } } return result; diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index add0754fe1..15a54e42a6 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -24,6 +24,7 @@ #include "InterfaceConfig.h" #include "Menu.h" #include "Util.h" +#include "SequenceNumberStats.h" using namespace std; @@ -341,8 +342,6 @@ void Stats::display( verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString4, color); - float packetLossRate, packetLossRate30s; - char downstreamLabelString[] = " Downstream:"; verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamLabelString, color); @@ -351,9 +350,8 @@ void Stats::display( AudioStreamStats downstreamAudioStreamStats = audio->getDownstreamAudioStreamStats(); - audio->calculatePacketLossRate(audio->getIncomingStreamPacketStatsHistory(), packetLossRate, packetLossRate30s); - - sprintf(downstreamAudioStatsString, " mix: %.1f%%/%.1f%%, %u/?/%u", packetLossRate*100.0f, packetLossRate30s*100.0f, + sprintf(downstreamAudioStatsString, " mix: %.1f%%/%.1f%%, %u/?/%u", downstreamAudioStreamStats._packetStreamStats.getLostRate()*100.0f, + downstreamAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f, downstreamAudioStreamStats._ringBufferFramesAvailable, downstreamAudioStreamStats._ringBufferDesiredJitterBufferFrames); verticalOffset += STATS_PELS_PER_LINE; @@ -381,10 +379,9 @@ void Stats::display( char upstreamAudioStatsString[30]; const AudioStreamStats& audioMixerAvatarAudioStreamStats = audio->getAudioMixerAvatarStreamAudioStats(); - - audio->calculatePacketLossRate(audio->getAudioMixerAvatarStreamPacketStatsHistory(), packetLossRate, packetLossRate30s); - sprintf(upstreamAudioStatsString, " mic: %.1f%%/%.1f%%, %u/%u/%u", packetLossRate*100.0f, packetLossRate30s*100.0f, + sprintf(upstreamAudioStatsString, " mic: %.1f%%/%.1f%%, %u/%u/%u", audioMixerAvatarAudioStreamStats._packetStreamStats.getLostRate()*100.0f, + audioMixerAvatarAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f, audioMixerAvatarAudioStreamStats._ringBufferFramesAvailable, audioMixerAvatarAudioStreamStats._ringBufferCurrentJitterBufferFrames, audioMixerAvatarAudioStreamStats._ringBufferDesiredJitterBufferFrames); @@ -405,15 +402,10 @@ void Stats::display( verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - QHash > audioMixerInjectedStreamPacketStatsHistoryMap - = audio->getAudioMixerInjectedStreamPacketStatsHistoryMap(); - foreach(const AudioStreamStats& injectedStreamAudioStats, audioMixerInjectedStreamAudioStatsMap) { - audio->calculatePacketLossRate(audioMixerInjectedStreamPacketStatsHistoryMap[injectedStreamAudioStats._streamIdentifier], - packetLossRate, packetLossRate30s); - - sprintf(upstreamAudioStatsString, " inj: %.1f%%/%.1f%%, %u/%u/%u", packetLossRate*100.0f, packetLossRate30s*100.0f, + sprintf(upstreamAudioStatsString, " inj: %.1f%%/%.1f%%, %u/%u/%u", injectedStreamAudioStats._packetStreamStats.getLostRate()*100.0f, + injectedStreamAudioStats._packetStreamWindowStats.getLostRate() * 100.0f, injectedStreamAudioStats._ringBufferFramesAvailable, injectedStreamAudioStats._ringBufferCurrentJitterBufferFrames, injectedStreamAudioStats._ringBufferDesiredJitterBufferFrames); From ae1d91b21d81be29195b6bdc5ddd140d607ce50d Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 10 Jul 2014 17:11:15 -0700 Subject: [PATCH 55/70] improved domain page stats string a bit --- .../src/audio/AudioMixerClientData.cpp | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 789e73eb86..bde4521d33 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -271,40 +271,40 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& QString AudioMixerClientData::getAudioStreamStatsString() const { QString result; AudioStreamStats streamStats = _downstreamAudioStreamStats; - result += "downstream.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) + result += "DOWNSTREAM.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) + " current: ?" + " available:" + QString::number(streamStats._ringBufferFramesAvailable) + " starves:" + QString::number(streamStats._ringBufferStarveCount) - + " not mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + + " not_mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + " overflows:" + QString::number(streamStats._ringBufferOverflowCount) - + " silents dropped: ?" - + " lost %:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'f', 2) - + " lost % 30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'f', 2) - + " min gap:" + QString::number(streamStats._timeGapMin) - + " max gap:" + QString::number(streamStats._timeGapMax) - + " avg gap:" + QString::number(streamStats._timeGapAverage, 'f', 2) - + " min 30s gap:" + QString::number(streamStats._timeGapWindowMin) - + " max 30s gap:" + QString::number(streamStats._timeGapWindowMax) - + " avg 30s gap:" + QString::number(streamStats._timeGapWindowAverage, 'f', 2); + + " silents_dropped: ?" + + " lost%:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'f', 2) + + " lost%_30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'f', 2) + + " min_gap:" + QString::number(streamStats._timeGapMin) + + " max_gap:" + QString::number(streamStats._timeGapMax) + + " avg_gap:" + QString::number(streamStats._timeGapAverage, 'f', 2) + + " min_gap_30s:" + QString::number(streamStats._timeGapWindowMin) + + " max_gap_30s:" + QString::number(streamStats._timeGapWindowMax) + + " avg_gap_30s:" + QString::number(streamStats._timeGapWindowAverage, 'f', 2); AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); if (avatarRingBuffer) { AudioStreamStats streamStats = getAudioStreamStatsOfStream(avatarRingBuffer); - result += " mic.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) + result += " UPSTREAM.mic.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) + " current:" + QString::number(streamStats._ringBufferCurrentJitterBufferFrames) + " available:" + QString::number(streamStats._ringBufferFramesAvailable) + " starves:" + QString::number(streamStats._ringBufferStarveCount) - + " not mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + + " not_mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + " overflows:" + QString::number(streamStats._ringBufferOverflowCount) - + " silents dropped:" + QString::number(streamStats._ringBufferSilentFramesDropped) - + " lost %:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'f', 2) - + " lost % 30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'f', 2) - + " min gap:" + QString::number(streamStats._timeGapMin) - + " max gap:" + QString::number(streamStats._timeGapMax) - + " avg gap:" + QString::number(streamStats._timeGapAverage, 'f', 2) - + " min 30s gap:" + QString::number(streamStats._timeGapWindowMin) - + " max 30s gap:" + QString::number(streamStats._timeGapWindowMax) - + " avg 30s gap:" + QString::number(streamStats._timeGapWindowAverage, 'f', 2); + + " silents_dropped:" + QString::number(streamStats._ringBufferSilentFramesDropped) + + " lost%:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'f', 2) + + " lost%_30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'f', 2) + + " min_gap:" + QString::number(streamStats._timeGapMin) + + " max_gap:" + QString::number(streamStats._timeGapMax) + + " avg_gap:" + QString::number(streamStats._timeGapAverage, 'f', 2) + + " min_gap_30s:" + QString::number(streamStats._timeGapWindowMin) + + " max_gap_30s:" + QString::number(streamStats._timeGapWindowMax) + + " avg_gap_30s:" + QString::number(streamStats._timeGapWindowAverage, 'f', 2); } else { result = "mic unknown"; } @@ -312,21 +312,21 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { for (int i = 0; i < _ringBuffers.size(); i++) { if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) { AudioStreamStats streamStats = getAudioStreamStatsOfStream(_ringBuffers[i]); - result += " inj.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) + result += " UPSTREAM.inj.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) + " current:" + QString::number(streamStats._ringBufferCurrentJitterBufferFrames) + " available:" + QString::number(streamStats._ringBufferFramesAvailable) + " starves:" + QString::number(streamStats._ringBufferStarveCount) - + " not mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + + " not_mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + " overflows:" + QString::number(streamStats._ringBufferOverflowCount) - + " silents dropped:" + QString::number(streamStats._ringBufferSilentFramesDropped) - + " lost %:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'f', 2) - + " lost % 30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'f', 2) - + " min gap:" + QString::number(streamStats._timeGapMin) - + " max gap:" + QString::number(streamStats._timeGapMax) - + " avg gap:" + QString::number(streamStats._timeGapAverage, 'f', 2) - + " min 30s gap:" + QString::number(streamStats._timeGapWindowMin) - + " max 30s gap:" + QString::number(streamStats._timeGapWindowMax) - + " avg 30s gap:" + QString::number(streamStats._timeGapWindowAverage, 'f', 2); + + " silents_dropped:" + QString::number(streamStats._ringBufferSilentFramesDropped) + + " lost%:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'f', 2) + + " lost%_30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'f', 2) + + " min_gap:" + QString::number(streamStats._timeGapMin) + + " max_gap:" + QString::number(streamStats._timeGapMax) + + " avg_gap:" + QString::number(streamStats._timeGapAverage, 'f', 2) + + " min_gap_30s:" + QString::number(streamStats._timeGapWindowMin) + + " max_gap_30s:" + QString::number(streamStats._timeGapWindowMax) + + " avg_gap_30s:" + QString::number(streamStats._timeGapWindowAverage, 'f', 2); } } return result; From df7f3073dc3efd4ce49f05598e2d1430f16e3459 Mon Sep 17 00:00:00 2001 From: TonyPeng Date: Thu, 10 Jul 2014 17:18:49 -0700 Subject: [PATCH 56/70] 2 local lights as default --- interface/src/avatar/Avatar.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index 83105c9022..febbba7536 100644 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -60,9 +60,9 @@ Avatar::Avatar() : _mouseRayDirection(0.0f, 0.0f, 0.0f), _moving(false), _collisionGroups(0), + _numLocalLights(2), _initialized(false), - _shouldRenderBillboard(true), - _numLocalLights(1) + _shouldRenderBillboard(true) { // we may have been created in the network thread, but we live in the main thread moveToThread(Application::getInstance()->thread()); @@ -89,14 +89,14 @@ void Avatar::init() { _localLightDirections[i] = glm::vec3(0.0f, 0.0f, 0.0f); } - glm::vec3 darkGrayColor(0.3f, 0.3f, 0.3f); + glm::vec3 darkGrayColor(0.4f, 0.4f, 0.4f); glm::vec3 greenColor(0.0f, 1.0f, 0.0f); glm::vec3 directionX(1.0f, 0.0f, 0.0f); glm::vec3 directionY(0.0f, 1.0f, 0.0f); // initialize local lights _localLightColors[0] = darkGrayColor; - _localLightColors[1] = greenColor; + _localLightColors[1] = darkGrayColor; _localLightDirections[0] = directionX; _localLightDirections[1] = directionY; From 69005242b9acc84aa2fa0a044d72bb50e32760fe Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 10 Jul 2014 17:18:54 -0700 Subject: [PATCH 57/70] forgot to multiply rates by 100% for domain stats page --- assignment-client/src/audio/AudioMixerClientData.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index bde4521d33..94bbdc6a6b 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -278,8 +278,8 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { + " not_mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + " overflows:" + QString::number(streamStats._ringBufferOverflowCount) + " silents_dropped: ?" - + " lost%:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'f', 2) - + " lost%_30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'f', 2) + + " lost%:" + QString::number(streamStats._packetStreamStats.getLostRate() * 100.0f, 'f', 2) + + " lost%_30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate() * 100.0f, 'f', 2) + " min_gap:" + QString::number(streamStats._timeGapMin) + " max_gap:" + QString::number(streamStats._timeGapMax) + " avg_gap:" + QString::number(streamStats._timeGapAverage, 'f', 2) @@ -297,8 +297,8 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { + " not_mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + " overflows:" + QString::number(streamStats._ringBufferOverflowCount) + " silents_dropped:" + QString::number(streamStats._ringBufferSilentFramesDropped) - + " lost%:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'f', 2) - + " lost%_30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'f', 2) + + " lost%:" + QString::number(streamStats._packetStreamStats.getLostRate() * 100.0f, 'f', 2) + + " lost%_30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate() * 100.0f, 'f', 2) + " min_gap:" + QString::number(streamStats._timeGapMin) + " max_gap:" + QString::number(streamStats._timeGapMax) + " avg_gap:" + QString::number(streamStats._timeGapAverage, 'f', 2) @@ -319,8 +319,8 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { + " not_mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) + " overflows:" + QString::number(streamStats._ringBufferOverflowCount) + " silents_dropped:" + QString::number(streamStats._ringBufferSilentFramesDropped) - + " lost%:" + QString::number(streamStats._packetStreamStats.getLostRate(), 'f', 2) - + " lost%_30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate(), 'f', 2) + + " lost%:" + QString::number(streamStats._packetStreamStats.getLostRate() * 100.0f, 'f', 2) + + " lost%_30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate() * 100.0f, 'f', 2) + " min_gap:" + QString::number(streamStats._timeGapMin) + " max_gap:" + QString::number(streamStats._timeGapMax) + " avg_gap:" + QString::number(streamStats._timeGapAverage, 'f', 2) From d121c8e07af33811b5ac636a00a2b4635c38c5de Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 10 Jul 2014 17:21:05 -0700 Subject: [PATCH 58/70] Fix for starting with empty spanners (due to LOD), then adding. --- libraries/metavoxels/src/AttributeRegistry.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libraries/metavoxels/src/AttributeRegistry.cpp b/libraries/metavoxels/src/AttributeRegistry.cpp index b43759fa4f..64b2646261 100644 --- a/libraries/metavoxels/src/AttributeRegistry.cpp +++ b/libraries/metavoxels/src/AttributeRegistry.cpp @@ -568,6 +568,10 @@ void SpannerSetAttribute::readMetavoxelRoot(MetavoxelData& data, MetavoxelStream } data.insert(state.attribute, object); } + // even if the root is empty, it should still exist + if (!data.getRoot(state.attribute)) { + data.createRoot(state.attribute); + } } void SpannerSetAttribute::writeMetavoxelRoot(const MetavoxelNode& root, MetavoxelStreamState& state) { @@ -586,6 +590,10 @@ void SpannerSetAttribute::readMetavoxelDelta(MetavoxelData& data, } data.toggle(state.attribute, object); } + // even if the root is empty, it should still exist + if (!data.getRoot(state.attribute)) { + data.createRoot(state.attribute); + } } void SpannerSetAttribute::writeMetavoxelDelta(const MetavoxelNode& root, From 1781a2d1253692666b03754274222bbd2a73b89b Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 10 Jul 2014 17:30:43 -0700 Subject: [PATCH 59/70] fixed unused variable warning --- interface/src/ui/ApplicationOverlay.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index fad4a64835..3ccb487716 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -702,7 +702,6 @@ void ApplicationOverlay::renderPointersOculus(const glm::vec3& eyePos) { PalmData& palm = myAvatar->getHand()->getPalms()[i]; if (palm.isActive()) { glm::vec3 tip = OculusManager::getLaserPointerTipPosition(&palm); - glm::quat orientation = glm::inverse(myAvatar->getOrientation()); glm::vec3 tipPos = (tip - eyePos); float length = glm::length(eyePos - tip); From 97165cadcc45443c3f81da10ed92f4c3de8b156b Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 10 Jul 2014 17:34:10 -0700 Subject: [PATCH 60/70] Removed unneeded variable --- interface/src/ui/ApplicationOverlay.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 3ccb487716..7b3db26d8d 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -166,7 +166,6 @@ void ApplicationOverlay::displayOverlayTexture() { void ApplicationOverlay::computeOculusPickRay(float x, float y, glm::vec3& direction) const { MyAvatar* myAvatar = Application::getInstance()->getAvatar(); - glm::quat rot = myAvatar->getOrientation(); //invert y direction y = 1.0 - y; @@ -179,7 +178,7 @@ void ApplicationOverlay::computeOculusPickRay(float x, float y, glm::vec3& direc float z = -sqrt(1.0f - dist * dist); glm::vec3 relativePosition = myAvatar->getHead()->calculateAverageEyePosition() + - glm::normalize(rot * glm::vec3(x, y, z)); + glm::normalize(myAvatar->getOrientation() * glm::vec3(x, y, z)); //Rotate the UI pick ray by the avatar orientation direction = glm::normalize(relativePosition - Application::getInstance()->getCamera()->getPosition()); From af3af15084ea77f3dfe90509220b2a3d3ce75c58 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 10 Jul 2014 18:11:51 -0700 Subject: [PATCH 61/70] Comment, formatting fix. --- assignment-client/src/metavoxels/MetavoxelServer.cpp | 2 +- interface/src/ui/Stats.cpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/assignment-client/src/metavoxels/MetavoxelServer.cpp b/assignment-client/src/metavoxels/MetavoxelServer.cpp index 956458404a..4d9c45ed6c 100644 --- a/assignment-client/src/metavoxels/MetavoxelServer.cpp +++ b/assignment-client/src/metavoxels/MetavoxelServer.cpp @@ -51,7 +51,7 @@ void MetavoxelServer::run() { // initialize Bitstream before using it in multiple threads Bitstream::preThreadingInit(); - // create the persister and start the it in its own thread + // create the persister and start it in its own thread _persister = new MetavoxelPersister(this); QThread* persistenceThread = new QThread(this); _persister->moveToThread(persistenceThread); diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 0d2762f923..eeff76395b 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -420,7 +420,6 @@ void Stats::display( verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, downloads.str().c_str(), color); - int internal = 0, leaves = 0; int sendProgress = 0, sendTotal = 0; int receiveProgress = 0, receiveTotal = 0; From 2347bd7eed458fd80b9fe81836723b69214f4a63 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 11 Jul 2014 10:47:48 -0700 Subject: [PATCH 62/70] Sitting position persists between sessions --- examples/sit.js | 54 +++++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/examples/sit.js b/examples/sit.js index 056a65fbf1..f7be681d25 100644 --- a/examples/sit.js +++ b/examples/sit.js @@ -43,7 +43,10 @@ var animationLenght = 2.0; var avatarOldPosition = { x: 0, y: 0, z: 0 }; -var sitting = false; +var sittingSettingsHandle = "SitJsSittingPosition"; +var sitting = Settings.getValue(sittingSettingsHandle, false) == "true"; +print("Original sitting status: " + sitting); +var frame = 0; var seat = new Object(); var hiddingSeats = false; @@ -123,10 +126,12 @@ var goToSeatAnimation = function(deltaTime) { function sitDown() { sitting = true; + Settings.setValue(sittingSettingsHandle, sitting); + print("sitDown sitting status: " + Settings.getValue(sittingSettingsHandle, false)); passedTime = 0.0; startPosition = MyAvatar.position; storeStartPoseAndTransition(); - try{ + try { Script.update.disconnect(standingUpAnimation); } catch(e){ // no need to handle. if it wasn't connected no harm done @@ -138,6 +143,8 @@ function sitDown() { function standUp() { sitting = false; + Settings.setValue(sittingSettingsHandle, sitting); + print("standUp sitting status: " + Settings.getValue(sittingSettingsHandle, false)); passedTime = 0.0; startPosition = MyAvatar.position; try{ @@ -218,33 +225,6 @@ Controller.mousePressEvent.connect(function(event) { try{ Script.update.disconnect(sittingDownAnimation); } catch(e){} Script.update.connect(goToSeatAnimation); } - - - - return; - var intersection = Models.findRayIntersection(pickRay); - - if (intersection.accurate && intersection.intersects && false) { - var properties = intersection.modelProperties; - print("Intersecting with model, let's check for seats."); - - if (properties.sittingPoints.length > 0) { - print("Available seats, going to the first one: " + properties.sittingPoints[0].name); - seat.position = Vec3.sum(properties.position, Vec3.multiplyQbyV(properties.modelRotation, properties.sittingPoints[0].position)); - Vec3.print("Seat position: ", seat.position); - seat.rotation = Quat.multiply(properties.modelRotation, properties.sittingPoints[0].rotation); - Quat.print("Seat rotation: ", seat.rotation); - - passedTime = 0.0; - startPosition = MyAvatar.position; - startRotation = MyAvatar.orientation; - try{ Script.update.disconnect(standingUpAnimation); } catch(e){} - try{ Script.update.disconnect(sittingDownAnimation); } catch(e){} - Script.update.connect(goToSeatAnimation); - } else { - print ("Sorry, no seats here."); - } - } } }) @@ -257,6 +237,22 @@ function update(deltaTime){ Overlays.editOverlay( standUpButton, {x: newX, y: newY} ); Overlays.editOverlay( sitDownButton, {x: newX, y: newY} ); } + + // For a weird reason avatar joint don't update till the 10th frame + // Set the update frame to 20 to be safe + var UPDATE_FRAME = 20; + if (frame <= UPDATE_FRAME) { + if (frame == UPDATE_FRAME) { + if (sitting == true) { + print("Was seated: " + sitting); + storeStartPoseAndTransition(); + updateJoints(1.0); + Overlays.editOverlay(sitDownButton, { visible: false }); + Overlays.editOverlay(standUpButton, { visible: true }); + } + } + frame++; + } if (MyAvatar.position.x != avatarOldPosition.x && MyAvatar.position.y != avatarOldPosition.y && From bd7b17a7878368dcdab23bb3b91c9289f8681a57 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 11 Jul 2014 10:50:11 -0700 Subject: [PATCH 63/70] Added subimage to Billboards overlays --- .../src/ui/overlays/BillboardOverlay.cpp | 51 ++++++++++++++++--- interface/src/ui/overlays/BillboardOverlay.h | 5 +- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/interface/src/ui/overlays/BillboardOverlay.cpp b/interface/src/ui/overlays/BillboardOverlay.cpp index 8742f19c3d..d88ee6abd2 100644 --- a/interface/src/ui/overlays/BillboardOverlay.cpp +++ b/interface/src/ui/overlays/BillboardOverlay.cpp @@ -14,7 +14,8 @@ #include "BillboardOverlay.h" BillboardOverlay::BillboardOverlay() -: _scale(1.0f), +: _fromImage(-1,-1,-1,-1), + _scale(1.0f), _isFacingAvatar(true) { } @@ -28,6 +29,9 @@ void BillboardOverlay::render() { image = image.convertToFormat(QImage::Format_ARGB32); } _size = image.size(); + if (_fromImage.x() == -1) { + _fromImage.setRect(0, 0, _size.width(), _size.height()); + } _billboardTexture.reset(new Texture()); glBindTexture(GL_TEXTURE_2D, _billboardTexture->getID()); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, _size.width(), _size.height(), 0, @@ -58,19 +62,23 @@ void BillboardOverlay::render() { } glScalef(_scale, _scale, _scale); - float maxSize = glm::max(_size.width(), _size.height()); - float x = _size.width() / (2.0f * maxSize); - float y = -_size.height() / (2.0f * maxSize); + float maxSize = glm::max(_fromImage.width(), _fromImage.height()); + float x = _fromImage.width() / (2.0f * maxSize); + float y = -_fromImage.height() / (2.0f * maxSize); glColor3f(1.0f, 1.0f, 1.0f); glBegin(GL_QUADS); { - glTexCoord2f(0.0f, 0.0f); + glTexCoord2f((float)_fromImage.x() / (float)_size.width(), + (float)_fromImage.y() / (float)_size.height()); glVertex2f(-x, -y); - glTexCoord2f(1.0f, 0.0f); + glTexCoord2f(((float)_fromImage.x() + (float)_fromImage.width()) / (float)_size.width(), + (float)_fromImage.y() / (float)_size.height()); glVertex2f(x, -y); - glTexCoord2f(1.0f, 1.0f); + glTexCoord2f(((float)_fromImage.x() + (float)_fromImage.width()) / (float)_size.width(), + ((float)_fromImage.y() + (float)_fromImage.height()) / _size.height()); glVertex2f(x, y); - glTexCoord2f(0.0f, 1.0f); + glTexCoord2f((float)_fromImage.x() / (float)_size.width(), + ((float)_fromImage.y() + (float)_fromImage.height()) / (float)_size.height()); glVertex2f(-x, y); } glEnd(); @@ -93,6 +101,33 @@ void BillboardOverlay::setProperties(const QScriptValue &properties) { setBillboardURL(_url); } + QScriptValue subImageBounds = properties.property("subImage"); + if (subImageBounds.isValid()) { + QRect oldSubImageRect = _fromImage; + QRect subImageRect = _fromImage; + if (subImageBounds.property("x").isValid()) { + subImageRect.setX(subImageBounds.property("x").toVariant().toInt()); + } else { + subImageRect.setX(oldSubImageRect.x()); + } + if (subImageBounds.property("y").isValid()) { + subImageRect.setY(subImageBounds.property("y").toVariant().toInt()); + } else { + subImageRect.setY(oldSubImageRect.y()); + } + if (subImageBounds.property("width").isValid()) { + subImageRect.setWidth(subImageBounds.property("width").toVariant().toInt()); + } else { + subImageRect.setWidth(oldSubImageRect.width()); + } + if (subImageBounds.property("height").isValid()) { + subImageRect.setHeight(subImageBounds.property("height").toVariant().toInt()); + } else { + subImageRect.setHeight(oldSubImageRect.height()); + } + setClipFromSource(subImageRect); + } + QScriptValue scaleValue = properties.property("scale"); if (scaleValue.isValid()) { _scale = scaleValue.toVariant().toFloat(); diff --git a/interface/src/ui/overlays/BillboardOverlay.h b/interface/src/ui/overlays/BillboardOverlay.h index 0037d1a4f7..5efb5767c6 100644 --- a/interface/src/ui/overlays/BillboardOverlay.h +++ b/interface/src/ui/overlays/BillboardOverlay.h @@ -25,7 +25,8 @@ public: virtual void render(); virtual void setProperties(const QScriptValue& properties); - + void setClipFromSource(const QRect& bounds) { _fromImage = bounds; } + private slots: void replyFinished(); @@ -37,6 +38,8 @@ private: QSize _size; QScopedPointer _billboardTexture; + QRect _fromImage; // where from in the image to sample + glm::quat _rotation; float _scale; bool _isFacingAvatar; From a9ceec46270eec7e3b4c56f68bc6442bf969a96a Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 11 Jul 2014 10:50:46 -0700 Subject: [PATCH 64/70] Switched sit.js spheres to billboards --- examples/sit.js | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/examples/sit.js b/examples/sit.js index f7be681d25..b51d59759a 100644 --- a/examples/sit.js +++ b/examples/sit.js @@ -166,14 +166,16 @@ function SeatIndicator(modelProperties, seatIndex) { modelProperties.sittingPoints[seatIndex].rotation); this.scale = MyAvatar.scale / 12; - this.sphere = Overlays.addOverlay("sphere", { - position: this.position, - size: this.scale, - solid: true, - color: { red: 0, green: 0, blue: 255 }, - alpha: 0.3, - visible: true - }); + this.sphere = Overlays.addOverlay("billboard", { + subImage: { x: 0, y: buttonHeight, width: buttonWidth, height: buttonHeight}, + url: buttonImageUrl, + position: this.position, + scale: this.scale * 4, + solid: true, + color: { red: 0, green: 0, blue: 255 }, + alpha: 0.3, + visible: true + }); this.show = function(doShow) { Overlays.editOverlay(this.sphere, { visible: doShow }); From 22bcb7d7e7040e052f0a386d6b8d5ca9c8384768 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 11 Jul 2014 11:22:06 -0700 Subject: [PATCH 65/70] Billboard overlays disply grey square while image is loading --- .../src/ui/overlays/BillboardOverlay.cpp | 85 ++++++++++--------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/interface/src/ui/overlays/BillboardOverlay.cpp b/interface/src/ui/overlays/BillboardOverlay.cpp index d88ee6abd2..4bd854114d 100644 --- a/interface/src/ui/overlays/BillboardOverlay.cpp +++ b/interface/src/ui/overlays/BillboardOverlay.cpp @@ -20,26 +20,25 @@ BillboardOverlay::BillboardOverlay() } void BillboardOverlay::render() { - if (_billboard.isEmpty()) { - return; - } - if (!_billboardTexture) { - QImage image = QImage::fromData(_billboard); - if (image.format() != QImage::Format_ARGB32) { - image = image.convertToFormat(QImage::Format_ARGB32); + if (!_billboard.isEmpty()) { + if (!_billboardTexture) { + QImage image = QImage::fromData(_billboard); + if (image.format() != QImage::Format_ARGB32) { + image = image.convertToFormat(QImage::Format_ARGB32); + } + _size = image.size(); + if (_fromImage.x() == -1) { + _fromImage.setRect(0, 0, _size.width(), _size.height()); + } + _billboardTexture.reset(new Texture()); + glBindTexture(GL_TEXTURE_2D, _billboardTexture->getID()); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, _size.width(), _size.height(), 0, + GL_BGRA, GL_UNSIGNED_BYTE, image.constBits()); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + + } else { + glBindTexture(GL_TEXTURE_2D, _billboardTexture->getID()); } - _size = image.size(); - if (_fromImage.x() == -1) { - _fromImage.setRect(0, 0, _size.width(), _size.height()); - } - _billboardTexture.reset(new Texture()); - glBindTexture(GL_TEXTURE_2D, _billboardTexture->getID()); - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, _size.width(), _size.height(), 0, - GL_BGRA, GL_UNSIGNED_BYTE, image.constBits()); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - - } else { - glBindTexture(GL_TEXTURE_2D, _billboardTexture->getID()); } glEnable(GL_ALPHA_TEST); @@ -62,25 +61,35 @@ void BillboardOverlay::render() { } glScalef(_scale, _scale, _scale); - float maxSize = glm::max(_fromImage.width(), _fromImage.height()); - float x = _fromImage.width() / (2.0f * maxSize); - float y = -_fromImage.height() / (2.0f * maxSize); - - glColor3f(1.0f, 1.0f, 1.0f); - glBegin(GL_QUADS); { - glTexCoord2f((float)_fromImage.x() / (float)_size.width(), - (float)_fromImage.y() / (float)_size.height()); - glVertex2f(-x, -y); - glTexCoord2f(((float)_fromImage.x() + (float)_fromImage.width()) / (float)_size.width(), - (float)_fromImage.y() / (float)_size.height()); - glVertex2f(x, -y); - glTexCoord2f(((float)_fromImage.x() + (float)_fromImage.width()) / (float)_size.width(), - ((float)_fromImage.y() + (float)_fromImage.height()) / _size.height()); - glVertex2f(x, y); - glTexCoord2f((float)_fromImage.x() / (float)_size.width(), - ((float)_fromImage.y() + (float)_fromImage.height()) / (float)_size.height()); - glVertex2f(-x, y); - } glEnd(); + if (_billboardTexture) { + float maxSize = glm::max(_fromImage.width(), _fromImage.height()); + float x = _fromImage.width() / (2.0f * maxSize); + float y = -_fromImage.height() / (2.0f * maxSize); + + glColor3f(1.0f, 1.0f, 1.0f); + glBegin(GL_QUADS); { + glTexCoord2f((float)_fromImage.x() / (float)_size.width(), + (float)_fromImage.y() / (float)_size.height()); + glVertex2f(-x, -y); + glTexCoord2f(((float)_fromImage.x() + (float)_fromImage.width()) / (float)_size.width(), + (float)_fromImage.y() / (float)_size.height()); + glVertex2f(x, -y); + glTexCoord2f(((float)_fromImage.x() + (float)_fromImage.width()) / (float)_size.width(), + ((float)_fromImage.y() + (float)_fromImage.height()) / _size.height()); + glVertex2f(x, y); + glTexCoord2f((float)_fromImage.x() / (float)_size.width(), + ((float)_fromImage.y() + (float)_fromImage.height()) / (float)_size.height()); + glVertex2f(-x, y); + } glEnd(); + } else { + glColor4f(0.5f, 0.5f, 0.5f, 1.0f); + glBegin(GL_QUADS); { + glVertex2f(-1.0f, -1.0f); + glVertex2f(1.0f, -1.0f); + glVertex2f(1.0f, 1.0f); + glVertex2f(-1.0f, 1.0f); + } glEnd(); + } } glPopMatrix(); From e067a013a0ee7643f1c2a5a5247a5c0be35aa83f Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 11 Jul 2014 11:22:38 -0700 Subject: [PATCH 66/70] Made sitting point more consistant --- examples/sit.js | 2 +- libraries/models/src/ModelsScriptingInterface.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/sit.js b/examples/sit.js index b51d59759a..c157d4854d 100644 --- a/examples/sit.js +++ b/examples/sit.js @@ -261,7 +261,7 @@ function update(deltaTime){ MyAvatar.position.z != avatarOldPosition.z) { avatarOldPosition = MyAvatar.position; - var SEARCH_RADIUS = 5; + var SEARCH_RADIUS = 10; var foundModels = Models.findModels(MyAvatar.position, SEARCH_RADIUS); // Let's remove indicator that got out of radius for (model in models) { diff --git a/libraries/models/src/ModelsScriptingInterface.cpp b/libraries/models/src/ModelsScriptingInterface.cpp index bac1213071..634039f949 100644 --- a/libraries/models/src/ModelsScriptingInterface.cpp +++ b/libraries/models/src/ModelsScriptingInterface.cpp @@ -69,8 +69,9 @@ ModelItemProperties ModelsScriptingInterface::getModelProperties(ModelItemID mod } if (_modelTree) { _modelTree->lockForRead(); - const ModelItem* model = _modelTree->findModelByID(identity.id, true); + ModelItem* model = const_cast(_modelTree->findModelByID(identity.id, true)); if (model) { + model->setSittingPoints(_modelTree->getGeometryForModel(*model)->sittingPoints); results.copyFromModelItem(*model); } else { results.setIsUnknownID(); From 4c3a1d192e8b4504be67e2575e5756f86c698693 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Fri, 11 Jul 2014 11:29:32 -0700 Subject: [PATCH 67/70] fix crash in particle and model server from attempting to edit items that dont exist in the server --- libraries/models/src/ModelItem.cpp | 12 ++++++------ libraries/particles/src/Particle.cpp | 14 ++++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/libraries/models/src/ModelItem.cpp b/libraries/models/src/ModelItem.cpp index 8c061102a0..750af8f1b6 100644 --- a/libraries/models/src/ModelItem.cpp +++ b/libraries/models/src/ModelItem.cpp @@ -331,6 +331,7 @@ ModelItem ModelItem::fromEditPacket(const unsigned char* data, int length, int& newModelItem.setCreatorTokenID(creatorTokenID); newModelItem._newlyCreated = true; + valid = true; } else { // look up the existing modelItem @@ -339,20 +340,19 @@ ModelItem ModelItem::fromEditPacket(const unsigned char* data, int length, int& // copy existing properties before over-writing with new properties if (existingModelItem) { newModelItem = *existingModelItem; + valid = true; } else { // the user attempted to edit a modelItem that doesn't exist - qDebug() << "user attempted to edit a modelItem that doesn't exist..."; + qDebug() << "user attempted to edit a modelItem that doesn't exist... editID=" << editID; + + // NOTE: even though this is a bad editID, we have to consume the edit details, so that + // the buffer doesn't get corrupted for further processing... valid = false; - return newModelItem; } newModelItem._id = editID; newModelItem._newlyCreated = false; } - // if we got this far, then our result will be valid - valid = true; - - // lastEdited memcpy(&newModelItem._lastEdited, dataAt, sizeof(newModelItem._lastEdited)); dataAt += sizeof(newModelItem._lastEdited); diff --git a/libraries/particles/src/Particle.cpp b/libraries/particles/src/Particle.cpp index 59265c00dc..4dfe4b588e 100644 --- a/libraries/particles/src/Particle.cpp +++ b/libraries/particles/src/Particle.cpp @@ -385,6 +385,8 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr newParticle._newlyCreated = true; newParticle.setAge(0); // this guy is new! + valid = true; + } else { // look up the existing particle const Particle* existingParticle = tree->findParticleByID(editID, true); @@ -392,20 +394,20 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr // copy existing properties before over-writing with new properties if (existingParticle) { newParticle = *existingParticle; + valid = true; + } else { // the user attempted to edit a particle that doesn't exist - qDebug() << "user attempted to edit a particle that doesn't exist..."; + qDebug() << "user attempted to edit a particle that doesn't exist... editID=" << editID; + + // NOTE: even though this is a bad particle ID, we have to consume the edit details, so that + // the buffer doesn't get corrupted for further processing... valid = false; - return newParticle; } newParticle._id = editID; newParticle._newlyCreated = false; } - // if we got this far, then our result will be valid - valid = true; - - // lastEdited memcpy(&newParticle._lastEdited, dataAt, sizeof(newParticle._lastEdited)); dataAt += sizeof(newParticle._lastEdited); From 7ad90ac7bbdb627cecf36565750814c704bfa327 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 11 Jul 2014 11:28:08 -0700 Subject: [PATCH 68/70] When visible is false, don't render billboards/models overlays --- interface/src/ui/overlays/BillboardOverlay.cpp | 4 ++++ interface/src/ui/overlays/ModelOverlay.cpp | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/interface/src/ui/overlays/BillboardOverlay.cpp b/interface/src/ui/overlays/BillboardOverlay.cpp index 4bd854114d..e7d5cef3be 100644 --- a/interface/src/ui/overlays/BillboardOverlay.cpp +++ b/interface/src/ui/overlays/BillboardOverlay.cpp @@ -20,6 +20,10 @@ BillboardOverlay::BillboardOverlay() } void BillboardOverlay::render() { + if (!_visible) { + return; + } + if (!_billboard.isEmpty()) { if (!_billboardTexture) { QImage image = QImage::fromData(_billboard); diff --git a/interface/src/ui/overlays/ModelOverlay.cpp b/interface/src/ui/overlays/ModelOverlay.cpp index bc0cc720c2..57f098aee3 100644 --- a/interface/src/ui/overlays/ModelOverlay.cpp +++ b/interface/src/ui/overlays/ModelOverlay.cpp @@ -35,6 +35,10 @@ void ModelOverlay::update(float deltatime) { } void ModelOverlay::render() { + if (!_visible) { + return; + } + if (_model.isActive()) { if (_model.isRenderable()) { From d832f5bec8f7e63aee3167a8299e0d9b697386b9 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Fri, 11 Jul 2014 16:08:37 -0700 Subject: [PATCH 69/70] fix menu config of visible stats --- interface/src/Menu.cpp | 10 ++++----- interface/src/Menu.h | 12 +++++------ interface/src/ui/Stats.cpp | 43 ++++++++++++++------------------------ 3 files changed, 25 insertions(+), 40 deletions(-) diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index f2ec217656..f18dde7cbf 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -441,13 +441,11 @@ Menu::Menu() : QMenu* timingMenu = developerMenu->addMenu("Timing and Statistics Tools"); QMenu* perfTimerMenu = timingMenu->addMenu("Performance Timer"); addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::DisplayTimingDetails, 0, true); - addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandDisplaySideTiming, 0, false); - addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandAvatarSimulateTiming, 0, false); - addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandAvatarUpdateTiming, 0, false); - addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandMiscAvatarTiming, 0, false); - addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandIdleTiming, 0, false); - addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandPaintGLTiming, 0, false); addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandUpdateTiming, 0, false); + addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandMyAvatarTiming, 0, false); + addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandMyAvatarSimulateTiming, 0, false); + addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandOtherAvatarTiming, 0, false); + addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandPaintGLTiming, 0, false); addCheckableActionToQMenuAndActionHash(timingMenu, MenuOption::TestPing, 0, true); addCheckableActionToQMenuAndActionHash(timingMenu, MenuOption::FrameTimer); diff --git a/interface/src/Menu.h b/interface/src/Menu.h index 4ca50f71b8..651cf3288a 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -357,13 +357,11 @@ namespace MenuOption { const QString EnableGlowEffect = "Enable Glow Effect (Warning: Poor Oculus Performance)"; const QString Enable3DTVMode = "Enable 3DTV Mode"; const QString EnableVRMode = "Enable VR Mode"; - const QString ExpandMiscAvatarTiming = "Expand Misc MyAvatar Timing"; - const QString ExpandAvatarUpdateTiming = "Expand MyAvatar update Timing"; - const QString ExpandAvatarSimulateTiming = "Expand MyAvatar simulate Timing"; - const QString ExpandDisplaySideTiming = "Expand Display Side Timing"; - const QString ExpandIdleTiming = "Expand Idle Timing"; - const QString ExpandPaintGLTiming = "Expand PaintGL Timing"; - const QString ExpandUpdateTiming = "Expand Update Timing"; + const QString ExpandMyAvatarSimulateTiming = "Expand /myAvatar/simulation"; + const QString ExpandMyAvatarTiming = "Expand /myAvatar"; + const QString ExpandOtherAvatarTiming = "Expand /otherAvatar"; + const QString ExpandPaintGLTiming = "Expand /paintGL"; + const QString ExpandUpdateTiming = "Expand /update"; const QString Faceplus = "Faceplus"; const QString Faceshift = "Faceshift"; const QString FilterSixense = "Smooth Sixense Movement"; diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 15a54e42a6..00f9450eda 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -162,36 +162,25 @@ void Stats::drawBackground(unsigned int rgba, int x, int y, int width, int heigh } bool Stats::includeTimingRecord(const QString& name) { - bool included = false; if (Menu::getInstance()->isOptionChecked(MenuOption::DisplayTimingDetails)) { - - if (name == "idle/update") { - included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandUpdateTiming) || - Menu::getInstance()->isOptionChecked(MenuOption::ExpandIdleTiming); - } else if (name == "idle/updateGL") { - included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandIdleTiming); - } else if (name.startsWith("idle/update")) { - included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandUpdateTiming); - } else if (name.startsWith("idle/")) { - included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandIdleTiming); - } else if (name.startsWith("MyAvatar::simulate")) { - included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandAvatarSimulateTiming); - } else if (name.startsWith("MyAvatar::update/") || name.startsWith("updateMyAvatar")) { - included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandAvatarUpdateTiming); - } else if (name.startsWith("MyAvatar::")) { - included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandMiscAvatarTiming); - } else if (name == "paintGL/displaySide") { - included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandDisplaySideTiming) || - Menu::getInstance()->isOptionChecked(MenuOption::ExpandPaintGLTiming); - } else if (name.startsWith("paintGL/displaySide/")) { - included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandDisplaySideTiming); - } else if (name.startsWith("paintGL/")) { - included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandPaintGLTiming); - } else { - included = true; // include everything else + if (name.startsWith("/idle/update/")) { + if (name.startsWith("/idle/update/myAvatar/")) { + if (name.startsWith("/idle/update/myAvatar/simulate/")) { + return Menu::getInstance()->isOptionChecked(MenuOption::ExpandMyAvatarSimulateTiming); + } + return Menu::getInstance()->isOptionChecked(MenuOption::ExpandMyAvatarTiming); + } else if (name.startsWith("/idle/update/otherAvatars/")) { + return Menu::getInstance()->isOptionChecked(MenuOption::ExpandOtherAvatarTiming); + } + return Menu::getInstance()->isOptionChecked(MenuOption::ExpandUpdateTiming); + } else if (name.startsWith("/idle/updateGL/paintGL/")) { + return Menu::getInstance()->isOptionChecked(MenuOption::ExpandPaintGLTiming); + } else if (name.startsWith("/paintGL/")) { + return Menu::getInstance()->isOptionChecked(MenuOption::ExpandPaintGLTiming); } + return true; } - return included; + return false; } // display expanded or contracted stats From 9939c00a40fad20c57db1b63cab664b67e74403c Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Fri, 11 Jul 2014 16:57:08 -0700 Subject: [PATCH 70/70] more guards to prevent crashing in bad particle edit packets --- libraries/particles/src/Particle.cpp | 114 ++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 2 deletions(-) diff --git a/libraries/particles/src/Particle.cpp b/libraries/particles/src/Particle.cpp index 4dfe4b588e..76890afafe 100644 --- a/libraries/particles/src/Particle.cpp +++ b/libraries/particles/src/Particle.cpp @@ -357,15 +357,23 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr processedBytes = 0; // the first part of the data is our octcode... - int octets = numberOfThreeBitSectionsInCode(data); + int octets = numberOfThreeBitSectionsInCode(data, length); int lengthOfOctcode = bytesRequiredForCodeLength(octets); // we don't actually do anything with this octcode... dataAt += lengthOfOctcode; processedBytes += lengthOfOctcode; - + // id uint32_t editID; + + // check to make sure we have enough content to keep reading... + if (processedBytes + sizeof(editID) > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } + memcpy(&editID, dataAt, sizeof(editID)); dataAt += sizeof(editID); processedBytes += sizeof(editID); @@ -377,6 +385,14 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr // If this is a NEW_PARTICLE, then we assume that there's an additional uint32_t creatorToken, that // we want to send back to the creator as an map to the actual id uint32_t creatorTokenID; + + // check to make sure we have enough content to keep reading... + if (processedBytes + sizeof(creatorTokenID) > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } + memcpy(&creatorTokenID, dataAt, sizeof(creatorTokenID)); dataAt += sizeof(creatorTokenID); processedBytes += sizeof(creatorTokenID); @@ -409,6 +425,12 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr } // lastEdited + // check to make sure we have enough content to keep reading... + if (processedBytes + sizeof(newParticle._lastEdited) > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } memcpy(&newParticle._lastEdited, dataAt, sizeof(newParticle._lastEdited)); dataAt += sizeof(newParticle._lastEdited); processedBytes += sizeof(newParticle._lastEdited); @@ -417,6 +439,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr // properties included bits uint16_t packetContainsBits = 0; if (!isNewParticle) { + if (processedBytes + sizeof(packetContainsBits) > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } memcpy(&packetContainsBits, dataAt, sizeof(packetContainsBits)); dataAt += sizeof(packetContainsBits); processedBytes += sizeof(packetContainsBits); @@ -425,6 +452,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr // radius if (isNewParticle || ((packetContainsBits & CONTAINS_RADIUS) == CONTAINS_RADIUS)) { + if (processedBytes + sizeof(newParticle._radius) > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } memcpy(&newParticle._radius, dataAt, sizeof(newParticle._radius)); dataAt += sizeof(newParticle._radius); processedBytes += sizeof(newParticle._radius); @@ -432,6 +464,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr // position if (isNewParticle || ((packetContainsBits & CONTAINS_POSITION) == CONTAINS_POSITION)) { + if (processedBytes + sizeof(newParticle._position) > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } memcpy(&newParticle._position, dataAt, sizeof(newParticle._position)); dataAt += sizeof(newParticle._position); processedBytes += sizeof(newParticle._position); @@ -439,6 +476,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr // color if (isNewParticle || ((packetContainsBits & CONTAINS_COLOR) == CONTAINS_COLOR)) { + if (processedBytes + sizeof(newParticle._color) > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } memcpy(newParticle._color, dataAt, sizeof(newParticle._color)); dataAt += sizeof(newParticle._color); processedBytes += sizeof(newParticle._color); @@ -446,6 +488,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr // velocity if (isNewParticle || ((packetContainsBits & CONTAINS_VELOCITY) == CONTAINS_VELOCITY)) { + if (processedBytes + sizeof(newParticle._velocity) > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } memcpy(&newParticle._velocity, dataAt, sizeof(newParticle._velocity)); dataAt += sizeof(newParticle._velocity); processedBytes += sizeof(newParticle._velocity); @@ -453,6 +500,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr // gravity if (isNewParticle || ((packetContainsBits & CONTAINS_GRAVITY) == CONTAINS_GRAVITY)) { + if (processedBytes + sizeof(newParticle._gravity) > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } memcpy(&newParticle._gravity, dataAt, sizeof(newParticle._gravity)); dataAt += sizeof(newParticle._gravity); processedBytes += sizeof(newParticle._gravity); @@ -460,6 +512,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr // damping if (isNewParticle || ((packetContainsBits & CONTAINS_DAMPING) == CONTAINS_DAMPING)) { + if (processedBytes + sizeof(newParticle._damping) > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } memcpy(&newParticle._damping, dataAt, sizeof(newParticle._damping)); dataAt += sizeof(newParticle._damping); processedBytes += sizeof(newParticle._damping); @@ -467,6 +524,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr // lifetime if (isNewParticle || ((packetContainsBits & CONTAINS_LIFETIME) == CONTAINS_LIFETIME)) { + if (processedBytes + sizeof(newParticle._lifetime) > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } memcpy(&newParticle._lifetime, dataAt, sizeof(newParticle._lifetime)); dataAt += sizeof(newParticle._lifetime); processedBytes += sizeof(newParticle._lifetime); @@ -475,6 +537,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr // TODO: make inHand and shouldDie into single bits // inHand if (isNewParticle || ((packetContainsBits & CONTAINS_INHAND) == CONTAINS_INHAND)) { + if (processedBytes + sizeof(newParticle._inHand) > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } memcpy(&newParticle._inHand, dataAt, sizeof(newParticle._inHand)); dataAt += sizeof(newParticle._inHand); processedBytes += sizeof(newParticle._inHand); @@ -482,6 +549,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr // shouldDie if (isNewParticle || ((packetContainsBits & CONTAINS_SHOULDDIE) == CONTAINS_SHOULDDIE)) { + if (processedBytes + sizeof(newParticle._shouldDie) > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } memcpy(&newParticle._shouldDie, dataAt, sizeof(newParticle._shouldDie)); dataAt += sizeof(newParticle._shouldDie); processedBytes += sizeof(newParticle._shouldDie); @@ -490,9 +562,20 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr // script if (isNewParticle || ((packetContainsBits & CONTAINS_SCRIPT) == CONTAINS_SCRIPT)) { uint16_t scriptLength; + if (processedBytes + sizeof(scriptLength) > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } memcpy(&scriptLength, dataAt, sizeof(scriptLength)); dataAt += sizeof(scriptLength); processedBytes += sizeof(scriptLength); + + if (processedBytes + scriptLength > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } QString tempString((const char*)dataAt); newParticle._script = tempString; dataAt += scriptLength; @@ -502,9 +585,20 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr // modelURL if (isNewParticle || ((packetContainsBits & CONTAINS_MODEL_URL) == CONTAINS_MODEL_URL)) { uint16_t modelURLLength; + if (processedBytes + sizeof(modelURLLength) > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } memcpy(&modelURLLength, dataAt, sizeof(modelURLLength)); dataAt += sizeof(modelURLLength); processedBytes += sizeof(modelURLLength); + + if (processedBytes + modelURLLength > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } QString tempString((const char*)dataAt); newParticle._modelURL = tempString; dataAt += modelURLLength; @@ -513,6 +607,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr // modelScale if (isNewParticle || ((packetContainsBits & CONTAINS_MODEL_SCALE) == CONTAINS_MODEL_SCALE)) { + if (processedBytes + sizeof(newParticle._modelScale) > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } memcpy(&newParticle._modelScale, dataAt, sizeof(newParticle._modelScale)); dataAt += sizeof(newParticle._modelScale); processedBytes += sizeof(newParticle._modelScale); @@ -520,6 +619,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr // modelTranslation if (isNewParticle || ((packetContainsBits & CONTAINS_MODEL_TRANSLATION) == CONTAINS_MODEL_TRANSLATION)) { + if (processedBytes + sizeof(newParticle._modelTranslation) > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } memcpy(&newParticle._modelTranslation, dataAt, sizeof(newParticle._modelTranslation)); dataAt += sizeof(newParticle._modelTranslation); processedBytes += sizeof(newParticle._modelTranslation); @@ -527,6 +631,12 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr // modelRotation if (isNewParticle || ((packetContainsBits & CONTAINS_MODEL_ROTATION) == CONTAINS_MODEL_ROTATION)) { + const int expectedBytesForPackedQuat = sizeof(uint16_t) * 4; // this is how we pack the quats + if (processedBytes + expectedBytesForPackedQuat > length) { + valid = false; + processedBytes = length; + return newParticle; // fail as if we read the entire buffer + } int bytes = unpackOrientationQuatFromBytes(dataAt, newParticle._modelRotation); dataAt += bytes; processedBytes += bytes;