From 905670ae7fc3fcbe206953467505df9f2b2abd71 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 16 Jul 2014 15:41:46 -0700 Subject: [PATCH 01/46] Added files for FaceshiftScriptingInterface --- .../src/FaceshiftScriptingInterface.cpp | 18 ++++++++++++ .../src/FaceshiftScriptingInterface.h | 29 +++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 libraries/animation/src/FaceshiftScriptingInterface.cpp create mode 100644 libraries/animation/src/FaceshiftScriptingInterface.h diff --git a/libraries/animation/src/FaceshiftScriptingInterface.cpp b/libraries/animation/src/FaceshiftScriptingInterface.cpp new file mode 100644 index 0000000000..0dd191c18b --- /dev/null +++ b/libraries/animation/src/FaceshiftScriptingInterface.cpp @@ -0,0 +1,18 @@ +// +// FaceshiftScriptingInterface.cpp +// interface/src/scripting +// +// Created by Ben Arnold on 7/38/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "Application.h" +#include "FaceshiftScriptingInterface.h" + +FaceshiftScriptingInterface* FaceshiftScriptingInterface::getInstance() { + static FaceshiftScriptingInterface sharedInstance; + return &sharedInstance; +} \ No newline at end of file diff --git a/libraries/animation/src/FaceshiftScriptingInterface.h b/libraries/animation/src/FaceshiftScriptingInterface.h new file mode 100644 index 0000000000..b365f66df6 --- /dev/null +++ b/libraries/animation/src/FaceshiftScriptingInterface.h @@ -0,0 +1,29 @@ +// +// FaceshiftScriptingInterface.h +// interface/src/scripting +// +// Created by Ben Arnold on 7/38/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_FaceshiftScriptingInterface_h +#define hifi_FaceshiftScriptingInterface_h + +#include +#include +#include + +class FaceshiftScriptingInterface : public QObject { + Q_OBJECT + FaceshiftScriptingInterface() { }; +public: + static FaceshiftScriptingInterface* getInstance(); + + public slots: + +}; + +#endif // hifi_FaceshiftScriptingInterface_h From 88de5949a4c19d39609d651667c551263c34daed Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 16 Jul 2014 16:34:30 -0700 Subject: [PATCH 02/46] Fleshed out FaceshiftScriptingInterface --- interface/src/Application.cpp | 2 + .../src/FaceshiftScriptingInterface.cpp | 89 +++++++++++++++++++ .../src/FaceshiftScriptingInterface.h | 36 +++++++- 3 files changed, 125 insertions(+), 2 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index bc6db5d7b6..045d54291e 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -70,6 +70,7 @@ #include "Menu.h" #include "ModelUploader.h" #include "Util.h" +#include "FaceshiftScriptingInterface.h" #include "devices/MIDIManager.h" #include "devices/OculusManager.h" #include "devices/TV3DManager.h" @@ -3638,6 +3639,7 @@ ScriptEngine* Application::loadScript(const QString& scriptName, bool loadScript scriptEngine->registerGlobalObject("Menu", MenuScriptingInterface::getInstance()); scriptEngine->registerGlobalObject("Settings", SettingsScriptingInterface::getInstance()); scriptEngine->registerGlobalObject("AudioDevice", AudioDeviceScriptingInterface::getInstance()); + scriptEngine->registerGlobalObject("Faceshift", FaceshiftScriptingInterface::getInstance()); scriptEngine->registerGlobalObject("AnimationCache", &_animationCache); scriptEngine->registerGlobalObject("AudioReflector", &_audioReflector); scriptEngine->registerGlobalObject("Account", AccountScriptingInterface::getInstance()); diff --git a/libraries/animation/src/FaceshiftScriptingInterface.cpp b/libraries/animation/src/FaceshiftScriptingInterface.cpp index 0dd191c18b..64118f0f2f 100644 --- a/libraries/animation/src/FaceshiftScriptingInterface.cpp +++ b/libraries/animation/src/FaceshiftScriptingInterface.cpp @@ -15,4 +15,93 @@ FaceshiftScriptingInterface* FaceshiftScriptingInterface::getInstance() { static FaceshiftScriptingInterface sharedInstance; return &sharedInstance; +} + + +bool FaceshiftScriptingInterface::isConnectedOrConnecting() const { + return Application::getInstance()->getFaceshift()->isConnectedOrConnecting(); +} + +bool FaceshiftScriptingInterface::isActive() const { + return Application::getInstance()->getFaceshift()->isActive(); +} + +const glm::vec3& FaceshiftScriptingInterface::getHeadAngularVelocity() const { + return Application::getInstance()->getFaceshift()->getHeadAngularVelocity(); +} + +// these pitch/yaw angles are in degrees +float FaceshiftScriptingInterface::getEyeGazeLeftPitch() const { + return Application::getInstance()->getFaceshift()->getEyeGazeLeftPitch(); +} +float FaceshiftScriptingInterface::getEyeGazeLeftYaw() const { + return Application::getInstance()->getFaceshift()->getEyeGazeLeftYaw(); +} + +float FaceshiftScriptingInterface::getEyeGazeRightPitch() const { + return Application::getInstance()->getFaceshift()->getEyeGazeRightPitch(); +} +float FaceshiftScriptingInterface::getEyeGazeRightYaw() const { + return Application::getInstance()->getFaceshift()->getEyeGazeRightYaw(); +} + +float FaceshiftScriptingInterface::getLeftBlink() const { + return Application::getInstance()->getFaceshift()->getLeftBlink(); +} + +float FaceshiftScriptingInterface::getRightBlink() const { + return Application::getInstance()->getFaceshift()->getRightBlink(); +} + +float FaceshiftScriptingInterface::getLeftEyeOpen() const { + return Application::getInstance()->getFaceshift()->getLeftEyeOpen(); +} + +float FaceshiftScriptingInterface::getRightEyeOpen() const { + return Application::getInstance()->getFaceshift()->getRightEyeOpen(); +} + +float FaceshiftScriptingInterface::getBrowDownLeft() const { + return Application::getInstance()->getFaceshift()->getBrowDownLeft(); +} + +float FaceshiftScriptingInterface::getBrowDownRight() const { + return Application::getInstance()->getFaceshift()->getBrowDownRight(); +} + +float FaceshiftScriptingInterface::getBrowUpCenter() const { + return Application::getInstance()->getFaceshift()->getBrowUpCenter(); +} + +float FaceshiftScriptingInterface::getBrowUpLeft() const { + return Application::getInstance()->getFaceshift()->getBrowUpLeft(); +} + +float FaceshiftScriptingInterface::getBrowUpRight() const { + return Application::getInstance()->getFaceshift()->getBrowUpRight(); +} + +float FaceshiftScriptingInterface::getMouthSize() const { + return Application::getInstance()->getFaceshift()->getMouthSize(); +} + +float FaceshiftScriptingInterface::getMouthSmileLeft() const { + return Application::getInstance()->getFaceshift()->getMouthSmileLeft(); +} + +float FaceshiftScriptingInterface::getMouthSmileRight() const { + return Application::getInstance()->getFaceshift()->getMouthSmileRight(); +} + +void FaceshiftScriptingInterface::update() { + Application::getInstance()->getFaceshift()->update(); +} + +void FaceshiftScriptingInterface::reset() { + Application::getInstance()->getFaceshift()->reset; +} + +void FaceshiftScriptingInterface::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, + float jawOpen, QVector& coefficients) const { + Application::getInstance()->getFaceshift()->updateFakeCoefficients(leftBlink, rightBlink, browUp, jawOpen, coefficients); } \ No newline at end of file diff --git a/libraries/animation/src/FaceshiftScriptingInterface.h b/libraries/animation/src/FaceshiftScriptingInterface.h index b365f66df6..f52dd953fa 100644 --- a/libraries/animation/src/FaceshiftScriptingInterface.h +++ b/libraries/animation/src/FaceshiftScriptingInterface.h @@ -22,8 +22,40 @@ class FaceshiftScriptingInterface : public QObject { public: static FaceshiftScriptingInterface* getInstance(); - public slots: - +public slots: + bool isConnectedOrConnecting() const; + + bool isActive() const; + + const glm::vec3& getHeadAngularVelocity() const; + + // these pitch/yaw angles are in degrees + float getEyeGazeLeftPitch() const; + float getEyeGazeLeftYaw() const; + + float getEyeGazeRightPitch() const; + float getEyeGazeRightYaw() const; + + float getLeftBlink() const; + float getRightBlink() const; + float getLeftEyeOpen() const; + float getRightEyeOpen() const; + + float getBrowDownLeft() const; + float getBrowDownRight() const; + float getBrowUpCenter() const; + float getBrowUpLeft() const; + float getBrowUpRight() const; + + float getMouthSize() const; + float getMouthSmileLeft() const; + float getMouthSmileRight() const; + + void update(); + void reset(); + + void updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, + float jawOpen, QVector& coefficients) const; }; #endif // hifi_FaceshiftScriptingInterface_h From 90dc3cf5de918405de811333d041e8c911c90700 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 16 Jul 2014 18:24:03 -0700 Subject: [PATCH 03/46] Small changes --- libraries/animation/src/FaceshiftScriptingInterface.cpp | 4 +--- libraries/animation/src/FaceshiftScriptingInterface.h | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/animation/src/FaceshiftScriptingInterface.cpp b/libraries/animation/src/FaceshiftScriptingInterface.cpp index 64118f0f2f..7de217e7d1 100644 --- a/libraries/animation/src/FaceshiftScriptingInterface.cpp +++ b/libraries/animation/src/FaceshiftScriptingInterface.cpp @@ -9,7 +9,6 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#include "Application.h" #include "FaceshiftScriptingInterface.h" FaceshiftScriptingInterface* FaceshiftScriptingInterface::getInstance() { @@ -17,7 +16,6 @@ FaceshiftScriptingInterface* FaceshiftScriptingInterface::getInstance() { return &sharedInstance; } - bool FaceshiftScriptingInterface::isConnectedOrConnecting() const { return Application::getInstance()->getFaceshift()->isConnectedOrConnecting(); } @@ -98,7 +96,7 @@ void FaceshiftScriptingInterface::update() { } void FaceshiftScriptingInterface::reset() { - Application::getInstance()->getFaceshift()->reset; + Application::getInstance()->getFaceshift()->reset(); } void FaceshiftScriptingInterface::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, diff --git a/libraries/animation/src/FaceshiftScriptingInterface.h b/libraries/animation/src/FaceshiftScriptingInterface.h index f52dd953fa..3d7a6000e5 100644 --- a/libraries/animation/src/FaceshiftScriptingInterface.h +++ b/libraries/animation/src/FaceshiftScriptingInterface.h @@ -16,6 +16,8 @@ #include #include +#include "Application.h" + class FaceshiftScriptingInterface : public QObject { Q_OBJECT FaceshiftScriptingInterface() { }; From f8772e10a1c8ebf79167a790518fcb509f4c8bd3 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 18 Jul 2014 15:38:36 -0700 Subject: [PATCH 04/46] Changed enum names --- interface/src/avatar/MyAvatar.cpp | 3 --- interface/src/avatar/MyAvatar.h | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index a58266fa71..bc955d88f5 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -137,9 +137,6 @@ void MyAvatar::simulate(float deltaTime) { } _skeletonModel.setShowTrueJointTransforms(! Menu::getInstance()->isOptionChecked(MenuOption::CollideAsRagdoll)); - // no extra movement of the hand here any more ... - _handState = HAND_STATE_NULL; - { PerformanceTimer perfTimer("transform"); updateOrientation(deltaTime); diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 0b2254bafa..5f2785e527 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -21,9 +21,9 @@ enum AvatarHandState { HAND_STATE_NULL = 0, - HAND_STATE_OPEN, - HAND_STATE_GRASPING, - HAND_STATE_POINTING, + HAND_STATE_LEFT_POINTING, + HAND_STATE_RIGTH_POINTING, + HAND_STATE_BOTH_POINTING, NUM_HAND_STATES }; From 35caef18825b66698871637053b83dce97b826b4 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 18 Jul 2014 15:39:12 -0700 Subject: [PATCH 05/46] Added pointing behaviour --- interface/src/avatar/Avatar.cpp | 46 ++++++++++++++++++++++++++++++ interface/src/avatar/Hand.cpp | 2 +- libraries/avatars/src/AvatarData.h | 4 +-- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index b5590dec09..4dd82a4652 100644 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -215,6 +215,52 @@ static TextRenderer* textRenderer(TextRendererType type) { } void Avatar::render(const glm::vec3& cameraPosition, RenderMode renderMode) { + + if (glm::distance(Application::getInstance()->getAvatar()->getPosition(), + _position) < 10.0f) { + // render pointing lasers + glm::vec3 laserColor = glm::vec3(1.0f, 0.0f, 1.0f); + float laserLength = 50.0f; + if (_handState == HAND_STATE_LEFT_POINTING || + _handState == HAND_STATE_BOTH_POINTING) { + int leftIndex = _skeletonModel.getLeftHandJointIndex(); + glm::vec3 leftPosition; + glm::quat leftRotation; + _skeletonModel.getJointPositionInWorldFrame(leftIndex, leftPosition); + _skeletonModel.getJointRotationInWorldFrame(leftIndex, leftRotation); + glPushMatrix(); { + glTranslatef(leftPosition.x, leftPosition.y, leftPosition.z); + float angle = glm::degrees(glm::angle(leftRotation)); + glm::vec3 axis = glm::axis(leftRotation); + glRotatef(angle, axis.x, axis.y, axis.z); + glBegin(GL_LINES); + glColor3f(laserColor.x, laserColor.y, laserColor.z); + glVertex3f(0.0f, 0.0f, 0.0f); + glVertex3f(0.0f, laserLength, 0.0f); + glEnd(); + } glPopMatrix(); + } + if (_handState == HAND_STATE_RIGTH_POINTING || + _handState == HAND_STATE_BOTH_POINTING) { + int rightIndex = _skeletonModel.getRightHandJointIndex(); + glm::vec3 rightPosition; + glm::quat rightRotation; + _skeletonModel.getJointPositionInWorldFrame(rightIndex, rightPosition); + _skeletonModel.getJointRotationInWorldFrame(rightIndex, rightRotation); + glPushMatrix(); { + glTranslatef(rightPosition.x, rightPosition.y, rightPosition.z); + float angle = glm::degrees(glm::angle(rightRotation)); + glm::vec3 axis = glm::axis(rightRotation); + glRotatef(angle, axis.x, axis.y, axis.z); + glBegin(GL_LINES); + glColor3f(laserColor.x, laserColor.y, laserColor.z); + glVertex3f(0.0f, 0.0f, 0.0f); + glVertex3f(0.0f, laserLength, 0.0f); + glEnd(); + } glPopMatrix(); + } + } + // simple frustum check float boundingRadius = getBillboardSize(); ViewFrustum* frustum = (renderMode == Avatar::SHADOW_RENDER_MODE) ? diff --git a/interface/src/avatar/Hand.cpp b/interface/src/avatar/Hand.cpp index 08e1cf83df..5ef1fbafea 100644 --- a/interface/src/avatar/Hand.cpp +++ b/interface/src/avatar/Hand.cpp @@ -125,7 +125,7 @@ void Hand::render(bool isMine, Model::RenderMode renderMode) { glEnable(GL_DEPTH_TEST); glEnable(GL_RESCALE_NORMAL); -} +} void Hand::renderHandTargets(bool isMine) { glPushMatrix(); diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 1b8f6e6007..433a52035b 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -185,8 +185,8 @@ public: void setClampedTargetScale(float targetScale); // Hand State - void setHandState(char s) { _handState = s; } - char getHandState() const { return _handState; } + Q_INVOKABLE void setHandState(char s) { _handState = s; } + Q_INVOKABLE char getHandState() const { return _handState; } const QVector& getJointData() const { return _jointData; } void setJointData(const QVector& jointData) { _jointData = jointData; } From 91e980f96efde4fd503f5fe132c696cf77590849 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 18 Jul 2014 15:39:47 -0700 Subject: [PATCH 06/46] Added laserPointer script --- examples/laserPointer.js | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 examples/laserPointer.js diff --git a/examples/laserPointer.js b/examples/laserPointer.js new file mode 100644 index 0000000000..bedafe6a18 --- /dev/null +++ b/examples/laserPointer.js @@ -0,0 +1,23 @@ +// +// laserPointer.js +// examples +// +// Created by Clément Brisset on 7/18/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +var LEFT = 0; +var RIGHT = 1; +var LEFT_HAND_FLAG = 1; +var RIGHT_HAND_FLAG = 2; + +function update() { + var state = ((Controller.getTriggerValue(LEFT) > 0.9) ? LEFT_HAND_FLAG : 0) + + ((Controller.getTriggerValue(RIGHT) > 0.9) ? RIGHT_HAND_FLAG : 0); + MyAvatar.setHandState(state); +} + +Script.update.connect(update); \ No newline at end of file From 17d827b66d4e960bb22507d44f19f8f7f8a08fc1 Mon Sep 17 00:00:00 2001 From: Mohammed Nafees Date: Tue, 22 Jul 2014 01:23:05 +0530 Subject: [PATCH 07/46] Show script instance with script name in Running Scripts Widget --- interface/src/ui/RunningScriptsWidget.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/interface/src/ui/RunningScriptsWidget.cpp b/interface/src/ui/RunningScriptsWidget.cpp index 8a7ebcbfd4..5bcb1a85de 100644 --- a/interface/src/ui/RunningScriptsWidget.cpp +++ b/interface/src/ui/RunningScriptsWidget.cpp @@ -23,7 +23,6 @@ #include "Menu.h" #include "ScriptsModel.h" - RunningScriptsWidget::RunningScriptsWidget(QWidget* parent) : FramelessDialog(parent, 0, POSITION_LEFT), ui(new Ui::RunningScriptsWidget), @@ -104,13 +103,21 @@ void RunningScriptsWidget::setRunningScripts(const QStringList& list) { delete widget->widget(); delete widget; } + QHash hash; const int CLOSE_ICON_HEIGHT = 12; for (int i = 0; i < list.size(); i++) { + if (!hash.contains(list.at(i))) { + hash.insert(list.at(i), 1); + } QWidget* row = new QWidget(ui->scrollAreaWidgetContents); row->setLayout(new QHBoxLayout(row)); QUrl url = QUrl(list.at(i)); QLabel* name = new QLabel(url.fileName(), row); + if (hash.find(list.at(i)).value() != 1) { + name->setText(name->text() + "(" + QString::number(hash.find(list.at(i)).value()) + ")"); + } + ++hash[list.at(i)]; QPushButton* closeButton = new QPushButton(row); closeButton->setFlat(true); closeButton->setIcon( From 192fe71ac464259e826062640449c6dd88ef7398 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Mon, 21 Jul 2014 14:24:05 -0700 Subject: [PATCH 08/46] removed faceShiftScriptingInterface --- interface/src/Application.cpp | 2 - .../src/FaceshiftScriptingInterface.cpp | 105 ------------------ .../src/FaceshiftScriptingInterface.h | 63 ----------- 3 files changed, 170 deletions(-) delete mode 100644 libraries/animation/src/FaceshiftScriptingInterface.cpp delete mode 100644 libraries/animation/src/FaceshiftScriptingInterface.h diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 045d54291e..bc6db5d7b6 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -70,7 +70,6 @@ #include "Menu.h" #include "ModelUploader.h" #include "Util.h" -#include "FaceshiftScriptingInterface.h" #include "devices/MIDIManager.h" #include "devices/OculusManager.h" #include "devices/TV3DManager.h" @@ -3639,7 +3638,6 @@ ScriptEngine* Application::loadScript(const QString& scriptName, bool loadScript scriptEngine->registerGlobalObject("Menu", MenuScriptingInterface::getInstance()); scriptEngine->registerGlobalObject("Settings", SettingsScriptingInterface::getInstance()); scriptEngine->registerGlobalObject("AudioDevice", AudioDeviceScriptingInterface::getInstance()); - scriptEngine->registerGlobalObject("Faceshift", FaceshiftScriptingInterface::getInstance()); scriptEngine->registerGlobalObject("AnimationCache", &_animationCache); scriptEngine->registerGlobalObject("AudioReflector", &_audioReflector); scriptEngine->registerGlobalObject("Account", AccountScriptingInterface::getInstance()); diff --git a/libraries/animation/src/FaceshiftScriptingInterface.cpp b/libraries/animation/src/FaceshiftScriptingInterface.cpp deleted file mode 100644 index 7de217e7d1..0000000000 --- a/libraries/animation/src/FaceshiftScriptingInterface.cpp +++ /dev/null @@ -1,105 +0,0 @@ -// -// FaceshiftScriptingInterface.cpp -// interface/src/scripting -// -// Created by Ben Arnold on 7/38/14. -// Copyright 2014 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "FaceshiftScriptingInterface.h" - -FaceshiftScriptingInterface* FaceshiftScriptingInterface::getInstance() { - static FaceshiftScriptingInterface sharedInstance; - return &sharedInstance; -} - -bool FaceshiftScriptingInterface::isConnectedOrConnecting() const { - return Application::getInstance()->getFaceshift()->isConnectedOrConnecting(); -} - -bool FaceshiftScriptingInterface::isActive() const { - return Application::getInstance()->getFaceshift()->isActive(); -} - -const glm::vec3& FaceshiftScriptingInterface::getHeadAngularVelocity() const { - return Application::getInstance()->getFaceshift()->getHeadAngularVelocity(); -} - -// these pitch/yaw angles are in degrees -float FaceshiftScriptingInterface::getEyeGazeLeftPitch() const { - return Application::getInstance()->getFaceshift()->getEyeGazeLeftPitch(); -} -float FaceshiftScriptingInterface::getEyeGazeLeftYaw() const { - return Application::getInstance()->getFaceshift()->getEyeGazeLeftYaw(); -} - -float FaceshiftScriptingInterface::getEyeGazeRightPitch() const { - return Application::getInstance()->getFaceshift()->getEyeGazeRightPitch(); -} -float FaceshiftScriptingInterface::getEyeGazeRightYaw() const { - return Application::getInstance()->getFaceshift()->getEyeGazeRightYaw(); -} - -float FaceshiftScriptingInterface::getLeftBlink() const { - return Application::getInstance()->getFaceshift()->getLeftBlink(); -} - -float FaceshiftScriptingInterface::getRightBlink() const { - return Application::getInstance()->getFaceshift()->getRightBlink(); -} - -float FaceshiftScriptingInterface::getLeftEyeOpen() const { - return Application::getInstance()->getFaceshift()->getLeftEyeOpen(); -} - -float FaceshiftScriptingInterface::getRightEyeOpen() const { - return Application::getInstance()->getFaceshift()->getRightEyeOpen(); -} - -float FaceshiftScriptingInterface::getBrowDownLeft() const { - return Application::getInstance()->getFaceshift()->getBrowDownLeft(); -} - -float FaceshiftScriptingInterface::getBrowDownRight() const { - return Application::getInstance()->getFaceshift()->getBrowDownRight(); -} - -float FaceshiftScriptingInterface::getBrowUpCenter() const { - return Application::getInstance()->getFaceshift()->getBrowUpCenter(); -} - -float FaceshiftScriptingInterface::getBrowUpLeft() const { - return Application::getInstance()->getFaceshift()->getBrowUpLeft(); -} - -float FaceshiftScriptingInterface::getBrowUpRight() const { - return Application::getInstance()->getFaceshift()->getBrowUpRight(); -} - -float FaceshiftScriptingInterface::getMouthSize() const { - return Application::getInstance()->getFaceshift()->getMouthSize(); -} - -float FaceshiftScriptingInterface::getMouthSmileLeft() const { - return Application::getInstance()->getFaceshift()->getMouthSmileLeft(); -} - -float FaceshiftScriptingInterface::getMouthSmileRight() const { - return Application::getInstance()->getFaceshift()->getMouthSmileRight(); -} - -void FaceshiftScriptingInterface::update() { - Application::getInstance()->getFaceshift()->update(); -} - -void FaceshiftScriptingInterface::reset() { - Application::getInstance()->getFaceshift()->reset(); -} - -void FaceshiftScriptingInterface::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, - float jawOpen, QVector& coefficients) const { - Application::getInstance()->getFaceshift()->updateFakeCoefficients(leftBlink, rightBlink, browUp, jawOpen, coefficients); -} \ No newline at end of file diff --git a/libraries/animation/src/FaceshiftScriptingInterface.h b/libraries/animation/src/FaceshiftScriptingInterface.h deleted file mode 100644 index 3d7a6000e5..0000000000 --- a/libraries/animation/src/FaceshiftScriptingInterface.h +++ /dev/null @@ -1,63 +0,0 @@ -// -// FaceshiftScriptingInterface.h -// interface/src/scripting -// -// Created by Ben Arnold on 7/38/14. -// Copyright 2014 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_FaceshiftScriptingInterface_h -#define hifi_FaceshiftScriptingInterface_h - -#include -#include -#include - -#include "Application.h" - -class FaceshiftScriptingInterface : public QObject { - Q_OBJECT - FaceshiftScriptingInterface() { }; -public: - static FaceshiftScriptingInterface* getInstance(); - -public slots: - bool isConnectedOrConnecting() const; - - bool isActive() const; - - const glm::vec3& getHeadAngularVelocity() const; - - // these pitch/yaw angles are in degrees - float getEyeGazeLeftPitch() const; - float getEyeGazeLeftYaw() const; - - float getEyeGazeRightPitch() const; - float getEyeGazeRightYaw() const; - - float getLeftBlink() const; - float getRightBlink() const; - float getLeftEyeOpen() const; - float getRightEyeOpen() const; - - float getBrowDownLeft() const; - float getBrowDownRight() const; - float getBrowUpCenter() const; - float getBrowUpLeft() const; - float getBrowUpRight() const; - - float getMouthSize() const; - float getMouthSmileLeft() const; - float getMouthSmileRight() const; - - void update(); - void reset(); - - void updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, - float jawOpen, QVector& coefficients) const; -}; - -#endif // hifi_FaceshiftScriptingInterface_h From 847bc28990cea9580c6e1b0814ce1c4cf3d04878 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 22 Jul 2014 10:27:26 -0700 Subject: [PATCH 09/46] Audio stats now reset when AudioMixer is killed --- interface/src/Application.cpp | 2 +- interface/src/Audio.cpp | 19 ++++++++++++++++++- interface/src/Audio.h | 3 ++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 1d5cbabd32..50f0af8723 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -3338,7 +3338,7 @@ void Application::nodeKilled(SharedNodePointer node) { _modelEditSender.nodeKilled(node); if (node->getType() == NodeType::AudioMixer) { - QMetaObject::invokeMethod(&_audio, "resetIncomingMixedAudioSequenceNumberStats"); + QMetaObject::invokeMethod(&_audio, "audioMixerKilled"); } if (node->getType() == NodeType::VoxelServer) { diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index be348ab635..7cf19fbcfe 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -152,11 +152,28 @@ void Audio::reset() { _starveCount = 0; _consecutiveNotMixedCount = 0; + _outgoingAvatarAudioSequenceNumber = 0; + + resetStats(); +} + +void Audio::resetStats() { _audioMixerAvatarStreamAudioStats = AudioStreamStats(); _audioMixerInjectedStreamAudioStatsMap.clear(); - _outgoingAvatarAudioSequenceNumber = 0; _incomingMixedAudioSequenceNumberStats.reset(); + + _interframeTimeGapStats.reset(); + + _inputRingBufferFramesAvailableStats.reset(); + + _outputRingBufferFramesAvailableStats.reset(); + _audioOutputBufferFramesAvailableStats.reset(); +} + +void Audio::audioMixerKilled() { + _outgoingAvatarAudioSequenceNumber = 0; + resetStats(); } QAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName) { diff --git a/interface/src/Audio.h b/interface/src/Audio.h index ed50815d78..f91208b6e4 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -93,7 +93,8 @@ public slots: void addSpatialAudioToBuffer(unsigned int sampleTime, const QByteArray& spatialAudio, unsigned int numSamples); void handleAudioInput(); void reset(); - void resetIncomingMixedAudioSequenceNumberStats() { _incomingMixedAudioSequenceNumberStats.reset(); } + void resetStats(); + void audioMixerKilled(); void toggleMute(); void toggleAudioNoiseReduction(); void toggleToneInjection(); From f7381c70bc9713aab8709bcbde1c93771d172cc5 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 22 Jul 2014 11:09:47 -0700 Subject: [PATCH 10/46] starvcount and notmixedcount also reset when audiomixer is killed --- interface/src/Audio.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 7cf19fbcfe..8e3662e790 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -148,9 +148,6 @@ void Audio::init(QGLWidget *parent) { void Audio::reset() { _ringBuffer.reset(); - - _starveCount = 0; - _consecutiveNotMixedCount = 0; _outgoingAvatarAudioSequenceNumber = 0; @@ -158,6 +155,9 @@ void Audio::reset() { } void Audio::resetStats() { + _starveCount = 0; + _consecutiveNotMixedCount = 0; + _audioMixerAvatarStreamAudioStats = AudioStreamStats(); _audioMixerInjectedStreamAudioStatsMap.clear(); From 17a6a72bdd5a6de499228c6b4ed8d491698e96a8 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 22 Jul 2014 11:39:49 -0700 Subject: [PATCH 11/46] ScriptEngine now inherit from QScriptEngine --- .../script-engine/src/ArrayBufferClass.cpp | 4 +- .../src/ArrayBufferViewClass.cpp | 4 +- libraries/script-engine/src/ScriptEngine.cpp | 142 +++++++++--------- libraries/script-engine/src/ScriptEngine.h | 4 +- 4 files changed, 74 insertions(+), 80 deletions(-) diff --git a/libraries/script-engine/src/ArrayBufferClass.cpp b/libraries/script-engine/src/ArrayBufferClass.cpp index ab33b5ffe7..b84188f707 100644 --- a/libraries/script-engine/src/ArrayBufferClass.cpp +++ b/libraries/script-engine/src/ArrayBufferClass.cpp @@ -23,8 +23,8 @@ static const QString CLASS_NAME = "ArrayBuffer"; Q_DECLARE_METATYPE(QByteArray*) ArrayBufferClass::ArrayBufferClass(ScriptEngine* scriptEngine) : -QObject(scriptEngine->getEngine()), -QScriptClass(scriptEngine->getEngine()), +QObject(scriptEngine), +QScriptClass(scriptEngine), _scriptEngine(scriptEngine) { qScriptRegisterMetaType(engine(), toScriptValue, fromScriptValue); QScriptValue global = engine()->globalObject(); diff --git a/libraries/script-engine/src/ArrayBufferViewClass.cpp b/libraries/script-engine/src/ArrayBufferViewClass.cpp index aad2e6add7..cf776ed834 100644 --- a/libraries/script-engine/src/ArrayBufferViewClass.cpp +++ b/libraries/script-engine/src/ArrayBufferViewClass.cpp @@ -14,8 +14,8 @@ Q_DECLARE_METATYPE(QByteArray*) ArrayBufferViewClass::ArrayBufferViewClass(ScriptEngine* scriptEngine) : -QObject(scriptEngine->getEngine()), -QScriptClass(scriptEngine->getEngine()), +QObject(scriptEngine), +QScriptClass(scriptEngine), _scriptEngine(scriptEngine) { // Save string handles for quick lookup _bufferName = engine()->toStringHandle(BUFFER_PROPERTY_NAME.toLatin1()); diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index fab21ea928..df66fa44d5 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -87,7 +87,6 @@ ScriptEngine::ScriptEngine(const QString& scriptContents, const QString& fileNam _isFinished(false), _isRunning(false), _isInitialized(false), - _engine(), _isAvatar(false), _avatarIdentityTimer(NULL), _avatarBillboardTimer(NULL), @@ -113,7 +112,6 @@ ScriptEngine::ScriptEngine(const QUrl& scriptURL, _isFinished(false), _isRunning(false), _isInitialized(false), - _engine(), _isAvatar(false), _avatarIdentityTimer(NULL), _avatarBillboardTimer(NULL), @@ -194,7 +192,7 @@ void ScriptEngine::setAvatarData(AvatarData* avatarData, const QString& objectNa _avatarData = avatarData; // remove the old Avatar property, if it exists - _engine.globalObject().setProperty(objectName, QScriptValue()); + globalObject().setProperty(objectName, QScriptValue()); // give the script engine the new Avatar script property registerGlobalObject(objectName, _avatarData); @@ -202,7 +200,7 @@ void ScriptEngine::setAvatarData(AvatarData* avatarData, const QString& objectNa void ScriptEngine::setAvatarHashMap(AvatarHashMap* avatarHashMap, const QString& objectName) { // remove the old Avatar property, if it exists - _engine.globalObject().setProperty(objectName, QScriptValue()); + globalObject().setProperty(objectName, QScriptValue()); // give the script engine the new avatar hash map registerGlobalObject(objectName, avatarHashMap); @@ -231,48 +229,48 @@ void ScriptEngine::init() { _particlesScriptingInterface.init(); // register various meta-types - registerMetaTypes(&_engine); - registerMIDIMetaTypes(&_engine); - registerVoxelMetaTypes(&_engine); - registerEventTypes(&_engine); - registerMenuItemProperties(&_engine); - registerAnimationTypes(&_engine); - registerAvatarTypes(&_engine); - Bitstream::registerTypes(&_engine); + registerMetaTypes(this); + registerMIDIMetaTypes(this); + registerVoxelMetaTypes(this); + registerEventTypes(this); + registerMenuItemProperties(this); + registerAnimationTypes(this); + registerAvatarTypes(this); + Bitstream::registerTypes(this); - qScriptRegisterMetaType(&_engine, ParticlePropertiesToScriptValue, ParticlePropertiesFromScriptValue); - qScriptRegisterMetaType(&_engine, ParticleIDtoScriptValue, ParticleIDfromScriptValue); - qScriptRegisterSequenceMetaType >(&_engine); + qScriptRegisterMetaType(this, ParticlePropertiesToScriptValue, ParticlePropertiesFromScriptValue); + qScriptRegisterMetaType(this, ParticleIDtoScriptValue, ParticleIDfromScriptValue); + qScriptRegisterSequenceMetaType >(this); - qScriptRegisterMetaType(&_engine, ModelItemPropertiesToScriptValue, ModelItemPropertiesFromScriptValue); - qScriptRegisterMetaType(&_engine, ModelItemIDtoScriptValue, ModelItemIDfromScriptValue); - qScriptRegisterMetaType(&_engine, RayToModelIntersectionResultToScriptValue, RayToModelIntersectionResultFromScriptValue); - qScriptRegisterSequenceMetaType >(&_engine); + qScriptRegisterMetaType(this, ModelItemPropertiesToScriptValue, ModelItemPropertiesFromScriptValue); + qScriptRegisterMetaType(this, ModelItemIDtoScriptValue, ModelItemIDfromScriptValue); + qScriptRegisterMetaType(this, RayToModelIntersectionResultToScriptValue, RayToModelIntersectionResultFromScriptValue); + qScriptRegisterSequenceMetaType >(this); - qScriptRegisterSequenceMetaType >(&_engine); - qScriptRegisterSequenceMetaType >(&_engine); - qScriptRegisterSequenceMetaType >(&_engine); + qScriptRegisterSequenceMetaType >(this); + qScriptRegisterSequenceMetaType >(this); + qScriptRegisterSequenceMetaType >(this); - QScriptValue xmlHttpRequestConstructorValue = _engine.newFunction(XMLHttpRequestClass::constructor); - _engine.globalObject().setProperty("XMLHttpRequest", xmlHttpRequestConstructorValue); + QScriptValue xmlHttpRequestConstructorValue = newFunction(XMLHttpRequestClass::constructor); + globalObject().setProperty("XMLHttpRequest", xmlHttpRequestConstructorValue); - QScriptValue printConstructorValue = _engine.newFunction(debugPrint); - _engine.globalObject().setProperty("print", printConstructorValue); + QScriptValue printConstructorValue = newFunction(debugPrint); + globalObject().setProperty("print", printConstructorValue); - QScriptValue soundConstructorValue = _engine.newFunction(soundConstructor); - QScriptValue soundMetaObject = _engine.newQMetaObject(&Sound::staticMetaObject, soundConstructorValue); - _engine.globalObject().setProperty("Sound", soundMetaObject); + QScriptValue soundConstructorValue = newFunction(soundConstructor); + QScriptValue soundMetaObject = newQMetaObject(&Sound::staticMetaObject, soundConstructorValue); + globalObject().setProperty("Sound", soundMetaObject); - QScriptValue injectionOptionValue = _engine.scriptValueFromQMetaObject(); - _engine.globalObject().setProperty("AudioInjectionOptions", injectionOptionValue); + QScriptValue injectionOptionValue = scriptValueFromQMetaObject(); + globalObject().setProperty("AudioInjectionOptions", injectionOptionValue); - QScriptValue localVoxelsValue = _engine.scriptValueFromQMetaObject(); - _engine.globalObject().setProperty("LocalVoxels", localVoxelsValue); + QScriptValue localVoxelsValue = scriptValueFromQMetaObject(); + globalObject().setProperty("LocalVoxels", localVoxelsValue); - qScriptRegisterMetaType(&_engine, injectorToScriptValue, injectorFromScriptValue); - qScriptRegisterMetaType( &_engine, injectorToScriptValueInputController, injectorFromScriptValueInputController); + qScriptRegisterMetaType(this, injectorToScriptValue, injectorFromScriptValue); + qScriptRegisterMetaType( this, injectorToScriptValueInputController, injectorFromScriptValueInputController); - qScriptRegisterMetaType(&_engine, animationDetailsToScriptValue, animationDetailsFromScriptValue); + qScriptRegisterMetaType(this, animationDetailsToScriptValue, animationDetailsFromScriptValue); registerGlobalObject("Script", this); registerGlobalObject("Audio", &_audioScriptingInterface); @@ -287,15 +285,14 @@ void ScriptEngine::init() { registerGlobalObject("Voxels", &_voxelsScriptingInterface); // constants - QScriptValue globalObject = _engine.globalObject(); - globalObject.setProperty("TREE_SCALE", _engine.newVariant(QVariant(TREE_SCALE))); - globalObject.setProperty("COLLISION_GROUP_ENVIRONMENT", _engine.newVariant(QVariant(COLLISION_GROUP_ENVIRONMENT))); - globalObject.setProperty("COLLISION_GROUP_AVATARS", _engine.newVariant(QVariant(COLLISION_GROUP_AVATARS))); - globalObject.setProperty("COLLISION_GROUP_VOXELS", _engine.newVariant(QVariant(COLLISION_GROUP_VOXELS))); - globalObject.setProperty("COLLISION_GROUP_PARTICLES", _engine.newVariant(QVariant(COLLISION_GROUP_PARTICLES))); + globalObject().setProperty("TREE_SCALE", newVariant(QVariant(TREE_SCALE))); + globalObject().setProperty("COLLISION_GROUP_ENVIRONMENT", newVariant(QVariant(COLLISION_GROUP_ENVIRONMENT))); + globalObject().setProperty("COLLISION_GROUP_AVATARS", newVariant(QVariant(COLLISION_GROUP_AVATARS))); + globalObject().setProperty("COLLISION_GROUP_VOXELS", newVariant(QVariant(COLLISION_GROUP_VOXELS))); + globalObject().setProperty("COLLISION_GROUP_PARTICLES", newVariant(QVariant(COLLISION_GROUP_PARTICLES))); - globalObject.setProperty("AVATAR_MOTION_OBEY_LOCAL_GRAVITY", _engine.newVariant(QVariant(AVATAR_MOTION_OBEY_LOCAL_GRAVITY))); - globalObject.setProperty("AVATAR_MOTION_OBEY_ENVIRONMENTAL_GRAVITY", _engine.newVariant(QVariant(AVATAR_MOTION_OBEY_ENVIRONMENTAL_GRAVITY))); + globalObject().setProperty("AVATAR_MOTION_OBEY_LOCAL_GRAVITY", newVariant(QVariant(AVATAR_MOTION_OBEY_LOCAL_GRAVITY))); + globalObject().setProperty("AVATAR_MOTION_OBEY_ENVIRONMENTAL_GRAVITY", newVariant(QVariant(AVATAR_MOTION_OBEY_ENVIRONMENTAL_GRAVITY))); // let the VoxelPacketSender know how frequently we plan to call it _voxelsScriptingInterface.getVoxelPacketSender()->setProcessCallIntervalHint(SCRIPT_DATA_CALLBACK_USECS); @@ -304,8 +301,8 @@ void ScriptEngine::init() { QScriptValue ScriptEngine::registerGlobalObject(const QString& name, QObject* object) { if (object) { - QScriptValue value = _engine.newQObject(object); - _engine.globalObject().setProperty(name, value); + QScriptValue value = newQObject(object); + globalObject().setProperty(name, value); return value; } return QScriptValue::NullValue; @@ -313,15 +310,15 @@ QScriptValue ScriptEngine::registerGlobalObject(const QString& name, QObject* ob void ScriptEngine::registerGetterSetter(const QString& name, QScriptEngine::FunctionSignature getter, QScriptEngine::FunctionSignature setter, QScriptValue object) { - QScriptValue setterFunction = _engine.newFunction(setter, 1); - QScriptValue getterFunction = _engine.newFunction(getter); + QScriptValue setterFunction = newFunction(setter, 1); + QScriptValue getterFunction = newFunction(getter); if (!object.isNull()) { object.setProperty(name, setterFunction, QScriptValue::PropertySetter); object.setProperty(name, getterFunction, QScriptValue::PropertyGetter); } else { - _engine.globalObject().setProperty(name, setterFunction, QScriptValue::PropertySetter); - _engine.globalObject().setProperty(name, getterFunction, QScriptValue::PropertyGetter); + globalObject().setProperty(name, setterFunction, QScriptValue::PropertySetter); + globalObject().setProperty(name, getterFunction, QScriptValue::PropertyGetter); } } @@ -330,25 +327,24 @@ void ScriptEngine::evaluate() { init(); } - QScriptValue result = _engine.evaluate(_scriptContents); + QScriptValue result = evaluate(_scriptContents); - if (_engine.hasUncaughtException()) { - int line = _engine.uncaughtExceptionLineNumber(); + if (hasUncaughtException()) { + int line = uncaughtExceptionLineNumber(); qDebug() << "Uncaught exception at (" << _fileNameString << ") line" << line << ":" << result.toString(); emit errorMessage("Uncaught exception at (" + _fileNameString + ") line" + QString::number(line) + ":" + result.toString()); - _engine.clearExceptions(); + clearExceptions(); } } QScriptValue ScriptEngine::evaluate(const QString& program, const QString& fileName, int lineNumber) { - QScriptValue result = _engine.evaluate(program, fileName, lineNumber); - bool hasUncaughtException = _engine.hasUncaughtException(); - if (hasUncaughtException) { - int line = _engine.uncaughtExceptionLineNumber(); + QScriptValue result = QScriptEngine::evaluate(program, fileName, lineNumber); + if (hasUncaughtException()) { + int line = uncaughtExceptionLineNumber(); qDebug() << "Uncaught exception at (" << _fileNameString << ") line" << line << ": " << result.toString(); } - emit evaluationFinished(result, hasUncaughtException); - _engine.clearExceptions(); + emit evaluationFinished(result, hasUncaughtException()); + clearExceptions(); return result; } @@ -372,12 +368,12 @@ void ScriptEngine::run() { _isFinished = false; emit runningStateChanged(); - QScriptValue result = _engine.evaluate(_scriptContents); - if (_engine.hasUncaughtException()) { - int line = _engine.uncaughtExceptionLineNumber(); + QScriptValue result = evaluate(_scriptContents); + if (hasUncaughtException()) { + int line = uncaughtExceptionLineNumber(); qDebug() << "Uncaught exception at (" << _fileNameString << ") line" << line << ":" << result.toString(); emit errorMessage("Uncaught exception at (" + _fileNameString + ") line" + QString::number(line) + ":" + result.toString()); - _engine.clearExceptions(); + clearExceptions(); } QElapsedTimer startTime; @@ -532,11 +528,11 @@ void ScriptEngine::run() { qint64 now = usecTimestampNow(); float deltaTime = (float) (now - lastUpdate) / (float) USECS_PER_SECOND; - if (_engine.hasUncaughtException()) { - int line = _engine.uncaughtExceptionLineNumber(); - qDebug() << "Uncaught exception at (" << _fileNameString << ") line" << line << ":" << _engine.uncaughtException().toString(); - emit errorMessage("Uncaught exception at (" + _fileNameString + ") line" + QString::number(line) + ":" + _engine.uncaughtException().toString()); - _engine.clearExceptions(); + if (hasUncaughtException()) { + int line = uncaughtExceptionLineNumber(); + qDebug() << "Uncaught exception at (" << _fileNameString << ") line" << line << ":" << uncaughtException().toString(); + emit errorMessage("Uncaught exception at (" + _fileNameString + ") line" + QString::number(line) + ":" + uncaughtException().toString()); + clearExceptions(); } emit update(deltaTime); @@ -694,12 +690,12 @@ void ScriptEngine::include(const QString& includeFile) { } } - QScriptValue result = _engine.evaluate(includeContents); - if (_engine.hasUncaughtException()) { - int line = _engine.uncaughtExceptionLineNumber(); + QScriptValue result = evaluate(includeContents); + if (hasUncaughtException()) { + int line = uncaughtExceptionLineNumber(); qDebug() << "Uncaught exception at (" << includeFile << ") line" << line << ":" << result.toString(); emit errorMessage("Uncaught exception at (" + includeFile + ") line" + QString::number(line) + ":" + result.toString()); - _engine.clearExceptions(); + clearExceptions(); } } diff --git a/libraries/script-engine/src/ScriptEngine.h b/libraries/script-engine/src/ScriptEngine.h index 0eda74914f..53c2a72b00 100644 --- a/libraries/script-engine/src/ScriptEngine.h +++ b/libraries/script-engine/src/ScriptEngine.h @@ -38,7 +38,7 @@ const QString NO_SCRIPT(""); const unsigned int SCRIPT_DATA_CALLBACK_USECS = floor(((1.0 / 60.0f) * 1000 * 1000) + 0.5); -class ScriptEngine : public QObject { +class ScriptEngine : public QScriptEngine { Q_OBJECT public: ScriptEngine(const QUrl& scriptURL, @@ -57,7 +57,6 @@ public: /// Access the ModelsScriptingInterface in order to initialize it with a custom packet sender and jurisdiction listener static ModelsScriptingInterface* getModelsScriptingInterface() { return &_modelsScriptingInterface; } - QScriptEngine* getEngine() { return &_engine; } ArrayBufferClass* getArrayBufferClass() { return _arrayBufferClass; } /// sets the script contents, will return false if failed, will fail if script is already running @@ -121,7 +120,6 @@ protected: bool _isFinished; bool _isRunning; bool _isInitialized; - QScriptEngine _engine; bool _isAvatar; QTimer* _avatarIdentityTimer; QTimer* _avatarBillboardTimer; From fc9fb11e4f6039f5c70bf0e00de2f4c82984929f Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Tue, 22 Jul 2014 11:56:19 -0700 Subject: [PATCH 12/46] Added FacialAnimationData class --- examples/bot.js | 2 +- interface/src/devices/Faceshift.cpp | 50 ++++++-------- interface/src/devices/Faceshift.h | 43 +++++------- .../animation/src/FacialAnimationData.cpp | 30 +++++++++ libraries/animation/src/FacialAnimationData.h | 65 +++++++++++++++++++ 5 files changed, 130 insertions(+), 60 deletions(-) create mode 100644 libraries/animation/src/FacialAnimationData.cpp create mode 100644 libraries/animation/src/FacialAnimationData.h diff --git a/examples/bot.js b/examples/bot.js index e42d234abf..edef280e21 100644 --- a/examples/bot.js +++ b/examples/bot.js @@ -92,7 +92,7 @@ function playRandomSound() { botNumber = getRandomInt(1, 100); if (botNumber <= 20) { - newFaceFilePrefix = "bot" + botNumber; + newFaceFilePrefix = "amber"; newBodyFilePrefix = "defaultAvatar_body" } else { if (botNumber <= 40) { diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index a7d50814e2..4105b17652 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -32,18 +32,6 @@ Faceshift::Faceshift() : _eyeGazeLeftYaw(0.0f), _eyeGazeRightPitch(0.0f), _eyeGazeRightYaw(0.0f), - _leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes - _rightBlinkIndex(1), - _leftEyeOpenIndex(8), - _rightEyeOpenIndex(9), - _browDownLeftIndex(14), - _browDownRightIndex(15), - _browUpCenterIndex(16), - _browUpLeftIndex(17), - _browUpRightIndex(18), - _mouthSmileLeftIndex(28), - _mouthSmileRightIndex(29), - _jawOpenIndex(21), _longTermAverageEyePitch(0.0f), _longTermAverageEyeYaw(0.0f), _longTermAverageInitialized(false) @@ -107,14 +95,14 @@ void Faceshift::reset() { void Faceshift::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, float jawOpen, QVector& coefficients) const { - coefficients.resize(max((int)coefficients.size(), _jawOpenIndex + 1)); + coefficients.resize(max((int)coefficients.size(), _facialAnimationData._jawOpenIndex + 1)); qFill(coefficients.begin(), coefficients.end(), 0.0f); - coefficients[_leftBlinkIndex] = leftBlink; - coefficients[_rightBlinkIndex] = rightBlink; - coefficients[_browUpCenterIndex] = browUp; - coefficients[_browUpLeftIndex] = browUp; - coefficients[_browUpRightIndex] = browUp; - coefficients[_jawOpenIndex] = jawOpen; + coefficients[_facialAnimationData._leftBlinkIndex] = leftBlink; + coefficients[_facialAnimationData._rightBlinkIndex] = rightBlink; + coefficients[_facialAnimationData._browUpCenterIndex] = browUp; + coefficients[_facialAnimationData._browUpLeftIndex] = browUp; + coefficients[_facialAnimationData._browUpRightIndex] = browUp; + coefficients[_facialAnimationData._jawOpenIndex] = jawOpen; } void Faceshift::setTCPEnabled(bool enabled) { @@ -217,40 +205,40 @@ void Faceshift::receive(const QByteArray& buffer) { const vector& names = static_cast(msg.get())->blendshape_names(); for (size_t i = 0; i < names.size(); i++) { if (names[i] == "EyeBlink_L") { - _leftBlinkIndex = i; + _facialAnimationData._leftBlinkIndex = i; } else if (names[i] == "EyeBlink_R") { - _rightBlinkIndex = i; + _facialAnimationData._rightBlinkIndex = i; } else if (names[i] == "EyeOpen_L") { - _leftEyeOpenIndex = i; + _facialAnimationData._leftEyeOpenIndex = i; } else if (names[i] == "EyeOpen_R") { - _rightEyeOpenIndex = i; + _facialAnimationData._rightEyeOpenIndex = i; } else if (names[i] == "BrowsD_L") { - _browDownLeftIndex = i; + _facialAnimationData._browDownLeftIndex = i; } else if (names[i] == "BrowsD_R") { - _browDownRightIndex = i; + _facialAnimationData._browDownRightIndex = i; } else if (names[i] == "BrowsU_C") { - _browUpCenterIndex = i; + _facialAnimationData._browUpCenterIndex = i; } else if (names[i] == "BrowsU_L") { - _browUpLeftIndex = i; + _facialAnimationData._browUpLeftIndex = i; } else if (names[i] == "BrowsU_R") { - _browUpRightIndex = i; + _facialAnimationData._browUpRightIndex = i; } else if (names[i] == "JawOpen") { - _jawOpenIndex = i; + _facialAnimationData._jawOpenIndex = i; } else if (names[i] == "MouthSmile_L") { - _mouthSmileLeftIndex = i; + _facialAnimationData._mouthSmileLeftIndex = i; } else if (names[i] == "MouthSmile_R") { - _mouthSmileRightIndex = i; + _facialAnimationData._mouthSmileRightIndex = i; } } break; diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 2fc1aaddb1..87fd1076ee 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -18,6 +18,7 @@ #include #include "FaceTracker.h" +#include "FacialAnimationData.h" /// Handles interaction with the Faceshift software, which provides head position/orientation and facial features. class Faceshift : public FaceTracker { @@ -42,20 +43,20 @@ public: float getEyeGazeRightPitch() const { return _eyeGazeRightPitch; } float getEyeGazeRightYaw() const { return _eyeGazeRightYaw; } - float getLeftBlink() const { return getBlendshapeCoefficient(_leftBlinkIndex); } - float getRightBlink() const { return getBlendshapeCoefficient(_rightBlinkIndex); } - float getLeftEyeOpen() const { return getBlendshapeCoefficient(_leftEyeOpenIndex); } - float getRightEyeOpen() const { return getBlendshapeCoefficient(_rightEyeOpenIndex); } + float getLeftBlink() const { return getBlendshapeCoefficient(_facialAnimationData._leftBlinkIndex); } + float getRightBlink() const { return getBlendshapeCoefficient(_facialAnimationData._rightBlinkIndex); } + float getLeftEyeOpen() const { return getBlendshapeCoefficient(_facialAnimationData._leftEyeOpenIndex); } + float getRightEyeOpen() const { return getBlendshapeCoefficient(_facialAnimationData._rightEyeOpenIndex); } - float getBrowDownLeft() const { return getBlendshapeCoefficient(_browDownLeftIndex); } - float getBrowDownRight() const { return getBlendshapeCoefficient(_browDownRightIndex); } - float getBrowUpCenter() const { return getBlendshapeCoefficient(_browUpCenterIndex); } - float getBrowUpLeft() const { return getBlendshapeCoefficient(_browUpLeftIndex); } - float getBrowUpRight() const { return getBlendshapeCoefficient(_browUpRightIndex); } + float getBrowDownLeft() const { return getBlendshapeCoefficient(_facialAnimationData._browDownLeftIndex); } + float getBrowDownRight() const { return getBlendshapeCoefficient(_facialAnimationData._browDownRightIndex); } + float getBrowUpCenter() const { return getBlendshapeCoefficient(_facialAnimationData._browUpCenterIndex); } + float getBrowUpLeft() const { return getBlendshapeCoefficient(_facialAnimationData._browUpLeftIndex); } + float getBrowUpRight() const { return getBlendshapeCoefficient(_facialAnimationData._browUpRightIndex); } - float getMouthSize() const { return getBlendshapeCoefficient(_jawOpenIndex); } - float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); } - float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); } + float getMouthSize() const { return getBlendshapeCoefficient(_facialAnimationData._jawOpenIndex); } + float getMouthSmileLeft() const { return getBlendshapeCoefficient(_facialAnimationData._mouthSmileLeftIndex); } + float getMouthSmileRight() const { return getBlendshapeCoefficient(_facialAnimationData._mouthSmileRightIndex); } void update(); void reset(); @@ -102,23 +103,9 @@ private: float _eyeGazeRightPitch; float _eyeGazeRightYaw; - int _leftBlinkIndex; - int _rightBlinkIndex; - int _leftEyeOpenIndex; - int _rightEyeOpenIndex; + // stores blendshape indexes + FacialAnimationData _facialAnimationData; - // Brows - int _browDownLeftIndex; - int _browDownRightIndex; - int _browUpCenterIndex; - int _browUpLeftIndex; - int _browUpRightIndex; - - int _mouthSmileLeftIndex; - int _mouthSmileRightIndex; - - int _jawOpenIndex; - // degrees float _longTermAverageEyePitch; float _longTermAverageEyeYaw; diff --git a/libraries/animation/src/FacialAnimationData.cpp b/libraries/animation/src/FacialAnimationData.cpp new file mode 100644 index 0000000000..0f45675083 --- /dev/null +++ b/libraries/animation/src/FacialAnimationData.cpp @@ -0,0 +1,30 @@ +// +// FacialAnimationData.cpp +// interface/src/devices +// +// Created by Ben Arnold on 7/21/2014. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "FacialAnimationData.h" + +FacialAnimationData::FacialAnimationData() :_leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes +_rightBlinkIndex(1), +_leftEyeOpenIndex(8), +_rightEyeOpenIndex(9), +_browDownLeftIndex(14), +_browDownRightIndex(15), +_browUpCenterIndex(16), +_browUpLeftIndex(17), +_browUpRightIndex(18), +_mouthSmileLeftIndex(28), +_mouthSmileRightIndex(29), +_jawOpenIndex(21) { +} + +float FacialAnimationData::getBlendshapeCoefficient(int index) const { + return (index >= 0 && index < (int)_blendshapeCoefficients.size()) ? _blendshapeCoefficients[index] : 0.0f; +} \ No newline at end of file diff --git a/libraries/animation/src/FacialAnimationData.h b/libraries/animation/src/FacialAnimationData.h new file mode 100644 index 0000000000..c8d740c1c9 --- /dev/null +++ b/libraries/animation/src/FacialAnimationData.h @@ -0,0 +1,65 @@ +// +// FacialAnimationData.h +// interface/src/devices +// +// Created by Ben Arnold on 7/21/2014. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_FacialAnimationData_h +#define hifi_FacialAnimationData_h + +#include +#include + +/// Stores facial animation data for use by faceshift or scripts +class FacialAnimationData : public QObject { + Q_OBJECT + +public: + friend class Faceshift; + + FacialAnimationData(); + + float getLeftBlink() const { return getBlendshapeCoefficient(_leftBlinkIndex); } + float getRightBlink() const { return getBlendshapeCoefficient(_rightBlinkIndex); } + float getLeftEyeOpen() const { return getBlendshapeCoefficient(_leftEyeOpenIndex); } + float getRightEyeOpen() const { return getBlendshapeCoefficient(_rightEyeOpenIndex); } + + float getBrowDownLeft() const { return getBlendshapeCoefficient(_browDownLeftIndex); } + float getBrowDownRight() const { return getBlendshapeCoefficient(_browDownRightIndex); } + float getBrowUpCenter() const { return getBlendshapeCoefficient(_browUpCenterIndex); } + float getBrowUpLeft() const { return getBlendshapeCoefficient(_browUpLeftIndex); } + float getBrowUpRight() const { return getBlendshapeCoefficient(_browUpRightIndex); } + + float getMouthSize() const { return getBlendshapeCoefficient(_jawOpenIndex); } + float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); } + float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); } + +private: + float getBlendshapeCoefficient(int index) const; + + int _leftBlinkIndex; + int _rightBlinkIndex; + int _leftEyeOpenIndex; + int _rightEyeOpenIndex; + + int _browDownLeftIndex; + int _browDownRightIndex; + int _browUpCenterIndex; + int _browUpLeftIndex; + int _browUpRightIndex; + + int _mouthSmileLeftIndex; + int _mouthSmileRightIndex; + + int _jawOpenIndex; + + //Only used by agents, since FaceTracker has its own _blendshapeCoefficients; + QVector _blendshapeCoefficients; +}; + +#endif // hifi_FacialAnimationData_h \ No newline at end of file From f3ae2280302f5e6da31202f9f01d44babd262927 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Tue, 22 Jul 2014 13:46:29 -0700 Subject: [PATCH 13/46] Moved some FaceShift stuff to FacialAnimationData. Made use of FacialAnimationData --- interface/src/avatar/Head.cpp | 11 +++++------ interface/src/devices/Faceshift.cpp | 12 ------------ interface/src/devices/Faceshift.h | 3 --- libraries/animation/src/FacialAnimationData.cpp | 16 ++++++++++++++++ libraries/animation/src/FacialAnimationData.h | 8 +++++++- libraries/avatars/src/AvatarData.cpp | 16 ++++++++++------ libraries/avatars/src/AvatarData.h | 5 +++++ libraries/avatars/src/HeadData.cpp | 6 ++++++ libraries/avatars/src/HeadData.h | 10 ++++++++-- 9 files changed, 57 insertions(+), 30 deletions(-) diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index ee242d179a..814377b3d2 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -66,7 +66,7 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { if (isMine) { FaceTracker* faceTracker = Application::getInstance()->getActiveFaceTracker(); if ((_isFaceshiftConnected = faceTracker)) { - _blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); + _facialAnimationData->setBlendshapeCoefficients(faceTracker->getBlendshapeCoefficients()); _isFaceshiftConnected = true; } } @@ -141,11 +141,10 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { // use data to update fake Faceshift blendshape coefficients const float JAW_OPEN_SCALE = 10.f; - Application::getInstance()->getFaceshift()->updateFakeCoefficients(_leftEyeBlink, - _rightEyeBlink, - _browAudioLift, - glm::clamp(log(_averageLoudness) / JAW_OPEN_SCALE, 0.0f, 1.0f), - _blendshapeCoefficients); + _facialAnimationData->updateFakeCoefficients(_leftEyeBlink, + _rightEyeBlink, + _browAudioLift, + glm::clamp(log(_averageLoudness) / JAW_OPEN_SCALE, 0.0f, 1.0f)); } if (!isMine) { diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 4105b17652..40ee12af86 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -93,18 +93,6 @@ void Faceshift::reset() { _longTermAverageInitialized = false; } -void Faceshift::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, - float jawOpen, QVector& coefficients) const { - coefficients.resize(max((int)coefficients.size(), _facialAnimationData._jawOpenIndex + 1)); - qFill(coefficients.begin(), coefficients.end(), 0.0f); - coefficients[_facialAnimationData._leftBlinkIndex] = leftBlink; - coefficients[_facialAnimationData._rightBlinkIndex] = rightBlink; - coefficients[_facialAnimationData._browUpCenterIndex] = browUp; - coefficients[_facialAnimationData._browUpLeftIndex] = browUp; - coefficients[_facialAnimationData._browUpRightIndex] = browUp; - coefficients[_facialAnimationData._jawOpenIndex] = jawOpen; -} - void Faceshift::setTCPEnabled(bool enabled) { if ((_tcpEnabled = enabled)) { connectSocket(); diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 87fd1076ee..2b720c843d 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -61,9 +61,6 @@ public: void update(); void reset(); - void updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, - float jawOpen, QVector& coefficients) const; - signals: void connectionStateChanged(); diff --git a/libraries/animation/src/FacialAnimationData.cpp b/libraries/animation/src/FacialAnimationData.cpp index 0f45675083..974cdec66f 100644 --- a/libraries/animation/src/FacialAnimationData.cpp +++ b/libraries/animation/src/FacialAnimationData.cpp @@ -11,6 +11,10 @@ #include "FacialAnimationData.h" +#ifndef max +inline int max(int a, int b) { return a > b ? a : b; } +#endif + FacialAnimationData::FacialAnimationData() :_leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes _rightBlinkIndex(1), _leftEyeOpenIndex(8), @@ -27,4 +31,16 @@ _jawOpenIndex(21) { float FacialAnimationData::getBlendshapeCoefficient(int index) const { return (index >= 0 && index < (int)_blendshapeCoefficients.size()) ? _blendshapeCoefficients[index] : 0.0f; +} + +void FacialAnimationData::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, + float jawOpen) { + _blendshapeCoefficients.resize(max((int)_blendshapeCoefficients.size(), _jawOpenIndex + 1)); + qFill(_blendshapeCoefficients.begin(), _blendshapeCoefficients.end(), 0.0f); + _blendshapeCoefficients[_leftBlinkIndex] = leftBlink; + _blendshapeCoefficients[_rightBlinkIndex] = rightBlink; + _blendshapeCoefficients[_browUpCenterIndex] = browUp; + _blendshapeCoefficients[_browUpLeftIndex] = browUp; + _blendshapeCoefficients[_browUpRightIndex] = browUp; + _blendshapeCoefficients[_jawOpenIndex] = jawOpen; } \ No newline at end of file diff --git a/libraries/animation/src/FacialAnimationData.h b/libraries/animation/src/FacialAnimationData.h index c8d740c1c9..514fd5a55f 100644 --- a/libraries/animation/src/FacialAnimationData.h +++ b/libraries/animation/src/FacialAnimationData.h @@ -21,6 +21,8 @@ class FacialAnimationData : public QObject { public: friend class Faceshift; + friend class HeadData; + friend class AvatarData; FacialAnimationData(); @@ -39,6 +41,11 @@ public: float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); } float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); } + void updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, float jawOpen); + + void setBlendshapeCoefficients(const QVector& coefficients) { _blendshapeCoefficients = coefficients; } + const QVector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } + private: float getBlendshapeCoefficient(int index) const; @@ -58,7 +65,6 @@ private: int _jawOpenIndex; - //Only used by agents, since FaceTracker has its own _blendshapeCoefficients; QVector _blendshapeCoefficients; }; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 6cfa4ba488..fc5b762a0f 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -29,6 +29,8 @@ #include "AvatarData.h" +#include "../animation/src/FacialAnimationData.h" + quint64 DEFAULT_FILTERED_LOG_EXPIRY = 2 * USECS_PER_SECOND; using namespace std; @@ -151,10 +153,12 @@ QByteArray AvatarData::toByteArray() { memcpy(destinationBuffer, &_headData->_browAudioLift, sizeof(float)); destinationBuffer += sizeof(float); - *destinationBuffer++ = _headData->_blendshapeCoefficients.size(); - memcpy(destinationBuffer, _headData->_blendshapeCoefficients.data(), - _headData->_blendshapeCoefficients.size() * sizeof(float)); - destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); + const QVector& blendshapeCoefficients = _headData->_facialAnimationData->getBlendshapeCoefficients(); + + *destinationBuffer++ = blendshapeCoefficients.size(); + memcpy(destinationBuffer, blendshapeCoefficients.data(), + blendshapeCoefficients.size() * sizeof(float)); + destinationBuffer += blendshapeCoefficients.size() * sizeof(float); } // pupil dilation @@ -429,8 +433,8 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { return maxAvailableSize; } - _headData->_blendshapeCoefficients.resize(numCoefficients); - memcpy(_headData->_blendshapeCoefficients.data(), sourceBuffer, blendDataSize); + _headData->_facialAnimationData->_blendshapeCoefficients.resize(numCoefficients); + memcpy(_headData->_facialAnimationData->_blendshapeCoefficients.data(), sourceBuffer, blendDataSize); sourceBuffer += numCoefficients * sizeof(float); //bitItemsDataSize = 4 * sizeof(float) + 1 + blendDataSize; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 1b8f6e6007..b1b48e3941 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -206,6 +206,11 @@ public: Q_INVOKABLE virtual QStringList getJointNames() const { return _jointNames; } + // Facial Animation + Q_INVOKABLE void setLeftEyebrow(float f) { + _headData->setLeftEyebrow(f); + } + // key state void setKeyState(KeyState s) { _keyState = s; } KeyState keyState() const { return _keyState; } diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index c691a2c28a..1f2bb73834 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -16,6 +16,8 @@ #include "AvatarData.h" #include "HeadData.h" +#include "../animation/src/FacialAnimationData.h" + HeadData::HeadData(AvatarData* owningAvatar) : _baseYaw(0.0f), _basePitch(0.0f), @@ -29,6 +31,7 @@ HeadData::HeadData(AvatarData* owningAvatar) : _rightEyeBlink(0.0f), _averageLoudness(0.0f), _browAudioLift(0.0f), + _facialAnimationData(new FacialAnimationData), _owningAvatar(owningAvatar) { @@ -64,3 +67,6 @@ void HeadData::addRoll(float roll) { setBaseRoll(_baseRoll + roll); } +const QVector& HeadData::getBlendshapeCoefficients() const { + return _facialAnimationData->_blendshapeCoefficients; +} \ No newline at end of file diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index 9e2920ae85..9920776b2a 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -28,6 +28,7 @@ const float MIN_HEAD_ROLL = -50.f; const float MAX_HEAD_ROLL = 50.f; class AvatarData; +class FacialAnimationData; class HeadData { public: @@ -54,11 +55,16 @@ public: float getAudioAverageLoudness() const { return _audioAverageLoudness; } void setAudioAverageLoudness(float audioAverageLoudness) { _audioAverageLoudness = audioAverageLoudness; } - const QVector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } + const QVector& getBlendshapeCoefficients() const; float getPupilDilation() const { return _pupilDilation; } void setPupilDilation(float pupilDilation) { _pupilDilation = pupilDilation; } + // Facial animation + void setLeftEyebrow(float f) { + + } + // degrees void addYaw(float yaw); void addPitch(float pitch); @@ -85,7 +91,7 @@ protected: float _averageLoudness; float _browAudioLift; float _audioAverageLoudness; - QVector _blendshapeCoefficients; + FacialAnimationData* _facialAnimationData; float _pupilDilation; AvatarData* _owningAvatar; From 1b2dc2cbe1c11768028bf13213dcd1eb6d2cc83a Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 22 Jul 2014 14:21:53 -0700 Subject: [PATCH 14/46] added central audio stats including pipeline latency; old stats still there --- interface/src/Audio.cpp | 205 ++++++++++++++++++++++++ interface/src/Audio.h | 6 + interface/src/Menu.cpp | 6 + interface/src/Menu.h | 1 + interface/src/ui/ApplicationOverlay.cpp | 2 + 5 files changed, 220 insertions(+) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 8e3662e790..d200bee043 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -43,6 +43,7 @@ #include "Audio.h" #include "Menu.h" #include "Util.h" +#include "AudioRingBuffer.h" static const float AUDIO_CALLBACK_MSECS = (float) NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; @@ -125,6 +126,7 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _scopeInput(0), _scopeOutputLeft(0), _scopeOutputRight(0), + _statsEnabled(false), _starveCount(0), _consecutiveNotMixedCount(0), _outgoingAvatarAudioSequenceNumber(0), @@ -1303,6 +1305,10 @@ void Audio::toggleScopePause() { _scopeEnabledPause = !_scopeEnabledPause; } +void Audio::toggleStats() { + _statsEnabled = !_statsEnabled; +} + void Audio::selectAudioScopeFiveFrames() { if (Menu::getInstance()->isOptionChecked(MenuOption::AudioScopeFiveFrames)) { reallocateScope(5); @@ -1382,6 +1388,205 @@ void Audio::addBufferToScope( } } +void Audio::renderStats(const float* color, int width, int height) { + if (!_statsEnabled) { + return; + } + + const int MIN_LINES = 20; + const int STATS_BACKGROUND_HEIGHT = STATS_HEIGHT_PER_LINE * MIN_LINES; + + int lines = _audioMixerInjectedStreamAudioStatsMap.size() * 3 + 23; + int statsHeight = STATS_HEIGHT_PER_LINE * std::max(lines, MIN_LINES); + + + static const float backgroundColor[4] = { 0.2f, 0.2f, 0.2f, 0.6f }; + + int x = (width - STATS_WIDTH) / 2; + int y = (height - STATS_BACKGROUND_HEIGHT) / 2; + int w = STATS_WIDTH; + int h = statsHeight; + renderBackground(backgroundColor, x, y, w, h); + + + int horizontalOffset = x + 5; + int verticalOffset = y; + + float scale = 0.10f; + float rotation = 0.0f; + int font = 2; + + + char latencyStatString[512]; + + const float BUFFER_SEND_INTERVAL_MSECS = BUFFER_SEND_INTERVAL_USECS / (float)USECS_PER_MSEC; + + float inputRingBufferLatency = 0.0f, networkRoundtripLatency = 0.0f, mixerRingBufferLatency = 0.0f, outputRingBufferLatency = 0.0f, audioOutputBufferLatency = 0.0f; + + SharedNodePointer audioMixerNodePointer = NodeList::getInstance()->soloNodeOfType(NodeType::AudioMixer); + if (!audioMixerNodePointer.isNull()) { + inputRingBufferLatency = _inputRingBufferFramesAvailableStats.getWindowAverage() * BUFFER_SEND_INTERVAL_MSECS; + networkRoundtripLatency = audioMixerNodePointer->getPingMs(); + mixerRingBufferLatency = _audioMixerAvatarStreamAudioStats._ringBufferFramesAvailableAverage * BUFFER_SEND_INTERVAL_MSECS; + outputRingBufferLatency = _outputRingBufferFramesAvailableStats.getWindowAverage() * BUFFER_SEND_INTERVAL_MSECS; + audioOutputBufferLatency = _audioOutputBufferFramesAvailableStats.getWindowAverage() * BUFFER_SEND_INTERVAL_MSECS; + } + float totalLatency = inputRingBufferLatency + networkRoundtripLatency + mixerRingBufferLatency + outputRingBufferLatency + audioOutputBufferLatency; + + + sprintf(latencyStatString, " Input ring buffer: %.1fms\n", inputRingBufferLatency); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); + + sprintf(latencyStatString, " Network to mixer: %.1fms / 2\n", networkRoundtripLatency); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); + + sprintf(latencyStatString, " AudioMixer ring buffer: %.1fms\n", mixerRingBufferLatency); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); + + sprintf(latencyStatString, " Network to client: %.1fms / 2\n", networkRoundtripLatency); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); + + sprintf(latencyStatString, " Output ring buffer: %.1fms\n", outputRingBufferLatency); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); + + sprintf(latencyStatString, " Audio output buffer: %.1fms\n", audioOutputBufferLatency); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); + + sprintf(latencyStatString, " TOTAL: %.1fms\n", totalLatency); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); + + verticalOffset += STATS_HEIGHT_PER_LINE; + + + char inputAudioLabelString[] = "Input: avail_avg_10s/avail"; + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, inputAudioLabelString, color); + + char inputAudioStatsString[512]; + sprintf(inputAudioStatsString, " %d/%d", getInputRingBufferAverageFramesAvailable(), getInputRingBufferFramesAvailable()); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, inputAudioStatsString, color); + + + char audioMixerStatsLabelString[] = "AudioMixer stats:"; + char streamStatsFormatLabelString[] = " lost%/lost_30s%, desr/avail_avg_10s/avail"; + char streamStatsFormatLabelString2[] = " gaps: min/max/avg, starv/ovfl"; + char streamStatsFormatLabelString3[] = " gaps_30s: (same), notmix/sdrop"; + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerStatsLabelString, color); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString, color); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString2, color); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString3, color); + + char downstreamLabelString[] = "Downstream:"; + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamLabelString, color); + + char downstreamAudioStatsString[512]; + + AudioStreamStats downstreamAudioStreamStats = getDownstreamAudioStreamStats(); + + sprintf(downstreamAudioStatsString, " mix: %.2f%%/%.2f%%, %u/%u+%d/%u+%d", downstreamAudioStreamStats._packetStreamStats.getLostRate()*100.0f, + downstreamAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f, + downstreamAudioStreamStats._ringBufferDesiredJitterBufferFrames, downstreamAudioStreamStats._ringBufferFramesAvailableAverage, + getOutputRingBufferAverageFramesAvailable(), + downstreamAudioStreamStats._ringBufferFramesAvailable, getOutputRingBufferFramesAvailable()); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); + + sprintf(downstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(downstreamAudioStreamStats._timeGapMin).toLatin1().data(), + formatUsecTime(downstreamAudioStreamStats._timeGapMax).toLatin1().data(), + formatUsecTime(downstreamAudioStreamStats._timeGapAverage).toLatin1().data(), + downstreamAudioStreamStats._ringBufferStarveCount, downstreamAudioStreamStats._ringBufferOverflowCount); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); + + sprintf(downstreamAudioStatsString, " %s/%s/%s, %u/?", formatUsecTime(downstreamAudioStreamStats._timeGapWindowMin).toLatin1().data(), + formatUsecTime(downstreamAudioStreamStats._timeGapWindowMax).toLatin1().data(), + formatUsecTime(downstreamAudioStreamStats._timeGapWindowAverage).toLatin1().data(), + downstreamAudioStreamStats._ringBufferConsecutiveNotMixedCount); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); + + + char upstreamLabelString[] = "Upstream:"; + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamLabelString, color); + + char upstreamAudioStatsString[512]; + + const AudioStreamStats& audioMixerAvatarAudioStreamStats = getAudioMixerAvatarStreamAudioStats(); + + sprintf(upstreamAudioStatsString, " mic: %.2f%%/%.2f%%, %u/%u/%u", audioMixerAvatarAudioStreamStats._packetStreamStats.getLostRate()*100.0f, + audioMixerAvatarAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f, + audioMixerAvatarAudioStreamStats._ringBufferDesiredJitterBufferFrames, audioMixerAvatarAudioStreamStats._ringBufferFramesAvailableAverage, + audioMixerAvatarAudioStreamStats._ringBufferFramesAvailable); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + + sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapMin).toLatin1().data(), + formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapMax).toLatin1().data(), + formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapAverage).toLatin1().data(), + audioMixerAvatarAudioStreamStats._ringBufferStarveCount, audioMixerAvatarAudioStreamStats._ringBufferOverflowCount); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + + sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowMin).toLatin1().data(), + formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowMax).toLatin1().data(), + formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowAverage).toLatin1().data(), + audioMixerAvatarAudioStreamStats._ringBufferConsecutiveNotMixedCount, audioMixerAvatarAudioStreamStats._ringBufferSilentFramesDropped); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + + foreach(const AudioStreamStats& injectedStreamAudioStats, _audioMixerInjectedStreamAudioStatsMap) { + + sprintf(upstreamAudioStatsString, " inj: %.2f%%/%.2f%%, %u/%u/%u", injectedStreamAudioStats._packetStreamStats.getLostRate()*100.0f, + injectedStreamAudioStats._packetStreamWindowStats.getLostRate() * 100.0f, + injectedStreamAudioStats._ringBufferDesiredJitterBufferFrames, injectedStreamAudioStats._ringBufferFramesAvailableAverage, + injectedStreamAudioStats._ringBufferFramesAvailable); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + + sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(injectedStreamAudioStats._timeGapMin).toLatin1().data(), + formatUsecTime(injectedStreamAudioStats._timeGapMax).toLatin1().data(), + formatUsecTime(injectedStreamAudioStats._timeGapAverage).toLatin1().data(), + injectedStreamAudioStats._ringBufferStarveCount, injectedStreamAudioStats._ringBufferOverflowCount); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + + sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(injectedStreamAudioStats._timeGapWindowMin).toLatin1().data(), + formatUsecTime(injectedStreamAudioStats._timeGapWindowMax).toLatin1().data(), + formatUsecTime(injectedStreamAudioStats._timeGapWindowAverage).toLatin1().data(), + injectedStreamAudioStats._ringBufferConsecutiveNotMixedCount, injectedStreamAudioStats._ringBufferSilentFramesDropped); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + } + + +} + void Audio::renderScope(int width, int height) { if (!_scopeEnabled) diff --git a/interface/src/Audio.h b/interface/src/Audio.h index f91208b6e4..fd10eeea11 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -71,6 +71,7 @@ public: void renderToolBox(int x, int y, bool boxed); void renderScope(int width, int height); + void renderStats(const float* color, int width, int height); int getNetworkSampleRate() { return SAMPLE_RATE; } int getNetworkBufferLengthSamplesPerChannel() { return NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; } @@ -100,6 +101,7 @@ public slots: void toggleToneInjection(); void toggleScope(); void toggleScopePause(); + void toggleStats(); void toggleAudioSpatialProcessing(); void toggleStereoInput(); void selectAudioScopeFiveFrames(); @@ -263,6 +265,10 @@ private: QByteArray* _scopeOutputLeft; QByteArray* _scopeOutputRight; + static const unsigned int STATS_WIDTH = 600; + static const unsigned int STATS_HEIGHT_PER_LINE = 20; + bool _statsEnabled; + int _starveCount; int _consecutiveNotMixedCount; diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index e21299da11..21edd11214 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -589,6 +589,12 @@ Menu::Menu() : Qt::CTRL | Qt::SHIFT | Qt::Key_U, false); + addCheckableActionToQMenuAndActionHash(audioDebugMenu, MenuOption::AudioStats, + 0, + false, + appInstance->getAudio(), + SLOT(toggleStats())); + addCheckableActionToQMenuAndActionHash(audioDebugMenu, MenuOption::DisableQAudioOutputOverflowCheck, 0, true); addActionToQMenuAndActionHash(developerMenu, MenuOption::PasteToVoxel, diff --git a/interface/src/Menu.h b/interface/src/Menu.h index efec231306..eb6a570ab1 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -315,6 +315,7 @@ namespace MenuOption { const QString AudioScopeFrames = "Display Frames"; const QString AudioScopePause = "Pause Audio Scope"; const QString AudioScopeTwentyFrames = "Twenty"; + const QString AudioStats = "Audio Stats"; const QString AudioSpatialProcessingAlternateDistanceAttenuate = "Alternate distance attenuation"; const QString AudioSpatialProcessing = "Audio Spatial Processing"; const QString AudioSpatialProcessingDontDistanceAttenuate = "Don't calculate distance attenuation"; diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 7b3db26d8d..fdaf2aa407 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -1003,6 +1003,8 @@ void ApplicationOverlay::renderAudioMeter() { audio->renderScope(glWidget->width(), glWidget->height()); + audio->renderStats(WHITE_TEXT, glWidget->width(), glWidget->height()); + glBegin(GL_QUADS); if (isClipping) { glColor3f(1, 0, 0); From c8ac1732ecaf55c6da48b3490be15acb0f897f99 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 22 Jul 2014 14:53:45 -0700 Subject: [PATCH 15/46] removed audio stats from old location --- interface/src/ui/Stats.cpp | 124 +------------------------------------ 1 file changed, 1 insertion(+), 123 deletions(-) diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 412de6e7b0..ef4603bc95 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -280,7 +280,7 @@ void Stats::display( Audio* audio = Application::getInstance()->getAudio(); const QHash& audioMixerInjectedStreamAudioStatsMap = audio->getAudioMixerInjectedStreamAudioStatsMap(); - lines = _expanded ? 13 + (audioMixerInjectedStreamAudioStatsMap.size() + 2) * 3 : 3; + lines = _expanded ? 4 : 3; drawBackground(backgroundColor, horizontalOffset, 0, _pingStatsWidth, lines * STATS_PELS_PER_LINE + 10); horizontalOffset += 5; @@ -313,128 +313,6 @@ void Stats::display( verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, voxelMaxPing, color); - - char inputAudioLabelString[] = "Input: avail_avg_10s/avail"; - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, inputAudioLabelString, color); - - char inputAudioStatsString[512]; - sprintf(inputAudioStatsString, " %d/%d", audio->getInputRingBufferAverageFramesAvailable(), - audio->getInputRingBufferFramesAvailable()); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, inputAudioStatsString, color); - - char audioMixerStatsLabelString[] = "AudioMixer stats:"; - char streamStatsFormatLabelString[] = "lost%/lost_30s%"; - char streamStatsFormatLabelString2[] = "desired/avail_avg_10s/avail"; - char streamStatsFormatLabelString3[] = "gaps: min/max/avg, starv/ovfl"; - char streamStatsFormatLabelString4[] = "gaps_30s: (same), notmix/sdrop"; - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerStatsLabelString, color); - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString, color); - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString2, color); - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString3, color); - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString4, color); - - char downstreamLabelString[] = " Downstream:"; - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamLabelString, color); - - char downstreamAudioStatsString[512]; - - AudioStreamStats downstreamAudioStreamStats = audio->getDownstreamAudioStreamStats(); - - sprintf(downstreamAudioStatsString, " mix: %.2f%%/%.2f%%, %u/%u+%d/%u+%d", downstreamAudioStreamStats._packetStreamStats.getLostRate()*100.0f, - downstreamAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f, - downstreamAudioStreamStats._ringBufferDesiredJitterBufferFrames, downstreamAudioStreamStats._ringBufferFramesAvailableAverage, - audio->getOutputRingBufferAverageFramesAvailable(), - downstreamAudioStreamStats._ringBufferFramesAvailable, audio->getOutputRingBufferFramesAvailable()); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); - - sprintf(downstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(downstreamAudioStreamStats._timeGapMin).toLatin1().data(), - formatUsecTime(downstreamAudioStreamStats._timeGapMax).toLatin1().data(), - formatUsecTime(downstreamAudioStreamStats._timeGapAverage).toLatin1().data(), - downstreamAudioStreamStats._ringBufferStarveCount, downstreamAudioStreamStats._ringBufferOverflowCount); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); - - sprintf(downstreamAudioStatsString, " %s/%s/%s, %u/?", formatUsecTime(downstreamAudioStreamStats._timeGapWindowMin).toLatin1().data(), - formatUsecTime(downstreamAudioStreamStats._timeGapWindowMax).toLatin1().data(), - formatUsecTime(downstreamAudioStreamStats._timeGapWindowAverage).toLatin1().data(), - downstreamAudioStreamStats._ringBufferConsecutiveNotMixedCount); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); - - - char upstreamLabelString[] = " Upstream:"; - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamLabelString, color); - - char upstreamAudioStatsString[512]; - - const AudioStreamStats& audioMixerAvatarAudioStreamStats = audio->getAudioMixerAvatarStreamAudioStats(); - - sprintf(upstreamAudioStatsString, " mic: %.2f%%/%.2f%%, %u/%u/%u", audioMixerAvatarAudioStreamStats._packetStreamStats.getLostRate()*100.0f, - audioMixerAvatarAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f, - audioMixerAvatarAudioStreamStats._ringBufferDesiredJitterBufferFrames, audioMixerAvatarAudioStreamStats._ringBufferFramesAvailableAverage, - audioMixerAvatarAudioStreamStats._ringBufferFramesAvailable); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - - sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapMin).toLatin1().data(), - formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapMax).toLatin1().data(), - formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapAverage).toLatin1().data(), - audioMixerAvatarAudioStreamStats._ringBufferStarveCount, audioMixerAvatarAudioStreamStats._ringBufferOverflowCount); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - - sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowMin).toLatin1().data(), - formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowMax).toLatin1().data(), - formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowAverage).toLatin1().data(), - audioMixerAvatarAudioStreamStats._ringBufferConsecutiveNotMixedCount, audioMixerAvatarAudioStreamStats._ringBufferSilentFramesDropped); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - - foreach(const AudioStreamStats& injectedStreamAudioStats, audioMixerInjectedStreamAudioStatsMap) { - - sprintf(upstreamAudioStatsString, " inj: %.2f%%/%.2f%%, %u/%u/%u", injectedStreamAudioStats._packetStreamStats.getLostRate()*100.0f, - injectedStreamAudioStats._packetStreamWindowStats.getLostRate() * 100.0f, - injectedStreamAudioStats._ringBufferDesiredJitterBufferFrames, injectedStreamAudioStats._ringBufferFramesAvailableAverage, - injectedStreamAudioStats._ringBufferFramesAvailable); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - - sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(injectedStreamAudioStats._timeGapMin).toLatin1().data(), - formatUsecTime(injectedStreamAudioStats._timeGapMax).toLatin1().data(), - formatUsecTime(injectedStreamAudioStats._timeGapAverage).toLatin1().data(), - injectedStreamAudioStats._ringBufferStarveCount, injectedStreamAudioStats._ringBufferOverflowCount); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - - sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(injectedStreamAudioStats._timeGapWindowMin).toLatin1().data(), - formatUsecTime(injectedStreamAudioStats._timeGapWindowMax).toLatin1().data(), - formatUsecTime(injectedStreamAudioStats._timeGapWindowAverage).toLatin1().data(), - injectedStreamAudioStats._ringBufferConsecutiveNotMixedCount, injectedStreamAudioStats._ringBufferSilentFramesDropped); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - } } verticalOffset = 0; From d9692e4af4cd009fc3a67e5c17a95f337b24946a Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 22 Jul 2014 17:34:35 -0700 Subject: [PATCH 16/46] changed a > to >= in handleAudioInput; added audio input buffer latency to stats; --- interface/src/Audio.cpp | 59 +++++++++++++++++++---------------------- interface/src/Audio.h | 9 ++++--- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index d200bee043..f351c73661 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -132,7 +132,8 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _outgoingAvatarAudioSequenceNumber(0), _incomingMixedAudioSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH), _interframeTimeGapStats(TIME_GAPS_STATS_INTERVAL_SAMPLES, TIME_GAP_STATS_WINDOW_INTERVALS), - _inputRingBufferFramesAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), + _audioInputBufferMsecsDataAvailableStats(MSECS_PER_SECOND / (float)AUDIO_CALLBACK_MSECS * CALLBACK_ACCELERATOR_RATIO, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), + _inputRingBufferMsecsDataAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), _outputRingBufferFramesAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), _audioOutputBufferFramesAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS) { @@ -167,7 +168,8 @@ void Audio::resetStats() { _interframeTimeGapStats.reset(); - _inputRingBufferFramesAvailableStats.reset(); + _audioInputBufferMsecsDataAvailableStats.reset(); + _inputRingBufferMsecsDataAvailableStats.reset(); _outputRingBufferFramesAvailableStats.reset(); _audioOutputBufferFramesAvailableStats.reset(); @@ -518,8 +520,11 @@ void Audio::handleAudioInput() { } _inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size()); + + float audioInputMsecsRead = inputByteArray.size() / (float)(_inputFormat.channelCount() * _inputFormat.sampleRate() * sizeof(int16_t)) * MSECS_PER_SECOND; + _audioInputBufferMsecsDataAvailableStats.update(audioInputMsecsRead); - while (_inputRingBuffer.samplesAvailable() > inputSamplesRequired) { + while (_inputRingBuffer.samplesAvailable() >= inputSamplesRequired) { int16_t* inputAudioSamples = new int16_t[inputSamplesRequired]; _inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired); @@ -830,9 +835,10 @@ AudioStreamStats Audio::getDownstreamAudioStreamStats() const { void Audio::sendDownstreamAudioStatsPacket() { - _inputRingBufferFramesAvailableStats.update(getInputRingBufferFramesAvailable()); + // since this function is called every second, we'll sample some of our stats here + + _inputRingBufferMsecsDataAvailableStats.update(getInputRingBufferMsecsDataAvailable()); - // since this function is called every second, we'll sample the number of audio frames available here. _outputRingBufferFramesAvailableStats.update(_ringBuffer.framesAvailable()); _audioOutputBufferFramesAvailableStats.update(getOutputRingBufferFramesAvailable()); @@ -1421,62 +1427,54 @@ void Audio::renderStats(const float* color, int width, int height) { const float BUFFER_SEND_INTERVAL_MSECS = BUFFER_SEND_INTERVAL_USECS / (float)USECS_PER_MSEC; - float inputRingBufferLatency = 0.0f, networkRoundtripLatency = 0.0f, mixerRingBufferLatency = 0.0f, outputRingBufferLatency = 0.0f, audioOutputBufferLatency = 0.0f; + float audioInputBufferLatency = 0.0f, inputRingBufferLatency = 0.0f, networkRoundtripLatency = 0.0f, mixerRingBufferLatency = 0.0f, outputRingBufferLatency = 0.0f, audioOutputBufferLatency = 0.0f; SharedNodePointer audioMixerNodePointer = NodeList::getInstance()->soloNodeOfType(NodeType::AudioMixer); if (!audioMixerNodePointer.isNull()) { - inputRingBufferLatency = _inputRingBufferFramesAvailableStats.getWindowAverage() * BUFFER_SEND_INTERVAL_MSECS; + audioInputBufferLatency = _audioInputBufferMsecsDataAvailableStats.getWindowAverage(); + inputRingBufferLatency = getInputRingBufferAverageMsecsDataAvailable(); networkRoundtripLatency = audioMixerNodePointer->getPingMs(); mixerRingBufferLatency = _audioMixerAvatarStreamAudioStats._ringBufferFramesAvailableAverage * BUFFER_SEND_INTERVAL_MSECS; outputRingBufferLatency = _outputRingBufferFramesAvailableStats.getWindowAverage() * BUFFER_SEND_INTERVAL_MSECS; audioOutputBufferLatency = _audioOutputBufferFramesAvailableStats.getWindowAverage() * BUFFER_SEND_INTERVAL_MSECS; } - float totalLatency = inputRingBufferLatency + networkRoundtripLatency + mixerRingBufferLatency + outputRingBufferLatency + audioOutputBufferLatency; + float totalLatency = audioInputBufferLatency + inputRingBufferLatency + networkRoundtripLatency + mixerRingBufferLatency + outputRingBufferLatency + audioOutputBufferLatency; - - sprintf(latencyStatString, " Input ring buffer: %.1fms\n", inputRingBufferLatency); + sprintf(latencyStatString, " Audio input buffer: %7.2fms\n", audioInputBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Network to mixer: %.1fms / 2\n", networkRoundtripLatency); + sprintf(latencyStatString, " Input ring buffer: %7.2fms\n", inputRingBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " AudioMixer ring buffer: %.1fms\n", mixerRingBufferLatency); + sprintf(latencyStatString, " Network to mixer: %7.2fms\n", networkRoundtripLatency / 2.0f); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Network to client: %.1fms / 2\n", networkRoundtripLatency); + sprintf(latencyStatString, " AudioMixer ring buffer: %7.2fms\n", mixerRingBufferLatency); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); + + sprintf(latencyStatString, " Network to client: %7.2fms\n", networkRoundtripLatency / 2.0f); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Output ring buffer: %.1fms\n", outputRingBufferLatency); + sprintf(latencyStatString, " Output ring buffer: %7.2fms\n", outputRingBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Audio output buffer: %.1fms\n", audioOutputBufferLatency); + sprintf(latencyStatString, " Audio output buffer: %7.2fms\n", audioOutputBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " TOTAL: %.1fms\n", totalLatency); + sprintf(latencyStatString, " TOTAL: %7.2fms\n", totalLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); verticalOffset += STATS_HEIGHT_PER_LINE; - char inputAudioLabelString[] = "Input: avail_avg_10s/avail"; - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, inputAudioLabelString, color); - - char inputAudioStatsString[512]; - sprintf(inputAudioStatsString, " %d/%d", getInputRingBufferAverageFramesAvailable(), getInputRingBufferFramesAvailable()); - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, inputAudioStatsString, color); - - char audioMixerStatsLabelString[] = "AudioMixer stats:"; char streamStatsFormatLabelString[] = " lost%/lost_30s%, desr/avail_avg_10s/avail"; char streamStatsFormatLabelString2[] = " gaps: min/max/avg, starv/ovfl"; @@ -1852,7 +1850,6 @@ int Audio::getOutputRingBufferFramesAvailable() const { / (sizeof(int16_t) * _ringBuffer.getNumFrameSamples()); } -int Audio::getInputRingBufferFramesAvailable() const { - float inputToNetworkInputRatio = calculateDeviceToNetworkInputRatio(_numInputCallbackBytes); - return _inputRingBuffer.samplesAvailable() / inputToNetworkInputRatio / _inputRingBuffer.getNumFrameSamples(); +float Audio::getInputRingBufferMsecsDataAvailable() const { + return _inputRingBuffer.samplesAvailable() / (float)(_inputFormat.channelCount() * _inputFormat.sampleRate()) * MSECS_PER_SECOND; } diff --git a/interface/src/Audio.h b/interface/src/Audio.h index fd10eeea11..feaa8d16b1 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -79,9 +79,9 @@ public: bool getProcessSpatialAudio() const { return _processSpatialAudio; } const SequenceNumberStats& getIncomingMixedAudioSequenceNumberStats() const { return _incomingMixedAudioSequenceNumberStats; } - - int getInputRingBufferFramesAvailable() const; - int getInputRingBufferAverageFramesAvailable() const { return (int)_inputRingBufferFramesAvailableStats.getWindowAverage(); } + + float getInputRingBufferMsecsDataAvailable() const; + float getInputRingBufferAverageMsecsDataAvailable() const { return (float)_inputRingBufferMsecsDataAvailableStats.getWindowAverage(); } int getOutputRingBufferFramesAvailable() const; int getOutputRingBufferAverageFramesAvailable() const { return (int)_audioOutputBufferFramesAvailableStats.getWindowAverage(); } @@ -280,7 +280,8 @@ private: MovingMinMaxAvg _interframeTimeGapStats; - MovingMinMaxAvg _inputRingBufferFramesAvailableStats; + MovingMinMaxAvg _audioInputBufferMsecsDataAvailableStats; + MovingMinMaxAvg _inputRingBufferMsecsDataAvailableStats; MovingMinMaxAvg _outputRingBufferFramesAvailableStats; MovingMinMaxAvg _audioOutputBufferFramesAvailableStats; From 5b0b4847278e47b554994df0755ac5cbc223b59c Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 11:00:48 -0700 Subject: [PATCH 17/46] finished new audio stats (looks good on Windows) --- interface/src/Audio.cpp | 175 ++++++++++++++++------------------------ interface/src/Audio.h | 6 +- 2 files changed, 75 insertions(+), 106 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index f351c73661..9c7a02a0a3 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -1399,17 +1399,17 @@ void Audio::renderStats(const float* color, int width, int height) { return; } - const int MIN_LINES = 20; - const int STATS_BACKGROUND_HEIGHT = STATS_HEIGHT_PER_LINE * MIN_LINES; + const int LINES_WHEN_CENTERED = 30; + const int CENTERED_BACKGROUND_HEIGHT = STATS_HEIGHT_PER_LINE * LINES_WHEN_CENTERED; - int lines = _audioMixerInjectedStreamAudioStatsMap.size() * 3 + 23; - int statsHeight = STATS_HEIGHT_PER_LINE * std::max(lines, MIN_LINES); + int lines = _audioMixerInjectedStreamAudioStatsMap.size() * 7 + 23; + int statsHeight = STATS_HEIGHT_PER_LINE * lines; static const float backgroundColor[4] = { 0.2f, 0.2f, 0.2f, 0.6f }; int x = (width - STATS_WIDTH) / 2; - int y = (height - STATS_BACKGROUND_HEIGHT) / 2; + int y = (height - CENTERED_BACKGROUND_HEIGHT) / 2; int w = STATS_WIDTH; int h = statsHeight; renderBackground(backgroundColor, x, y, w, h); @@ -1440,31 +1440,31 @@ void Audio::renderStats(const float* color, int width, int height) { } float totalLatency = audioInputBufferLatency + inputRingBufferLatency + networkRoundtripLatency + mixerRingBufferLatency + outputRingBufferLatency + audioOutputBufferLatency; - sprintf(latencyStatString, " Audio input buffer: %7.2fms\n", audioInputBufferLatency); + sprintf(latencyStatString, " Audio input buffer: %7.2fms - avg msecs of samples read to the input ring buffer in last 10s", audioInputBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Input ring buffer: %7.2fms\n", inputRingBufferLatency); + sprintf(latencyStatString, " Input ring buffer: %7.2fms - avg msecs of samples in input ring buffer in last 10s", inputRingBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Network to mixer: %7.2fms\n", networkRoundtripLatency / 2.0f); + sprintf(latencyStatString, " Network to mixer: %7.2fms - half of last ping value calculated by the node list", networkRoundtripLatency / 2.0f); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " AudioMixer ring buffer: %7.2fms\n", mixerRingBufferLatency); + sprintf(latencyStatString, " AudioMixer ring buffer: %7.2fms - avg msecs of samples in audio mixer's ring buffer in last 10s", mixerRingBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Network to client: %7.2fms\n", networkRoundtripLatency / 2.0f); + sprintf(latencyStatString, " Network to client: %7.2fms - half of last ping value calculated by the node list", networkRoundtripLatency / 2.0f); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Output ring buffer: %7.2fms\n", outputRingBufferLatency); + sprintf(latencyStatString, " Output ring buffer: %7.2fms - avg msecs of samples in output ring buffer in last 10s", outputRingBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Audio output buffer: %7.2fms\n", audioOutputBufferLatency); + sprintf(latencyStatString, " Audio output buffer: %7.2fms - avg msecs of samples in audio output buffer in last 10s", audioOutputBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); @@ -1472,119 +1472,84 @@ void Audio::renderStats(const float* color, int width, int height) { verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - verticalOffset += STATS_HEIGHT_PER_LINE; + + verticalOffset += STATS_HEIGHT_PER_LINE; // blank line - char audioMixerStatsLabelString[] = "AudioMixer stats:"; - char streamStatsFormatLabelString[] = " lost%/lost_30s%, desr/avail_avg_10s/avail"; - char streamStatsFormatLabelString2[] = " gaps: min/max/avg, starv/ovfl"; - char streamStatsFormatLabelString3[] = " gaps_30s: (same), notmix/sdrop"; - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerStatsLabelString, color); - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString, color); - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString2, color); - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString3, color); - - char downstreamLabelString[] = "Downstream:"; + char downstreamLabelString[] = "Downstream mixed audio stats:"; verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamLabelString, color); - char downstreamAudioStatsString[512]; + renderAudioStreamStats(getDownstreamAudioStreamStats(), horizontalOffset, verticalOffset, scale, rotation, font, color); - AudioStreamStats downstreamAudioStreamStats = getDownstreamAudioStreamStats(); - sprintf(downstreamAudioStatsString, " mix: %.2f%%/%.2f%%, %u/%u+%d/%u+%d", downstreamAudioStreamStats._packetStreamStats.getLostRate()*100.0f, - downstreamAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f, - downstreamAudioStreamStats._ringBufferDesiredJitterBufferFrames, downstreamAudioStreamStats._ringBufferFramesAvailableAverage, - getOutputRingBufferAverageFramesAvailable(), - downstreamAudioStreamStats._ringBufferFramesAvailable, getOutputRingBufferFramesAvailable()); + verticalOffset += STATS_HEIGHT_PER_LINE; // blank line + char upstreamMicLabelString[] = "Upstream mic audio stats:"; verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamMicLabelString, color); - sprintf(downstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(downstreamAudioStreamStats._timeGapMin).toLatin1().data(), - formatUsecTime(downstreamAudioStreamStats._timeGapMax).toLatin1().data(), - formatUsecTime(downstreamAudioStreamStats._timeGapAverage).toLatin1().data(), - downstreamAudioStreamStats._ringBufferStarveCount, downstreamAudioStreamStats._ringBufferOverflowCount); + renderAudioStreamStats(_audioMixerAvatarStreamAudioStats, horizontalOffset, verticalOffset, scale, rotation, font, color); - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); - - sprintf(downstreamAudioStatsString, " %s/%s/%s, %u/?", formatUsecTime(downstreamAudioStreamStats._timeGapWindowMin).toLatin1().data(), - formatUsecTime(downstreamAudioStreamStats._timeGapWindowMax).toLatin1().data(), - formatUsecTime(downstreamAudioStreamStats._timeGapWindowAverage).toLatin1().data(), - downstreamAudioStreamStats._ringBufferConsecutiveNotMixedCount); - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); - - - char upstreamLabelString[] = "Upstream:"; - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamLabelString, color); - - char upstreamAudioStatsString[512]; - - const AudioStreamStats& audioMixerAvatarAudioStreamStats = getAudioMixerAvatarStreamAudioStats(); - - sprintf(upstreamAudioStatsString, " mic: %.2f%%/%.2f%%, %u/%u/%u", audioMixerAvatarAudioStreamStats._packetStreamStats.getLostRate()*100.0f, - audioMixerAvatarAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f, - audioMixerAvatarAudioStreamStats._ringBufferDesiredJitterBufferFrames, audioMixerAvatarAudioStreamStats._ringBufferFramesAvailableAverage, - audioMixerAvatarAudioStreamStats._ringBufferFramesAvailable); - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - - sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapMin).toLatin1().data(), - formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapMax).toLatin1().data(), - formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapAverage).toLatin1().data(), - audioMixerAvatarAudioStreamStats._ringBufferStarveCount, audioMixerAvatarAudioStreamStats._ringBufferOverflowCount); - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - - sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowMin).toLatin1().data(), - formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowMax).toLatin1().data(), - formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowAverage).toLatin1().data(), - audioMixerAvatarAudioStreamStats._ringBufferConsecutiveNotMixedCount, audioMixerAvatarAudioStreamStats._ringBufferSilentFramesDropped); - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); foreach(const AudioStreamStats& injectedStreamAudioStats, _audioMixerInjectedStreamAudioStatsMap) { - sprintf(upstreamAudioStatsString, " inj: %.2f%%/%.2f%%, %u/%u/%u", injectedStreamAudioStats._packetStreamStats.getLostRate()*100.0f, - injectedStreamAudioStats._packetStreamWindowStats.getLostRate() * 100.0f, - injectedStreamAudioStats._ringBufferDesiredJitterBufferFrames, injectedStreamAudioStats._ringBufferFramesAvailableAverage, - injectedStreamAudioStats._ringBufferFramesAvailable); + verticalOffset += STATS_HEIGHT_PER_LINE; // blank line + char upstreamInjectedLabelString[512]; + sprintf(upstreamInjectedLabelString, "Upstream injected audio stats: stream ID: %s", + injectedStreamAudioStats._streamIdentifier.toString().toLatin1().data()); verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamInjectedLabelString, color); - sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(injectedStreamAudioStats._timeGapMin).toLatin1().data(), - formatUsecTime(injectedStreamAudioStats._timeGapMax).toLatin1().data(), - formatUsecTime(injectedStreamAudioStats._timeGapAverage).toLatin1().data(), - injectedStreamAudioStats._ringBufferStarveCount, injectedStreamAudioStats._ringBufferOverflowCount); - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - - sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(injectedStreamAudioStats._timeGapWindowMin).toLatin1().data(), - formatUsecTime(injectedStreamAudioStats._timeGapWindowMax).toLatin1().data(), - formatUsecTime(injectedStreamAudioStats._timeGapWindowAverage).toLatin1().data(), - injectedStreamAudioStats._ringBufferConsecutiveNotMixedCount, injectedStreamAudioStats._ringBufferSilentFramesDropped); - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + renderAudioStreamStats(injectedStreamAudioStats, horizontalOffset, verticalOffset, scale, rotation, font, color); } - - } +void Audio::renderAudioStreamStats(const AudioStreamStats& streamStats, int horizontalOffset, int& verticalOffset, + float scale, float rotation, int font, const float* color) { + + char stringBuffer[512]; + + sprintf(stringBuffer, " Packet loss | overall: %5.2f%%, last_30s: %5.2f%%", + streamStats._packetStreamStats.getLostRate()*100.0f, + streamStats._packetStreamWindowStats.getLostRate() * 100.0f); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); + + sprintf(stringBuffer, " Ringbuffer frames | desired: %u, avg_available(10s): %u+%d, available: %u+%d", + streamStats._ringBufferDesiredJitterBufferFrames, + streamStats._ringBufferFramesAvailableAverage, + getOutputRingBufferAverageFramesAvailable(), + streamStats._ringBufferFramesAvailable, + getOutputRingBufferFramesAvailable()); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); + + sprintf(stringBuffer, " Ringbuffer stats | starves: %u, prev_starve_lasted: %u, frames_dropped: %u, overflows: %u", + streamStats._ringBufferStarveCount, + streamStats._ringBufferStarveCount, + streamStats._ringBufferConsecutiveNotMixedCount, + streamStats._ringBufferOverflowCount); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); + + sprintf(stringBuffer, " Inter-packet timegaps (overall) | min: %9s, max: %9s, avg: %9s", + formatUsecTime(streamStats._timeGapMin).toLatin1().data(), + formatUsecTime(streamStats._timeGapMax).toLatin1().data(), + formatUsecTime(streamStats._timeGapAverage).toLatin1().data()); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); + + sprintf(stringBuffer, " Inter-packet timegaps (last 30s) | min: %9s, max: %9s, avg: %9s", + formatUsecTime(streamStats._timeGapWindowMin).toLatin1().data(), + formatUsecTime(streamStats._timeGapWindowMax).toLatin1().data(), + formatUsecTime(streamStats._timeGapWindowAverage).toLatin1().data()); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); +} + + void Audio::renderScope(int width, int height) { if (!_scopeEnabled) diff --git a/interface/src/Audio.h b/interface/src/Audio.h index feaa8d16b1..857044c182 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -248,6 +248,10 @@ private: void renderGrid(const float* color, int x, int y, int width, int height, int rows, int cols); void renderLineStrip(const float* color, int x, int y, int n, int offset, const QByteArray* byteArray); + // audio stats methods for rendering + void renderAudioStreamStats(const AudioStreamStats& streamStats, int horizontalOffset, int& verticalOffset, + float scale, float rotation, int font, const float* color); + // Audio scope data static const unsigned int NETWORK_SAMPLES_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; static const unsigned int DEFAULT_FRAMES_PER_SCOPE = 5; @@ -265,7 +269,7 @@ private: QByteArray* _scopeOutputLeft; QByteArray* _scopeOutputRight; - static const unsigned int STATS_WIDTH = 600; + static const unsigned int STATS_WIDTH = 1500; static const unsigned int STATS_HEIGHT_PER_LINE = 20; bool _statsEnabled; From ba4403b06315d0d5bc201b68ee42b5010c51ce74 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 11:18:25 -0700 Subject: [PATCH 18/46] audio stats look good on Mac now --- interface/src/Audio.cpp | 4 ++-- interface/src/Audio.h | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 9c7a02a0a3..e55d30d753 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -1408,8 +1408,8 @@ void Audio::renderStats(const float* color, int width, int height) { static const float backgroundColor[4] = { 0.2f, 0.2f, 0.2f, 0.6f }; - int x = (width - STATS_WIDTH) / 2; - int y = (height - CENTERED_BACKGROUND_HEIGHT) / 2; + int x = std::max((width - (int)STATS_WIDTH) / 2, 0); + int y = std::max((height - CENTERED_BACKGROUND_HEIGHT) / 2, 0); int w = STATS_WIDTH; int h = statsHeight; renderBackground(backgroundColor, x, y, w, h); diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 857044c182..1692a1b0ce 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -268,8 +268,11 @@ private: QByteArray* _scopeInput; QByteArray* _scopeOutputLeft; QByteArray* _scopeOutputRight; - +#ifdef _WIN32 static const unsigned int STATS_WIDTH = 1500; +#else + static const unsigned int STATS_WIDTH = 650; +#endif static const unsigned int STATS_HEIGHT_PER_LINE = 20; bool _statsEnabled; From 849c489b4c041aff183b39f4c35f78c735f47ec1 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 11:20:30 -0700 Subject: [PATCH 19/46] removed warnings about parseData() overloading in ringbuffer classes --- assignment-client/src/audio/AudioMixerClientData.cpp | 4 ++-- assignment-client/src/audio/AvatarAudioRingBuffer.h | 2 +- libraries/audio/src/InjectedAudioRingBuffer.h | 2 +- libraries/audio/src/PositionalAudioRingBuffer.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index b42a690d97..1536da7aa6 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -95,7 +95,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { } case SequenceNumberStats::OnTime: { // ask the AvatarAudioRingBuffer instance to parse the data - avatarRingBuffer->parseData(packet); + avatarRingBuffer->parseData(packet, 0); break; } default: { @@ -139,7 +139,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { } case SequenceNumberStats::OnTime: { // ask the AvatarAudioRingBuffer instance to parse the data - matchingInjectedRingBuffer->parseData(packet); + matchingInjectedRingBuffer->parseData(packet, 0); break; } default: { diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.h b/assignment-client/src/audio/AvatarAudioRingBuffer.h index df9dc3787a..22444cf32a 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.h +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.h @@ -20,7 +20,7 @@ class AvatarAudioRingBuffer : public PositionalAudioRingBuffer { public: AvatarAudioRingBuffer(bool isStereo = false, bool dynamicJitterBuffer = false); - int parseData(const QByteArray& packet, int packetsSkipped = 0); + int parseData(const QByteArray& packet, int packetsSkipped); private: // disallow copying of AvatarAudioRingBuffer objects AvatarAudioRingBuffer(const AvatarAudioRingBuffer&); diff --git a/libraries/audio/src/InjectedAudioRingBuffer.h b/libraries/audio/src/InjectedAudioRingBuffer.h index f226d99b12..856121e2dd 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.h +++ b/libraries/audio/src/InjectedAudioRingBuffer.h @@ -20,7 +20,7 @@ class InjectedAudioRingBuffer : public PositionalAudioRingBuffer { public: InjectedAudioRingBuffer(const QUuid& streamIdentifier = QUuid(), bool dynamicJitterBuffer = false); - int parseData(const QByteArray& packet, int packetsSkipped = 0); + int parseData(const QByteArray& packet, int packetsSkipped); const QUuid& getStreamIdentifier() const { return _streamIdentifier; } float getRadius() const { return _radius; } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index f0d6aff80b..7b7d81d216 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -45,7 +45,7 @@ public: PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo = false, bool dynamicJitterBuffers = false); - virtual int parseData(const QByteArray& packet, int packetsSkipped = 0) = 0; + virtual int parseData(const QByteArray& packet, int packetsSkipped) = 0; int parsePositionalData(const QByteArray& positionalByteArray); int parseListenModeData(const QByteArray& listenModeByteArray); From 4a4d065f00ce19485050b38cbfa8f7c8c0964836 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 12:06:31 -0700 Subject: [PATCH 20/46] another parseData change --- assignment-client/src/audio/AudioMixerClientData.cpp | 8 ++++---- assignment-client/src/audio/AvatarAudioRingBuffer.cpp | 2 +- assignment-client/src/audio/AvatarAudioRingBuffer.h | 2 +- libraries/audio/src/InjectedAudioRingBuffer.cpp | 2 +- libraries/audio/src/InjectedAudioRingBuffer.h | 2 +- libraries/audio/src/PositionalAudioRingBuffer.h | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 1536da7aa6..a4983e6a95 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -90,12 +90,12 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { switch (packetArrivalInfo._status) { case SequenceNumberStats::Early: { int packetsLost = packetArrivalInfo._seqDiffFromExpected; - avatarRingBuffer->parseData(packet, packetsLost); + avatarRingBuffer->parseDataAndHandleDroppedPackets(packet, packetsLost); break; } case SequenceNumberStats::OnTime: { // ask the AvatarAudioRingBuffer instance to parse the data - avatarRingBuffer->parseData(packet, 0); + avatarRingBuffer->parseDataAndHandleDroppedPackets(packet, 0); break; } default: { @@ -134,12 +134,12 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { switch (packetArrivalInfo._status) { case SequenceNumberStats::Early: { int packetsLost = packetArrivalInfo._seqDiffFromExpected; - matchingInjectedRingBuffer->parseData(packet, packetsLost); + matchingInjectedRingBuffer->parseDataAndHandleDroppedPackets(packet, packetsLost); break; } case SequenceNumberStats::OnTime: { // ask the AvatarAudioRingBuffer instance to parse the data - matchingInjectedRingBuffer->parseData(packet, 0); + matchingInjectedRingBuffer->parseDataAndHandleDroppedPackets(packet, 0); break; } default: { diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp index f6edde7ac4..382e8de68b 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp @@ -18,7 +18,7 @@ AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo, bool dynamicJitterBu } -int AvatarAudioRingBuffer::parseData(const QByteArray& packet, int packetsSkipped) { +int AvatarAudioRingBuffer::parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped) { frameReceivedUpdateTimingStats(); _shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho); diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.h b/assignment-client/src/audio/AvatarAudioRingBuffer.h index 22444cf32a..96233220cd 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.h +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.h @@ -20,7 +20,7 @@ class AvatarAudioRingBuffer : public PositionalAudioRingBuffer { public: AvatarAudioRingBuffer(bool isStereo = false, bool dynamicJitterBuffer = false); - int parseData(const QByteArray& packet, int packetsSkipped); + int parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped); private: // disallow copying of AvatarAudioRingBuffer objects AvatarAudioRingBuffer(const AvatarAudioRingBuffer&); diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index da2d8336de..4723bca906 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -30,7 +30,7 @@ InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier, const uchar MAX_INJECTOR_VOLUME = 255; -int InjectedAudioRingBuffer::parseData(const QByteArray& packet, int packetsSkipped) { +int InjectedAudioRingBuffer::parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped) { frameReceivedUpdateTimingStats(); // setup a data stream to read from this packet diff --git a/libraries/audio/src/InjectedAudioRingBuffer.h b/libraries/audio/src/InjectedAudioRingBuffer.h index 856121e2dd..4a1f8b5292 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.h +++ b/libraries/audio/src/InjectedAudioRingBuffer.h @@ -20,7 +20,7 @@ class InjectedAudioRingBuffer : public PositionalAudioRingBuffer { public: InjectedAudioRingBuffer(const QUuid& streamIdentifier = QUuid(), bool dynamicJitterBuffer = false); - int parseData(const QByteArray& packet, int packetsSkipped); + int parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped); const QUuid& getStreamIdentifier() const { return _streamIdentifier; } float getRadius() const { return _radius; } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 7b7d81d216..0b14a12858 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -45,7 +45,7 @@ public: PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo = false, bool dynamicJitterBuffers = false); - virtual int parseData(const QByteArray& packet, int packetsSkipped) = 0; + virtual int parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped) = 0; int parsePositionalData(const QByteArray& positionalByteArray); int parseListenModeData(const QByteArray& listenModeByteArray); From 5eaa57b3ade0edbdeb6bd97a0cdec28e97fba7b8 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 12:10:25 -0700 Subject: [PATCH 21/46] removed more warnings about unused vars --- assignment-client/src/audio/AudioMixer.cpp | 8 -------- interface/src/ui/Stats.cpp | 1 - 2 files changed, 9 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index b008a20aa7..42136f3e9d 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -237,14 +237,6 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf _clientSamples[delayedChannelIndex + SINGLE_STEREO_OFFSET] += delayBufferSample[1]; } - // The following code is pretty gross and redundant, but AFAIK it's the best way to avoid - // too many conditionals in handling the delay samples at the beginning of _clientSamples. - // Basically we try to take the samples in batches of four, and then handle the remainder - // conditionally to get rid of the rest. - - const int DOUBLE_STEREO_OFFSET = 4; - const int TRIPLE_STEREO_OFFSET = 6; - if (numSamplesDelay > 0) { // if there was a sample delay for this buffer, we need to pull samples prior to the nextOutput // to stick at the beginning diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index ef4603bc95..a5ff5e9cb1 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -278,7 +278,6 @@ void Stats::display( Audio* audio = Application::getInstance()->getAudio(); - const QHash& audioMixerInjectedStreamAudioStatsMap = audio->getAudioMixerInjectedStreamAudioStatsMap(); lines = _expanded ? 4 : 3; drawBackground(backgroundColor, horizontalOffset, 0, _pingStatsWidth, lines * STATS_PELS_PER_LINE + 10); From 142bda3ed481d4e656a9de70c0d4988e9644ce18 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 12:43:26 -0700 Subject: [PATCH 22/46] fixed audio stats typos --- interface/src/Audio.cpp | 32 +++++++++++++++++++++----------- interface/src/Audio.h | 2 +- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index e55d30d753..ede27d4b20 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -1480,7 +1480,7 @@ void Audio::renderStats(const float* color, int width, int height) { verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamLabelString, color); - renderAudioStreamStats(getDownstreamAudioStreamStats(), horizontalOffset, verticalOffset, scale, rotation, font, color); + renderAudioStreamStats(getDownstreamAudioStreamStats(), horizontalOffset, verticalOffset, scale, rotation, font, color, true); verticalOffset += STATS_HEIGHT_PER_LINE; // blank line @@ -1507,29 +1507,39 @@ void Audio::renderStats(const float* color, int width, int height) { } void Audio::renderAudioStreamStats(const AudioStreamStats& streamStats, int horizontalOffset, int& verticalOffset, - float scale, float rotation, int font, const float* color) { + float scale, float rotation, int font, const float* color, bool isDownstreamStats) { char stringBuffer[512]; - sprintf(stringBuffer, " Packet loss | overall: %5.2f%%, last_30s: %5.2f%%", + sprintf(stringBuffer, " Packet loss | overall: %5.2f%% (%d lost), last_30s: %5.2f%% (%d lost)", streamStats._packetStreamStats.getLostRate()*100.0f, - streamStats._packetStreamWindowStats.getLostRate() * 100.0f); + streamStats._packetStreamStats._numLost, + streamStats._packetStreamWindowStats.getLostRate() * 100.0f, + streamStats._packetStreamWindowStats._numLost); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); - sprintf(stringBuffer, " Ringbuffer frames | desired: %u, avg_available(10s): %u+%d, available: %u+%d", - streamStats._ringBufferDesiredJitterBufferFrames, - streamStats._ringBufferFramesAvailableAverage, - getOutputRingBufferAverageFramesAvailable(), - streamStats._ringBufferFramesAvailable, - getOutputRingBufferFramesAvailable()); + if (isDownstreamStats) { + sprintf(stringBuffer, " Ringbuffer frames | desired: %u, avg_available(10s): %u+%d, available: %u+%d", + streamStats._ringBufferDesiredJitterBufferFrames, + streamStats._ringBufferFramesAvailableAverage, + getOutputRingBufferAverageFramesAvailable(), + streamStats._ringBufferFramesAvailable, + getOutputRingBufferFramesAvailable()); + } else { + sprintf(stringBuffer, " Ringbuffer frames | desired: %u, avg_available(10s): %u, available: %u", + streamStats._ringBufferDesiredJitterBufferFrames, + streamStats._ringBufferFramesAvailableAverage, + streamStats._ringBufferFramesAvailable); + } + verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); sprintf(stringBuffer, " Ringbuffer stats | starves: %u, prev_starve_lasted: %u, frames_dropped: %u, overflows: %u", - streamStats._ringBufferStarveCount, streamStats._ringBufferStarveCount, streamStats._ringBufferConsecutiveNotMixedCount, + streamStats._ringBufferSilentFramesDropped, streamStats._ringBufferOverflowCount); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 1692a1b0ce..4346a5f352 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -250,7 +250,7 @@ private: // audio stats methods for rendering void renderAudioStreamStats(const AudioStreamStats& streamStats, int horizontalOffset, int& verticalOffset, - float scale, float rotation, int font, const float* color); + float scale, float rotation, int font, const float* color, bool isDownstreamStats = false); // Audio scope data static const unsigned int NETWORK_SAMPLES_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; From bd8b7eb71085b06d795337f2edd4ba3aec2d9282 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 12:44:16 -0700 Subject: [PATCH 23/46] minor spacing issue --- interface/src/Audio.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index ede27d4b20..ca98ae194c 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -1512,7 +1512,7 @@ void Audio::renderAudioStreamStats(const AudioStreamStats& streamStats, int hori char stringBuffer[512]; sprintf(stringBuffer, " Packet loss | overall: %5.2f%% (%d lost), last_30s: %5.2f%% (%d lost)", - streamStats._packetStreamStats.getLostRate()*100.0f, + streamStats._packetStreamStats.getLostRate() * 100.0f, streamStats._packetStreamStats._numLost, streamStats._packetStreamWindowStats.getLostRate() * 100.0f, streamStats._packetStreamWindowStats._numLost); From 06ed8e4b63867136da7b69a2518b59c4b9f8617c Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 13:47:03 -0700 Subject: [PATCH 24/46] fixed some typos in audio stats; cleaned up code --- interface/src/Audio.cpp | 46 +++++++++++++++++++++-------------------- interface/src/Audio.h | 14 ++++++------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index ca98ae194c..27a7dfc6e4 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -132,10 +132,10 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _outgoingAvatarAudioSequenceNumber(0), _incomingMixedAudioSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH), _interframeTimeGapStats(TIME_GAPS_STATS_INTERVAL_SAMPLES, TIME_GAP_STATS_WINDOW_INTERVALS), - _audioInputBufferMsecsDataAvailableStats(MSECS_PER_SECOND / (float)AUDIO_CALLBACK_MSECS * CALLBACK_ACCELERATOR_RATIO, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), - _inputRingBufferMsecsDataAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), + _audioInputMsecsReadStats(MSECS_PER_SECOND / (float)AUDIO_CALLBACK_MSECS * CALLBACK_ACCELERATOR_RATIO, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), + _inputRingBufferMsecsAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), _outputRingBufferFramesAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), - _audioOutputBufferFramesAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS) + _audioOutputMsecsUnplayedStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS) { // clear the array of locally injected samples memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); @@ -168,11 +168,11 @@ void Audio::resetStats() { _interframeTimeGapStats.reset(); - _audioInputBufferMsecsDataAvailableStats.reset(); - _inputRingBufferMsecsDataAvailableStats.reset(); + _audioInputMsecsReadStats.reset(); + _inputRingBufferMsecsAvailableStats.reset(); _outputRingBufferFramesAvailableStats.reset(); - _audioOutputBufferFramesAvailableStats.reset(); + _audioOutputMsecsUnplayedStats.reset(); } void Audio::audioMixerKilled() { @@ -522,7 +522,7 @@ void Audio::handleAudioInput() { _inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size()); float audioInputMsecsRead = inputByteArray.size() / (float)(_inputFormat.channelCount() * _inputFormat.sampleRate() * sizeof(int16_t)) * MSECS_PER_SECOND; - _audioInputBufferMsecsDataAvailableStats.update(audioInputMsecsRead); + _audioInputMsecsReadStats.update(audioInputMsecsRead); while (_inputRingBuffer.samplesAvailable() >= inputSamplesRequired) { @@ -837,10 +837,10 @@ void Audio::sendDownstreamAudioStatsPacket() { // since this function is called every second, we'll sample some of our stats here - _inputRingBufferMsecsDataAvailableStats.update(getInputRingBufferMsecsDataAvailable()); + _inputRingBufferMsecsAvailableStats.update(getInputRingBufferMsecsAvailable()); _outputRingBufferFramesAvailableStats.update(_ringBuffer.framesAvailable()); - _audioOutputBufferFramesAvailableStats.update(getOutputRingBufferFramesAvailable()); + _audioOutputMsecsUnplayedStats.update(getAudioOutputMsecsUnplayed()); // push the current seq number stats into history, which moves the history window forward 1s // (since that's how often pushStatsToHistory() is called) @@ -1431,12 +1431,12 @@ void Audio::renderStats(const float* color, int width, int height) { SharedNodePointer audioMixerNodePointer = NodeList::getInstance()->soloNodeOfType(NodeType::AudioMixer); if (!audioMixerNodePointer.isNull()) { - audioInputBufferLatency = _audioInputBufferMsecsDataAvailableStats.getWindowAverage(); - inputRingBufferLatency = getInputRingBufferAverageMsecsDataAvailable(); + audioInputBufferLatency = _audioInputMsecsReadStats.getWindowAverage(); + inputRingBufferLatency = getInputRingBufferAverageMsecsAvailable(); networkRoundtripLatency = audioMixerNodePointer->getPingMs(); mixerRingBufferLatency = _audioMixerAvatarStreamAudioStats._ringBufferFramesAvailableAverage * BUFFER_SEND_INTERVAL_MSECS; outputRingBufferLatency = _outputRingBufferFramesAvailableStats.getWindowAverage() * BUFFER_SEND_INTERVAL_MSECS; - audioOutputBufferLatency = _audioOutputBufferFramesAvailableStats.getWindowAverage() * BUFFER_SEND_INTERVAL_MSECS; + audioOutputBufferLatency = _audioOutputMsecsUnplayedStats.getWindowAverage(); } float totalLatency = audioInputBufferLatency + inputRingBufferLatency + networkRoundtripLatency + mixerRingBufferLatency + outputRingBufferLatency + audioOutputBufferLatency; @@ -1520,12 +1520,14 @@ void Audio::renderAudioStreamStats(const AudioStreamStats& streamStats, int hori drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); if (isDownstreamStats) { + + const float BUFFER_SEND_INTERVAL_MSECS = BUFFER_SEND_INTERVAL_USECS / (float)USECS_PER_MSEC; sprintf(stringBuffer, " Ringbuffer frames | desired: %u, avg_available(10s): %u+%d, available: %u+%d", streamStats._ringBufferDesiredJitterBufferFrames, streamStats._ringBufferFramesAvailableAverage, - getOutputRingBufferAverageFramesAvailable(), + (int)(getAudioOutputAverageMsecsUnplayed() / BUFFER_SEND_INTERVAL_MSECS), streamStats._ringBufferFramesAvailable, - getOutputRingBufferFramesAvailable()); + (int)(getAudioOutputMsecsUnplayed() / BUFFER_SEND_INTERVAL_MSECS)); } else { sprintf(stringBuffer, " Ringbuffer frames | desired: %u, avg_available(10s): %u, available: %u", streamStats._ringBufferDesiredJitterBufferFrames, @@ -1817,14 +1819,14 @@ int Audio::calculateNumberOfFrameSamples(int numBytes) const { return frameSamples; } -int Audio::getOutputRingBufferFramesAvailable() const { - float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float)_outputFormat.sampleRate()) - * (_desiredOutputFormat.channelCount() / (float)_outputFormat.channelCount()); - - return (_audioOutput->bufferSize() - _audioOutput->bytesFree()) * networkOutputToOutputRatio - / (sizeof(int16_t) * _ringBuffer.getNumFrameSamples()); +float Audio::getAudioOutputMsecsUnplayed() const { + int bytesAudioOutputUnplayed = _audioOutput->bufferSize() - _audioOutput->bytesFree(); + float msecsAudioOutputUnplayed = bytesAudioOutputUnplayed / (float)_outputFormat.bytesForDuration(USECS_PER_MSEC); + return msecsAudioOutputUnplayed; } -float Audio::getInputRingBufferMsecsDataAvailable() const { - return _inputRingBuffer.samplesAvailable() / (float)(_inputFormat.channelCount() * _inputFormat.sampleRate()) * MSECS_PER_SECOND; +float Audio::getInputRingBufferMsecsAvailable() const { + int bytesInInputRingBuffer = _inputRingBuffer.samplesAvailable() * sizeof(int16_t); + float msecsInInputRingBuffer = bytesInInputRingBuffer / (float)(_inputFormat.bytesForDuration(USECS_PER_MSEC)); + return msecsInInputRingBuffer; } diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 4346a5f352..67a951b8d9 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -80,11 +80,11 @@ public: const SequenceNumberStats& getIncomingMixedAudioSequenceNumberStats() const { return _incomingMixedAudioSequenceNumberStats; } - float getInputRingBufferMsecsDataAvailable() const; - float getInputRingBufferAverageMsecsDataAvailable() const { return (float)_inputRingBufferMsecsDataAvailableStats.getWindowAverage(); } + float getInputRingBufferMsecsAvailable() const; + float getInputRingBufferAverageMsecsAvailable() const { return (float)_inputRingBufferMsecsAvailableStats.getWindowAverage(); } - int getOutputRingBufferFramesAvailable() const; - int getOutputRingBufferAverageFramesAvailable() const { return (int)_audioOutputBufferFramesAvailableStats.getWindowAverage(); } + float getAudioOutputMsecsUnplayed() const; + float getAudioOutputAverageMsecsUnplayed() const { return (float)_audioOutputMsecsUnplayedStats.getWindowAverage(); } public slots: void start(); @@ -287,11 +287,11 @@ private: MovingMinMaxAvg _interframeTimeGapStats; - MovingMinMaxAvg _audioInputBufferMsecsDataAvailableStats; - MovingMinMaxAvg _inputRingBufferMsecsDataAvailableStats; + MovingMinMaxAvg _audioInputMsecsReadStats; + MovingMinMaxAvg _inputRingBufferMsecsAvailableStats; MovingMinMaxAvg _outputRingBufferFramesAvailableStats; - MovingMinMaxAvg _audioOutputBufferFramesAvailableStats; + MovingMinMaxAvg _audioOutputMsecsUnplayedStats; }; From e1365a29279f6c642f4d823337a3851d9d8cd84c Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 23 Jul 2014 14:19:47 -0700 Subject: [PATCH 25/46] Forgot one change --- interface/src/Application.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 8b00bface7..02f4842e88 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -3628,7 +3628,7 @@ ScriptEngine* Application::loadScript(const QString& scriptName, bool loadScript scriptEngine->getModelsScriptingInterface()->setModelTree(_models.getTree()); // model has some custom types - Model::registerMetaTypes(scriptEngine->getEngine()); + Model::registerMetaTypes(scriptEngine); // hook our avatar object into this script engine scriptEngine->setAvatarData(_myAvatar, "MyAvatar"); // leave it as a MyAvatar class to expose thrust features From 663cbb76dd8b122a4373848a6b607835f60a8c47 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 14:37:11 -0700 Subject: [PATCH 26/46] removed seq num reset from Audio::reset() --- interface/src/Audio.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 27a7dfc6e4..bd6e51b820 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -152,7 +152,8 @@ void Audio::init(QGLWidget *parent) { void Audio::reset() { _ringBuffer.reset(); - _outgoingAvatarAudioSequenceNumber = 0; + // we don't want to reset seq numbers when space-bar reset occurs. + //_outgoingAvatarAudioSequenceNumber = 0; resetStats(); } From 0d039b1cdb16c06af447d9bc54afec8fe699545d Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 23 Jul 2014 14:38:37 -0700 Subject: [PATCH 27/46] Fixed invokeMethod bug + removed debug --- examples/editModels.js | 1 - interface/src/models/ModelTreeRenderer.h | 3 ++- libraries/models/src/ModelsScriptingInterface.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/editModels.js b/examples/editModels.js index 458ddf7b4a..ea88c87bea 100644 --- a/examples/editModels.js +++ b/examples/editModels.js @@ -1168,7 +1168,6 @@ function handeMenuEvent(menuItem){ angles.z = array[7].value; selectedModelProperties.modelRotation = Quat.fromVec3Degrees(angles); selectedModelProperties.radius = array[8].value / 2; - print(selectedModelProperties.radius); Models.editModel(selectedModelID, selectedModelProperties); } diff --git a/interface/src/models/ModelTreeRenderer.h b/interface/src/models/ModelTreeRenderer.h index b6df71565d..d69b85efe9 100644 --- a/interface/src/models/ModelTreeRenderer.h +++ b/interface/src/models/ModelTreeRenderer.h @@ -27,6 +27,7 @@ // Generic client side Octree renderer class. class ModelTreeRenderer : public OctreeRenderer, public ModelItemFBXService { + Q_OBJECT public: ModelTreeRenderer(); virtual ~ModelTreeRenderer(); @@ -56,7 +57,7 @@ public: protected: void clearModelsCache(); - Model* getModel(const ModelItem& modelItem); + Q_INVOKABLE Model* getModel(const ModelItem& modelItem); QMap _knownModelsItemModels; QMap _unknownModelsItemModels; }; diff --git a/libraries/models/src/ModelsScriptingInterface.cpp b/libraries/models/src/ModelsScriptingInterface.cpp index 634039f949..d2c3ed1ba7 100644 --- a/libraries/models/src/ModelsScriptingInterface.cpp +++ b/libraries/models/src/ModelsScriptingInterface.cpp @@ -70,7 +70,7 @@ ModelItemProperties ModelsScriptingInterface::getModelProperties(ModelItemID mod if (_modelTree) { _modelTree->lockForRead(); ModelItem* model = const_cast(_modelTree->findModelByID(identity.id, true)); - if (model) { + if (model && _modelTree->getGeometryForModel(*model)) { model->setSittingPoints(_modelTree->getGeometryForModel(*model)->sittingPoints); results.copyFromModelItem(*model); } else { From 512034fd7cb5f10ef3123218210a3f873b43d82a Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 23 Jul 2014 15:23:27 -0700 Subject: [PATCH 28/46] Ability to modify AC avatar blendshapes from JS --- assignment-client/src/Agent.cpp | 7 ++- examples/bot.js | 15 ++++-- interface/src/avatar/Head.cpp | 2 +- .../animation/src/FacialAnimationData.cpp | 10 +++- libraries/animation/src/FacialAnimationData.h | 18 +++++++ libraries/avatars/src/AvatarData.cpp | 5 ++ libraries/avatars/src/AvatarData.h | 18 +++++-- libraries/avatars/src/HeadData.cpp | 49 +++++++++++++++++++ libraries/avatars/src/HeadData.h | 15 +++++- 9 files changed, 125 insertions(+), 14 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 0449e0d682..0f485dec14 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -229,11 +229,13 @@ void Agent::run() { // setup an Avatar for the script to use AvatarData scriptedAvatar; - + // Pretend faceshift is active so we can modify facial data + scriptedAvatar.setIsFaceshiftConnected(true); + // call model URL setters with empty URLs so our avatar, if user, will have the default models scriptedAvatar.setFaceModelURL(QUrl()); scriptedAvatar.setSkeletonModelURL(QUrl()); - + // give this AvatarData object to the script engine _scriptEngine.setAvatarData(&scriptedAvatar, "Avatar"); _scriptEngine.setAvatarHashMap(&_avatarHashMap, "AvatarList"); @@ -263,6 +265,7 @@ void Agent::run() { _scriptEngine.getModelsScriptingInterface()->setModelTree(_modelViewer.getTree()); _scriptEngine.setScriptContents(scriptContents); + _scriptEngine.run(); setFinished(true); } diff --git a/examples/bot.js b/examples/bot.js index edef280e21..1c8fd218d9 100644 --- a/examples/bot.js +++ b/examples/bot.js @@ -24,11 +24,11 @@ function printVector(string, vector) { print(string + " " + vector.x + ", " + vector.y + ", " + vector.z); } -var CHANCE_OF_MOVING = 0.005; -var CHANCE_OF_SOUND = 0.005; +var CHANCE_OF_MOVING = 0.0; +var CHANCE_OF_SOUND = 0.0; var CHANCE_OF_HEAD_TURNING = 0.05; -var CHANCE_OF_BIG_MOVE = 0.1; -var CHANCE_OF_WAVING = 0.009; +var CHANCE_OF_BIG_MOVE = 0.0; +var CHANCE_OF_WAVING = 0.0; var shouldReceiveVoxels = true; var VOXEL_FPS = 60.0; @@ -147,6 +147,13 @@ function updateBehavior(deltaTime) { if (CHANCE_OF_MOVING == 0.000) { Avatar.position = firstPosition; } + //print("hi"); + Avatar.setLeftBlink(1.0); + Avatar.setRightBlink(0.5); + Avatar.setBrowDownLeft(1.0); + Avatar.setMouthSize(1.0); + Avatar.setMouthSmileRight(1.0); + //print("ho"); if (shouldReceiveVoxels && ((cumulativeTime - lastVoxelQueryTime) > (1.0 / VOXEL_FPS))) { VoxelViewer.setPosition(Avatar.position); diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index 814377b3d2..244eaab8ab 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -146,7 +146,7 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { _browAudioLift, glm::clamp(log(_averageLoudness) / JAW_OPEN_SCALE, 0.0f, 1.0f)); } - + if (!isMine) { _faceModel.setLODDistance(static_cast(_owningAvatar)->getLODDistance()); } diff --git a/libraries/animation/src/FacialAnimationData.cpp b/libraries/animation/src/FacialAnimationData.cpp index 974cdec66f..68e61bbe72 100644 --- a/libraries/animation/src/FacialAnimationData.cpp +++ b/libraries/animation/src/FacialAnimationData.cpp @@ -29,12 +29,18 @@ _mouthSmileRightIndex(29), _jawOpenIndex(21) { } +void FacialAnimationData::setBlendshapeCoefficient(int index, float val) { + _blendshapeCoefficients.resize(max((int)_blendshapeCoefficients.size(), _jawOpenIndex + 1)); + if (index >= 0 && index < (int)_blendshapeCoefficients.size()) { + _blendshapeCoefficients[index] = val; + } +} + float FacialAnimationData::getBlendshapeCoefficient(int index) const { return (index >= 0 && index < (int)_blendshapeCoefficients.size()) ? _blendshapeCoefficients[index] : 0.0f; } -void FacialAnimationData::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, - float jawOpen) { +void FacialAnimationData::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, float jawOpen) { _blendshapeCoefficients.resize(max((int)_blendshapeCoefficients.size(), _jawOpenIndex + 1)); qFill(_blendshapeCoefficients.begin(), _blendshapeCoefficients.end(), 0.0f); _blendshapeCoefficients[_leftBlinkIndex] = leftBlink; diff --git a/libraries/animation/src/FacialAnimationData.h b/libraries/animation/src/FacialAnimationData.h index 514fd5a55f..f87f3a9748 100644 --- a/libraries/animation/src/FacialAnimationData.h +++ b/libraries/animation/src/FacialAnimationData.h @@ -26,6 +26,23 @@ public: FacialAnimationData(); + // Setters + void setLeftBlink(float val) { setBlendshapeCoefficient(_leftBlinkIndex, val); } + void setRightBlink(float val) { setBlendshapeCoefficient(_rightBlinkIndex, val); } + void setLeftEyeOpen(float val) { setBlendshapeCoefficient(_leftBlinkIndex, val); } + void setRightEyeOpen(float val) { setBlendshapeCoefficient(_rightBlinkIndex, val); } + + void setBrowDownLeft(float val) { setBlendshapeCoefficient(_browDownLeftIndex, val); } + void setBrowDownRight(float val) { setBlendshapeCoefficient(_browDownRightIndex, val); } + void setBrowUpCenter(float val) { setBlendshapeCoefficient(_browUpCenterIndex, val); } + void setBrowUpLeft(float val) { setBlendshapeCoefficient(_browUpLeftIndex, val); } + void setBrowUpRight(float val) { setBlendshapeCoefficient(_browUpRightIndex, val); } + + void setMouthSize(float val) { setBlendshapeCoefficient(_jawOpenIndex, val); } + void setMouthSmileLeft(float val) { setBlendshapeCoefficient(_mouthSmileLeftIndex, val); } + void setMouthSmileRight(float val) { setBlendshapeCoefficient(_mouthSmileRightIndex, val); } + + // Getters float getLeftBlink() const { return getBlendshapeCoefficient(_leftBlinkIndex); } float getRightBlink() const { return getBlendshapeCoefficient(_rightBlinkIndex); } float getLeftEyeOpen() const { return getBlendshapeCoefficient(_leftEyeOpenIndex); } @@ -47,6 +64,7 @@ public: const QVector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } private: + void setBlendshapeCoefficient(int index, float val); float getBlendshapeCoefficient(int index) const; int _leftBlinkIndex; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index fc5b762a0f..93186eb78a 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -45,6 +45,7 @@ AvatarData::AvatarData() : _handState(0), _keyState(NO_KEY_DOWN), _isChatCirclingEnabled(false), + _forceFaceshiftIsConnected(false), _hasNewJointRotations(true), _headData(NULL), _handData(NULL), @@ -82,6 +83,10 @@ QByteArray AvatarData::toByteArray() { // lazily allocate memory for HeadData in case we're not an Avatar instance if (!_headData) { _headData = new HeadData(this); + // In some cases, for instance for AC avatars, we might want to force faceshift to appear connected + if (_forceFaceshiftIsConnected) { + _headData->setIsFaceshiftConnected(true); + } } QByteArray avatarDataByteArray; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index b1b48e3941..33fbd39616 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -207,9 +207,20 @@ public: Q_INVOKABLE virtual QStringList getJointNames() const { return _jointNames; } // Facial Animation - Q_INVOKABLE void setLeftEyebrow(float f) { - _headData->setLeftEyebrow(f); - } + Q_INVOKABLE void setLeftBlink(float val) { _headData->setLeftBlink(val); } + Q_INVOKABLE void setRightBlink(float val) { _headData->setRightBlink(val); } + Q_INVOKABLE void setLeftEyeOpen(float val) { _headData->setLeftEyeOpen(val); } + Q_INVOKABLE void setRightEyeOpen(float val) { _headData->setRightEyeOpen(val); } + Q_INVOKABLE void setBrowDownLeft(float val) { _headData->setBrowDownLeft(val); } + Q_INVOKABLE void setBrowDownRight(float val) { _headData->setBrowDownRight(val); } + Q_INVOKABLE void setBrowUpCenter(float val) { _headData->setBrowUpCenter(val); } + Q_INVOKABLE void setBrowUpLeft(float val) { _headData->setBrowUpLeft(val); } + Q_INVOKABLE void setBrowUpRight(float val) { _headData->setBrowUpRight(val); } + Q_INVOKABLE void setMouthSize(float val) { _headData->setMouthSize(val); } + Q_INVOKABLE void setMouthSmileLeft(float val) { _headData->setMouthSmileLeft(val); } + Q_INVOKABLE void setMouthSmileRight(float val) { _headData->setMouthSmileRight(val); } + + void setIsFaceshiftConnected(bool isFaceshift) { _forceFaceshiftIsConnected = isFaceshift; } // key state void setKeyState(KeyState s) { _keyState = s; } @@ -305,6 +316,7 @@ protected: std::string _chatMessage; bool _isChatCirclingEnabled; + bool _forceFaceshiftIsConnected; bool _hasNewJointRotations; // set in AvatarData, cleared in Avatar diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 1f2bb73834..919adcd642 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -55,6 +55,55 @@ void HeadData::setOrientation(const glm::quat& orientation) { _baseRoll = eulers.z; } +void HeadData::setLeftBlink(float val) { + _facialAnimationData->setLeftBlink(val); +} + +void HeadData::setRightBlink(float val) { + _facialAnimationData->setRightBlink(val); +} + +void HeadData::setLeftEyeOpen(float val) { + _facialAnimationData->setLeftEyeOpen(val); +} + +void HeadData::setRightEyeOpen(float val) { + _facialAnimationData->setRightEyeOpen(val); +} + +void HeadData::setBrowDownLeft(float val) { + _facialAnimationData->setBrowDownLeft(val); +} + +void HeadData::setBrowDownRight(float val) { + _facialAnimationData->setBrowDownRight(val); +} + +void HeadData::setBrowUpCenter(float val) { + _facialAnimationData->setBrowUpCenter(val); +} + +void HeadData::setBrowUpLeft(float val) { + _facialAnimationData->setBrowUpLeft(val); +} + +void HeadData::setBrowUpRight(float val) { + _facialAnimationData->setBrowUpRight(val); +} + +void HeadData::setMouthSize(float val) { + _facialAnimationData->setMouthSize(val); +} + +void HeadData::setMouthSmileLeft(float val) { + _facialAnimationData->setLeftBlink(val); +} + +void HeadData::setMouthSmileRight(float val) { + _facialAnimationData->setMouthSmileRight(val); +} + + void HeadData::addYaw(float yaw) { setBaseYaw(_baseYaw + yaw); } diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index 9920776b2a..b6e877564a 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -61,9 +61,20 @@ public: void setPupilDilation(float pupilDilation) { _pupilDilation = pupilDilation; } // Facial animation - void setLeftEyebrow(float f) { + void setLeftBlink(float val); + void setRightBlink(float val); + void setLeftEyeOpen(float val); + void setRightEyeOpen(float val); + void setBrowDownLeft(float val); + void setBrowDownRight(float val); + void setBrowUpCenter(float val); + void setBrowUpLeft(float val); + void setBrowUpRight(float val); + void setMouthSize(float val); + void setMouthSmileLeft(float val); + void setMouthSmileRight(float val); - } + void setIsFaceshiftConnected(bool isFaceshiftConnected) { _isFaceshiftConnected = isFaceshiftConnected; } // degrees void addYaw(float yaw); From 3abd383cd3feac91d8a8aac37c38de3d2a7ba6df Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 15:26:49 -0700 Subject: [PATCH 29/46] cleaned up code more --- interface/src/Audio.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index bd6e51b820..5054537287 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -522,7 +522,7 @@ void Audio::handleAudioInput() { _inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size()); - float audioInputMsecsRead = inputByteArray.size() / (float)(_inputFormat.channelCount() * _inputFormat.sampleRate() * sizeof(int16_t)) * MSECS_PER_SECOND; + float audioInputMsecsRead = inputByteArray.size() / (float)(_inputFormat.bytesForDuration(USECS_PER_MSEC)); _audioInputMsecsReadStats.update(audioInputMsecsRead); while (_inputRingBuffer.samplesAvailable() >= inputSamplesRequired) { From 6aa9e6b7367ee4a95d339512a2106ced04c99326 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 23 Jul 2014 15:54:40 -0700 Subject: [PATCH 30/46] Revert "Ability to modify AC avatar blendshapes from JS" This reverts commit 512034fd7cb5f10ef3123218210a3f873b43d82a. --- assignment-client/src/Agent.cpp | 7 +-- examples/bot.js | 15 ++---- interface/src/avatar/Head.cpp | 2 +- .../animation/src/FacialAnimationData.cpp | 10 +--- libraries/animation/src/FacialAnimationData.h | 18 ------- libraries/avatars/src/AvatarData.cpp | 5 -- libraries/avatars/src/AvatarData.h | 18 ++----- libraries/avatars/src/HeadData.cpp | 49 ------------------- libraries/avatars/src/HeadData.h | 15 +----- 9 files changed, 14 insertions(+), 125 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 0f485dec14..0449e0d682 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -229,13 +229,11 @@ void Agent::run() { // setup an Avatar for the script to use AvatarData scriptedAvatar; - // Pretend faceshift is active so we can modify facial data - scriptedAvatar.setIsFaceshiftConnected(true); - + // call model URL setters with empty URLs so our avatar, if user, will have the default models scriptedAvatar.setFaceModelURL(QUrl()); scriptedAvatar.setSkeletonModelURL(QUrl()); - + // give this AvatarData object to the script engine _scriptEngine.setAvatarData(&scriptedAvatar, "Avatar"); _scriptEngine.setAvatarHashMap(&_avatarHashMap, "AvatarList"); @@ -265,7 +263,6 @@ void Agent::run() { _scriptEngine.getModelsScriptingInterface()->setModelTree(_modelViewer.getTree()); _scriptEngine.setScriptContents(scriptContents); - _scriptEngine.run(); setFinished(true); } diff --git a/examples/bot.js b/examples/bot.js index 1c8fd218d9..edef280e21 100644 --- a/examples/bot.js +++ b/examples/bot.js @@ -24,11 +24,11 @@ function printVector(string, vector) { print(string + " " + vector.x + ", " + vector.y + ", " + vector.z); } -var CHANCE_OF_MOVING = 0.0; -var CHANCE_OF_SOUND = 0.0; +var CHANCE_OF_MOVING = 0.005; +var CHANCE_OF_SOUND = 0.005; var CHANCE_OF_HEAD_TURNING = 0.05; -var CHANCE_OF_BIG_MOVE = 0.0; -var CHANCE_OF_WAVING = 0.0; +var CHANCE_OF_BIG_MOVE = 0.1; +var CHANCE_OF_WAVING = 0.009; var shouldReceiveVoxels = true; var VOXEL_FPS = 60.0; @@ -147,13 +147,6 @@ function updateBehavior(deltaTime) { if (CHANCE_OF_MOVING == 0.000) { Avatar.position = firstPosition; } - //print("hi"); - Avatar.setLeftBlink(1.0); - Avatar.setRightBlink(0.5); - Avatar.setBrowDownLeft(1.0); - Avatar.setMouthSize(1.0); - Avatar.setMouthSmileRight(1.0); - //print("ho"); if (shouldReceiveVoxels && ((cumulativeTime - lastVoxelQueryTime) > (1.0 / VOXEL_FPS))) { VoxelViewer.setPosition(Avatar.position); diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index 244eaab8ab..814377b3d2 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -146,7 +146,7 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { _browAudioLift, glm::clamp(log(_averageLoudness) / JAW_OPEN_SCALE, 0.0f, 1.0f)); } - + if (!isMine) { _faceModel.setLODDistance(static_cast(_owningAvatar)->getLODDistance()); } diff --git a/libraries/animation/src/FacialAnimationData.cpp b/libraries/animation/src/FacialAnimationData.cpp index 68e61bbe72..974cdec66f 100644 --- a/libraries/animation/src/FacialAnimationData.cpp +++ b/libraries/animation/src/FacialAnimationData.cpp @@ -29,18 +29,12 @@ _mouthSmileRightIndex(29), _jawOpenIndex(21) { } -void FacialAnimationData::setBlendshapeCoefficient(int index, float val) { - _blendshapeCoefficients.resize(max((int)_blendshapeCoefficients.size(), _jawOpenIndex + 1)); - if (index >= 0 && index < (int)_blendshapeCoefficients.size()) { - _blendshapeCoefficients[index] = val; - } -} - float FacialAnimationData::getBlendshapeCoefficient(int index) const { return (index >= 0 && index < (int)_blendshapeCoefficients.size()) ? _blendshapeCoefficients[index] : 0.0f; } -void FacialAnimationData::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, float jawOpen) { +void FacialAnimationData::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, + float jawOpen) { _blendshapeCoefficients.resize(max((int)_blendshapeCoefficients.size(), _jawOpenIndex + 1)); qFill(_blendshapeCoefficients.begin(), _blendshapeCoefficients.end(), 0.0f); _blendshapeCoefficients[_leftBlinkIndex] = leftBlink; diff --git a/libraries/animation/src/FacialAnimationData.h b/libraries/animation/src/FacialAnimationData.h index f87f3a9748..514fd5a55f 100644 --- a/libraries/animation/src/FacialAnimationData.h +++ b/libraries/animation/src/FacialAnimationData.h @@ -26,23 +26,6 @@ public: FacialAnimationData(); - // Setters - void setLeftBlink(float val) { setBlendshapeCoefficient(_leftBlinkIndex, val); } - void setRightBlink(float val) { setBlendshapeCoefficient(_rightBlinkIndex, val); } - void setLeftEyeOpen(float val) { setBlendshapeCoefficient(_leftBlinkIndex, val); } - void setRightEyeOpen(float val) { setBlendshapeCoefficient(_rightBlinkIndex, val); } - - void setBrowDownLeft(float val) { setBlendshapeCoefficient(_browDownLeftIndex, val); } - void setBrowDownRight(float val) { setBlendshapeCoefficient(_browDownRightIndex, val); } - void setBrowUpCenter(float val) { setBlendshapeCoefficient(_browUpCenterIndex, val); } - void setBrowUpLeft(float val) { setBlendshapeCoefficient(_browUpLeftIndex, val); } - void setBrowUpRight(float val) { setBlendshapeCoefficient(_browUpRightIndex, val); } - - void setMouthSize(float val) { setBlendshapeCoefficient(_jawOpenIndex, val); } - void setMouthSmileLeft(float val) { setBlendshapeCoefficient(_mouthSmileLeftIndex, val); } - void setMouthSmileRight(float val) { setBlendshapeCoefficient(_mouthSmileRightIndex, val); } - - // Getters float getLeftBlink() const { return getBlendshapeCoefficient(_leftBlinkIndex); } float getRightBlink() const { return getBlendshapeCoefficient(_rightBlinkIndex); } float getLeftEyeOpen() const { return getBlendshapeCoefficient(_leftEyeOpenIndex); } @@ -64,7 +47,6 @@ public: const QVector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } private: - void setBlendshapeCoefficient(int index, float val); float getBlendshapeCoefficient(int index) const; int _leftBlinkIndex; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 93186eb78a..fc5b762a0f 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -45,7 +45,6 @@ AvatarData::AvatarData() : _handState(0), _keyState(NO_KEY_DOWN), _isChatCirclingEnabled(false), - _forceFaceshiftIsConnected(false), _hasNewJointRotations(true), _headData(NULL), _handData(NULL), @@ -83,10 +82,6 @@ QByteArray AvatarData::toByteArray() { // lazily allocate memory for HeadData in case we're not an Avatar instance if (!_headData) { _headData = new HeadData(this); - // In some cases, for instance for AC avatars, we might want to force faceshift to appear connected - if (_forceFaceshiftIsConnected) { - _headData->setIsFaceshiftConnected(true); - } } QByteArray avatarDataByteArray; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 33fbd39616..b1b48e3941 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -207,20 +207,9 @@ public: Q_INVOKABLE virtual QStringList getJointNames() const { return _jointNames; } // Facial Animation - Q_INVOKABLE void setLeftBlink(float val) { _headData->setLeftBlink(val); } - Q_INVOKABLE void setRightBlink(float val) { _headData->setRightBlink(val); } - Q_INVOKABLE void setLeftEyeOpen(float val) { _headData->setLeftEyeOpen(val); } - Q_INVOKABLE void setRightEyeOpen(float val) { _headData->setRightEyeOpen(val); } - Q_INVOKABLE void setBrowDownLeft(float val) { _headData->setBrowDownLeft(val); } - Q_INVOKABLE void setBrowDownRight(float val) { _headData->setBrowDownRight(val); } - Q_INVOKABLE void setBrowUpCenter(float val) { _headData->setBrowUpCenter(val); } - Q_INVOKABLE void setBrowUpLeft(float val) { _headData->setBrowUpLeft(val); } - Q_INVOKABLE void setBrowUpRight(float val) { _headData->setBrowUpRight(val); } - Q_INVOKABLE void setMouthSize(float val) { _headData->setMouthSize(val); } - Q_INVOKABLE void setMouthSmileLeft(float val) { _headData->setMouthSmileLeft(val); } - Q_INVOKABLE void setMouthSmileRight(float val) { _headData->setMouthSmileRight(val); } - - void setIsFaceshiftConnected(bool isFaceshift) { _forceFaceshiftIsConnected = isFaceshift; } + Q_INVOKABLE void setLeftEyebrow(float f) { + _headData->setLeftEyebrow(f); + } // key state void setKeyState(KeyState s) { _keyState = s; } @@ -316,7 +305,6 @@ protected: std::string _chatMessage; bool _isChatCirclingEnabled; - bool _forceFaceshiftIsConnected; bool _hasNewJointRotations; // set in AvatarData, cleared in Avatar diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 919adcd642..1f2bb73834 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -55,55 +55,6 @@ void HeadData::setOrientation(const glm::quat& orientation) { _baseRoll = eulers.z; } -void HeadData::setLeftBlink(float val) { - _facialAnimationData->setLeftBlink(val); -} - -void HeadData::setRightBlink(float val) { - _facialAnimationData->setRightBlink(val); -} - -void HeadData::setLeftEyeOpen(float val) { - _facialAnimationData->setLeftEyeOpen(val); -} - -void HeadData::setRightEyeOpen(float val) { - _facialAnimationData->setRightEyeOpen(val); -} - -void HeadData::setBrowDownLeft(float val) { - _facialAnimationData->setBrowDownLeft(val); -} - -void HeadData::setBrowDownRight(float val) { - _facialAnimationData->setBrowDownRight(val); -} - -void HeadData::setBrowUpCenter(float val) { - _facialAnimationData->setBrowUpCenter(val); -} - -void HeadData::setBrowUpLeft(float val) { - _facialAnimationData->setBrowUpLeft(val); -} - -void HeadData::setBrowUpRight(float val) { - _facialAnimationData->setBrowUpRight(val); -} - -void HeadData::setMouthSize(float val) { - _facialAnimationData->setMouthSize(val); -} - -void HeadData::setMouthSmileLeft(float val) { - _facialAnimationData->setLeftBlink(val); -} - -void HeadData::setMouthSmileRight(float val) { - _facialAnimationData->setMouthSmileRight(val); -} - - void HeadData::addYaw(float yaw) { setBaseYaw(_baseYaw + yaw); } diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index b6e877564a..9920776b2a 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -61,20 +61,9 @@ public: void setPupilDilation(float pupilDilation) { _pupilDilation = pupilDilation; } // Facial animation - void setLeftBlink(float val); - void setRightBlink(float val); - void setLeftEyeOpen(float val); - void setRightEyeOpen(float val); - void setBrowDownLeft(float val); - void setBrowDownRight(float val); - void setBrowUpCenter(float val); - void setBrowUpLeft(float val); - void setBrowUpRight(float val); - void setMouthSize(float val); - void setMouthSmileLeft(float val); - void setMouthSmileRight(float val); + void setLeftEyebrow(float f) { - void setIsFaceshiftConnected(bool isFaceshiftConnected) { _isFaceshiftConnected = isFaceshiftConnected; } + } // degrees void addYaw(float yaw); From 7c89201ce8f83720c267f519fba6004ab591bd99 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 23 Jul 2014 15:54:45 -0700 Subject: [PATCH 31/46] Revert "Moved some FaceShift stuff to FacialAnimationData. Made use of" This reverts commit f3ae2280302f5e6da31202f9f01d44babd262927. --- interface/src/avatar/Head.cpp | 11 ++++++----- interface/src/devices/Faceshift.cpp | 12 ++++++++++++ interface/src/devices/Faceshift.h | 3 +++ libraries/animation/src/FacialAnimationData.cpp | 16 ---------------- libraries/animation/src/FacialAnimationData.h | 8 +------- libraries/avatars/src/AvatarData.cpp | 16 ++++++---------- libraries/avatars/src/AvatarData.h | 5 ----- libraries/avatars/src/HeadData.cpp | 6 ------ libraries/avatars/src/HeadData.h | 10 ++-------- 9 files changed, 30 insertions(+), 57 deletions(-) diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index 814377b3d2..ee242d179a 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -66,7 +66,7 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { if (isMine) { FaceTracker* faceTracker = Application::getInstance()->getActiveFaceTracker(); if ((_isFaceshiftConnected = faceTracker)) { - _facialAnimationData->setBlendshapeCoefficients(faceTracker->getBlendshapeCoefficients()); + _blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); _isFaceshiftConnected = true; } } @@ -141,10 +141,11 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { // use data to update fake Faceshift blendshape coefficients const float JAW_OPEN_SCALE = 10.f; - _facialAnimationData->updateFakeCoefficients(_leftEyeBlink, - _rightEyeBlink, - _browAudioLift, - glm::clamp(log(_averageLoudness) / JAW_OPEN_SCALE, 0.0f, 1.0f)); + Application::getInstance()->getFaceshift()->updateFakeCoefficients(_leftEyeBlink, + _rightEyeBlink, + _browAudioLift, + glm::clamp(log(_averageLoudness) / JAW_OPEN_SCALE, 0.0f, 1.0f), + _blendshapeCoefficients); } if (!isMine) { diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 40ee12af86..4105b17652 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -93,6 +93,18 @@ void Faceshift::reset() { _longTermAverageInitialized = false; } +void Faceshift::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, + float jawOpen, QVector& coefficients) const { + coefficients.resize(max((int)coefficients.size(), _facialAnimationData._jawOpenIndex + 1)); + qFill(coefficients.begin(), coefficients.end(), 0.0f); + coefficients[_facialAnimationData._leftBlinkIndex] = leftBlink; + coefficients[_facialAnimationData._rightBlinkIndex] = rightBlink; + coefficients[_facialAnimationData._browUpCenterIndex] = browUp; + coefficients[_facialAnimationData._browUpLeftIndex] = browUp; + coefficients[_facialAnimationData._browUpRightIndex] = browUp; + coefficients[_facialAnimationData._jawOpenIndex] = jawOpen; +} + void Faceshift::setTCPEnabled(bool enabled) { if ((_tcpEnabled = enabled)) { connectSocket(); diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 2b720c843d..87fd1076ee 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -61,6 +61,9 @@ public: void update(); void reset(); + void updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, + float jawOpen, QVector& coefficients) const; + signals: void connectionStateChanged(); diff --git a/libraries/animation/src/FacialAnimationData.cpp b/libraries/animation/src/FacialAnimationData.cpp index 974cdec66f..0f45675083 100644 --- a/libraries/animation/src/FacialAnimationData.cpp +++ b/libraries/animation/src/FacialAnimationData.cpp @@ -11,10 +11,6 @@ #include "FacialAnimationData.h" -#ifndef max -inline int max(int a, int b) { return a > b ? a : b; } -#endif - FacialAnimationData::FacialAnimationData() :_leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes _rightBlinkIndex(1), _leftEyeOpenIndex(8), @@ -31,16 +27,4 @@ _jawOpenIndex(21) { float FacialAnimationData::getBlendshapeCoefficient(int index) const { return (index >= 0 && index < (int)_blendshapeCoefficients.size()) ? _blendshapeCoefficients[index] : 0.0f; -} - -void FacialAnimationData::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, - float jawOpen) { - _blendshapeCoefficients.resize(max((int)_blendshapeCoefficients.size(), _jawOpenIndex + 1)); - qFill(_blendshapeCoefficients.begin(), _blendshapeCoefficients.end(), 0.0f); - _blendshapeCoefficients[_leftBlinkIndex] = leftBlink; - _blendshapeCoefficients[_rightBlinkIndex] = rightBlink; - _blendshapeCoefficients[_browUpCenterIndex] = browUp; - _blendshapeCoefficients[_browUpLeftIndex] = browUp; - _blendshapeCoefficients[_browUpRightIndex] = browUp; - _blendshapeCoefficients[_jawOpenIndex] = jawOpen; } \ No newline at end of file diff --git a/libraries/animation/src/FacialAnimationData.h b/libraries/animation/src/FacialAnimationData.h index 514fd5a55f..c8d740c1c9 100644 --- a/libraries/animation/src/FacialAnimationData.h +++ b/libraries/animation/src/FacialAnimationData.h @@ -21,8 +21,6 @@ class FacialAnimationData : public QObject { public: friend class Faceshift; - friend class HeadData; - friend class AvatarData; FacialAnimationData(); @@ -41,11 +39,6 @@ public: float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); } float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); } - void updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, float jawOpen); - - void setBlendshapeCoefficients(const QVector& coefficients) { _blendshapeCoefficients = coefficients; } - const QVector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } - private: float getBlendshapeCoefficient(int index) const; @@ -65,6 +58,7 @@ private: int _jawOpenIndex; + //Only used by agents, since FaceTracker has its own _blendshapeCoefficients; QVector _blendshapeCoefficients; }; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index fc5b762a0f..6cfa4ba488 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -29,8 +29,6 @@ #include "AvatarData.h" -#include "../animation/src/FacialAnimationData.h" - quint64 DEFAULT_FILTERED_LOG_EXPIRY = 2 * USECS_PER_SECOND; using namespace std; @@ -153,12 +151,10 @@ QByteArray AvatarData::toByteArray() { memcpy(destinationBuffer, &_headData->_browAudioLift, sizeof(float)); destinationBuffer += sizeof(float); - const QVector& blendshapeCoefficients = _headData->_facialAnimationData->getBlendshapeCoefficients(); - - *destinationBuffer++ = blendshapeCoefficients.size(); - memcpy(destinationBuffer, blendshapeCoefficients.data(), - blendshapeCoefficients.size() * sizeof(float)); - destinationBuffer += blendshapeCoefficients.size() * sizeof(float); + *destinationBuffer++ = _headData->_blendshapeCoefficients.size(); + memcpy(destinationBuffer, _headData->_blendshapeCoefficients.data(), + _headData->_blendshapeCoefficients.size() * sizeof(float)); + destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); } // pupil dilation @@ -433,8 +429,8 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { return maxAvailableSize; } - _headData->_facialAnimationData->_blendshapeCoefficients.resize(numCoefficients); - memcpy(_headData->_facialAnimationData->_blendshapeCoefficients.data(), sourceBuffer, blendDataSize); + _headData->_blendshapeCoefficients.resize(numCoefficients); + memcpy(_headData->_blendshapeCoefficients.data(), sourceBuffer, blendDataSize); sourceBuffer += numCoefficients * sizeof(float); //bitItemsDataSize = 4 * sizeof(float) + 1 + blendDataSize; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index b1b48e3941..1b8f6e6007 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -206,11 +206,6 @@ public: Q_INVOKABLE virtual QStringList getJointNames() const { return _jointNames; } - // Facial Animation - Q_INVOKABLE void setLeftEyebrow(float f) { - _headData->setLeftEyebrow(f); - } - // key state void setKeyState(KeyState s) { _keyState = s; } KeyState keyState() const { return _keyState; } diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 1f2bb73834..c691a2c28a 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -16,8 +16,6 @@ #include "AvatarData.h" #include "HeadData.h" -#include "../animation/src/FacialAnimationData.h" - HeadData::HeadData(AvatarData* owningAvatar) : _baseYaw(0.0f), _basePitch(0.0f), @@ -31,7 +29,6 @@ HeadData::HeadData(AvatarData* owningAvatar) : _rightEyeBlink(0.0f), _averageLoudness(0.0f), _browAudioLift(0.0f), - _facialAnimationData(new FacialAnimationData), _owningAvatar(owningAvatar) { @@ -67,6 +64,3 @@ void HeadData::addRoll(float roll) { setBaseRoll(_baseRoll + roll); } -const QVector& HeadData::getBlendshapeCoefficients() const { - return _facialAnimationData->_blendshapeCoefficients; -} \ No newline at end of file diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index 9920776b2a..9e2920ae85 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -28,7 +28,6 @@ const float MIN_HEAD_ROLL = -50.f; const float MAX_HEAD_ROLL = 50.f; class AvatarData; -class FacialAnimationData; class HeadData { public: @@ -55,16 +54,11 @@ public: float getAudioAverageLoudness() const { return _audioAverageLoudness; } void setAudioAverageLoudness(float audioAverageLoudness) { _audioAverageLoudness = audioAverageLoudness; } - const QVector& getBlendshapeCoefficients() const; + const QVector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } float getPupilDilation() const { return _pupilDilation; } void setPupilDilation(float pupilDilation) { _pupilDilation = pupilDilation; } - // Facial animation - void setLeftEyebrow(float f) { - - } - // degrees void addYaw(float yaw); void addPitch(float pitch); @@ -91,7 +85,7 @@ protected: float _averageLoudness; float _browAudioLift; float _audioAverageLoudness; - FacialAnimationData* _facialAnimationData; + QVector _blendshapeCoefficients; float _pupilDilation; AvatarData* _owningAvatar; From ba9d89685678ae38433abe45c5b2f4ad340e3921 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 23 Jul 2014 15:54:47 -0700 Subject: [PATCH 32/46] Revert "Added FacialAnimationData class" This reverts commit fc9fb11e4f6039f5c70bf0e00de2f4c82984929f. --- examples/bot.js | 2 +- interface/src/devices/Faceshift.cpp | 50 ++++++++------ interface/src/devices/Faceshift.h | 43 +++++++----- .../animation/src/FacialAnimationData.cpp | 30 --------- libraries/animation/src/FacialAnimationData.h | 65 ------------------- 5 files changed, 60 insertions(+), 130 deletions(-) delete mode 100644 libraries/animation/src/FacialAnimationData.cpp delete mode 100644 libraries/animation/src/FacialAnimationData.h diff --git a/examples/bot.js b/examples/bot.js index edef280e21..e42d234abf 100644 --- a/examples/bot.js +++ b/examples/bot.js @@ -92,7 +92,7 @@ function playRandomSound() { botNumber = getRandomInt(1, 100); if (botNumber <= 20) { - newFaceFilePrefix = "amber"; + newFaceFilePrefix = "bot" + botNumber; newBodyFilePrefix = "defaultAvatar_body" } else { if (botNumber <= 40) { diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 4105b17652..a7d50814e2 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -32,6 +32,18 @@ Faceshift::Faceshift() : _eyeGazeLeftYaw(0.0f), _eyeGazeRightPitch(0.0f), _eyeGazeRightYaw(0.0f), + _leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes + _rightBlinkIndex(1), + _leftEyeOpenIndex(8), + _rightEyeOpenIndex(9), + _browDownLeftIndex(14), + _browDownRightIndex(15), + _browUpCenterIndex(16), + _browUpLeftIndex(17), + _browUpRightIndex(18), + _mouthSmileLeftIndex(28), + _mouthSmileRightIndex(29), + _jawOpenIndex(21), _longTermAverageEyePitch(0.0f), _longTermAverageEyeYaw(0.0f), _longTermAverageInitialized(false) @@ -95,14 +107,14 @@ void Faceshift::reset() { void Faceshift::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, float jawOpen, QVector& coefficients) const { - coefficients.resize(max((int)coefficients.size(), _facialAnimationData._jawOpenIndex + 1)); + coefficients.resize(max((int)coefficients.size(), _jawOpenIndex + 1)); qFill(coefficients.begin(), coefficients.end(), 0.0f); - coefficients[_facialAnimationData._leftBlinkIndex] = leftBlink; - coefficients[_facialAnimationData._rightBlinkIndex] = rightBlink; - coefficients[_facialAnimationData._browUpCenterIndex] = browUp; - coefficients[_facialAnimationData._browUpLeftIndex] = browUp; - coefficients[_facialAnimationData._browUpRightIndex] = browUp; - coefficients[_facialAnimationData._jawOpenIndex] = jawOpen; + coefficients[_leftBlinkIndex] = leftBlink; + coefficients[_rightBlinkIndex] = rightBlink; + coefficients[_browUpCenterIndex] = browUp; + coefficients[_browUpLeftIndex] = browUp; + coefficients[_browUpRightIndex] = browUp; + coefficients[_jawOpenIndex] = jawOpen; } void Faceshift::setTCPEnabled(bool enabled) { @@ -205,40 +217,40 @@ void Faceshift::receive(const QByteArray& buffer) { const vector& names = static_cast(msg.get())->blendshape_names(); for (size_t i = 0; i < names.size(); i++) { if (names[i] == "EyeBlink_L") { - _facialAnimationData._leftBlinkIndex = i; + _leftBlinkIndex = i; } else if (names[i] == "EyeBlink_R") { - _facialAnimationData._rightBlinkIndex = i; + _rightBlinkIndex = i; } else if (names[i] == "EyeOpen_L") { - _facialAnimationData._leftEyeOpenIndex = i; + _leftEyeOpenIndex = i; } else if (names[i] == "EyeOpen_R") { - _facialAnimationData._rightEyeOpenIndex = i; + _rightEyeOpenIndex = i; } else if (names[i] == "BrowsD_L") { - _facialAnimationData._browDownLeftIndex = i; + _browDownLeftIndex = i; } else if (names[i] == "BrowsD_R") { - _facialAnimationData._browDownRightIndex = i; + _browDownRightIndex = i; } else if (names[i] == "BrowsU_C") { - _facialAnimationData._browUpCenterIndex = i; + _browUpCenterIndex = i; } else if (names[i] == "BrowsU_L") { - _facialAnimationData._browUpLeftIndex = i; + _browUpLeftIndex = i; } else if (names[i] == "BrowsU_R") { - _facialAnimationData._browUpRightIndex = i; + _browUpRightIndex = i; } else if (names[i] == "JawOpen") { - _facialAnimationData._jawOpenIndex = i; + _jawOpenIndex = i; } else if (names[i] == "MouthSmile_L") { - _facialAnimationData._mouthSmileLeftIndex = i; + _mouthSmileLeftIndex = i; } else if (names[i] == "MouthSmile_R") { - _facialAnimationData._mouthSmileRightIndex = i; + _mouthSmileRightIndex = i; } } break; diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 87fd1076ee..2fc1aaddb1 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -18,7 +18,6 @@ #include #include "FaceTracker.h" -#include "FacialAnimationData.h" /// Handles interaction with the Faceshift software, which provides head position/orientation and facial features. class Faceshift : public FaceTracker { @@ -43,20 +42,20 @@ public: float getEyeGazeRightPitch() const { return _eyeGazeRightPitch; } float getEyeGazeRightYaw() const { return _eyeGazeRightYaw; } - float getLeftBlink() const { return getBlendshapeCoefficient(_facialAnimationData._leftBlinkIndex); } - float getRightBlink() const { return getBlendshapeCoefficient(_facialAnimationData._rightBlinkIndex); } - float getLeftEyeOpen() const { return getBlendshapeCoefficient(_facialAnimationData._leftEyeOpenIndex); } - float getRightEyeOpen() const { return getBlendshapeCoefficient(_facialAnimationData._rightEyeOpenIndex); } + float getLeftBlink() const { return getBlendshapeCoefficient(_leftBlinkIndex); } + float getRightBlink() const { return getBlendshapeCoefficient(_rightBlinkIndex); } + float getLeftEyeOpen() const { return getBlendshapeCoefficient(_leftEyeOpenIndex); } + float getRightEyeOpen() const { return getBlendshapeCoefficient(_rightEyeOpenIndex); } - float getBrowDownLeft() const { return getBlendshapeCoefficient(_facialAnimationData._browDownLeftIndex); } - float getBrowDownRight() const { return getBlendshapeCoefficient(_facialAnimationData._browDownRightIndex); } - float getBrowUpCenter() const { return getBlendshapeCoefficient(_facialAnimationData._browUpCenterIndex); } - float getBrowUpLeft() const { return getBlendshapeCoefficient(_facialAnimationData._browUpLeftIndex); } - float getBrowUpRight() const { return getBlendshapeCoefficient(_facialAnimationData._browUpRightIndex); } + float getBrowDownLeft() const { return getBlendshapeCoefficient(_browDownLeftIndex); } + float getBrowDownRight() const { return getBlendshapeCoefficient(_browDownRightIndex); } + float getBrowUpCenter() const { return getBlendshapeCoefficient(_browUpCenterIndex); } + float getBrowUpLeft() const { return getBlendshapeCoefficient(_browUpLeftIndex); } + float getBrowUpRight() const { return getBlendshapeCoefficient(_browUpRightIndex); } - float getMouthSize() const { return getBlendshapeCoefficient(_facialAnimationData._jawOpenIndex); } - float getMouthSmileLeft() const { return getBlendshapeCoefficient(_facialAnimationData._mouthSmileLeftIndex); } - float getMouthSmileRight() const { return getBlendshapeCoefficient(_facialAnimationData._mouthSmileRightIndex); } + float getMouthSize() const { return getBlendshapeCoefficient(_jawOpenIndex); } + float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); } + float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); } void update(); void reset(); @@ -103,9 +102,23 @@ private: float _eyeGazeRightPitch; float _eyeGazeRightYaw; - // stores blendshape indexes - FacialAnimationData _facialAnimationData; + int _leftBlinkIndex; + int _rightBlinkIndex; + int _leftEyeOpenIndex; + int _rightEyeOpenIndex; + // Brows + int _browDownLeftIndex; + int _browDownRightIndex; + int _browUpCenterIndex; + int _browUpLeftIndex; + int _browUpRightIndex; + + int _mouthSmileLeftIndex; + int _mouthSmileRightIndex; + + int _jawOpenIndex; + // degrees float _longTermAverageEyePitch; float _longTermAverageEyeYaw; diff --git a/libraries/animation/src/FacialAnimationData.cpp b/libraries/animation/src/FacialAnimationData.cpp deleted file mode 100644 index 0f45675083..0000000000 --- a/libraries/animation/src/FacialAnimationData.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// -// FacialAnimationData.cpp -// interface/src/devices -// -// Created by Ben Arnold on 7/21/2014. -// Copyright 2014 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "FacialAnimationData.h" - -FacialAnimationData::FacialAnimationData() :_leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes -_rightBlinkIndex(1), -_leftEyeOpenIndex(8), -_rightEyeOpenIndex(9), -_browDownLeftIndex(14), -_browDownRightIndex(15), -_browUpCenterIndex(16), -_browUpLeftIndex(17), -_browUpRightIndex(18), -_mouthSmileLeftIndex(28), -_mouthSmileRightIndex(29), -_jawOpenIndex(21) { -} - -float FacialAnimationData::getBlendshapeCoefficient(int index) const { - return (index >= 0 && index < (int)_blendshapeCoefficients.size()) ? _blendshapeCoefficients[index] : 0.0f; -} \ No newline at end of file diff --git a/libraries/animation/src/FacialAnimationData.h b/libraries/animation/src/FacialAnimationData.h deleted file mode 100644 index c8d740c1c9..0000000000 --- a/libraries/animation/src/FacialAnimationData.h +++ /dev/null @@ -1,65 +0,0 @@ -// -// FacialAnimationData.h -// interface/src/devices -// -// Created by Ben Arnold on 7/21/2014. -// Copyright 2014 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_FacialAnimationData_h -#define hifi_FacialAnimationData_h - -#include -#include - -/// Stores facial animation data for use by faceshift or scripts -class FacialAnimationData : public QObject { - Q_OBJECT - -public: - friend class Faceshift; - - FacialAnimationData(); - - float getLeftBlink() const { return getBlendshapeCoefficient(_leftBlinkIndex); } - float getRightBlink() const { return getBlendshapeCoefficient(_rightBlinkIndex); } - float getLeftEyeOpen() const { return getBlendshapeCoefficient(_leftEyeOpenIndex); } - float getRightEyeOpen() const { return getBlendshapeCoefficient(_rightEyeOpenIndex); } - - float getBrowDownLeft() const { return getBlendshapeCoefficient(_browDownLeftIndex); } - float getBrowDownRight() const { return getBlendshapeCoefficient(_browDownRightIndex); } - float getBrowUpCenter() const { return getBlendshapeCoefficient(_browUpCenterIndex); } - float getBrowUpLeft() const { return getBlendshapeCoefficient(_browUpLeftIndex); } - float getBrowUpRight() const { return getBlendshapeCoefficient(_browUpRightIndex); } - - float getMouthSize() const { return getBlendshapeCoefficient(_jawOpenIndex); } - float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); } - float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); } - -private: - float getBlendshapeCoefficient(int index) const; - - int _leftBlinkIndex; - int _rightBlinkIndex; - int _leftEyeOpenIndex; - int _rightEyeOpenIndex; - - int _browDownLeftIndex; - int _browDownRightIndex; - int _browUpCenterIndex; - int _browUpLeftIndex; - int _browUpRightIndex; - - int _mouthSmileLeftIndex; - int _mouthSmileRightIndex; - - int _jawOpenIndex; - - //Only used by agents, since FaceTracker has its own _blendshapeCoefficients; - QVector _blendshapeCoefficients; -}; - -#endif // hifi_FacialAnimationData_h \ No newline at end of file From aba031a1259936eca0bbda55a6ffb72d9b8c33fe Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 23 Jul 2014 16:25:45 -0700 Subject: [PATCH 33/46] Proper blendshape JS api. --- assignment-client/src/Agent.cpp | 3 ++- libraries/avatars/src/AvatarData.cpp | 4 ++++ libraries/avatars/src/AvatarData.h | 6 +++++- libraries/avatars/src/HeadData.cpp | 22 ++++++++++++++++++++++ libraries/avatars/src/HeadData.h | 1 + libraries/fbx/src/FBXReader.cpp | 2 ++ libraries/fbx/src/FBXReader.h | 2 ++ 7 files changed, 38 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 0449e0d682..c961d8f19c 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -229,7 +229,8 @@ void Agent::run() { // setup an Avatar for the script to use AvatarData scriptedAvatar; - + scriptedAvatar.setForceFaceshiftConnected(true); + // call model URL setters with empty URLs so our avatar, if user, will have the default models scriptedAvatar.setFaceModelURL(QUrl()); scriptedAvatar.setSkeletonModelURL(QUrl()); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 6cfa4ba488..c3ea2f8b50 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -43,6 +43,7 @@ AvatarData::AvatarData() : _handState(0), _keyState(NO_KEY_DOWN), _isChatCirclingEnabled(false), + _forceFaceshiftConnected(false), _hasNewJointRotations(true), _headData(NULL), _handData(NULL), @@ -80,6 +81,9 @@ QByteArray AvatarData::toByteArray() { // lazily allocate memory for HeadData in case we're not an Avatar instance if (!_headData) { _headData = new HeadData(this); + if (_forceFaceshiftConnected) { + _headData->_isFaceshiftConnected = true; + } } QByteArray avatarDataByteArray; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 1b8f6e6007..768633c366 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -206,6 +206,10 @@ public: Q_INVOKABLE virtual QStringList getJointNames() const { return _jointNames; } + Q_INVOKABLE void setBlendshape(QString name, float val) { _headData->setBlendshape(name, val); } + + void setForceFaceshiftConnected(bool connected) { _forceFaceshiftConnected = connected; } + // key state void setKeyState(KeyState s) { _keyState = s; } KeyState keyState() const { return _keyState; } @@ -300,7 +304,7 @@ protected: std::string _chatMessage; bool _isChatCirclingEnabled; - + bool _forceFaceshiftConnected; bool _hasNewJointRotations; // set in AvatarData, cleared in Avatar HeadData* _headData; diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index c691a2c28a..31001bf162 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -16,6 +16,8 @@ #include "AvatarData.h" #include "HeadData.h" +#include "../fbx/src/FBXReader.h" + HeadData::HeadData(AvatarData* owningAvatar) : _baseYaw(0.0f), _basePitch(0.0f), @@ -52,6 +54,26 @@ void HeadData::setOrientation(const glm::quat& orientation) { _baseRoll = eulers.z; } +void HeadData::setBlendshape(QString name, float val) { + static bool hasInitializedLookupMap = false; + static QMap blendshapeLookupMap; + //Lazily construct a lookup map from the blendshapes + if (!hasInitializedLookupMap) { + for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { + blendshapeLookupMap[FACESHIFT_BLENDSHAPES[i]] = i; + } + } + + //Check to see if the named blendshape exists, and then set its value if it does + auto it = blendshapeLookupMap.find(name); + if (it != blendshapeLookupMap.end()) { + if (_blendshapeCoefficients.size() <= it.value()) { + _blendshapeCoefficients.resize(it.value() + 1); + } + _blendshapeCoefficients[it.value()] = val; + } +} + void HeadData::addYaw(float yaw) { setBaseYaw(_baseYaw + yaw); } diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index 9e2920ae85..782386c649 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -54,6 +54,7 @@ public: float getAudioAverageLoudness() const { return _audioAverageLoudness; } void setAudioAverageLoudness(float audioAverageLoudness) { _audioAverageLoudness = audioAverageLoudness; } + void setBlendshape(QString name, float val); const QVector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } float getPupilDilation() const { return _pupilDilation; } diff --git a/libraries/fbx/src/FBXReader.cpp b/libraries/fbx/src/FBXReader.cpp index f366b6a5c3..cf726800a1 100644 --- a/libraries/fbx/src/FBXReader.cpp +++ b/libraries/fbx/src/FBXReader.cpp @@ -577,6 +577,8 @@ const char* FACESHIFT_BLENDSHAPES[] = { "" }; +const int NUM_FACESHIFT_BLENDSHAPES = sizeof(FACESHIFT_BLENDSHAPES) / sizeof(char*); + const char* HUMANIK_JOINTS[] = { "RightHand", "RightForeArm", diff --git a/libraries/fbx/src/FBXReader.h b/libraries/fbx/src/FBXReader.h index c336252574..b5340978c1 100644 --- a/libraries/fbx/src/FBXReader.h +++ b/libraries/fbx/src/FBXReader.h @@ -29,6 +29,8 @@ typedef QList FBXNodeList; /// The names of the blendshapes expected by Faceshift, terminated with an empty string. extern const char* FACESHIFT_BLENDSHAPES[]; +/// The size of FACESHIFT_BLENDSHAPES +extern const int NUM_FACESHIFT_BLENDSHAPES; /// The names of the joints in the Maya HumanIK rig, terminated with an empty string. extern const char* HUMANIK_JOINTS[]; From c0e02107c56ac6d68f11d0c2ba44c6f09c6b142b Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Wed, 23 Jul 2014 16:51:26 -0700 Subject: [PATCH 34/46] minor cleanup --- libraries/shared/src/Ragdoll.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/libraries/shared/src/Ragdoll.h b/libraries/shared/src/Ragdoll.h index 59c1291725..682f0f8dae 100644 --- a/libraries/shared/src/Ragdoll.h +++ b/libraries/shared/src/Ragdoll.h @@ -28,11 +28,7 @@ public: void applyAccumulatedDelta(); glm::vec3 getAccumulatedDelta() const { - glm::vec3 foo(0.0f); - if (_numDeltas > 0) { - foo = _accumulatedDelta / (float)_numDeltas; - } - return foo; + return (_numDeltas > 0) ? _accumulatedDelta / (float)_numDeltas : glm::vec3(0.0f); } glm::vec3 _position; From b62f04e9fe498e411ac448307f89c475571255d5 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Wed, 23 Jul 2014 16:51:53 -0700 Subject: [PATCH 35/46] check for nan's coming out of rotationBetween() --- libraries/shared/src/SharedUtil.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libraries/shared/src/SharedUtil.cpp b/libraries/shared/src/SharedUtil.cpp index b5be502ed5..e795ea746c 100644 --- a/libraries/shared/src/SharedUtil.cpp +++ b/libraries/shared/src/SharedUtil.cpp @@ -733,6 +733,13 @@ glm::quat rotationBetween(const glm::vec3& v1, const glm::vec3& v2) { } } else { axis = glm::normalize(glm::cross(v1, v2)); + // It is possible for axis to be nan even when angle is not less than EPSILON. + // For example when angle is small but not tiny but v1 and v2 and have very short lengths. + if (glm::isnan(glm::dot(axis, axis))) { + // set angle and axis to values that will generate an identity rotation + angle = 0.0f; + axis = glm::vec3(1.0f, 0.0f, 0.0f); + } } return glm::angleAxis(angle, axis); } From 72c74984540a6f6bc593be7136871e9d6de06d28 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Wed, 23 Jul 2014 16:56:40 -0700 Subject: [PATCH 36/46] add JointState::getDistanceToParent() --- interface/src/renderer/JointState.cpp | 4 ++++ interface/src/renderer/JointState.h | 2 ++ 2 files changed, 6 insertions(+) diff --git a/interface/src/renderer/JointState.cpp b/interface/src/renderer/JointState.cpp index 865036df8e..26fa29c4a9 100644 --- a/interface/src/renderer/JointState.cpp +++ b/interface/src/renderer/JointState.cpp @@ -20,6 +20,7 @@ JointState::JointState() : _animationPriority(0.0f), _positionInParentFrame(0.0f), + _distanceToParent(0.0f), _fbxJoint(NULL), _constraint(NULL) { } @@ -29,6 +30,7 @@ JointState::JointState(const JointState& other) : _constraint(NULL) { _rotation = other._rotation; _rotationInConstrainedFrame = other._rotationInConstrainedFrame; _positionInParentFrame = other._positionInParentFrame; + _distanceToParent = other._distanceToParent; _animationPriority = other._animationPriority; _fbxJoint = other._fbxJoint; // DO NOT copy _constraint @@ -72,6 +74,7 @@ void JointState::copyState(const JointState& state) { _rotation = extractRotation(_transform); _rotationInConstrainedFrame = state._rotationInConstrainedFrame; _positionInParentFrame = state._positionInParentFrame; + _distanceToParent = state._distanceToParent; _visibleTransform = state._visibleTransform; _visibleRotation = extractRotation(_visibleTransform); @@ -82,6 +85,7 @@ void JointState::copyState(const JointState& state) { void JointState::initTransform(const glm::mat4& parentTransform) { computeTransform(parentTransform); _positionInParentFrame = glm::inverse(extractRotation(parentTransform)) * (extractTranslation(_transform) - extractTranslation(parentTransform)); + _distanceToParent = glm::length(_positionInParentFrame); } void JointState::computeTransform(const glm::mat4& parentTransform) { diff --git a/interface/src/renderer/JointState.h b/interface/src/renderer/JointState.h index 86ee5d01b7..a3b792abc4 100644 --- a/interface/src/renderer/JointState.h +++ b/interface/src/renderer/JointState.h @@ -51,6 +51,7 @@ public: glm::quat getRotationInParentFrame() const; glm::quat getVisibleRotationInParentFrame() const; const glm::vec3& getPositionInParentFrame() const { return _positionInParentFrame; } + float getDistanceToParent() const { return _distanceToParent; } int getParentIndex() const { return _fbxJoint->parentIndex; } @@ -104,6 +105,7 @@ private: glm::quat _rotation; // joint- to model-frame glm::quat _rotationInConstrainedFrame; // rotation in frame where angular constraints would be applied glm::vec3 _positionInParentFrame; // only changes when the Model is scaled + float _distanceToParent; glm::mat4 _visibleTransform; glm::quat _visibleRotation; From 559188caced5c9cb3c2bb3eeb993588c1e018605 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Wed, 23 Jul 2014 17:00:17 -0700 Subject: [PATCH 37/46] parent-relative linear slaving of verlet shapes --- interface/src/avatar/SkeletonModel.cpp | 57 ++++++++++++++++++++------ 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 59f39834e3..1a68b86a14 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -533,6 +533,7 @@ void SkeletonModel::buildRagdollConstraints() { _ragdollConstraints.push_back(anchor); } else { DistanceConstraint* bone = new DistanceConstraint(&(_ragdollPoints[i]), &(_ragdollPoints[parentIndex])); + bone->setDistance(state.getDistanceToParent()); _ragdollConstraints.push_back(bone); families.insert(parentIndex, i); } @@ -597,11 +598,7 @@ void SkeletonModel::updateVisibleJointStates() { // virtual void SkeletonModel::stepRagdollForward(float deltaTime) { - // NOTE: increasing this timescale reduces vibrations in the ragdoll solution and reduces tunneling - // but makes the shapes slower to follow the body (introduces lag). - const float RAGDOLL_FOLLOWS_JOINTS_TIMESCALE = 0.05f; - float fraction = glm::clamp(deltaTime / RAGDOLL_FOLLOWS_JOINTS_TIMESCALE, 0.0f, 1.0f); - moveShapesTowardJoints(fraction); + moveShapesTowardJoints(deltaTime); } float DENSITY_OF_WATER = 1000.0f; // kg/m^3 @@ -676,17 +673,51 @@ void SkeletonModel::buildShapes() { enforceRagdollConstraints(); } -void SkeletonModel::moveShapesTowardJoints(float fraction) { +void SkeletonModel::moveShapesTowardJoints(float deltaTime) { const int numStates = _jointStates.size(); assert(_jointStates.size() == _ragdollPoints.size()); - assert(fraction >= 0.0f && fraction <= 1.0f); - if (_ragdollPoints.size() == numStates) { - float oneMinusFraction = 1.0f - fraction; - int numJoints = _jointStates.size(); - for (int i = 0; i < numJoints; ++i) { - _ragdollPoints[i]._lastPosition = _ragdollPoints[i]._position; - _ragdollPoints[i]._position = oneMinusFraction * _ragdollPoints[i]._position + fraction * _jointStates.at(i).getPosition(); + if (_ragdollPoints.size() != numStates) { + return; + } + + const float RAGDOLL_FOLLOWS_JOINTS_TIMESCALE = 0.05f; + float fraction = glm::clamp(deltaTime / RAGDOLL_FOLLOWS_JOINTS_TIMESCALE, 0.0f, 1.0f); + + // SIMPLE LINEAR_SLAVING -- KEEP this implementation for reference + //float oneMinusFraction = 1.0f - fraction; + //for (int i = 0; i < numStates; ++i) { + // _ragdollPoints[i]._lastPosition = _ragdollPoints[i]._position; + // _ragdollPoints[i]._position = oneMinusFraction * _ragdollPoints[i]._position + fraction * _jointStates.at(i).getPosition(); + //} + // LINEAR_SLAVING -- KEEP + + // parent-relative linear slaving + for (int i = 0; i < numStates; ++i) { + JointState& state = _jointStates[i]; + _ragdollPoints[i]._lastPosition = _ragdollPoints.at(i)._position; + + int p = state.getParentIndex(); + if (p == -1) { + _ragdollPoints[i]._position = glm::vec3(0.0f); + continue; } + if (state.getDistanceToParent() < EPSILON) { + _ragdollPoints[i]._position = _ragdollPoints.at(p)._position; + continue; + } + + const JointState& parentState = _jointStates.at(p); + glm::vec3 targetBone = state.getPosition() - parentState.getPosition(); + glm::vec3 bone = _ragdollPoints.at(i)._lastPosition - _ragdollPoints.at(p)._lastPosition; + + glm::vec3 newBone = (1.0f - fraction) * bone + fraction * targetBone; + float boneLength = glm::length(newBone); + if (boneLength > EPSILON) { + // slam newBone's lenght to that of the joint, which helps maintain distance constraints + newBone *= state.getDistanceToParent() / boneLength; + } + // we set the new position relative to parent + _ragdollPoints[i]._position = _ragdollPoints.at(p)._position + newBone; } } From 3d3b4572e30a756c61fa97f3f9b7ad7127919635 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Wed, 23 Jul 2014 17:19:41 -0700 Subject: [PATCH 38/46] Added bot_randomExpression.js to test blendshapes --- examples/bot_randomExpression.js | 144 +++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 examples/bot_randomExpression.js diff --git a/examples/bot_randomExpression.js b/examples/bot_randomExpression.js new file mode 100644 index 0000000000..2d2653e6d5 --- /dev/null +++ b/examples/bot_randomExpression.js @@ -0,0 +1,144 @@ +// +// bot_randomExpression.js +// examples +// +// Created by Ben Arnold on 7/23/14. +// Copyright 2014 High Fidelity, Inc. +// +// This is an example script that demonstrates an NPC avatar with +// random facial expressions. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +function getRandomFloat(min, max) { + return Math.random() * (max - min) + min; +} + +function getRandomInt (min, max) { + return Math.floor(Math.random() * (max - min + 1)) + min; +} + +function printVector(string, vector) { + print(string + " " + vector.x + ", " + vector.y + ", " + vector.z); +} + +var timePassed = 0.0; +var updateSpeed = 3.0; + +var X_MIN = 5.0; +var X_MAX = 15.0; +var Z_MIN = 5.0; +var Z_MAX = 15.0; +var Y_PELVIS = 1.0; + +// pick an integer between 1 and 100 for the body model for this bot +botNumber = getRandomInt(1, 100); + +newFaceFilePrefix = "ron"; + +newBodyFilePrefix = "bot" + botNumber; + +// set the face model fst using the bot number +// there is no need to change the body model - we're using the default +Avatar.faceModelURL = "https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/" + newFaceFilePrefix + ".fst"; +Avatar.skeletonModelURL = "https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/" + newBodyFilePrefix + ".fst"; +Avatar.billboardURL = "https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/billboards/bot" + botNumber + ".png"; + +Agent.isAvatar = true; +Agent.isListeningToAudioStream = true; + +// change the avatar's position to the random one +Avatar.position = { x: getRandomFloat(X_MIN, X_MAX), y: Y_PELVIS, z: getRandomFloat(Z_MIN, Z_MAX) };; +printVector("New bot, position = ", Avatar.position); + +var allBlendShapes = []; +var targetBlendCoefficient = []; +var currentBlendCoefficient = []; + +function addBlendShape(s) { + allBlendShapes[allBlendShapes.length] = s; +} + +//It is imperative that the following blendshapes are all present and are in the correct order +addBlendShape("EyeBlink_L"); +addBlendShape("EyeBlink_R"); +addBlendShape("EyeSquint_L"); +addBlendShape("EyeSquint_R"); +addBlendShape("EyeDown_L"); +addBlendShape("EyeDown_R"); +addBlendShape("EyeIn_L"); +addBlendShape("EyeIn_R"); +addBlendShape("EyeOpen_L"); +addBlendShape("EyeOpen_R"); +addBlendShape("EyeOut_L"); +addBlendShape("EyeOut_R"); +addBlendShape("EyeUp_L"); +addBlendShape("EyeUp_R"); +addBlendShape("BrowsD_L"); +addBlendShape("BrowsD_R"); +addBlendShape("BrowsU_C"); +addBlendShape("BrowsU_L"); +addBlendShape("BrowsU_R"); +addBlendShape("JawFwd"); +addBlendShape("JawLeft"); +addBlendShape("JawOpen"); +addBlendShape("JawChew"); +addBlendShape("JawRight"); +addBlendShape("MouthLeft"); +addBlendShape("MouthRight"); +addBlendShape("MouthFrown_L"); +addBlendShape("MouthFrown_R"); +addBlendShape("MouthSmile_L"); +addBlendShape("MouthSmile_R"); +addBlendShape("MouthDimple_L"); +addBlendShape("MouthDimple_R"); +addBlendShape("LipsStretch_L"); +addBlendShape("LipsStretch_R"); +addBlendShape("LipsUpperClose"); +addBlendShape("LipsLowerClose"); +addBlendShape("LipsUpperUp"); +addBlendShape("LipsLowerDown"); +addBlendShape("LipsUpperOpen"); +addBlendShape("LipsLowerOpen"); +addBlendShape("LipsFunnel"); +addBlendShape("LipsPucker"); +addBlendShape("ChinLowerRaise"); +addBlendShape("ChinUpperRaise"); +addBlendShape("Sneer"); +addBlendShape("Puff"); +addBlendShape("CheekSquint_L"); +addBlendShape("CheekSquint_R"); + +for (var i = 0; i < allBlendShapes.length; i++) { + targetBlendCoefficient[i] = 0; + currentBlendCoefficient[i] = 0; +} + +function setRandomExpression() { + for (var i = 0; i < allBlendShapes.length; i++) { + targetBlendCoefficient[i] = Math.random(); + } +} + +var expressionChangeSpeed = 0.1; + +function updateBlendShapes(deltaTime) { + + for (var i = 0; i < allBlendShapes.length; i++) { + currentBlendCoefficient[i] += (targetBlendCoefficient[i] - currentBlendCoefficient[i]) * expressionChangeSpeed; + Avatar.setBlendshape(allBlendShapes[i], currentBlendCoefficient[i]); + } +} + +function update(deltaTime) { + timePassed += deltaTime; + if (timePassed > updateSpeed) { + timePassed = 0; + setRandomExpression(); + } + updateBlendShapes(deltaTime); +} + +Script.update.connect(update); \ No newline at end of file From 00fdc3782233a9746ea2bdb12f432db1158c315f Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Wed, 23 Jul 2014 22:45:50 -0700 Subject: [PATCH 39/46] add velocity filtering to eyePosition to reduce camera jitter from faceshift --- interface/src/Application.cpp | 2 +- interface/src/avatar/Head.cpp | 4 ++++ interface/src/avatar/Head.h | 4 +++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 8b00bface7..1dc2c15031 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -592,7 +592,7 @@ void Application::paintGL() { if (_myCamera.getMode() == CAMERA_MODE_FIRST_PERSON) { _myCamera.setTightness(0.0f); // In first person, camera follows (untweaked) head exactly without delay - _myCamera.setTargetPosition(_myAvatar->getHead()->calculateAverageEyePosition()); + _myCamera.setTargetPosition(_myAvatar->getHead()->getFilteredEyePosition()); _myCamera.setTargetRotation(_myAvatar->getHead()->getCameraOrientation()); } else if (_myCamera.getMode() == CAMERA_MODE_THIRD_PERSON) { diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index ee242d179a..d3d1e74fc8 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -159,6 +159,10 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { } } _eyePosition = calculateAverageEyePosition(); + + float velocityFilter = glm::clamp(1.0f - glm::length(_filteredEyePosition - _eyePosition), 0.0f, 1.0f); + _filteredEyePosition = velocityFilter * _filteredEyePosition + (1.0f - velocityFilter) * _eyePosition; + } void Head::relaxLean(float deltaTime) { diff --git a/interface/src/avatar/Head.h b/interface/src/avatar/Head.h index 36df51fa6f..63667914ac 100644 --- a/interface/src/avatar/Head.h +++ b/interface/src/avatar/Head.h @@ -89,7 +89,7 @@ public: const bool getReturnToCenter() const { return _returnHeadToCenter; } // Do you want head to try to return to center (depends on interface detected) float getAverageLoudness() const { return _averageLoudness; } glm::vec3 calculateAverageEyePosition() const { return _leftEyePosition + (_rightEyePosition - _leftEyePosition ) * ONE_HALF; } - + glm::vec3 getFilteredEyePosition() const { return _filteredEyePosition; } /// \return the point about which scaling occurs. glm::vec3 getScalePivot() const; @@ -120,6 +120,8 @@ private: glm::vec3 _leftEyePosition; glm::vec3 _rightEyePosition; glm::vec3 _eyePosition; + glm::vec3 _filteredEyePosition; // velocity filtered world space eye position + float _scale; float _lastLoudness; float _audioAttack; From 5aecbe690dcf8ac7c06c87780ff456048041077e Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Wed, 23 Jul 2014 22:56:59 -0700 Subject: [PATCH 40/46] made calculateAverageEyePosition private, changed to getEyePosition() where used outside head --- interface/src/Application.cpp | 14 +++++++------- interface/src/avatar/Head.h | 3 ++- interface/src/avatar/MyAvatar.cpp | 2 +- interface/src/ui/ApplicationOverlay.cpp | 10 +++++----- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 1dc2c15031..aa362b790b 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -611,10 +611,10 @@ void Application::paintGL() { if (OculusManager::isConnected()) { _myCamera.setDistance(MIRROR_FULLSCREEN_DISTANCE * _scaleMirror); _myCamera.setTargetRotation(_myAvatar->getWorldAlignedOrientation() * glm::quat(glm::vec3(0.0f, PI + _rotateMirror, 0.0f))); - _myCamera.setTargetPosition(_myAvatar->getHead()->calculateAverageEyePosition() + glm::vec3(0, _raiseMirror * _myAvatar->getScale(), 0)); + _myCamera.setTargetPosition(_myAvatar->getHead()->getEyePosition() + glm::vec3(0, _raiseMirror * _myAvatar->getScale(), 0)); } else { _myCamera.setTightness(0.0f); - glm::vec3 eyePosition = _myAvatar->getHead()->calculateAverageEyePosition(); + glm::vec3 eyePosition = _myAvatar->getHead()->getFilteredEyePosition(); float headHeight = eyePosition.y - _myAvatar->getPosition().y; _myCamera.setDistance(MIRROR_FULLSCREEN_DISTANCE * _scaleMirror); _myCamera.setTargetPosition(_myAvatar->getPosition() + glm::vec3(0, headHeight + (_raiseMirror * _myAvatar->getScale()), 0)); @@ -1910,14 +1910,14 @@ void Application::updateMyAvatarLookAtPosition() { } } else { // I am not looking at anyone else, so just look forward - lookAtSpot = _myAvatar->getHead()->calculateAverageEyePosition() + + lookAtSpot = _myAvatar->getHead()->getEyePosition() + (_myAvatar->getHead()->getFinalOrientationInWorldFrame() * glm::vec3(0.f, 0.f, -TREE_SCALE)); } // TODO: Add saccade to mouse pointer when stable, IF not looking at someone (since we know we are looking at it) /* const float FIXED_MIN_EYE_DISTANCE = 0.3f; float minEyeDistance = FIXED_MIN_EYE_DISTANCE + (_myCamera.getMode() == CAMERA_MODE_FIRST_PERSON ? 0.0f : - glm::distance(_mouseRayOrigin, _myAvatar->getHead()->calculateAverageEyePosition())); + glm::distance(_mouseRayOrigin, _myAvatar->getHead()->getAverageEyePosition())); lookAtSpot = _mouseRayOrigin + _mouseRayDirection * qMax(minEyeDistance, distance); */ @@ -1930,7 +1930,7 @@ void Application::updateMyAvatarLookAtPosition() { float eyeYaw = tracker->getEstimatedEyeYaw(); const float GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT = 0.1f; // deflect using Faceshift gaze data - glm::vec3 origin = _myAvatar->getHead()->calculateAverageEyePosition(); + glm::vec3 origin = _myAvatar->getHead()->getEyePosition(); float pitchSign = (_myCamera.getMode() == CAMERA_MODE_MIRROR) ? -1.0f : 1.0f; float deflection = Menu::getInstance()->getFaceshiftEyeDeflection(); if (isLookingAtSomeone) { @@ -2934,7 +2934,7 @@ void Application::renderRearViewMirror(const QRect& region, bool billboard) { _mirrorCamera.setTargetPosition(glm::vec3()); } else { - _mirrorCamera.setTargetPosition(_myAvatar->getHead()->calculateAverageEyePosition()); + _mirrorCamera.setTargetPosition(_myAvatar->getHead()->getEyePosition()); } } _mirrorCamera.setAspectRatio((float)region.width() / region.height()); @@ -2963,7 +2963,7 @@ void Application::renderRearViewMirror(const QRect& region, bool billboard) { _myAvatar->getSkeletonModel().getNeckPosition(neckPosition); // get the eye position relative to the body - glm::vec3 eyePosition = _myAvatar->getHead()->calculateAverageEyePosition(); + glm::vec3 eyePosition = _myAvatar->getHead()->getEyePosition(); float eyeHeight = eyePosition.y - _myAvatar->getPosition().y; // set the translation of the face relative to the neck position diff --git a/interface/src/avatar/Head.h b/interface/src/avatar/Head.h index 63667914ac..6d1e82b97f 100644 --- a/interface/src/avatar/Head.h +++ b/interface/src/avatar/Head.h @@ -88,7 +88,6 @@ public: const bool getReturnToCenter() const { return _returnHeadToCenter; } // Do you want head to try to return to center (depends on interface detected) float getAverageLoudness() const { return _averageLoudness; } - glm::vec3 calculateAverageEyePosition() const { return _leftEyePosition + (_rightEyePosition - _leftEyePosition ) * ONE_HALF; } glm::vec3 getFilteredEyePosition() const { return _filteredEyePosition; } /// \return the point about which scaling occurs. glm::vec3 getScalePivot() const; @@ -110,6 +109,8 @@ public: void addLeanDeltas(float sideways, float forward); private: + glm::vec3 calculateAverageEyePosition() const { return _leftEyePosition + (_rightEyePosition - _leftEyePosition ) * ONE_HALF; } + // disallow copies of the Head, copy of owning Avatar is disallowed too Head(const Head&); Head& operator= (const Head&); diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 556f9dfc68..8cf6cd69a3 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -908,7 +908,7 @@ const float RENDER_HEAD_CUTOFF_DISTANCE = 0.50f; bool MyAvatar::shouldRenderHead(const glm::vec3& cameraPosition, RenderMode renderMode) const { const Head* head = getHead(); return (renderMode != NORMAL_RENDER_MODE) || - (glm::length(cameraPosition - head->calculateAverageEyePosition()) > RENDER_HEAD_CUTOFF_DISTANCE * _scale); + (glm::length(cameraPosition - head->getEyePosition()) > RENDER_HEAD_CUTOFF_DISTANCE * _scale); } float MyAvatar::computeDistanceToFloor(const glm::vec3& startPoint) { diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 89ae2fbb46..990e4e5c95 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -178,7 +178,7 @@ void ApplicationOverlay::computeOculusPickRay(float x, float y, glm::vec3& direc float dist = sqrt(x * x + y * y); float z = -sqrt(1.0f - dist * dist); - glm::vec3 relativePosition = myAvatar->getHead()->calculateAverageEyePosition() + + glm::vec3 relativePosition = myAvatar->getHead()->getEyePosition() + glm::normalize(myAvatar->getOrientation() * glm::vec3(x, y, z)); //Rotate the UI pick ray by the avatar orientation @@ -274,7 +274,7 @@ QPoint ApplicationOverlay::getPalmClickLocation(const PalmData *palm) const { MyAvatar* myAvatar = application->getAvatar(); glm::vec3 tip = myAvatar->getLaserPointerTipPosition(palm); - glm::vec3 eyePos = myAvatar->getHead()->calculateAverageEyePosition(); + glm::vec3 eyePos = myAvatar->getHead()->getEyePosition(); glm::quat orientation = glm::inverse(myAvatar->getOrientation()); glm::vec3 dir = orientation * glm::normalize(application->getCamera()->getPosition() - tip); //direction of ray goes towards camera glm::vec3 tipPos = orientation * (tip - eyePos); @@ -331,7 +331,7 @@ bool ApplicationOverlay::calculateRayUICollisionPoint(const glm::vec3& position, glm::quat orientation = myAvatar->getOrientation(); - glm::vec3 relativePosition = orientation * (position - myAvatar->getHead()->calculateAverageEyePosition()); + glm::vec3 relativePosition = orientation * (position - myAvatar->getHead()->getEyePosition()); glm::vec3 relativeDirection = orientation * direction; float t; @@ -375,7 +375,7 @@ void ApplicationOverlay::displayOverlayTextureOculus(Camera& whichCamera) { glPushMatrix(); const glm::quat& orientation = myAvatar->getOrientation(); - const glm::vec3& position = myAvatar->getHead()->calculateAverageEyePosition(); + const glm::vec3& position = myAvatar->getHead()->getEyePosition(); glm::mat4 rotation = glm::toMat4(orientation); @@ -1210,7 +1210,7 @@ void ApplicationOverlay::renderTexturedHemisphere() { Application* application = Application::getInstance(); MyAvatar* myAvatar = application->getAvatar(); const glm::quat& orientation = myAvatar->getOrientation(); - const glm::vec3& position = myAvatar->getHead()->calculateAverageEyePosition(); + const glm::vec3& position = myAvatar->getHead()->getEyePosition(); glm::mat4 rotation = glm::toMat4(orientation); From a1b74c5272b8faaff31cb361a606226a1a437032 Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Wed, 23 Jul 2014 23:07:37 -0700 Subject: [PATCH 41/46] remove unused code --- interface/src/Application.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index aa362b790b..d2a91c9480 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1913,14 +1913,6 @@ void Application::updateMyAvatarLookAtPosition() { lookAtSpot = _myAvatar->getHead()->getEyePosition() + (_myAvatar->getHead()->getFinalOrientationInWorldFrame() * glm::vec3(0.f, 0.f, -TREE_SCALE)); } - // TODO: Add saccade to mouse pointer when stable, IF not looking at someone (since we know we are looking at it) - /* - const float FIXED_MIN_EYE_DISTANCE = 0.3f; - float minEyeDistance = FIXED_MIN_EYE_DISTANCE + (_myCamera.getMode() == CAMERA_MODE_FIRST_PERSON ? 0.0f : - glm::distance(_mouseRayOrigin, _myAvatar->getHead()->getAverageEyePosition())); - lookAtSpot = _mouseRayOrigin + _mouseRayDirection * qMax(minEyeDistance, distance); - */ - } // // Deflect the eyes a bit to match the detected Gaze from 3D camera if active From 2aaa628a6727096c6f1f9a406bd98504caeaca74 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Thu, 24 Jul 2014 08:42:34 -0700 Subject: [PATCH 42/46] send visible joint state when colliding as ragdoll --- interface/src/avatar/MyAvatar.cpp | 13 ++++++++++--- interface/src/renderer/JointState.cpp | 8 ++++++++ interface/src/renderer/JointState.h | 3 +++ interface/src/renderer/Model.cpp | 18 ++++++++++++------ interface/src/renderer/Model.h | 4 ++++ 5 files changed, 37 insertions(+), 9 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 556f9dfc68..37558adbc7 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -165,9 +165,16 @@ void MyAvatar::simulate(float deltaTime) { PerformanceTimer perfTimer("joints"); // copy out the skeleton joints from the model _jointData.resize(_skeletonModel.getJointStateCount()); - for (int i = 0; i < _jointData.size(); i++) { - JointData& data = _jointData[i]; - data.valid = _skeletonModel.getJointState(i, data.rotation); + if (Menu::getInstance()->isOptionChecked(MenuOption::CollideAsRagdoll)) { + for (int i = 0; i < _jointData.size(); i++) { + JointData& data = _jointData[i]; + data.valid = _skeletonModel.getVisibleJointState(i, data.rotation); + } + } else { + for (int i = 0; i < _jointData.size(); i++) { + JointData& data = _jointData[i]; + data.valid = _skeletonModel.getJointState(i, data.rotation); + } } } diff --git a/interface/src/renderer/JointState.cpp b/interface/src/renderer/JointState.cpp index 26fa29c4a9..2a4372401e 100644 --- a/interface/src/renderer/JointState.cpp +++ b/interface/src/renderer/JointState.cpp @@ -218,6 +218,14 @@ void JointState::setVisibleRotationInConstrainedFrame(const glm::quat& targetRot _visibleRotation = parentRotation * _fbxJoint->preRotation * _visibleRotationInConstrainedFrame * _fbxJoint->postRotation; } +const bool JointState::rotationIsDefault(const glm::quat& rotation, float tolerance) const { + glm::quat defaultRotation = _fbxJoint->rotation; + return glm::abs(rotation.x - defaultRotation.x) < tolerance && + glm::abs(rotation.y - defaultRotation.y) < tolerance && + glm::abs(rotation.z - defaultRotation.z) < tolerance && + glm::abs(rotation.w - defaultRotation.w) < tolerance; +} + const glm::vec3& JointState::getDefaultTranslationInConstrainedFrame() const { assert(_fbxJoint != NULL); return _fbxJoint->translation; diff --git a/interface/src/renderer/JointState.h b/interface/src/renderer/JointState.h index a3b792abc4..94811fe13c 100644 --- a/interface/src/renderer/JointState.h +++ b/interface/src/renderer/JointState.h @@ -82,6 +82,9 @@ public: void setRotationInConstrainedFrame(const glm::quat& targetRotation); void setVisibleRotationInConstrainedFrame(const glm::quat& targetRotation); const glm::quat& getRotationInConstrainedFrame() const { return _rotationInConstrainedFrame; } + const glm::quat& getVisibleRotationInConstrainedFrame() const { return _visibleRotationInConstrainedFrame; } + + const bool rotationIsDefault(const glm::quat& rotation, float tolerance = EPSILON) const; const glm::vec3& getDefaultTranslationInConstrainedFrame() const; diff --git a/interface/src/renderer/Model.cpp b/interface/src/renderer/Model.cpp index 8fac5d3d03..8c19c11ed3 100644 --- a/interface/src/renderer/Model.cpp +++ b/interface/src/renderer/Model.cpp @@ -676,12 +676,18 @@ bool Model::getJointState(int index, glm::quat& rotation) const { if (index == -1 || index >= _jointStates.size()) { return false; } - rotation = _jointStates.at(index).getRotationInConstrainedFrame(); - const glm::quat& defaultRotation = _geometry->getFBXGeometry().joints.at(index).rotation; - return glm::abs(rotation.x - defaultRotation.x) >= EPSILON || - glm::abs(rotation.y - defaultRotation.y) >= EPSILON || - glm::abs(rotation.z - defaultRotation.z) >= EPSILON || - glm::abs(rotation.w - defaultRotation.w) >= EPSILON; + const JointState& state = _jointStates.at(index); + rotation = state.getRotationInConstrainedFrame(); + return !state.rotationIsDefault(rotation); +} + +bool Model::getVisibleJointState(int index, glm::quat& rotation) const { + if (index == -1 || index >= _jointStates.size()) { + return false; + } + const JointState& state = _jointStates.at(index); + rotation = state.getVisibleRotationInConstrainedFrame(); + return !state.rotationIsDefault(rotation); } void Model::setJointState(int index, bool valid, const glm::quat& rotation, float priority) { diff --git a/interface/src/renderer/Model.h b/interface/src/renderer/Model.h index a4eae8fd9a..cbed941791 100644 --- a/interface/src/renderer/Model.h +++ b/interface/src/renderer/Model.h @@ -113,6 +113,10 @@ public: /// Fetches the joint state at the specified index. /// \return whether or not the joint state is "valid" (that is, non-default) bool getJointState(int index, glm::quat& rotation) const; + + /// Fetches the visible joint state at the specified index. + /// \return whether or not the joint state is "valid" (that is, non-default) + bool getVisibleJointState(int index, glm::quat& rotation) const; /// Sets the joint state at the specified index. void setJointState(int index, bool valid, const glm::quat& rotation = glm::quat(), float priority = 1.0f); From 1d124ebc61e3df0942d81a25cabf58518a9dad6d Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Thu, 24 Jul 2014 08:49:41 -0700 Subject: [PATCH 43/46] fix comments --- interface/src/avatar/SkeletonModel.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 1a68b86a14..b81949d8c3 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -680,16 +680,17 @@ void SkeletonModel::moveShapesTowardJoints(float deltaTime) { return; } + // fraction = 0 means keep old position, = 1 means slave 100% to target position const float RAGDOLL_FOLLOWS_JOINTS_TIMESCALE = 0.05f; float fraction = glm::clamp(deltaTime / RAGDOLL_FOLLOWS_JOINTS_TIMESCALE, 0.0f, 1.0f); - // SIMPLE LINEAR_SLAVING -- KEEP this implementation for reference + // SIMPLE LINEAR SLAVING -- KEEP this implementation for reference //float oneMinusFraction = 1.0f - fraction; //for (int i = 0; i < numStates; ++i) { // _ragdollPoints[i]._lastPosition = _ragdollPoints[i]._position; // _ragdollPoints[i]._position = oneMinusFraction * _ragdollPoints[i]._position + fraction * _jointStates.at(i).getPosition(); //} - // LINEAR_SLAVING -- KEEP + // SIMPLE LINEAR SLAVING -- KEEP // parent-relative linear slaving for (int i = 0; i < numStates; ++i) { @@ -706,17 +707,17 @@ void SkeletonModel::moveShapesTowardJoints(float deltaTime) { continue; } + glm::vec3 bone = _ragdollPoints.at(i)._lastPosition - _ragdollPoints.at(p)._lastPosition; const JointState& parentState = _jointStates.at(p); glm::vec3 targetBone = state.getPosition() - parentState.getPosition(); - glm::vec3 bone = _ragdollPoints.at(i)._lastPosition - _ragdollPoints.at(p)._lastPosition; glm::vec3 newBone = (1.0f - fraction) * bone + fraction * targetBone; float boneLength = glm::length(newBone); if (boneLength > EPSILON) { - // slam newBone's lenght to that of the joint, which helps maintain distance constraints + // slam newBone's length to that of the joint helps maintain distance constraints newBone *= state.getDistanceToParent() / boneLength; } - // we set the new position relative to parent + // set the new position relative to parent's new position _ragdollPoints[i]._position = _ragdollPoints.at(p)._position + newBone; } } From 9ac8b732f09d377b9b26e6811b55fb0a45db6b0e Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 24 Jul 2014 10:32:34 -0700 Subject: [PATCH 44/46] FIxed error on ubuntu --- libraries/avatars/src/HeadData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 31001bf162..b29277ddeb 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -65,7 +65,7 @@ void HeadData::setBlendshape(QString name, float val) { } //Check to see if the named blendshape exists, and then set its value if it does - auto it = blendshapeLookupMap.find(name); + QMap::iterator it = blendshapeLookupMap.find(name); if (it != blendshapeLookupMap.end()) { if (_blendshapeCoefficients.size() <= it.value()) { _blendshapeCoefficients.resize(it.value() + 1); From 9ca00b55a54d5093040b1da80c39b20c5a2e809c Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 24 Jul 2014 17:19:49 -0700 Subject: [PATCH 45/46] Spelling --- interface/src/avatar/Avatar.cpp | 2 +- interface/src/avatar/MyAvatar.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index 4dd82a4652..33f85ad11a 100644 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -240,7 +240,7 @@ void Avatar::render(const glm::vec3& cameraPosition, RenderMode renderMode) { glEnd(); } glPopMatrix(); } - if (_handState == HAND_STATE_RIGTH_POINTING || + if (_handState == HAND_STATE_RIGHT_POINTING || _handState == HAND_STATE_BOTH_POINTING) { int rightIndex = _skeletonModel.getRightHandJointIndex(); glm::vec3 rightPosition; diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 7020222056..581044c522 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -22,7 +22,7 @@ enum AvatarHandState { HAND_STATE_NULL = 0, HAND_STATE_LEFT_POINTING, - HAND_STATE_RIGTH_POINTING, + HAND_STATE_RIGHT_POINTING, HAND_STATE_BOTH_POINTING, NUM_HAND_STATES }; From ce1b8cdafc3e313b4f73629ab80399d51d96c9b0 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 24 Jul 2014 17:32:41 -0700 Subject: [PATCH 46/46] Multicursor editvoxels support --- examples/editVoxels.js | 121 +++++++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 52 deletions(-) diff --git a/examples/editVoxels.js b/examples/editVoxels.js index 8d2a516f55..306156cc18 100644 --- a/examples/editVoxels.js +++ b/examples/editVoxels.js @@ -51,7 +51,6 @@ var lastVoxelScale = 0; var dragStart = { x: 0, y: 0 }; var wheelPixelsMoved = 0; - var mouseX = 0; var mouseY = 0; @@ -168,7 +167,16 @@ var voxelPreview = Overlays.addOverlay("cube", { lineWidth: 4 }); -var linePreviewTop = Overlays.addOverlay("line3d", { +var linePreviewTop = []; +var linePreviewBottom = []; +var linePreviewLeft = []; +var linePreviewRight = []; + +// Currend cursor index +var currentCursor = 0; + +function addLineOverlay() { + return Overlays.addOverlay("line3d", { position: { x: 0, y: 0, z: 0}, end: { x: 0, y: 0, z: 0}, color: { red: 255, green: 255, blue: 255}, @@ -176,34 +184,24 @@ var linePreviewTop = Overlays.addOverlay("line3d", { visible: false, lineWidth: previewLineWidth }); +} + +//Cursor line previews for up to three cursors +linePreviewTop[0] = addLineOverlay(); +linePreviewTop[1] = addLineOverlay(); +linePreviewTop[2] = addLineOverlay(); -var linePreviewBottom = Overlays.addOverlay("line3d", { - position: { x: 0, y: 0, z: 0}, - end: { x: 0, y: 0, z: 0}, - color: { red: 255, green: 255, blue: 255}, - alpha: 1, - visible: false, - lineWidth: previewLineWidth - }); - -var linePreviewLeft = Overlays.addOverlay("line3d", { - position: { x: 0, y: 0, z: 0}, - end: { x: 0, y: 0, z: 0}, - color: { red: 255, green: 255, blue: 255}, - alpha: 1, - visible: false, - lineWidth: previewLineWidth - }); - -var linePreviewRight = Overlays.addOverlay("line3d", { - position: { x: 0, y: 0, z: 0}, - end: { x: 0, y: 0, z: 0}, - color: { red: 255, green: 255, blue: 255}, - alpha: 1, - visible: false, - lineWidth: previewLineWidth - }); - +linePreviewBottom[0] = addLineOverlay(); +linePreviewBottom[1] = addLineOverlay(); +linePreviewBottom[2] = addLineOverlay(); + +linePreviewLeft[0] = addLineOverlay(); +linePreviewLeft[1] = addLineOverlay(); +linePreviewLeft[2] = addLineOverlay(); + +linePreviewRight[0] = addLineOverlay(); +linePreviewRight[1] = addLineOverlay(); +linePreviewRight[2] = addLineOverlay(); // these will be used below var scaleSelectorWidth = 144; @@ -809,21 +807,21 @@ function showPreviewLines() { var pasteVoxel = getNewPasteVoxel(pickRay); // X axis - Overlays.editOverlay(linePreviewBottom, { + Overlays.editOverlay(linePreviewBottom[currentCursor], { position: pasteVoxel.origin, end: {x: pasteVoxel.origin.x + pasteVoxel.voxelSize, y: pasteVoxel.origin.y, z: pasteVoxel.origin.z }, visible: true }); // Y axis - Overlays.editOverlay(linePreviewRight, { + Overlays.editOverlay(linePreviewRight[currentCursor], { position: pasteVoxel.origin, end: {x: pasteVoxel.origin.x, y: pasteVoxel.origin.y + pasteVoxel.voxelSize, z: pasteVoxel.origin.z }, visible: true }); // Z axis - Overlays.editOverlay(linePreviewTop, { + Overlays.editOverlay(linePreviewTop[currentCursor], { position: pasteVoxel.origin, end: {x: pasteVoxel.origin.x, y: pasteVoxel.origin.y, z: pasteVoxel.origin.z - pasteVoxel.voxelSize }, visible: true @@ -837,10 +835,10 @@ function showPreviewLines() { if (intersection.intersects) { resultVoxel = calculateVoxelFromIntersection(intersection,""); Overlays.editOverlay(voxelPreview, { visible: false }); - Overlays.editOverlay(linePreviewTop, { position: resultVoxel.topLeft, end: resultVoxel.topRight, visible: true }); - Overlays.editOverlay(linePreviewBottom, { position: resultVoxel.bottomLeft, end: resultVoxel.bottomRight, visible: true }); - Overlays.editOverlay(linePreviewLeft, { position: resultVoxel.topLeft, end: resultVoxel.bottomLeft, visible: true }); - Overlays.editOverlay(linePreviewRight, { position: resultVoxel.topRight, end: resultVoxel.bottomRight, visible: true }); + Overlays.editOverlay(linePreviewTop[currentCursor], { position: resultVoxel.topLeft, end: resultVoxel.topRight, visible: true }); + Overlays.editOverlay(linePreviewBottom[currentCursor], { position: resultVoxel.bottomLeft, end: resultVoxel.bottomRight, visible: true }); + Overlays.editOverlay(linePreviewLeft[currentCursor], { position: resultVoxel.topLeft, end: resultVoxel.bottomLeft, visible: true }); + Overlays.editOverlay(linePreviewRight[currentCursor], { position: resultVoxel.topRight, end: resultVoxel.bottomRight, visible: true }); colors[0] = {red: intersection.voxel.red, green: intersection.voxel.green , blue: intersection.voxel.blue }; if (copyScale) { @@ -849,10 +847,10 @@ function showPreviewLines() { moveTools(); } else { Overlays.editOverlay(voxelPreview, { visible: false }); - Overlays.editOverlay(linePreviewTop, { visible: false }); - Overlays.editOverlay(linePreviewBottom, { visible: false }); - Overlays.editOverlay(linePreviewLeft, { visible: false }); - Overlays.editOverlay(linePreviewRight, { visible: false }); + Overlays.editOverlay(linePreviewTop[currentCursor], { visible: false }); + Overlays.editOverlay(linePreviewBottom[currentCursor], { visible: false }); + Overlays.editOverlay(linePreviewLeft[currentCursor], { visible: false }); + Overlays.editOverlay(linePreviewRight[currentCursor], { visible: false }); } } @@ -862,20 +860,20 @@ function showPreviewGuides() { showPreviewVoxel(); // make sure alternative is hidden - Overlays.editOverlay(linePreviewTop, { visible: false }); - Overlays.editOverlay(linePreviewBottom, { visible: false }); - Overlays.editOverlay(linePreviewLeft, { visible: false }); - Overlays.editOverlay(linePreviewRight, { visible: false }); + Overlays.editOverlay(linePreviewTop[currentCursor], { visible: false }); + Overlays.editOverlay(linePreviewBottom[currentCursor], { visible: false }); + Overlays.editOverlay(linePreviewLeft[currentCursor], { visible: false }); + Overlays.editOverlay(linePreviewRight[currentCursor], { visible: false }); } else { showPreviewLines(); } } else { // make sure all previews are off Overlays.editOverlay(voxelPreview, { visible: false }); - Overlays.editOverlay(linePreviewTop, { visible: false }); - Overlays.editOverlay(linePreviewBottom, { visible: false }); - Overlays.editOverlay(linePreviewLeft, { visible: false }); - Overlays.editOverlay(linePreviewRight, { visible: false }); + Overlays.editOverlay(linePreviewTop[currentCursor], { visible: false }); + Overlays.editOverlay(linePreviewBottom[currentCursor], { visible: false }); + Overlays.editOverlay(linePreviewLeft[currentCursor], { visible: false }); + Overlays.editOverlay(linePreviewRight[currentCursor], { visible: false }); } } @@ -968,6 +966,14 @@ function mousePressEvent(event) { return; } + if (event.deviceID == 1500) { // Left Hydra Controller + currentCursor = 0; + } else if (event.deviceID == 1501) { // Right Hydra Controller + currentCursor = 1; + } else { + currentCursor = 2; + } + var clickedOnSomething = false; var clickedOverlay = Overlays.getOverlayAtPoint({x: event.x, y: event.y}); @@ -1220,6 +1226,7 @@ function menuItemEvent(menuItem) { } function mouseMoveEvent(event) { + if (!editToolsOn) { return; } @@ -1227,6 +1234,14 @@ function mouseMoveEvent(event) { return; } + if (event.deviceID == 1500) { // Left Hydra Controller + currentCursor = 0; + } else if (event.deviceID == 1501) { // Right Hydra Controller + currentCursor = 1; + } else { + currentCursor = 2; + } + // Move Import Preview if (isImporting) { var pickRay = Camera.computePickRay(event.x, event.y); @@ -1475,10 +1490,12 @@ Controller.captureKeyEvents({ text: "-" }); function scriptEnding() { Overlays.deleteOverlay(voxelPreview); - Overlays.deleteOverlay(linePreviewTop); - Overlays.deleteOverlay(linePreviewBottom); - Overlays.deleteOverlay(linePreviewLeft); - Overlays.deleteOverlay(linePreviewRight); + for (var i = 0; i < linePreviewTop.length; i++) { + Overlays.deleteOverlay(linePreviewTop[i]); + Overlays.deleteOverlay(linePreviewBottom[i]); + Overlays.deleteOverlay(linePreviewLeft[i]); + Overlays.deleteOverlay(linePreviewRight[i]); + } for (s = 0; s < numColors; s++) { Overlays.deleteOverlay(swatches[s]); }