From da0911e01b8a88b630513788601cee15650c31be Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 8 Oct 2019 16:08:29 -0700 Subject: [PATCH] Delete legacy blendshape code Move FaceshiftConstants to BlendshapeConstants. Delete FaceTracker and DdeFaceTracker classes. Delete old facetracker menu and preferences --- interface/CMakeLists.txt | 4 +- interface/src/Application.cpp | 113 +-- interface/src/Application.h | 9 - interface/src/Menu.cpp | 42 -- interface/src/avatar/MyAvatar.cpp | 73 +- interface/src/avatar/MyAvatar.h | 4 +- interface/src/avatar/MyHead.cpp | 4 +- interface/src/devices/DdeFaceTracker.cpp | 686 ------------------ interface/src/devices/DdeFaceTracker.h | 181 ----- interface/src/ui/AvatarInputs.cpp | 10 - interface/src/ui/AvatarInputs.h | 33 - interface/src/ui/PreferencesDialog.cpp | 17 - libraries/avatars-renderer/CMakeLists.txt | 2 +- .../src/avatars-renderer/Head.cpp | 19 +- libraries/avatars/src/HeadData.cpp | 2 +- libraries/avatars/src/HeadData.h | 2 +- libraries/fbx/src/FBXSerializer.cpp | 2 +- libraries/fbx/src/GLTFSerializer.cpp | 2 +- ...tConstants.cpp => BlendshapeConstants.cpp} | 14 +- libraries/shared/src/BlendshapeConstants.h | 76 ++ libraries/shared/src/FaceshiftConstants.h | 25 - libraries/trackers/CMakeLists.txt | 7 - .../trackers/src/trackers/FaceTracker.cpp | 133 ---- libraries/trackers/src/trackers/FaceTracker.h | 131 ---- libraries/trackers/src/trackers/Logging.cpp | 11 - libraries/trackers/src/trackers/Logging.h | 16 - tools/jsdoc/plugins/hifi.js | 2 - 27 files changed, 110 insertions(+), 1510 deletions(-) delete mode 100644 interface/src/devices/DdeFaceTracker.cpp delete mode 100644 interface/src/devices/DdeFaceTracker.h rename libraries/shared/src/{FaceshiftConstants.cpp => BlendshapeConstants.cpp} (84%) create mode 100644 libraries/shared/src/BlendshapeConstants.h delete mode 100644 libraries/shared/src/FaceshiftConstants.h delete mode 100644 libraries/trackers/CMakeLists.txt delete mode 100644 libraries/trackers/src/trackers/FaceTracker.cpp delete mode 100644 libraries/trackers/src/trackers/FaceTracker.h delete mode 100644 libraries/trackers/src/trackers/Logging.cpp delete mode 100644 libraries/trackers/src/trackers/Logging.h diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index bcd3f269e8..9030666609 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -211,10 +211,10 @@ endif() link_hifi_libraries( shared workload task octree ktx gpu gl procedural graphics graphics-scripting render pointers recording hfm fbx networking material-networking - model-networking model-baker entities avatars trackers + model-networking model-baker entities avatars audio audio-client animation script-engine physics render-utils entities-renderer avatars-renderer ui qml auto-updater midi - controllers plugins image trackers platform + controllers plugins image platform ui-plugins display-plugins input-plugins # Platform specific GL libraries ${PLATFORM_GL_BACKEND} diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 92bc54d43f..677137148c 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -170,7 +170,6 @@ #include "avatar/MyCharacterController.h" #include "CrashRecoveryHandler.h" #include "CrashHandler.h" -#include "devices/DdeFaceTracker.h" #include "DiscoverabilityManager.h" #include "GLCanvas.h" #include "InterfaceDynamicFactory.h" @@ -887,11 +886,6 @@ bool setupEssentials(int& argc, char** argv, bool runningMarkerExisted) { DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); - -#ifdef HAVE_DDE - DependencyManager::set(); -#endif - DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); @@ -1067,7 +1061,6 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo _lastSendDownstreamAudioStats(usecTimestampNow()), _notifiedPacketVersionMismatchThisDomain(false), _maxOctreePPS(maxOctreePacketsPerSecond.get()), - _lastFaceTrackerUpdate(0), _snapshotSound(nullptr), _sampleSound(nullptr) { @@ -2014,13 +2007,6 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo this->installEventFilter(this); - -#ifdef HAVE_DDE - auto ddeTracker = DependencyManager::get(); - ddeTracker->init(); - connect(ddeTracker.data(), &FaceTracker::muteToggled, this, &Application::faceTrackerMuteToggled); -#endif - // If launched from Steam, let it handle updates const QString HIFI_NO_UPDATER_COMMAND_LINE_KEY = "--no-updater"; bool noUpdater = arguments().indexOf(HIFI_NO_UPDATER_COMMAND_LINE_KEY) != -1; @@ -2762,9 +2748,6 @@ void Application::cleanupBeforeQuit() { } // Stop third party processes so that they're not left running in the event of a subsequent shutdown crash. -#ifdef HAVE_DDE - DependencyManager::get()->setEnabled(false); -#endif AnimDebugDraw::getInstance().shutdown(); // FIXME: once we move to shared pointer for the INputDevice we shoud remove this naked delete: @@ -2835,10 +2818,6 @@ void Application::cleanupBeforeQuit() { _window->saveGeometry(); // Destroy third party processes after scripts have finished using them. -#ifdef HAVE_DDE - DependencyManager::destroy(); -#endif - DependencyManager::destroy(); // Must be destroyed before TabletScriptingInterface // stop QML @@ -3457,9 +3436,6 @@ void Application::onDesktopRootContextCreated(QQmlContext* surfaceContext) { surfaceContext->setContextProperty("AccountServices", AccountServicesScriptingInterface::getInstance()); surfaceContext->setContextProperty("DialogsManager", _dialogsManagerScriptingInterface); -#ifdef HAVE_DDE - surfaceContext->setContextProperty("FaceTracker", DependencyManager::get().data()); -#endif surfaceContext->setContextProperty("AvatarManager", DependencyManager::get().data()); surfaceContext->setContextProperty("LODManager", DependencyManager::get().data()); surfaceContext->setContextProperty("HMD", DependencyManager::get().data()); @@ -3723,16 +3699,6 @@ void Application::runTests() { runUnitTests(); } -void Application::faceTrackerMuteToggled() { - - QAction* muteAction = Menu::getInstance()->getActionForOption(MenuOption::MuteFaceTracking); - Q_CHECK_PTR(muteAction); - bool isMuted = getSelectedFaceTracker()->isMuted(); - muteAction->setChecked(isMuted); - getSelectedFaceTracker()->setEnabled(!isMuted); - Menu::getInstance()->getActionForOption(MenuOption::CalibrateCamera)->setEnabled(!isMuted); -} - void Application::setFieldOfView(float fov) { if (fov != _fieldOfView.get()) { _fieldOfView.set(fov); @@ -5307,43 +5273,6 @@ ivec2 Application::getMouse() const { return getApplicationCompositor().getReticlePosition(); } -FaceTracker* Application::getActiveFaceTracker() { -#ifdef HAVE_DDE - auto dde = DependencyManager::get(); - - if (dde && dde->isActive()) { - return static_cast(dde.data()); - } -#endif - - return nullptr; -} - -FaceTracker* Application::getSelectedFaceTracker() { - FaceTracker* faceTracker = nullptr; -#ifdef HAVE_DDE - if (Menu::getInstance()->isOptionChecked(MenuOption::UseCamera)) { - faceTracker = DependencyManager::get().data(); - } -#endif - return faceTracker; -} - -void Application::setActiveFaceTracker() const { -#ifdef HAVE_DDE - bool isMuted = Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking); - bool isUsingDDE = Menu::getInstance()->isOptionChecked(MenuOption::UseCamera); - Menu::getInstance()->getActionForOption(MenuOption::BinaryEyelidControl)->setVisible(isUsingDDE); - Menu::getInstance()->getActionForOption(MenuOption::CoupleEyelids)->setVisible(isUsingDDE); - Menu::getInstance()->getActionForOption(MenuOption::UseAudioForMouth)->setVisible(isUsingDDE); - Menu::getInstance()->getActionForOption(MenuOption::VelocityFilter)->setVisible(isUsingDDE); - Menu::getInstance()->getActionForOption(MenuOption::CalibrateCamera)->setVisible(isUsingDDE); - auto ddeTracker = DependencyManager::get(); - ddeTracker->setIsMuted(isMuted); - ddeTracker->setEnabled(isUsingDDE && !isMuted); -#endif -} - bool Application::exportEntities(const QString& filename, const QVector& entityIDs, const glm::vec3* givenOffset) { @@ -5827,8 +5756,7 @@ void Application::updateMyAvatarLookAtPosition() { PerformanceWarning warn(showWarnings, "Application::updateMyAvatarLookAtPosition()"); auto myAvatar = getMyAvatar(); - FaceTracker* faceTracker = getActiveFaceTracker(); - myAvatar->updateLookAtPosition(faceTracker, _myCamera); + myAvatar->updateLookAtPosition(_myCamera); } void Application::updateThreads(float deltaTime) { @@ -6254,37 +6182,6 @@ void Application::update(float deltaTime) { auto myAvatar = getMyAvatar(); { PerformanceTimer perfTimer("devices"); - - FaceTracker* tracker = getSelectedFaceTracker(); - if (tracker && Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking) != tracker->isMuted()) { - tracker->toggleMute(); - } - - tracker = getActiveFaceTracker(); - if (tracker && !tracker->isMuted()) { - tracker->update(deltaTime); - - // Auto-mute microphone after losing face tracking? - if (tracker->isTracking()) { - _lastFaceTrackerUpdate = usecTimestampNow(); - } else { - const quint64 MUTE_MICROPHONE_AFTER_USECS = 5000000; //5 secs - Menu* menu = Menu::getInstance(); - auto audioClient = DependencyManager::get(); - if (menu->isOptionChecked(MenuOption::AutoMuteAudio) && !audioClient->isMuted()) { - if (_lastFaceTrackerUpdate > 0 - && ((usecTimestampNow() - _lastFaceTrackerUpdate) > MUTE_MICROPHONE_AFTER_USECS)) { - audioClient->setMuted(true); - _lastFaceTrackerUpdate = 0; - } - } else { - _lastFaceTrackerUpdate = 0; - } - } - } else { - _lastFaceTrackerUpdate = 0; - } - auto userInputMapper = DependencyManager::get(); controller::HmdAvatarAlignmentType hmdAvatarAlignmentType; @@ -7080,10 +6977,6 @@ void Application::copyDisplayViewFrustum(ViewFrustum& viewOut) const { // feature. However, we still use this to reset face trackers, eye trackers, audio and to optionally re-load the avatar // rig and animations from scratch. void Application::resetSensors(bool andReload) { -#ifdef HAVE_DDE - DependencyManager::get()->reset(); -#endif - _overlayConductor.centerUI(); getActiveDisplayPlugin()->resetSensors(); getMyAvatar()->reset(true, andReload); @@ -7485,10 +7378,6 @@ void Application::registerScriptEngineWithApplicationServices(const ScriptEngine scriptEngine->registerGlobalObject("AccountServices", AccountServicesScriptingInterface::getInstance()); qScriptRegisterMetaType(scriptEngine.data(), DownloadInfoResultToScriptValue, DownloadInfoResultFromScriptValue); -#ifdef HAVE_DDE - scriptEngine->registerGlobalObject("FaceTracker", DependencyManager::get().data()); -#endif - scriptEngine->registerGlobalObject("AvatarManager", DependencyManager::get().data()); scriptEngine->registerGlobalObject("LODManager", DependencyManager::get().data()); diff --git a/interface/src/Application.h b/interface/src/Application.h index e3334d12d6..80b10697b8 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -81,7 +81,6 @@ #include "VisionSqueeze.h" class GLCanvas; -class FaceTracker; class MainWindow; class AssetUpload; class CompositorHelper; @@ -191,9 +190,6 @@ public: ivec2 getMouse() const; - FaceTracker* getActiveFaceTracker(); - FaceTracker* getSelectedFaceTracker(); - ApplicationOverlay& getApplicationOverlay() { return _applicationOverlay; } const ApplicationOverlay& getApplicationOverlay() const { return _applicationOverlay; } CompositorHelper& getApplicationCompositor() const; @@ -423,7 +419,6 @@ public slots: static void packageModel(); void resetSensors(bool andReload = false); - void setActiveFaceTracker() const; void hmdVisibleChanged(bool visible); @@ -497,8 +492,6 @@ private slots: void resettingDomain(); - void faceTrackerMuteToggled(); - void activeChanged(Qt::ApplicationState state); void windowMinimizedChanged(bool minimized); @@ -736,8 +729,6 @@ private: PerformanceManager _performanceManager; RefreshRateManager _refreshRateManager; - quint64 _lastFaceTrackerUpdate; - GameWorkload _gameWorkload; GraphicsEngine _graphicsEngine; diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 5be4db46a0..c0df4ff159 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -37,7 +37,6 @@ #include "avatar/AvatarManager.h" #include "avatar/AvatarPackager.h" #include "AvatarBookmarks.h" -#include "devices/DdeFaceTracker.h" #include "MainWindow.h" #include "render/DrawStatus.h" #include "scripting/MenuScriptingInterface.h" @@ -493,47 +492,6 @@ Menu::Menu() { // Developer > Avatar >>> MenuWrapper* avatarDebugMenu = developerMenu->addMenu("Avatar"); - // Developer > Avatar > Face Tracking - MenuWrapper* faceTrackingMenu = avatarDebugMenu->addMenu("Face Tracking"); - { - QActionGroup* faceTrackerGroup = new QActionGroup(avatarDebugMenu); - - bool defaultNoFaceTracking = true; -#ifdef HAVE_DDE - defaultNoFaceTracking = false; -#endif - QAction* noFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::NoFaceTracking, - 0, defaultNoFaceTracking, - qApp, SLOT(setActiveFaceTracker())); - faceTrackerGroup->addAction(noFaceTracker); - -#ifdef HAVE_DDE - QAction* ddeFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::UseCamera, - 0, true, - qApp, SLOT(setActiveFaceTracker())); - faceTrackerGroup->addAction(ddeFaceTracker); -#endif - } -#ifdef HAVE_DDE - faceTrackingMenu->addSeparator(); - QAction* binaryEyelidControl = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::BinaryEyelidControl, 0, true); - binaryEyelidControl->setVisible(true); // DDE face tracking is on by default - QAction* coupleEyelids = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::CoupleEyelids, 0, true); - coupleEyelids->setVisible(true); // DDE face tracking is on by default - QAction* useAudioForMouth = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::UseAudioForMouth, 0, true); - useAudioForMouth->setVisible(true); // DDE face tracking is on by default - QAction* ddeFiltering = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::VelocityFilter, 0, true); - ddeFiltering->setVisible(true); // DDE face tracking is on by default - QAction* ddeCalibrate = addActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::CalibrateCamera, 0, - DependencyManager::get().data(), SLOT(calibrate())); - ddeCalibrate->setVisible(true); // DDE face tracking is on by default - faceTrackingMenu->addSeparator(); - addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::MuteFaceTracking, - [](bool mute) { FaceTracker::setIsMuted(mute); }, - Qt::CTRL | Qt::SHIFT | Qt::Key_F, FaceTracker::isMuted()); - addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::AutoMuteAudio, 0, false); -#endif - action = addCheckableActionToQMenuAndActionHash(avatarDebugMenu, MenuOption::AvatarReceiveStats, 0, false); connect(action, &QAction::triggered, [this]{ Avatar::setShowReceiveStats(isOptionChecked(MenuOption::AvatarReceiveStats)); }); action = addCheckableActionToQMenuAndActionHash(avatarDebugMenu, MenuOption::ShowBoundingCollisionShapes, 0, false); diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 3c27ef52c2..fb42f89048 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -48,7 +48,6 @@ #include #include #include -#include #include #include @@ -749,7 +748,6 @@ void MyAvatar::update(float deltaTime) { Head* head = getHead(); head->relax(deltaTime); - updateFromTrackers(deltaTime); if (getIsInWalkingState() && glm::length(getControllerPoseInAvatarFrame(controller::Action::HEAD).getVelocity()) < DEFAULT_AVATAR_WALK_SPEED_THRESHOLD) { setIsInWalkingState(false); @@ -1140,60 +1138,6 @@ void MyAvatar::updateSensorToWorldMatrix() { } -// Update avatar head rotation with sensor data -void MyAvatar::updateFromTrackers(float deltaTime) { - glm::vec3 estimatedRotation; - - bool hasHead = getControllerPoseInAvatarFrame(controller::Action::HEAD).isValid(); - bool playing = DependencyManager::get()->isPlaying(); - if (hasHead && playing) { - return; - } - - FaceTracker* tracker = qApp->getActiveFaceTracker(); - bool inFacetracker = tracker && !FaceTracker::isMuted(); - - if (inFacetracker) { - estimatedRotation = glm::degrees(safeEulerAngles(tracker->getHeadRotation())); - } - - // Rotate the body if the head is turned beyond the screen - if (Menu::getInstance()->isOptionChecked(MenuOption::TurnWithHead)) { - const float TRACKER_YAW_TURN_SENSITIVITY = 0.5f; - const float TRACKER_MIN_YAW_TURN = 15.0f; - const float TRACKER_MAX_YAW_TURN = 50.0f; - if ( (fabs(estimatedRotation.y) > TRACKER_MIN_YAW_TURN) && - (fabs(estimatedRotation.y) < TRACKER_MAX_YAW_TURN) ) { - if (estimatedRotation.y > 0.0f) { - _bodyYawDelta += (estimatedRotation.y - TRACKER_MIN_YAW_TURN) * TRACKER_YAW_TURN_SENSITIVITY; - } else { - _bodyYawDelta += (estimatedRotation.y + TRACKER_MIN_YAW_TURN) * TRACKER_YAW_TURN_SENSITIVITY; - } - } - } - - // Set the rotation of the avatar's head (as seen by others, not affecting view frustum) - // to be scaled such that when the user's physical head is pointing at edge of screen, the - // avatar head is at the edge of the in-world view frustum. So while a real person may move - // their head only 30 degrees or so, this may correspond to a 90 degree field of view. - // Note that roll is magnified by a constant because it is not related to field of view. - - - Head* head = getHead(); - if (hasHead || playing) { - head->setDeltaPitch(estimatedRotation.x); - head->setDeltaYaw(estimatedRotation.y); - head->setDeltaRoll(estimatedRotation.z); - } else { - ViewFrustum viewFrustum; - qApp->copyViewFrustum(viewFrustum); - float magnifyFieldOfView = viewFrustum.getFieldOfView() / _realWorldFieldOfView.get(); - head->setDeltaPitch(estimatedRotation.x * magnifyFieldOfView); - head->setDeltaYaw(estimatedRotation.y * magnifyFieldOfView); - head->setDeltaRoll(estimatedRotation.z); - } -} - glm::vec3 MyAvatar::getLeftHandPosition() const { auto pose = getControllerPoseInAvatarFrame(controller::Action::LEFT_HAND); return pose.isValid() ? pose.getTranslation() : glm::vec3(0.0f); @@ -6585,7 +6529,7 @@ bool MyAvatar::getIsJointOverridden(int jointIndex) const { return _skeletonModel->getIsJointOverridden(jointIndex); } -void MyAvatar::updateLookAtPosition(FaceTracker* faceTracker, Camera& myCamera) { +void MyAvatar::updateLookAtPosition(Camera& myCamera) { updateLookAtTargetAvatar(); @@ -6681,21 +6625,6 @@ void MyAvatar::updateLookAtPosition(FaceTracker* faceTracker, Camera& myCamera) (getHead()->getFinalOrientationInWorldFrame() * glm::vec3(0.0f, 0.0f, -TREE_SCALE)); } } - - // Deflect the eyes a bit to match the detected gaze from the face tracker if active. - if (faceTracker && !faceTracker->isMuted()) { - float eyePitch = faceTracker->getEstimatedEyePitch(); - float eyeYaw = faceTracker->getEstimatedEyeYaw(); - const float GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT = 0.1f; - glm::vec3 origin = myHead->getEyePosition(); - float deflection = faceTracker->getEyeDeflection(); - if (isLookingAtSomeone) { - deflection *= GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT; - } - lookAtSpot = origin + myCamera.getOrientation() * glm::quat(glm::radians(glm::vec3( - eyePitch * deflection, eyeYaw * deflection, 0.0f))) * - glm::inverse(myCamera.getOrientation()) * (lookAtSpot - origin); - } } } diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index acbd0f27a5..d67a2567f8 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -34,7 +34,6 @@ #include "AtRestDetector.h" #include "MyCharacterController.h" #include "RingBufferHistory.h" -#include "devices/DdeFaceTracker.h" class AvatarActionHold; class ModelItemID; @@ -1899,7 +1898,7 @@ public: bool getFlowActive() const; bool getNetworkGraphActive() const; - void updateLookAtPosition(FaceTracker* faceTracker, Camera& myCamera); + void updateLookAtPosition(Camera& myCamera); // sets the reaction enabled and triggered parameters of the passed in params // also clears internal reaction triggers @@ -2542,7 +2541,6 @@ private: virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail, bool dropFaceTracking) override; void simulate(float deltaTime, bool inView) override; - void updateFromTrackers(float deltaTime); void saveAvatarUrl(); virtual void render(RenderArgs* renderArgs) override; virtual bool shouldRenderHead(const RenderArgs* renderArgs) const override; diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index 4705d2d765..1c6e4690dd 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -14,10 +14,8 @@ #include #include #include -#include -#include +#include -#include "devices/DdeFaceTracker.h" #include "Application.h" #include "MyAvatar.h" diff --git a/interface/src/devices/DdeFaceTracker.cpp b/interface/src/devices/DdeFaceTracker.cpp deleted file mode 100644 index b9dc8326e8..0000000000 --- a/interface/src/devices/DdeFaceTracker.cpp +++ /dev/null @@ -1,686 +0,0 @@ -// -// DdeFaceTracker.cpp -// -// -// Created by Clement on 8/2/14. -// Copyright 2014 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "DdeFaceTracker.h" - -#include - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "Application.h" -#include "InterfaceLogging.h" -#include "Menu.h" - - -static const QHostAddress DDE_SERVER_ADDR("127.0.0.1"); -static const quint16 DDE_SERVER_PORT = 64204; -static const quint16 DDE_CONTROL_PORT = 64205; -#if defined(Q_OS_WIN) -static const QString DDE_PROGRAM_PATH = "/dde/dde.exe"; -#elif defined(Q_OS_MAC) -static const QString DDE_PROGRAM_PATH = "/dde.app/Contents/MacOS/dde"; -#endif -static const QStringList DDE_ARGUMENTS = QStringList() - << "--udp=" + DDE_SERVER_ADDR.toString() + ":" + QString::number(DDE_SERVER_PORT) - << "--receiver=" + QString::number(DDE_CONTROL_PORT) - << "--facedet_interval=500" // ms - << "--headless"; - -static const int NUM_EXPRESSIONS = 46; -static const int MIN_PACKET_SIZE = (8 + NUM_EXPRESSIONS) * sizeof(float) + sizeof(int); -static const int MAX_NAME_SIZE = 31; - -// There's almost but not quite a 1-1 correspondence between DDE's 46 and Faceshift 1.3's 48 packets. -// The best guess at mapping is to: -// - Swap L and R values -// - Skip two Faceshift values: JawChew (22) and LipsLowerDown (37) -static const int DDE_TO_FACESHIFT_MAPPING[] = { - 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, - 16, - 18, 17, - 19, - 23, - 21, - // Skip JawChew - 20, - 25, 24, 27, 26, 29, 28, 31, 30, 33, 32, - 34, 35, 36, - // Skip LipsLowerDown - 38, 39, 40, 41, 42, 43, 44, 45, - 47, 46 -}; - -// The DDE coefficients, overall, range from -0.2 to 1.5 or so. However, individual coefficients typically vary much -// less than this. -static const float DDE_COEFFICIENT_SCALES[] = { - 1.0f, // EyeBlink_L - 1.0f, // EyeBlink_R - 1.0f, // EyeSquint_L - 1.0f, // EyeSquint_R - 1.0f, // EyeDown_L - 1.0f, // EyeDown_R - 1.0f, // EyeIn_L - 1.0f, // EyeIn_R - 1.0f, // EyeOpen_L - 1.0f, // EyeOpen_R - 1.0f, // EyeOut_L - 1.0f, // EyeOut_R - 1.0f, // EyeUp_L - 1.0f, // EyeUp_R - 3.0f, // BrowsD_L - 3.0f, // BrowsD_R - 3.0f, // BrowsU_C - 3.0f, // BrowsU_L - 3.0f, // BrowsU_R - 1.0f, // JawFwd - 2.0f, // JawLeft - 1.8f, // JawOpen - 1.0f, // JawChew - 2.0f, // JawRight - 1.5f, // MouthLeft - 1.5f, // MouthRight - 1.5f, // MouthFrown_L - 1.5f, // MouthFrown_R - 2.5f, // MouthSmile_L - 2.5f, // MouthSmile_R - 1.0f, // MouthDimple_L - 1.0f, // MouthDimple_R - 1.0f, // LipsStretch_L - 1.0f, // LipsStretch_R - 1.0f, // LipsUpperClose - 1.0f, // LipsLowerClose - 1.0f, // LipsUpperUp - 1.0f, // LipsLowerDown - 1.0f, // LipsUpperOpen - 1.0f, // LipsLowerOpen - 1.5f, // LipsFunnel - 2.5f, // LipsPucker - 1.5f, // ChinLowerRaise - 1.5f, // ChinUpperRaise - 1.0f, // Sneer - 3.0f, // Puff - 1.0f, // CheekSquint_L - 1.0f // CheekSquint_R -}; - -struct DDEPacket { - //roughly in mm - float focal_length[1]; - float translation[3]; - - //quaternion - float rotation[4]; - - // The DDE coefficients, overall, range from -0.2 to 1.5 or so. However, individual coefficients typically vary much - // less than this. - float expressions[NUM_EXPRESSIONS]; - - //avatar id selected on the UI - int avatar_id; - - //client name, arbitrary length - char name[MAX_NAME_SIZE + 1]; -}; - -static const float STARTING_DDE_MESSAGE_TIME = 0.033f; -static const float DEFAULT_DDE_EYE_CLOSING_THRESHOLD = 0.8f; -static const int CALIBRATION_SAMPLES = 150; - -DdeFaceTracker::DdeFaceTracker() : - DdeFaceTracker(QHostAddress::Any, DDE_SERVER_PORT, DDE_CONTROL_PORT) -{ - -} - -DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, quint16 controlPort) : - _ddeProcess(NULL), - _ddeStopping(false), - _host(host), - _serverPort(serverPort), - _controlPort(controlPort), - _lastReceiveTimestamp(0), - _reset(false), - _leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes - _rightBlinkIndex(1), - _leftEyeDownIndex(4), - _rightEyeDownIndex(5), - _leftEyeInIndex(6), - _rightEyeInIndex(7), - _leftEyeOpenIndex(8), - _rightEyeOpenIndex(9), - _browDownLeftIndex(14), - _browDownRightIndex(15), - _browUpCenterIndex(16), - _browUpLeftIndex(17), - _browUpRightIndex(18), - _mouthSmileLeftIndex(28), - _mouthSmileRightIndex(29), - _jawOpenIndex(21), - _lastMessageReceived(0), - _averageMessageTime(STARTING_DDE_MESSAGE_TIME), - _lastHeadTranslation(glm::vec3(0.0f)), - _filteredHeadTranslation(glm::vec3(0.0f)), - _lastBrowUp(0.0f), - _filteredBrowUp(0.0f), - _eyePitch(0.0f), - _eyeYaw(0.0f), - _lastEyePitch(0.0f), - _lastEyeYaw(0.0f), - _filteredEyePitch(0.0f), - _filteredEyeYaw(0.0f), - _longTermAverageEyePitch(0.0f), - _longTermAverageEyeYaw(0.0f), - _lastEyeBlinks(), - _filteredEyeBlinks(), - _lastEyeCoefficients(), - _eyeClosingThreshold("ddeEyeClosingThreshold", DEFAULT_DDE_EYE_CLOSING_THRESHOLD), - _isCalibrating(false), - _calibrationCount(0), - _calibrationValues(), - _calibrationBillboard(NULL), - _calibrationMessage(QString()), - _isCalibrated(false) -{ - _coefficients.resize(NUM_FACESHIFT_BLENDSHAPES); - _blendshapeCoefficients.resize(NUM_FACESHIFT_BLENDSHAPES); - _coefficientAverages.resize(NUM_FACESHIFT_BLENDSHAPES); - _calibrationValues.resize(NUM_FACESHIFT_BLENDSHAPES); - - _eyeStates[0] = EYE_UNCONTROLLED; - _eyeStates[1] = EYE_UNCONTROLLED; - - connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams())); - connect(&_udpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(socketErrorOccurred(QAbstractSocket::SocketError))); - connect(&_udpSocket, SIGNAL(stateChanged(QAbstractSocket::SocketState)), - SLOT(socketStateChanged(QAbstractSocket::SocketState))); -} - -DdeFaceTracker::~DdeFaceTracker() { - setEnabled(false); - - if (_isCalibrating) { - cancelCalibration(); - } -} - -void DdeFaceTracker::init() { - FaceTracker::init(); - setEnabled(Menu::getInstance()->isOptionChecked(MenuOption::UseCamera) && !_isMuted); - Menu::getInstance()->getActionForOption(MenuOption::CalibrateCamera)->setEnabled(!_isMuted); -} - -void DdeFaceTracker::setEnabled(bool enabled) { - if (!_isInitialized) { - // Don't enable until have explicitly initialized - return; - } -#ifdef HAVE_DDE - - if (_isCalibrating) { - cancelCalibration(); - } - - // isOpen() does not work as one might expect on QUdpSocket; don't test isOpen() before closing socket. - _udpSocket.close(); - - // Terminate any existing DDE process, perhaps left running after an Interface crash. - // Do this even if !enabled in case user reset their settings after crash. - const char* DDE_EXIT_COMMAND = "exit"; - _udpSocket.bind(_host, _serverPort); - _udpSocket.writeDatagram(DDE_EXIT_COMMAND, DDE_SERVER_ADDR, _controlPort); - - if (enabled && !_ddeProcess) { - _ddeStopping = false; - qCDebug(interfaceapp) << "DDE Face Tracker: Starting"; - _ddeProcess = new QProcess(qApp); - connect(_ddeProcess, SIGNAL(finished(int, QProcess::ExitStatus)), SLOT(processFinished(int, QProcess::ExitStatus))); - _ddeProcess->start(QCoreApplication::applicationDirPath() + DDE_PROGRAM_PATH, DDE_ARGUMENTS); - } - - if (!enabled && _ddeProcess) { - _ddeStopping = true; - qCDebug(interfaceapp) << "DDE Face Tracker: Stopping"; - } -#endif -} - -void DdeFaceTracker::processFinished(int exitCode, QProcess::ExitStatus exitStatus) { - if (_ddeProcess) { - if (_ddeStopping) { - qCDebug(interfaceapp) << "DDE Face Tracker: Stopped"; - - } else { - qCWarning(interfaceapp) << "DDE Face Tracker: Stopped unexpectedly"; - Menu::getInstance()->setIsOptionChecked(MenuOption::NoFaceTracking, true); - } - _udpSocket.close(); - delete _ddeProcess; - _ddeProcess = NULL; - } -} - -void DdeFaceTracker::reset() { - if (_udpSocket.state() == QAbstractSocket::BoundState) { - _reset = true; - - qCDebug(interfaceapp) << "DDE Face Tracker: Reset"; - - const char* DDE_RESET_COMMAND = "reset"; - _udpSocket.writeDatagram(DDE_RESET_COMMAND, DDE_SERVER_ADDR, _controlPort); - - FaceTracker::reset(); - - _reset = true; - } -} - -void DdeFaceTracker::update(float deltaTime) { - if (!isActive()) { - return; - } - FaceTracker::update(deltaTime); - - glm::vec3 headEulers = glm::degrees(glm::eulerAngles(_headRotation)); - _estimatedEyePitch = _eyePitch - headEulers.x; - _estimatedEyeYaw = _eyeYaw - headEulers.y; -} - -bool DdeFaceTracker::isActive() const { - return (_ddeProcess != NULL); -} - -bool DdeFaceTracker::isTracking() const { - static const quint64 ACTIVE_TIMEOUT_USECS = 3000000; //3 secs - return (usecTimestampNow() - _lastReceiveTimestamp < ACTIVE_TIMEOUT_USECS); -} - -//private slots and methods -void DdeFaceTracker::socketErrorOccurred(QAbstractSocket::SocketError socketError) { - qCWarning(interfaceapp) << "DDE Face Tracker: Socket error: " << _udpSocket.errorString(); -} - -void DdeFaceTracker::socketStateChanged(QAbstractSocket::SocketState socketState) { - QString state; - switch(socketState) { - case QAbstractSocket::BoundState: - state = "Bound"; - break; - case QAbstractSocket::ClosingState: - state = "Closing"; - break; - case QAbstractSocket::ConnectedState: - state = "Connected"; - break; - case QAbstractSocket::ConnectingState: - state = "Connecting"; - break; - case QAbstractSocket::HostLookupState: - state = "Host Lookup"; - break; - case QAbstractSocket::ListeningState: - state = "Listening"; - break; - case QAbstractSocket::UnconnectedState: - state = "Unconnected"; - break; - } - qCDebug(interfaceapp) << "DDE Face Tracker: Socket: " << state; -} - -void DdeFaceTracker::readPendingDatagrams() { - QByteArray buffer; - while (_udpSocket.hasPendingDatagrams()) { - buffer.resize(_udpSocket.pendingDatagramSize()); - _udpSocket.readDatagram(buffer.data(), buffer.size()); - } - decodePacket(buffer); -} - -float DdeFaceTracker::getBlendshapeCoefficient(int index) const { - return (index >= 0 && index < (int)_blendshapeCoefficients.size()) ? _blendshapeCoefficients[index] : 0.0f; -} - -void DdeFaceTracker::decodePacket(const QByteArray& buffer) { - _lastReceiveTimestamp = usecTimestampNow(); - - if (buffer.size() > MIN_PACKET_SIZE) { - if (!_isCalibrated) { - calibrate(); - } - - bool isFiltering = Menu::getInstance()->isOptionChecked(MenuOption::VelocityFilter); - - DDEPacket packet; - int bytesToCopy = glm::min((int)sizeof(packet), buffer.size()); - memset(&packet.name, '\n', MAX_NAME_SIZE + 1); - memcpy(&packet, buffer.data(), bytesToCopy); - - glm::vec3 translation; - memcpy(&translation, packet.translation, sizeof(packet.translation)); - glm::quat rotation; - memcpy(&rotation, &packet.rotation, sizeof(packet.rotation)); - if (_reset || (_lastMessageReceived == 0)) { - memcpy(&_referenceTranslation, &translation, sizeof(glm::vec3)); - memcpy(&_referenceRotation, &rotation, sizeof(glm::quat)); - _reset = false; - } - - // Compute relative translation - float LEAN_DAMPING_FACTOR = 75.0f; - translation -= _referenceTranslation; - translation /= LEAN_DAMPING_FACTOR; - translation.x *= -1; - if (isFiltering) { - glm::vec3 linearVelocity = (translation - _lastHeadTranslation) / _averageMessageTime; - const float LINEAR_VELOCITY_FILTER_STRENGTH = 0.3f; - float velocityFilter = glm::clamp(1.0f - glm::length(linearVelocity) * - LINEAR_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); - _filteredHeadTranslation = velocityFilter * _filteredHeadTranslation + (1.0f - velocityFilter) * translation; - _lastHeadTranslation = translation; - _headTranslation = _filteredHeadTranslation; - } else { - _headTranslation = translation; - } - - // Compute relative rotation - rotation = glm::inverse(_referenceRotation) * rotation; - if (isFiltering) { - glm::quat r = glm::normalize(rotation * glm::inverse(_headRotation)); - float theta = 2 * acos(r.w); - glm::vec3 angularVelocity; - if (theta > EPSILON) { - float rMag = glm::length(glm::vec3(r.x, r.y, r.z)); - angularVelocity = theta / _averageMessageTime * glm::vec3(r.x, r.y, r.z) / rMag; - } else { - angularVelocity = glm::vec3(0, 0, 0); - } - const float ANGULAR_VELOCITY_FILTER_STRENGTH = 0.3f; - _headRotation = safeMix(_headRotation, rotation, glm::clamp(glm::length(angularVelocity) * - ANGULAR_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f)); - } else { - _headRotation = rotation; - } - - // Translate DDE coefficients to Faceshift compatible coefficients - for (int i = 0; i < NUM_EXPRESSIONS; i++) { - _coefficients[DDE_TO_FACESHIFT_MAPPING[i]] = packet.expressions[i]; - } - - // Calibration - if (_isCalibrating) { - addCalibrationDatum(); - } - for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { - _coefficients[i] -= _coefficientAverages[i]; - } - - // Use BrowsU_C to control both brows' up and down - float browUp = _coefficients[_browUpCenterIndex]; - if (isFiltering) { - const float BROW_VELOCITY_FILTER_STRENGTH = 0.5f; - float velocity = fabsf(browUp - _lastBrowUp) / _averageMessageTime; - float velocityFilter = glm::clamp(velocity * BROW_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); - _filteredBrowUp = velocityFilter * browUp + (1.0f - velocityFilter) * _filteredBrowUp; - _lastBrowUp = browUp; - browUp = _filteredBrowUp; - _coefficients[_browUpCenterIndex] = browUp; - } - _coefficients[_browUpLeftIndex] = browUp; - _coefficients[_browUpRightIndex] = browUp; - _coefficients[_browDownLeftIndex] = -browUp; - _coefficients[_browDownRightIndex] = -browUp; - - // Offset jaw open coefficient - static const float JAW_OPEN_THRESHOLD = 0.1f; - _coefficients[_jawOpenIndex] = _coefficients[_jawOpenIndex] - JAW_OPEN_THRESHOLD; - - // Offset smile coefficients - static const float SMILE_THRESHOLD = 0.5f; - _coefficients[_mouthSmileLeftIndex] = _coefficients[_mouthSmileLeftIndex] - SMILE_THRESHOLD; - _coefficients[_mouthSmileRightIndex] = _coefficients[_mouthSmileRightIndex] - SMILE_THRESHOLD; - - // Eye pitch and yaw - // EyeDown coefficients work better over both +ve and -ve values than EyeUp values. - // EyeIn coefficients work better over both +ve and -ve values than EyeOut values. - // Pitch and yaw values are relative to the screen. - const float EYE_PITCH_SCALE = -1500.0f; // Sign, scale, and average to be similar to Faceshift values. - _eyePitch = EYE_PITCH_SCALE * (_coefficients[_leftEyeDownIndex] + _coefficients[_rightEyeDownIndex]); - const float EYE_YAW_SCALE = 2000.0f; // Scale and average to be similar to Faceshift values. - _eyeYaw = EYE_YAW_SCALE * (_coefficients[_leftEyeInIndex] + _coefficients[_rightEyeInIndex]); - if (isFiltering) { - const float EYE_VELOCITY_FILTER_STRENGTH = 0.005f; - float pitchVelocity = fabsf(_eyePitch - _lastEyePitch) / _averageMessageTime; - float pitchVelocityFilter = glm::clamp(pitchVelocity * EYE_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); - _filteredEyePitch = pitchVelocityFilter * _eyePitch + (1.0f - pitchVelocityFilter) * _filteredEyePitch; - _lastEyePitch = _eyePitch; - _eyePitch = _filteredEyePitch; - float yawVelocity = fabsf(_eyeYaw - _lastEyeYaw) / _averageMessageTime; - float yawVelocityFilter = glm::clamp(yawVelocity * EYE_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); - _filteredEyeYaw = yawVelocityFilter * _eyeYaw + (1.0f - yawVelocityFilter) * _filteredEyeYaw; - _lastEyeYaw = _eyeYaw; - _eyeYaw = _filteredEyeYaw; - } - - // Velocity filter EyeBlink values - const float DDE_EYEBLINK_SCALE = 3.0f; - float eyeBlinks[] = { DDE_EYEBLINK_SCALE * _coefficients[_leftBlinkIndex], - DDE_EYEBLINK_SCALE * _coefficients[_rightBlinkIndex] }; - if (isFiltering) { - const float BLINK_VELOCITY_FILTER_STRENGTH = 0.3f; - for (int i = 0; i < 2; i++) { - float velocity = fabsf(eyeBlinks[i] - _lastEyeBlinks[i]) / _averageMessageTime; - float velocityFilter = glm::clamp(velocity * BLINK_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); - _filteredEyeBlinks[i] = velocityFilter * eyeBlinks[i] + (1.0f - velocityFilter) * _filteredEyeBlinks[i]; - _lastEyeBlinks[i] = eyeBlinks[i]; - } - } - - // Finesse EyeBlink values - float eyeCoefficients[2]; - if (Menu::getInstance()->isOptionChecked(MenuOption::BinaryEyelidControl)) { - if (_eyeStates[0] == EYE_UNCONTROLLED) { - _eyeStates[0] = EYE_OPEN; - _eyeStates[1] = EYE_OPEN; - } - - for (int i = 0; i < 2; i++) { - // Scale EyeBlink values so that they can be used to control both EyeBlink and EyeOpen - // -ve values control EyeOpen; +ve values control EyeBlink - static const float EYE_CONTROL_THRESHOLD = 0.5f; // Resting eye value - eyeCoefficients[i] = (_filteredEyeBlinks[i] - EYE_CONTROL_THRESHOLD) / (1.0f - EYE_CONTROL_THRESHOLD); - - // Change to closing or opening states - const float EYE_CONTROL_HYSTERISIS = 0.25f; - float eyeClosingThreshold = getEyeClosingThreshold(); - float eyeOpeningThreshold = eyeClosingThreshold - EYE_CONTROL_HYSTERISIS; - if ((_eyeStates[i] == EYE_OPEN || _eyeStates[i] == EYE_OPENING) && eyeCoefficients[i] > eyeClosingThreshold) { - _eyeStates[i] = EYE_CLOSING; - } else if ((_eyeStates[i] == EYE_CLOSED || _eyeStates[i] == EYE_CLOSING) - && eyeCoefficients[i] < eyeOpeningThreshold) { - _eyeStates[i] = EYE_OPENING; - } - - const float EYELID_MOVEMENT_RATE = 10.0f; // units/second - const float EYE_OPEN_SCALE = 0.2f; - if (_eyeStates[i] == EYE_CLOSING) { - // Close eyelid until it's fully closed - float closingValue = _lastEyeCoefficients[i] + EYELID_MOVEMENT_RATE * _averageMessageTime; - if (closingValue >= 1.0f) { - _eyeStates[i] = EYE_CLOSED; - eyeCoefficients[i] = 1.0f; - } else { - eyeCoefficients[i] = closingValue; - } - } else if (_eyeStates[i] == EYE_OPENING) { - // Open eyelid until it meets the current adjusted value - float openingValue = _lastEyeCoefficients[i] - EYELID_MOVEMENT_RATE * _averageMessageTime; - if (openingValue < eyeCoefficients[i] * EYE_OPEN_SCALE) { - _eyeStates[i] = EYE_OPEN; - eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE; - } else { - eyeCoefficients[i] = openingValue; - } - } else if (_eyeStates[i] == EYE_OPEN) { - // Reduce eyelid movement - eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE; - } else if (_eyeStates[i] == EYE_CLOSED) { - // Keep eyelid fully closed - eyeCoefficients[i] = 1.0; - } - } - - if (_eyeStates[0] == EYE_OPEN && _eyeStates[1] == EYE_OPEN) { - // Couple eyelids - eyeCoefficients[0] = eyeCoefficients[1] = (eyeCoefficients[0] + eyeCoefficients[0]) / 2.0f; - } - - _lastEyeCoefficients[0] = eyeCoefficients[0]; - _lastEyeCoefficients[1] = eyeCoefficients[1]; - } else { - _eyeStates[0] = EYE_UNCONTROLLED; - _eyeStates[1] = EYE_UNCONTROLLED; - - eyeCoefficients[0] = _filteredEyeBlinks[0]; - eyeCoefficients[1] = _filteredEyeBlinks[1]; - } - - // Couple eyelid values if configured - use the most "open" value for both - if (Menu::getInstance()->isOptionChecked(MenuOption::CoupleEyelids)) { - float eyeCoefficient = std::min(eyeCoefficients[0], eyeCoefficients[1]); - eyeCoefficients[0] = eyeCoefficient; - eyeCoefficients[1] = eyeCoefficient; - } - - // Use EyeBlink values to control both EyeBlink and EyeOpen - if (eyeCoefficients[0] > 0) { - _coefficients[_leftBlinkIndex] = eyeCoefficients[0]; - _coefficients[_leftEyeOpenIndex] = 0.0f; - } else { - _coefficients[_leftBlinkIndex] = 0.0f; - _coefficients[_leftEyeOpenIndex] = -eyeCoefficients[0]; - } - if (eyeCoefficients[1] > 0) { - _coefficients[_rightBlinkIndex] = eyeCoefficients[1]; - _coefficients[_rightEyeOpenIndex] = 0.0f; - } else { - _coefficients[_rightBlinkIndex] = 0.0f; - _coefficients[_rightEyeOpenIndex] = -eyeCoefficients[1]; - } - - // Scale all coefficients - for (int i = 0; i < NUM_EXPRESSIONS; i++) { - _blendshapeCoefficients[i] - = glm::clamp(DDE_COEFFICIENT_SCALES[i] * _coefficients[i], 0.0f, 1.0f); - } - - // Calculate average frame time - const float FRAME_AVERAGING_FACTOR = 0.99f; - quint64 usecsNow = usecTimestampNow(); - if (_lastMessageReceived != 0) { - _averageMessageTime = FRAME_AVERAGING_FACTOR * _averageMessageTime - + (1.0f - FRAME_AVERAGING_FACTOR) * (float)(usecsNow - _lastMessageReceived) / 1000000.0f; - } - _lastMessageReceived = usecsNow; - - FaceTracker::countFrame(); - - } else { - qCWarning(interfaceapp) << "DDE Face Tracker: Decode error"; - } - - if (_isCalibrating && _calibrationCount > CALIBRATION_SAMPLES) { - finishCalibration(); - } -} - -void DdeFaceTracker::setEyeClosingThreshold(float eyeClosingThreshold) { - _eyeClosingThreshold.set(eyeClosingThreshold); -} - -static const int CALIBRATION_BILLBOARD_WIDTH = 300; -static const int CALIBRATION_BILLBOARD_HEIGHT = 120; -static QString CALIBRATION_INSTRUCTION_MESSAGE = "Hold still to calibrate camera"; - -void DdeFaceTracker::calibrate() { - if (!Menu::getInstance()->isOptionChecked(MenuOption::UseCamera) || _isMuted) { - return; - } - - if (!_isCalibrating) { - qCDebug(interfaceapp) << "DDE Face Tracker: Calibration started"; - - _isCalibrating = true; - _calibrationCount = 0; - _calibrationMessage = CALIBRATION_INSTRUCTION_MESSAGE + "\n\n"; - - // FIXME: this overlay probably doesn't work anymore - _calibrationBillboard = new TextOverlay(); - glm::vec2 viewport = qApp->getCanvasSize(); - _calibrationBillboard->setX((viewport.x - CALIBRATION_BILLBOARD_WIDTH) / 2); - _calibrationBillboard->setY((viewport.y - CALIBRATION_BILLBOARD_HEIGHT) / 2); - _calibrationBillboard->setWidth(CALIBRATION_BILLBOARD_WIDTH); - _calibrationBillboard->setHeight(CALIBRATION_BILLBOARD_HEIGHT); - _calibrationBillboardID = qApp->getOverlays().addOverlay(_calibrationBillboard); - - for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { - _calibrationValues[i] = 0.0f; - } - } -} - -void DdeFaceTracker::addCalibrationDatum() { - const int LARGE_TICK_INTERVAL = 30; - const int SMALL_TICK_INTERVAL = 6; - int samplesLeft = CALIBRATION_SAMPLES - _calibrationCount; - if (samplesLeft % LARGE_TICK_INTERVAL == 0) { - _calibrationMessage += QString::number(samplesLeft / LARGE_TICK_INTERVAL); - // FIXME: set overlay text - } else if (samplesLeft % SMALL_TICK_INTERVAL == 0) { - _calibrationMessage += "."; - // FIXME: set overlay text - } - - for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { - _calibrationValues[i] += _coefficients[i]; - } - - _calibrationCount += 1; -} - -void DdeFaceTracker::cancelCalibration() { - qApp->getOverlays().deleteOverlay(_calibrationBillboardID); - _calibrationBillboard = NULL; - _isCalibrating = false; - qCDebug(interfaceapp) << "DDE Face Tracker: Calibration cancelled"; -} - -void DdeFaceTracker::finishCalibration() { - qApp->getOverlays().deleteOverlay(_calibrationBillboardID); - _calibrationBillboard = NULL; - _isCalibrating = false; - _isCalibrated = true; - - for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { - _coefficientAverages[i] = _calibrationValues[i] / (float)CALIBRATION_SAMPLES; - } - - reset(); - - qCDebug(interfaceapp) << "DDE Face Tracker: Calibration finished"; -} diff --git a/interface/src/devices/DdeFaceTracker.h b/interface/src/devices/DdeFaceTracker.h deleted file mode 100644 index dc451134f0..0000000000 --- a/interface/src/devices/DdeFaceTracker.h +++ /dev/null @@ -1,181 +0,0 @@ -// -// DdeFaceTracker.h -// -// -// Created by Clement on 8/2/14. -// Copyright 2014 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_DdeFaceTracker_h -#define hifi_DdeFaceTracker_h - -#include - -//Disabling dde due to random crashes with closing the socket on macos. all the accompanying code is wrapped with the ifdef HAVE_DDE. uncomment the define below to enable -#if defined(Q_OS_WIN) || defined(Q_OS_OSX) - //#define HAVE_DDE -#endif - -#include -#include - -#include -#include - -#include - -/**jsdoc - * The FaceTracker API helps manage facial tracking hardware. - * @namespace FaceTracker - * - * @hifi-interface - * @hifi-client-entity - * @hifi-avatar - */ - -class DdeFaceTracker : public FaceTracker, public Dependency { - Q_OBJECT - SINGLETON_DEPENDENCY - -public: - virtual void init() override; - virtual void reset() override; - virtual void update(float deltaTime) override; - - virtual bool isActive() const override; - virtual bool isTracking() const override; - - float getLeftBlink() const { return getBlendshapeCoefficient(_leftBlinkIndex); } - float getRightBlink() const { return getBlendshapeCoefficient(_rightBlinkIndex); } - float getLeftEyeOpen() const { return getBlendshapeCoefficient(_leftEyeOpenIndex); } - float getRightEyeOpen() const { return getBlendshapeCoefficient(_rightEyeOpenIndex); } - - float getBrowDownLeft() const { return getBlendshapeCoefficient(_browDownLeftIndex); } - float getBrowDownRight() const { return getBlendshapeCoefficient(_browDownRightIndex); } - float getBrowUpCenter() const { return getBlendshapeCoefficient(_browUpCenterIndex); } - float getBrowUpLeft() const { return getBlendshapeCoefficient(_browUpLeftIndex); } - float getBrowUpRight() const { return getBlendshapeCoefficient(_browUpRightIndex); } - - float getMouthSize() const { return getBlendshapeCoefficient(_jawOpenIndex); } - float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); } - float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); } - - float getEyeClosingThreshold() { return _eyeClosingThreshold.get(); } - void setEyeClosingThreshold(float eyeClosingThreshold); - -public slots: - - /**jsdoc - * @function FaceTracker.setEnabled - * @param {boolean} enabled - */ - void setEnabled(bool enabled) override; - - /**jsdoc - * @function FaceTracker.calibrate - */ - void calibrate(); - -private slots: - void processFinished(int exitCode, QProcess::ExitStatus exitStatus); - - //sockets - void socketErrorOccurred(QAbstractSocket::SocketError socketError); - void readPendingDatagrams(); - void socketStateChanged(QAbstractSocket::SocketState socketState); - -private: - DdeFaceTracker(); - DdeFaceTracker(const QHostAddress& host, quint16 serverPort, quint16 controlPort); - virtual ~DdeFaceTracker(); - - QProcess* _ddeProcess; - bool _ddeStopping; - - QHostAddress _host; - quint16 _serverPort; - quint16 _controlPort; - - float getBlendshapeCoefficient(int index) const; - void decodePacket(const QByteArray& buffer); - - // sockets - QUdpSocket _udpSocket; - quint64 _lastReceiveTimestamp; - - bool _reset; - glm::vec3 _referenceTranslation; - glm::quat _referenceRotation; - - int _leftBlinkIndex; - int _rightBlinkIndex; - int _leftEyeDownIndex; - int _rightEyeDownIndex; - int _leftEyeInIndex; - int _rightEyeInIndex; - int _leftEyeOpenIndex; - int _rightEyeOpenIndex; - - int _browDownLeftIndex; - int _browDownRightIndex; - int _browUpCenterIndex; - int _browUpLeftIndex; - int _browUpRightIndex; - - int _mouthSmileLeftIndex; - int _mouthSmileRightIndex; - - int _jawOpenIndex; - - QVector _coefficients; - - quint64 _lastMessageReceived; - float _averageMessageTime; - - glm::vec3 _lastHeadTranslation; - glm::vec3 _filteredHeadTranslation; - - float _lastBrowUp; - float _filteredBrowUp; - - float _eyePitch; // Degrees, relative to screen - float _eyeYaw; - float _lastEyePitch; - float _lastEyeYaw; - float _filteredEyePitch; - float _filteredEyeYaw; - float _longTermAverageEyePitch = 0.0f; - float _longTermAverageEyeYaw = 0.0f; - bool _longTermAverageInitialized = false; - - enum EyeState { - EYE_UNCONTROLLED, - EYE_OPEN, - EYE_CLOSING, - EYE_CLOSED, - EYE_OPENING - }; - EyeState _eyeStates[2]; - float _lastEyeBlinks[2]; - float _filteredEyeBlinks[2]; - float _lastEyeCoefficients[2]; - Setting::Handle _eyeClosingThreshold; - - QVector _coefficientAverages; - - bool _isCalibrating; - int _calibrationCount; - QVector _calibrationValues; - TextOverlay* _calibrationBillboard; - QUuid _calibrationBillboardID; - QString _calibrationMessage; - bool _isCalibrated; - void addCalibrationDatum(); - void cancelCalibration(); - void finishCalibration(); -}; - -#endif // hifi_DdeFaceTracker_h diff --git a/interface/src/ui/AvatarInputs.cpp b/interface/src/ui/AvatarInputs.cpp index a6bc7cf84f..3986cb1533 100644 --- a/interface/src/ui/AvatarInputs.cpp +++ b/interface/src/ui/AvatarInputs.cpp @@ -11,7 +11,6 @@ #include #include -#include #include #include "Application.h" @@ -76,8 +75,6 @@ void AvatarInputs::update() { return; } - AI_UPDATE(cameraEnabled, !Menu::getInstance()->isOptionChecked(MenuOption::NoFaceTracking)); - AI_UPDATE(cameraMuted, Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking)); AI_UPDATE(isHMD, qApp->isHMDMode()); } @@ -103,13 +100,6 @@ bool AvatarInputs::getIgnoreRadiusEnabled() const { return DependencyManager::get()->getIgnoreRadiusEnabled(); } -void AvatarInputs::toggleCameraMute() { - FaceTracker* faceTracker = qApp->getSelectedFaceTracker(); - if (faceTracker) { - faceTracker->toggleMute(); - } -} - void AvatarInputs::resetSensors() { qApp->resetSensors(); } diff --git a/interface/src/ui/AvatarInputs.h b/interface/src/ui/AvatarInputs.h index dca39fd433..40ee0ea7d4 100644 --- a/interface/src/ui/AvatarInputs.h +++ b/interface/src/ui/AvatarInputs.h @@ -32,14 +32,6 @@ class AvatarInputs : public QObject { * @hifi-client-entity * @hifi-avatar * - * @property {boolean} cameraEnabled - true if webcam face tracking is enabled, false if it is - * disabled. - * Read-only. - *

Deprecated: This property is deprecated and will be removed.

- * @property {boolean} cameraMuted - true if webcam face tracking is muted (temporarily disabled), - * false it if isn't. - * Read-only. - *

Deprecated: This property is deprecated and will be removed.

* @property {boolean} ignoreRadiusEnabled - true if the privacy shield is enabled, false if it * is disabled. * Read-only. @@ -51,8 +43,6 @@ class AvatarInputs : public QObject { * it is hidden. */ - AI_PROPERTY(bool, cameraEnabled, false) - AI_PROPERTY(bool, cameraMuted, false) AI_PROPERTY(bool, isHMD, false) Q_PROPERTY(bool showAudioTools READ showAudioTools WRITE setShowAudioTools NOTIFY showAudioToolsChanged) @@ -97,22 +87,6 @@ public slots: signals: - /**jsdoc - * Triggered when webcam face tracking is enabled or disabled. - * @deprecated This signal is deprecated and will be removed. - * @function AvatarInputs.cameraEnabledChanged - * @returns {Signal} - */ - void cameraEnabledChanged(); - - /**jsdoc - * Triggered when webcam face tracking is muted (temporarily disabled) or unmuted. - * @deprecated This signal is deprecated and will be removed. - * @function AvatarInputs.cameraMutedChanged - * @returns {Signal} - */ - void cameraMutedChanged(); - /**jsdoc * Triggered when the display mode changes between desktop and HMD. * @function AvatarInputs.isHMDChanged @@ -183,13 +157,6 @@ protected: */ Q_INVOKABLE void resetSensors(); - /**jsdoc - * Toggles the muting (temporary disablement) of webcam face tracking on/off. - *

Deprecated: This function is deprecated and will be removed.

- * @function AvatarInputs.toggleCameraMute - */ - Q_INVOKABLE void toggleCameraMute(); - private: void onAvatarEnteredIgnoreRadius(); void onAvatarLeftIgnoreRadius(); diff --git a/interface/src/ui/PreferencesDialog.cpp b/interface/src/ui/PreferencesDialog.cpp index ec15dd8111..5f033fe008 100644 --- a/interface/src/ui/PreferencesDialog.cpp +++ b/interface/src/ui/PreferencesDialog.cpp @@ -10,7 +10,6 @@ #include #include -#include #include #include #include @@ -285,22 +284,6 @@ void setupPreferences() { preferences->addPreference(preference); } - static const QString FACE_TRACKING{ "Face Tracking" }; - { -#ifdef HAVE_DDE - auto getter = []()->float { return DependencyManager::get()->getEyeClosingThreshold(); }; - auto setter = [](float value) { DependencyManager::get()->setEyeClosingThreshold(value); }; - preferences->addPreference(new SliderPreference(FACE_TRACKING, "Eye Closing Threshold", getter, setter)); -#endif - } - - - { - auto getter = []()->float { return FaceTracker::getEyeDeflection(); }; - auto setter = [](float value) { FaceTracker::setEyeDeflection(value); }; - preferences->addPreference(new SliderPreference(FACE_TRACKING, "Eye Deflection", getter, setter)); - } - static const QString VR_MOVEMENT{ "VR Movement" }; { auto getter = [myAvatar]()->bool { return myAvatar->getAllowTeleporting(); }; diff --git a/libraries/avatars-renderer/CMakeLists.txt b/libraries/avatars-renderer/CMakeLists.txt index de1ac1a7c2..7929be8b04 100644 --- a/libraries/avatars-renderer/CMakeLists.txt +++ b/libraries/avatars-renderer/CMakeLists.txt @@ -1,6 +1,6 @@ set(TARGET_NAME avatars-renderer) setup_hifi_library(Network Script) -link_hifi_libraries(shared shaders gpu graphics animation material-networking model-networking script-engine render render-utils image trackers entities-renderer) +link_hifi_libraries(shared shaders gpu graphics animation material-networking model-networking script-engine render render-utils image entities-renderer) include_hifi_library_headers(avatars) include_hifi_library_headers(networking) include_hifi_library_headers(hfm) diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 63d8e2981c..ea6cdd7087 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include "Logging.h" @@ -26,6 +25,22 @@ using namespace std; static bool disableEyelidAdjustment { false }; +static void updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, + float jawOpen, float mouth2, float mouth3, float mouth4, QVector& coefficients) { + + coefficients.resize(std::max((int)coefficients.size(), (int)Blendshapes::BlendshapeCount)); + qFill(coefficients.begin(), coefficients.end(), 0.0f); + coefficients[(int)Blendshapes::EyeBlink_L] = leftBlink; + coefficients[(int)Blendshapes::EyeBlink_R] = rightBlink; + coefficients[(int)Blendshapes::BrowsU_C] = browUp; + coefficients[(int)Blendshapes::BrowsU_L] = browUp; + coefficients[(int)Blendshapes::BrowsU_R] = browUp; + coefficients[(int)Blendshapes::JawOpen] = jawOpen; + coefficients[(int)Blendshapes::MouthSmile_L] = coefficients[(int)Blendshapes::MouthSmile_R] = mouth4; + coefficients[(int)Blendshapes::LipsUpperClose] = mouth2; + coefficients[(int)Blendshapes::LipsFunnel] = mouth3; +} + Head::Head(Avatar* owningAvatar) : HeadData(owningAvatar), _leftEyeLookAtID(DependencyManager::get()->allocateID()), @@ -153,7 +168,7 @@ void Head::simulate(float deltaTime) { _mouthTime = 0.0f; } - FaceTracker::updateFakeCoefficients( + updateFakeCoefficients( _leftEyeBlink, _rightEyeBlink, _browAudioLift, diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 0e0dd9887f..561f2c798a 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -71,7 +71,7 @@ void HeadData::setOrientation(const glm::quat& orientation) { } void HeadData::computeBlendshapesLookupMap(){ - for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { + for (int i = 0; i < (int)Blendshapes::BlendshapeCount; i++) { _blendshapeLookupMap[FACESHIFT_BLENDSHAPES[i]] = i; } } diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index e5dd158505..c6bcc2599b 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -20,7 +20,7 @@ #include #include -#include +#include // degrees const float MIN_HEAD_YAW = -180.0f; diff --git a/libraries/fbx/src/FBXSerializer.cpp b/libraries/fbx/src/FBXSerializer.cpp index f8339ddd31..100f6ee98e 100644 --- a/libraries/fbx/src/FBXSerializer.cpp +++ b/libraries/fbx/src/FBXSerializer.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include diff --git a/libraries/fbx/src/GLTFSerializer.cpp b/libraries/fbx/src/GLTFSerializer.cpp index da21c7995e..5d4daf53f7 100755 --- a/libraries/fbx/src/GLTFSerializer.cpp +++ b/libraries/fbx/src/GLTFSerializer.cpp @@ -33,7 +33,7 @@ #include #include #include -#include +#include #include "FBXSerializer.h" diff --git a/libraries/shared/src/FaceshiftConstants.cpp b/libraries/shared/src/BlendshapeConstants.cpp similarity index 84% rename from libraries/shared/src/FaceshiftConstants.cpp rename to libraries/shared/src/BlendshapeConstants.cpp index b847837a7a..528b941b73 100644 --- a/libraries/shared/src/FaceshiftConstants.cpp +++ b/libraries/shared/src/BlendshapeConstants.cpp @@ -1,5 +1,5 @@ // -// FaceshiftConstants.cpp +// BlendshapeConstants.cpp // // // Created by Clement on 1/23/15. @@ -9,7 +9,7 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#include "FaceshiftConstants.h" +#include "BlendshapeConstants.h" const char* FACESHIFT_BLENDSHAPES[] = { "EyeBlink_L", @@ -34,7 +34,7 @@ const char* FACESHIFT_BLENDSHAPES[] = { "JawFwd", "JawLeft", "JawOpen", - "JawChew", // legacy not in ARKit + "JawChew", "JawRight", "MouthLeft", "MouthRight", @@ -48,15 +48,15 @@ const char* FACESHIFT_BLENDSHAPES[] = { "LipsStretch_R", "LipsUpperClose", "LipsLowerClose", - "LipsUpperUp", // legacy, split in ARKit - "LipsLowerDown", // legacy, split in ARKit + "LipsUpperUp", + "LipsLowerDown", "LipsUpperOpen", "LipsLowerOpen", "LipsFunnel", "LipsPucker", "ChinLowerRaise", "ChinUpperRaise", - "Sneer", // legacy, split in ARKit + "Sneer", "Puff", "CheekSquint_L", "CheekSquint_R", @@ -71,8 +71,6 @@ const char* FACESHIFT_BLENDSHAPES[] = { // MouthShrugUpper // TongueOut -const int NUM_FACESHIFT_BLENDSHAPES = sizeof(FACESHIFT_BLENDSHAPES) / sizeof(char*); - const int EYE_BLINK_L_INDEX = 0; const int EYE_BLINK_R_INDEX = 1; const int EYE_SQUINT_L_INDEX = 2; diff --git a/libraries/shared/src/BlendshapeConstants.h b/libraries/shared/src/BlendshapeConstants.h new file mode 100644 index 0000000000..0fca03e34b --- /dev/null +++ b/libraries/shared/src/BlendshapeConstants.h @@ -0,0 +1,76 @@ +// +// BlendshapeConstants.h +// +// +// Created by Clement on 1/23/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_BlendshapeConstants_h +#define hifi_BlendshapeConstants_h + +/// The names of the blendshapes expected by Faceshift, terminated with an empty string. +extern const char* FACESHIFT_BLENDSHAPES[]; + +// Eyes and Brows indices +extern const int EYE_BLINK_INDICES[]; +extern const int EYE_OPEN_INDICES[]; +extern const int BROWS_U_INDICES[]; +extern const int EYE_SQUINT_INDICES[]; + +enum class Blendshapes : int { + EyeBlink_L = 0, + EyeBlink_R, + EyeSquint_L, + EyeSquint_R, + EyeDown_L, + EyeDown_R, + EyeIn_L, + EyeIn_R, + EyeOpen_L, + EyeOpen_R, + EyeOut_L, // 10 + EyeOut_R, + EyeUp_L, + EyeUp_R, + BrowsD_L, + BrowsD_R, + BrowsU_C, + BrowsU_L, + BrowsU_R, + JawFwd, + JawLeft, // 20 + JawOpen, + JawChew, // legacy not in ARKit + JawRight, + MouthLeft, + MouthRight, + MouthFrown_L, + MouthFrown_R, + MouthSmile_L, + MouthSmile_R, + MouthDimple_L, // 30 + MouthDimple_R, + LipsStretch_L, + LipsStretch_R, + LipsUpperClose, + LipsLowerClose, + LipsUpperUp, // legacy, split in ARKit + LipsLowerDown, // legacy, split in ARKit + LipsUpperOpen, + LipsLowerOpen, + LipsFunnel, // 40 + LipsPucker, + ChinLowerRaise, + ChinUpperRaise, + Sneer, // legacy, split in ARKit + Puff, + CheekSquint_L, + CheekSquint_R, + BlendshapeCount +}; + +#endif // hifi_BlendshapeConstants_h diff --git a/libraries/shared/src/FaceshiftConstants.h b/libraries/shared/src/FaceshiftConstants.h deleted file mode 100644 index 4349a3a21e..0000000000 --- a/libraries/shared/src/FaceshiftConstants.h +++ /dev/null @@ -1,25 +0,0 @@ -// -// FaceshiftConstants.h -// -// -// Created by Clement on 1/23/15. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_FaceshiftConstants_h -#define hifi_FaceshiftConstants_h - -/// The names of the blendshapes expected by Faceshift, terminated with an empty string. -extern const char* FACESHIFT_BLENDSHAPES[]; -/// The size of FACESHIFT_BLENDSHAPES -extern const int NUM_FACESHIFT_BLENDSHAPES; -// Eyes and Brows indices -extern const int EYE_BLINK_INDICES[]; -extern const int EYE_OPEN_INDICES[]; -extern const int BROWS_U_INDICES[]; -extern const int EYE_SQUINT_INDICES[]; - -#endif // hifi_FaceshiftConstants_h diff --git a/libraries/trackers/CMakeLists.txt b/libraries/trackers/CMakeLists.txt deleted file mode 100644 index 6943f1a197..0000000000 --- a/libraries/trackers/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(TARGET_NAME trackers) -setup_hifi_library() -GroupSources("src") -link_hifi_libraries(shared) -include_hifi_library_headers(octree) - -target_bullet() diff --git a/libraries/trackers/src/trackers/FaceTracker.cpp b/libraries/trackers/src/trackers/FaceTracker.cpp deleted file mode 100644 index 034787f19a..0000000000 --- a/libraries/trackers/src/trackers/FaceTracker.cpp +++ /dev/null @@ -1,133 +0,0 @@ -// -// Created by Andrzej Kapolka on 4/9/14. -// Copyright 2014 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "FaceTracker.h" - -#include -#include -#include "Logging.h" -//#include "Menu.h" - -const int FPS_TIMER_DELAY = 2000; // ms -const int FPS_TIMER_DURATION = 2000; // ms - -const float DEFAULT_EYE_DEFLECTION = 0.25f; -Setting::Handle FaceTracker::_eyeDeflection("faceshiftEyeDeflection", DEFAULT_EYE_DEFLECTION); -bool FaceTracker::_isMuted { true }; - -void FaceTracker::init() { - _isInitialized = true; // FaceTracker can be used now -} - -inline float FaceTracker::getBlendshapeCoefficient(int index) const { - return isValidBlendshapeIndex(index) ? glm::mix(0.0f, _blendshapeCoefficients[index], getFadeCoefficient()) - : 0.0f; -} - -const QVector& FaceTracker::getBlendshapeCoefficients() const { - static QVector blendshapes; - float fadeCoefficient = getFadeCoefficient(); - if (fadeCoefficient == 1.0f) { - return _blendshapeCoefficients; - } else { - blendshapes.resize(_blendshapeCoefficients.size()); - for (int i = 0; i < _blendshapeCoefficients.size(); i++) { - blendshapes[i] = glm::mix(0.0f, _blendshapeCoefficients[i], fadeCoefficient); - } - return blendshapes; - } -} - -float FaceTracker::getFadeCoefficient() const { - return _fadeCoefficient; -} - -const glm::vec3 FaceTracker::getHeadTranslation() const { - return glm::mix(glm::vec3(0.0f), _headTranslation, getFadeCoefficient()); -} - -const glm::quat FaceTracker::getHeadRotation() const { - return safeMix(glm::quat(), _headRotation, getFadeCoefficient()); -} - -void FaceTracker::update(float deltaTime) { - // Based on exponential distributions: http://en.wikipedia.org/wiki/Exponential_distribution - static const float EPSILON = 0.02f; // MUST BE < 1.0f - static const float INVERSE_AT_EPSILON = -std::log(EPSILON); // So that f(1.0f) = EPSILON ~ 0.0f - static const float RELAXATION_TIME = 0.8f; // sec - - if (isTracking()) { - if (_relaxationStatus == 1.0f) { - _fadeCoefficient = 1.0f; - return; - } - _relaxationStatus = glm::clamp(_relaxationStatus + deltaTime / RELAXATION_TIME, 0.0f, 1.0f); - _fadeCoefficient = 1.0f - std::exp(-_relaxationStatus * INVERSE_AT_EPSILON); - } else { - if (_relaxationStatus == 0.0f) { - _fadeCoefficient = 0.0f; - return; - } - _relaxationStatus = glm::clamp(_relaxationStatus - deltaTime / RELAXATION_TIME, 0.0f, 1.0f); - _fadeCoefficient = std::exp(-(1.0f - _relaxationStatus) * INVERSE_AT_EPSILON); - } -} - -void FaceTracker::reset() { - if (isActive() && !_isCalculatingFPS) { - QTimer::singleShot(FPS_TIMER_DELAY, this, SLOT(startFPSTimer())); - _isCalculatingFPS = true; - } -} - -void FaceTracker::startFPSTimer() { - _frameCount = 0; - QTimer::singleShot(FPS_TIMER_DURATION, this, SLOT(finishFPSTimer())); -} - -void FaceTracker::countFrame() { - if (_isCalculatingFPS) { - _frameCount++; - } -} - -void FaceTracker::finishFPSTimer() { - qCDebug(trackers) << "Face tracker FPS =" << (float)_frameCount / ((float)FPS_TIMER_DURATION / 1000.0f); - _isCalculatingFPS = false; -} - -void FaceTracker::toggleMute() { - _isMuted = !_isMuted; - emit muteToggled(); -} - -void FaceTracker::setEyeDeflection(float eyeDeflection) { - _eyeDeflection.set(eyeDeflection); -} - -void FaceTracker::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, - float jawOpen, float mouth2, float mouth3, float mouth4, QVector& coefficients) { - const int MMMM_BLENDSHAPE = 34; - const int FUNNEL_BLENDSHAPE = 40; - const int SMILE_LEFT_BLENDSHAPE = 28; - const int SMILE_RIGHT_BLENDSHAPE = 29; - const int MAX_FAKE_BLENDSHAPE = 40; // Largest modified blendshape from above and below - - coefficients.resize(std::max((int)coefficients.size(), MAX_FAKE_BLENDSHAPE + 1)); - qFill(coefficients.begin(), coefficients.end(), 0.0f); - coefficients[_leftBlinkIndex] = leftBlink; - coefficients[_rightBlinkIndex] = rightBlink; - coefficients[_browUpCenterIndex] = browUp; - coefficients[_browUpLeftIndex] = browUp; - coefficients[_browUpRightIndex] = browUp; - coefficients[_jawOpenIndex] = jawOpen; - coefficients[SMILE_LEFT_BLENDSHAPE] = coefficients[SMILE_RIGHT_BLENDSHAPE] = mouth4; - coefficients[MMMM_BLENDSHAPE] = mouth2; - coefficients[FUNNEL_BLENDSHAPE] = mouth3; -} - diff --git a/libraries/trackers/src/trackers/FaceTracker.h b/libraries/trackers/src/trackers/FaceTracker.h deleted file mode 100644 index 47fbf72616..0000000000 --- a/libraries/trackers/src/trackers/FaceTracker.h +++ /dev/null @@ -1,131 +0,0 @@ -// -// Created by Andrzej Kapolka on 4/9/14. -// Copyright 2014 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_FaceTracker_h -#define hifi_FaceTracker_h - -#include -#include - -#include -#include - -#include - -/// Base class for face trackers (DDE, BinaryVR). - -class FaceTracker : public QObject { - Q_OBJECT - -public: - virtual bool isActive() const { return false; } - virtual bool isTracking() const { return false; } - - virtual void init(); - virtual void update(float deltaTime); - virtual void reset(); - - float getFadeCoefficient() const; - - const glm::vec3 getHeadTranslation() const; - const glm::quat getHeadRotation() const; - - float getEstimatedEyePitch() const { return _estimatedEyePitch; } - float getEstimatedEyeYaw() const { return _estimatedEyeYaw; } - - int getNumBlendshapes() const { return _blendshapeCoefficients.size(); } - bool isValidBlendshapeIndex(int index) const { return index >= 0 && index < getNumBlendshapes(); } - const QVector& getBlendshapeCoefficients() const; - float getBlendshapeCoefficient(int index) const; - - static bool isMuted() { return _isMuted; } - static void setIsMuted(bool isMuted) { _isMuted = isMuted; } - - static float getEyeDeflection() { return _eyeDeflection.get(); } - static void setEyeDeflection(float eyeDeflection); - - static void updateFakeCoefficients(float leftBlink, - float rightBlink, - float browUp, - float jawOpen, - float mouth2, - float mouth3, - float mouth4, - QVector& coefficients); - -signals: - - /**jsdoc - * @function FaceTracker.muteToggled - * @returns {Signal} - */ - void muteToggled(); - -public slots: - - // No JSDoc here because it's overridden in DdeFaceTracker. - virtual void setEnabled(bool enabled) = 0; - - /**jsdoc - * @function FaceTracker.toggleMute - */ - void toggleMute(); - - /**jsdoc - * @function FaceTracker.getMuted - * @returns {boolean} - */ - bool getMuted() { return _isMuted; } - -protected: - virtual ~FaceTracker() {}; - - bool _isInitialized = false; - static bool _isMuted; - - glm::vec3 _headTranslation = glm::vec3(0.0f); - glm::quat _headRotation = glm::quat(); - float _estimatedEyePitch = 0.0f; - float _estimatedEyeYaw = 0.0f; - QVector _blendshapeCoefficients; - - float _relaxationStatus = 0.0f; // Between 0.0f and 1.0f - float _fadeCoefficient = 0.0f; // Between 0.0f and 1.0f - - void countFrame(); - -private slots: - void startFPSTimer(); - void finishFPSTimer(); - -private: - bool _isCalculatingFPS = false; - int _frameCount = 0; - - // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes - static const int _leftBlinkIndex = 0; - static const int _rightBlinkIndex = 1; - static const int _leftEyeOpenIndex = 8; - static const int _rightEyeOpenIndex = 9; - - // Brows - static const int _browDownLeftIndex = 14; - static const int _browDownRightIndex = 15; - static const int _browUpCenterIndex = 16; - static const int _browUpLeftIndex = 17; - static const int _browUpRightIndex = 18; - - static const int _mouthSmileLeftIndex = 28; - static const int _mouthSmileRightIndex = 29; - - static const int _jawOpenIndex = 21; - - static Setting::Handle _eyeDeflection; -}; - -#endif // hifi_FaceTracker_h diff --git a/libraries/trackers/src/trackers/Logging.cpp b/libraries/trackers/src/trackers/Logging.cpp deleted file mode 100644 index a4dcf1b711..0000000000 --- a/libraries/trackers/src/trackers/Logging.cpp +++ /dev/null @@ -1,11 +0,0 @@ -// -// Created by Bradley Austin Davis on 2017/04/25 -// Copyright 2013-2017 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "Logging.h" - -Q_LOGGING_CATEGORY(trackers, "hifi.trackers") diff --git a/libraries/trackers/src/trackers/Logging.h b/libraries/trackers/src/trackers/Logging.h deleted file mode 100644 index 554429b61d..0000000000 --- a/libraries/trackers/src/trackers/Logging.h +++ /dev/null @@ -1,16 +0,0 @@ -// -// Created by Bradley Austin Davis on 2017/04/25 -// Copyright 2013-2017 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_TrackersLogging_h -#define hifi_TrackersLogging_h - -#include - -Q_DECLARE_LOGGING_CATEGORY(trackers) - -#endif // hifi_TrackersLogging_h diff --git a/tools/jsdoc/plugins/hifi.js b/tools/jsdoc/plugins/hifi.js index 07549530ce..8b18d02458 100644 --- a/tools/jsdoc/plugins/hifi.js +++ b/tools/jsdoc/plugins/hifi.js @@ -29,7 +29,6 @@ exports.handlers = { '../../interface/src/audio', '../../interface/src/avatar', '../../interface/src/commerce', - '../../interface/src/devices', '../../interface/src/java', '../../interface/src/networking', '../../interface/src/raypick', @@ -64,7 +63,6 @@ exports.handlers = { '../../libraries/shared/src', '../../libraries/shared/src/shared', '../../libraries/task/src/task', - '../../libraries/trackers/src/trackers', '../../libraries/ui/src', '../../libraries/ui/src/ui', '../../plugins/oculus/src',