From 28d3eae12b08eed5655a3419d3eace5036d068da Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 9 Apr 2014 14:52:27 -0700 Subject: [PATCH] Move Faceplus reading to its own thread to avoid blocking the main one. --- interface/src/Application.cpp | 11 -- interface/src/Application.h | 1 - interface/src/devices/Faceplus.cpp | 173 ++++++++++++++++++----------- interface/src/devices/Faceplus.h | 24 +++- 4 files changed, 127 insertions(+), 82 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index dedbbced30..6e05167d6f 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1689,15 +1689,6 @@ void Application::updateMouseRay() { _myAvatar->setMouseRay(_mouseRayOrigin, _mouseRayDirection); } -void Application::updateFaceplus() { - - bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings); - PerformanceWarning warn(showWarnings, "Application::updateFaceplus()"); - - // Update faceplus - _faceplus.update(); -} - void Application::updateFaceshift() { bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings); @@ -1894,7 +1885,6 @@ void Application::update(float deltaTime) { // check what's under the mouse and update the mouse voxel updateMouseRay(); - updateFaceplus(); updateFaceshift(); updateVisage(); _myAvatar->updateLookAtTargetAvatar(); @@ -2945,7 +2935,6 @@ void Application::resetSensors() { _mouseX = _glWidget->width() / 2; _mouseY = _glWidget->height() / 2; - _faceplus.reset(); _faceshift.reset(); _visage.reset(); diff --git a/interface/src/Application.h b/interface/src/Application.h index c6b38b1644..7182010050 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -174,7 +174,6 @@ public: bool isMouseHidden() const { return _mouseHidden; } const glm::vec3& getMouseRayOrigin() const { return _mouseRayOrigin; } const glm::vec3& getMouseRayDirection() const { return _mouseRayDirection; } - Faceplus* getFaceplus() { return &_faceplus; } Faceshift* getFaceshift() { return &_faceshift; } Visage* getVisage() { return &_visage; } FaceTracker* getActiveFaceTracker(); diff --git a/interface/src/devices/Faceplus.cpp b/interface/src/devices/Faceplus.cpp index 9a3067e48f..34547c2a6a 100644 --- a/interface/src/devices/Faceplus.cpp +++ b/interface/src/devices/Faceplus.cpp @@ -6,6 +6,8 @@ // Copyright (c) 2014 High Fidelity, Inc. All rights reserved. // +#include + #ifdef HAVE_FACEPLUS #include #endif @@ -33,6 +35,41 @@ void Faceplus::init() { updateEnabled(); } +void Faceplus::setState(const glm::quat& headRotation, float estimatedEyePitch, float estimatedEyeYaw, + const QVector& blendshapeCoefficients) { + _headRotation = headRotation; + _estimatedEyePitch = estimatedEyePitch; + _estimatedEyeYaw = estimatedEyeYaw; + _blendshapeCoefficients = blendshapeCoefficients; + _active = true; +} + +void Faceplus::updateEnabled() { + setEnabled(Menu::getInstance()->isOptionChecked(MenuOption::Faceplus) && + !(Menu::getInstance()->isOptionChecked(MenuOption::Faceshift) && + Application::getInstance()->getFaceshift()->isConnectedOrConnecting())); +} + +void Faceplus::setEnabled(bool enabled) { + if (_enabled == enabled) { + return; + } + if ((_enabled = enabled)) { + _reader = new FaceplusReader(); + QThread* readerThread = new QThread(this); + _reader->moveToThread(readerThread); + readerThread->start(); + QMetaObject::invokeMethod(_reader, "init"); + + } else { + QThread* readerThread = _reader->thread(); + QMetaObject::invokeMethod(_reader, "shutdown"); + readerThread->wait(); + delete readerThread; + _active = false; + } +} + #ifdef HAVE_FACEPLUS static QMultiHash > createChannelNameMap() { QMultiHash > blendshapeMap; @@ -97,18 +134,76 @@ static const QMultiHash >& getChannelNameMap() { } #endif -void Faceplus::update() { +void FaceplusReader::init() { #ifdef HAVE_FACEPLUS - if (!_active) { + if (!faceplus_init("VGA")) { + qDebug() << "Failed to initialized Faceplus."; return; } - if (!(_active = faceplus_synchronous_track() && faceplus_current_output_vector(_outputVector.data()))) { + qDebug() << "Faceplus initialized."; + + int channelCount = faceplus_output_channels_count(); + _outputVector.resize(channelCount); + + int maxIndex = -1; + _channelIndexMap.clear(); + for (int i = 0; i < channelCount; i++) { + QByteArray name = faceplus_output_channel_name(i); + if (name == "Head_Joint::Rotation_X") { + _headRotationIndices[0] = i; + + } else if (name == "Head_Joint::Rotation_Y") { + _headRotationIndices[1] = i; + + } else if (name == "Head_Joint::Rotation_Z") { + _headRotationIndices[2] = i; + + } else if (name == "Left_Eye_Joint::Rotation_X") { + _leftEyeRotationIndices[0] = i; + + } else if (name == "Left_Eye_Joint::Rotation_Y") { + _leftEyeRotationIndices[1] = i; + + } else if (name == "Right_Eye_Joint::Rotation_X") { + _rightEyeRotationIndices[0] = i; + + } else if (name == "Right_Eye_Joint::Rotation_Y") { + _rightEyeRotationIndices[1] = i; + } + for (QMultiHash >::const_iterator it = getChannelNameMap().constFind(name); + it != getChannelNameMap().constEnd() && it.key() == name; it++) { + _channelIndexMap.insert(i, it.value()); + maxIndex = qMax(maxIndex, it.value().first); + } + } + _blendshapeCoefficients.resize(maxIndex + 1); + + QMetaObject::invokeMethod(this, "update", Qt::QueuedConnection); +#endif +} + +void FaceplusReader::shutdown() { +#ifdef HAVE_FACEPLUS + if (faceplus_teardown()) { + qDebug() << "Faceplus torn down."; + } +#endif + deleteLater(); + thread()->quit(); +} + +void FaceplusReader::update() { +#ifdef HAVE_FACEPLUS + if (!(faceplus_synchronous_track() && faceplus_current_output_vector(_outputVector.data()))) { + QMetaObject::invokeMethod(this, "update", Qt::QueuedConnection); return; } - _headRotation = glm::quat(glm::radians(glm::vec3(-_outputVector.at(_headRotationIndices[0]), + glm::quat headRotation(glm::radians(glm::vec3(-_outputVector.at(_headRotationIndices[0]), _outputVector.at(_headRotationIndices[1]), -_outputVector.at(_headRotationIndices[2])))); - _estimatedEyePitch = (_outputVector.at(_leftEyeRotationIndices[0]) + _outputVector.at(_rightEyeRotationIndices[0])) * -0.5f; - _estimatedEyeYaw = (_outputVector.at(_leftEyeRotationIndices[1]) + _outputVector.at(_rightEyeRotationIndices[1])) * 0.5f; + float estimatedEyePitch = (_outputVector.at(_leftEyeRotationIndices[0]) + + _outputVector.at(_rightEyeRotationIndices[0])) * -0.5f; + float estimatedEyeYaw = (_outputVector.at(_leftEyeRotationIndices[1]) + + _outputVector.at(_rightEyeRotationIndices[1])) * 0.5f; qFill(_blendshapeCoefficients.begin(), _blendshapeCoefficients.end(), 0.0f); for (int i = 0; i < _outputVector.size(); i++) { @@ -117,67 +212,11 @@ void Faceplus::update() { _blendshapeCoefficients[it.value().first] += _outputVector.at(i) * it.value().second; } } + + QMetaObject::invokeMethod(Application::getInstance()->getFaceplus(), "setState", Q_ARG(const glm::quat&, headRotation), + Q_ARG(float, estimatedEyePitch), Q_ARG(float, estimatedEyeYaw), Q_ARG(const QVector&, _blendshapeCoefficients)); + + QMetaObject::invokeMethod(this, "update", Qt::QueuedConnection); #endif } -void Faceplus::reset() { -} - -void Faceplus::updateEnabled() { - setEnabled(Menu::getInstance()->isOptionChecked(MenuOption::Faceplus) && - !(Menu::getInstance()->isOptionChecked(MenuOption::Faceshift) && - Application::getInstance()->getFaceshift()->isConnectedOrConnecting())); -} - -void Faceplus::setEnabled(bool enabled) { -#ifdef HAVE_FACEPLUS - if (_enabled == enabled) { - return; - } - if ((_enabled = enabled)) { - if (faceplus_init("VGA")) { - qDebug() << "Faceplus initialized."; - _active = true; - - int channelCount = faceplus_output_channels_count(); - _outputVector.resize(channelCount); - - int maxIndex = -1; - _channelIndexMap.clear(); - for (int i = 0; i < channelCount; i++) { - QByteArray name = faceplus_output_channel_name(i); - if (name == "Head_Joint::Rotation_X") { - _headRotationIndices[0] = i; - - } else if (name == "Head_Joint::Rotation_Y") { - _headRotationIndices[1] = i; - - } else if (name == "Head_Joint::Rotation_Z") { - _headRotationIndices[2] = i; - - } else if (name == "Left_Eye_Joint::Rotation_X") { - _leftEyeRotationIndices[0] = i; - - } else if (name == "Left_Eye_Joint::Rotation_Y") { - _leftEyeRotationIndices[1] = i; - - } else if (name == "Right_Eye_Joint::Rotation_X") { - _rightEyeRotationIndices[0] = i; - - } else if (name == "Right_Eye_Joint::Rotation_Y") { - _rightEyeRotationIndices[1] = i; - } - for (QMultiHash >::const_iterator it = getChannelNameMap().constFind(name); - it != getChannelNameMap().constEnd() && it.key() == name; it++) { - _channelIndexMap.insert(i, it.value()); - maxIndex = qMax(maxIndex, it.value().first); - } - } - _blendshapeCoefficients.resize(maxIndex + 1); - } - } else if (faceplus_teardown()) { - qDebug() << "Faceplus torn down."; - _active = false; - } -#endif -} diff --git a/interface/src/devices/Faceplus.h b/interface/src/devices/Faceplus.h index 164cf4d031..5c5f26e64d 100644 --- a/interface/src/devices/Faceplus.h +++ b/interface/src/devices/Faceplus.h @@ -15,6 +15,8 @@ #include "FaceTracker.h" +class FaceplusReader; + /// Interface for Mixamo FacePlus. class Faceplus : public FaceTracker { Q_OBJECT @@ -28,9 +30,9 @@ public: bool isActive() const { return _active; } - void update(); - void reset(); - + Q_INVOKABLE void setState(const glm::quat& headRotation, float estimatedEyePitch, float estimatedEyeYaw, + const QVector& blendshapeCoefficients); + public slots: void updateEnabled(); @@ -42,12 +44,28 @@ private: bool _enabled; bool _active; + FaceplusReader* _reader; +}; + +/// The reader object that lives in its own thread. +class FaceplusReader : public QObject { + Q_OBJECT + +public: + + Q_INVOKABLE void init(); + Q_INVOKABLE void shutdown(); + Q_INVOKABLE void update(); + +private: + #ifdef HAVE_FACEPLUS QMultiHash > _channelIndexMap; QVector _outputVector; int _headRotationIndices[3]; int _leftEyeRotationIndices[2]; int _rightEyeRotationIndices[2]; + QVector _blendshapeIndices; #endif };