From c56778c3bff79a611cf2d2667bc53f08b11e1ef0 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 10 Sep 2013 11:14:36 -0700 Subject: [PATCH 01/17] When devices become inactive (including when Faceshift reports loss of tracking), smoothly restore neutral head rotation/lean. --- interface/src/avatar/MyAvatar.cpp | 17 +++++++++++------ interface/src/devices/Faceshift.cpp | 3 ++- interface/src/devices/Faceshift.h | 3 ++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 4b7f64cb6f..abe9975a50 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -347,14 +347,19 @@ void MyAvatar::updateFromGyrosAndOrWebcam(bool gyroLook, } else if (webcam->isActive()) { estimatedRotation = webcam->getEstimatedRotation(); - } else if (_leadingAvatar) { - _head.getFace().clearFrame(); - return; - } else { - _head.setMousePitch(pitchFromTouch); - _head.setPitch(pitchFromTouch); + if (!_leadingAvatar) { + _head.setMousePitch(pitchFromTouch); + _head.setPitch(pitchFromTouch); + } _head.getFace().clearFrame(); + + // restore rotation, lean to neutral positions + const float RESTORE_RATE = 0.5f; + _head.setYaw(glm::mix(_head.getYaw(), 0.0f, RESTORE_RATE)); + _head.setRoll(glm::mix(_head.getRoll(), 0.0f, RESTORE_RATE)); + _head.setLeanSideways(glm::mix(_head.getLeanSideways(), 0.0f, RESTORE_RATE)); + _head.setLeanForward(glm::mix(_head.getLeanForward(), 0.0f, RESTORE_RATE)); return; } _head.setMousePitch(pitchFromTouch); diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 399369cdd3..991a406cc7 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -73,6 +73,7 @@ void Faceshift::connectSocket() { const quint16 FACESHIFT_PORT = 33433; _socket.connectToHost("localhost", FACESHIFT_PORT); + _tracking = false; } } @@ -102,7 +103,7 @@ void Faceshift::readFromSocket() { switch (msg->id()) { case fsMsg::MSG_OUT_TRACKING_STATE: { const fsTrackingData& data = static_cast(msg.get())->tracking_data(); - if (data.m_trackingSuccessful) { + if ((_tracking = data.m_trackingSuccessful)) { _headRotation = glm::quat(data.m_headRotation.w, -data.m_headRotation.x, data.m_headRotation.y, -data.m_headRotation.z); const float TRANSLATION_SCALE = 0.02f; diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 408513e5d7..586f336c04 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -24,7 +24,7 @@ public: Faceshift(); - bool isActive() const { return _socket.state() == QAbstractSocket::ConnectedState; } + bool isActive() const { return _socket.state() == QAbstractSocket::ConnectedState && _tracking; } const glm::quat& getHeadRotation() const { return _headRotation; } const glm::vec3& getHeadTranslation() const { return _headTranslation; } @@ -66,6 +66,7 @@ private: QTcpSocket _socket; fs::fsBinaryStream _stream; bool _enabled; + bool _tracking; glm::quat _headRotation; glm::vec3 _headTranslation; From db8575b59aa8b40bc2eea6126eaf3c27f9d7d17d Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 10 Sep 2013 11:25:30 -0700 Subject: [PATCH 02/17] Reduced the restore rate. --- interface/src/avatar/MyAvatar.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index abe9975a50..0dcfe32236 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -355,7 +355,7 @@ void MyAvatar::updateFromGyrosAndOrWebcam(bool gyroLook, _head.getFace().clearFrame(); // restore rotation, lean to neutral positions - const float RESTORE_RATE = 0.5f; + const float RESTORE_RATE = 0.05f; _head.setYaw(glm::mix(_head.getYaw(), 0.0f, RESTORE_RATE)); _head.setRoll(glm::mix(_head.getRoll(), 0.0f, RESTORE_RATE)); _head.setLeanSideways(glm::mix(_head.getLeanSideways(), 0.0f, RESTORE_RATE)); From 6614a13c64183bb20eb24703e47974450ac9bc9f Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 10 Sep 2013 11:58:45 -0700 Subject: [PATCH 03/17] Align the eyelids to the top of the iris. --- interface/src/avatar/Head.cpp | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index abf16248c2..6a9f61b7d4 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -661,21 +661,22 @@ void Head::renderEyeBalls() { glm::vec3 front = orientation * IDENTITY_FRONT; // render left iris + glm::quat leftIrisRotation; glPushMatrix(); { glTranslatef(_leftEyePosition.x, _leftEyePosition.y, _leftEyePosition.z); //translate to eyeball position //rotate the eyeball to aim towards the lookat position glm::vec3 targetLookatVector = _lookAtPosition + _saccade - _leftEyePosition; - glm::quat rotation = rotationBetween(front, targetLookatVector) * orientation; - glm::vec3 rotationAxis = glm::axis(rotation); - glRotatef(glm::angle(rotation), rotationAxis.x, rotationAxis.y, rotationAxis.z); + leftIrisRotation = rotationBetween(front, targetLookatVector) * orientation; + glm::vec3 rotationAxis = glm::axis(leftIrisRotation); + glRotatef(glm::angle(leftIrisRotation), rotationAxis.x, rotationAxis.y, rotationAxis.z); glTranslatef(0.0f, 0.0f, -_scale * IRIS_PROTRUSION); glScalef(_scale * IRIS_RADIUS * 2.0f, _scale * IRIS_RADIUS * 2.0f, _scale * IRIS_RADIUS); // flatten the iris // this ugliness is simply to invert the model transform and get the eye position in model space - _irisProgram.setUniform(_eyePositionLocation, (glm::inverse(rotation) * + _irisProgram.setUniform(_eyePositionLocation, (glm::inverse(leftIrisRotation) * (Application::getInstance()->getCamera()->getPosition() - _leftEyePosition) + glm::vec3(0.0f, 0.0f, _scale * IRIS_PROTRUSION)) * glm::vec3(1.0f / (_scale * IRIS_RADIUS * 2.0f), 1.0f / (_scale * IRIS_RADIUS * 2.0f), 1.0f / (_scale * IRIS_RADIUS))); @@ -685,21 +686,22 @@ void Head::renderEyeBalls() { glPopMatrix(); // render right iris + glm::quat rightIrisRotation; glPushMatrix(); { glTranslatef(_rightEyePosition.x, _rightEyePosition.y, _rightEyePosition.z); //translate to eyeball position //rotate the eyeball to aim towards the lookat position glm::vec3 targetLookatVector = _lookAtPosition + _saccade - _rightEyePosition; - glm::quat rotation = rotationBetween(front, targetLookatVector) * orientation; - glm::vec3 rotationAxis = glm::axis(rotation); - glRotatef(glm::angle(rotation), rotationAxis.x, rotationAxis.y, rotationAxis.z); + rightIrisRotation = rotationBetween(front, targetLookatVector) * orientation; + glm::vec3 rotationAxis = glm::axis(rightIrisRotation); + glRotatef(glm::angle(rightIrisRotation), rotationAxis.x, rotationAxis.y, rotationAxis.z); glTranslatef(0.0f, 0.0f, -_scale * IRIS_PROTRUSION); glScalef(_scale * IRIS_RADIUS * 2.0f, _scale * IRIS_RADIUS * 2.0f, _scale * IRIS_RADIUS); // flatten the iris // this ugliness is simply to invert the model transform and get the eye position in model space - _irisProgram.setUniform(_eyePositionLocation, (glm::inverse(rotation) * + _irisProgram.setUniform(_eyePositionLocation, (glm::inverse(rightIrisRotation) * (Application::getInstance()->getCamera()->getPosition() - _rightEyePosition) + glm::vec3(0.0f, 0.0f, _scale * IRIS_PROTRUSION)) * glm::vec3(1.0f / (_scale * IRIS_RADIUS * 2.0f), 1.0f / (_scale * IRIS_RADIUS * 2.0f), 1.0f / (_scale * IRIS_RADIUS))); @@ -718,10 +720,10 @@ void Head::renderEyeBalls() { // left eyelid glPushMatrix(); { glTranslatef(_leftEyePosition.x, _leftEyePosition.y, _leftEyePosition.z); //translate to eyeball position - glm::vec3 rotationAxis = glm::axis(orientation); - glRotatef(glm::angle(orientation), rotationAxis.x, rotationAxis.y, rotationAxis.z); + glm::vec3 rotationAxis = glm::axis(leftIrisRotation); + glRotatef(glm::angle(leftIrisRotation), rotationAxis.x, rotationAxis.y, rotationAxis.z); glScalef(_scale * EYELID_RADIUS, _scale * EYELID_RADIUS, _scale * EYELID_RADIUS); - glRotatef(-40 - 50 * _leftEyeBlink, 1, 0, 0); + glRotatef(-70 - 50 * _leftEyeBlink, 1, 0, 0); Application::getInstance()->getGeometryCache()->renderHemisphere(15, 10); glRotatef(180 * _leftEyeBlink, 1, 0, 0); Application::getInstance()->getGeometryCache()->renderHemisphere(15, 10); @@ -731,10 +733,10 @@ void Head::renderEyeBalls() { // right eyelid glPushMatrix(); { glTranslatef(_rightEyePosition.x, _rightEyePosition.y, _rightEyePosition.z); //translate to eyeball position - glm::vec3 rotationAxis = glm::axis(orientation); - glRotatef(glm::angle(orientation), rotationAxis.x, rotationAxis.y, rotationAxis.z); + glm::vec3 rotationAxis = glm::axis(rightIrisRotation); + glRotatef(glm::angle(rightIrisRotation), rotationAxis.x, rotationAxis.y, rotationAxis.z); glScalef(_scale * EYELID_RADIUS, _scale * EYELID_RADIUS, _scale * EYELID_RADIUS); - glRotatef(-40 - 50 * _rightEyeBlink, 1, 0, 0); + glRotatef(-70 - 50 * _rightEyeBlink, 1, 0, 0); Application::getInstance()->getGeometryCache()->renderHemisphere(15, 10); glRotatef(180 * _rightEyeBlink, 1, 0, 0); Application::getInstance()->getGeometryCache()->renderHemisphere(15, 10); From cea645dfc7fd5253712290349846c53ad709d0c2 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 10 Sep 2013 12:34:41 -0700 Subject: [PATCH 04/17] Tweak to blink angles. --- interface/src/avatar/Head.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index 6a9f61b7d4..517d062494 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -723,9 +723,10 @@ void Head::renderEyeBalls() { glm::vec3 rotationAxis = glm::axis(leftIrisRotation); glRotatef(glm::angle(leftIrisRotation), rotationAxis.x, rotationAxis.y, rotationAxis.z); glScalef(_scale * EYELID_RADIUS, _scale * EYELID_RADIUS, _scale * EYELID_RADIUS); - glRotatef(-70 - 50 * _leftEyeBlink, 1, 0, 0); + float angle = -67.5f - 50.0f * _leftEyeBlink; + glRotatef(angle, 1, 0, 0); Application::getInstance()->getGeometryCache()->renderHemisphere(15, 10); - glRotatef(180 * _leftEyeBlink, 1, 0, 0); + glRotatef(glm::mix(-angle, 180.0f, _leftEyeBlink), 1, 0, 0); Application::getInstance()->getGeometryCache()->renderHemisphere(15, 10); } glPopMatrix(); @@ -736,9 +737,10 @@ void Head::renderEyeBalls() { glm::vec3 rotationAxis = glm::axis(rightIrisRotation); glRotatef(glm::angle(rightIrisRotation), rotationAxis.x, rotationAxis.y, rotationAxis.z); glScalef(_scale * EYELID_RADIUS, _scale * EYELID_RADIUS, _scale * EYELID_RADIUS); - glRotatef(-70 - 50 * _rightEyeBlink, 1, 0, 0); + float angle = -67.5f - 50.0f * _rightEyeBlink; + glRotatef(angle, 1, 0, 0); Application::getInstance()->getGeometryCache()->renderHemisphere(15, 10); - glRotatef(180 * _rightEyeBlink, 1, 0, 0); + glRotatef(glm::mix(-angle, 180.0f, _rightEyeBlink), 1, 0, 0); Application::getInstance()->getGeometryCache()->renderHemisphere(15, 10); } glPopMatrix(); From 5c7913068e6aa13bc8e937ac60c252c1fbd8a6d9 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 10 Sep 2013 16:21:08 -0700 Subject: [PATCH 05/17] Support for UDP input from Faceshift. --- interface/src/devices/Faceshift.cpp | 35 ++++++++++++++++++++--------- interface/src/devices/Faceshift.h | 7 ++++-- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 991a406cc7..66ba958bbd 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -8,13 +8,18 @@ #include +#include + #include "Faceshift.h" using namespace fs; using namespace std; +const quint16 FACESHIFT_PORT = 33433; + Faceshift::Faceshift() : _enabled(false), + _lastMessageReceived(0), _eyeGazeLeftPitch(0.0f), _eyeGazeLeftYaw(0.0f), _eyeGazeRightPitch(0.0f), @@ -32,9 +37,19 @@ Faceshift::Faceshift() : _estimatedEyePitch(0.0f), _estimatedEyeYaw(0.0f) { - connect(&_socket, SIGNAL(connected()), SLOT(noteConnected())); - connect(&_socket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(noteError(QAbstractSocket::SocketError))); - connect(&_socket, SIGNAL(readyRead()), SLOT(readFromSocket())); + connect(&_tcpSocket, SIGNAL(connected()), SLOT(noteConnected())); + connect(&_tcpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(noteError(QAbstractSocket::SocketError))); + connect(&_tcpSocket, SIGNAL(readyRead()), SLOT(readFromSocket())); + + connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readFromSocket())); + + _udpSocket.bind(FACESHIFT_PORT); +} + +bool Faceshift::isActive() const { + const uint64_t ACTIVE_TIMEOUT_USECS = 1000000; + return (_tcpSocket.state() == QAbstractSocket::ConnectedState || + (usecTimestampNow() - _lastMessageReceived) < ACTIVE_TIMEOUT_USECS) && _tracking; } void Faceshift::update() { @@ -51,7 +66,7 @@ void Faceshift::update() { } void Faceshift::reset() { - if (isActive()) { + if (_tcpSocket.state() == QAbstractSocket::ConnectedState) { string message; fsBinaryStream::encode_message(message, fsMsgCalibrateNeutral()); send(message); @@ -63,7 +78,7 @@ void Faceshift::setEnabled(bool enabled) { connectSocket(); } else { - _socket.disconnectFromHost(); + _tcpSocket.disconnectFromHost(); } } @@ -71,8 +86,7 @@ void Faceshift::connectSocket() { if (_enabled) { qDebug("Faceshift: Connecting...\n"); - const quint16 FACESHIFT_PORT = 33433; - _socket.connectToHost("localhost", FACESHIFT_PORT); + _tcpSocket.connectToHost("localhost", FACESHIFT_PORT); _tracking = false; } } @@ -87,7 +101,7 @@ void Faceshift::noteConnected() { } void Faceshift::noteError(QAbstractSocket::SocketError error) { - qDebug() << "Faceshift: " << _socket.errorString() << "\n"; + qDebug() << "Faceshift: " << _tcpSocket.errorString() << "\n"; // reconnect after a delay if (_enabled) { @@ -96,7 +110,7 @@ void Faceshift::noteError(QAbstractSocket::SocketError error) { } void Faceshift::readFromSocket() { - QByteArray buffer = _socket.readAll(); + QByteArray buffer = static_cast(sender())->readAll(); _stream.received(buffer.size(), buffer.constData()); fsMsgPtr msg; for (fsMsgPtr msg; (msg = _stream.get_message()); ) { @@ -151,8 +165,9 @@ void Faceshift::readFromSocket() { break; } } + _lastMessageReceived = usecTimestampNow(); } void Faceshift::send(const std::string& message) { - _socket.write(message.data(), message.size()); + _tcpSocket.write(message.data(), message.size()); } diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 586f336c04..f93bbba2b4 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -10,6 +10,7 @@ #define __interface__Faceshift__ #include +#include #include #include @@ -24,7 +25,7 @@ public: Faceshift(); - bool isActive() const { return _socket.state() == QAbstractSocket::ConnectedState && _tracking; } + bool isActive() const; const glm::quat& getHeadRotation() const { return _headRotation; } const glm::vec3& getHeadTranslation() const { return _headTranslation; } @@ -63,10 +64,12 @@ private: void send(const std::string& message); - QTcpSocket _socket; + QTcpSocket _tcpSocket; + QUdpSocket _udpSocket; fs::fsBinaryStream _stream; bool _enabled; bool _tracking; + uint64_t _lastMessageReceived; glm::quat _headRotation; glm::vec3 _headTranslation; From b9878bbbd68228e44d8cf1e2b7d0274c0e7cd11d Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 10 Sep 2013 16:55:59 -0700 Subject: [PATCH 06/17] Adjustment to UDP code. --- interface/src/devices/Faceshift.cpp | 32 ++++++++++++++++++++--------- interface/src/devices/Faceshift.h | 2 ++ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 66ba958bbd..0df7ac79c4 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -26,12 +26,12 @@ Faceshift::Faceshift() : _eyeGazeRightYaw(0.0f), _leftBlink(0.0f), _rightBlink(0.0f), - _leftBlinkIndex(-1), - _rightBlinkIndex(-1), + _leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes + _rightBlinkIndex(1), _browHeight(0.0f), - _browUpCenterIndex(-1), + _browUpCenterIndex(16), _mouthSize(0.0f), - _jawOpenIndex(-1), + _jawOpenIndex(21), _longTermAverageEyePitch(0.0f), _longTermAverageEyeYaw(0.0f), _estimatedEyePitch(0.0f), @@ -41,7 +41,7 @@ Faceshift::Faceshift() : connect(&_tcpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(noteError(QAbstractSocket::SocketError))); connect(&_tcpSocket, SIGNAL(readyRead()), SLOT(readFromSocket())); - connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readFromSocket())); + connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams())); _udpSocket.bind(FACESHIFT_PORT); } @@ -109,8 +109,24 @@ void Faceshift::noteError(QAbstractSocket::SocketError error) { } } +void Faceshift::readPendingDatagrams() { + QByteArray buffer; + while (_udpSocket.hasPendingDatagrams()) { + buffer.resize(_udpSocket.pendingDatagramSize()); + _udpSocket.readDatagram(buffer.data(), buffer.size()); + receive(buffer); + } +} + void Faceshift::readFromSocket() { - QByteArray buffer = static_cast(sender())->readAll(); + receive(_tcpSocket.readAll()); +} + +void Faceshift::send(const std::string& message) { + _tcpSocket.write(message.data(), message.size()); +} + +void Faceshift::receive(const QByteArray& buffer) { _stream.received(buffer.size(), buffer.constData()); fsMsgPtr msg; for (fsMsgPtr msg; (msg = _stream.get_message()); ) { @@ -167,7 +183,3 @@ void Faceshift::readFromSocket() { } _lastMessageReceived = usecTimestampNow(); } - -void Faceshift::send(const std::string& message) { - _tcpSocket.write(message.data(), message.size()); -} diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index f93bbba2b4..f8ec6f122d 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -58,11 +58,13 @@ private slots: void connectSocket(); void noteConnected(); void noteError(QAbstractSocket::SocketError error); + void readPendingDatagrams(); void readFromSocket(); private: void send(const std::string& message); + void receive(const QByteArray& buffer); QTcpSocket _tcpSocket; QUdpSocket _udpSocket; From 4b19e3d0fe348b2a09e5863ba186e6f8a8c8c474 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 10 Sep 2013 17:43:06 -0700 Subject: [PATCH 07/17] Incorporate the "eye open" coefficients for eye widening. --- interface/src/devices/Faceshift.cpp | 15 ++++++++++++--- interface/src/devices/Faceshift.h | 3 +++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 0df7ac79c4..5da06ae912 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -28,6 +28,8 @@ Faceshift::Faceshift() : _rightBlink(0.0f), _leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes _rightBlinkIndex(1), + _leftEyeOpenIndex(8), + _rightEyeOpenIndex(9), _browHeight(0.0f), _browUpCenterIndex(16), _mouthSize(0.0f), @@ -144,11 +146,12 @@ void Faceshift::receive(const QByteArray& buffer) { _eyeGazeRightPitch = -data.m_eyeGazeRightPitch; _eyeGazeRightYaw = data.m_eyeGazeRightYaw; + const float EYE_OPEN_SCALE = 0.25f; if (_leftBlinkIndex != -1) { - _leftBlink = data.m_coeffs[_leftBlinkIndex]; + _leftBlink = data.m_coeffs[_leftBlinkIndex] - data.m_coeffs[_leftEyeOpenIndex] * EYE_OPEN_SCALE; } if (_rightBlinkIndex != -1) { - _rightBlink = data.m_coeffs[_rightBlinkIndex]; + _rightBlink = data.m_coeffs[_rightBlinkIndex] - data.m_coeffs[_rightEyeOpenIndex] * EYE_OPEN_SCALE; } if (_browUpCenterIndex != -1) { _browHeight = data.m_coeffs[_browUpCenterIndex]; @@ -167,7 +170,13 @@ void Faceshift::receive(const QByteArray& buffer) { } else if (names[i] == "EyeBlink_R") { _rightBlinkIndex = i; - + + } else if (names[i] == "EyeOpen_L") { + _leftEyeOpenIndex = i; + + } else if (names[i] == "EyeOpen_R") { + _rightEyeOpenIndex = i; + } else if (names[i] == "BrowsU_C") { _browUpCenterIndex = i; diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index f8ec6f122d..1bdb0cde59 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -88,6 +88,9 @@ private: int _leftBlinkIndex; int _rightBlinkIndex; + int _leftEyeOpenIndex; + int _rightEyeOpenIndex; + float _browHeight; int _browUpCenterIndex; From feecb4fa25daf267ae81068aed23951c04f7ada2 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 11 Sep 2013 10:44:38 -0700 Subject: [PATCH 08/17] Fix for eyelids, change Faceshift menu option to indicate that it applies specifically to TCP connection. --- interface/src/Menu.cpp | 4 ++-- interface/src/Menu.h | 2 +- interface/src/avatar/Head.cpp | 4 ++-- interface/src/devices/Faceshift.cpp | 10 +++++----- interface/src/devices/Faceshift.h | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 1d9e12cc96..7e9dfe2355 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -370,11 +370,11 @@ Menu::Menu() : SLOT(setDepthOnly(bool))); addCheckableActionToQMenuAndActionHash(developerMenu, - MenuOption::Faceshift, + MenuOption::FaceshiftTCP, 0, false, appInstance->getFaceshift(), - SLOT(setEnabled(bool))); + SLOT(setTCPEnabled(bool))); QMenu* audioDebugMenu = developerMenu->addMenu("Audio Debugging Tools"); addCheckableActionToQMenuAndActionHash(audioDebugMenu, MenuOption::EchoAudio); diff --git a/interface/src/Menu.h b/interface/src/Menu.h index 934cb1b7ae..2f486da031 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -137,7 +137,7 @@ namespace MenuOption { const QString ExportVoxels = "Export Voxels"; const QString HeadMouse = "Head Mouse"; const QString FaceMode = "Cycle Face Mode"; - const QString Faceshift = "Faceshift"; + const QString FaceshiftTCP = "Faceshift (TCP)"; const QString FalseColorByDistance = "FALSE Color By Distance"; const QString FalseColorBySource = "FALSE Color By Source"; const QString FalseColorEveryOtherVoxel = "FALSE Color Every Other Randomly"; diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index 517d062494..b439d56568 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -726,7 +726,7 @@ void Head::renderEyeBalls() { float angle = -67.5f - 50.0f * _leftEyeBlink; glRotatef(angle, 1, 0, 0); Application::getInstance()->getGeometryCache()->renderHemisphere(15, 10); - glRotatef(glm::mix(-angle, 180.0f, _leftEyeBlink), 1, 0, 0); + glRotatef(glm::mix(-angle, 180.0f, max(0.0f, _leftEyeBlink)), 1, 0, 0); Application::getInstance()->getGeometryCache()->renderHemisphere(15, 10); } glPopMatrix(); @@ -740,7 +740,7 @@ void Head::renderEyeBalls() { float angle = -67.5f - 50.0f * _rightEyeBlink; glRotatef(angle, 1, 0, 0); Application::getInstance()->getGeometryCache()->renderHemisphere(15, 10); - glRotatef(glm::mix(-angle, 180.0f, _rightEyeBlink), 1, 0, 0); + glRotatef(glm::mix(-angle, 180.0f, max(0.0f, _rightEyeBlink)), 1, 0, 0); Application::getInstance()->getGeometryCache()->renderHemisphere(15, 10); } glPopMatrix(); diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 5da06ae912..4bcab43cf3 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -18,7 +18,7 @@ using namespace std; const quint16 FACESHIFT_PORT = 33433; Faceshift::Faceshift() : - _enabled(false), + _tcpEnabled(false), _lastMessageReceived(0), _eyeGazeLeftPitch(0.0f), _eyeGazeLeftYaw(0.0f), @@ -75,8 +75,8 @@ void Faceshift::reset() { } } -void Faceshift::setEnabled(bool enabled) { - if ((_enabled = enabled)) { +void Faceshift::setTCPEnabled(bool enabled) { + if ((_tcpEnabled = enabled)) { connectSocket(); } else { @@ -85,7 +85,7 @@ void Faceshift::setEnabled(bool enabled) { } void Faceshift::connectSocket() { - if (_enabled) { + if (_tcpEnabled) { qDebug("Faceshift: Connecting...\n"); _tcpSocket.connectToHost("localhost", FACESHIFT_PORT); @@ -106,7 +106,7 @@ void Faceshift::noteError(QAbstractSocket::SocketError error) { qDebug() << "Faceshift: " << _tcpSocket.errorString() << "\n"; // reconnect after a delay - if (_enabled) { + if (_tcpEnabled) { QTimer::singleShot(1000, this, SLOT(connectSocket())); } } diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 1bdb0cde59..29a25f9c4c 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -51,7 +51,7 @@ public: public slots: - void setEnabled(bool enabled); + void setTCPEnabled(bool enabled); private slots: @@ -69,7 +69,7 @@ private: QTcpSocket _tcpSocket; QUdpSocket _udpSocket; fs::fsBinaryStream _stream; - bool _enabled; + bool _tcpEnabled; bool _tracking; uint64_t _lastMessageReceived; From 80cc278bfaf7a1acee7d81a6b00ae4b24504cb26 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 11 Sep 2013 10:49:12 -0700 Subject: [PATCH 09/17] Increase the eye open scale. --- interface/src/devices/Faceshift.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 4bcab43cf3..bb4538b0e8 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -146,7 +146,7 @@ void Faceshift::receive(const QByteArray& buffer) { _eyeGazeRightPitch = -data.m_eyeGazeRightPitch; _eyeGazeRightYaw = data.m_eyeGazeRightYaw; - const float EYE_OPEN_SCALE = 0.25f; + const float EYE_OPEN_SCALE = 0.5f; if (_leftBlinkIndex != -1) { _leftBlink = data.m_coeffs[_leftBlinkIndex] - data.m_coeffs[_leftEyeOpenIndex] * EYE_OPEN_SCALE; } From b7aa49cf97452744bfcb9ab08d61389f65afdda9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 11 Sep 2013 12:16:47 -0700 Subject: [PATCH 10/17] have DS act as AS and hand assignments directly --- assignment-client/src/main.cpp | 47 +++-- domain-server/src/main.cpp | 285 ++++++++++++++++------------ libraries/shared/src/Assignment.cpp | 45 +---- libraries/shared/src/Assignment.h | 25 ++- libraries/shared/src/NodeList.cpp | 2 +- 5 files changed, 220 insertions(+), 184 deletions(-) diff --git a/assignment-client/src/main.cpp b/assignment-client/src/main.cpp index 96652318ed..564d827e0e 100644 --- a/assignment-client/src/main.cpp +++ b/assignment-client/src/main.cpp @@ -26,7 +26,6 @@ const char CHILD_TARGET_NAME[] = "assignment-client"; pid_t* childForks = NULL; sockaddr_in customAssignmentSocket = {}; -const char* assignmentPool = NULL; int numForks = 0; void childClient() { @@ -51,8 +50,10 @@ void childClient() { unsigned char packetData[MAX_PACKET_SIZE]; ssize_t receivedBytes = 0; + sockaddr_in senderSocket = {}; + // create a request assignment, accept all assignments, pass the desired pool (if it exists) - Assignment requestAssignment(Assignment::Request, Assignment::All, assignmentPool); + Assignment requestAssignment(Assignment::RequestDirection, Assignment::AllTypes); while (true) { if (usecTimestampNow() - usecTimestamp(&lastRequest) >= ASSIGNMENT_REQUEST_INTERVAL_USECS) { @@ -61,22 +62,34 @@ void childClient() { nodeList->sendAssignment(requestAssignment); } - if (nodeList->getNodeSocket()->receive(packetData, &receivedBytes) && - packetData[0] == PACKET_TYPE_DEPLOY_ASSIGNMENT && packetVersionMatch(packetData)) { + if (nodeList->getNodeSocket()->receive((sockaddr*) &senderSocket, packetData, &receivedBytes) && + (packetData[0] == PACKET_TYPE_DEPLOY_ASSIGNMENT || packetData[0] == PACKET_TYPE_CREATE_ASSIGNMENT) + && packetVersionMatch(packetData)) { // construct the deployed assignment from the packet data Assignment deployedAssignment(packetData, receivedBytes); qDebug() << "Received an assignment -" << deployedAssignment << "\n"; - // switch our nodelist DOMAIN_IP to the ip receieved in the assignment - if (deployedAssignment.getAttachedPublicSocket()->sa_family == AF_INET) { - in_addr domainSocketAddr = ((sockaddr_in*) deployedAssignment.getAttachedPublicSocket())->sin_addr; + // switch our nodelist DOMAIN_IP + if (packetData[0] == PACKET_TYPE_CREATE_ASSIGNMENT || + deployedAssignment.getAttachedPublicSocket()->sa_family == AF_INET) { + + in_addr domainSocketAddr = {}; + + if (packetData[0] == PACKET_TYPE_CREATE_ASSIGNMENT) { + // the domain server IP address is the address we got this packet from + domainSocketAddr = senderSocket.sin_addr; + } else { + // grab the domain server IP address from the packet from the AS + domainSocketAddr = ((sockaddr_in*) deployedAssignment.getAttachedPublicSocket())->sin_addr; + } + nodeList->setDomainIP(inet_ntoa(domainSocketAddr)); qDebug("Destination IP for assignment is %s\n", inet_ntoa(domainSocketAddr)); - if (deployedAssignment.getType() == Assignment::AudioMixer) { + if (deployedAssignment.getType() == Assignment::AudioMixerType) { AudioMixer::run(); } else { AvatarMixer::run(); @@ -165,19 +178,23 @@ int main(int argc, const char* argv[]) { // start the Logging class with the parent's target name Logging::setTargetName(PARENT_TARGET_NAME); + const char CUSTOM_ASSIGNMENT_SERVER_HOSTNAME_OPTION[] = "-a"; + const char CUSTOM_ASSIGNMENT_SERVER_PORT_OPTION[] = "-p"; + // grab the overriden assignment-server hostname from argv, if it exists - const char* customAssignmentServer = getCmdOption(argc, argv, "-a"); - if (customAssignmentServer) { - ::customAssignmentSocket = socketForHostnameAndHostOrderPort(customAssignmentServer, ASSIGNMENT_SERVER_PORT); + const char* customAssignmentServerHostname = getCmdOption(argc, argv, CUSTOM_ASSIGNMENT_SERVER_HOSTNAME_OPTION); + + if (customAssignmentServerHostname) { + const char* customAssignmentServerPortString = getCmdOption(argc, argv, CUSTOM_ASSIGNMENT_SERVER_PORT_OPTION); + unsigned short assignmentServerPort = customAssignmentServerPortString + ? atoi(customAssignmentServerPortString) : ASSIGNMENT_SERVER_PORT; + + ::customAssignmentSocket = socketForHostnameAndHostOrderPort(customAssignmentServerHostname, assignmentServerPort); } const char* NUM_FORKS_PARAMETER = "-n"; const char* numForksString = getCmdOption(argc, argv, NUM_FORKS_PARAMETER); - // grab the assignment pool from argv, if it was passed - const char* ASSIGNMENT_POOL_PARAMETER = "-p"; - ::assignmentPool = getCmdOption(argc, argv, ASSIGNMENT_POOL_PARAMETER); - int processID = 0; if (numForksString) { diff --git a/domain-server/src/main.cpp b/domain-server/src/main.cpp index 7b44648736..11ccbf6bb7 100644 --- a/domain-server/src/main.cpp +++ b/domain-server/src/main.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -50,6 +51,7 @@ unsigned char* addNodeToBroadcastPacket(unsigned char* currentPosition, Node* no int main(int argc, const char* argv[]) { NodeList* nodeList = NodeList::createInstance(NODE_TYPE_DOMAIN, DOMAIN_LISTEN_PORT); + // If user asks to run in "local" mode then we do NOT replace the IP // with the EC2 IP. Otherwise, we will replace the IP like we used to // this allows developers to run a local domain without recompiling the @@ -71,7 +73,6 @@ int main(int argc, const char* argv[]) { char nodeType = '\0'; unsigned char broadcastPacket[MAX_PACKET_SIZE]; - int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_DOMAIN); unsigned char* currentBufferPos; unsigned char* startPointer; @@ -84,13 +85,8 @@ int main(int argc, const char* argv[]) { nodeList->startSilentNodeRemovalThread(); timeval lastStatSendTime = {}; - - const char ASSIGNMENT_POOL_OPTION[] = "-p"; const char ASSIGNMENT_SERVER_OPTION[] = "-a"; - // set our assignment pool from argv, if it exists - const char* assignmentPool = getCmdOption(argc, argv, ASSIGNMENT_POOL_OPTION); - // grab the overriden assignment-server hostname from argv, if it exists const char* customAssignmentServer = getCmdOption(argc, argv, ASSIGNMENT_SERVER_OPTION); if (customAssignmentServer) { @@ -99,149 +95,197 @@ int main(int argc, const char* argv[]) { } // use a map to keep track of iterations of silence for assignment creation requests - const long long ASSIGNMENT_SILENCE_MAX_USECS = 5 * 1000 * 1000; + const long long GLOBAL_ASSIGNMENT_REQUEST_INTERVAL_USECS = 1 * 1000 * 1000; + timeval lastGlobalAssignmentRequest = {}; + + // setup the assignment queue + std::deque assignmentQueue; // as a domain-server we will always want an audio mixer and avatar mixer - // setup the create assignment pointers for those - Assignment* audioAssignment = NULL; - Assignment* avatarAssignment = NULL; + // setup the create assignments for those + Assignment audioMixerAssignment(Assignment::CreateDirection, + Assignment::AudioMixerType); - // construct a local socket to send with our created assignments + Assignment avatarMixerAssignment(Assignment::CreateDirection, + Assignment::AvatarMixerType, + Assignment::LocalLocation); + + // construct a local socket to send with our created assignments to the global AS sockaddr_in localSocket = {}; localSocket.sin_family = AF_INET; localSocket.sin_port = htons(nodeList->getInstance()->getNodeSocket()->getListeningPort()); localSocket.sin_addr.s_addr = serverLocalAddress; while (true) { - if (!nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER)) { - if (!audioAssignment - || usecTimestampNow() - usecTimestamp(&audioAssignment->getTime()) >= ASSIGNMENT_SILENCE_MAX_USECS) { - - if (!audioAssignment) { - audioAssignment = new Assignment(Assignment::Create, Assignment::AudioMixer, assignmentPool); - audioAssignment->setAttachedLocalSocket((sockaddr*) &localSocket); - } - - nodeList->sendAssignment(*audioAssignment); - audioAssignment->setCreateTimeToNow(); - } + + // check if our audio-mixer or avatar-mixer are dead and we don't have existing assignments in the queue + // so we can add those assignments back to the front of the queue since they are high-priority + if (!nodeList->soloNodeOfType(NODE_TYPE_AVATAR_MIXER) && + std::find(assignmentQueue.begin(), assignmentQueue.end(), &avatarMixerAssignment) == assignmentQueue.end()) { + qDebug("Missing an avatar mixer and assignment not in queue. Adding.\n"); + assignmentQueue.push_front(&avatarMixerAssignment); } - if (!nodeList->soloNodeOfType(NODE_TYPE_AVATAR_MIXER)) { - if (!avatarAssignment - || usecTimestampNow() - usecTimestamp(&avatarAssignment->getTime()) >= ASSIGNMENT_SILENCE_MAX_USECS) { - if (!avatarAssignment) { - avatarAssignment = new Assignment(Assignment::Create, Assignment::AvatarMixer, assignmentPool); - avatarAssignment->setAttachedLocalSocket((sockaddr*) &localSocket); - } - - nodeList->sendAssignment(*avatarAssignment); - - // reset the create time on the assignment so re-request is in ASSIGNMENT_SILENCE_MAX_USECS - avatarAssignment->setCreateTimeToNow(); - } - + if (!nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER) && + std::find(assignmentQueue.begin(), assignmentQueue.end(), &audioMixerAssignment) == assignmentQueue.end()) { + qDebug("Missing an audio mixer and assignment not in queue. Adding.\n"); + assignmentQueue.push_front(&audioMixerAssignment); } - if (nodeList->getNodeSocket()->receive((sockaddr *)&nodePublicAddress, packetData, &receivedBytes) && - (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY || packetData[0] == PACKET_TYPE_DOMAIN_LIST_REQUEST) && - packetVersionMatch(packetData)) { - // this is an RFD or domain list request packet, and there is a version match - std::map newestSoloNodes; - - int numBytesSenderHeader = numBytesForPacketHeader(packetData); - - nodeType = *(packetData + numBytesSenderHeader); - int numBytesSocket = unpackSocket(packetData + numBytesSenderHeader + sizeof(NODE_TYPE), - (sockaddr*) &nodeLocalAddress); - - sockaddr* destinationSocket = (sockaddr*) &nodePublicAddress; - - // check the node public address - // if it matches our local address we're on the same box - // so hardcode the EC2 public address for now - if (nodePublicAddress.sin_addr.s_addr == serverLocalAddress) { - // If we're not running "local" then we do replace the IP - // with 0. This designates to clients that the server is reachable - // at the same IP address - if (!isLocalMode) { - nodePublicAddress.sin_addr.s_addr = 0; - destinationSocket = (sockaddr*) &nodeLocalAddress; - } - } - - Node* newNode = nodeList->addOrUpdateNode((sockaddr*) &nodePublicAddress, - (sockaddr*) &nodeLocalAddress, - nodeType, - nodeList->getLastNodeID()); - - // if addOrUpdateNode returns NULL this was a solo node we already have, don't talk back to it - if (newNode) { - if (newNode->getNodeID() == nodeList->getLastNodeID()) { - nodeList->increaseNodeID(); + while (nodeList->getNodeSocket()->receive((sockaddr *)&nodePublicAddress, packetData, &receivedBytes) && + packetVersionMatch(packetData)) { + if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY || packetData[0] == PACKET_TYPE_DOMAIN_LIST_REQUEST) { + // this is an RFD or domain list request packet, and there is a version match + std::map newestSoloNodes; + + int numBytesSenderHeader = numBytesForPacketHeader(packetData); + + nodeType = *(packetData + numBytesSenderHeader); + int numBytesSocket = unpackSocket(packetData + numBytesSenderHeader + sizeof(NODE_TYPE), + (sockaddr*) &nodeLocalAddress); + + sockaddr* destinationSocket = (sockaddr*) &nodePublicAddress; + + // check the node public address + // if it matches our local address we're on the same box + // so hardcode the EC2 public address for now + if (nodePublicAddress.sin_addr.s_addr == serverLocalAddress) { + // If we're not running "local" then we do replace the IP + // with 0. This designates to clients that the server is reachable + // at the same IP address + if (!isLocalMode) { + nodePublicAddress.sin_addr.s_addr = 0; + destinationSocket = (sockaddr*) &nodeLocalAddress; + } } - currentBufferPos = broadcastPacket + numHeaderBytes; - startPointer = currentBufferPos; + Node* newNode = nodeList->addOrUpdateNode((sockaddr*) &nodePublicAddress, + (sockaddr*) &nodeLocalAddress, + nodeType, + nodeList->getLastNodeID()); - unsigned char* nodeTypesOfInterest = packetData + numBytesSenderHeader + sizeof(NODE_TYPE) - + numBytesSocket + sizeof(unsigned char); - int numInterestTypes = *(nodeTypesOfInterest - 1); - - if (numInterestTypes > 0) { - // if the node has sent no types of interest, assume they want nothing but their own ID back - for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { - if (!node->matches((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType) && - memchr(nodeTypesOfInterest, node->getType(), numInterestTypes)) { - // this is not the node themselves - // and this is an node of a type in the passed node types of interest - // or the node did not pass us any specific types they are interested in - - if (memchr(SOLO_NODE_TYPES, node->getType(), sizeof(SOLO_NODE_TYPES)) == NULL) { - // this is an node of which there can be multiple, just add them to the packet - // don't send avatar nodes to other avatars, that will come from avatar mixer - if (nodeType != NODE_TYPE_AGENT || node->getType() != NODE_TYPE_AGENT) { - currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, &(*node)); - } + // if addOrUpdateNode returns NULL this was a solo node we already have, don't talk back to it + if (newNode) { + if (newNode->getNodeID() == nodeList->getLastNodeID()) { + nodeList->increaseNodeID(); + } + + int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_DOMAIN); + + currentBufferPos = broadcastPacket + numHeaderBytes; + startPointer = currentBufferPos; + + unsigned char* nodeTypesOfInterest = packetData + numBytesSenderHeader + sizeof(NODE_TYPE) + + numBytesSocket + sizeof(unsigned char); + int numInterestTypes = *(nodeTypesOfInterest - 1); + + if (numInterestTypes > 0) { + // if the node has sent no types of interest, assume they want nothing but their own ID back + for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { + if (!node->matches((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType) && + memchr(nodeTypesOfInterest, node->getType(), numInterestTypes)) { + // this is not the node themselves + // and this is an node of a type in the passed node types of interest + // or the node did not pass us any specific types they are interested in - } else { - // solo node, we need to only send newest - if (newestSoloNodes[node->getType()] == NULL || - newestSoloNodes[node->getType()]->getWakeMicrostamp() < node->getWakeMicrostamp()) { - // we have to set the newer solo node to add it to the broadcast later - newestSoloNodes[node->getType()] = &(*node); + if (memchr(SOLO_NODE_TYPES, node->getType(), sizeof(SOLO_NODE_TYPES)) == NULL) { + // this is an node of which there can be multiple, just add them to the packet + // don't send avatar nodes to other avatars, that will come from avatar mixer + if (nodeType != NODE_TYPE_AGENT || node->getType() != NODE_TYPE_AGENT) { + currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, &(*node)); + } + + } else { + // solo node, we need to only send newest + if (newestSoloNodes[node->getType()] == NULL || + newestSoloNodes[node->getType()]->getWakeMicrostamp() < node->getWakeMicrostamp()) { + // we have to set the newer solo node to add it to the broadcast later + newestSoloNodes[node->getType()] = &(*node); + } } } } + + for (std::map::iterator soloNode = newestSoloNodes.begin(); + soloNode != newestSoloNodes.end(); + soloNode++) { + // this is the newest alive solo node, add them to the packet + currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, soloNode->second); + } } - for (std::map::iterator soloNode = newestSoloNodes.begin(); - soloNode != newestSoloNodes.end(); - soloNode++) { - // this is the newest alive solo node, add them to the packet - currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, soloNode->second); + // update last receive to now + uint64_t timeNow = usecTimestampNow(); + newNode->setLastHeardMicrostamp(timeNow); + + if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY + && memchr(SOLO_NODE_TYPES, nodeType, sizeof(SOLO_NODE_TYPES))) { + newNode->setWakeMicrostamp(timeNow); } + + // add the node ID to the end of the pointer + currentBufferPos += packNodeId(currentBufferPos, newNode->getNodeID()); + + // send the constructed list back to this node + nodeList->getNodeSocket()->send(destinationSocket, + broadcastPacket, + (currentBufferPos - startPointer) + numHeaderBytes); } + } else if (packetData[0] == PACKET_TYPE_REQUEST_ASSIGNMENT) { - // update last receive to now - uint64_t timeNow = usecTimestampNow(); - newNode->setLastHeardMicrostamp(timeNow); + qDebug("Received a request for assignment.\n"); - if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY - && memchr(SOLO_NODE_TYPES, nodeType, sizeof(SOLO_NODE_TYPES))) { - newNode->setWakeMicrostamp(timeNow); + // this is an unassigned client talking to us directly for an assignment + // go through our queue and see if there are any assignments to give out + std::deque::iterator assignment = assignmentQueue.begin(); + + while (assignment != assignmentQueue.end()) { + + // give this assignment out, no conditions stop us from giving it to the local assignment client + int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_CREATE_ASSIGNMENT); + int numAssignmentBytes = (*assignment)->packToBuffer(broadcastPacket + numHeaderBytes); + + nodeList->getNodeSocket()->send((sockaddr*) &nodePublicAddress, + broadcastPacket, + numHeaderBytes + numAssignmentBytes); + + // remove the assignment from the queue + assignmentQueue.erase(assignment); + + // stop looping, we've handed out an assignment + break; } - - // add the node ID to the end of the pointer - currentBufferPos += packNodeId(currentBufferPos, newNode->getNodeID()); - - // send the constructed list back to this node - nodeList->getNodeSocket()->send(destinationSocket, - broadcastPacket, - (currentBufferPos - startPointer) + numHeaderBytes); } } + // if ASSIGNMENT_REQUEST_INTERVAL_USECS have passed since last global assignment request then fire off another + if (usecTimestampNow() - usecTimestamp(&lastGlobalAssignmentRequest) >= GLOBAL_ASSIGNMENT_REQUEST_INTERVAL_USECS) { + gettimeofday(&lastGlobalAssignmentRequest, NULL); + + // go through our queue and see if there are any assignments to send to the global assignment server + std::deque::iterator assignment = assignmentQueue.begin(); + + while (assignment != assignmentQueue.end()) { + + if ((*assignment)->getLocation() != Assignment::LocalLocation) { + // attach our local socket to the assignment so the assignment-server can optionally hand it out + (*assignment)->setAttachedLocalSocket((sockaddr*) &localSocket); + + nodeList->sendAssignment(*(*assignment)); + + // remove the assignment from the queue + assignmentQueue.erase(assignment); + + // stop looping, we've handed out an assignment + break; + } else { + // push forward the iterator to check the next assignment + assignment++; + } + } + + } + if (Logging::shouldSendStats()) { if (usecTimestampNow() - usecTimestamp(&lastStatSendTime) >= (NODE_COUNT_STAT_INTERVAL_MSECS * 1000)) { // time to send our count of nodes and servers to logstash @@ -253,9 +297,6 @@ int main(int argc, const char* argv[]) { } } } - - delete audioAssignment; - delete avatarAssignment; return 0; } diff --git a/libraries/shared/src/Assignment.cpp b/libraries/shared/src/Assignment.cpp index 1f8ab1c388..117d6716e0 100644 --- a/libraries/shared/src/Assignment.cpp +++ b/libraries/shared/src/Assignment.cpp @@ -13,28 +13,19 @@ const char IPv4_ADDRESS_DESIGNATOR = 4; const char IPv6_ADDRESS_DESIGNATOR = 6; -Assignment::Assignment(Assignment::Direction direction, Assignment::Type type, const char* pool) : +Assignment::Assignment(Assignment::Direction direction, Assignment::Type type, Assignment::Location location) : _direction(direction), _type(type), - _pool(NULL), + _location(location), _attachedPublicSocket(NULL), _attachedLocalSocket(NULL) { // set the create time on this assignment gettimeofday(&_time, NULL); - - // copy the pool, if we got one - if (pool) { - int poolLength = strlen(pool); - - // create the char array and make it large enough for string and null termination - _pool = new char[poolLength + sizeof(char)]; - strcpy(_pool, pool); - } } Assignment::Assignment(const unsigned char* dataBuffer, int numBytes) : - _pool(NULL), + _location(GlobalLocation), _attachedPublicSocket(NULL), _attachedLocalSocket(NULL) { @@ -44,11 +35,11 @@ Assignment::Assignment(const unsigned char* dataBuffer, int numBytes) : int numBytesRead = 0; if (dataBuffer[0] == PACKET_TYPE_REQUEST_ASSIGNMENT) { - _direction = Assignment::Request; + _direction = Assignment::RequestDirection; } else if (dataBuffer[0] == PACKET_TYPE_CREATE_ASSIGNMENT) { - _direction = Assignment::Create; + _direction = Assignment::CreateDirection; } else if (dataBuffer[0] == PACKET_TYPE_DEPLOY_ASSIGNMENT) { - _direction = Assignment::Deploy; + _direction = Assignment::DeployDirection; } numBytesRead += numBytesForPacketHeader(dataBuffer); @@ -56,15 +47,6 @@ Assignment::Assignment(const unsigned char* dataBuffer, int numBytes) : memcpy(&_type, dataBuffer + numBytesRead, sizeof(Assignment::Type)); numBytesRead += sizeof(Assignment::Type); - if (dataBuffer[numBytesRead] != 0) { - int poolLength = strlen((const char*) dataBuffer + numBytesRead); - _pool = new char[poolLength + sizeof(char)]; - strcpy(_pool, (char*) dataBuffer + numBytesRead); - numBytesRead += poolLength + sizeof(char); - } else { - numBytesRead += sizeof(char); - } - if (numBytes > numBytesRead) { sockaddr* newSocket = NULL; @@ -78,7 +60,7 @@ Assignment::Assignment(const unsigned char* dataBuffer, int numBytes) : qDebug("Received a socket that cannot be unpacked!\n"); } - if (_direction == Assignment::Create) { + if (_direction == Assignment::CreateDirection) { delete _attachedLocalSocket; _attachedLocalSocket = newSocket; } else { @@ -91,7 +73,6 @@ Assignment::Assignment(const unsigned char* dataBuffer, int numBytes) : Assignment::~Assignment() { delete _attachedPublicSocket; delete _attachedLocalSocket; - delete _pool; } void Assignment::setAttachedPublicSocket(const sockaddr* attachedPublicSocket) { @@ -124,16 +105,6 @@ int Assignment::packToBuffer(unsigned char* buffer) { memcpy(buffer + numPackedBytes, &_type, sizeof(_type)); numPackedBytes += sizeof(_type); - if (_pool) { - int poolLength = strlen(_pool); - strcpy((char*) buffer + numPackedBytes, _pool); - - numPackedBytes += poolLength + sizeof(char); - } else { - buffer[numPackedBytes] = '\0'; - numPackedBytes += sizeof(char); - } - if (_attachedPublicSocket || _attachedLocalSocket) { sockaddr* socketToPack = (_attachedPublicSocket) ? _attachedPublicSocket : _attachedLocalSocket; @@ -148,6 +119,6 @@ int Assignment::packToBuffer(unsigned char* buffer) { } QDebug operator<<(QDebug debug, const Assignment &assignment) { - debug << "T:" << assignment.getType() << "P:" << assignment.getPool(); + debug << "T:" << assignment.getType(); return debug.nospace(); } \ No newline at end of file diff --git a/libraries/shared/src/Assignment.h b/libraries/shared/src/Assignment.h index 11966b2dc7..3d364945ba 100644 --- a/libraries/shared/src/Assignment.h +++ b/libraries/shared/src/Assignment.h @@ -18,18 +18,25 @@ class Assignment { public: enum Type { - AudioMixer, - AvatarMixer, - All + AudioMixerType, + AvatarMixerType, + AllTypes }; enum Direction { - Create, - Deploy, - Request + CreateDirection, + DeployDirection, + RequestDirection }; - Assignment(Assignment::Direction direction, Assignment::Type type, const char* pool = NULL); + enum Location { + GlobalLocation, + LocalLocation + }; + + Assignment(Assignment::Direction direction, + Assignment::Type type, + Assignment::Location location = Assignment::GlobalLocation); /// Constructs an Assignment from the data in the buffer /// \param dataBuffer the source buffer to un-pack the assignment from @@ -40,7 +47,7 @@ public: Assignment::Direction getDirection() const { return _direction; } Assignment::Type getType() const { return _type; } - const char* getPool() const { return _pool; } + Assignment::Location getLocation() const { return _location; } const timeval& getTime() const { return _time; } const sockaddr* getAttachedPublicSocket() { return _attachedPublicSocket; } @@ -60,7 +67,7 @@ public: private: Assignment::Direction _direction; /// the direction of the assignment (Create, Deploy, Request) Assignment::Type _type; /// the type of the assignment, defines what the assignee will do - char* _pool; /// the pool this assignment is for/from + Assignment::Location _location; /// the location of the assignment, allows a domain to preferentially use local ACs sockaddr* _attachedPublicSocket; /// pointer to a public socket that relates to assignment, depends on direction sockaddr* _attachedLocalSocket; /// pointer to a local socket that relates to assignment, depends on direction timeval _time; /// time the assignment was created (set in constructor) diff --git a/libraries/shared/src/NodeList.cpp b/libraries/shared/src/NodeList.cpp index a982e22242..a51cf12ab3 100644 --- a/libraries/shared/src/NodeList.cpp +++ b/libraries/shared/src/NodeList.cpp @@ -382,7 +382,7 @@ const sockaddr_in GLOBAL_ASSIGNMENT_SOCKET = socketForHostnameAndHostOrderPort(G void NodeList::sendAssignment(Assignment& assignment) { unsigned char assignmentPacket[MAX_PACKET_SIZE]; - PACKET_TYPE assignmentPacketType = assignment.getDirection() == Assignment::Create + PACKET_TYPE assignmentPacketType = assignment.getDirection() == Assignment::CreateDirection ? PACKET_TYPE_CREATE_ASSIGNMENT : PACKET_TYPE_REQUEST_ASSIGNMENT; From ff6fd7318760c2a680581e5419a2e4f07d8e3b3e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 11 Sep 2013 12:17:30 -0700 Subject: [PATCH 11/17] have the domain-server use the new verbose logging message handler --- domain-server/src/main.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/domain-server/src/main.cpp b/domain-server/src/main.cpp index 11ccbf6bb7..1dedf910e9 100644 --- a/domain-server/src/main.cpp +++ b/domain-server/src/main.cpp @@ -50,6 +50,9 @@ unsigned char* addNodeToBroadcastPacket(unsigned char* currentPosition, Node* no } int main(int argc, const char* argv[]) { + + qInstallMessageHandler(Logging::verboseMessageHandler); + NodeList* nodeList = NodeList::createInstance(NODE_TYPE_DOMAIN, DOMAIN_LISTEN_PORT); // If user asks to run in "local" mode then we do NOT replace the IP From 3149431e1663364674f70e90e55381b44fd3fcc7 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 11 Sep 2013 12:23:15 -0700 Subject: [PATCH 12/17] remove notion of pool from assignment-server --- assignment-server/src/main.cpp | 59 +++++++++++++--------------------- 1 file changed, 23 insertions(+), 36 deletions(-) diff --git a/assignment-server/src/main.cpp b/assignment-server/src/main.cpp index 248b9fdc83..73c5d10156 100644 --- a/assignment-server/src/main.cpp +++ b/assignment-server/src/main.cpp @@ -59,43 +59,30 @@ int main(int argc, const char* argv[]) { continue; } - bool eitherHasPool = ((*assignment)->getPool() || requestAssignment.getPool()); - bool bothHavePool = ((*assignment)->getPool() && requestAssignment.getPool()); - - // make sure there is a pool match for the created and requested assignment - // or that neither has a designated pool - if ((eitherHasPool && bothHavePool - && strcmp((*assignment)->getPool(), requestAssignment.getPool()) == 0) - || !eitherHasPool) { - - // check if the requestor is on the same network as the destination for the assignment - if (senderSocket.sin_addr.s_addr == - ((sockaddr_in*) (*assignment)->getAttachedPublicSocket())->sin_addr.s_addr) { - // if this is the case we remove the public socket on the assignment by setting it to NULL - // this ensures the local IP and port sent to the requestor is the local address of destination - (*assignment)->setAttachedPublicSocket(NULL); - } - - - int numAssignmentBytes = (*assignment)->packToBuffer(assignmentPacket + numSendHeaderBytes); - - // send the assignment - serverSocket.send((sockaddr*) &senderSocket, - assignmentPacket, - numSendHeaderBytes + numAssignmentBytes); - - - // delete this assignment now that it has been sent out - delete *assignment; - // remove it from the deque and make the iterator the next assignment - assignmentQueue.erase(assignment); - - // stop looping - we've handed out an assignment - break; - } else { - // push forward the iterator - assignment++; + // check if the requestor is on the same network as the destination for the assignment + if (senderSocket.sin_addr.s_addr == + ((sockaddr_in*) (*assignment)->getAttachedPublicSocket())->sin_addr.s_addr) { + // if this is the case we remove the public socket on the assignment by setting it to NULL + // this ensures the local IP and port sent to the requestor is the local address of destination + (*assignment)->setAttachedPublicSocket(NULL); } + + + int numAssignmentBytes = (*assignment)->packToBuffer(assignmentPacket + numSendHeaderBytes); + + // send the assignment + serverSocket.send((sockaddr*) &senderSocket, + assignmentPacket, + numSendHeaderBytes + numAssignmentBytes); + + + // delete this assignment now that it has been sent out + delete *assignment; + // remove it from the deque and make the iterator the next assignment + assignmentQueue.erase(assignment); + + // stop looping - we've handed out an assignment + break; } } } else if (senderData[0] == PACKET_TYPE_CREATE_ASSIGNMENT && packetVersionMatch(senderData)) { From e1005288cbcb0e9d9b9f08ee6ec9c1f61ca953d9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 11 Sep 2013 12:25:59 -0700 Subject: [PATCH 13/17] prefer that audio-mixer assignment is fulfilled locally --- domain-server/src/main.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/domain-server/src/main.cpp b/domain-server/src/main.cpp index 1dedf910e9..32b784ac9c 100644 --- a/domain-server/src/main.cpp +++ b/domain-server/src/main.cpp @@ -107,7 +107,8 @@ int main(int argc, const char* argv[]) { // as a domain-server we will always want an audio mixer and avatar mixer // setup the create assignments for those Assignment audioMixerAssignment(Assignment::CreateDirection, - Assignment::AudioMixerType); + Assignment::AudioMixerType, + Assignment::LocalLocationg); Assignment avatarMixerAssignment(Assignment::CreateDirection, Assignment::AvatarMixerType, From 04a40be97ad52808dcb06e04433bf2e79cb5f9e0 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 11 Sep 2013 12:29:07 -0700 Subject: [PATCH 14/17] remove an extra g --- domain-server/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domain-server/src/main.cpp b/domain-server/src/main.cpp index 32b784ac9c..18f2ccdfde 100644 --- a/domain-server/src/main.cpp +++ b/domain-server/src/main.cpp @@ -108,7 +108,7 @@ int main(int argc, const char* argv[]) { // setup the create assignments for those Assignment audioMixerAssignment(Assignment::CreateDirection, Assignment::AudioMixerType, - Assignment::LocalLocationg); + Assignment::LocalLocation); Assignment avatarMixerAssignment(Assignment::CreateDirection, Assignment::AvatarMixerType, From fe3b4a66544a2fa78811fb4fbdab813fd44e6761 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 11 Sep 2013 12:29:50 -0700 Subject: [PATCH 15/17] use the verbose message handler in the assignment-server --- assignment-server/src/main.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/assignment-server/src/main.cpp b/assignment-server/src/main.cpp index 73c5d10156..3608f6e360 100644 --- a/assignment-server/src/main.cpp +++ b/assignment-server/src/main.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -22,6 +23,8 @@ const long long NUM_DEFAULT_ASSIGNMENT_STALENESS_USECS = 10 * 1000 * 1000; int main(int argc, const char* argv[]) { + qInstallMessageHandler(Logging::verboseMessageHandler); + std::deque assignmentQueue; sockaddr_in senderSocket; From e73ef6204deace68f649638d84be73be8641b5dd Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 11 Sep 2013 12:37:14 -0700 Subject: [PATCH 16/17] add newlines to assignment-server logging so it doesn't clog up --- assignment-server/src/main.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/assignment-server/src/main.cpp b/assignment-server/src/main.cpp index 3608f6e360..4d2603de44 100644 --- a/assignment-server/src/main.cpp +++ b/assignment-server/src/main.cpp @@ -42,8 +42,8 @@ int main(int argc, const char* argv[]) { // construct the requested assignment from the packet data Assignment requestAssignment(senderData, receivedBytes); - qDebug() << "Received request for assignment:" << requestAssignment; - qDebug() << "Current queue size is" << assignmentQueue.size(); + qDebug() << "Received request for assignment:" << requestAssignment << "\n"; + qDebug() << "Current queue size is" << assignmentQueue.size() << "\n"; // make sure there are assignments in the queue at all if (assignmentQueue.size() > 0) { @@ -92,8 +92,8 @@ int main(int argc, const char* argv[]) { // construct the create assignment from the packet data Assignment* createdAssignment = new Assignment(senderData, receivedBytes); - qDebug() << "Received a created assignment:" << *createdAssignment; - qDebug() << "Current queue size is" << assignmentQueue.size(); + qDebug() << "Received a created assignment:" << *createdAssignment << "\n"; + qDebug() << "Current queue size is" << assignmentQueue.size() << "\n"; // assignment server is likely on a public server // assume that the address we now have for the sender is the public address/port From 803d2975dce6fcaafada489adfba5a9cb6f73bc6 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 11 Sep 2013 12:43:57 -0700 Subject: [PATCH 17/17] fix queue include --- domain-server/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domain-server/src/main.cpp b/domain-server/src/main.cpp index 18f2ccdfde..a362be2520 100644 --- a/domain-server/src/main.cpp +++ b/domain-server/src/main.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include #include #include