From 37aa90932237b981e9e00730afd73daf1d82e4a4 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 9 Sep 2013 11:22:46 -0700 Subject: [PATCH 1/9] Try using the eye coefficients rather than the eye directions reported by Faceshift. --- interface/src/devices/Faceshift.cpp | 45 ++++++++++++++++++++++++++++- interface/src/devices/Faceshift.h | 10 +++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 4418d19223..5e35d23415 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -26,7 +26,11 @@ Faceshift::Faceshift() : _browHeight(0.0f), _browUpCenterIndex(-1), _mouthSize(0.0f), - _jawOpenIndex(-1) + _jawOpenIndex(-1), + _leftEyeUpIndex(-1), + _leftEyeInIndex(-1), + _rightEyeUpIndex(-1), + _rightEyeInIndex(-1) { connect(&_socket, SIGNAL(connected()), SLOT(noteConnected())); connect(&_socket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(noteError(QAbstractSocket::SocketError))); @@ -108,6 +112,21 @@ void Faceshift::readFromSocket() { if (_jawOpenIndex != -1) { _mouthSize = data.m_coeffs[_jawOpenIndex]; } + const float PITCH_SCALE = 45.0f; + if (_leftEyeUpIndex != -1) { + _eyeGazeLeftPitch = PITCH_SCALE * (data.m_coeffs[_leftEyeUpIndex] - data.m_coeffs[_leftEyeDownIndex]); + } + const float YAW_SCALE = 45.0f; + if (_leftEyeInIndex != -1) { + _eyeGazeLeftYaw = YAW_SCALE * (data.m_coeffs[_leftEyeOutIndex] - data.m_coeffs[_leftEyeInIndex]); + } + if (_rightEyeUpIndex != -1) { + _eyeGazeRightPitch = PITCH_SCALE * (data.m_coeffs[_rightEyeUpIndex] - + data.m_coeffs[_rightEyeDownIndex]); + } + if (_rightEyeInIndex != -1) { + _eyeGazeRightYaw = YAW_SCALE * (data.m_coeffs[_rightEyeInIndex] - data.m_coeffs[_rightEyeOutIndex]); + } } break; } @@ -125,6 +144,30 @@ void Faceshift::readFromSocket() { } else if (names[i] == "JawOpen") { _jawOpenIndex = i; + + } else if (names[i] == "EyeUp_L") { + _leftEyeUpIndex = i; + + } else if (names[i] == "EyeDown_L") { + _leftEyeDownIndex = i; + + } else if (names[i] == "EyeIn_L") { + _leftEyeInIndex = i; + + } else if (names[i] == "EyeOut_L") { + _leftEyeOutIndex = i; + + } else if (names[i] == "EyeUp_R") { + _rightEyeUpIndex = i; + + } else if (names[i] == "EyeDown_R") { + _rightEyeDownIndex = i; + + } else if (names[i] == "EyeIn_R") { + _rightEyeInIndex = i; + + } else if (names[i] == "EyeOut_R") { + _rightEyeOutIndex = i; } } break; diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 286a7a56a1..745ce6f099 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -85,6 +85,16 @@ private: float _mouthSize; int _jawOpenIndex; + + int _leftEyeUpIndex; + int _leftEyeDownIndex; + int _leftEyeInIndex; + int _leftEyeOutIndex; + + int _rightEyeUpIndex; + int _rightEyeDownIndex; + int _rightEyeInIndex; + int _rightEyeOutIndex; }; #endif /* defined(__interface__Faceshift__) */ From 597c57a11740b529d80bded8a692ad9f05aa4d40 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 9 Sep 2013 12:17:12 -0700 Subject: [PATCH 2/9] Revert "Try using the eye coefficients rather than the eye directions reported by" This reverts commit 37aa90932237b981e9e00730afd73daf1d82e4a4. --- interface/src/devices/Faceshift.cpp | 45 +---------------------------- interface/src/devices/Faceshift.h | 10 ------- 2 files changed, 1 insertion(+), 54 deletions(-) diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 5e35d23415..4418d19223 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -26,11 +26,7 @@ Faceshift::Faceshift() : _browHeight(0.0f), _browUpCenterIndex(-1), _mouthSize(0.0f), - _jawOpenIndex(-1), - _leftEyeUpIndex(-1), - _leftEyeInIndex(-1), - _rightEyeUpIndex(-1), - _rightEyeInIndex(-1) + _jawOpenIndex(-1) { connect(&_socket, SIGNAL(connected()), SLOT(noteConnected())); connect(&_socket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(noteError(QAbstractSocket::SocketError))); @@ -112,21 +108,6 @@ void Faceshift::readFromSocket() { if (_jawOpenIndex != -1) { _mouthSize = data.m_coeffs[_jawOpenIndex]; } - const float PITCH_SCALE = 45.0f; - if (_leftEyeUpIndex != -1) { - _eyeGazeLeftPitch = PITCH_SCALE * (data.m_coeffs[_leftEyeUpIndex] - data.m_coeffs[_leftEyeDownIndex]); - } - const float YAW_SCALE = 45.0f; - if (_leftEyeInIndex != -1) { - _eyeGazeLeftYaw = YAW_SCALE * (data.m_coeffs[_leftEyeOutIndex] - data.m_coeffs[_leftEyeInIndex]); - } - if (_rightEyeUpIndex != -1) { - _eyeGazeRightPitch = PITCH_SCALE * (data.m_coeffs[_rightEyeUpIndex] - - data.m_coeffs[_rightEyeDownIndex]); - } - if (_rightEyeInIndex != -1) { - _eyeGazeRightYaw = YAW_SCALE * (data.m_coeffs[_rightEyeInIndex] - data.m_coeffs[_rightEyeOutIndex]); - } } break; } @@ -144,30 +125,6 @@ void Faceshift::readFromSocket() { } else if (names[i] == "JawOpen") { _jawOpenIndex = i; - - } else if (names[i] == "EyeUp_L") { - _leftEyeUpIndex = i; - - } else if (names[i] == "EyeDown_L") { - _leftEyeDownIndex = i; - - } else if (names[i] == "EyeIn_L") { - _leftEyeInIndex = i; - - } else if (names[i] == "EyeOut_L") { - _leftEyeOutIndex = i; - - } else if (names[i] == "EyeUp_R") { - _rightEyeUpIndex = i; - - } else if (names[i] == "EyeDown_R") { - _rightEyeDownIndex = i; - - } else if (names[i] == "EyeIn_R") { - _rightEyeInIndex = i; - - } else if (names[i] == "EyeOut_R") { - _rightEyeOutIndex = i; } } break; diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 745ce6f099..286a7a56a1 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -85,16 +85,6 @@ private: float _mouthSize; int _jawOpenIndex; - - int _leftEyeUpIndex; - int _leftEyeDownIndex; - int _leftEyeInIndex; - int _leftEyeOutIndex; - - int _rightEyeUpIndex; - int _rightEyeDownIndex; - int _rightEyeInIndex; - int _rightEyeOutIndex; }; #endif /* defined(__interface__Faceshift__) */ From 8fec78d82afb2be6ad9b0b93bcc935c77b94d6f2 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 9 Sep 2013 13:48:46 -0700 Subject: [PATCH 3/9] Disable random saccades for own eyes. --- interface/src/avatar/Head.cpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index 08efb32062..abf16248c2 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -146,19 +146,6 @@ void Head::resetHairPhysics() { void Head::simulate(float deltaTime, bool isMine, float gyroCameraSensitivity) { - // Update eye saccades - const float AVERAGE_MICROSACCADE_INTERVAL = 0.50f; - const float AVERAGE_SACCADE_INTERVAL = 4.0f; - const float MICROSACCADE_MAGNITUDE = 0.002f; - const float SACCADE_MAGNITUDE = 0.04; - - if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) { - _saccadeTarget = MICROSACCADE_MAGNITUDE * randVector(); - } else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) { - _saccadeTarget = SACCADE_MAGNITUDE * randVector(); - } - _saccade += (_saccadeTarget - _saccade) * 0.50f; - // Update audio trailing average for rendering facial animations Faceshift* faceshift = Application::getInstance()->getFaceshift(); if (isMine && faceshift->isActive()) { @@ -173,6 +160,19 @@ void Head::simulate(float deltaTime, bool isMine, float gyroCameraSensitivity) { _browAudioLift = faceshift->getBrowHeight() * BROW_HEIGHT_SCALE; } else { + // Update eye saccades + const float AVERAGE_MICROSACCADE_INTERVAL = 0.50f; + const float AVERAGE_SACCADE_INTERVAL = 4.0f; + const float MICROSACCADE_MAGNITUDE = 0.002f; + const float SACCADE_MAGNITUDE = 0.04f; + + if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) { + _saccadeTarget = MICROSACCADE_MAGNITUDE * randVector(); + } else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) { + _saccadeTarget = SACCADE_MAGNITUDE * randVector(); + } + _saccade += (_saccadeTarget - _saccade) * 0.50f; + const float AUDIO_AVERAGING_SECS = 0.05; _averageLoudness = (1.f - deltaTime / AUDIO_AVERAGING_SECS) * _averageLoudness + (deltaTime / AUDIO_AVERAGING_SECS) * _audioLoudness; From f8aee88a5b03d41e5f171fffdb42b01405b3ea7b Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 9 Sep 2013 14:30:26 -0700 Subject: [PATCH 4/9] Let's try subtracting the long-term average from the eye directions. --- interface/src/Application.cpp | 11 +++++------ interface/src/devices/Faceshift.cpp | 19 ++++++++++++++++++- interface/src/devices/Faceshift.h | 10 ++++++++++ 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index ea61179057..36e6147d1d 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1531,14 +1531,15 @@ void Application::update(float deltaTime) { // Set where I am looking based on my mouse ray (so that other people can see) glm::vec3 lookAtSpot; + // Update faceshift + _faceshift.update(); + // if we have faceshift, use that to compute the lookat direction glm::vec3 lookAtRayOrigin = mouseRayOrigin, lookAtRayDirection = mouseRayDirection; if (_faceshift.isActive()) { lookAtRayOrigin = _myAvatar.getHead().calculateAverageEyePosition(); - float averagePitch = (_faceshift.getEyeGazeLeftPitch() + _faceshift.getEyeGazeRightPitch()) / 2.0f; - float averageYaw = (_faceshift.getEyeGazeLeftYaw() + _faceshift.getEyeGazeRightYaw()) / 2.0f; - lookAtRayDirection = _myAvatar.getHead().getOrientation() * - glm::quat(glm::radians(glm::vec3(averagePitch, averageYaw, 0.0f))) * glm::vec3(0.0f, 0.0f, -1.0f); + lookAtRayDirection = _myAvatar.getHead().getOrientation() * glm::quat(glm::radians(glm::vec3( + _faceshift.getEstimatedEyePitch(), _faceshift.getEstimatedEyeYaw(), 0.0f))) * glm::vec3(0.0f, 0.0f, -1.0f); } _isLookingAtOtherAvatar = isLookingAtOtherAvatar(lookAtRayOrigin, lookAtRayDirection, lookAtSpot); @@ -1707,8 +1708,6 @@ void Application::update(float deltaTime) { _serialHeadSensor.readData(deltaTime); } - // Update transmitter - // Sample hardware, update view frustum if needed, and send avatar data to mixer/nodes updateAvatar(deltaTime); diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 4418d19223..399369cdd3 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -26,13 +26,30 @@ Faceshift::Faceshift() : _browHeight(0.0f), _browUpCenterIndex(-1), _mouthSize(0.0f), - _jawOpenIndex(-1) + _jawOpenIndex(-1), + _longTermAverageEyePitch(0.0f), + _longTermAverageEyeYaw(0.0f), + _estimatedEyePitch(0.0f), + _estimatedEyeYaw(0.0f) { connect(&_socket, SIGNAL(connected()), SLOT(noteConnected())); connect(&_socket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(noteError(QAbstractSocket::SocketError))); connect(&_socket, SIGNAL(readyRead()), SLOT(readFromSocket())); } +void Faceshift::update() { + if (!isActive()) { + return; + } + float averageEyePitch = (_eyeGazeLeftPitch + _eyeGazeRightPitch) / 2.0f; + float averageEyeYaw = (_eyeGazeLeftYaw + _eyeGazeRightYaw) / 2.0f; + const float LONG_TERM_AVERAGE_SMOOTHING = 0.999f; + _longTermAverageEyePitch = glm::mix(averageEyePitch, _longTermAverageEyePitch, LONG_TERM_AVERAGE_SMOOTHING); + _longTermAverageEyeYaw = glm::mix(averageEyeYaw, _longTermAverageEyeYaw, LONG_TERM_AVERAGE_SMOOTHING); + _estimatedEyePitch = averageEyePitch - _longTermAverageEyePitch; + _estimatedEyeYaw = averageEyeYaw - _longTermAverageEyeYaw; +} + void Faceshift::reset() { if (isActive()) { string message; diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 286a7a56a1..408513e5d7 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -35,6 +35,9 @@ public: float getEyeGazeRightPitch() const { return _eyeGazeRightPitch; } float getEyeGazeRightYaw() const { return _eyeGazeRightYaw; } + float getEstimatedEyePitch() const { return _estimatedEyePitch; } + float getEstimatedEyeYaw() const { return _estimatedEyeYaw; } + float getLeftBlink() const { return _leftBlink; } float getRightBlink() const { return _rightBlink; } @@ -42,6 +45,7 @@ public: float getMouthSize() const { return _mouthSize; } + void update(); void reset(); public slots: @@ -85,6 +89,12 @@ private: float _mouthSize; int _jawOpenIndex; + + float _longTermAverageEyePitch; + float _longTermAverageEyeYaw; + + float _estimatedEyePitch; + float _estimatedEyeYaw; }; #endif /* defined(__interface__Faceshift__) */ From 97803b827e520189be2ba64297ba27de75b8b712 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 9 Sep 2013 16:45:44 -0700 Subject: [PATCH 5/9] pack destination socket for assignment as tightly as possible --- assignment-server/src/main.cpp | 2 +- libraries/shared/src/Assignment.cpp | 48 +++++++++++++---------------- libraries/shared/src/Assignment.h | 6 ++-- libraries/shared/src/UDPSocket.cpp | 2 +- libraries/shared/src/UDPSocket.h | 2 +- 5 files changed, 28 insertions(+), 32 deletions(-) diff --git a/assignment-server/src/main.cpp b/assignment-server/src/main.cpp index dbbd6b2607..d92ab46f63 100644 --- a/assignment-server/src/main.cpp +++ b/assignment-server/src/main.cpp @@ -99,7 +99,7 @@ int main(int argc, const char* argv[]) { // assignment server is on a public server // assume that the address we now have for the sender is the public address/port // and store that with the assignment so it can be given to the requestor later - createdAssignment->setDomainSocket((sockaddr*) &senderSocket); + createdAssignment->setDestinationSocket((sockaddr*) &senderSocket); // add this assignment to the queue assignmentQueue.push_back(createdAssignment); diff --git a/libraries/shared/src/Assignment.cpp b/libraries/shared/src/Assignment.cpp index 850922396f..7151259b7c 100644 --- a/libraries/shared/src/Assignment.cpp +++ b/libraries/shared/src/Assignment.cpp @@ -17,7 +17,7 @@ Assignment::Assignment(Assignment::Direction direction, Assignment::Type type, c _direction(direction), _type(type), _pool(NULL), - _domainSocket(NULL) + _destinationSocket(NULL) { // set the create time on this assignment gettimeofday(&_time, NULL); @@ -34,7 +34,7 @@ Assignment::Assignment(Assignment::Direction direction, Assignment::Type type, c Assignment::Assignment(const unsigned char* dataBuffer, int numBytes) : _pool(NULL), - _domainSocket(NULL) + _destinationSocket(NULL) { // set the create time on this assignment gettimeofday(&_time, NULL); @@ -64,41 +64,39 @@ Assignment::Assignment(const unsigned char* dataBuffer, int numBytes) : } if (numBytes > numBytesRead) { + if (dataBuffer[numBytesRead++] == IPv4_ADDRESS_DESIGNATOR) { // IPv4 address - sockaddr_in destinationSocket = {}; - memcpy(&destinationSocket, dataBuffer + numBytesRead, sizeof(sockaddr_in)); - destinationSocket.sin_family = AF_INET; - setDomainSocket((sockaddr*) &destinationSocket); + delete _destinationSocket; + _destinationSocket = (sockaddr*) new sockaddr_in; + unpackSocket(dataBuffer + numBytesRead, _destinationSocket); } else { - // IPv6 address - sockaddr_in6 destinationSocket = {}; - memcpy(&destinationSocket, dataBuffer + numBytesRead, sizeof(sockaddr_in6)); - setDomainSocket((sockaddr*) &destinationSocket); + // IPv6 address, or bad designator + qDebug("Received a socket that cannot be unpacked!\n"); } } } Assignment::~Assignment() { - delete _domainSocket; + delete _destinationSocket; delete _pool; } -void Assignment::setDomainSocket(const sockaddr* domainSocket) { +void Assignment::setDestinationSocket(const sockaddr* destinationSocket) { - if (_domainSocket) { + if (_destinationSocket) { // delete the old _domainSocket if it exists - delete _domainSocket; - _domainSocket = NULL; + delete _destinationSocket; + _destinationSocket = NULL; } // create a new sockaddr or sockaddr_in depending on what type of address this is - if (domainSocket->sa_family == AF_INET) { - _domainSocket = (sockaddr*) new sockaddr_in; - memcpy(_domainSocket, domainSocket, sizeof(sockaddr_in)); + if (destinationSocket->sa_family == AF_INET) { + _destinationSocket = (sockaddr*) new sockaddr_in; + memcpy(_destinationSocket, destinationSocket, sizeof(sockaddr_in)); } else { - _domainSocket = (sockaddr*) new sockaddr_in6; - memcpy(_domainSocket, domainSocket, sizeof(sockaddr_in6)); + _destinationSocket = (sockaddr*) new sockaddr_in6; + memcpy(_destinationSocket, destinationSocket, sizeof(sockaddr_in6)); } } @@ -118,13 +116,11 @@ int Assignment::packToBuffer(unsigned char* buffer) { numPackedBytes += sizeof(char); } - if (_domainSocket) { - buffer[numPackedBytes++] = (_domainSocket->sa_family == AF_INET) ? IPv4_ADDRESS_DESIGNATOR : IPv6_ADDRESS_DESIGNATOR; + if (_destinationSocket) { + buffer[numPackedBytes++] = (_destinationSocket->sa_family == AF_INET) + ? IPv4_ADDRESS_DESIGNATOR : IPv6_ADDRESS_DESIGNATOR; - int numSocketBytes = (_domainSocket->sa_family == AF_INET) ? sizeof(sockaddr_in) : sizeof(sockaddr_in6); - - memcpy(buffer + numPackedBytes, _domainSocket, numSocketBytes); - numPackedBytes += numSocketBytes; + numPackedBytes += packSocket(buffer + numPackedBytes, _destinationSocket); } return numPackedBytes; diff --git a/libraries/shared/src/Assignment.h b/libraries/shared/src/Assignment.h index 693cc42577..de56b684b6 100644 --- a/libraries/shared/src/Assignment.h +++ b/libraries/shared/src/Assignment.h @@ -43,8 +43,8 @@ public: const char* getPool() const { return _pool; } const timeval& getTime() const { return _time; } - const sockaddr* getDomainSocket() { return _domainSocket; } - void setDomainSocket(const sockaddr* domainSocket); + const sockaddr* getDestinationSocket() { return _destinationSocket; } + void setDestinationSocket(const sockaddr* destinationSocket); /// Packs the assignment to the passed buffer /// \param buffer the buffer in which to pack the assignment @@ -58,7 +58,7 @@ private: Assignment::Direction _direction; /// the direction of the assignment (Create, Deploy, Request) Assignment::Type _type; /// the type of the assignment, defines what the assignee will do char* _pool; /// the pool this assignment is for/from - sockaddr* _domainSocket; /// pointer to socket for domain server that created assignment + sockaddr* _destinationSocket; /// pointer to destination socket for assignment timeval _time; /// time the assignment was created (set in constructor) }; diff --git a/libraries/shared/src/UDPSocket.cpp b/libraries/shared/src/UDPSocket.cpp index d89265b471..9ce9ba695c 100644 --- a/libraries/shared/src/UDPSocket.cpp +++ b/libraries/shared/src/UDPSocket.cpp @@ -68,7 +68,7 @@ int packSocket(unsigned char* packStore, sockaddr* socketToPack) { return packSocket(packStore, ((sockaddr_in*) socketToPack)->sin_addr.s_addr, ((sockaddr_in*) socketToPack)->sin_port); } -int unpackSocket(unsigned char* packedData, sockaddr* unpackDestSocket) { +int unpackSocket(const unsigned char* packedData, sockaddr* unpackDestSocket) { sockaddr_in* destinationSocket = (sockaddr_in*) unpackDestSocket; destinationSocket->sin_addr.s_addr = (packedData[0] << 24) + (packedData[1] << 16) + (packedData[2] << 8) + packedData[3]; destinationSocket->sin_port = (packedData[4] << 8) + packedData[5]; diff --git a/libraries/shared/src/UDPSocket.h b/libraries/shared/src/UDPSocket.h index 1b627dbd41..447d234ca5 100644 --- a/libraries/shared/src/UDPSocket.h +++ b/libraries/shared/src/UDPSocket.h @@ -44,7 +44,7 @@ private: bool socketMatch(const sockaddr* first, const sockaddr* second); int packSocket(unsigned char* packStore, in_addr_t inAddress, in_port_t networkOrderPort); int packSocket(unsigned char* packStore, sockaddr* socketToPack); -int unpackSocket(unsigned char* packedData, sockaddr* unpackDestSocket); +int unpackSocket(const unsigned char* packedData, sockaddr* unpackDestSocket); int getLocalAddress(); unsigned short loadBufferWithSocketInfo(char* addressBuffer, sockaddr* socket); sockaddr_in socketForHostnameAndHostOrderPort(const char* hostname, unsigned short port = 0); From 93ca5278da424757eecec41169faa39d3cdf0f5a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 9 Sep 2013 16:47:48 -0700 Subject: [PATCH 6/9] cleanup is assignment-client destination socket handling --- assignment-client/src/main.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/assignment-client/src/main.cpp b/assignment-client/src/main.cpp index 0579b8bb56..f5137aac39 100644 --- a/assignment-client/src/main.cpp +++ b/assignment-client/src/main.cpp @@ -64,25 +64,25 @@ void childClient() { if (nodeList->getNodeSocket()->receive(packetData, &receivedBytes) && packetData[0] == PACKET_TYPE_DEPLOY_ASSIGNMENT && packetVersionMatch(packetData)) { - - // construct the deployed assignment from the packet data Assignment deployedAssignment(packetData, receivedBytes); qDebug() << "Received an assignment -" << deployedAssignment << "\n"; // switch our nodelist DOMAIN_IP to the ip receieved in the assignment - if (deployedAssignment.getDomainSocket()->sa_family == AF_INET) { - in_addr domainSocketAddr = ((sockaddr_in*) deployedAssignment.getDomainSocket())->sin_addr; + if (deployedAssignment.getDestinationSocket()->sa_family == AF_INET) { + in_addr domainSocketAddr = ((sockaddr_in*) deployedAssignment.getDestinationSocket())->sin_addr; nodeList->setDomainIP(inet_ntoa(domainSocketAddr)); - qDebug("Changed Domain IP to %s\n", inet_ntoa(domainSocketAddr)); - } - - if (deployedAssignment.getType() == Assignment::AudioMixer) { - AudioMixer::run(); + qDebug("Destination IP for assignment is %s\n", inet_ntoa(domainSocketAddr)); + + if (deployedAssignment.getType() == Assignment::AudioMixer) { + AudioMixer::run(); + } else { + AvatarMixer::run(); + } } else { - AvatarMixer::run(); + qDebug("Received a bad destination socket for assignment.\n"); } qDebug("Assignment finished or never started - waiting for new assignment\n"); From e53807ca82b80b0e0f1d1c1e8bdea194ea9c4a23 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 9 Sep 2013 17:29:33 -0700 Subject: [PATCH 7/9] store an attached local and public socket with assignment --- assignment-client/src/main.cpp | 4 +-- assignment-server/src/main.cpp | 6 ++-- libraries/shared/src/Assignment.cpp | 55 ++++++++++++++++++----------- libraries/shared/src/Assignment.h | 10 ++++-- libraries/shared/src/UDPSocket.cpp | 11 ++++++ libraries/shared/src/UDPSocket.h | 1 + 6 files changed, 58 insertions(+), 29 deletions(-) diff --git a/assignment-client/src/main.cpp b/assignment-client/src/main.cpp index f5137aac39..96652318ed 100644 --- a/assignment-client/src/main.cpp +++ b/assignment-client/src/main.cpp @@ -70,8 +70,8 @@ void childClient() { qDebug() << "Received an assignment -" << deployedAssignment << "\n"; // switch our nodelist DOMAIN_IP to the ip receieved in the assignment - if (deployedAssignment.getDestinationSocket()->sa_family == AF_INET) { - in_addr domainSocketAddr = ((sockaddr_in*) deployedAssignment.getDestinationSocket())->sin_addr; + if (deployedAssignment.getAttachedPublicSocket()->sa_family == AF_INET) { + in_addr domainSocketAddr = ((sockaddr_in*) deployedAssignment.getAttachedPublicSocket())->sin_addr; nodeList->setDomainIP(inet_ntoa(domainSocketAddr)); qDebug("Destination IP for assignment is %s\n", inet_ntoa(domainSocketAddr)); diff --git a/assignment-server/src/main.cpp b/assignment-server/src/main.cpp index d92ab46f63..764d68f879 100644 --- a/assignment-server/src/main.cpp +++ b/assignment-server/src/main.cpp @@ -96,10 +96,10 @@ int main(int argc, const char* argv[]) { qDebug() << "Received a created assignment:" << *createdAssignment; qDebug() << "Current queue size is" << assignmentQueue.size(); - // assignment server is on a public server + // assignment server is likely on a public server // assume that the address we now have for the sender is the public address/port - // and store that with the assignment so it can be given to the requestor later - createdAssignment->setDestinationSocket((sockaddr*) &senderSocket); + // and store that with the assignment so it can be given to the requestor later if necessary + createdAssignment->setAttachedPublicSocket((sockaddr*) &senderSocket); // add this assignment to the queue assignmentQueue.push_back(createdAssignment); diff --git a/libraries/shared/src/Assignment.cpp b/libraries/shared/src/Assignment.cpp index 7151259b7c..5da675ced9 100644 --- a/libraries/shared/src/Assignment.cpp +++ b/libraries/shared/src/Assignment.cpp @@ -17,7 +17,8 @@ Assignment::Assignment(Assignment::Direction direction, Assignment::Type type, c _direction(direction), _type(type), _pool(NULL), - _destinationSocket(NULL) + _attachedPublicSocket(NULL), + _attachedLocalSocket(NULL) { // set the create time on this assignment gettimeofday(&_time, NULL); @@ -34,7 +35,8 @@ Assignment::Assignment(Assignment::Direction direction, Assignment::Type type, c Assignment::Assignment(const unsigned char* dataBuffer, int numBytes) : _pool(NULL), - _destinationSocket(NULL) + _attachedPublicSocket(NULL), + _attachedLocalSocket(NULL) { // set the create time on this assignment gettimeofday(&_time, NULL); @@ -65,11 +67,15 @@ Assignment::Assignment(const unsigned char* dataBuffer, int numBytes) : if (numBytes > numBytesRead) { + sockaddr* socketDestination = (_direction == Assignment::Create) + ? _attachedLocalSocket + : _attachedPublicSocket; + if (dataBuffer[numBytesRead++] == IPv4_ADDRESS_DESIGNATOR) { // IPv4 address - delete _destinationSocket; - _destinationSocket = (sockaddr*) new sockaddr_in; - unpackSocket(dataBuffer + numBytesRead, _destinationSocket); + delete socketDestination; + socketDestination = (sockaddr*) new sockaddr_in; + unpackSocket(dataBuffer + numBytesRead, socketDestination); } else { // IPv6 address, or bad designator qDebug("Received a socket that cannot be unpacked!\n"); @@ -78,26 +84,30 @@ Assignment::Assignment(const unsigned char* dataBuffer, int numBytes) : } Assignment::~Assignment() { - delete _destinationSocket; + delete _attachedPublicSocket; + delete _attachedLocalSocket; delete _pool; } -void Assignment::setDestinationSocket(const sockaddr* destinationSocket) { +void Assignment::setAttachedPublicSocket(const sockaddr* attachedPublicSocket) { - if (_destinationSocket) { - // delete the old _domainSocket if it exists - delete _destinationSocket; - _destinationSocket = NULL; + if (_attachedPublicSocket) { + // delete the old socket if it exists + delete _attachedPublicSocket; + _attachedPublicSocket = NULL; } - // create a new sockaddr or sockaddr_in depending on what type of address this is - if (destinationSocket->sa_family == AF_INET) { - _destinationSocket = (sockaddr*) new sockaddr_in; - memcpy(_destinationSocket, destinationSocket, sizeof(sockaddr_in)); - } else { - _destinationSocket = (sockaddr*) new sockaddr_in6; - memcpy(_destinationSocket, destinationSocket, sizeof(sockaddr_in6)); + copySocketToEmptySocketPointer(_attachedPublicSocket, attachedPublicSocket); +} + +void Assignment::setAttachedLocalSocket(const sockaddr* attachedLocalSocket) { + if (_attachedLocalSocket) { + // delete the old socket if it exists + delete _attachedLocalSocket; + _attachedLocalSocket = NULL; } + + copySocketToEmptySocketPointer(_attachedLocalSocket, attachedLocalSocket); } int Assignment::packToBuffer(unsigned char* buffer) { @@ -116,11 +126,14 @@ int Assignment::packToBuffer(unsigned char* buffer) { numPackedBytes += sizeof(char); } - if (_destinationSocket) { - buffer[numPackedBytes++] = (_destinationSocket->sa_family == AF_INET) + if (_attachedPublicSocket || _attachedLocalSocket) { + sockaddr* socketToPack = (_attachedPublicSocket) ? _attachedPublicSocket : _attachedLocalSocket; + + // we have a socket to pack, add the designator + buffer[numPackedBytes++] = (socketToPack->sa_family == AF_INET) ? IPv4_ADDRESS_DESIGNATOR : IPv6_ADDRESS_DESIGNATOR; - numPackedBytes += packSocket(buffer + numPackedBytes, _destinationSocket); + numPackedBytes += packSocket(buffer + numPackedBytes, socketToPack); } return numPackedBytes; diff --git a/libraries/shared/src/Assignment.h b/libraries/shared/src/Assignment.h index de56b684b6..b72aafb67e 100644 --- a/libraries/shared/src/Assignment.h +++ b/libraries/shared/src/Assignment.h @@ -43,8 +43,11 @@ public: const char* getPool() const { return _pool; } const timeval& getTime() const { return _time; } - const sockaddr* getDestinationSocket() { return _destinationSocket; } - void setDestinationSocket(const sockaddr* destinationSocket); + const sockaddr* getAttachedPublicSocket() { return _attachedPublicSocket; } + void setAttachedPublicSocket(const sockaddr* attachedPublicSocket); + + const sockaddr* getAttachedLocalSocket() { return _attachedLocalSocket; } + void setAttachedLocalSocket(const sockaddr* attachedLocalSocket); /// Packs the assignment to the passed buffer /// \param buffer the buffer in which to pack the assignment @@ -58,7 +61,8 @@ private: Assignment::Direction _direction; /// the direction of the assignment (Create, Deploy, Request) Assignment::Type _type; /// the type of the assignment, defines what the assignee will do char* _pool; /// the pool this assignment is for/from - sockaddr* _destinationSocket; /// pointer to destination socket for assignment + sockaddr* _attachedPublicSocket; /// pointer to a public socket that relates to assignment, depends on direction + sockaddr* _attachedLocalSocket; /// pointer to a local socket that relates to assignment, depends on direction timeval _time; /// time the assignment was created (set in constructor) }; diff --git a/libraries/shared/src/UDPSocket.cpp b/libraries/shared/src/UDPSocket.cpp index 9ce9ba695c..0405f83c85 100644 --- a/libraries/shared/src/UDPSocket.cpp +++ b/libraries/shared/src/UDPSocket.cpp @@ -75,6 +75,17 @@ int unpackSocket(const unsigned char* packedData, sockaddr* unpackDestSocket) { return 6; // this could be more if we ever need IPv6 } +void copySocketToEmptySocketPointer(sockaddr* destination, const sockaddr* source) { + // create a new sockaddr or sockaddr_in depending on what type of address this is + if (source->sa_family == AF_INET) { + destination = (sockaddr*) new sockaddr_in; + memcpy(destination, source, sizeof(sockaddr_in)); + } else { + destination = (sockaddr*) new sockaddr_in6; + memcpy(destination, source, sizeof(sockaddr_in6)); + } +} + int getLocalAddress() { // get this node's local address so we can pass that to DS struct ifaddrs* ifAddrStruct = NULL; diff --git a/libraries/shared/src/UDPSocket.h b/libraries/shared/src/UDPSocket.h index 447d234ca5..21b19920b2 100644 --- a/libraries/shared/src/UDPSocket.h +++ b/libraries/shared/src/UDPSocket.h @@ -45,6 +45,7 @@ bool socketMatch(const sockaddr* first, const sockaddr* second); int packSocket(unsigned char* packStore, in_addr_t inAddress, in_port_t networkOrderPort); int packSocket(unsigned char* packStore, sockaddr* socketToPack); int unpackSocket(const unsigned char* packedData, sockaddr* unpackDestSocket); +void copySocketToEmptySocketPointer(sockaddr* destination, const sockaddr* source); int getLocalAddress(); unsigned short loadBufferWithSocketInfo(char* addressBuffer, sockaddr* socket); sockaddr_in socketForHostnameAndHostOrderPort(const char* hostname, unsigned short port = 0); From fa521a5dc99597ca7ad76a78c0483acf8b1eef3c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 9 Sep 2013 18:15:33 -0700 Subject: [PATCH 8/9] handle case where AC is on same network as DS but not AS --- assignment-server/src/main.cpp | 9 +++++++++ domain-server/src/main.cpp | 8 ++++++++ libraries/shared/src/Assignment.cpp | 26 +++++++++++++++++--------- libraries/shared/src/UDPSocket.cpp | 11 ++++++----- libraries/shared/src/UDPSocket.h | 2 +- 5 files changed, 41 insertions(+), 15 deletions(-) diff --git a/assignment-server/src/main.cpp b/assignment-server/src/main.cpp index 764d68f879..248b9fdc83 100644 --- a/assignment-server/src/main.cpp +++ b/assignment-server/src/main.cpp @@ -68,6 +68,15 @@ int main(int argc, const char* argv[]) { && strcmp((*assignment)->getPool(), requestAssignment.getPool()) == 0) || !eitherHasPool) { + // check if the requestor is on the same network as the destination for the assignment + if (senderSocket.sin_addr.s_addr == + ((sockaddr_in*) (*assignment)->getAttachedPublicSocket())->sin_addr.s_addr) { + // if this is the case we remove the public socket on the assignment by setting it to NULL + // this ensures the local IP and port sent to the requestor is the local address of destination + (*assignment)->setAttachedPublicSocket(NULL); + } + + int numAssignmentBytes = (*assignment)->packToBuffer(assignmentPacket + numSendHeaderBytes); // send the assignment diff --git a/domain-server/src/main.cpp b/domain-server/src/main.cpp index 5739639f95..7b44648736 100644 --- a/domain-server/src/main.cpp +++ b/domain-server/src/main.cpp @@ -106,6 +106,12 @@ int main(int argc, const char* argv[]) { Assignment* audioAssignment = NULL; Assignment* avatarAssignment = NULL; + // construct a local socket to send with our created assignments + sockaddr_in localSocket = {}; + localSocket.sin_family = AF_INET; + localSocket.sin_port = htons(nodeList->getInstance()->getNodeSocket()->getListeningPort()); + localSocket.sin_addr.s_addr = serverLocalAddress; + while (true) { if (!nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER)) { if (!audioAssignment @@ -113,6 +119,7 @@ int main(int argc, const char* argv[]) { if (!audioAssignment) { audioAssignment = new Assignment(Assignment::Create, Assignment::AudioMixer, assignmentPool); + audioAssignment->setAttachedLocalSocket((sockaddr*) &localSocket); } nodeList->sendAssignment(*audioAssignment); @@ -125,6 +132,7 @@ int main(int argc, const char* argv[]) { || usecTimestampNow() - usecTimestamp(&avatarAssignment->getTime()) >= ASSIGNMENT_SILENCE_MAX_USECS) { if (!avatarAssignment) { avatarAssignment = new Assignment(Assignment::Create, Assignment::AvatarMixer, assignmentPool); + avatarAssignment->setAttachedLocalSocket((sockaddr*) &localSocket); } nodeList->sendAssignment(*avatarAssignment); diff --git a/libraries/shared/src/Assignment.cpp b/libraries/shared/src/Assignment.cpp index 5da675ced9..1f8ab1c388 100644 --- a/libraries/shared/src/Assignment.cpp +++ b/libraries/shared/src/Assignment.cpp @@ -67,19 +67,24 @@ Assignment::Assignment(const unsigned char* dataBuffer, int numBytes) : if (numBytes > numBytesRead) { - sockaddr* socketDestination = (_direction == Assignment::Create) - ? _attachedLocalSocket - : _attachedPublicSocket; + sockaddr* newSocket = NULL; if (dataBuffer[numBytesRead++] == IPv4_ADDRESS_DESIGNATOR) { // IPv4 address - delete socketDestination; - socketDestination = (sockaddr*) new sockaddr_in; - unpackSocket(dataBuffer + numBytesRead, socketDestination); + newSocket = (sockaddr*) new sockaddr_in; + unpackSocket(dataBuffer + numBytesRead, newSocket); } else { // IPv6 address, or bad designator qDebug("Received a socket that cannot be unpacked!\n"); } + + if (_direction == Assignment::Create) { + delete _attachedLocalSocket; + _attachedLocalSocket = newSocket; + } else { + delete _attachedPublicSocket; + _attachedPublicSocket = newSocket; + } } } @@ -90,14 +95,15 @@ Assignment::~Assignment() { } void Assignment::setAttachedPublicSocket(const sockaddr* attachedPublicSocket) { - if (_attachedPublicSocket) { // delete the old socket if it exists delete _attachedPublicSocket; _attachedPublicSocket = NULL; } - copySocketToEmptySocketPointer(_attachedPublicSocket, attachedPublicSocket); + if (attachedPublicSocket) { + copySocketToEmptySocketPointer(&_attachedPublicSocket, attachedPublicSocket); + } } void Assignment::setAttachedLocalSocket(const sockaddr* attachedLocalSocket) { @@ -107,7 +113,9 @@ void Assignment::setAttachedLocalSocket(const sockaddr* attachedLocalSocket) { _attachedLocalSocket = NULL; } - copySocketToEmptySocketPointer(_attachedLocalSocket, attachedLocalSocket); + if (attachedLocalSocket) { + copySocketToEmptySocketPointer(&_attachedLocalSocket, attachedLocalSocket); + } } int Assignment::packToBuffer(unsigned char* buffer) { diff --git a/libraries/shared/src/UDPSocket.cpp b/libraries/shared/src/UDPSocket.cpp index 0405f83c85..11fd218a56 100644 --- a/libraries/shared/src/UDPSocket.cpp +++ b/libraries/shared/src/UDPSocket.cpp @@ -70,19 +70,20 @@ int packSocket(unsigned char* packStore, sockaddr* socketToPack) { int unpackSocket(const unsigned char* packedData, sockaddr* unpackDestSocket) { sockaddr_in* destinationSocket = (sockaddr_in*) unpackDestSocket; + destinationSocket->sin_family = AF_INET; destinationSocket->sin_addr.s_addr = (packedData[0] << 24) + (packedData[1] << 16) + (packedData[2] << 8) + packedData[3]; destinationSocket->sin_port = (packedData[4] << 8) + packedData[5]; return 6; // this could be more if we ever need IPv6 } -void copySocketToEmptySocketPointer(sockaddr* destination, const sockaddr* source) { +void copySocketToEmptySocketPointer(sockaddr** destination, const sockaddr* source) { // create a new sockaddr or sockaddr_in depending on what type of address this is if (source->sa_family == AF_INET) { - destination = (sockaddr*) new sockaddr_in; - memcpy(destination, source, sizeof(sockaddr_in)); + *destination = (sockaddr*) new sockaddr_in; + memcpy(*destination, source, sizeof(sockaddr_in)); } else { - destination = (sockaddr*) new sockaddr_in6; - memcpy(destination, source, sizeof(sockaddr_in6)); + *destination = (sockaddr*) new sockaddr_in6; + memcpy(*destination, source, sizeof(sockaddr_in6)); } } diff --git a/libraries/shared/src/UDPSocket.h b/libraries/shared/src/UDPSocket.h index 21b19920b2..d3f7cfac58 100644 --- a/libraries/shared/src/UDPSocket.h +++ b/libraries/shared/src/UDPSocket.h @@ -45,7 +45,7 @@ bool socketMatch(const sockaddr* first, const sockaddr* second); int packSocket(unsigned char* packStore, in_addr_t inAddress, in_port_t networkOrderPort); int packSocket(unsigned char* packStore, sockaddr* socketToPack); int unpackSocket(const unsigned char* packedData, sockaddr* unpackDestSocket); -void copySocketToEmptySocketPointer(sockaddr* destination, const sockaddr* source); +void copySocketToEmptySocketPointer(sockaddr** destination, const sockaddr* source); int getLocalAddress(); unsigned short loadBufferWithSocketInfo(char* addressBuffer, sockaddr* socket); sockaddr_in socketForHostnameAndHostOrderPort(const char* hostname, unsigned short port = 0); From fb8c9dbfeaf30eb64d9d7419d5ee28dc2eadbc8e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 10 Sep 2013 09:58:42 -0700 Subject: [PATCH 9/9] fix spacing in Assignment --- libraries/shared/src/Assignment.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/shared/src/Assignment.h b/libraries/shared/src/Assignment.h index b72aafb67e..11966b2dc7 100644 --- a/libraries/shared/src/Assignment.h +++ b/libraries/shared/src/Assignment.h @@ -61,7 +61,7 @@ private: Assignment::Direction _direction; /// the direction of the assignment (Create, Deploy, Request) Assignment::Type _type; /// the type of the assignment, defines what the assignee will do char* _pool; /// the pool this assignment is for/from - sockaddr* _attachedPublicSocket; /// pointer to a public socket that relates to assignment, depends on direction + sockaddr* _attachedPublicSocket; /// pointer to a public socket that relates to assignment, depends on direction sockaddr* _attachedLocalSocket; /// pointer to a local socket that relates to assignment, depends on direction timeval _time; /// time the assignment was created (set in constructor) };