diff --git a/assignment-client/src/main.cpp b/assignment-client/src/main.cpp index 0579b8bb56..96652318ed 100644 --- a/assignment-client/src/main.cpp +++ b/assignment-client/src/main.cpp @@ -64,25 +64,25 @@ void childClient() { if (nodeList->getNodeSocket()->receive(packetData, &receivedBytes) && packetData[0] == PACKET_TYPE_DEPLOY_ASSIGNMENT && packetVersionMatch(packetData)) { - - // construct the deployed assignment from the packet data Assignment deployedAssignment(packetData, receivedBytes); qDebug() << "Received an assignment -" << deployedAssignment << "\n"; // switch our nodelist DOMAIN_IP to the ip receieved in the assignment - if (deployedAssignment.getDomainSocket()->sa_family == AF_INET) { - in_addr domainSocketAddr = ((sockaddr_in*) deployedAssignment.getDomainSocket())->sin_addr; + if (deployedAssignment.getAttachedPublicSocket()->sa_family == AF_INET) { + in_addr domainSocketAddr = ((sockaddr_in*) deployedAssignment.getAttachedPublicSocket())->sin_addr; nodeList->setDomainIP(inet_ntoa(domainSocketAddr)); - qDebug("Changed Domain IP to %s\n", inet_ntoa(domainSocketAddr)); - } - - if (deployedAssignment.getType() == Assignment::AudioMixer) { - AudioMixer::run(); + qDebug("Destination IP for assignment is %s\n", inet_ntoa(domainSocketAddr)); + + if (deployedAssignment.getType() == Assignment::AudioMixer) { + AudioMixer::run(); + } else { + AvatarMixer::run(); + } } else { - AvatarMixer::run(); + qDebug("Received a bad destination socket for assignment.\n"); } qDebug("Assignment finished or never started - waiting for new assignment\n"); diff --git a/assignment-server/src/main.cpp b/assignment-server/src/main.cpp index dbbd6b2607..248b9fdc83 100644 --- a/assignment-server/src/main.cpp +++ b/assignment-server/src/main.cpp @@ -68,6 +68,15 @@ int main(int argc, const char* argv[]) { && strcmp((*assignment)->getPool(), requestAssignment.getPool()) == 0) || !eitherHasPool) { + // check if the requestor is on the same network as the destination for the assignment + if (senderSocket.sin_addr.s_addr == + ((sockaddr_in*) (*assignment)->getAttachedPublicSocket())->sin_addr.s_addr) { + // if this is the case we remove the public socket on the assignment by setting it to NULL + // this ensures the local IP and port sent to the requestor is the local address of destination + (*assignment)->setAttachedPublicSocket(NULL); + } + + int numAssignmentBytes = (*assignment)->packToBuffer(assignmentPacket + numSendHeaderBytes); // send the assignment @@ -96,10 +105,10 @@ int main(int argc, const char* argv[]) { qDebug() << "Received a created assignment:" << *createdAssignment; qDebug() << "Current queue size is" << assignmentQueue.size(); - // assignment server is on a public server + // assignment server is likely on a public server // assume that the address we now have for the sender is the public address/port - // and store that with the assignment so it can be given to the requestor later - createdAssignment->setDomainSocket((sockaddr*) &senderSocket); + // and store that with the assignment so it can be given to the requestor later if necessary + createdAssignment->setAttachedPublicSocket((sockaddr*) &senderSocket); // add this assignment to the queue assignmentQueue.push_back(createdAssignment); diff --git a/domain-server/src/main.cpp b/domain-server/src/main.cpp index 5739639f95..7b44648736 100644 --- a/domain-server/src/main.cpp +++ b/domain-server/src/main.cpp @@ -106,6 +106,12 @@ int main(int argc, const char* argv[]) { Assignment* audioAssignment = NULL; Assignment* avatarAssignment = NULL; + // construct a local socket to send with our created assignments + sockaddr_in localSocket = {}; + localSocket.sin_family = AF_INET; + localSocket.sin_port = htons(nodeList->getInstance()->getNodeSocket()->getListeningPort()); + localSocket.sin_addr.s_addr = serverLocalAddress; + while (true) { if (!nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER)) { if (!audioAssignment @@ -113,6 +119,7 @@ int main(int argc, const char* argv[]) { if (!audioAssignment) { audioAssignment = new Assignment(Assignment::Create, Assignment::AudioMixer, assignmentPool); + audioAssignment->setAttachedLocalSocket((sockaddr*) &localSocket); } nodeList->sendAssignment(*audioAssignment); @@ -125,6 +132,7 @@ int main(int argc, const char* argv[]) { || usecTimestampNow() - usecTimestamp(&avatarAssignment->getTime()) >= ASSIGNMENT_SILENCE_MAX_USECS) { if (!avatarAssignment) { avatarAssignment = new Assignment(Assignment::Create, Assignment::AvatarMixer, assignmentPool); + avatarAssignment->setAttachedLocalSocket((sockaddr*) &localSocket); } nodeList->sendAssignment(*avatarAssignment); diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 7f055abe88..9b1c445e78 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1531,14 +1531,15 @@ void Application::update(float deltaTime) { // Set where I am looking based on my mouse ray (so that other people can see) glm::vec3 lookAtSpot; + // Update faceshift + _faceshift.update(); + // if we have faceshift, use that to compute the lookat direction glm::vec3 lookAtRayOrigin = mouseRayOrigin, lookAtRayDirection = mouseRayDirection; if (_faceshift.isActive()) { lookAtRayOrigin = _myAvatar.getHead().calculateAverageEyePosition(); - float averagePitch = (_faceshift.getEyeGazeLeftPitch() + _faceshift.getEyeGazeRightPitch()) / 2.0f; - float averageYaw = (_faceshift.getEyeGazeLeftYaw() + _faceshift.getEyeGazeRightYaw()) / 2.0f; - lookAtRayDirection = _myAvatar.getHead().getOrientation() * - glm::quat(glm::radians(glm::vec3(averagePitch, averageYaw, 0.0f))) * glm::vec3(0.0f, 0.0f, -1.0f); + lookAtRayDirection = _myAvatar.getHead().getOrientation() * glm::quat(glm::radians(glm::vec3( + _faceshift.getEstimatedEyePitch(), _faceshift.getEstimatedEyeYaw(), 0.0f))) * glm::vec3(0.0f, 0.0f, -1.0f); } _isLookingAtOtherAvatar = isLookingAtOtherAvatar(lookAtRayOrigin, lookAtRayDirection, lookAtSpot); @@ -1707,8 +1708,6 @@ void Application::update(float deltaTime) { _serialHeadSensor.readData(deltaTime); } - // Update transmitter - // Sample hardware, update view frustum if needed, and send avatar data to mixer/nodes updateAvatar(deltaTime); diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index 08efb32062..abf16248c2 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -146,19 +146,6 @@ void Head::resetHairPhysics() { void Head::simulate(float deltaTime, bool isMine, float gyroCameraSensitivity) { - // Update eye saccades - const float AVERAGE_MICROSACCADE_INTERVAL = 0.50f; - const float AVERAGE_SACCADE_INTERVAL = 4.0f; - const float MICROSACCADE_MAGNITUDE = 0.002f; - const float SACCADE_MAGNITUDE = 0.04; - - if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) { - _saccadeTarget = MICROSACCADE_MAGNITUDE * randVector(); - } else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) { - _saccadeTarget = SACCADE_MAGNITUDE * randVector(); - } - _saccade += (_saccadeTarget - _saccade) * 0.50f; - // Update audio trailing average for rendering facial animations Faceshift* faceshift = Application::getInstance()->getFaceshift(); if (isMine && faceshift->isActive()) { @@ -173,6 +160,19 @@ void Head::simulate(float deltaTime, bool isMine, float gyroCameraSensitivity) { _browAudioLift = faceshift->getBrowHeight() * BROW_HEIGHT_SCALE; } else { + // Update eye saccades + const float AVERAGE_MICROSACCADE_INTERVAL = 0.50f; + const float AVERAGE_SACCADE_INTERVAL = 4.0f; + const float MICROSACCADE_MAGNITUDE = 0.002f; + const float SACCADE_MAGNITUDE = 0.04f; + + if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) { + _saccadeTarget = MICROSACCADE_MAGNITUDE * randVector(); + } else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) { + _saccadeTarget = SACCADE_MAGNITUDE * randVector(); + } + _saccade += (_saccadeTarget - _saccade) * 0.50f; + const float AUDIO_AVERAGING_SECS = 0.05; _averageLoudness = (1.f - deltaTime / AUDIO_AVERAGING_SECS) * _averageLoudness + (deltaTime / AUDIO_AVERAGING_SECS) * _audioLoudness; diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 4418d19223..399369cdd3 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -26,13 +26,30 @@ Faceshift::Faceshift() : _browHeight(0.0f), _browUpCenterIndex(-1), _mouthSize(0.0f), - _jawOpenIndex(-1) + _jawOpenIndex(-1), + _longTermAverageEyePitch(0.0f), + _longTermAverageEyeYaw(0.0f), + _estimatedEyePitch(0.0f), + _estimatedEyeYaw(0.0f) { connect(&_socket, SIGNAL(connected()), SLOT(noteConnected())); connect(&_socket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(noteError(QAbstractSocket::SocketError))); connect(&_socket, SIGNAL(readyRead()), SLOT(readFromSocket())); } +void Faceshift::update() { + if (!isActive()) { + return; + } + float averageEyePitch = (_eyeGazeLeftPitch + _eyeGazeRightPitch) / 2.0f; + float averageEyeYaw = (_eyeGazeLeftYaw + _eyeGazeRightYaw) / 2.0f; + const float LONG_TERM_AVERAGE_SMOOTHING = 0.999f; + _longTermAverageEyePitch = glm::mix(averageEyePitch, _longTermAverageEyePitch, LONG_TERM_AVERAGE_SMOOTHING); + _longTermAverageEyeYaw = glm::mix(averageEyeYaw, _longTermAverageEyeYaw, LONG_TERM_AVERAGE_SMOOTHING); + _estimatedEyePitch = averageEyePitch - _longTermAverageEyePitch; + _estimatedEyeYaw = averageEyeYaw - _longTermAverageEyeYaw; +} + void Faceshift::reset() { if (isActive()) { string message; diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 286a7a56a1..408513e5d7 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -35,6 +35,9 @@ public: float getEyeGazeRightPitch() const { return _eyeGazeRightPitch; } float getEyeGazeRightYaw() const { return _eyeGazeRightYaw; } + float getEstimatedEyePitch() const { return _estimatedEyePitch; } + float getEstimatedEyeYaw() const { return _estimatedEyeYaw; } + float getLeftBlink() const { return _leftBlink; } float getRightBlink() const { return _rightBlink; } @@ -42,6 +45,7 @@ public: float getMouthSize() const { return _mouthSize; } + void update(); void reset(); public slots: @@ -85,6 +89,12 @@ private: float _mouthSize; int _jawOpenIndex; + + float _longTermAverageEyePitch; + float _longTermAverageEyeYaw; + + float _estimatedEyePitch; + float _estimatedEyeYaw; }; #endif /* defined(__interface__Faceshift__) */ diff --git a/libraries/shared/src/Assignment.cpp b/libraries/shared/src/Assignment.cpp index 850922396f..1f8ab1c388 100644 --- a/libraries/shared/src/Assignment.cpp +++ b/libraries/shared/src/Assignment.cpp @@ -17,7 +17,8 @@ Assignment::Assignment(Assignment::Direction direction, Assignment::Type type, c _direction(direction), _type(type), _pool(NULL), - _domainSocket(NULL) + _attachedPublicSocket(NULL), + _attachedLocalSocket(NULL) { // set the create time on this assignment gettimeofday(&_time, NULL); @@ -34,7 +35,8 @@ Assignment::Assignment(Assignment::Direction direction, Assignment::Type type, c Assignment::Assignment(const unsigned char* dataBuffer, int numBytes) : _pool(NULL), - _domainSocket(NULL) + _attachedPublicSocket(NULL), + _attachedLocalSocket(NULL) { // set the create time on this assignment gettimeofday(&_time, NULL); @@ -64,41 +66,55 @@ Assignment::Assignment(const unsigned char* dataBuffer, int numBytes) : } if (numBytes > numBytesRead) { + + sockaddr* newSocket = NULL; + if (dataBuffer[numBytesRead++] == IPv4_ADDRESS_DESIGNATOR) { // IPv4 address - sockaddr_in destinationSocket = {}; - memcpy(&destinationSocket, dataBuffer + numBytesRead, sizeof(sockaddr_in)); - destinationSocket.sin_family = AF_INET; - setDomainSocket((sockaddr*) &destinationSocket); + newSocket = (sockaddr*) new sockaddr_in; + unpackSocket(dataBuffer + numBytesRead, newSocket); } else { - // IPv6 address - sockaddr_in6 destinationSocket = {}; - memcpy(&destinationSocket, dataBuffer + numBytesRead, sizeof(sockaddr_in6)); - setDomainSocket((sockaddr*) &destinationSocket); + // IPv6 address, or bad designator + qDebug("Received a socket that cannot be unpacked!\n"); + } + + if (_direction == Assignment::Create) { + delete _attachedLocalSocket; + _attachedLocalSocket = newSocket; + } else { + delete _attachedPublicSocket; + _attachedPublicSocket = newSocket; } } } Assignment::~Assignment() { - delete _domainSocket; + delete _attachedPublicSocket; + delete _attachedLocalSocket; delete _pool; } -void Assignment::setDomainSocket(const sockaddr* domainSocket) { - - if (_domainSocket) { - // delete the old _domainSocket if it exists - delete _domainSocket; - _domainSocket = NULL; +void Assignment::setAttachedPublicSocket(const sockaddr* attachedPublicSocket) { + if (_attachedPublicSocket) { + // delete the old socket if it exists + delete _attachedPublicSocket; + _attachedPublicSocket = NULL; } - // create a new sockaddr or sockaddr_in depending on what type of address this is - if (domainSocket->sa_family == AF_INET) { - _domainSocket = (sockaddr*) new sockaddr_in; - memcpy(_domainSocket, domainSocket, sizeof(sockaddr_in)); - } else { - _domainSocket = (sockaddr*) new sockaddr_in6; - memcpy(_domainSocket, domainSocket, sizeof(sockaddr_in6)); + if (attachedPublicSocket) { + copySocketToEmptySocketPointer(&_attachedPublicSocket, attachedPublicSocket); + } +} + +void Assignment::setAttachedLocalSocket(const sockaddr* attachedLocalSocket) { + if (_attachedLocalSocket) { + // delete the old socket if it exists + delete _attachedLocalSocket; + _attachedLocalSocket = NULL; + } + + if (attachedLocalSocket) { + copySocketToEmptySocketPointer(&_attachedLocalSocket, attachedLocalSocket); } } @@ -118,13 +134,14 @@ int Assignment::packToBuffer(unsigned char* buffer) { numPackedBytes += sizeof(char); } - if (_domainSocket) { - buffer[numPackedBytes++] = (_domainSocket->sa_family == AF_INET) ? IPv4_ADDRESS_DESIGNATOR : IPv6_ADDRESS_DESIGNATOR; + if (_attachedPublicSocket || _attachedLocalSocket) { + sockaddr* socketToPack = (_attachedPublicSocket) ? _attachedPublicSocket : _attachedLocalSocket; - int numSocketBytes = (_domainSocket->sa_family == AF_INET) ? sizeof(sockaddr_in) : sizeof(sockaddr_in6); + // we have a socket to pack, add the designator + buffer[numPackedBytes++] = (socketToPack->sa_family == AF_INET) + ? IPv4_ADDRESS_DESIGNATOR : IPv6_ADDRESS_DESIGNATOR; - memcpy(buffer + numPackedBytes, _domainSocket, numSocketBytes); - numPackedBytes += numSocketBytes; + numPackedBytes += packSocket(buffer + numPackedBytes, socketToPack); } return numPackedBytes; diff --git a/libraries/shared/src/Assignment.h b/libraries/shared/src/Assignment.h index 693cc42577..11966b2dc7 100644 --- a/libraries/shared/src/Assignment.h +++ b/libraries/shared/src/Assignment.h @@ -43,8 +43,11 @@ public: const char* getPool() const { return _pool; } const timeval& getTime() const { return _time; } - const sockaddr* getDomainSocket() { return _domainSocket; } - void setDomainSocket(const sockaddr* domainSocket); + const sockaddr* getAttachedPublicSocket() { return _attachedPublicSocket; } + void setAttachedPublicSocket(const sockaddr* attachedPublicSocket); + + const sockaddr* getAttachedLocalSocket() { return _attachedLocalSocket; } + void setAttachedLocalSocket(const sockaddr* attachedLocalSocket); /// Packs the assignment to the passed buffer /// \param buffer the buffer in which to pack the assignment @@ -58,7 +61,8 @@ private: Assignment::Direction _direction; /// the direction of the assignment (Create, Deploy, Request) Assignment::Type _type; /// the type of the assignment, defines what the assignee will do char* _pool; /// the pool this assignment is for/from - sockaddr* _domainSocket; /// pointer to socket for domain server that created assignment + sockaddr* _attachedPublicSocket; /// pointer to a public socket that relates to assignment, depends on direction + sockaddr* _attachedLocalSocket; /// pointer to a local socket that relates to assignment, depends on direction timeval _time; /// time the assignment was created (set in constructor) }; diff --git a/libraries/shared/src/UDPSocket.cpp b/libraries/shared/src/UDPSocket.cpp index d89265b471..11fd218a56 100644 --- a/libraries/shared/src/UDPSocket.cpp +++ b/libraries/shared/src/UDPSocket.cpp @@ -68,13 +68,25 @@ int packSocket(unsigned char* packStore, sockaddr* socketToPack) { return packSocket(packStore, ((sockaddr_in*) socketToPack)->sin_addr.s_addr, ((sockaddr_in*) socketToPack)->sin_port); } -int unpackSocket(unsigned char* packedData, sockaddr* unpackDestSocket) { +int unpackSocket(const unsigned char* packedData, sockaddr* unpackDestSocket) { sockaddr_in* destinationSocket = (sockaddr_in*) unpackDestSocket; + destinationSocket->sin_family = AF_INET; destinationSocket->sin_addr.s_addr = (packedData[0] << 24) + (packedData[1] << 16) + (packedData[2] << 8) + packedData[3]; destinationSocket->sin_port = (packedData[4] << 8) + packedData[5]; return 6; // this could be more if we ever need IPv6 } +void copySocketToEmptySocketPointer(sockaddr** destination, const sockaddr* source) { + // create a new sockaddr or sockaddr_in depending on what type of address this is + if (source->sa_family == AF_INET) { + *destination = (sockaddr*) new sockaddr_in; + memcpy(*destination, source, sizeof(sockaddr_in)); + } else { + *destination = (sockaddr*) new sockaddr_in6; + memcpy(*destination, source, sizeof(sockaddr_in6)); + } +} + int getLocalAddress() { // get this node's local address so we can pass that to DS struct ifaddrs* ifAddrStruct = NULL; diff --git a/libraries/shared/src/UDPSocket.h b/libraries/shared/src/UDPSocket.h index 1b627dbd41..d3f7cfac58 100644 --- a/libraries/shared/src/UDPSocket.h +++ b/libraries/shared/src/UDPSocket.h @@ -44,7 +44,8 @@ private: bool socketMatch(const sockaddr* first, const sockaddr* second); int packSocket(unsigned char* packStore, in_addr_t inAddress, in_port_t networkOrderPort); int packSocket(unsigned char* packStore, sockaddr* socketToPack); -int unpackSocket(unsigned char* packedData, sockaddr* unpackDestSocket); +int unpackSocket(const unsigned char* packedData, sockaddr* unpackDestSocket); +void copySocketToEmptySocketPointer(sockaddr** destination, const sockaddr* source); int getLocalAddress(); unsigned short loadBufferWithSocketInfo(char* addressBuffer, sockaddr* socket); sockaddr_in socketForHostnameAndHostOrderPort(const char* hostname, unsigned short port = 0);