From 4d3659b627530250e256d944d7f7494d0c58f983 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 8 Jul 2015 16:50:35 -0700 Subject: [PATCH] update packet creation in AvatarData to new API --- libraries/avatars/src/AvatarData.cpp | 176 ++++++++++++++------------- 1 file changed, 90 insertions(+), 86 deletions(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index efc8e5c157..dfac2f9b46 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -87,7 +87,7 @@ glm::quat AvatarData::getOrientation() const { if (_referential) { _referential->update(); } - + return glm::quat(glm::radians(glm::vec3(_bodyPitch, _bodyYaw, _bodyRoll))); } @@ -104,7 +104,7 @@ float AvatarData::getTargetScale() const { if (_referential) { _referential->update(); } - + return _targetScale; } @@ -115,9 +115,9 @@ void AvatarData::setTargetScale(float targetScale, bool overideReferential) { } void AvatarData::setClampedTargetScale(float targetScale, bool overideReferential) { - + targetScale = glm::clamp(targetScale, MIN_AVATAR_SCALE, MAX_AVATAR_SCALE); - + setTargetScale(targetScale, overideReferential); qCDebug(avatars) << "Changed scale to " << _targetScale; } @@ -135,7 +135,7 @@ QByteArray AvatarData::toByteArray() { // TODO: DRY this up to a shared method // that can pack any type given the number of bytes // and return the number of bytes to push the pointer - + // lazily allocate memory for HeadData in case we're not an Avatar instance if (!_headData) { _headData = new HeadData(this); @@ -143,21 +143,21 @@ QByteArray AvatarData::toByteArray() { if (_forceFaceTrackerConnected) { _headData->_isFaceTrackerConnected = true; } - + QByteArray avatarDataByteArray; avatarDataByteArray.resize(MAX_PACKET_SIZE); - + unsigned char* destinationBuffer = reinterpret_cast(avatarDataByteArray.data()); unsigned char* startPosition = destinationBuffer; - + memcpy(destinationBuffer, &_position, sizeof(_position)); destinationBuffer += sizeof(_position); - + // Body rotation (NOTE: This needs to become a quaternion to save two bytes) destinationBuffer += packFloatAngleToTwoByte(destinationBuffer, _bodyYaw); destinationBuffer += packFloatAngleToTwoByte(destinationBuffer, _bodyPitch); destinationBuffer += packFloatAngleToTwoByte(destinationBuffer, _bodyRoll); - + // Body scale destinationBuffer += packFloatRatioToTwoByte(destinationBuffer, _targetScale); @@ -178,11 +178,11 @@ QByteArray AvatarData::toByteArray() { // Lookat Position memcpy(destinationBuffer, &_headData->_lookAtPosition, sizeof(_headData->_lookAtPosition)); destinationBuffer += sizeof(_headData->_lookAtPosition); - + // Instantaneous audio loudness (used to drive facial animation) memcpy(destinationBuffer, &_headData->_audioLoudness, sizeof(float)); destinationBuffer += sizeof(float); - + // bitMask of less than byte wide items unsigned char bitItems = 0; @@ -202,7 +202,7 @@ QByteArray AvatarData::toByteArray() { setAtBit(bitItems, HAS_REFERENTIAL); } *destinationBuffer++ = bitItems; - + // Add referential if (_referential != NULL && _referential->isValid()) { destinationBuffer += _referential->packReferential(destinationBuffer); @@ -221,13 +221,13 @@ QByteArray AvatarData::toByteArray() { memcpy(destinationBuffer, &_headData->_browAudioLift, sizeof(float)); destinationBuffer += sizeof(float); - + *destinationBuffer++ = _headData->_blendshapeCoefficients.size(); memcpy(destinationBuffer, _headData->_blendshapeCoefficients.data(), _headData->_blendshapeCoefficients.size() * sizeof(float)); destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); } - + // pupil dilation destinationBuffer += packFloatToByte(destinationBuffer, _headData->_pupilDilation, 1.0f); @@ -252,7 +252,7 @@ QByteArray AvatarData::toByteArray() { destinationBuffer += packOrientationQuatToBytes(destinationBuffer, data.rotation); } } - + return avatarDataByteArray.left(destinationBuffer - startPosition); } @@ -266,17 +266,17 @@ bool AvatarData::shouldLogError(const quint64& now) { // read data in packet starting at byte offset and return number of bytes parsed int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { - + // lazily allocate memory for HeadData in case we're not an Avatar instance if (!_headData) { _headData = new HeadData(this); } - + // lazily allocate memory for HandData in case we're not an Avatar instance if (!_handData) { _handData = new HandData(this); } - + const unsigned char* startPosition = reinterpret_cast(packet.data()) + offset; const unsigned char* sourceBuffer = startPosition; quint64 now = usecTimestampNow(); @@ -298,13 +298,13 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { // + 1 byte for numJoints (0) // = 45 bytes int minPossibleSize = 45; - + int maxAvailableSize = packet.size() - offset; if (minPossibleSize > maxAvailableSize) { if (shouldLogError(now)) { qCDebug(avatars) << "Malformed AvatarData packet at the start; " << " displayName = '" << _displayName << "'" - << " minPossibleSize = " << minPossibleSize + << " minPossibleSize = " << minPossibleSize << " maxAvailableSize = " << maxAvailableSize; } // this packet is malformed so we report all bytes as consumed @@ -316,7 +316,7 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { glm::vec3 position; memcpy(&position, sourceBuffer, sizeof(position)); sourceBuffer += sizeof(position); - + if (glm::isnan(position.x) || glm::isnan(position.y) || glm::isnan(position.z)) { if (shouldLogError(now)) { qCDebug(avatars) << "Discard nan AvatarData::position; displayName = '" << _displayName << "'"; @@ -324,7 +324,7 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { return maxAvailableSize; } setPosition(position); - + // rotation (NOTE: This needs to become a quaternion to save two bytes) float yaw, pitch, roll; sourceBuffer += unpackFloatAngleFromTwoByte((uint16_t*) sourceBuffer, &yaw); @@ -342,7 +342,7 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { _bodyPitch = pitch; _bodyRoll = roll; } - + // scale float scale; sourceBuffer += unpackFloatRatioFromTwoByte(sourceBuffer, scale); @@ -354,8 +354,8 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { } _targetScale = scale; } // 20 bytes - - { // Head rotation + + { // Head rotation //(NOTE: This needs to become a quaternion to save two bytes) float headYaw, headPitch, headRoll; sourceBuffer += unpackFloatAngleFromTwoByte((uint16_t*) sourceBuffer, &headPitch); @@ -371,7 +371,7 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { _headData->setBaseYaw(headYaw); _headData->setBaseRoll(headRoll); } // 6 bytes - + { // Lookat Position glm::vec3 lookAt; memcpy(&lookAt, sourceBuffer, sizeof(lookAt)); @@ -384,7 +384,7 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { } _headData->_lookAtPosition = lookAt; } // 12 bytes - + { // AudioLoudness // Instantaneous audio loudness (used to drive facial animation) float audioLoudness; @@ -398,26 +398,26 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { } _headData->_audioLoudness = audioLoudness; } // 4 bytes - + { // bitFlags and face data unsigned char bitItems = *sourceBuffer++; - + // key state, stored as a semi-nibble in the bitItems _keyState = (KeyState)getSemiNibbleAt(bitItems,KEY_STATE_START_BIT); // hand state, stored as a semi-nibble plus a bit in the bitItems - // we store the hand state as well as other items in a shared bitset. The hand state is an octal, but is split + // we store the hand state as well as other items in a shared bitset. The hand state is an octal, but is split // into two sections to maintain backward compatibility. The bits are ordered as such (0-7 left to right). // +---+-----+-----+--+ // |x,x|H0,H1|x,x,x|H2| // +---+-----+-----+--+ // Hand state - H0,H1,H2 is found in the 3rd, 4th, and 8th bits - _handState = getSemiNibbleAt(bitItems, HAND_STATE_START_BIT) + _handState = getSemiNibbleAt(bitItems, HAND_STATE_START_BIT) + (oneAtBit(bitItems, HAND_STATE_FINGER_POINTING_BIT) ? IS_FINGER_POINTING_FLAG : 0); - + _headData->_isFaceTrackerConnected = oneAtBit(bitItems, IS_FACESHIFT_CONNECTED); bool hasReferential = oneAtBit(bitItems, HAS_REFERENTIAL); - + // Referential if (hasReferential) { Referential* ref = new Referential(sourceBuffer, this); @@ -431,8 +431,8 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { } else if (_referential != NULL) { changeReferential(NULL); } - - + + if (_headData->_isFaceTrackerConnected) { float leftEyeBlink, rightEyeBlink, averageLoudness, browAudioLift; minPossibleSize += sizeof(leftEyeBlink) + sizeof(rightEyeBlink) + sizeof(averageLoudness) + sizeof(browAudioLift); @@ -441,7 +441,7 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { if (shouldLogError(now)) { qCDebug(avatars) << "Malformed AvatarData packet after BitItems;" << " displayName = '" << _displayName << "'" - << " minPossibleSize = " << minPossibleSize + << " minPossibleSize = " << minPossibleSize << " maxAvailableSize = " << maxAvailableSize; } return maxAvailableSize; @@ -449,17 +449,17 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { // unpack face data memcpy(&leftEyeBlink, sourceBuffer, sizeof(float)); sourceBuffer += sizeof(float); - + memcpy(&rightEyeBlink, sourceBuffer, sizeof(float)); sourceBuffer += sizeof(float); - + memcpy(&averageLoudness, sourceBuffer, sizeof(float)); sourceBuffer += sizeof(float); - + memcpy(&browAudioLift, sourceBuffer, sizeof(float)); sourceBuffer += sizeof(float); - - if (glm::isnan(leftEyeBlink) || glm::isnan(rightEyeBlink) + + if (glm::isnan(leftEyeBlink) || glm::isnan(rightEyeBlink) || glm::isnan(averageLoudness) || glm::isnan(browAudioLift)) { if (shouldLogError(now)) { qCDebug(avatars) << "Discard nan AvatarData::faceData; displayName = '" << _displayName << "'"; @@ -470,7 +470,7 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { _headData->_rightEyeBlink = rightEyeBlink; _headData->_averageLoudness = averageLoudness; _headData->_browAudioLift = browAudioLift; - + int numCoefficients = (int)(*sourceBuffer++); int blendDataSize = numCoefficients * sizeof(float); minPossibleSize += blendDataSize; @@ -478,7 +478,7 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { if (shouldLogError(now)) { qCDebug(avatars) << "Malformed AvatarData packet after Blendshapes;" << " displayName = '" << _displayName << "'" - << " minPossibleSize = " << minPossibleSize + << " minPossibleSize = " << minPossibleSize << " maxAvailableSize = " << maxAvailableSize; } return maxAvailableSize; @@ -487,15 +487,15 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { _headData->_blendshapeCoefficients.resize(numCoefficients); memcpy(_headData->_blendshapeCoefficients.data(), sourceBuffer, blendDataSize); sourceBuffer += numCoefficients * sizeof(float); - + //bitItemsDataSize = 4 * sizeof(float) + 1 + blendDataSize; } } // 1 + bitItemsDataSize bytes - + { // pupil dilation sourceBuffer += unpackFloatFromByte(sourceBuffer, _headData->_pupilDilation, 1.0f); } // 1 byte - + // joint data int numJoints = *sourceBuffer++; int bytesOfValidity = (int)ceil((float)numJoints / (float)BITS_IN_BYTE); @@ -504,7 +504,7 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { if (shouldLogError(now)) { qCDebug(avatars) << "Malformed AvatarData packet after JointValidityBits;" << " displayName = '" << _displayName << "'" - << " minPossibleSize = " << minPossibleSize + << " minPossibleSize = " << minPossibleSize << " maxAvailableSize = " << maxAvailableSize; } return maxAvailableSize; @@ -523,7 +523,7 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { ++numValidJoints; } _jointData[i].valid = valid; - validityBit = (validityBit + 1) % BITS_IN_BYTE; + validityBit = (validityBit + 1) % BITS_IN_BYTE; } } // 1 + bytesOfValidity bytes @@ -535,7 +535,7 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { if (shouldLogError(now)) { qCDebug(avatars) << "Malformed AvatarData packet after JointData;" << " displayName = '" << _displayName << "'" - << " minPossibleSize = " << minPossibleSize + << " minPossibleSize = " << minPossibleSize << " maxAvailableSize = " << maxAvailableSize; } return maxAvailableSize; @@ -550,18 +550,18 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { } } } // numJoints * 8 bytes - + int numBytesRead = sourceBuffer - startPosition; - _averageBytesReceived.updateAverage(numBytesRead); + _averageBytesReceived.updateAverage(numBytesRead); return numBytesRead; } int AvatarData::getAverageBytesReceivedPerSecond() const { - return lrint(_averageBytesReceived.getAverageSampleValuePerSecond()); + return lrint(_averageBytesReceived.getAverageSampleValuePerSecond()); } int AvatarData::getReceiveRate() const { - return lrint(1.0f / _averageBytesReceived.getEventDeltaAverage()); + return lrint(1.0f / _averageBytesReceived.getEventDeltaAverage()); } bool AvatarData::hasReferential() { @@ -619,7 +619,7 @@ void AvatarData::loadRecording(QString filename) { if (!_player) { _player = PlayerPointer(new Player(this)); } - + _player->loadFromFile(filename); } @@ -700,7 +700,7 @@ void AvatarData::play() { QMetaObject::invokeMethod(this, "play", Qt::BlockingQueuedConnection); return; } - + _player->play(); } } @@ -870,20 +870,20 @@ void AvatarData::clearJointsData() { bool AvatarData::hasIdentityChangedAfterParsing(const QByteArray &packet) { QDataStream packetStream(packet); packetStream.skipRawData(numBytesForPacketHeader(packet)); - + QUuid avatarUUID; QUrl faceModelURL, skeletonModelURL; QVector attachmentData; QString displayName; packetStream >> avatarUUID >> faceModelURL >> skeletonModelURL >> attachmentData >> displayName; - + bool hasIdentityChanged = false; - + if (faceModelURL != _faceModelURL) { setFaceModelURL(faceModelURL); hasIdentityChanged = true; } - + if (skeletonModelURL != _skeletonModelURL) { setSkeletonModelURL(skeletonModelURL); hasIdentityChanged = true; @@ -893,12 +893,12 @@ bool AvatarData::hasIdentityChangedAfterParsing(const QByteArray &packet) { setDisplayName(displayName); hasIdentityChanged = true; } - + if (attachmentData != _attachmentData) { setAttachmentData(attachmentData); hasIdentityChanged = true; } - + return hasIdentityChanged; } @@ -907,7 +907,7 @@ QByteArray AvatarData::identityByteArray() { QDataStream identityStream(&identityData, QIODevice::Append); identityStream << QUuid() << _faceModelURL << _skeletonModelURL << _attachmentData << _displayName; - + return identityData; } @@ -922,15 +922,15 @@ bool AvatarData::hasBillboardChangedAfterParsing(const QByteArray& packet) { void AvatarData::setFaceModelURL(const QUrl& faceModelURL) { _faceModelURL = faceModelURL; - + qCDebug(avatars) << "Changing face model for avatar to" << _faceModelURL.toString(); } void AvatarData::setSkeletonModelURL(const QUrl& skeletonModelURL) { _skeletonModelURL = skeletonModelURL.isEmpty() ? DEFAULT_BODY_MODEL_URL : skeletonModelURL; - + qCDebug(avatars) << "Changing skeleton model for avatar to" << _skeletonModelURL.toString(); - + updateJointMappings(); } @@ -1017,20 +1017,20 @@ void AvatarData::detachAll(const QString& modelURL, const QString& jointName) { void AvatarData::setBillboard(const QByteArray& billboard) { _billboard = billboard; - + qCDebug(avatars) << "Changing billboard for avatar."; } void AvatarData::setBillboardFromURL(const QString &billboardURL) { _billboardURL = billboardURL; - - + + qCDebug(avatars) << "Changing billboard for avatar to PNG at" << qPrintable(billboardURL); - + QNetworkRequest billboardRequest; billboardRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); billboardRequest.setUrl(QUrl(billboardURL)); - + QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); QNetworkReply* networkReply = networkAccessManager.get(billboardRequest); connect(networkReply, SIGNAL(finished()), this, SLOT(setBillboardFromNetworkReply())); @@ -1044,7 +1044,7 @@ void AvatarData::setBillboardFromNetworkReply() { void AvatarData::setJointMappingsFromNetworkReply() { QNetworkReply* networkReply = static_cast(sender()); - + QByteArray line; while (!(line = networkReply->readLine()).isEmpty()) { if (!(line = line.trimmed()).startsWith("jointIndex")) { @@ -1071,43 +1071,47 @@ void AvatarData::setJointMappingsFromNetworkReply() { for (int i = 0; i < _jointNames.size(); i++) { _jointIndices.insert(_jointNames.at(i), i + 1); } - + networkReply->deleteLater(); } void AvatarData::sendAvatarDataPacket() { auto nodeList = DependencyManager::get(); - QByteArray dataPacket = nodeList->byteArrayWithPopulatedHeader(PacketType::AvatarData); - dataPacket.append(toByteArray()); - - nodeList->broadcastToNodes(dataPacket, NodeSet() << NodeType::AvatarMixer); + QByteArray avatarByteArray = toByteArray(); + + auto avatarPacket = NLPacket::create(PacketType::AvatarData, avatarByteArray.size()); + avatarPacket->write(avatarByteArray); + + nodeList->broadcastToNodes(std::move(avatarPacket), NodeSet() << NodeType::AvatarMixer); } void AvatarData::sendIdentityPacket() { auto nodeList = DependencyManager::get(); - QByteArray identityPacket = nodeList->byteArrayWithPopulatedHeader(PacketType::AvatarIdentity); - identityPacket.append(identityByteArray()); - - nodeList->broadcastToNodes(identityPacket, NodeSet() << NodeType::AvatarMixer); + QByteArray identityByteArray = identityByteArray(); + + auto identityPacket = NLPacket::create(PacketType::AvatarIdentity, identityByteArray.size()); + identityPacket->write(identityByteArray); + + nodeList->broadcastToNodes(std::move(identityPacket), NodeSet() << NodeType::AvatarMixer); } void AvatarData::sendBillboardPacket() { if (!_billboard.isEmpty()) { auto nodeList = DependencyManager::get(); - QByteArray billboardPacket = nodeList->byteArrayWithPopulatedHeader(PacketType::AvatarBillboard); - billboardPacket.append(_billboard); - - nodeList->broadcastToNodes(billboardPacket, NodeSet() << NodeType::AvatarMixer); + auto billboardPacket = NLPacket::create(PacketType::AvatarBillboard, _billboard.size()); + billboardPacket->write(_billboard); + + nodeList->broadcastToNodes(std::move(billboardPacket), NodeSet() << NodeType::AvatarMixer); } } void AvatarData::updateJointMappings() { _jointIndices.clear(); _jointNames.clear(); - + if (_skeletonModelURL.fileName().toLower().endsWith(".fst")) { QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); QNetworkRequest networkRequest = QNetworkRequest(_skeletonModelURL);