From f234dbf9a10a3909c439cf2da22b72d1346e66a9 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Fri, 5 Sep 2014 10:51:08 -0700 Subject: [PATCH] converting voxel files to new versioned format with buffer breaks --- libraries/networking/src/PacketHeaders.cpp | 64 ++++++++++++++++++++++ libraries/networking/src/PacketHeaders.h | 4 +- libraries/octree/src/Octree.cpp | 27 ++++++--- libraries/octree/src/OctreeConstants.h | 2 + libraries/voxels/src/VoxelTree.h | 16 ++++++ 5 files changed, 105 insertions(+), 8 deletions(-) diff --git a/libraries/networking/src/PacketHeaders.cpp b/libraries/networking/src/PacketHeaders.cpp index 91de48296f..f3937dd2e3 100644 --- a/libraries/networking/src/PacketHeaders.cpp +++ b/libraries/networking/src/PacketHeaders.cpp @@ -83,11 +83,75 @@ PacketVersion versionForPacketType(PacketType type) { return 1; case PacketTypeMetavoxelData: return 3; + case PacketTypeVoxelData: + return VERSION_VOXELS_HAS_FILE_BREAKS; default: return 0; } } +#define PACKET_TYPE_NAME_LOOKUP(x) case x: return QString(#x); + +QString nameForPacketType(PacketType type) { + switch (type) { + PACKET_TYPE_NAME_LOOKUP(PacketTypeUnknown); + PACKET_TYPE_NAME_LOOKUP(PacketTypeStunResponse); + PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainList); + PACKET_TYPE_NAME_LOOKUP(PacketTypePing); + PACKET_TYPE_NAME_LOOKUP(PacketTypePingReply); + PACKET_TYPE_NAME_LOOKUP(PacketTypeKillAvatar); + PACKET_TYPE_NAME_LOOKUP(PacketTypeAvatarData); + PACKET_TYPE_NAME_LOOKUP(PacketTypeInjectAudio); + PACKET_TYPE_NAME_LOOKUP(PacketTypeMixedAudio); + PACKET_TYPE_NAME_LOOKUP(PacketTypeMicrophoneAudioNoEcho); + PACKET_TYPE_NAME_LOOKUP(PacketTypeMicrophoneAudioWithEcho); + PACKET_TYPE_NAME_LOOKUP(PacketTypeBulkAvatarData); + PACKET_TYPE_NAME_LOOKUP(PacketTypeSilentAudioFrame); + PACKET_TYPE_NAME_LOOKUP(PacketTypeEnvironmentData); + PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainListRequest); + PACKET_TYPE_NAME_LOOKUP(PacketTypeRequestAssignment); + PACKET_TYPE_NAME_LOOKUP(PacketTypeCreateAssignment); + PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainOAuthRequest); + PACKET_TYPE_NAME_LOOKUP(PacketTypeMuteEnvironment); + PACKET_TYPE_NAME_LOOKUP(PacketTypeAudioStreamStats); + PACKET_TYPE_NAME_LOOKUP(PacketTypeDataServerConfirm); + PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelQuery); + PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelData); + PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelSet); + PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelSetDestructive); + PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelErase); + PACKET_TYPE_NAME_LOOKUP(PacketTypeOctreeStats); + PACKET_TYPE_NAME_LOOKUP(PacketTypeJurisdiction); + PACKET_TYPE_NAME_LOOKUP(PacketTypeJurisdictionRequest); + PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleQuery); + PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleData); + PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleAddOrEdit); + PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleErase); + PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleAddResponse); + PACKET_TYPE_NAME_LOOKUP(PacketTypeMetavoxelData); + PACKET_TYPE_NAME_LOOKUP(PacketTypeAvatarIdentity); + PACKET_TYPE_NAME_LOOKUP(PacketTypeAvatarBillboard); + PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainConnectRequest); + PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainServerRequireDTLS); + PACKET_TYPE_NAME_LOOKUP(PacketTypeNodeJsonStats); + PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityQuery); + PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityData); + PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityAddOrEdit); + PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityErase); + PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityAddResponse); + PACKET_TYPE_NAME_LOOKUP(PacketTypeOctreeDataNack); + PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelEditNack); + PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleEditNack); + PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityEditNack); + PACKET_TYPE_NAME_LOOKUP(PacketTypeSignedTransactionPayment); + default: + return QString("Type: ") + QString::number((int)type); + } + return QString("unexpected"); +} + + + QByteArray byteArrayWithPopulatedHeader(PacketType type, const QUuid& connectionUUID) { QByteArray freshByteArray(MAX_PACKET_HEADER_BYTES, 0); freshByteArray.resize(populatePacketHeader(freshByteArray, type, connectionUUID)); diff --git a/libraries/networking/src/PacketHeaders.h b/libraries/networking/src/PacketHeaders.h index a558710c67..bb64388dd6 100644 --- a/libraries/networking/src/PacketHeaders.h +++ b/libraries/networking/src/PacketHeaders.h @@ -19,7 +19,7 @@ #include "UUID.h" // NOTE: if adding a new packet type, you can replace one marked usable or add at the end - +// NOTE: if you want the name of the packet type to be available for debugging or logging, update nameForPacketType() as well enum PacketType { PacketTypeUnknown, PacketTypeStunResponse, @@ -87,6 +87,7 @@ const int NUM_STATIC_HEADER_BYTES = sizeof(PacketVersion) + NUM_BYTES_RFC4122_UU const int MAX_PACKET_HEADER_BYTES = sizeof(PacketType) + NUM_BYTES_MD5_HASH + NUM_STATIC_HEADER_BYTES; PacketVersion versionForPacketType(PacketType type); +QString nameForPacketType(PacketType type); const QUuid nullUUID = QUuid(); @@ -116,5 +117,6 @@ const PacketVersion VERSION_ENTITIES_HAVE_ANIMATION = 1; const PacketVersion VERSION_ROOT_ELEMENT_HAS_DATA = 2; const PacketVersion VERSION_ENTITIES_SUPPORT_SPLIT_MTU = 3; const PacketVersion VERSION_ENTITIES_HAS_FILE_BREAKS = VERSION_ENTITIES_SUPPORT_SPLIT_MTU; +const PacketVersion VERSION_VOXELS_HAS_FILE_BREAKS = 1; #endif // hifi_PacketHeaders_h diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp index abb98f56e8..7f21d519f6 100644 --- a/libraries/octree/src/Octree.cpp +++ b/libraries/octree/src/Octree.cpp @@ -247,7 +247,7 @@ int Octree::readElementData(OctreeElement* destinationElement, const unsigned ch return bytesAvailable; // assume we read the entire buffer... } - if (destinationElement->getScale() < SMALLEST_REASONABLE_OCTREE_ELEMENT_SCALE) { + if (destinationElement->getScale() < SCALE_AT_DANGEROUSLY_DEEP_RECURSION) { qDebug() << "UNEXPECTED: readElementData() destination element is unreasonably small [" << destinationElement->getScale() * (float)TREE_SCALE << " meters] " << " Discarding " << bytesAvailable << " remaining bytes."; @@ -1893,11 +1893,24 @@ bool Octree::readFromSVOFile(const char* fileName) { // if so, read the first byte of the file and see if it matches the expected version code PacketType gotType; memcpy(&gotType, dataAt, sizeof(gotType)); + + dataAt += sizeof(expectedType); + dataLength -= sizeof(expectedType); + gotVersion = *dataAt; + + // NOTE: SPECIAL CASE for old voxel svo files. The old voxel SVO files didn't have header + // details. They started with the the octalcode for the root. Which was always 00 which matches PacketTypeUnknown + unsigned char* firstByteAt = (unsigned char*)&fileHeader; + unsigned char firstByteValue = *firstByteAt; + if (expectedType == PacketTypeVoxelData && firstByteValue == 0) { + gotType = PacketTypeVoxelData; + gotVersion = 0; + qDebug() << "Detected OLD Voxels format."; + headerLength = 0; // old format files don't have headers + file.seekg( 0, std::ios::beg ); // rewind to the beginning so old logic will work + } if (gotType == expectedType) { - dataAt += sizeof(expectedType); - dataLength -= sizeof(expectedType); - gotVersion = *dataAt; if (canProcessVersion(gotVersion)) { dataAt += sizeof(gotVersion); dataLength -= sizeof(gotVersion); @@ -1911,7 +1924,8 @@ bool Octree::readFromSVOFile(const char* fileName) { versionForPacketType(expectedDataPacketType()), gotVersion); } } else { - qDebug("SVO file type mismatch. Expected: %c Got: %c", expectedType, gotType); + qDebug() << "SVO file type mismatch. Expected: " << nameForPacketType(expectedType) + << " Got: " << nameForPacketType(gotType); } } else { @@ -2008,8 +2022,7 @@ void Octree::writeToSVOFile(const char* fileName, OctreeElement* element) { // if so, read the first byte of the file and see if it matches the expected version code file.write(reinterpret_cast(&expectedType), sizeof(expectedType)); file.write(&expectedVersion, sizeof(expectedVersion)); - - qDebug("SVO file type: %c version: %d", expectedType, expectedVersion); + qDebug() << "SVO file type: " << nameForPacketType(expectedType) << " version: " << (int)expectedVersion; hasBufferBreaks = versionHasSVOfileBreaks(expectedVersion); } diff --git a/libraries/octree/src/OctreeConstants.h b/libraries/octree/src/OctreeConstants.h index 0f836c7515..3fdb7b4af1 100644 --- a/libraries/octree/src/OctreeConstants.h +++ b/libraries/octree/src/OctreeConstants.h @@ -37,6 +37,8 @@ const float VIEW_FRUSTUM_FOV_OVERSEND = 60.0f; // These are guards to prevent our voxel tree recursive routines from spinning out of control const int UNREASONABLY_DEEP_RECURSION = 20; // use this for something that you want to be shallow, but not spin out const int DANGEROUSLY_DEEP_RECURSION = 200; // use this for something that needs to go deeper +const float SCALE_AT_UNREASONABLY_DEEP_RECURSION = (1.0f / powf(2.0f, DANGEROUSLY_DEEP_RECURSION)); +const float SCALE_AT_DANGEROUSLY_DEEP_RECURSION = (1.0f / powf(2.0f, DANGEROUSLY_DEEP_RECURSION)); const int DEFAULT_MAX_OCTREE_PPS = 600; // the default maximum PPS we think any octree based server should send to a client diff --git a/libraries/voxels/src/VoxelTree.h b/libraries/voxels/src/VoxelTree.h index 220c6a0ecd..547f590270 100644 --- a/libraries/voxels/src/VoxelTree.h +++ b/libraries/voxels/src/VoxelTree.h @@ -51,12 +51,28 @@ public: void readCodeColorBufferToTree(const unsigned char* codeColorBuffer, bool destructive = false); + + virtual bool getWantSVOfileVersions() const { return true; } + virtual bool canProcessVersion(PacketVersion thisVersion) const { + return thisVersion == 0 || thisVersion == versionForPacketType(expectedDataPacketType()); } + virtual PacketVersion expectedVersion() const { return versionForPacketType(expectedDataPacketType()); } + virtual PacketType expectedDataPacketType() const { return PacketTypeVoxelData; } virtual bool handlesEditPacketType(PacketType packetType) const; virtual int processEditPacketData(PacketType packetType, const unsigned char* packetData, int packetLength, const unsigned char* editData, int maxLength, const SharedNodePointer& node); virtual bool recurseChildrenWithData() const { return false; } + /// some versions of the SVO file will include breaks with buffer lengths between each buffer chunk in the SVO + /// file. If the Octree subclass expects this for this particular version of the file, it should override this + /// method and return true. + virtual bool versionHasSVOfileBreaks(PacketVersion thisVersion) const { + if (thisVersion == 0) { + return false; // old versions didn't have buffer breaks + } + return true; + } + virtual void dumpTree(); private: