converting voxel files to new versioned format with buffer breaks

libraries/networking/src/PacketHeaders.cpp

@@ -83,11 +83,75 @@ PacketVersion versionForPacketType(PacketType type) {
             return 1;
         case PacketTypeMetavoxelData:
             return 3;
+        case PacketTypeVoxelData:
+            return VERSION_VOXELS_HAS_FILE_BREAKS;
         default:
             return 0;
     }
 }
 
+#define PACKET_TYPE_NAME_LOOKUP(x) case x:  return QString(#x);
+
+QString nameForPacketType(PacketType type) {
+    switch (type) {
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeUnknown);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeStunResponse);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainList);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypePing);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypePingReply);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeKillAvatar);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeAvatarData);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeInjectAudio);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeMixedAudio);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeMicrophoneAudioNoEcho);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeMicrophoneAudioWithEcho);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeBulkAvatarData);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeSilentAudioFrame);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeEnvironmentData);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainListRequest);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeRequestAssignment);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeCreateAssignment);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainOAuthRequest);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeMuteEnvironment);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeAudioStreamStats);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeDataServerConfirm);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelQuery);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelData);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelSet);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelSetDestructive);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelErase);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeOctreeStats);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeJurisdiction);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeJurisdictionRequest);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleQuery);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleData);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleAddOrEdit);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleErase);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleAddResponse);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeMetavoxelData);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeAvatarIdentity);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeAvatarBillboard);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainConnectRequest);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainServerRequireDTLS);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeNodeJsonStats);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityQuery);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityData);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityAddOrEdit);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityErase);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityAddResponse);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeOctreeDataNack);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelEditNack);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleEditNack);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityEditNack);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeSignedTransactionPayment);
+        default:
+            return QString("Type: ") + QString::number((int)type);
+    }
+    return QString("unexpected");
+}
+
+
+
 QByteArray byteArrayWithPopulatedHeader(PacketType type, const QUuid& connectionUUID) {
     QByteArray freshByteArray(MAX_PACKET_HEADER_BYTES, 0);
     freshByteArray.resize(populatePacketHeader(freshByteArray, type, connectionUUID));

libraries/networking/src/PacketHeaders.h

@@ -19,7 +19,7 @@
 #include "UUID.h"
 
 // NOTE: if adding a new packet type, you can replace one marked usable or add at the end
-
+// NOTE: if you want the name of the packet type to be available for debugging or logging, update nameForPacketType() as well
 enum PacketType {
     PacketTypeUnknown,
     PacketTypeStunResponse,
@@ -87,6 +87,7 @@ const int NUM_STATIC_HEADER_BYTES = sizeof(PacketVersion) + NUM_BYTES_RFC4122_UU
 const int MAX_PACKET_HEADER_BYTES = sizeof(PacketType) + NUM_BYTES_MD5_HASH + NUM_STATIC_HEADER_BYTES;
 
 PacketVersion versionForPacketType(PacketType type);
+QString nameForPacketType(PacketType type);
 
 const QUuid nullUUID = QUuid();
 
@@ -116,5 +117,6 @@ const PacketVersion VERSION_ENTITIES_HAVE_ANIMATION = 1;
 const PacketVersion VERSION_ROOT_ELEMENT_HAS_DATA = 2;
 const PacketVersion VERSION_ENTITIES_SUPPORT_SPLIT_MTU = 3;
 const PacketVersion VERSION_ENTITIES_HAS_FILE_BREAKS = VERSION_ENTITIES_SUPPORT_SPLIT_MTU;
+const PacketVersion VERSION_VOXELS_HAS_FILE_BREAKS = 1;
 
 #endif // hifi_PacketHeaders_h

libraries/octree/src/Octree.cpp

@@ -247,7 +247,7 @@ int Octree::readElementData(OctreeElement* destinationElement, const unsigned ch
         return bytesAvailable; // assume we read the entire buffer...
     }
     
-    if (destinationElement->getScale() < SMALLEST_REASONABLE_OCTREE_ELEMENT_SCALE) {
+    if (destinationElement->getScale() < SCALE_AT_DANGEROUSLY_DEEP_RECURSION) {
         qDebug() << "UNEXPECTED: readElementData() destination element is unreasonably small [" 
                 << destinationElement->getScale() * (float)TREE_SCALE << " meters] "
                 << " Discarding " << bytesAvailable << " remaining bytes.";
@@ -1893,11 +1893,24 @@ bool Octree::readFromSVOFile(const char* fileName) {
             // if so, read the first byte of the file and see if it matches the expected version code
             PacketType gotType;
             memcpy(&gotType, dataAt, sizeof(gotType));
+
+            dataAt += sizeof(expectedType);
+            dataLength -= sizeof(expectedType);
+            gotVersion = *dataAt;
+            
+            // NOTE: SPECIAL CASE for old voxel svo files. The old voxel SVO files didn't have header
+            // details. They started with the the octalcode for the root. Which was always 00 which matches PacketTypeUnknown
+            unsigned char* firstByteAt = (unsigned char*)&fileHeader;
+            unsigned char firstByteValue = *firstByteAt;
+            if (expectedType == PacketTypeVoxelData && firstByteValue == 0) {
+                gotType = PacketTypeVoxelData;
+                gotVersion = 0;
+                qDebug() << "Detected OLD Voxels format.";
+                headerLength = 0; // old format files don't have headers
+                file.seekg( 0, std::ios::beg ); // rewind to the beginning so old logic will work
+            }
             
             if (gotType == expectedType) {
-                dataAt += sizeof(expectedType);
-                dataLength -= sizeof(expectedType);
-                gotVersion = *dataAt;
                 if (canProcessVersion(gotVersion)) {
                     dataAt += sizeof(gotVersion);
                     dataLength -= sizeof(gotVersion);
@@ -1911,7 +1924,8 @@ bool Octree::readFromSVOFile(const char* fileName) {
                                 versionForPacketType(expectedDataPacketType()), gotVersion);
                 }
             } else {
-                qDebug("SVO file type mismatch. Expected: %c Got: %c", expectedType, gotType);
+                qDebug() << "SVO file type mismatch. Expected: " << nameForPacketType(expectedType) 
+                            << " Got: " << nameForPacketType(gotType);
             }
 
         } else {
@@ -2008,8 +2022,7 @@ void Octree::writeToSVOFile(const char* fileName, OctreeElement* element) {
             // if so, read the first byte of the file and see if it matches the expected version code
             file.write(reinterpret_cast<char*>(&expectedType), sizeof(expectedType));
             file.write(&expectedVersion, sizeof(expectedVersion));
-
+            qDebug() << "SVO file type: " << nameForPacketType(expectedType) << " version: " << (int)expectedVersion;
-            qDebug("SVO file type: %c version: %d", expectedType, expectedVersion);
 
             hasBufferBreaks = versionHasSVOfileBreaks(expectedVersion);
         }

libraries/octree/src/OctreeConstants.h

@@ -37,6 +37,8 @@ const float VIEW_FRUSTUM_FOV_OVERSEND = 60.0f;
 // These are guards to prevent our voxel tree recursive routines from spinning out of control
 const int UNREASONABLY_DEEP_RECURSION = 20; // use this for something that you want to be shallow, but not spin out
 const int DANGEROUSLY_DEEP_RECURSION = 200; // use this for something that needs to go deeper
+const float SCALE_AT_UNREASONABLY_DEEP_RECURSION = (1.0f / powf(2.0f, DANGEROUSLY_DEEP_RECURSION));
+const float SCALE_AT_DANGEROUSLY_DEEP_RECURSION = (1.0f / powf(2.0f, DANGEROUSLY_DEEP_RECURSION));
 
 const int DEFAULT_MAX_OCTREE_PPS = 600; // the default maximum PPS we think any octree based server should send to a client
 

libraries/voxels/src/VoxelTree.h

@@ -51,12 +51,28 @@ public:
 
     void readCodeColorBufferToTree(const unsigned char* codeColorBuffer, bool destructive = false);
 
+
+    virtual bool getWantSVOfileVersions() const { return true; }
+    virtual bool canProcessVersion(PacketVersion thisVersion) const { 
+                    return thisVersion == 0 || thisVersion == versionForPacketType(expectedDataPacketType()); }
+    virtual PacketVersion expectedVersion() const { return versionForPacketType(expectedDataPacketType()); }
+
     virtual PacketType expectedDataPacketType() const { return PacketTypeVoxelData; }
     virtual bool handlesEditPacketType(PacketType packetType) const;
     virtual int processEditPacketData(PacketType packetType, const unsigned char* packetData, int packetLength,
                     const unsigned char* editData, int maxLength, const SharedNodePointer& node);
     virtual bool recurseChildrenWithData() const { return false; }
 
+    /// some versions of the SVO file will include breaks with buffer lengths between each buffer chunk in the SVO
+    /// file. If the Octree subclass expects this for this particular version of the file, it should override this
+    /// method and return true.
+    virtual bool versionHasSVOfileBreaks(PacketVersion thisVersion) const { 
+        if (thisVersion == 0) {
+            return false; // old versions didn't have buffer breaks
+        }
+        return true; 
+    }
+
     virtual void dumpTree();
 
 private: