Merge pull request #3356 from ZappoMan/fixingOctreeSave

voxel file corruption fixes

libraries/entities/src/EntityTree.cpp

@@ -467,7 +467,7 @@ int EntityTree::processEditPacketData(PacketType packetType, const unsigned char
                     if (existingEntity) {
                         updateEntity(entityItemID, properties);
                     } else {
-                        qDebug() << "User attempted to edit an unknown entity.";
+                        qDebug() << "User attempted to edit an unknown entity. ID:" << entityItemID;
                     }
                 } else {
                     // this is a new entity... assign a new entityID

libraries/entities/src/EntityTree.h

@@ -67,7 +67,8 @@ public:
     virtual int minimumRequiredRootDataBytes() const { return sizeof(uint16_t); }
     virtual bool suppressEmptySubtrees() const { return false; }
     virtual void releaseSceneEncodeData(OctreeElementExtraEncodeData* extraEncodeData) const;
- 
+    virtual bool mustIncludeAllChildData() const { return false; }
+
     virtual bool versionHasSVOfileBreaks(PacketVersion thisVersion) const 
                     { return thisVersion >= VERSION_ENTITIES_HAS_FILE_BREAKS; }
                     
@@ -126,7 +127,7 @@ public:
     void setContainingElement(const EntityItemID& entityItemID, EntityTreeElement* element);
     void resetContainingElement(const EntityItemID& entityItemID, EntityTreeElement* element);
     void debugDumpMap();
-    void dumpTree();
+    virtual void dumpTree();
 
     void sendEntities(EntityEditPacketSender* packetSender, EntityTree* localTree, float x, float y, float z);
 

libraries/entities/src/EntityTreeElement.cpp

@@ -721,9 +721,17 @@ int EntityTreeElement::readElementDataFromBuffer(const unsigned char* data, int
         if (bytesLeftToRead >= (int)(numberOfEntities * expectedBytesPerEntity)) {
             for (uint16_t i = 0; i < numberOfEntities; i++) {
                 int bytesForThisEntity = 0;
-                EntityItemID entityItemID = EntityItemID::readEntityItemIDFromBuffer(dataAt, bytesLeftToRead);
+                EntityItemID entityItemID;
-                EntityItem* entityItem = _myTree->findEntityByEntityItemID(entityItemID);
+                EntityItem* entityItem = NULL;
                 bool newEntity = false;
+
+                // Old model files don't have UUIDs in them. So we don't want to try to read those IDs from the stream.
+                // Since this can only happen on loading an old file, we can safely treat these as new entity cases,
+                // which will correctly handle the case of creating models and letting them parse the old format.
+                if (args.bitstreamVersion >= VERSION_ENTITIES_SUPPORT_SPLIT_MTU) {
+                    entityItemID = EntityItemID::readEntityItemIDFromBuffer(dataAt, bytesLeftToRead);
+                    entityItem = _myTree->findEntityByEntityItemID(entityItemID);
+                }
                 
                 // If the item already exists in our tree, we want do the following...
                 // 1) allow the existing item to read from the databuffer
@@ -758,6 +766,7 @@ int EntityTreeElement::readElementDataFromBuffer(const unsigned char* data, int
                     if (entityItem) {
                         bytesForThisEntity = entityItem->readEntityDataFromBuffer(dataAt, bytesLeftToRead, args);
                         addEntityItem(entityItem); // add this new entity to this elements entities
+                        entityItemID = entityItem->getEntityItemID();
                         _myTree->setContainingElement(entityItemID, this);
                         newEntity = true;
                         EntityItem::SimulationState newState = entityItem->getSimulationState();

libraries/entities/src/EntityTypes.cpp

@@ -65,7 +65,9 @@ bool EntityTypes::registerEntityType(EntityType entityType, const char* name, En
     return false;
 }
 
-EntityItem* EntityTypes::constructEntityItem(EntityType entityType, const EntityItemID& entityID, const EntityItemProperties& properties) {
+EntityItem* EntityTypes::constructEntityItem(EntityType entityType, const EntityItemID& entityID,
+                                                    const EntityItemProperties& properties) {
+                            
     EntityItem* newEntityItem = NULL;
     EntityTypeFactory factory = NULL;
     if (entityType >= 0 && entityType <= LAST) {

libraries/entities/src/ModelEntityItem.cpp

@@ -165,7 +165,8 @@ int ModelEntityItem::oldVersionReadEntityDataFromBuffer(const unsigned char* dat
         
         QString ageAsString = formatSecondsElapsed(getAge());
         qDebug() << "Loading old model file, _created = _lastEdited =" << _created 
-                        << " age=" << getAge() << "seconds - " << ageAsString;
+                        << " age=" << getAge() << "seconds - " << ageAsString
+                        << "old ID=" << oldID << "new ID=" << _id;
 
         // radius
         memcpy(&_radius, dataAt, sizeof(_radius));

libraries/fbx/src/FBXReader.cpp

@@ -2053,19 +2053,64 @@ FBXGeometry readSVO(const QByteArray& model) {
     mesh.parts.append(part);
 
     VoxelTree tree;
-    ReadBitstreamToTreeParams args(WANT_COLOR, NO_EXISTS_BITS);
 
     unsigned char* dataAt = (unsigned char*)model.data();
     size_t dataSize = model.size();
 
-    if (tree.getWantSVOfileVersions()) {
+    PacketVersion gotVersion = 0;
+
+    // NOTE: SPECIAL CASE for old voxel svo files. The old voxel SVO files didn't have header
+    // details. They started with the the octalcode for the root. Which was always 00 which matches PacketTypeUnknown
+    unsigned char* firstByteAt = (unsigned char*)model.data();
+    unsigned char firstByteValue = *firstByteAt;
+    if (tree.expectedDataPacketType() == PacketTypeVoxelData && firstByteValue == 0) {
+        qDebug() << "Detected OLD Voxels format.";
+        gotVersion = 0;
+    } else if (tree.getWantSVOfileVersions()) {
         // skip the type/version
         dataAt += sizeof(PacketType);
         dataSize -= sizeof(PacketType);
+
+        gotVersion = *dataAt;
         dataAt += sizeof(PacketVersion);
         dataSize -= sizeof(PacketVersion);
-    }   
+    }
-    tree.readBitstreamToTree(dataAt, dataSize, args);
+    bool hasBufferBreaks = tree.versionHasSVOfileBreaks(gotVersion);
+
+    ReadBitstreamToTreeParams args(WANT_COLOR, NO_EXISTS_BITS, NULL, 0, 
+                                        SharedNodePointer(), false, gotVersion);
+    
+    if (!hasBufferBreaks) {
+        tree.readBitstreamToTree(dataAt, dataSize, args);
+    } else {
+        const unsigned long MAX_CHUNK_LENGTH = MAX_OCTREE_PACKET_SIZE * 2;
+        while (dataSize > 0) {
+            quint16 chunkLength = 0;
+
+            chunkLength = *dataAt;
+            dataAt += sizeof(chunkLength);
+            dataSize -= sizeof(chunkLength);
+            
+            if (chunkLength > dataSize) {
+                qDebug() << "UNEXPECTED chunk size of:" << chunkLength 
+                            << "greater than remaining length:" << dataSize;
+                break;
+            }
+
+            if (chunkLength > MAX_CHUNK_LENGTH) {
+                qDebug() << "UNEXPECTED chunk size of:" << chunkLength 
+                            << "greater than MAX_CHUNK_LENGTH:" << MAX_CHUNK_LENGTH;
+                break;
+            }
+            
+            ReadBitstreamToTreeParams args(WANT_COLOR, NO_EXISTS_BITS, NULL, 0, 
+                                                SharedNodePointer(), false, gotVersion);
+
+            tree.readBitstreamToTree(dataAt, chunkLength, args);
+            dataAt += chunkLength;
+            dataSize -= chunkLength;
+        }
+    }
     tree.recurseTreeWithOperation(addMeshVoxelsOperation, &mesh);
 
     geometry.meshes.append(mesh);

libraries/networking/src/PacketHeaders.cpp

@@ -83,11 +83,75 @@ PacketVersion versionForPacketType(PacketType type) {
             return 1;
         case PacketTypeMetavoxelData:
             return 3;
+        case PacketTypeVoxelData:
+            return VERSION_VOXELS_HAS_FILE_BREAKS;
         default:
             return 0;
     }
 }
 
+#define PACKET_TYPE_NAME_LOOKUP(x) case x:  return QString(#x);
+
+QString nameForPacketType(PacketType type) {
+    switch (type) {
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeUnknown);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeStunResponse);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainList);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypePing);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypePingReply);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeKillAvatar);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeAvatarData);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeInjectAudio);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeMixedAudio);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeMicrophoneAudioNoEcho);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeMicrophoneAudioWithEcho);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeBulkAvatarData);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeSilentAudioFrame);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeEnvironmentData);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainListRequest);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeRequestAssignment);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeCreateAssignment);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainOAuthRequest);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeMuteEnvironment);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeAudioStreamStats);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeDataServerConfirm);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelQuery);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelData);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelSet);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelSetDestructive);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelErase);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeOctreeStats);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeJurisdiction);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeJurisdictionRequest);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleQuery);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleData);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleAddOrEdit);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleErase);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleAddResponse);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeMetavoxelData);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeAvatarIdentity);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeAvatarBillboard);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainConnectRequest);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainServerRequireDTLS);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeNodeJsonStats);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityQuery);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityData);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityAddOrEdit);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityErase);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityAddResponse);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeOctreeDataNack);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelEditNack);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleEditNack);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityEditNack);
+        PACKET_TYPE_NAME_LOOKUP(PacketTypeSignedTransactionPayment);
+        default:
+            return QString("Type: ") + QString::number((int)type);
+    }
+    return QString("unexpected");
+}
+
+
+
 QByteArray byteArrayWithPopulatedHeader(PacketType type, const QUuid& connectionUUID) {
     QByteArray freshByteArray(MAX_PACKET_HEADER_BYTES, 0);
     freshByteArray.resize(populatePacketHeader(freshByteArray, type, connectionUUID));

libraries/networking/src/PacketHeaders.h

@@ -19,7 +19,7 @@
 #include "UUID.h"
 
 // NOTE: if adding a new packet type, you can replace one marked usable or add at the end
-
+// NOTE: if you want the name of the packet type to be available for debugging or logging, update nameForPacketType() as well
 enum PacketType {
     PacketTypeUnknown,
     PacketTypeStunResponse,
@@ -87,6 +87,7 @@ const int NUM_STATIC_HEADER_BYTES = sizeof(PacketVersion) + NUM_BYTES_RFC4122_UU
 const int MAX_PACKET_HEADER_BYTES = sizeof(PacketType) + NUM_BYTES_MD5_HASH + NUM_STATIC_HEADER_BYTES;
 
 PacketVersion versionForPacketType(PacketType type);
+QString nameForPacketType(PacketType type);
 
 const QUuid nullUUID = QUuid();
 
@@ -116,5 +117,6 @@ const PacketVersion VERSION_ENTITIES_HAVE_ANIMATION = 1;
 const PacketVersion VERSION_ROOT_ELEMENT_HAS_DATA = 2;
 const PacketVersion VERSION_ENTITIES_SUPPORT_SPLIT_MTU = 3;
 const PacketVersion VERSION_ENTITIES_HAS_FILE_BREAKS = VERSION_ENTITIES_SUPPORT_SPLIT_MTU;
+const PacketVersion VERSION_VOXELS_HAS_FILE_BREAKS = 1;
 
 #endif // hifi_PacketHeaders_h

libraries/octree/src/Octree.cpp

@@ -233,15 +233,31 @@ OctreeElement* Octree::createMissingElement(OctreeElement* lastParentElement, co
     }
 }
 
-int Octree::readElementData(OctreeElement* destinationElement, const unsigned char* nodeData, int bytesLeftToRead,
+int Octree::readElementData(OctreeElement* destinationElement, const unsigned char* nodeData, int bytesAvailable,
                             ReadBitstreamToTreeParams& args) {
 
+    int bytesLeftToRead = bytesAvailable;
+    int bytesRead = 0;
+
     // give this destination element the child mask from the packet
     const unsigned char ALL_CHILDREN_ASSUMED_TO_EXIST = 0xFF;
+    
+    if (bytesLeftToRead < sizeof(unsigned char)) {
+        qDebug() << "UNEXPECTED: readElementData() only had " << bytesLeftToRead << " bytes. Not enough for meaningful data.";
+        return bytesAvailable; // assume we read the entire buffer...
+    }
+    
+    if (destinationElement->getScale() < SCALE_AT_DANGEROUSLY_DEEP_RECURSION) {
+        qDebug() << "UNEXPECTED: readElementData() destination element is unreasonably small [" 
+                << destinationElement->getScale() * (float)TREE_SCALE << " meters] "
+                << " Discarding " << bytesAvailable << " remaining bytes.";
+        return bytesAvailable; // assume we read the entire buffer...
+    }
+    
     unsigned char colorInPacketMask = *nodeData;
+    bytesRead += sizeof(colorInPacketMask);
+    bytesLeftToRead -= sizeof(colorInPacketMask);
 
-    // instantiate variable for bytes already read
-    int bytesRead = sizeof(colorInPacketMask);
     for (int i = 0; i < NUMBER_OF_CHILDREN; i++) {
         // check the colors mask to see if we have a child to color in
         if (oneAtBit(colorInPacketMask, i)) {
@@ -256,9 +272,13 @@ int Octree::readElementData(OctreeElement* destinationElement, const unsigned ch
             OctreeElement* childElementAt = destinationElement->getChildAtIndex(i);
             bool nodeIsDirty = false;
             if (childElementAt) {
-                bytesRead += childElementAt->readElementDataFromBuffer(nodeData + bytesRead, bytesLeftToRead, args);
+
+                int childElementDataRead = childElementAt->readElementDataFromBuffer(nodeData + bytesRead, bytesLeftToRead, args);
                 childElementAt->setSourceUUID(args.sourceUUID);
 
+                bytesRead += childElementDataRead;
+                bytesLeftToRead -= childElementDataRead;
+
                 // if we had a local version of the element already, it's possible that we have it already but
                 // with the same color data, so this won't count as a change. To address this we check the following
                 if (!childElementAt->isDirty() && childElementAt->getShouldRender() && !childElementAt->isRendered()) {
@@ -273,17 +293,28 @@ int Octree::readElementData(OctreeElement* destinationElement, const unsigned ch
         }
     }
 
-    // give this destination element the child mask from the packet
+    unsigned char childrenInTreeMask = ALL_CHILDREN_ASSUMED_TO_EXIST;
-    unsigned char childrenInTreeMask = args.includeExistsBits ? *(nodeData + bytesRead) : ALL_CHILDREN_ASSUMED_TO_EXIST;
+    unsigned char childInBufferMask = 0;
-    unsigned char childMask = *(nodeData + bytesRead + (args.includeExistsBits ? sizeof(childrenInTreeMask) : 0));
+    int bytesForMasks = args.includeExistsBits ? sizeof(childrenInTreeMask) + sizeof(childInBufferMask) 
+                                                : sizeof(childInBufferMask);
+
+    if (bytesLeftToRead < bytesForMasks) {
+        qDebug() << "UNEXPECTED: readElementDataFromBuffer() only had " << bytesLeftToRead << " bytes before masks. "
+                    "Not enough for meaningful data.";
+        return bytesAvailable; // assume we read the entire buffer...
+    }
+    
+    childrenInTreeMask = args.includeExistsBits ? *(nodeData + bytesRead) : ALL_CHILDREN_ASSUMED_TO_EXIST;
+    childInBufferMask = *(nodeData + bytesRead + (args.includeExistsBits ? sizeof(childrenInTreeMask) : 0));
 
     int childIndex = 0;
-    bytesRead += args.includeExistsBits ? sizeof(childrenInTreeMask) + sizeof(childMask) : sizeof(childMask);
+    bytesRead += bytesForMasks;
+    bytesLeftToRead -= bytesForMasks;
 
-    while (bytesLeftToRead - bytesRead > 0 && childIndex < NUMBER_OF_CHILDREN) {
+    while (bytesLeftToRead > 0 && childIndex < NUMBER_OF_CHILDREN) {
         // check the exists mask to see if we have a child to traverse into
 
-        if (oneAtBit(childMask, childIndex)) {
+        if (oneAtBit(childInBufferMask, childIndex)) {
             if (!destinationElement->getChildAtIndex(childIndex)) {
                 // add a child at that index, if it doesn't exist
                 destinationElement->addChildAtIndex(childIndex);
@@ -294,8 +325,11 @@ int Octree::readElementData(OctreeElement* destinationElement, const unsigned ch
             }
 
             // tell the child to read the subsequent data
-            bytesRead += readElementData(destinationElement->getChildAtIndex(childIndex),
+            int lowerLevelBytes = readElementData(destinationElement->getChildAtIndex(childIndex),
-                                      nodeData + bytesRead, bytesLeftToRead - bytesRead, args);
+                                      nodeData + bytesRead, bytesLeftToRead, args);
+
+            bytesRead += lowerLevelBytes;
+            bytesLeftToRead -= lowerLevelBytes;
         }
         childIndex++;
     }
@@ -314,7 +348,9 @@ int Octree::readElementData(OctreeElement* destinationElement, const unsigned ch
     // if this is the root, and there is more data to read, allow it to read it's element data...
     if (destinationElement == _rootElement  && rootElementHasData() && (bytesLeftToRead - bytesRead) > 0) {
         // tell the element to read the subsequent data
-        bytesRead += _rootElement->readElementDataFromBuffer(nodeData + bytesRead, bytesLeftToRead - bytesRead, args);
+        int rootDataSize = _rootElement->readElementDataFromBuffer(nodeData + bytesRead, bytesLeftToRead - bytesRead, args);
+        bytesRead += rootDataSize;
+        bytesLeftToRead -= rootDataSize;
     }
 
     return bytesRead;
@@ -322,7 +358,6 @@ int Octree::readElementData(OctreeElement* destinationElement, const unsigned ch
 
 void Octree::readBitstreamToTree(const unsigned char * bitstream, unsigned long int bufferSizeBytes,
                                     ReadBitstreamToTreeParams& args) {
-
     int bytesRead = 0;
     const unsigned char* bitstreamAt = bitstream;
 
@@ -337,9 +372,18 @@ void Octree::readBitstreamToTree(const unsigned char * bitstream, unsigned long
 
     while (bitstreamAt < bitstream + bufferSizeBytes) {
         OctreeElement* bitstreamRootElement = nodeForOctalCode(args.destinationElement, (unsigned char *)bitstreamAt, NULL);
-        if (*bitstreamAt != *bitstreamRootElement->getOctalCode()) {
+
-            // if the octal code returned is not on the same level as
+        int numberOfThreeBitSectionsInStream = numberOfThreeBitSectionsInCode(bitstreamAt, bufferSizeBytes);
-            // the code being searched for, we have OctreeElements to create
+        
+        if (numberOfThreeBitSectionsInStream == OVERFLOWED_OCTCODE_BUFFER) {
+            qDebug() << "UNEXPECTED: parsing of the octal code would overflow the buffer. This buffer is corrupt. Returning.";
+            return;
+        }
+        
+        int numberOfThreeBitSectionsFromNode = numberOfThreeBitSectionsInCode(bitstreamRootElement->getOctalCode());
+
+        // if the octal code returned is not on the same level as the code being searched for, we have OctreeElements to create
+        if (numberOfThreeBitSectionsInStream != numberOfThreeBitSectionsFromNode) {
 
             // Note: we need to create this element relative to root, because we're assuming that the bitstream for the initial
             // octal code is always relative to root!
@@ -349,16 +393,18 @@ void Octree::readBitstreamToTree(const unsigned char * bitstream, unsigned long
             }
         }
 
-        int octalCodeBytes = bytesRequiredForCodeLength(*bitstreamAt);
+        int octalCodeBytes = bytesRequiredForCodeLength(numberOfThreeBitSectionsInStream);
 
         int theseBytesRead = 0;
         theseBytesRead += octalCodeBytes;
-        theseBytesRead += readElementData(bitstreamRootElement, bitstreamAt + octalCodeBytes,
+        int lowerLevelBytes = readElementData(bitstreamRootElement, bitstreamAt + octalCodeBytes,
                                        bufferSizeBytes - (bytesRead + octalCodeBytes), args);
 
+        theseBytesRead += lowerLevelBytes;
+
         // skip bitstream to new startPoint
         bitstreamAt += theseBytesRead;
-        bytesRead +=  theseBytesRead;
+        bytesRead += theseBytesRead;
 
         if (args.wantImportProgress) {
             emit importProgress((100 * (bitstreamAt - bitstream)) / bufferSizeBytes);
@@ -1004,7 +1050,7 @@ int Octree::encodeTreeBitstream(OctreeElement* element,
         params.stopReason = EncodeBitstreamParams::DIDNT_FIT;
         return bytesWritten;
     }
-
+    
     bytesWritten += codeLength; // keep track of byte count
 
     int currentEncodeLevel = 0;
@@ -1421,6 +1467,7 @@ int Octree::encodeTreeBitstreamRecursion(OctreeElement* element,
     // We wil write this bit mask but we may come back later and update the bits that are actually included
     packetData->releaseReservedBytes(sizeof(childrenDataBits));
     continueThisLevel = packetData->appendBitMask(childrenDataBits);
+
     int childDataBitsPlaceHolder = packetData->getUncompressedByteOffset(sizeof(childrenDataBits));
     unsigned char actualChildrenDataBits = 0;
 
@@ -1470,6 +1517,13 @@ int Octree::encodeTreeBitstreamRecursion(OctreeElement* element,
                     // Continue this level so long as some part of this child element was appended.
                     bool childFit = (childAppendState != OctreeElement::NONE);
                     
+                    // some datatypes (like Voxels) assume that all child data will fit, if it doesn't fit
+                    // the data type wants to bail on this element level completely
+                    if (!childFit && mustIncludeAllChildData()) {
+                        continueThisLevel = false;
+                        break;   
+                    }
+                    
                     // If the child was partially or fully appended, then mark the actualChildrenDataBits as including
                     // this child data
                     if (childFit) {
@@ -1500,15 +1554,14 @@ int Octree::encodeTreeBitstreamRecursion(OctreeElement* element,
         }
     }
     
-    if (!continueThisLevel) {
+    if (!mustIncludeAllChildData() && !continueThisLevel) {
         qDebug() << "WARNING UNEXPECTED CASE: reached end of child element data loop with continueThisLevel=FALSE";
         qDebug() << "This is not expected!!!!  -- continueThisLevel=FALSE....";
     }
 
-    if (actualChildrenDataBits != childrenDataBits) {
+    if (continueThisLevel && actualChildrenDataBits != childrenDataBits) {
         // repair the child data mask
         continueThisLevel = packetData->updatePriorBitMask(childDataBitsPlaceHolder, actualChildrenDataBits);
-
         if (!continueThisLevel) {
             qDebug() << "WARNING UNEXPECTED CASE: Failed to update childDataBitsPlaceHolder";
             qDebug() << "This is not expected!!!!  -- continueThisLevel=FALSE....";
@@ -1764,17 +1817,22 @@ int Octree::encodeTreeBitstreamRecursion(OctreeElement* element,
         continueThisLevel = packetData->endLevel(thisLevelKey);
     } else {
         packetData->discardLevel(thisLevelKey);
-        qDebug() << "WARNING UNEXPECTED CASE: Something failed in attempting to pack this element";
+        
-        qDebug() << "This is not expected!!!! -- continueThisLevel=FALSE....";
+        if (!mustIncludeAllChildData()) {
+            qDebug() << "WARNING UNEXPECTED CASE: Something failed in attempting to pack this element";
+            qDebug() << "This is not expected!!!! -- continueThisLevel=FALSE....";
+        }
     }
 
     // This happens if the element could not be written at all. In the case of Octree's that support partial
     // element data, continueThisLevel will be true. So this only happens if the full element needs to be
     // added back to the element bag.
     if (!continueThisLevel) {
-        qDebug() << "WARNING UNEXPECTED CASE - Something failed in attempting to pack this element";
+        if (!mustIncludeAllChildData()) {
-        qDebug() << "IS THIS EVER EXPECTED???? -- continueThisLevel=FALSE...." ;
+            qDebug() << "WARNING UNEXPECTED CASE - Something failed in attempting to pack this element.";
-        qDebug() << "   calling bag.insert(element);.....";
+            qDebug() << "   If the datatype requires all child data, then this might happen. Otherwise" ;
+            qDebug() << "   this is an unexpected case and we should research a potential logic error." ;
+        }
 
         bag.insert(element);
 
@@ -1826,6 +1884,10 @@ bool Octree::readFromSVOFile(const char* fileName) {
         
         bool wantImportProgress = true;
 
+        PacketType expectedType = expectedDataPacketType();
+        PacketVersion expectedVersion = versionForPacketType(expectedType);
+        bool hasBufferBreaks = versionHasSVOfileBreaks(expectedVersion);
+
         // before reading the file, check to see if this version of the Octree supports file versions
         if (getWantSVOfileVersions()) {
 
@@ -1840,15 +1902,26 @@ bool Octree::readFromSVOFile(const char* fileName) {
             unsigned long  dataLength = HEADER_LENGTH;
 
             // if so, read the first byte of the file and see if it matches the expected version code
-            PacketType expectedType = expectedDataPacketType();
-            
             PacketType gotType;
             memcpy(&gotType, dataAt, sizeof(gotType));
+
+            dataAt += sizeof(expectedType);
+            dataLength -= sizeof(expectedType);
+            gotVersion = *dataAt;
+            
+            // NOTE: SPECIAL CASE for old voxel svo files. The old voxel SVO files didn't have header
+            // details. They started with the the octalcode for the root. Which was always 00 which matches PacketTypeUnknown
+            unsigned char* firstByteAt = (unsigned char*)&fileHeader;
+            unsigned char firstByteValue = *firstByteAt;
+            if (expectedType == PacketTypeVoxelData && firstByteValue == 0) {
+                gotType = PacketTypeVoxelData;
+                gotVersion = 0;
+                qDebug() << "Detected OLD Voxels format.";
+                headerLength = 0; // old format files don't have headers
+                file.seekg( 0, std::ios::beg ); // rewind to the beginning so old logic will work
+            }
             
             if (gotType == expectedType) {
-                dataAt += sizeof(expectedType);
-                dataLength -= sizeof(expectedType);
-                gotVersion = *dataAt;
                 if (canProcessVersion(gotVersion)) {
                     dataAt += sizeof(gotVersion);
                     dataLength -= sizeof(gotVersion);
@@ -1857,25 +1930,25 @@ bool Octree::readFromSVOFile(const char* fileName) {
                                 versionForPacketType(expectedDataPacketType()), gotVersion);
 
                     hasBufferBreaks = versionHasSVOfileBreaks(gotVersion);
-                    if (hasBufferBreaks) {
-                        qDebug() << "    this version includes buffer breaks";
-                    } else {
-                        qDebug() << "    this version does not include buffer breaks";
-                    }
-
                 } else {
                     qDebug("SVO file version mismatch. Expected: %d Got: %d", 
                                 versionForPacketType(expectedDataPacketType()), gotVersion);
                 }
             } else {
-                qDebug("SVO file type mismatch. Expected: %c Got: %c", expectedType, gotType);
+                qDebug() << "SVO file type mismatch. Expected: " << nameForPacketType(expectedType) 
+                            << " Got: " << nameForPacketType(gotType);
             }
 
         } else {
             qDebug() << "   NOTE: this file type does not include type and version information.";
             fileOk = true; // assume the file is ok
         }
-
+        
+        if (hasBufferBreaks) {
+            qDebug() << "    this version includes buffer breaks";
+        } else {
+            qDebug() << "    this version does not include buffer breaks";
+        }
 
         if (fileOk) {
         
@@ -1906,7 +1979,6 @@ bool Octree::readFromSVOFile(const char* fileName) {
                     quint16 chunkLength = 0;
 
                     file.read((char*)&chunkLength, sizeof(chunkLength)); // read the chunk size from the file
-                    
                     remainingLength -= sizeof(chunkLength);
                     
                     if (chunkLength > remainingLength) {
@@ -1942,6 +2014,7 @@ bool Octree::readFromSVOFile(const char* fileName) {
 
         file.close();
     }
+    
     return fileOk;
 }
 
@@ -1951,25 +2024,25 @@ void Octree::writeToSVOFile(const char* fileName, OctreeElement* element) {
     if(file.is_open()) {
         qDebug("Saving to file %s...", fileName);
 
-        bool hasBufferBreaks = false;
+        PacketType expectedType = expectedDataPacketType();
+        PacketVersion expectedVersion = versionForPacketType(expectedType);
+        bool hasBufferBreaks = versionHasSVOfileBreaks(expectedVersion);
 
         // before reading the file, check to see if this version of the Octree supports file versions
         if (getWantSVOfileVersions()) {
             // if so, read the first byte of the file and see if it matches the expected version code
-            PacketType expectedType = expectedDataPacketType();
-            PacketVersion expectedVersion = versionForPacketType(expectedType);
             file.write(reinterpret_cast<char*>(&expectedType), sizeof(expectedType));
             file.write(&expectedVersion, sizeof(expectedVersion));
-
+            qDebug() << "SVO file type: " << nameForPacketType(expectedType) << " version: " << (int)expectedVersion;
-            qDebug("SVO file type: %c version: %d", expectedType, expectedVersion);
 
             hasBufferBreaks = versionHasSVOfileBreaks(expectedVersion);
-            if (hasBufferBreaks) {
-                qDebug() << "    this version includes buffer breaks";
-            } else {
-                qDebug() << "    this version does not include buffer breaks";
-            }
         }
+        if (hasBufferBreaks) {
+            qDebug() << "    this version includes buffer breaks";
+        } else {
+            qDebug() << "    this version does not include buffer breaks";
+        }
+        
 
         OctreeElementBag elementBag;
         OctreeElementExtraEncodeData extraEncodeData;

libraries/octree/src/Octree.h

@@ -238,6 +238,7 @@ public:
     virtual int minimumRequiredRootDataBytes() const { return 0; }
     virtual bool suppressEmptySubtrees() const { return true; }
     virtual void releaseSceneEncodeData(OctreeElementExtraEncodeData* extraEncodeData) const { }
+    virtual bool mustIncludeAllChildData() const { return true; }
     
     /// some versions of the SVO file will include breaks with buffer lengths between each buffer chunk in the SVO
     /// file. If the Octree subclass expects this for this particular version of the file, it should override this
@@ -352,6 +353,7 @@ public:
     bool getIsClient() const { return !_isServer; } /// Is this a client based tree. Allows guards for certain operations
     void setIsClient(bool isClient) { _isServer = !isClient; }
     
+    virtual void dumpTree() { };
 
 signals:
     void importSize(float x, float y, float z);

libraries/octree/src/OctreeConstants.h

@@ -37,6 +37,8 @@ const float VIEW_FRUSTUM_FOV_OVERSEND = 60.0f;
 // These are guards to prevent our voxel tree recursive routines from spinning out of control
 const int UNREASONABLY_DEEP_RECURSION = 20; // use this for something that you want to be shallow, but not spin out
 const int DANGEROUSLY_DEEP_RECURSION = 200; // use this for something that needs to go deeper
+const float SCALE_AT_UNREASONABLY_DEEP_RECURSION = (1.0f / powf(2.0f, UNREASONABLY_DEEP_RECURSION));
+const float SCALE_AT_DANGEROUSLY_DEEP_RECURSION = (1.0f / powf(2.0f, DANGEROUSLY_DEEP_RECURSION));
 
 const int DEFAULT_MAX_OCTREE_PPS = 600; // the default maximum PPS we think any octree based server should send to a client
 

libraries/voxels/src/VoxelTree.cpp

@@ -11,11 +11,10 @@
 
 #include <algorithm>
 
-#include <QtCore/QDebug>
+#include <QDebug>
 #include <QImage>
 #include <QRgb>
 
-
 #include "VoxelTree.h"
 #include "Tags.h"
 
@@ -567,3 +566,26 @@ int VoxelTree::processEditPacketData(PacketType packetType, const unsigned char*
             return 0;
     }
 }
+
+class VoxelTreeDebugOperator : public RecurseOctreeOperator {
+public:
+    virtual bool preRecursion(OctreeElement* element);
+    virtual bool postRecursion(OctreeElement* element) { return true; }
+};
+
+bool VoxelTreeDebugOperator::preRecursion(OctreeElement* element) {
+    VoxelTreeElement* treeElement = static_cast<VoxelTreeElement*>(element);
+    qDebug() << "VoxelTreeElement [" << treeElement << ":" << treeElement->getAACube() << "]";
+    qDebug() << "    isLeaf:" << treeElement->isLeaf();
+    qDebug() << "    color:" << treeElement->getColor()[0] << ", "
+                             << treeElement->getColor()[1] << ", " 
+                             << treeElement->getColor()[2];
+    return true;
+}
+
+void VoxelTree::dumpTree() {
+    // First, look for the existing entity in the tree..
+    VoxelTreeDebugOperator theOperator;
+    recurseTreeWithOperator(&theOperator);
+}
+

libraries/voxels/src/VoxelTree.h

@@ -51,12 +51,30 @@ public:
 
     void readCodeColorBufferToTree(const unsigned char* codeColorBuffer, bool destructive = false);
 
+
+    virtual bool getWantSVOfileVersions() const { return true; }
+    virtual bool canProcessVersion(PacketVersion thisVersion) const { 
+                    return thisVersion == 0 || thisVersion == versionForPacketType(expectedDataPacketType()); }
+    virtual PacketVersion expectedVersion() const { return versionForPacketType(expectedDataPacketType()); }
+
     virtual PacketType expectedDataPacketType() const { return PacketTypeVoxelData; }
     virtual bool handlesEditPacketType(PacketType packetType) const;
     virtual int processEditPacketData(PacketType packetType, const unsigned char* packetData, int packetLength,
                     const unsigned char* editData, int maxLength, const SharedNodePointer& node);
     virtual bool recurseChildrenWithData() const { return false; }
 
+    /// some versions of the SVO file will include breaks with buffer lengths between each buffer chunk in the SVO
+    /// file. If the Octree subclass expects this for this particular version of the file, it should override this
+    /// method and return true.
+    virtual bool versionHasSVOfileBreaks(PacketVersion thisVersion) const { 
+        if (thisVersion == 0) {
+            return false; // old versions didn't have buffer breaks
+        }
+        return true; 
+    }
+
+    virtual void dumpTree();
+
 private:
     // helper functions for nudgeSubTree
     void recurseNodeForNudge(VoxelTreeElement* element, RecurseOctreeOperation operation, void* extraData);

libraries/voxels/src/VoxelTreeElement.cpp

@@ -73,6 +73,13 @@ OctreeElement::AppendState VoxelTreeElement::appendElementData(OctreePacketData*
 int VoxelTreeElement::readElementDataFromBuffer(const unsigned char* data, int bytesLeftToRead,
             ReadBitstreamToTreeParams& args) {
     const int BYTES_PER_COLOR = 3;
+    
+    if (bytesLeftToRead < BYTES_PER_COLOR) {
+        qDebug() << "UNEXPECTED: readElementDataFromBuffer() only had " << bytesLeftToRead << " bytes. "
+                        "Not enough for meaningful data.";
+        return bytesLeftToRead;
+    }
+    
 
     // pull the color for this child
     nodeColor newColor = { 128, 128, 128, 1};