diff --git a/libraries/entities/src/EntityTree.cpp b/libraries/entities/src/EntityTree.cpp index 6cc140e408..b7fd267c30 100644 --- a/libraries/entities/src/EntityTree.cpp +++ b/libraries/entities/src/EntityTree.cpp @@ -467,7 +467,7 @@ int EntityTree::processEditPacketData(PacketType packetType, const unsigned char if (existingEntity) { updateEntity(entityItemID, properties); } else { - qDebug() << "User attempted to edit an unknown entity."; + qDebug() << "User attempted to edit an unknown entity. ID:" << entityItemID; } } else { // this is a new entity... assign a new entityID diff --git a/libraries/entities/src/EntityTree.h b/libraries/entities/src/EntityTree.h index f05eb69538..47ae04846d 100644 --- a/libraries/entities/src/EntityTree.h +++ b/libraries/entities/src/EntityTree.h @@ -67,7 +67,8 @@ public: virtual int minimumRequiredRootDataBytes() const { return sizeof(uint16_t); } virtual bool suppressEmptySubtrees() const { return false; } virtual void releaseSceneEncodeData(OctreeElementExtraEncodeData* extraEncodeData) const; - + virtual bool mustIncludeAllChildData() const { return false; } + virtual bool versionHasSVOfileBreaks(PacketVersion thisVersion) const { return thisVersion >= VERSION_ENTITIES_HAS_FILE_BREAKS; } @@ -126,7 +127,7 @@ public: void setContainingElement(const EntityItemID& entityItemID, EntityTreeElement* element); void resetContainingElement(const EntityItemID& entityItemID, EntityTreeElement* element); void debugDumpMap(); - void dumpTree(); + virtual void dumpTree(); void sendEntities(EntityEditPacketSender* packetSender, EntityTree* localTree, float x, float y, float z); diff --git a/libraries/entities/src/EntityTreeElement.cpp b/libraries/entities/src/EntityTreeElement.cpp index 0d952f9fef..e1aa9b33e1 100644 --- a/libraries/entities/src/EntityTreeElement.cpp +++ b/libraries/entities/src/EntityTreeElement.cpp @@ -721,9 +721,17 @@ int EntityTreeElement::readElementDataFromBuffer(const unsigned char* data, int if (bytesLeftToRead >= (int)(numberOfEntities * expectedBytesPerEntity)) { for (uint16_t i = 0; i < numberOfEntities; i++) { int bytesForThisEntity = 0; - EntityItemID entityItemID = EntityItemID::readEntityItemIDFromBuffer(dataAt, bytesLeftToRead); - EntityItem* entityItem = _myTree->findEntityByEntityItemID(entityItemID); + EntityItemID entityItemID; + EntityItem* entityItem = NULL; bool newEntity = false; + + // Old model files don't have UUIDs in them. So we don't want to try to read those IDs from the stream. + // Since this can only happen on loading an old file, we can safely treat these as new entity cases, + // which will correctly handle the case of creating models and letting them parse the old format. + if (args.bitstreamVersion >= VERSION_ENTITIES_SUPPORT_SPLIT_MTU) { + entityItemID = EntityItemID::readEntityItemIDFromBuffer(dataAt, bytesLeftToRead); + entityItem = _myTree->findEntityByEntityItemID(entityItemID); + } // If the item already exists in our tree, we want do the following... // 1) allow the existing item to read from the databuffer @@ -758,6 +766,7 @@ int EntityTreeElement::readElementDataFromBuffer(const unsigned char* data, int if (entityItem) { bytesForThisEntity = entityItem->readEntityDataFromBuffer(dataAt, bytesLeftToRead, args); addEntityItem(entityItem); // add this new entity to this elements entities + entityItemID = entityItem->getEntityItemID(); _myTree->setContainingElement(entityItemID, this); newEntity = true; EntityItem::SimulationState newState = entityItem->getSimulationState(); diff --git a/libraries/entities/src/EntityTypes.cpp b/libraries/entities/src/EntityTypes.cpp index 51890ad7d6..7505c3f768 100644 --- a/libraries/entities/src/EntityTypes.cpp +++ b/libraries/entities/src/EntityTypes.cpp @@ -65,7 +65,9 @@ bool EntityTypes::registerEntityType(EntityType entityType, const char* name, En return false; } -EntityItem* EntityTypes::constructEntityItem(EntityType entityType, const EntityItemID& entityID, const EntityItemProperties& properties) { +EntityItem* EntityTypes::constructEntityItem(EntityType entityType, const EntityItemID& entityID, + const EntityItemProperties& properties) { + EntityItem* newEntityItem = NULL; EntityTypeFactory factory = NULL; if (entityType >= 0 && entityType <= LAST) { diff --git a/libraries/entities/src/ModelEntityItem.cpp b/libraries/entities/src/ModelEntityItem.cpp index 99eafc012e..1c1cd19831 100644 --- a/libraries/entities/src/ModelEntityItem.cpp +++ b/libraries/entities/src/ModelEntityItem.cpp @@ -165,7 +165,8 @@ int ModelEntityItem::oldVersionReadEntityDataFromBuffer(const unsigned char* dat QString ageAsString = formatSecondsElapsed(getAge()); qDebug() << "Loading old model file, _created = _lastEdited =" << _created - << " age=" << getAge() << "seconds - " << ageAsString; + << " age=" << getAge() << "seconds - " << ageAsString + << "old ID=" << oldID << "new ID=" << _id; // radius memcpy(&_radius, dataAt, sizeof(_radius)); diff --git a/libraries/fbx/src/FBXReader.cpp b/libraries/fbx/src/FBXReader.cpp index d8b52fb794..79deef236a 100644 --- a/libraries/fbx/src/FBXReader.cpp +++ b/libraries/fbx/src/FBXReader.cpp @@ -2053,19 +2053,64 @@ FBXGeometry readSVO(const QByteArray& model) { mesh.parts.append(part); VoxelTree tree; - ReadBitstreamToTreeParams args(WANT_COLOR, NO_EXISTS_BITS); unsigned char* dataAt = (unsigned char*)model.data(); size_t dataSize = model.size(); - if (tree.getWantSVOfileVersions()) { + PacketVersion gotVersion = 0; + + // NOTE: SPECIAL CASE for old voxel svo files. The old voxel SVO files didn't have header + // details. They started with the the octalcode for the root. Which was always 00 which matches PacketTypeUnknown + unsigned char* firstByteAt = (unsigned char*)model.data(); + unsigned char firstByteValue = *firstByteAt; + if (tree.expectedDataPacketType() == PacketTypeVoxelData && firstByteValue == 0) { + qDebug() << "Detected OLD Voxels format."; + gotVersion = 0; + } else if (tree.getWantSVOfileVersions()) { // skip the type/version dataAt += sizeof(PacketType); dataSize -= sizeof(PacketType); + + gotVersion = *dataAt; dataAt += sizeof(PacketVersion); dataSize -= sizeof(PacketVersion); - } - tree.readBitstreamToTree(dataAt, dataSize, args); + } + bool hasBufferBreaks = tree.versionHasSVOfileBreaks(gotVersion); + + ReadBitstreamToTreeParams args(WANT_COLOR, NO_EXISTS_BITS, NULL, 0, + SharedNodePointer(), false, gotVersion); + + if (!hasBufferBreaks) { + tree.readBitstreamToTree(dataAt, dataSize, args); + } else { + const unsigned long MAX_CHUNK_LENGTH = MAX_OCTREE_PACKET_SIZE * 2; + while (dataSize > 0) { + quint16 chunkLength = 0; + + chunkLength = *dataAt; + dataAt += sizeof(chunkLength); + dataSize -= sizeof(chunkLength); + + if (chunkLength > dataSize) { + qDebug() << "UNEXPECTED chunk size of:" << chunkLength + << "greater than remaining length:" << dataSize; + break; + } + + if (chunkLength > MAX_CHUNK_LENGTH) { + qDebug() << "UNEXPECTED chunk size of:" << chunkLength + << "greater than MAX_CHUNK_LENGTH:" << MAX_CHUNK_LENGTH; + break; + } + + ReadBitstreamToTreeParams args(WANT_COLOR, NO_EXISTS_BITS, NULL, 0, + SharedNodePointer(), false, gotVersion); + + tree.readBitstreamToTree(dataAt, chunkLength, args); + dataAt += chunkLength; + dataSize -= chunkLength; + } + } tree.recurseTreeWithOperation(addMeshVoxelsOperation, &mesh); geometry.meshes.append(mesh); diff --git a/libraries/networking/src/PacketHeaders.cpp b/libraries/networking/src/PacketHeaders.cpp index 91de48296f..f3937dd2e3 100644 --- a/libraries/networking/src/PacketHeaders.cpp +++ b/libraries/networking/src/PacketHeaders.cpp @@ -83,11 +83,75 @@ PacketVersion versionForPacketType(PacketType type) { return 1; case PacketTypeMetavoxelData: return 3; + case PacketTypeVoxelData: + return VERSION_VOXELS_HAS_FILE_BREAKS; default: return 0; } } +#define PACKET_TYPE_NAME_LOOKUP(x) case x: return QString(#x); + +QString nameForPacketType(PacketType type) { + switch (type) { + PACKET_TYPE_NAME_LOOKUP(PacketTypeUnknown); + PACKET_TYPE_NAME_LOOKUP(PacketTypeStunResponse); + PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainList); + PACKET_TYPE_NAME_LOOKUP(PacketTypePing); + PACKET_TYPE_NAME_LOOKUP(PacketTypePingReply); + PACKET_TYPE_NAME_LOOKUP(PacketTypeKillAvatar); + PACKET_TYPE_NAME_LOOKUP(PacketTypeAvatarData); + PACKET_TYPE_NAME_LOOKUP(PacketTypeInjectAudio); + PACKET_TYPE_NAME_LOOKUP(PacketTypeMixedAudio); + PACKET_TYPE_NAME_LOOKUP(PacketTypeMicrophoneAudioNoEcho); + PACKET_TYPE_NAME_LOOKUP(PacketTypeMicrophoneAudioWithEcho); + PACKET_TYPE_NAME_LOOKUP(PacketTypeBulkAvatarData); + PACKET_TYPE_NAME_LOOKUP(PacketTypeSilentAudioFrame); + PACKET_TYPE_NAME_LOOKUP(PacketTypeEnvironmentData); + PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainListRequest); + PACKET_TYPE_NAME_LOOKUP(PacketTypeRequestAssignment); + PACKET_TYPE_NAME_LOOKUP(PacketTypeCreateAssignment); + PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainOAuthRequest); + PACKET_TYPE_NAME_LOOKUP(PacketTypeMuteEnvironment); + PACKET_TYPE_NAME_LOOKUP(PacketTypeAudioStreamStats); + PACKET_TYPE_NAME_LOOKUP(PacketTypeDataServerConfirm); + PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelQuery); + PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelData); + PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelSet); + PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelSetDestructive); + PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelErase); + PACKET_TYPE_NAME_LOOKUP(PacketTypeOctreeStats); + PACKET_TYPE_NAME_LOOKUP(PacketTypeJurisdiction); + PACKET_TYPE_NAME_LOOKUP(PacketTypeJurisdictionRequest); + PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleQuery); + PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleData); + PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleAddOrEdit); + PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleErase); + PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleAddResponse); + PACKET_TYPE_NAME_LOOKUP(PacketTypeMetavoxelData); + PACKET_TYPE_NAME_LOOKUP(PacketTypeAvatarIdentity); + PACKET_TYPE_NAME_LOOKUP(PacketTypeAvatarBillboard); + PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainConnectRequest); + PACKET_TYPE_NAME_LOOKUP(PacketTypeDomainServerRequireDTLS); + PACKET_TYPE_NAME_LOOKUP(PacketTypeNodeJsonStats); + PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityQuery); + PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityData); + PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityAddOrEdit); + PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityErase); + PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityAddResponse); + PACKET_TYPE_NAME_LOOKUP(PacketTypeOctreeDataNack); + PACKET_TYPE_NAME_LOOKUP(PacketTypeVoxelEditNack); + PACKET_TYPE_NAME_LOOKUP(PacketTypeParticleEditNack); + PACKET_TYPE_NAME_LOOKUP(PacketTypeEntityEditNack); + PACKET_TYPE_NAME_LOOKUP(PacketTypeSignedTransactionPayment); + default: + return QString("Type: ") + QString::number((int)type); + } + return QString("unexpected"); +} + + + QByteArray byteArrayWithPopulatedHeader(PacketType type, const QUuid& connectionUUID) { QByteArray freshByteArray(MAX_PACKET_HEADER_BYTES, 0); freshByteArray.resize(populatePacketHeader(freshByteArray, type, connectionUUID)); diff --git a/libraries/networking/src/PacketHeaders.h b/libraries/networking/src/PacketHeaders.h index a558710c67..bb64388dd6 100644 --- a/libraries/networking/src/PacketHeaders.h +++ b/libraries/networking/src/PacketHeaders.h @@ -19,7 +19,7 @@ #include "UUID.h" // NOTE: if adding a new packet type, you can replace one marked usable or add at the end - +// NOTE: if you want the name of the packet type to be available for debugging or logging, update nameForPacketType() as well enum PacketType { PacketTypeUnknown, PacketTypeStunResponse, @@ -87,6 +87,7 @@ const int NUM_STATIC_HEADER_BYTES = sizeof(PacketVersion) + NUM_BYTES_RFC4122_UU const int MAX_PACKET_HEADER_BYTES = sizeof(PacketType) + NUM_BYTES_MD5_HASH + NUM_STATIC_HEADER_BYTES; PacketVersion versionForPacketType(PacketType type); +QString nameForPacketType(PacketType type); const QUuid nullUUID = QUuid(); @@ -116,5 +117,6 @@ const PacketVersion VERSION_ENTITIES_HAVE_ANIMATION = 1; const PacketVersion VERSION_ROOT_ELEMENT_HAS_DATA = 2; const PacketVersion VERSION_ENTITIES_SUPPORT_SPLIT_MTU = 3; const PacketVersion VERSION_ENTITIES_HAS_FILE_BREAKS = VERSION_ENTITIES_SUPPORT_SPLIT_MTU; +const PacketVersion VERSION_VOXELS_HAS_FILE_BREAKS = 1; #endif // hifi_PacketHeaders_h diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp index 3611605515..ff9ebbcd87 100644 --- a/libraries/octree/src/Octree.cpp +++ b/libraries/octree/src/Octree.cpp @@ -233,15 +233,31 @@ OctreeElement* Octree::createMissingElement(OctreeElement* lastParentElement, co } } -int Octree::readElementData(OctreeElement* destinationElement, const unsigned char* nodeData, int bytesLeftToRead, +int Octree::readElementData(OctreeElement* destinationElement, const unsigned char* nodeData, int bytesAvailable, ReadBitstreamToTreeParams& args) { + int bytesLeftToRead = bytesAvailable; + int bytesRead = 0; + // give this destination element the child mask from the packet const unsigned char ALL_CHILDREN_ASSUMED_TO_EXIST = 0xFF; + + if (bytesLeftToRead < sizeof(unsigned char)) { + qDebug() << "UNEXPECTED: readElementData() only had " << bytesLeftToRead << " bytes. Not enough for meaningful data."; + return bytesAvailable; // assume we read the entire buffer... + } + + if (destinationElement->getScale() < SCALE_AT_DANGEROUSLY_DEEP_RECURSION) { + qDebug() << "UNEXPECTED: readElementData() destination element is unreasonably small [" + << destinationElement->getScale() * (float)TREE_SCALE << " meters] " + << " Discarding " << bytesAvailable << " remaining bytes."; + return bytesAvailable; // assume we read the entire buffer... + } + unsigned char colorInPacketMask = *nodeData; + bytesRead += sizeof(colorInPacketMask); + bytesLeftToRead -= sizeof(colorInPacketMask); - // instantiate variable for bytes already read - int bytesRead = sizeof(colorInPacketMask); for (int i = 0; i < NUMBER_OF_CHILDREN; i++) { // check the colors mask to see if we have a child to color in if (oneAtBit(colorInPacketMask, i)) { @@ -256,9 +272,13 @@ int Octree::readElementData(OctreeElement* destinationElement, const unsigned ch OctreeElement* childElementAt = destinationElement->getChildAtIndex(i); bool nodeIsDirty = false; if (childElementAt) { - bytesRead += childElementAt->readElementDataFromBuffer(nodeData + bytesRead, bytesLeftToRead, args); + + int childElementDataRead = childElementAt->readElementDataFromBuffer(nodeData + bytesRead, bytesLeftToRead, args); childElementAt->setSourceUUID(args.sourceUUID); + bytesRead += childElementDataRead; + bytesLeftToRead -= childElementDataRead; + // if we had a local version of the element already, it's possible that we have it already but // with the same color data, so this won't count as a change. To address this we check the following if (!childElementAt->isDirty() && childElementAt->getShouldRender() && !childElementAt->isRendered()) { @@ -273,17 +293,28 @@ int Octree::readElementData(OctreeElement* destinationElement, const unsigned ch } } - // give this destination element the child mask from the packet - unsigned char childrenInTreeMask = args.includeExistsBits ? *(nodeData + bytesRead) : ALL_CHILDREN_ASSUMED_TO_EXIST; - unsigned char childMask = *(nodeData + bytesRead + (args.includeExistsBits ? sizeof(childrenInTreeMask) : 0)); + unsigned char childrenInTreeMask = ALL_CHILDREN_ASSUMED_TO_EXIST; + unsigned char childInBufferMask = 0; + int bytesForMasks = args.includeExistsBits ? sizeof(childrenInTreeMask) + sizeof(childInBufferMask) + : sizeof(childInBufferMask); + + if (bytesLeftToRead < bytesForMasks) { + qDebug() << "UNEXPECTED: readElementDataFromBuffer() only had " << bytesLeftToRead << " bytes before masks. " + "Not enough for meaningful data."; + return bytesAvailable; // assume we read the entire buffer... + } + + childrenInTreeMask = args.includeExistsBits ? *(nodeData + bytesRead) : ALL_CHILDREN_ASSUMED_TO_EXIST; + childInBufferMask = *(nodeData + bytesRead + (args.includeExistsBits ? sizeof(childrenInTreeMask) : 0)); int childIndex = 0; - bytesRead += args.includeExistsBits ? sizeof(childrenInTreeMask) + sizeof(childMask) : sizeof(childMask); + bytesRead += bytesForMasks; + bytesLeftToRead -= bytesForMasks; - while (bytesLeftToRead - bytesRead > 0 && childIndex < NUMBER_OF_CHILDREN) { + while (bytesLeftToRead > 0 && childIndex < NUMBER_OF_CHILDREN) { // check the exists mask to see if we have a child to traverse into - if (oneAtBit(childMask, childIndex)) { + if (oneAtBit(childInBufferMask, childIndex)) { if (!destinationElement->getChildAtIndex(childIndex)) { // add a child at that index, if it doesn't exist destinationElement->addChildAtIndex(childIndex); @@ -294,8 +325,11 @@ int Octree::readElementData(OctreeElement* destinationElement, const unsigned ch } // tell the child to read the subsequent data - bytesRead += readElementData(destinationElement->getChildAtIndex(childIndex), - nodeData + bytesRead, bytesLeftToRead - bytesRead, args); + int lowerLevelBytes = readElementData(destinationElement->getChildAtIndex(childIndex), + nodeData + bytesRead, bytesLeftToRead, args); + + bytesRead += lowerLevelBytes; + bytesLeftToRead -= lowerLevelBytes; } childIndex++; } @@ -314,7 +348,9 @@ int Octree::readElementData(OctreeElement* destinationElement, const unsigned ch // if this is the root, and there is more data to read, allow it to read it's element data... if (destinationElement == _rootElement && rootElementHasData() && (bytesLeftToRead - bytesRead) > 0) { // tell the element to read the subsequent data - bytesRead += _rootElement->readElementDataFromBuffer(nodeData + bytesRead, bytesLeftToRead - bytesRead, args); + int rootDataSize = _rootElement->readElementDataFromBuffer(nodeData + bytesRead, bytesLeftToRead - bytesRead, args); + bytesRead += rootDataSize; + bytesLeftToRead -= rootDataSize; } return bytesRead; @@ -322,7 +358,6 @@ int Octree::readElementData(OctreeElement* destinationElement, const unsigned ch void Octree::readBitstreamToTree(const unsigned char * bitstream, unsigned long int bufferSizeBytes, ReadBitstreamToTreeParams& args) { - int bytesRead = 0; const unsigned char* bitstreamAt = bitstream; @@ -337,9 +372,18 @@ void Octree::readBitstreamToTree(const unsigned char * bitstream, unsigned long while (bitstreamAt < bitstream + bufferSizeBytes) { OctreeElement* bitstreamRootElement = nodeForOctalCode(args.destinationElement, (unsigned char *)bitstreamAt, NULL); - if (*bitstreamAt != *bitstreamRootElement->getOctalCode()) { - // if the octal code returned is not on the same level as - // the code being searched for, we have OctreeElements to create + + int numberOfThreeBitSectionsInStream = numberOfThreeBitSectionsInCode(bitstreamAt, bufferSizeBytes); + + if (numberOfThreeBitSectionsInStream == OVERFLOWED_OCTCODE_BUFFER) { + qDebug() << "UNEXPECTED: parsing of the octal code would overflow the buffer. This buffer is corrupt. Returning."; + return; + } + + int numberOfThreeBitSectionsFromNode = numberOfThreeBitSectionsInCode(bitstreamRootElement->getOctalCode()); + + // if the octal code returned is not on the same level as the code being searched for, we have OctreeElements to create + if (numberOfThreeBitSectionsInStream != numberOfThreeBitSectionsFromNode) { // Note: we need to create this element relative to root, because we're assuming that the bitstream for the initial // octal code is always relative to root! @@ -349,16 +393,18 @@ void Octree::readBitstreamToTree(const unsigned char * bitstream, unsigned long } } - int octalCodeBytes = bytesRequiredForCodeLength(*bitstreamAt); + int octalCodeBytes = bytesRequiredForCodeLength(numberOfThreeBitSectionsInStream); int theseBytesRead = 0; theseBytesRead += octalCodeBytes; - theseBytesRead += readElementData(bitstreamRootElement, bitstreamAt + octalCodeBytes, + int lowerLevelBytes = readElementData(bitstreamRootElement, bitstreamAt + octalCodeBytes, bufferSizeBytes - (bytesRead + octalCodeBytes), args); + theseBytesRead += lowerLevelBytes; + // skip bitstream to new startPoint bitstreamAt += theseBytesRead; - bytesRead += theseBytesRead; + bytesRead += theseBytesRead; if (args.wantImportProgress) { emit importProgress((100 * (bitstreamAt - bitstream)) / bufferSizeBytes); @@ -1004,7 +1050,7 @@ int Octree::encodeTreeBitstream(OctreeElement* element, params.stopReason = EncodeBitstreamParams::DIDNT_FIT; return bytesWritten; } - + bytesWritten += codeLength; // keep track of byte count int currentEncodeLevel = 0; @@ -1421,6 +1467,7 @@ int Octree::encodeTreeBitstreamRecursion(OctreeElement* element, // We wil write this bit mask but we may come back later and update the bits that are actually included packetData->releaseReservedBytes(sizeof(childrenDataBits)); continueThisLevel = packetData->appendBitMask(childrenDataBits); + int childDataBitsPlaceHolder = packetData->getUncompressedByteOffset(sizeof(childrenDataBits)); unsigned char actualChildrenDataBits = 0; @@ -1470,6 +1517,13 @@ int Octree::encodeTreeBitstreamRecursion(OctreeElement* element, // Continue this level so long as some part of this child element was appended. bool childFit = (childAppendState != OctreeElement::NONE); + // some datatypes (like Voxels) assume that all child data will fit, if it doesn't fit + // the data type wants to bail on this element level completely + if (!childFit && mustIncludeAllChildData()) { + continueThisLevel = false; + break; + } + // If the child was partially or fully appended, then mark the actualChildrenDataBits as including // this child data if (childFit) { @@ -1500,15 +1554,14 @@ int Octree::encodeTreeBitstreamRecursion(OctreeElement* element, } } - if (!continueThisLevel) { + if (!mustIncludeAllChildData() && !continueThisLevel) { qDebug() << "WARNING UNEXPECTED CASE: reached end of child element data loop with continueThisLevel=FALSE"; qDebug() << "This is not expected!!!! -- continueThisLevel=FALSE...."; } - if (actualChildrenDataBits != childrenDataBits) { + if (continueThisLevel && actualChildrenDataBits != childrenDataBits) { // repair the child data mask continueThisLevel = packetData->updatePriorBitMask(childDataBitsPlaceHolder, actualChildrenDataBits); - if (!continueThisLevel) { qDebug() << "WARNING UNEXPECTED CASE: Failed to update childDataBitsPlaceHolder"; qDebug() << "This is not expected!!!! -- continueThisLevel=FALSE...."; @@ -1764,17 +1817,22 @@ int Octree::encodeTreeBitstreamRecursion(OctreeElement* element, continueThisLevel = packetData->endLevel(thisLevelKey); } else { packetData->discardLevel(thisLevelKey); - qDebug() << "WARNING UNEXPECTED CASE: Something failed in attempting to pack this element"; - qDebug() << "This is not expected!!!! -- continueThisLevel=FALSE...."; + + if (!mustIncludeAllChildData()) { + qDebug() << "WARNING UNEXPECTED CASE: Something failed in attempting to pack this element"; + qDebug() << "This is not expected!!!! -- continueThisLevel=FALSE...."; + } } // This happens if the element could not be written at all. In the case of Octree's that support partial // element data, continueThisLevel will be true. So this only happens if the full element needs to be // added back to the element bag. if (!continueThisLevel) { - qDebug() << "WARNING UNEXPECTED CASE - Something failed in attempting to pack this element"; - qDebug() << "IS THIS EVER EXPECTED???? -- continueThisLevel=FALSE...." ; - qDebug() << " calling bag.insert(element);....."; + if (!mustIncludeAllChildData()) { + qDebug() << "WARNING UNEXPECTED CASE - Something failed in attempting to pack this element."; + qDebug() << " If the datatype requires all child data, then this might happen. Otherwise" ; + qDebug() << " this is an unexpected case and we should research a potential logic error." ; + } bag.insert(element); @@ -1826,6 +1884,10 @@ bool Octree::readFromSVOFile(const char* fileName) { bool wantImportProgress = true; + PacketType expectedType = expectedDataPacketType(); + PacketVersion expectedVersion = versionForPacketType(expectedType); + bool hasBufferBreaks = versionHasSVOfileBreaks(expectedVersion); + // before reading the file, check to see if this version of the Octree supports file versions if (getWantSVOfileVersions()) { @@ -1840,15 +1902,26 @@ bool Octree::readFromSVOFile(const char* fileName) { unsigned long dataLength = HEADER_LENGTH; // if so, read the first byte of the file and see if it matches the expected version code - PacketType expectedType = expectedDataPacketType(); - PacketType gotType; memcpy(&gotType, dataAt, sizeof(gotType)); + + dataAt += sizeof(expectedType); + dataLength -= sizeof(expectedType); + gotVersion = *dataAt; + + // NOTE: SPECIAL CASE for old voxel svo files. The old voxel SVO files didn't have header + // details. They started with the the octalcode for the root. Which was always 00 which matches PacketTypeUnknown + unsigned char* firstByteAt = (unsigned char*)&fileHeader; + unsigned char firstByteValue = *firstByteAt; + if (expectedType == PacketTypeVoxelData && firstByteValue == 0) { + gotType = PacketTypeVoxelData; + gotVersion = 0; + qDebug() << "Detected OLD Voxels format."; + headerLength = 0; // old format files don't have headers + file.seekg( 0, std::ios::beg ); // rewind to the beginning so old logic will work + } if (gotType == expectedType) { - dataAt += sizeof(expectedType); - dataLength -= sizeof(expectedType); - gotVersion = *dataAt; if (canProcessVersion(gotVersion)) { dataAt += sizeof(gotVersion); dataLength -= sizeof(gotVersion); @@ -1857,25 +1930,25 @@ bool Octree::readFromSVOFile(const char* fileName) { versionForPacketType(expectedDataPacketType()), gotVersion); hasBufferBreaks = versionHasSVOfileBreaks(gotVersion); - if (hasBufferBreaks) { - qDebug() << " this version includes buffer breaks"; - } else { - qDebug() << " this version does not include buffer breaks"; - } - } else { qDebug("SVO file version mismatch. Expected: %d Got: %d", versionForPacketType(expectedDataPacketType()), gotVersion); } } else { - qDebug("SVO file type mismatch. Expected: %c Got: %c", expectedType, gotType); + qDebug() << "SVO file type mismatch. Expected: " << nameForPacketType(expectedType) + << " Got: " << nameForPacketType(gotType); } } else { qDebug() << " NOTE: this file type does not include type and version information."; fileOk = true; // assume the file is ok } - + + if (hasBufferBreaks) { + qDebug() << " this version includes buffer breaks"; + } else { + qDebug() << " this version does not include buffer breaks"; + } if (fileOk) { @@ -1906,7 +1979,6 @@ bool Octree::readFromSVOFile(const char* fileName) { quint16 chunkLength = 0; file.read((char*)&chunkLength, sizeof(chunkLength)); // read the chunk size from the file - remainingLength -= sizeof(chunkLength); if (chunkLength > remainingLength) { @@ -1942,6 +2014,7 @@ bool Octree::readFromSVOFile(const char* fileName) { file.close(); } + return fileOk; } @@ -1951,25 +2024,25 @@ void Octree::writeToSVOFile(const char* fileName, OctreeElement* element) { if(file.is_open()) { qDebug("Saving to file %s...", fileName); - bool hasBufferBreaks = false; + PacketType expectedType = expectedDataPacketType(); + PacketVersion expectedVersion = versionForPacketType(expectedType); + bool hasBufferBreaks = versionHasSVOfileBreaks(expectedVersion); // before reading the file, check to see if this version of the Octree supports file versions if (getWantSVOfileVersions()) { // if so, read the first byte of the file and see if it matches the expected version code - PacketType expectedType = expectedDataPacketType(); - PacketVersion expectedVersion = versionForPacketType(expectedType); file.write(reinterpret_cast(&expectedType), sizeof(expectedType)); file.write(&expectedVersion, sizeof(expectedVersion)); - - qDebug("SVO file type: %c version: %d", expectedType, expectedVersion); + qDebug() << "SVO file type: " << nameForPacketType(expectedType) << " version: " << (int)expectedVersion; hasBufferBreaks = versionHasSVOfileBreaks(expectedVersion); - if (hasBufferBreaks) { - qDebug() << " this version includes buffer breaks"; - } else { - qDebug() << " this version does not include buffer breaks"; - } } + if (hasBufferBreaks) { + qDebug() << " this version includes buffer breaks"; + } else { + qDebug() << " this version does not include buffer breaks"; + } + OctreeElementBag elementBag; OctreeElementExtraEncodeData extraEncodeData; diff --git a/libraries/octree/src/Octree.h b/libraries/octree/src/Octree.h index f58b5f0cbd..5bb33d8c75 100644 --- a/libraries/octree/src/Octree.h +++ b/libraries/octree/src/Octree.h @@ -238,6 +238,7 @@ public: virtual int minimumRequiredRootDataBytes() const { return 0; } virtual bool suppressEmptySubtrees() const { return true; } virtual void releaseSceneEncodeData(OctreeElementExtraEncodeData* extraEncodeData) const { } + virtual bool mustIncludeAllChildData() const { return true; } /// some versions of the SVO file will include breaks with buffer lengths between each buffer chunk in the SVO /// file. If the Octree subclass expects this for this particular version of the file, it should override this @@ -352,6 +353,7 @@ public: bool getIsClient() const { return !_isServer; } /// Is this a client based tree. Allows guards for certain operations void setIsClient(bool isClient) { _isServer = !isClient; } + virtual void dumpTree() { }; signals: void importSize(float x, float y, float z); diff --git a/libraries/octree/src/OctreeConstants.h b/libraries/octree/src/OctreeConstants.h index 0f836c7515..4186b90888 100644 --- a/libraries/octree/src/OctreeConstants.h +++ b/libraries/octree/src/OctreeConstants.h @@ -37,6 +37,8 @@ const float VIEW_FRUSTUM_FOV_OVERSEND = 60.0f; // These are guards to prevent our voxel tree recursive routines from spinning out of control const int UNREASONABLY_DEEP_RECURSION = 20; // use this for something that you want to be shallow, but not spin out const int DANGEROUSLY_DEEP_RECURSION = 200; // use this for something that needs to go deeper +const float SCALE_AT_UNREASONABLY_DEEP_RECURSION = (1.0f / powf(2.0f, UNREASONABLY_DEEP_RECURSION)); +const float SCALE_AT_DANGEROUSLY_DEEP_RECURSION = (1.0f / powf(2.0f, DANGEROUSLY_DEEP_RECURSION)); const int DEFAULT_MAX_OCTREE_PPS = 600; // the default maximum PPS we think any octree based server should send to a client diff --git a/libraries/voxels/src/VoxelTree.cpp b/libraries/voxels/src/VoxelTree.cpp index 5fe3fbdf05..0809e922e1 100644 --- a/libraries/voxels/src/VoxelTree.cpp +++ b/libraries/voxels/src/VoxelTree.cpp @@ -11,11 +11,10 @@ #include -#include +#include #include #include - #include "VoxelTree.h" #include "Tags.h" @@ -567,3 +566,26 @@ int VoxelTree::processEditPacketData(PacketType packetType, const unsigned char* return 0; } } + +class VoxelTreeDebugOperator : public RecurseOctreeOperator { +public: + virtual bool preRecursion(OctreeElement* element); + virtual bool postRecursion(OctreeElement* element) { return true; } +}; + +bool VoxelTreeDebugOperator::preRecursion(OctreeElement* element) { + VoxelTreeElement* treeElement = static_cast(element); + qDebug() << "VoxelTreeElement [" << treeElement << ":" << treeElement->getAACube() << "]"; + qDebug() << " isLeaf:" << treeElement->isLeaf(); + qDebug() << " color:" << treeElement->getColor()[0] << ", " + << treeElement->getColor()[1] << ", " + << treeElement->getColor()[2]; + return true; +} + +void VoxelTree::dumpTree() { + // First, look for the existing entity in the tree.. + VoxelTreeDebugOperator theOperator; + recurseTreeWithOperator(&theOperator); +} + diff --git a/libraries/voxels/src/VoxelTree.h b/libraries/voxels/src/VoxelTree.h index 03d07a1649..547f590270 100644 --- a/libraries/voxels/src/VoxelTree.h +++ b/libraries/voxels/src/VoxelTree.h @@ -51,12 +51,30 @@ public: void readCodeColorBufferToTree(const unsigned char* codeColorBuffer, bool destructive = false); + + virtual bool getWantSVOfileVersions() const { return true; } + virtual bool canProcessVersion(PacketVersion thisVersion) const { + return thisVersion == 0 || thisVersion == versionForPacketType(expectedDataPacketType()); } + virtual PacketVersion expectedVersion() const { return versionForPacketType(expectedDataPacketType()); } + virtual PacketType expectedDataPacketType() const { return PacketTypeVoxelData; } virtual bool handlesEditPacketType(PacketType packetType) const; virtual int processEditPacketData(PacketType packetType, const unsigned char* packetData, int packetLength, const unsigned char* editData, int maxLength, const SharedNodePointer& node); virtual bool recurseChildrenWithData() const { return false; } + /// some versions of the SVO file will include breaks with buffer lengths between each buffer chunk in the SVO + /// file. If the Octree subclass expects this for this particular version of the file, it should override this + /// method and return true. + virtual bool versionHasSVOfileBreaks(PacketVersion thisVersion) const { + if (thisVersion == 0) { + return false; // old versions didn't have buffer breaks + } + return true; + } + + virtual void dumpTree(); + private: // helper functions for nudgeSubTree void recurseNodeForNudge(VoxelTreeElement* element, RecurseOctreeOperation operation, void* extraData); diff --git a/libraries/voxels/src/VoxelTreeElement.cpp b/libraries/voxels/src/VoxelTreeElement.cpp index 1ab1dd6686..f81e8c073f 100644 --- a/libraries/voxels/src/VoxelTreeElement.cpp +++ b/libraries/voxels/src/VoxelTreeElement.cpp @@ -73,6 +73,13 @@ OctreeElement::AppendState VoxelTreeElement::appendElementData(OctreePacketData* int VoxelTreeElement::readElementDataFromBuffer(const unsigned char* data, int bytesLeftToRead, ReadBitstreamToTreeParams& args) { const int BYTES_PER_COLOR = 3; + + if (bytesLeftToRead < BYTES_PER_COLOR) { + qDebug() << "UNEXPECTED: readElementDataFromBuffer() only had " << bytesLeftToRead << " bytes. " + "Not enough for meaningful data."; + return bytesLeftToRead; + } + // pull the color for this child nodeColor newColor = { 128, 128, 128, 1};