cleaned up const semantics in streaming voxel packets, fixed double reset bug

libraries/voxel-server-library/src/VoxelSendThread.cpp

@@ -212,8 +212,6 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod
     int packetsSentThisInterval = 0;
     bool somethingToSend = true; // assume we have something
 
-    _tempPacket.reset(); // reset at top of distributor
-
     // FOR NOW... node tells us if it wants to receive only view frustum deltas
     bool wantDelta = viewFrustumChanged && nodeData->getWantDelta();
 
@@ -404,7 +402,8 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod
             // We only consider sending anything if there is something in the _tempPacket to send... But
             // if bytesWritten == 0 it means either the subTree couldn't fit or we had an empty bag... Both cases
             // mean we should send the previous packet contents and reset it. 
-            if (_tempPacket.hasContent() && bytesWritten == 0) {
+            bool sendNow = (bytesWritten == 0);
+            if (_tempPacket.hasContent() && sendNow) {
                 nodeData->writeToPacket(_tempPacket.getFinalizedData(), _tempPacket.getFinalizedSize());
                 packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);
                 _tempPacket.reset();

libraries/voxels/src/VoxelPacket.cpp

@@ -16,7 +16,6 @@ void VoxelPacket::reset() {
     _bytesInUse = 0;
     _bytesAvailable = MAX_VOXEL_PACKET_SIZE;
     _subTreeAt = 0;
-    _levelAt = 0;
 }
 
 VoxelPacket::~VoxelPacket() {
@@ -54,6 +53,15 @@ bool VoxelPacket::updatePriorBitMask(int offset, unsigned char bitmask) {
     return success;
 }
 
+bool VoxelPacket::updatePriorBytes(int offset, const unsigned char* replacementBytes, int length) {
+    bool success = false;
+    if (length >= 0 && offset >= 0 && ((offset + length) <= _bytesInUse)) {
+        memcpy(&_buffer[offset], replacementBytes, length);
+        success = true;
+    }
+    return success;
+}
+
 bool VoxelPacket::startSubTree(const unsigned char* octcode) {
     bool success = false;
     int possibleStartAt = _bytesInUse;
@@ -79,6 +87,7 @@ void VoxelPacket::discardSubTree() {
     int bytesInSubTree = _bytesInUse - _subTreeAt;
     _bytesInUse -= bytesInSubTree;
     _bytesAvailable += bytesInSubTree; 
+    _subTreeAt = _bytesInUse; // should be the same actually...
 }
 
 int VoxelPacket::startLevel() {
@@ -96,7 +105,6 @@ void VoxelPacket::endLevel() {
     // nothing to do
 }
 
-
 bool VoxelPacket::appendBitMask(unsigned char bitmask) {
     bool success = false;
     if (_bytesAvailable > 0) {

libraries/voxels/src/VoxelPacket.h

@@ -38,6 +38,10 @@ public:
     /// bitmask would cause packet to be less compressed, or if offset was out of range.
     bool updatePriorBitMask(int offset, unsigned char bitmask); 
 
+    /// updates the uncompressed content of the stream starting at byte offset with replacementBytes for length.
+    /// Might fail if the new bytes would cause packet to be less compressed, or if offset and length was out of range.
+    bool updatePriorBytes(int offset, const unsigned char* replacementBytes, int length);
+
     bool appendColor(rgbColor color);
 
     /// returns a byte offset from beginning of the uncompressed stream based on offset from end. 
@@ -50,24 +54,15 @@ public:
     int getFinalizedSize() const { return _bytesInUse; }
 
     /// get pointer to the start of uncompressed stream buffer
-    unsigned char* getUncompressedData() { return &_buffer[0]; }
+    const unsigned char* getUncompressedData() { return &_buffer[0]; }
     /// the size of the packet in uncompressed form
     int getUncompressedSize() { return _bytesInUse; }
 
     /// has some content been written to the packet
     bool hasContent() const { return (_bytesInUse > 0); }
     
-    ////////////////////////////////////
+    // XXXBHG: TO DO...
-    // XXXBHG: Questions...
+    //  *  add stats tracking for number of bytes of octal code, bitmasks, and colors in a packet.
-    //  Slice Reshuffle...
-    //
-    //  1) getEndOfBuffer() is used by recursive slice shuffling... is there a safer API for that? This usage would probably
-    //     break badly with compression... Especially since we do a memcpy into the uncompressed buffer, we'd need to
-    //     add an "updateBytes()" method... which could definitely fail on compression.... It would also break any RLE we
-    //     might implement, since the order of the colors would clearly change.
-    //
-    //  2) add stats tracking for number of bytes of octal code, bitmasks, and colors in a packet.
-    unsigned char* getEndOfBuffer() { return &_buffer[_bytesInUse]; } /// get pointer to current end of stream buffer
     
 private:
     /// appends raw bytes, might fail if byte would cause packet to be too large
@@ -80,7 +75,6 @@ private:
     int _bytesInUse;
     int _bytesAvailable;
     int _subTreeAt;
-    int _levelAt;
 };
 
 #endif /* defined(__hifi__VoxelPacket__) */

libraries/voxels/src/VoxelTree.cpp

@@ -170,7 +170,7 @@ VoxelNode* VoxelTree::nodeForOctalCode(VoxelNode* ancestorNode,
 }
 
 // returns the node created!
-VoxelNode* VoxelTree::createMissingNode(VoxelNode* lastParentNode, unsigned char* codeToReach) {
+VoxelNode* VoxelTree::createMissingNode(VoxelNode* lastParentNode, const unsigned char* codeToReach) {
     int indexOfNewChild = branchIndexWithDescendant(lastParentNode->getOctalCode(), codeToReach);
     // If this parent node is a leaf, then you know the child path doesn't exist, so deal with
     // breaking up the leaf first, which will also create a child path
@@ -193,7 +193,7 @@ VoxelNode* VoxelTree::createMissingNode(VoxelNode* lastParentNode, unsigned char
     }
 }
 
-int VoxelTree::readNodeData(VoxelNode* destinationNode, unsigned char* nodeData, int bytesLeftToRead,
+int VoxelTree::readNodeData(VoxelNode* destinationNode, const unsigned char* nodeData, int bytesLeftToRead,
                             ReadBitstreamToTreeParams& args) {
     // give this destination node the child mask from the packet
     const unsigned char ALL_CHILDREN_ASSUMED_TO_EXIST = 0xFF;
@@ -296,10 +296,10 @@ printf("CALLING destinationNode->safeDeepDeleteChildAtIndex(i) because of EXISTS
     return bytesRead;
 }
 
-void VoxelTree::readBitstreamToTree(unsigned char * bitstream, unsigned long int bufferSizeBytes, 
+void VoxelTree::readBitstreamToTree(const unsigned char * bitstream, unsigned long int bufferSizeBytes,
                                     ReadBitstreamToTreeParams& args) {
     int bytesRead = 0;
-    unsigned char* bitstreamAt = bitstream;
+    const unsigned char* bitstreamAt = bitstream;
 
     // If destination node is not included, set it to root
     if (!args.destinationNode) {
@@ -330,9 +330,10 @@ void VoxelTree::readBitstreamToTree(unsigned char * bitstream, unsigned long int
         int octalCodeBytes = bytesRequiredForCodeLength(*bitstreamAt);
         int theseBytesRead = 0;
         theseBytesRead += octalCodeBytes;
+        
         theseBytesRead += readNodeData(bitstreamRootNode, bitstreamAt + octalCodeBytes, 
                                        bufferSizeBytes - (bytesRead + octalCodeBytes), args);
-
+                                       
         // skip bitstream to new startPoint
         bitstreamAt += theseBytesRead;
         bytesRead +=  theseBytesRead;
@@ -354,16 +355,16 @@ void VoxelTree::deleteVoxelAt(float x, float y, float z, float s) {
 
 class DeleteVoxelCodeFromTreeArgs {
 public:
-    bool            collapseEmptyTrees;
+    bool collapseEmptyTrees;
-    unsigned char*  codeBuffer;
+    const unsigned char* codeBuffer;
-    int             lengthOfCode;
+    int lengthOfCode;
-    bool            deleteLastChild;
+    bool deleteLastChild;
-    bool            pathChanged;
+    bool pathChanged;
 };
 
 // Note: uses the codeColorBuffer format, but the color's are ignored, because
 // this only finds and deletes the node from the tree.
-void VoxelTree::deleteVoxelCodeFromTree(unsigned char* codeBuffer, bool collapseEmptyTrees) {
+void VoxelTree::deleteVoxelCodeFromTree(const unsigned char* codeBuffer, bool collapseEmptyTrees) {
     // recurse the tree while decoding the codeBuffer, once you find the node in question, recurse
     // back and implement color reaveraging, and marking of lastChanged
     DeleteVoxelCodeFromTreeArgs args;
@@ -492,13 +493,13 @@ void VoxelTree::eraseAllVoxels() {
 
 class ReadCodeColorBufferToTreeArgs {
 public:
-    unsigned char*  codeColorBuffer;
+    const unsigned char* codeColorBuffer;
-    int             lengthOfCode;
+    int lengthOfCode;
-    bool            destructive;
+    bool destructive;
-    bool            pathChanged;
+    bool pathChanged;
 };
 
-void VoxelTree::readCodeColorBufferToTree(unsigned char* codeColorBuffer, bool destructive) {
+void VoxelTree::readCodeColorBufferToTree(const unsigned char* codeColorBuffer, bool destructive) {
     ReadCodeColorBufferToTreeArgs args;
     args.codeColorBuffer = codeColorBuffer;
     args.lengthOfCode    = numberOfThreeBitSectionsInCode(codeColorBuffer);
@@ -578,7 +579,7 @@ void VoxelTree::readCodeColorBufferToTreeRecursion(VoxelNode* node, void* extraD
     }
 }
 
-void VoxelTree::processRemoveVoxelBitstream(unsigned char* bitstream, int bufferSizeBytes) {
+void VoxelTree::processRemoveVoxelBitstream(const unsigned char* bitstream, int bufferSizeBytes) {
     //unsigned short int itemNumber = (*((unsigned short int*)&bitstream[sizeof(PACKET_HEADER)]));
 
     int numBytesPacketHeader = numBytesForPacketHeader(bitstream);
@@ -1043,14 +1044,14 @@ int VoxelTree::encodeTreeBitstream(VoxelNode* node,
         doneEncoding(node);
         return bytesWritten;
     }
-    
+
     // write the octal code
     bool roomForOctalCode = false; // assume the worst
     int codeLength;
     if (params.chopLevels) {
         unsigned char* newCode = chopOctalCode(node->getOctalCode(), params.chopLevels);
         roomForOctalCode = packet->startSubTree(newCode);
-        
+
         if (newCode) {
             // old way...
             codeLength = bytesRequiredForCodeLength(numberOfThreeBitSectionsInCode(newCode));
@@ -1072,7 +1073,7 @@ int VoxelTree::encodeTreeBitstream(VoxelNode* node,
         // old way
         memcpy(outputBuffer, node->getOctalCode(), codeLength);
     }
-    
+
     // If the octalcode couldn't fit, then we can return, because no nodes below us will fit...
     if (!roomForOctalCode) {
         doneEncoding(node);
@@ -1562,22 +1563,7 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node,
         continueThisLevel = false; // ???
      }
  
-    // if we were unable to fit this level in our packet, then rewind and add it to the node bag for 
+    if (continueThisLevel && keepDiggingDeeper) {
-    // sending later...
-    if (!continueThisLevel) {
-        packet->discardLevel(thisLevelKey);
-        bag.insert(node);
-
-        // don't need to check node here, because we can't get here with no node
-        if (params.stats) {
-            params.stats->didntFit(node);
-        }
-
-        //printf("line: %d <<!continueThisLevel>> returning 0...\n",__LINE__);
-        return 0;
-    }
-
-    if (keepDiggingDeeper) {
         // at this point, we need to iterate the children who are in view, even if not colored
         // and we need to determine if there's a deeper tree below them that we care about.
         //
@@ -1596,9 +1582,9 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node,
         // and then later reshuffle these sections of our output buffer back into normal order. This allows us to make
         // a single recursive pass in distance sorted order, but retain standard order in our encoded packet
         int recursiveSliceSizes[NUMBER_OF_CHILDREN];
-        unsigned char* recursiveSliceStarts[NUMBER_OF_CHILDREN];
+        const unsigned char* recursiveSliceStarts[NUMBER_OF_CHILDREN];
-        unsigned char* recursiveSliceStartsOLD[NUMBER_OF_CHILDREN];
+        const unsigned char* recursiveSliceStartsOLD[NUMBER_OF_CHILDREN];
-        unsigned char* firstRecursiveSlice = packet->getEndOfBuffer();
+        int firstRecursiveSliceOffset = packet->getUncompressedByteOffset();
         unsigned char* firstRecursiveSliceOLD = outputBuffer;
         int allSlicesSize = 0;
 
@@ -1612,7 +1598,7 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node,
 
                 int thisLevel = currentEncodeLevel;
                 // remember this for reshuffling
-                recursiveSliceStarts[originalIndex] = packet->getEndOfBuffer();
+                recursiveSliceStarts[originalIndex] = packet->getUncompressedData() + packet->getUncompressedSize();
                 recursiveSliceStartsOLD[originalIndex] = outputBuffer;
 
                 int childTreeBytesOut = encodeTreeBitstreamRecursion(childNode, 
@@ -1689,7 +1675,7 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node,
             for (int originalIndex = 0; originalIndex < NUMBER_OF_CHILDREN; originalIndex++) {
                 if (oneAtBit(childrenExistInPacketBits, originalIndex)) {
                     int thisSliceSize = recursiveSliceSizes[originalIndex];
-                    unsigned char* thisSliceStarts = recursiveSliceStarts[originalIndex];
+                    const unsigned char* thisSliceStarts = recursiveSliceStarts[originalIndex];
 
                     memcpy(tempBufferTo, thisSliceStarts, thisSliceSize);
                     tempBufferTo += thisSliceSize;
@@ -1697,8 +1683,7 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node,
             }
 
             // now that all slices are back in the correct order, copy them to the correct output buffer
-            memcpy(firstRecursiveSlice, &tempReshuffleBuffer[0], allSlicesSize);
+            packet->updatePriorBytes(firstRecursiveSliceOffset, &tempReshuffleBuffer[0], allSlicesSize);
-
 
             // DO IT AGAIN FOR OLD WAY....
             unsigned char* tempBufferToOLD = &tempReshuffleBuffer[0]; // this is our temporary destination
@@ -1709,7 +1694,7 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node,
             for (int originalIndex = 0; originalIndex < NUMBER_OF_CHILDREN; originalIndex++) {
                 if (oneAtBit(childrenExistInPacketBits, originalIndex)) {
                     int thisSliceSize = recursiveSliceSizes[originalIndex];
-                    unsigned char* thisSliceStartsOLD = recursiveSliceStartsOLD[originalIndex];
+                    const unsigned char* thisSliceStartsOLD = recursiveSliceStartsOLD[originalIndex];
 
                     memcpy(tempBufferToOLD, thisSliceStartsOLD, thisSliceSize);
                     tempBufferToOLD += thisSliceSize;
@@ -1723,6 +1708,21 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node,
 
     } // end keepDiggingDeeper
 
+    // if we were unable to fit this level in our packet, then rewind and add it to the node bag for 
+    // sending later...
+    if (!continueThisLevel) {
+        packet->discardLevel(thisLevelKey);
+        bag.insert(node);
+
+        // don't need to check node here, because we can't get here with no node
+        if (params.stats) {
+            params.stats->didntFit(node);
+        }
+
+        //printf("line: %d <<!continueThisLevel>> returning 0...\n",__LINE__);
+        return 0;
+    }
+
     return bytesAtThisLevel;
 }
 
@@ -2059,26 +2059,26 @@ void VoxelTree::doneEncoding(VoxelNode* node) {
     emptyDeleteQueue();
 }
 
-void VoxelTree::startDeleting(unsigned char* code) {
+void VoxelTree::startDeleting(const unsigned char* code) {
     pthread_mutex_lock(&_deleteSetLock);
     _codesBeingDeleted.insert(code);
     pthread_mutex_unlock(&_deleteSetLock);
 }
 
-void VoxelTree::doneDeleting(unsigned char* code) {
+void VoxelTree::doneDeleting(const unsigned char* code) {
     pthread_mutex_lock(&_deleteSetLock);
     _codesBeingDeleted.erase(code);
     pthread_mutex_unlock(&_deleteSetLock);
 }
 
-bool VoxelTree::isEncoding(unsigned char* codeBuffer) {
+bool VoxelTree::isEncoding(const unsigned char* codeBuffer) {
     pthread_mutex_lock(&_encodeSetLock);
     bool isEncoding = (_codesBeingEncoded.find(codeBuffer) != _codesBeingEncoded.end());
     pthread_mutex_unlock(&_encodeSetLock);
     return isEncoding;
 }
 
-void VoxelTree::queueForLaterDelete(unsigned char* codeBuffer) {
+void VoxelTree::queueForLaterDelete(const unsigned char* codeBuffer) {
     pthread_mutex_lock(&_deletePendingSetLock);
     _codesPendingDelete.insert(codeBuffer);
     pthread_mutex_unlock(&_deletePendingSetLock);
@@ -2086,8 +2086,8 @@ void VoxelTree::queueForLaterDelete(unsigned char* codeBuffer) {
 
 void VoxelTree::emptyDeleteQueue() {
     pthread_mutex_lock(&_deletePendingSetLock);
-    for (std::set<unsigned char*>::iterator i = _codesPendingDelete.begin(); i != _codesPendingDelete.end(); ++i) {
+    for (std::set<const unsigned char*>::iterator i = _codesPendingDelete.begin(); i != _codesPendingDelete.end(); ++i) {
-        unsigned char* codeToDelete = *i;
+        const unsigned char* codeToDelete = *i;
         _codesBeingDeleted.erase(codeToDelete);
         deleteVoxelCodeFromTree(codeToDelete, COLLAPSE_EMPTY_TREE);
     }

libraries/voxels/src/VoxelTree.h

@@ -144,10 +144,10 @@ public:
 
     void eraseAllVoxels();
 
-    void processRemoveVoxelBitstream(unsigned char* bitstream, int bufferSizeBytes);
+    void processRemoveVoxelBitstream(const unsigned char* bitstream, int bufferSizeBytes);
-    void readBitstreamToTree(unsigned char* bitstream,  unsigned long int bufferSizeBytes, ReadBitstreamToTreeParams& args);
+    void readBitstreamToTree(const unsigned char* bitstream,  unsigned long int bufferSizeBytes, ReadBitstreamToTreeParams& args);
-    void readCodeColorBufferToTree(unsigned char* codeColorBuffer, bool destructive = false);
+    void readCodeColorBufferToTree(const unsigned char* codeColorBuffer, bool destructive = false);
-    void deleteVoxelCodeFromTree(unsigned char* codeBuffer, bool collapseEmptyTrees = DONT_COLLAPSE);
+    void deleteVoxelCodeFromTree(const unsigned char* codeBuffer, bool collapseEmptyTrees = DONT_COLLAPSE);
     void printTreeForDebugging(VoxelNode* startNode);
     void reaverageVoxelColors(VoxelNode* startNode);
 
@@ -231,8 +231,9 @@ private:
     static bool countVoxelsOperation(VoxelNode* node, void* extraData);
 
     VoxelNode* nodeForOctalCode(VoxelNode* ancestorNode, const unsigned char* needleCode, VoxelNode** parentOfFoundNode) const;
-    VoxelNode* createMissingNode(VoxelNode* lastParentNode, unsigned char* deepestCodeToCreate);
+    VoxelNode* createMissingNode(VoxelNode* lastParentNode, const unsigned char* codeToReach);
-    int readNodeData(VoxelNode *destinationNode, unsigned char* nodeData, int bufferSizeBytes, ReadBitstreamToTreeParams& args);
+    int readNodeData(VoxelNode *destinationNode, const unsigned char* nodeData, 
+                int bufferSizeBytes, ReadBitstreamToTreeParams& args);
     
     bool _isDirty;
     unsigned long int _nodesChangedFromBitstream;
@@ -250,26 +251,26 @@ private:
     /// Called to indicate that a VoxelNode is done being encoded.
     void doneEncoding(VoxelNode* node);
     /// Is the Octal Code currently being deleted?
-    bool isEncoding(unsigned char* codeBuffer);
+    bool isEncoding(const unsigned char* codeBuffer);
 
     /// Octal Codes of any subtrees currently being deleted. While any of these codes is being deleted, ancestors and 
     /// descendants of them can not be encoded.
-    std::set<unsigned char*>  _codesBeingDeleted;
+    std::set<const unsigned char*> _codesBeingDeleted;
     /// mutex lock to protect the deleting set
     pthread_mutex_t _deleteSetLock;
 
     /// Called to indicate that an octal code is in the process of being deleted.
-    void startDeleting(unsigned char* code);
+    void startDeleting(const unsigned char* code);
     /// Called to indicate that an octal code is done being deleted.
-    void doneDeleting(unsigned char* code);
+    void doneDeleting(const unsigned char* code);
     /// Octal Codes that were attempted to be deleted but couldn't be because they were actively being encoded, and were
     /// instead queued for later delete
-    std::set<unsigned char*>  _codesPendingDelete;
+    std::set<const unsigned char*> _codesPendingDelete;
     /// mutex lock to protect the deleting set
     pthread_mutex_t _deletePendingSetLock;
 
     /// Adds an Octal Code to the set of codes that needs to be deleted
-    void queueForLaterDelete(unsigned char* codeBuffer);
+    void queueForLaterDelete(const unsigned char* codeBuffer);
     /// flushes out any Octal Codes that had to be queued
     void emptyDeleteQueue();