better compressed packets with fewer calls to checkCompress

libraries/voxel-server-library/src/VoxelSendThread.cpp

@@ -311,6 +311,8 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod
     if (!nodeData->nodeBag.isEmpty()) {
         int bytesWritten = 0;
         uint64_t start = usecTimestampNow();
+        uint64_t startCompressTimeMsecs = VoxelPacket::_checkCompressTime / 1000;
+        uint64_t startCompressCalls = VoxelPacket::_checkCompressCalls;
 
         bool shouldSendEnvironments = _myServer->wantSendEnvironments() && shouldDo(ENVIRONMENT_SEND_INTERVAL_USECS, VOXEL_SEND_INTERVAL_USECS);
 
@@ -429,9 +431,15 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod
                 printf("WARNING! packetLoop() took %d milliseconds to generate %d bytes in %d packets, %d nodes still to send\n",
                         elapsedmsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count());
             }
-        } else if (_myServer->wantsDebugVoxelSending() && _myServer->wantsVerboseDebug()) {
+        } else if (truePacketsSent > 0 /*_myServer->wantsDebugVoxelSending() && _myServer->wantsVerboseDebug()*/) {
-            printf("packetLoop() took %d milliseconds to generate %d bytes in %d packets, %d nodes still to send\n",
+
-                    elapsedmsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count());
+            uint64_t endCompressCalls = VoxelPacket::_checkCompressCalls;
+            int elapsedCompressCalls = endCompressCalls - startCompressCalls;
+        
+            uint64_t endCompressTimeMsecs = VoxelPacket::_checkCompressTime / 1000;
+            int elapsedCompressTimeMsecs = endCompressTimeMsecs - startCompressTimeMsecs;
+            printf("packetLoop() took %d milliseconds [%d milliseconds %d calls in compress] to generate %d bytes in %d packets, %d nodes still to send\n",
+                    elapsedmsec, elapsedCompressTimeMsecs, elapsedCompressCalls, trueBytesSent, truePacketsSent, nodeData->nodeBag.count());
         }
         
         // if after sending packets we've emptied our bag, then we want to remember that we've sent all 

libraries/voxels/src/VoxelConstants.h

@@ -35,7 +35,7 @@ const float MAX_LOD_SIZE_MULTIPLIER = 2000.0f;
 
 const int NUMBER_OF_CHILDREN = 8;
 const int MAX_VOXEL_PACKET_SIZE = MAX_PACKET_SIZE - (sizeof(PACKET_TYPE) + sizeof(PACKET_VERSION));
-const int MAX_VOXEL_PACKET_COMPRESSION_RATIO = 1;
+const int MAX_VOXEL_PACKET_COMPRESSION_RATIO = 3;
 const int MAX_VOXEL_UNCOMRESSED_PACKET_SIZE = MAX_VOXEL_PACKET_SIZE * MAX_VOXEL_PACKET_COMPRESSION_RATIO;
 
 const int MAX_TREE_SLICE_BYTES = 26;

libraries/voxels/src/VoxelPacket.cpp

@@ -6,6 +6,7 @@
 //  Copyright (c) 2013 HighFidelity, Inc. All rights reserved.
 //
 
+#include <PerfStat.h>
 #include "VoxelPacket.h"
 
 VoxelPacket::VoxelPacket() {
@@ -30,6 +31,9 @@ bool VoxelPacket::append(const unsigned char* data, int length) {
         _bytesInUse += length;
         _bytesAvailable -= length;
         
+        success = true;
+        
+        /****
         // Now, check for compression, if we fit, then proceed, otherwise, rollback.
         if (checkCompress()) {
             success = true;
@@ -38,6 +42,7 @@ bool VoxelPacket::append(const unsigned char* data, int length) {
             _bytesInUse -= length;
             _bytesAvailable += length;
         }
+        ***/
     }
     return success;
 }
@@ -49,6 +54,9 @@ bool VoxelPacket::append(unsigned char byte) {
         _bytesInUse++;
         _bytesAvailable--; 
 
+        success = true;
+        
+        /****
         // Now, check for compression, if we fit, then proceed, otherwise, rollback.
         if (checkCompress()) {
             success = true;
@@ -57,6 +65,7 @@ bool VoxelPacket::append(unsigned char byte) {
             _bytesInUse--;
             _bytesAvailable++;
         }
+        ****/
     }
     return success;
 }
@@ -66,6 +75,10 @@ bool VoxelPacket::updatePriorBitMask(int offset, unsigned char bitmask) {
     if (offset >= 0 && offset < _bytesInUse) {
         unsigned char oldValue = _uncompressed[offset];
         _uncompressed[offset] = bitmask;
+
+        success = true;
+        
+        /****
         // Now, check for compression, if we fit, then proceed, otherwise, rollback.
         if (checkCompress()) {
             success = true;
@@ -73,6 +86,7 @@ bool VoxelPacket::updatePriorBitMask(int offset, unsigned char bitmask) {
             // rollback is easy, the length didn't change, but we need to restore the previous value
             _uncompressed[offset] = oldValue;
         }
+        ****/
     }
     return success;
 }
@@ -84,6 +98,9 @@ bool VoxelPacket::updatePriorBytes(int offset, const unsigned char* replacementB
         memcpy(&oldValues[0], &_uncompressed[offset], length); // save the old values for restore
         memcpy(&_uncompressed[offset], replacementBytes, length); // copy new content
 
+        success = true;
+        
+        /****
         // Now, check for compression, if we fit, then proceed, otherwise, rollback.
         if (checkCompress()) {
             success = true;
@@ -91,6 +108,7 @@ bool VoxelPacket::updatePriorBytes(int offset, const unsigned char* replacementB
             // rollback is easy, the length didn't change, but we need to restore the previous values
             memcpy(&_uncompressed[offset], &oldValues[0], length); // restore the old values
         }
+        ****/
     }
     return success;
 }
@@ -144,8 +162,12 @@ void VoxelPacket::discardLevel(int key) {
     checkCompress(); 
 }
 
-void VoxelPacket::endLevel() {
+bool VoxelPacket::endLevel(int key) {
-    // nothing to do
+    bool success = checkCompress();
+    if (!success) {
+        discardLevel(key);
+    }
+    return success;
 }
 
 bool VoxelPacket::appendBitMask(unsigned char bitmask) {
@@ -169,7 +191,13 @@ bool VoxelPacket::appendColor(const nodeColor& color) {
     return success;
 }
 
+uint64_t VoxelPacket::_checkCompressTime = 0;
+uint64_t VoxelPacket::_checkCompressCalls = 0;
+
 bool VoxelPacket::checkCompress() { 
+    PerformanceWarning warn(false,"VoxelPacket::checkCompress()",false,&_checkCompressTime,&_checkCompressCalls);
+
+
     bool success = false;
     const int MAX_COMPRESSION = 9;
 
@@ -243,31 +271,3 @@ void VoxelPacket::debugContent() {
     }
     printf("\n");
 }
-
-
-/***
-void VoxelPacket::compressPacket() { 
-    int uncompressedLength = getPacketLengthUncompressed();
-    const int MAX_COMPRESSION = 9;
-    // we only want to compress the data payload, not the message header
-    int numBytesPacketHeader = numBytesForPacketHeader(_voxelPacket);
-    QByteArray compressedData = qCompress(_voxelPacket+numBytesPacketHeader, 
-                                    uncompressedLength-numBytesPacketHeader, MAX_COMPRESSION);
-    _compressedPacket.clear();
-    _compressedPacket.append((const char*)_voxelPacket, numBytesPacketHeader);
-    _compressedPacket.append(compressedData);
-}
-
-void VoxelPacket::uncompressPacket() { 
-    int numBytesPacketHeader = numBytesForPacketHeader(packetData);
-    QByteArray compressedData((const char*)packetData + numBytesPacketHeader, 
-                            messageLength - numBytesPacketHeader);
-    QByteArray uncompressedData = qUncompress(compressedData);
-    QByteArray uncompressedPacket((const char*)packetData, numBytesPacketHeader);
-    uncompressedPacket.append(uncompressedData);
-    //app->_voxels.parseData((unsigned char*)uncompressedPacket.data(), uncompressedPacket.size());
-}
-***/
-
-
-

libraries/voxels/src/VoxelPacket.h

@@ -46,8 +46,9 @@ public:
     /// discards all content back to a previous marker key
     void discardLevel(int key);
     
-    /// ends a level without discarding it
+    /// ends a level, and performs any expensive finalization. may fail if finalization creates a stream which is too large
-    void endLevel();
+    /// if the finalization would fail, the packet will automatically discard the previous level.
+    bool endLevel(int key);
 
     /// appends a bitmask to the end of the stream, may fail if new data stream is too long to fit in packet
     bool appendBitMask(unsigned char bitmask);
@@ -84,7 +85,10 @@ public:
     void loadCompressedContent(const unsigned char* data, int length);
 
     void debugContent();
-    
+
+    static uint64_t _checkCompressTime;
+    static uint64_t _checkCompressCalls;
+     
 private:
     /// appends raw bytes, might fail if byte would cause packet to be too large
     bool append(const unsigned char* data, int length);
@@ -101,6 +105,8 @@ private:
     
     unsigned char _compressed[MAX_VOXEL_PACKET_SIZE];
     int _compressedBytes;
+    
+   
 };
 
 #endif /* defined(__hifi__VoxelPacket__) */

libraries/voxels/src/VoxelTree.cpp

@@ -1600,8 +1600,14 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node,
 
     // if we were unable to fit this level in our packet, then rewind and add it to the node bag for 
     // sending later...
-    if (!continueThisLevel) {
+    
+    if (continueThisLevel) {
+        continueThisLevel = packet->endLevel(thisLevelKey);
+    } else {
         packet->discardLevel(thisLevelKey);
+    }
+    
+    if (!continueThisLevel) {
         bag.insert(node);
 
         // don't need to check node here, because we can't get here with no node