first cut at compresses voxel packets

interface/src/VoxelPacketProcessor.cpp

@@ -8,6 +8,8 @@
 //  Threaded or non-threaded voxel packet receiver for the Application
 //
 
+#include <QByteArray>
+
 #include <PerfStat.h>
 
 #include "Application.h"
@@ -56,7 +58,17 @@ void VoxelPacketProcessor::processPacket(sockaddr& senderAddress, unsigned char*
                 app->_environment.parseData(&senderAddress, packetData, messageLength);
             } else {
                 app->_voxels.setDataSourceUUID(voxelServer->getUUID());
-                app->_voxels.parseData(packetData, messageLength);
+                
+                // thse packets are commpressed...
+                
+                int numBytesPacketHeader = numBytesForPacketHeader(packetData);
+                QByteArray compressedData((const char*)packetData + numBytesPacketHeader, 
+                                        messageLength - numBytesPacketHeader);
+                QByteArray uncompressedData = qUncompress(compressedData);
+                QByteArray uncompressedPacket((const char*)packetData, numBytesPacketHeader);
+                uncompressedPacket.append(uncompressedData);
+
+                app->_voxels.parseData((unsigned char*)uncompressedPacket.data(), uncompressedPacket.size());
                 app->_voxels.setDataSourceUUID(QUuid());
             }
         }

interface/src/VoxelSystem.cpp

@@ -595,7 +595,7 @@ int VoxelSystem::parseData(unsigned char* sourceBuffer, int numBytes) {
             PerformanceWarning warn(Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings),
                                     "readBitstreamToTree()");
             // ask the VoxelTree to read the bitstream into the tree
-            ReadBitstreamToTreeParams args(WANT_COLOR, WANT_EXISTS_BITS, NULL, getDataSourceUUID());
+            ReadBitstreamToTreeParams args(WANT_COLOR, NO_EXISTS_BITS /*WANT_EXISTS_BITS*/, NULL, getDataSourceUUID());
             lockTree();
             _tree->readBitstreamToTree(voxelData, numBytes - numBytesPacketHeader, args);
             unlockTree();
@@ -605,7 +605,7 @@ int VoxelSystem::parseData(unsigned char* sourceBuffer, int numBytes) {
             PerformanceWarning warn(Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings),
                                     "readBitstreamToTree()");
             // ask the VoxelTree to read the MONOCHROME bitstream into the tree
-            ReadBitstreamToTreeParams args(NO_COLOR, WANT_EXISTS_BITS, NULL, getDataSourceUUID());
+            ReadBitstreamToTreeParams args(NO_COLOR, NO_EXISTS_BITS /*WANT_EXISTS_BITS*/, NULL, getDataSourceUUID());
             lockTree();
             _tree->readBitstreamToTree(voxelData, numBytes - numBytesPacketHeader, args);
             unlockTree();

libraries/voxel-server-library/src/VoxelNodeData.cpp

@@ -13,10 +13,12 @@
 #include <cstdio>
 #include "VoxelSendThread.h"
 
+const int UNCOMPRESSED_SIZE_MULTIPLE = 20;
+
 VoxelNodeData::VoxelNodeData(Node* owningNode) :
     VoxelQuery(owningNode),
     _viewSent(false),
-    _voxelPacketAvailableBytes(MAX_VOXEL_PACKET_SIZE),
+    _voxelPacketAvailableBytes(MAX_VOXEL_PACKET_SIZE * UNCOMPRESSED_SIZE_MULTIPLE),
     _maxSearchLevel(1),
     _maxLevelReachedInLastSearch(1),
     _lastTimeBagEmpty(0),
@@ -29,9 +31,9 @@ VoxelNodeData::VoxelNodeData(Node* owningNode) :
     _lodChanged(false),
     _lodInitialized(false)
 {
-    _voxelPacket = new unsigned char[MAX_VOXEL_PACKET_SIZE];
+    _voxelPacket = new unsigned char[MAX_VOXEL_PACKET_SIZE * UNCOMPRESSED_SIZE_MULTIPLE];
     _voxelPacketAt = _voxelPacket;
-    _lastVoxelPacket = new unsigned char[MAX_VOXEL_PACKET_SIZE];
+    _lastVoxelPacket = new unsigned char[MAX_VOXEL_PACKET_SIZE * UNCOMPRESSED_SIZE_MULTIPLE];
     _lastVoxelPacketLength = 0;
     _duplicatePacketCount = 0;
     resetVoxelPacket();
@@ -44,9 +46,9 @@ void VoxelNodeData::initializeVoxelSendThread(VoxelServer* voxelServer) {
     _voxelSendThread->initialize(true);
 }
 
-bool VoxelNodeData::packetIsDuplicate() const {
+bool VoxelNodeData::packetIsDuplicate() {
     if (_lastVoxelPacketLength == getPacketLength()) {
-        return memcmp(_lastVoxelPacket, _voxelPacket, getPacketLength()) == 0;
+        return memcmp(_lastVoxelPacket, getPacket(), getPacketLength()) == 0;
     }
     return false;
 }
@@ -91,7 +93,7 @@ void VoxelNodeData::resetVoxelPacket() {
     // scene information, (e.g. the root node packet of a static scene), we can use this as a strategy for reducing
     // packet send rate.
     _lastVoxelPacketLength = getPacketLength();
-    memcpy(_lastVoxelPacket, _voxelPacket, _lastVoxelPacketLength);
+    memcpy(_lastVoxelPacket, getPacket(), _lastVoxelPacketLength);
 
     // If we're moving, and the client asked for low res, then we force monochrome, otherwise, use 
     // the clients requested color state.    
@@ -100,15 +102,63 @@ void VoxelNodeData::resetVoxelPacket() {
 
     int numBytesPacketHeader = populateTypeAndVersion(_voxelPacket, voxelPacketType);
     _voxelPacketAt = _voxelPacket + numBytesPacketHeader;
-    _voxelPacketAvailableBytes = MAX_VOXEL_PACKET_SIZE - numBytesPacketHeader;
+    _voxelPacketAvailableBytes = (MAX_VOXEL_PACKET_SIZE * UNCOMPRESSED_SIZE_MULTIPLE) - numBytesPacketHeader;
+    compressPacket();
     _voxelPacketWaiting = false;
 }
 
+bool VoxelNodeData::willFit(unsigned char* buffer, int bytes) {
+    int uncompressedLength = getPacketLengthUncompressed();
+    const int MAX_COMPRESSION = 9;
+    // we only want to compress the data payload, not the message header
+    int numBytesPacketHeader = numBytesForPacketHeader(_voxelPacket);
+    
+    // start with the current uncompressed data
+    QByteArray uncompressedTestData((const char*)_voxelPacket + numBytesPacketHeader, 
+                                    uncompressedLength - numBytesPacketHeader);
+
+    // append this next buffer...
+    uncompressedTestData.append((const char*)buffer, bytes);
+
+    // test compress it
+    QByteArray compressedData = qCompress(uncompressedTestData, MAX_COMPRESSION);
+    
+    bool wouldFit = (compressedData.size() + numBytesPacketHeader) <= MAX_VOXEL_PACKET_SIZE;
+    
+    /*
+    if (!wouldFit) {
+        printf("would not fit... previous size: %d, buffer size: %d, would be:%d MAX_VOXEL_PACKET_SIZE: %d\n",
+            getPacketLength(), bytes, (compressedData.size() + numBytesPacketHeader), MAX_VOXEL_PACKET_SIZE);
+    } else {
+        printf("would fit... previous size: %d, buffer size: %d, would be:%d MAX_VOXEL_PACKET_SIZE: %d\n",
+            getPacketLength(), bytes, (compressedData.size() + numBytesPacketHeader), MAX_VOXEL_PACKET_SIZE);
+    }
+    */
+    return wouldFit;
+}
+
 void VoxelNodeData::writeToPacket(unsigned char* buffer, int bytes) {
     memcpy(_voxelPacketAt, buffer, bytes);
     _voxelPacketAvailableBytes -= bytes;
     _voxelPacketAt += bytes;
     _voxelPacketWaiting = true;
+    compressPacket();
+}
+
+int VoxelNodeData::getPacketLengthUncompressed() const { 
+    return (MAX_VOXEL_PACKET_SIZE * UNCOMPRESSED_SIZE_MULTIPLE) - _voxelPacketAvailableBytes;
+}
+
+void VoxelNodeData::compressPacket() { 
+    int uncompressedLength = getPacketLengthUncompressed();
+    const int MAX_COMPRESSION = 9;
+    // we only want to compress the data payload, not the message header
+    int numBytesPacketHeader = numBytesForPacketHeader(_voxelPacket);
+    QByteArray compressedData = qCompress(_voxelPacket+numBytesPacketHeader, 
+                                    uncompressedLength-numBytesPacketHeader, MAX_COMPRESSION);
+    _compressedPacket.clear();
+    _compressedPacket.append((const char*)_voxelPacket, numBytesPacketHeader);
+    _compressedPacket.append(compressedData);
 }
 
 VoxelNodeData::~VoxelNodeData() {

libraries/voxel-server-library/src/VoxelNodeData.h

@@ -29,15 +29,18 @@ public:
     void resetVoxelPacket();  // resets voxel packet to after "V" header
 
     void writeToPacket(unsigned char* buffer, int bytes); // writes to end of packet
+    bool willFit(unsigned char* buffer, int bytes); // tests to see if the bytes will fit
+
+    const unsigned char* getPacket() const { return (const unsigned char*)_compressedPacket.constData(); }
+    int getPacketLength() const { return _compressedPacket.size(); }
+    int getPacketLengthUncompressed() const;
 
-    const unsigned char* getPacket() const { return _voxelPacket; }
-    int getPacketLength() const { return (MAX_VOXEL_PACKET_SIZE - _voxelPacketAvailableBytes); }
     bool isPacketWaiting() const { return _voxelPacketWaiting; }
 
-    bool packetIsDuplicate() const;
+    bool packetIsDuplicate();
     bool shouldSuppressDuplicatePacket();
 
-    int getAvailable() const { return _voxelPacketAvailableBytes; }
+    int getAvailable() const { return MAX_VOXEL_PACKET_SIZE - getPacketLength(); }
     int getMaxSearchLevel() const { return _maxSearchLevel; };
     void resetMaxSearchLevel() { _maxSearchLevel = 1; };
     void incrementMaxSearchLevel() { _maxSearchLevel++; };
@@ -109,6 +112,9 @@ private:
     float _lastClientVoxelSizeScale;
     bool _lodChanged;
     bool _lodInitialized;
+    
+    QByteArray _compressedPacket;
+    void compressPacket();
 };
 
 #endif /* defined(__hifi__VoxelNodeData__) */

libraries/voxel-server-library/src/VoxelSendThread.cpp

@@ -78,6 +78,11 @@ bool VoxelSendThread::process() {
 }
 
 
+int totalUncompressed = 0;
+int totalCompressed = 0;
+int totalWastedBytes = 0;
+int totalPackets = 0;
+
 int VoxelSendThread::handlePacketSend(Node* node, VoxelNodeData* nodeData, int& trueBytesSent, int& truePacketsSent) {
 
     int packetsSent = 0;
@@ -102,6 +107,17 @@ int VoxelSendThread::handlePacketSend(Node* node, VoxelNodeData* nodeData, int&
             memcpy(statsMessage + statsMessageLength, nodeData->getPacket(), nodeData->getPacketLength());
             statsMessageLength += nodeData->getPacketLength();
 
+            int thisWastedBytes = MAX_PACKET_SIZE - nodeData->getPacketLength();
+            ::totalWastedBytes += thisWastedBytes;
+            ::totalUncompressed += nodeData->getPacketLengthUncompressed();
+            ::totalCompressed += nodeData->getPacketLength();
+            ::totalPackets++;
+            qDebug("Adding stats to packet [%d]: uncompress size:%d [%d], compressed size:%d [%d] thisWastedBytes:%d [%d]\n",
+                totalPackets,
+                nodeData->getPacketLengthUncompressed(), ::totalUncompressed,
+                nodeData->getPacketLength(), ::totalCompressed,
+                thisWastedBytes, ::totalWastedBytes);
+
             // actually send it
             NodeList::getInstance()->getNodeSocket()->send(node->getActiveSocket(), statsMessage, statsMessageLength);
         } else {
@@ -113,12 +129,34 @@ int VoxelSendThread::handlePacketSend(Node* node, VoxelNodeData* nodeData, int&
 
             NodeList::getInstance()->getNodeSocket()->send(node->getActiveSocket(),
                                             nodeData->getPacket(), nodeData->getPacketLength());
+
+            int thisWastedBytes = MAX_PACKET_SIZE - nodeData->getPacketLength();
+            ::totalWastedBytes += thisWastedBytes;
+            ::totalUncompressed += nodeData->getPacketLengthUncompressed();
+            ::totalCompressed += nodeData->getPacketLength();
+            ::totalPackets++;
+            qDebug("Sending packet [%d]: uncompress size:%d [%d], compressed size:%d [%d] thisWastedBytes:%d [%d]\n",
+                totalPackets,
+                nodeData->getPacketLengthUncompressed(), ::totalUncompressed,
+                nodeData->getPacketLength(), ::totalCompressed,
+                thisWastedBytes, ::totalWastedBytes);
         }
         nodeData->stats.markAsSent();
     } else {
         // just send the voxel packet
         NodeList::getInstance()->getNodeSocket()->send(node->getActiveSocket(),
                                                        nodeData->getPacket(), nodeData->getPacketLength());
+
+            int thisWastedBytes = MAX_PACKET_SIZE - nodeData->getPacketLength();
+            ::totalWastedBytes += thisWastedBytes;
+            ::totalUncompressed += nodeData->getPacketLengthUncompressed();
+            ::totalCompressed += nodeData->getPacketLength();
+            ::totalPackets++;
+            qDebug("Sending packet [%d]: uncompress size:%d [%d], compressed size:%d [%d] thisWastedBytes:%d [%d]\n",
+                totalPackets,
+                nodeData->getPacketLengthUncompressed(), ::totalUncompressed,
+                nodeData->getPacketLength(), ::totalCompressed,
+                thisWastedBytes, ::totalWastedBytes);
     }
     // remember to track our stats
     nodeData->stats.packetSent(nodeData->getPacketLength());
@@ -297,7 +335,7 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod
                                  nodeData->getViewFrustumJustStoppedChanging()) || nodeData->hasLodChanged();
                 
                 EncodeBitstreamParams params(INT_MAX, &nodeData->getCurrentViewFrustum(), wantColor, 
-                                             WANT_EXISTS_BITS, DONT_CHOP, wantDelta, lastViewFrustum,
+                                             NO_EXISTS_BITS /*WANT_EXISTS_BITS*/, DONT_CHOP, wantDelta, lastViewFrustum,
                                              wantOcclusionCulling, coverageMap, boundaryLevelAdjust, voxelSizeScale,
                                              nodeData->getLastTimeBagEmpty(),
                                              isFullScene, &nodeData->stats, _myServer->getJurisdiction());
@@ -310,7 +348,8 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod
                 nodeData->stats.encodeStopped();
                 _myServer->getServerTree().unlock();
 
-                if (nodeData->getAvailable() >= bytesWritten) {
+                // NOTE: could be better, the bytesWritten might be compressable...
+                if (nodeData->willFit(_tempOutputBuffer, bytesWritten)) {
                     nodeData->writeToPacket(_tempOutputBuffer, bytesWritten);
                 } else {
                     packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);