From e2f5069e6a7de944453ca9f0b96eb58bb5fb2abd Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Sun, 24 Nov 2013 14:41:50 -0800 Subject: [PATCH] more VoxelPacket compression tuning --- interface/src/VoxelSystem.cpp | 4 +-- .../src/VoxelSendThread.cpp | 25 +++++++++------- libraries/voxels/src/VoxelConstants.h | 8 +++-- libraries/voxels/src/VoxelPacket.cpp | 30 ++++++++++++++----- libraries/voxels/src/VoxelPacket.h | 1 + 5 files changed, 46 insertions(+), 22 deletions(-) diff --git a/interface/src/VoxelSystem.cpp b/interface/src/VoxelSystem.cpp index 21e7809014..622247df8a 100644 --- a/interface/src/VoxelSystem.cpp +++ b/interface/src/VoxelSystem.cpp @@ -597,7 +597,7 @@ int VoxelSystem::parseData(unsigned char* sourceBuffer, int numBytes) { // ask the VoxelTree to read the bitstream into the tree ReadBitstreamToTreeParams args(WANT_COLOR, WANT_EXISTS_BITS, NULL, getDataSourceUUID()); lockTree(); - VoxelPacket packet; + VoxelPacket packet(VOXEL_PACKET_COMPRESSION_DEFAULT); int compressedSize = numBytes - numBytesPacketHeader; packet.loadCompressedContent(voxelData, compressedSize); printf("got packet numBytes=%d compressed size %d uncompressed size %d\n",numBytes, compressedSize, packet.getUncompressedSize()); @@ -611,7 +611,7 @@ printf("got packet numBytes=%d compressed size %d uncompressed size %d\n",numByt // ask the VoxelTree to read the MONOCHROME bitstream into the tree ReadBitstreamToTreeParams args(NO_COLOR, WANT_EXISTS_BITS, NULL, getDataSourceUUID()); lockTree(); - VoxelPacket packet; + VoxelPacket packet(VOXEL_PACKET_COMPRESSION_DEFAULT); int compressedSize = numBytes - numBytesPacketHeader; packet.loadCompressedContent(voxelData, compressedSize); printf("got packet numBytes=%d compressed size %d uncompressed size %d\n",numBytes, compressedSize, packet.getUncompressedSize()); diff --git a/libraries/voxel-server-library/src/VoxelSendThread.cpp b/libraries/voxel-server-library/src/VoxelSendThread.cpp index 5d96cedb5d..4137ad67cc 100644 --- a/libraries/voxel-server-library/src/VoxelSendThread.cpp +++ b/libraries/voxel-server-library/src/VoxelSendThread.cpp @@ -21,7 +21,9 @@ extern EnvironmentData environmentData[3]; VoxelSendThread::VoxelSendThread(const QUuid& nodeUUID, VoxelServer* myServer) : _nodeUUID(nodeUUID), _myServer(myServer), - _encodedSomething(false) { + _tempPacket(VOXEL_PACKET_COMPRESSION_DEFAULT), + _encodedSomething(false) +{ } bool VoxelSendThread::process() { @@ -422,22 +424,25 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod uint64_t end = usecTimestampNow(); int elapsedmsec = (end - start)/1000; + + uint64_t endCompressCalls = VoxelPacket::_checkCompressCalls; + int elapsedCompressCalls = endCompressCalls - startCompressCalls; + + uint64_t endCompressTimeMsecs = VoxelPacket::_checkCompressTime / 1000; + int elapsedCompressTimeMsecs = endCompressTimeMsecs - startCompressTimeMsecs; + + if (elapsedmsec > 100) { if (elapsedmsec > 1000) { int elapsedsec = (end - start)/1000000; - printf("WARNING! packetLoop() took %d seconds to generate %d bytes in %d packets %d nodes still to send\n", - elapsedsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count()); + printf("WARNING! packetLoop() took %d seconds [%d milliseconds %d calls in compress] to generate %d bytes in %d packets %d nodes still to send\n", + elapsedsec, elapsedCompressTimeMsecs, elapsedCompressCalls, trueBytesSent, truePacketsSent, nodeData->nodeBag.count()); } else { - printf("WARNING! packetLoop() took %d milliseconds to generate %d bytes in %d packets, %d nodes still to send\n", - elapsedmsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count()); + printf("WARNING! packetLoop() took %d milliseconds [%d milliseconds %d calls in compress] to generate %d bytes in %d packets, %d nodes still to send\n", + elapsedmsec, elapsedCompressTimeMsecs, elapsedCompressCalls, trueBytesSent, truePacketsSent, nodeData->nodeBag.count()); } } else if (truePacketsSent > 0 /*_myServer->wantsDebugVoxelSending() && _myServer->wantsVerboseDebug()*/) { - uint64_t endCompressCalls = VoxelPacket::_checkCompressCalls; - int elapsedCompressCalls = endCompressCalls - startCompressCalls; - - uint64_t endCompressTimeMsecs = VoxelPacket::_checkCompressTime / 1000; - int elapsedCompressTimeMsecs = endCompressTimeMsecs - startCompressTimeMsecs; printf("packetLoop() took %d milliseconds [%d milliseconds %d calls in compress] to generate %d bytes in %d packets, %d nodes still to send\n", elapsedmsec, elapsedCompressTimeMsecs, elapsedCompressCalls, trueBytesSent, truePacketsSent, nodeData->nodeBag.count()); } diff --git a/libraries/voxels/src/VoxelConstants.h b/libraries/voxels/src/VoxelConstants.h index e53107f80e..7fcbd6e308 100644 --- a/libraries/voxels/src/VoxelConstants.h +++ b/libraries/voxels/src/VoxelConstants.h @@ -35,9 +35,11 @@ const float MAX_LOD_SIZE_MULTIPLIER = 2000.0f; const int NUMBER_OF_CHILDREN = 8; const int MAX_VOXEL_PACKET_SIZE = MAX_PACKET_SIZE - (sizeof(PACKET_TYPE) + sizeof(PACKET_VERSION)); -const int MAX_VOXEL_PACKET_COMPRESSION_RATIO = 3; -const int MAX_VOXEL_UNCOMRESSED_PACKET_SIZE = 3000; // MAX_VOXEL_PACKET_SIZE * MAX_VOXEL_PACKET_COMPRESSION_RATIO; -const int COMPRESSION_TEST_THRESHOLD = 2000; +const int MAX_VOXEL_UNCOMRESSED_PACKET_SIZE = 4500; +const int VOXEL_PACKET_ALWAYS_TEST_COMPRESSED_THRESHOLD = 1400; +const int VOXEL_PACKET_TEST_UNCOMPRESSED_THRESHOLD = 4000; +const int VOXEL_PACKET_TEST_UNCOMPRESSED_CHANGE_THRESHOLD = 20; +const int VOXEL_PACKET_COMPRESSION_DEFAULT = true; const int MAX_TREE_SLICE_BYTES = 26; const int DEFAULT_MAX_VOXELS_PER_SYSTEM = 200000; diff --git a/libraries/voxels/src/VoxelPacket.cpp b/libraries/voxels/src/VoxelPacket.cpp index 9911055d3c..9d0d062c1f 100644 --- a/libraries/voxels/src/VoxelPacket.cpp +++ b/libraries/voxels/src/VoxelPacket.cpp @@ -27,6 +27,7 @@ void VoxelPacket::reset() { } _subTreeAt = 0; _compressedBytes = 0; + _bytesInUseLastCheck = 0; _dirty = false; } @@ -101,7 +102,9 @@ const unsigned char* VoxelPacket::getFinalizedData() { } if (_dirty) { -if (_debug) printf("getFinalizedData() _compressedBytes=%d _bytesInUse=%d\n",_compressedBytes, _bytesInUse); + if (_debug) { + printf("getFinalizedData() _compressedBytes=%d _bytesInUse=%d\n",_compressedBytes, _bytesInUse); + } checkCompress(); } @@ -114,7 +117,9 @@ int VoxelPacket::getFinalizedSize() { } if (_dirty) { -if (_debug) printf("getFinalizedSize() _compressedBytes=%d _bytesInUse=%d\n",_compressedBytes, _bytesInUse); + if (_debug) { + printf("getFinalizedSize() _compressedBytes=%d _bytesInUse=%d\n",_compressedBytes, _bytesInUse); + } checkCompress(); } @@ -156,11 +161,21 @@ if (_debug) printf("discardLevel() AFTER _dirty=%s bytesInLevel=%d _compressedBy bool VoxelPacket::endLevel(int key) { bool success = true; - // if we've never compressed, then compress to see our size - // or, if we're within 50% of our MAX_VOXEL_PACKET_SIZE then check... - // otherwise assume it's safe... - if (_dirty && (_compressedBytes == 0 || _bytesInUse > COMPRESSION_TEST_THRESHOLD)) { -if (_debug) printf("endLevel() _dirty=%s _compressedBytes=%d _bytesInUse=%d\n",debug::valueOf(_dirty), _compressedBytes, _bytesInUse); + // if we are dirty (something has changed) then try a compression test in the following cases... + // 1) If we've previously compressed and our _compressedBytes are "close enough to our limit" that we want to keep + // testing to make sure we don't overflow... VOXEL_PACKET_ALWAYS_TEST_COMPRESSION_THRESHOLD + // 2) If we've passed the uncompressed size where we believe we might pass the compressed threshold, and we've added + // a sufficient number of uncompressed bytes + if (_dirty && ( + (_compressedBytes > VOXEL_PACKET_ALWAYS_TEST_COMPRESSED_THRESHOLD) || + ( (_bytesInUse > VOXEL_PACKET_TEST_UNCOMPRESSED_THRESHOLD) && + (_bytesInUse - _bytesInUseLastCheck > VOXEL_PACKET_TEST_UNCOMPRESSED_CHANGE_THRESHOLD) + ) + )) { + if (_debug) { + printf("endLevel() _dirty=%s _compressedBytes=%d _bytesInUse=%d\n", + debug::valueOf(_dirty), _compressedBytes, _bytesInUse); + } success = checkCompress(); } if (!success) { @@ -201,6 +216,7 @@ bool VoxelPacket::checkCompress() { return true; } + _bytesInUseLastCheck = _bytesInUse; bool success = false; const int MAX_COMPRESSION = 2; diff --git a/libraries/voxels/src/VoxelPacket.h b/libraries/voxels/src/VoxelPacket.h index 4930f7f51f..2122467af1 100644 --- a/libraries/voxels/src/VoxelPacket.h +++ b/libraries/voxels/src/VoxelPacket.h @@ -109,6 +109,7 @@ private: unsigned char _compressed[MAX_VOXEL_UNCOMRESSED_PACKET_SIZE]; int _compressedBytes; + int _bytesInUseLastCheck; bool _dirty; static bool _debug;