From c28027b7ca5153c491fe48cc993288f3a06e7ba9 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Sat, 23 Nov 2013 22:30:15 -0800 Subject: [PATCH] better compressed packets with fewer calls to checkCompress --- .../src/VoxelSendThread.cpp | 14 ++++- libraries/voxels/src/VoxelConstants.h | 2 +- libraries/voxels/src/VoxelPacket.cpp | 60 +++++++++---------- libraries/voxels/src/VoxelPacket.h | 12 +++- libraries/voxels/src/VoxelTree.cpp | 8 ++- 5 files changed, 58 insertions(+), 38 deletions(-) diff --git a/libraries/voxel-server-library/src/VoxelSendThread.cpp b/libraries/voxel-server-library/src/VoxelSendThread.cpp index de4d592a98..391722ce7f 100644 --- a/libraries/voxel-server-library/src/VoxelSendThread.cpp +++ b/libraries/voxel-server-library/src/VoxelSendThread.cpp @@ -311,6 +311,8 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod if (!nodeData->nodeBag.isEmpty()) { int bytesWritten = 0; uint64_t start = usecTimestampNow(); + uint64_t startCompressTimeMsecs = VoxelPacket::_checkCompressTime / 1000; + uint64_t startCompressCalls = VoxelPacket::_checkCompressCalls; bool shouldSendEnvironments = _myServer->wantSendEnvironments() && shouldDo(ENVIRONMENT_SEND_INTERVAL_USECS, VOXEL_SEND_INTERVAL_USECS); @@ -429,9 +431,15 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod printf("WARNING! packetLoop() took %d milliseconds to generate %d bytes in %d packets, %d nodes still to send\n", elapsedmsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count()); } - } else if (_myServer->wantsDebugVoxelSending() && _myServer->wantsVerboseDebug()) { - printf("packetLoop() took %d milliseconds to generate %d bytes in %d packets, %d nodes still to send\n", - elapsedmsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count()); + } else if (truePacketsSent > 0 /*_myServer->wantsDebugVoxelSending() && _myServer->wantsVerboseDebug()*/) { + + uint64_t endCompressCalls = VoxelPacket::_checkCompressCalls; + int elapsedCompressCalls = endCompressCalls - startCompressCalls; + + uint64_t endCompressTimeMsecs = VoxelPacket::_checkCompressTime / 1000; + int elapsedCompressTimeMsecs = endCompressTimeMsecs - startCompressTimeMsecs; + printf("packetLoop() took %d milliseconds [%d milliseconds %d calls in compress] to generate %d bytes in %d packets, %d nodes still to send\n", + elapsedmsec, elapsedCompressTimeMsecs, elapsedCompressCalls, trueBytesSent, truePacketsSent, nodeData->nodeBag.count()); } // if after sending packets we've emptied our bag, then we want to remember that we've sent all diff --git a/libraries/voxels/src/VoxelConstants.h b/libraries/voxels/src/VoxelConstants.h index f2fa4eca92..c30d39dbf1 100644 --- a/libraries/voxels/src/VoxelConstants.h +++ b/libraries/voxels/src/VoxelConstants.h @@ -35,7 +35,7 @@ const float MAX_LOD_SIZE_MULTIPLIER = 2000.0f; const int NUMBER_OF_CHILDREN = 8; const int MAX_VOXEL_PACKET_SIZE = MAX_PACKET_SIZE - (sizeof(PACKET_TYPE) + sizeof(PACKET_VERSION)); -const int MAX_VOXEL_PACKET_COMPRESSION_RATIO = 1; +const int MAX_VOXEL_PACKET_COMPRESSION_RATIO = 3; const int MAX_VOXEL_UNCOMRESSED_PACKET_SIZE = MAX_VOXEL_PACKET_SIZE * MAX_VOXEL_PACKET_COMPRESSION_RATIO; const int MAX_TREE_SLICE_BYTES = 26; diff --git a/libraries/voxels/src/VoxelPacket.cpp b/libraries/voxels/src/VoxelPacket.cpp index 5031e2b89e..d1405cd1e6 100644 --- a/libraries/voxels/src/VoxelPacket.cpp +++ b/libraries/voxels/src/VoxelPacket.cpp @@ -6,6 +6,7 @@ // Copyright (c) 2013 HighFidelity, Inc. All rights reserved. // +#include #include "VoxelPacket.h" VoxelPacket::VoxelPacket() { @@ -30,6 +31,9 @@ bool VoxelPacket::append(const unsigned char* data, int length) { _bytesInUse += length; _bytesAvailable -= length; + success = true; + + /**** // Now, check for compression, if we fit, then proceed, otherwise, rollback. if (checkCompress()) { success = true; @@ -38,6 +42,7 @@ bool VoxelPacket::append(const unsigned char* data, int length) { _bytesInUse -= length; _bytesAvailable += length; } + ***/ } return success; } @@ -49,6 +54,9 @@ bool VoxelPacket::append(unsigned char byte) { _bytesInUse++; _bytesAvailable--; + success = true; + + /**** // Now, check for compression, if we fit, then proceed, otherwise, rollback. if (checkCompress()) { success = true; @@ -57,6 +65,7 @@ bool VoxelPacket::append(unsigned char byte) { _bytesInUse--; _bytesAvailable++; } + ****/ } return success; } @@ -66,6 +75,10 @@ bool VoxelPacket::updatePriorBitMask(int offset, unsigned char bitmask) { if (offset >= 0 && offset < _bytesInUse) { unsigned char oldValue = _uncompressed[offset]; _uncompressed[offset] = bitmask; + + success = true; + + /**** // Now, check for compression, if we fit, then proceed, otherwise, rollback. if (checkCompress()) { success = true; @@ -73,6 +86,7 @@ bool VoxelPacket::updatePriorBitMask(int offset, unsigned char bitmask) { // rollback is easy, the length didn't change, but we need to restore the previous value _uncompressed[offset] = oldValue; } + ****/ } return success; } @@ -84,6 +98,9 @@ bool VoxelPacket::updatePriorBytes(int offset, const unsigned char* replacementB memcpy(&oldValues[0], &_uncompressed[offset], length); // save the old values for restore memcpy(&_uncompressed[offset], replacementBytes, length); // copy new content + success = true; + + /**** // Now, check for compression, if we fit, then proceed, otherwise, rollback. if (checkCompress()) { success = true; @@ -91,6 +108,7 @@ bool VoxelPacket::updatePriorBytes(int offset, const unsigned char* replacementB // rollback is easy, the length didn't change, but we need to restore the previous values memcpy(&_uncompressed[offset], &oldValues[0], length); // restore the old values } + ****/ } return success; } @@ -144,8 +162,12 @@ void VoxelPacket::discardLevel(int key) { checkCompress(); } -void VoxelPacket::endLevel() { - // nothing to do +bool VoxelPacket::endLevel(int key) { + bool success = checkCompress(); + if (!success) { + discardLevel(key); + } + return success; } bool VoxelPacket::appendBitMask(unsigned char bitmask) { @@ -169,7 +191,13 @@ bool VoxelPacket::appendColor(const nodeColor& color) { return success; } +uint64_t VoxelPacket::_checkCompressTime = 0; +uint64_t VoxelPacket::_checkCompressCalls = 0; + bool VoxelPacket::checkCompress() { + PerformanceWarning warn(false,"VoxelPacket::checkCompress()",false,&_checkCompressTime,&_checkCompressCalls); + + bool success = false; const int MAX_COMPRESSION = 9; @@ -243,31 +271,3 @@ void VoxelPacket::debugContent() { } printf("\n"); } - - -/*** -void VoxelPacket::compressPacket() { - int uncompressedLength = getPacketLengthUncompressed(); - const int MAX_COMPRESSION = 9; - // we only want to compress the data payload, not the message header - int numBytesPacketHeader = numBytesForPacketHeader(_voxelPacket); - QByteArray compressedData = qCompress(_voxelPacket+numBytesPacketHeader, - uncompressedLength-numBytesPacketHeader, MAX_COMPRESSION); - _compressedPacket.clear(); - _compressedPacket.append((const char*)_voxelPacket, numBytesPacketHeader); - _compressedPacket.append(compressedData); -} - -void VoxelPacket::uncompressPacket() { - int numBytesPacketHeader = numBytesForPacketHeader(packetData); - QByteArray compressedData((const char*)packetData + numBytesPacketHeader, - messageLength - numBytesPacketHeader); - QByteArray uncompressedData = qUncompress(compressedData); - QByteArray uncompressedPacket((const char*)packetData, numBytesPacketHeader); - uncompressedPacket.append(uncompressedData); - //app->_voxels.parseData((unsigned char*)uncompressedPacket.data(), uncompressedPacket.size()); -} -***/ - - - diff --git a/libraries/voxels/src/VoxelPacket.h b/libraries/voxels/src/VoxelPacket.h index c557726324..037b49e9ba 100644 --- a/libraries/voxels/src/VoxelPacket.h +++ b/libraries/voxels/src/VoxelPacket.h @@ -46,8 +46,9 @@ public: /// discards all content back to a previous marker key void discardLevel(int key); - /// ends a level without discarding it - void endLevel(); + /// ends a level, and performs any expensive finalization. may fail if finalization creates a stream which is too large + /// if the finalization would fail, the packet will automatically discard the previous level. + bool endLevel(int key); /// appends a bitmask to the end of the stream, may fail if new data stream is too long to fit in packet bool appendBitMask(unsigned char bitmask); @@ -84,7 +85,10 @@ public: void loadCompressedContent(const unsigned char* data, int length); void debugContent(); - + + static uint64_t _checkCompressTime; + static uint64_t _checkCompressCalls; + private: /// appends raw bytes, might fail if byte would cause packet to be too large bool append(const unsigned char* data, int length); @@ -101,6 +105,8 @@ private: unsigned char _compressed[MAX_VOXEL_PACKET_SIZE]; int _compressedBytes; + + }; #endif /* defined(__hifi__VoxelPacket__) */ \ No newline at end of file diff --git a/libraries/voxels/src/VoxelTree.cpp b/libraries/voxels/src/VoxelTree.cpp index e301cbc4cd..8ef2cd7d66 100644 --- a/libraries/voxels/src/VoxelTree.cpp +++ b/libraries/voxels/src/VoxelTree.cpp @@ -1600,8 +1600,14 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, // if we were unable to fit this level in our packet, then rewind and add it to the node bag for // sending later... - if (!continueThisLevel) { + + if (continueThisLevel) { + continueThisLevel = packet->endLevel(thisLevelKey); + } else { packet->discardLevel(thisLevelKey); + } + + if (!continueThisLevel) { bag.insert(node); // don't need to check node here, because we can't get here with no node