diff --git a/interface/src/VoxelSystem.cpp b/interface/src/VoxelSystem.cpp index 62419565c6..21e7809014 100644 --- a/interface/src/VoxelSystem.cpp +++ b/interface/src/VoxelSystem.cpp @@ -597,7 +597,11 @@ int VoxelSystem::parseData(unsigned char* sourceBuffer, int numBytes) { // ask the VoxelTree to read the bitstream into the tree ReadBitstreamToTreeParams args(WANT_COLOR, WANT_EXISTS_BITS, NULL, getDataSourceUUID()); lockTree(); - _tree->readBitstreamToTree(voxelData, numBytes - numBytesPacketHeader, args); + VoxelPacket packet; + int compressedSize = numBytes - numBytesPacketHeader; + packet.loadCompressedContent(voxelData, compressedSize); +printf("got packet numBytes=%d compressed size %d uncompressed size %d\n",numBytes, compressedSize, packet.getUncompressedSize()); + _tree->readBitstreamToTree(packet.getUncompressedData(), packet.getUncompressedSize(), args); unlockTree(); } break; @@ -607,7 +611,11 @@ int VoxelSystem::parseData(unsigned char* sourceBuffer, int numBytes) { // ask the VoxelTree to read the MONOCHROME bitstream into the tree ReadBitstreamToTreeParams args(NO_COLOR, WANT_EXISTS_BITS, NULL, getDataSourceUUID()); lockTree(); - _tree->readBitstreamToTree(voxelData, numBytes - numBytesPacketHeader, args); + VoxelPacket packet; + int compressedSize = numBytes - numBytesPacketHeader; + packet.loadCompressedContent(voxelData, compressedSize); +printf("got packet numBytes=%d compressed size %d uncompressed size %d\n",numBytes, compressedSize, packet.getUncompressedSize()); + _tree->readBitstreamToTree(packet.getUncompressedData(), packet.getUncompressedSize(), args); unlockTree(); } break; diff --git a/libraries/shared/src/OctalCode.cpp b/libraries/shared/src/OctalCode.cpp index 027a6a4d68..5f79a7dfa5 100644 --- a/libraries/shared/src/OctalCode.cpp +++ b/libraries/shared/src/OctalCode.cpp @@ -32,12 +32,12 @@ int numberOfThreeBitSectionsInCode(const unsigned char* octalCode, int maxBytes) void printOctalCode(const unsigned char* octalCode) { if (!octalCode) { - qDebug("NULL\n"); + printf("NULL\n"); } else { for (int i = 0; i < bytesRequiredForCodeLength(numberOfThreeBitSectionsInCode(octalCode)); i++) { outputBits(octalCode[i],false); } - qDebug("\n"); + printf("\n"); } } diff --git a/libraries/shared/src/SharedUtil.cpp b/libraries/shared/src/SharedUtil.cpp index 8c56c250e0..b9d5f9da60 100644 --- a/libraries/shared/src/SharedUtil.cpp +++ b/libraries/shared/src/SharedUtil.cpp @@ -71,18 +71,18 @@ void outputBufferBits(const unsigned char* buffer, int length, bool withNewLine) void outputBits(unsigned char byte, bool withNewLine) { if (isalnum(byte)) { - qDebug("[ %d (%c): ", byte, byte); + printf("[ %d (%c): ", byte, byte); } else { - qDebug("[ %d (0x%x): ", byte, byte); + printf("[ %d (0x%x): ", byte, byte); } for (int i = 0; i < 8; i++) { - qDebug("%d", byte >> (7 - i) & 1); + printf("%d", byte >> (7 - i) & 1); } - qDebug(" ] "); + printf(" ] "); if (withNewLine) { - qDebug("\n"); + printf("\n"); } } diff --git a/libraries/voxel-server-library/src/VoxelSendThread.cpp b/libraries/voxel-server-library/src/VoxelSendThread.cpp index d7b007088b..de4d592a98 100644 --- a/libraries/voxel-server-library/src/VoxelSendThread.cpp +++ b/libraries/voxel-server-library/src/VoxelSendThread.cpp @@ -371,7 +371,14 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod _myServer->getServerTree().lockForRead(); nodeData->stats.encodeStarted(); + //printf("calling nodeBag.count()=%d encode()... subTree=%p \n", nodeData->nodeBag.count(), subTree); + //printOctalCode(subTree->getOctalCode()); + bytesWritten = _myServer->getServerTree().encodeTreeBitstream(subTree, &_tempPacket, nodeData->nodeBag, params); + + + //printf("called encode()... bytesWritten=%d compressedSize=%d uncompressedSize=%d\n",bytesWritten, _tempPacket.getFinalizedSize(), _tempPacket.getUncompressedSize()); + if (bytesWritten > 0) { _encodedSomething = true; } @@ -388,6 +395,7 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod // mean we should send the previous packet contents and reset it. bool sendNow = (bytesWritten == 0); if (_tempPacket.hasContent() && sendNow) { + printf("calling writeToPacket() compressedSize=%d uncompressedSize=%d\n",_tempPacket.getFinalizedSize(), _tempPacket.getUncompressedSize()); nodeData->writeToPacket(_tempPacket.getFinalizedData(), _tempPacket.getFinalizedSize()); packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent); _tempPacket.reset(); diff --git a/libraries/voxel-server-library/src/VoxelSendThread.h b/libraries/voxel-server-library/src/VoxelSendThread.h index 1c60be701d..607e1a9e0d 100644 --- a/libraries/voxel-server-library/src/VoxelSendThread.h +++ b/libraries/voxel-server-library/src/VoxelSendThread.h @@ -33,7 +33,7 @@ private: int handlePacketSend(Node* node, VoxelNodeData* nodeData, int& trueBytesSent, int& truePacketsSent); int deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nodeData, bool viewFrustumChanged); - unsigned char _tempOutputBuffer[MAX_VOXEL_PACKET_SIZE]; + unsigned char _tempOutputBuffer[MAX_VOXEL_PACKET_SIZE]; // used by environment sending code VoxelPacket _tempPacket; bool _encodedSomething; diff --git a/libraries/voxels/src/VoxelConstants.h b/libraries/voxels/src/VoxelConstants.h index 0151789c4e..f2fa4eca92 100644 --- a/libraries/voxels/src/VoxelConstants.h +++ b/libraries/voxels/src/VoxelConstants.h @@ -35,6 +35,9 @@ const float MAX_LOD_SIZE_MULTIPLIER = 2000.0f; const int NUMBER_OF_CHILDREN = 8; const int MAX_VOXEL_PACKET_SIZE = MAX_PACKET_SIZE - (sizeof(PACKET_TYPE) + sizeof(PACKET_VERSION)); +const int MAX_VOXEL_PACKET_COMPRESSION_RATIO = 1; +const int MAX_VOXEL_UNCOMRESSED_PACKET_SIZE = MAX_VOXEL_PACKET_SIZE * MAX_VOXEL_PACKET_COMPRESSION_RATIO; + const int MAX_TREE_SLICE_BYTES = 26; const int DEFAULT_MAX_VOXELS_PER_SYSTEM = 200000; const int VERTICES_PER_VOXEL = 24; // 6 sides * 4 corners per side diff --git a/libraries/voxels/src/VoxelPacket.cpp b/libraries/voxels/src/VoxelPacket.cpp index 657f2a99be..5031e2b89e 100644 --- a/libraries/voxels/src/VoxelPacket.cpp +++ b/libraries/voxels/src/VoxelPacket.cpp @@ -14,8 +14,9 @@ VoxelPacket::VoxelPacket() { void VoxelPacket::reset() { _bytesInUse = 0; - _bytesAvailable = MAX_VOXEL_PACKET_SIZE; + _bytesAvailable = MAX_VOXEL_UNCOMRESSED_PACKET_SIZE; _subTreeAt = 0; + _compressedBytes = 0; } VoxelPacket::~VoxelPacket() { @@ -25,10 +26,18 @@ bool VoxelPacket::append(const unsigned char* data, int length) { bool success = false; if (length <= _bytesAvailable) { - memcpy(&_buffer[_bytesInUse], data, length); + memcpy(&_uncompressed[_bytesInUse], data, length); _bytesInUse += length; _bytesAvailable -= length; - success = true; + + // Now, check for compression, if we fit, then proceed, otherwise, rollback. + if (checkCompress()) { + success = true; + } else { + // rollback is easy, we just reset _bytesInUse and _bytesAvailable + _bytesInUse -= length; + _bytesAvailable += length; + } } return success; } @@ -36,10 +45,18 @@ bool VoxelPacket::append(const unsigned char* data, int length) { bool VoxelPacket::append(unsigned char byte) { bool success = false; if (_bytesAvailable > 0) { - _buffer[_bytesInUse] = byte; + _uncompressed[_bytesInUse] = byte; _bytesInUse++; _bytesAvailable--; - success = true; + + // Now, check for compression, if we fit, then proceed, otherwise, rollback. + if (checkCompress()) { + success = true; + } else { + // rollback is easy, we just reset _bytesInUse and _bytesAvailable + _bytesInUse--; + _bytesAvailable++; + } } return success; } @@ -47,8 +64,15 @@ bool VoxelPacket::append(unsigned char byte) { bool VoxelPacket::updatePriorBitMask(int offset, unsigned char bitmask) { bool success = false; if (offset >= 0 && offset < _bytesInUse) { - _buffer[offset] = bitmask; - success = true; + unsigned char oldValue = _uncompressed[offset]; + _uncompressed[offset] = bitmask; + // Now, check for compression, if we fit, then proceed, otherwise, rollback. + if (checkCompress()) { + success = true; + } else { + // rollback is easy, the length didn't change, but we need to restore the previous value + _uncompressed[offset] = oldValue; + } } return success; } @@ -56,8 +80,17 @@ bool VoxelPacket::updatePriorBitMask(int offset, unsigned char bitmask) { bool VoxelPacket::updatePriorBytes(int offset, const unsigned char* replacementBytes, int length) { bool success = false; if (length >= 0 && offset >= 0 && ((offset + length) <= _bytesInUse)) { - memcpy(&_buffer[offset], replacementBytes, length); - success = true; + unsigned char oldValues[length]; + memcpy(&oldValues[0], &_uncompressed[offset], length); // save the old values for restore + memcpy(&_uncompressed[offset], replacementBytes, length); // copy new content + + // Now, check for compression, if we fit, then proceed, otherwise, rollback. + if (checkCompress()) { + success = true; + } else { + // rollback is easy, the length didn't change, but we need to restore the previous values + memcpy(&_uncompressed[offset], &oldValues[0], length); // restore the old values + } } return success; } @@ -67,11 +100,11 @@ bool VoxelPacket::startSubTree(const unsigned char* octcode) { int possibleStartAt = _bytesInUse; if (octcode) { int length = bytesRequiredForCodeLength(numberOfThreeBitSectionsInCode(octcode)); - success = append(octcode, length); + success = append(octcode, length); // handles checking compression } else { // NULL case, means root node, which is 0 unsigned char byte = 0; - success = append(byte); + success = append(byte); // handles checking compression } if (success) { _subTreeAt = possibleStartAt; @@ -88,6 +121,11 @@ void VoxelPacket::discardSubTree() { _bytesInUse -= bytesInSubTree; _bytesAvailable += bytesInSubTree; _subTreeAt = _bytesInUse; // should be the same actually... + + // we can be confident that it will compress, because we can't get here without previously being able to compress + // the content up to this point in the uncompressed buffer. But we still call this because it cleans up the compressed + // buffer with the correct content + checkCompress(); } int VoxelPacket::startLevel() { @@ -99,6 +137,11 @@ void VoxelPacket::discardLevel(int key) { int bytesInLevel = _bytesInUse - key; _bytesInUse -= bytesInLevel; _bytesAvailable += bytesInLevel; + + // we can be confident that it will compress, because we can't get here without previously being able to compress + // the content up to this point in the uncompressed buffer. But we still call this because it cleans up the compressed + // buffer with the correct content + checkCompress(); } void VoxelPacket::endLevel() { @@ -106,7 +149,7 @@ void VoxelPacket::endLevel() { } bool VoxelPacket::appendBitMask(unsigned char bitmask) { - return append(bitmask); + return append(bitmask); // handles checking compression } bool VoxelPacket::appendColor(const nodeColor& color) { @@ -114,14 +157,94 @@ bool VoxelPacket::appendColor(const nodeColor& color) { bool success = false; const int BYTES_PER_COLOR = 3; if (_bytesAvailable > BYTES_PER_COLOR) { - append(color[RED_INDEX]); - append(color[GREEN_INDEX]); - append(color[BLUE_INDEX]); + // handles checking compression... + if (append(color[RED_INDEX])) { + if (append(color[GREEN_INDEX])) { + if (append(color[BLUE_INDEX])) { + success = true; + } + } + } + } + return success; +} + +bool VoxelPacket::checkCompress() { + bool success = false; + const int MAX_COMPRESSION = 9; + + // we only want to compress the data payload, not the message header + QByteArray compressedData = qCompress(_uncompressed,_bytesInUse, MAX_COMPRESSION); + if (compressedData.size() < MAX_VOXEL_PACKET_SIZE) { + //memcpy(&_compressed[0], compressedData.constData(), compressedData.size()); + + _compressedBytes = compressedData.size(); + for (int i = 0; i < _compressedBytes; i++) { + _compressed[i] = compressedData[i]; + } + +//printf("compressed %d bytes from %d original bytes\n", _compressedBytes, _bytesInUse); success = true; } return success; } + +void VoxelPacket::loadCompressedContent(const unsigned char* data, int length) { + reset(); // by definition we reset upon loading compressed content + + if (length > 0) { + QByteArray compressedData; + + for (int i = 0; i < length; i++) { + compressedData[i] = data[i]; + _compressed[i] = compressedData[i]; + } + _compressedBytes = length; + + QByteArray uncompressedData = qUncompress(compressedData); + + if (uncompressedData.size() <= _bytesAvailable) { + _bytesInUse = uncompressedData.size(); + _bytesAvailable -= uncompressedData.size(); + + for (int i = 0; i < _bytesInUse; i++) { + _uncompressed[i] = uncompressedData[i]; + } + } + } else { + printf("VoxelPacket::loadCompressedContent()... length = 0, nothing to do...\n"); + } +} + +void VoxelPacket::debugContent() { + + printf("VoxelPacket::debugContent()... COMPRESSED DATA.... size=%d\n",_compressedBytes); + int perline=0; + for (int i = 0; i < _compressedBytes; i++) { + printf("%.2x ",_compressed[i]); + perline++; + if (perline >= 30) { + printf("\n"); + perline=0; + } + } + printf("\n"); + + printf("VoxelPacket::debugContent()... UNCOMPRESSED DATA.... size=%d\n",_bytesInUse); + perline=0; + for (int i = 0; i < _bytesInUse; i++) { + printf("%.2x ",_uncompressed[i]); + perline++; + if (perline >= 30) { + printf("\n"); + perline=0; + } + } + printf("\n"); +} + + /*** void VoxelPacket::compressPacket() { int uncompressedLength = getPacketLengthUncompressed(); diff --git a/libraries/voxels/src/VoxelPacket.h b/libraries/voxels/src/VoxelPacket.h index 66591bd568..c557726324 100644 --- a/libraries/voxels/src/VoxelPacket.h +++ b/libraries/voxels/src/VoxelPacket.h @@ -20,6 +20,8 @@ #include "VoxelConstants.h" #include "VoxelNode.h" + + class VoxelPacket { public: VoxelPacket(); @@ -66,17 +68,22 @@ public: int getUncompressedByteOffset(int offsetFromEnd = 0) const { return _bytesInUse - offsetFromEnd; } /// get access to the finalized data (it may be compressed or rewritten into optimal form) - const unsigned char* getFinalizedData() { return &_buffer[0]; } + const unsigned char* getFinalizedData() { return &_compressed[0]; } /// get size of the finalized data (it may be compressed or rewritten into optimal form) - int getFinalizedSize() const { return _bytesInUse; } + int getFinalizedSize() const { return _compressedBytes; } /// get pointer to the start of uncompressed stream buffer - const unsigned char* getUncompressedData() { return &_buffer[0]; } + const unsigned char* getUncompressedData() { return &_uncompressed[0]; } /// the size of the packet in uncompressed form int getUncompressedSize() { return _bytesInUse; } /// has some content been written to the packet bool hasContent() const { return (_bytesInUse > 0); } + + /// load compressed content to allow access to decoded content for parsing + void loadCompressedContent(const unsigned char* data, int length); + + void debugContent(); private: /// appends raw bytes, might fail if byte would cause packet to be too large @@ -85,10 +92,15 @@ private: /// append a single byte, might fail if byte would cause packet to be too large bool append(unsigned char byte); - unsigned char _buffer[MAX_VOXEL_PACKET_SIZE]; + unsigned char _uncompressed[MAX_VOXEL_UNCOMRESSED_PACKET_SIZE]; int _bytesInUse; int _bytesAvailable; int _subTreeAt; + + bool checkCompress(); + + unsigned char _compressed[MAX_VOXEL_PACKET_SIZE]; + int _compressedBytes; }; #endif /* defined(__hifi__VoxelPacket__) */ \ No newline at end of file diff --git a/libraries/voxels/src/VoxelTree.cpp b/libraries/voxels/src/VoxelTree.cpp index 348e6a1899..e301cbc4cd 100644 --- a/libraries/voxels/src/VoxelTree.cpp +++ b/libraries/voxels/src/VoxelTree.cpp @@ -1576,7 +1576,7 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, // reshuffle here... if (continueThisLevel && params.wantOcclusionCulling) { - unsigned char tempReshuffleBuffer[MAX_VOXEL_PACKET_SIZE]; + unsigned char tempReshuffleBuffer[MAX_VOXEL_UNCOMRESSED_PACKET_SIZE]; unsigned char* tempBufferTo = &tempReshuffleBuffer[0]; // this is our temporary destination