Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-07-10 16:18:37 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

moved compression into VoxelPacket class. works, but too slow for larger sizes

Browse source
This commit is contained in:
ZappoMan 2013-11-23 21:40:50 -08:00
parent cd1d3765a9
commit e65f74e06b
9 changed files with 184 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
interface/src/VoxelSystem.cpp
View file

@ -597,7 +597,11 @@ int VoxelSystem::parseData(unsigned char* sourceBuffer, int numBytes) {
// ask the VoxelTree to read the bitstream into the tree
ReadBitstreamToTreeParams args(WANT_COLOR, WANT_EXISTS_BITS, NULL, getDataSourceUUID());
lockTree();
_tree->readBitstreamToTree(voxelData, numBytes - numBytesPacketHeader, args);
VoxelPacket packet;
int compressedSize = numBytes - numBytesPacketHeader;
packet.loadCompressedContent(voxelData, compressedSize);
printf("got packet numBytes=%d compressed size %d uncompressed size %d\n",numBytes, compressedSize, packet.getUncompressedSize());
_tree->readBitstreamToTree(packet.getUncompressedData(), packet.getUncompressedSize(), args);
unlockTree();
}
break;
@ -607,7 +611,11 @@ int VoxelSystem::parseData(unsigned char* sourceBuffer, int numBytes) {
// ask the VoxelTree to read the MONOCHROME bitstream into the tree
ReadBitstreamToTreeParams args(NO_COLOR, WANT_EXISTS_BITS, NULL, getDataSourceUUID());
lockTree();
_tree->readBitstreamToTree(voxelData, numBytes - numBytesPacketHeader, args);
VoxelPacket packet;
int compressedSize = numBytes - numBytesPacketHeader;
packet.loadCompressedContent(voxelData, compressedSize);
printf("got packet numBytes=%d compressed size %d uncompressed size %d\n",numBytes, compressedSize, packet.getUncompressedSize());
_tree->readBitstreamToTree(packet.getUncompressedData(), packet.getUncompressedSize(), args);
unlockTree();
}
break;

4
libraries/shared/src/OctalCode.cpp
View file

@ -32,12 +32,12 @@ int numberOfThreeBitSectionsInCode(const unsigned char* octalCode, int maxBytes)
void printOctalCode(const unsigned char* octalCode) {
if (!octalCode) {
qDebug("NULL\n");
printf("NULL\n");
} else {
for (int i = 0; i < bytesRequiredForCodeLength(numberOfThreeBitSectionsInCode(octalCode)); i++) {
outputBits(octalCode[i],false);
}
qDebug("\n");
printf("\n");
}
}

10
libraries/shared/src/SharedUtil.cpp
View file

@ -71,18 +71,18 @@ void outputBufferBits(const unsigned char* buffer, int length, bool withNewLine)
void outputBits(unsigned char byte, bool withNewLine) {
if (isalnum(byte)) {
qDebug("[ %d (%c): ", byte, byte);
printf("[ %d (%c): ", byte, byte);
} else {
qDebug("[ %d (0x%x): ", byte, byte);
printf("[ %d (0x%x): ", byte, byte);
}
for (int i = 0; i < 8; i++) {
qDebug("%d", byte >> (7 - i) & 1);
printf("%d", byte >> (7 - i) & 1);
}
qDebug(" ] ");
printf(" ] ");
if (withNewLine) {
qDebug("\n");
printf("\n");
}
}

8
libraries/voxel-server-library/src/VoxelSendThread.cpp
View file

@ -371,7 +371,14 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod
_myServer->getServerTree().lockForRead();
nodeData->stats.encodeStarted();
//printf("calling nodeBag.count()=%d encode()... subTree=%p \n", nodeData->nodeBag.count(), subTree);
//printOctalCode(subTree->getOctalCode());
bytesWritten = _myServer->getServerTree().encodeTreeBitstream(subTree, &_tempPacket, nodeData->nodeBag, params);
//printf("called encode()... bytesWritten=%d compressedSize=%d uncompressedSize=%d\n",bytesWritten, _tempPacket.getFinalizedSize(), _tempPacket.getUncompressedSize());
if (bytesWritten > 0) {
_encodedSomething = true;
}
@ -388,6 +395,7 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod
// mean we should send the previous packet contents and reset it.
bool sendNow = (bytesWritten == 0);
if (_tempPacket.hasContent() && sendNow) {
printf("calling writeToPacket() compressedSize=%d uncompressedSize=%d\n",_tempPacket.getFinalizedSize(), _tempPacket.getUncompressedSize());
nodeData->writeToPacket(_tempPacket.getFinalizedData(), _tempPacket.getFinalizedSize());
packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);
_tempPacket.reset();

2
libraries/voxel-server-library/src/VoxelSendThread.h
View file

@ -33,7 +33,7 @@ private:
int handlePacketSend(Node* node, VoxelNodeData* nodeData, int& trueBytesSent, int& truePacketsSent);
int deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nodeData, bool viewFrustumChanged);
unsigned char _tempOutputBuffer[MAX_VOXEL_PACKET_SIZE];
unsigned char _tempOutputBuffer[MAX_VOXEL_PACKET_SIZE]; // used by environment sending code
VoxelPacket _tempPacket;
bool _encodedSomething;

3
libraries/voxels/src/VoxelConstants.h
View file

@ -35,6 +35,9 @@ const float MAX_LOD_SIZE_MULTIPLIER = 2000.0f;
const int NUMBER_OF_CHILDREN = 8;
const int MAX_VOXEL_PACKET_SIZE = MAX_PACKET_SIZE - (sizeof(PACKET_TYPE) + sizeof(PACKET_VERSION));
const int MAX_VOXEL_PACKET_COMPRESSION_RATIO = 1;
const int MAX_VOXEL_UNCOMRESSED_PACKET_SIZE = MAX_VOXEL_PACKET_SIZE * MAX_VOXEL_PACKET_COMPRESSION_RATIO;
const int MAX_TREE_SLICE_BYTES = 26;
const int DEFAULT_MAX_VOXELS_PER_SYSTEM = 200000;
const int VERTICES_PER_VOXEL = 24; // 6 sides * 4 corners per side

145
libraries/voxels/src/VoxelPacket.cpp
View file

@ -14,8 +14,9 @@ VoxelPacket::VoxelPacket() {
void VoxelPacket::reset() {
_bytesInUse = 0;
_bytesAvailable = MAX_VOXEL_PACKET_SIZE;
_bytesAvailable = MAX_VOXEL_UNCOMRESSED_PACKET_SIZE;
_subTreeAt = 0;
_compressedBytes = 0;
}
VoxelPacket::~VoxelPacket() {
@ -25,10 +26,18 @@ bool VoxelPacket::append(const unsigned char* data, int length) {
bool success = false;
if (length <= _bytesAvailable) {
memcpy(&_buffer[_bytesInUse], data, length);
memcpy(&_uncompressed[_bytesInUse], data, length);
_bytesInUse += length;
_bytesAvailable -= length;
// Now, check for compression, if we fit, then proceed, otherwise, rollback.
if (checkCompress()) {
success = true;
} else {
// rollback is easy, we just reset _bytesInUse and _bytesAvailable
_bytesInUse -= length;
_bytesAvailable += length;
}
}
return success;
}
@ -36,10 +45,18 @@ bool VoxelPacket::append(const unsigned char* data, int length) {
bool VoxelPacket::append(unsigned char byte) {
bool success = false;
if (_bytesAvailable > 0) {
_buffer[_bytesInUse] = byte;
_uncompressed[_bytesInUse] = byte;
_bytesInUse++;
_bytesAvailable--;
// Now, check for compression, if we fit, then proceed, otherwise, rollback.
if (checkCompress()) {
success = true;
} else {
// rollback is easy, we just reset _bytesInUse and _bytesAvailable
_bytesInUse--;
_bytesAvailable++;
}
}
return success;
}
@ -47,8 +64,15 @@ bool VoxelPacket::append(unsigned char byte) {
bool VoxelPacket::updatePriorBitMask(int offset, unsigned char bitmask) {
bool success = false;
if (offset >= 0 && offset < _bytesInUse) {
_buffer[offset] = bitmask;
unsigned char oldValue = _uncompressed[offset];
_uncompressed[offset] = bitmask;
// Now, check for compression, if we fit, then proceed, otherwise, rollback.
if (checkCompress()) {
success = true;
} else {
// rollback is easy, the length didn't change, but we need to restore the previous value
_uncompressed[offset] = oldValue;
}
}
return success;
}
@ -56,8 +80,17 @@ bool VoxelPacket::updatePriorBitMask(int offset, unsigned char bitmask) {
bool VoxelPacket::updatePriorBytes(int offset, const unsigned char* replacementBytes, int length) {
bool success = false;
if (length >= 0 && offset >= 0 && ((offset + length) <= _bytesInUse)) {
memcpy(&_buffer[offset], replacementBytes, length);
unsigned char oldValues[length];
memcpy(&oldValues[0], &_uncompressed[offset], length); // save the old values for restore
memcpy(&_uncompressed[offset], replacementBytes, length); // copy new content
// Now, check for compression, if we fit, then proceed, otherwise, rollback.
if (checkCompress()) {
success = true;
} else {
// rollback is easy, the length didn't change, but we need to restore the previous values
memcpy(&_uncompressed[offset], &oldValues[0], length); // restore the old values
}
}
return success;
}
@ -67,11 +100,11 @@ bool VoxelPacket::startSubTree(const unsigned char* octcode) {
int possibleStartAt = _bytesInUse;
if (octcode) {
int length = bytesRequiredForCodeLength(numberOfThreeBitSectionsInCode(octcode));
success = append(octcode, length);
success = append(octcode, length); // handles checking compression
} else {
// NULL case, means root node, which is 0
unsigned char byte = 0;
success = append(byte);
success = append(byte); // handles checking compression
}
if (success) {
_subTreeAt = possibleStartAt;
@ -88,6 +121,11 @@ void VoxelPacket::discardSubTree() {
_bytesInUse -= bytesInSubTree;
_bytesAvailable += bytesInSubTree;
_subTreeAt = _bytesInUse; // should be the same actually...
// we can be confident that it will compress, because we can't get here without previously being able to compress
// the content up to this point in the uncompressed buffer. But we still call this because it cleans up the compressed
// buffer with the correct content
checkCompress();
}
int VoxelPacket::startLevel() {
@ -99,6 +137,11 @@ void VoxelPacket::discardLevel(int key) {
int bytesInLevel = _bytesInUse - key;
_bytesInUse -= bytesInLevel;
_bytesAvailable += bytesInLevel;
// we can be confident that it will compress, because we can't get here without previously being able to compress
// the content up to this point in the uncompressed buffer. But we still call this because it cleans up the compressed
// buffer with the correct content
checkCompress();
}
void VoxelPacket::endLevel() {
@ -106,7 +149,7 @@ void VoxelPacket::endLevel() {
}
bool VoxelPacket::appendBitMask(unsigned char bitmask) {
return append(bitmask);
return append(bitmask); // handles checking compression
}
bool VoxelPacket::appendColor(const nodeColor& color) {
@ -114,14 +157,94 @@ bool VoxelPacket::appendColor(const nodeColor& color) {
bool success = false;
const int BYTES_PER_COLOR = 3;
if (_bytesAvailable > BYTES_PER_COLOR) {
append(color[RED_INDEX]);
append(color[GREEN_INDEX]);
append(color[BLUE_INDEX]);
// handles checking compression...
if (append(color[RED_INDEX])) {
if (append(color[GREEN_INDEX])) {
if (append(color[BLUE_INDEX])) {
success = true;
}
}
}
}
return success;
}
bool VoxelPacket::checkCompress() {
bool success = false;
const int MAX_COMPRESSION = 9;
// we only want to compress the data payload, not the message header
QByteArray compressedData = qCompress(_uncompressed,_bytesInUse, MAX_COMPRESSION);
if (compressedData.size() < MAX_VOXEL_PACKET_SIZE) {
//memcpy(&_compressed[0], compressedData.constData(), compressedData.size());
_compressedBytes = compressedData.size();
for (int i = 0; i < _compressedBytes; i++) {
_compressed[i] = compressedData[i];
}
//printf("compressed %d bytes from %d original bytes\n", _compressedBytes, _bytesInUse);
success = true;
}
return success;
}
void VoxelPacket::loadCompressedContent(const unsigned char* data, int length) {
reset(); // by definition we reset upon loading compressed content
if (length > 0) {
QByteArray compressedData;
for (int i = 0; i < length; i++) {
compressedData[i] = data[i];
_compressed[i] = compressedData[i];
}
_compressedBytes = length;
QByteArray uncompressedData = qUncompress(compressedData);
if (uncompressedData.size() <= _bytesAvailable) {
_bytesInUse = uncompressedData.size();
_bytesAvailable -= uncompressedData.size();
for (int i = 0; i < _bytesInUse; i++) {
_uncompressed[i] = uncompressedData[i];
}
}
} else {
printf("VoxelPacket::loadCompressedContent()... length = 0, nothing to do...\n");
}
}
void VoxelPacket::debugContent() {
printf("VoxelPacket::debugContent()... COMPRESSED DATA.... size=%d\n",_compressedBytes);
int perline=0;
for (int i = 0; i < _compressedBytes; i++) {
printf("%.2x ",_compressed[i]);
perline++;
if (perline >= 30) {
printf("\n");
perline=0;
}
}
printf("\n");
printf("VoxelPacket::debugContent()... UNCOMPRESSED DATA.... size=%d\n",_bytesInUse);
perline=0;
for (int i = 0; i < _bytesInUse; i++) {
printf("%.2x ",_uncompressed[i]);
perline++;
if (perline >= 30) {
printf("\n");
perline=0;
}
}
printf("\n");
}
/***
void VoxelPacket::compressPacket() {
int uncompressedLength = getPacketLengthUncompressed();

20
libraries/voxels/src/VoxelPacket.h
View file

@ -20,6 +20,8 @@
#include "VoxelConstants.h"
#include "VoxelNode.h"
class VoxelPacket {
public:
VoxelPacket();
@ -66,18 +68,23 @@ public:
int getUncompressedByteOffset(int offsetFromEnd = 0) const { return _bytesInUse - offsetFromEnd; }
/// get access to the finalized data (it may be compressed or rewritten into optimal form)
const unsigned char* getFinalizedData() { return &_buffer[0]; }
const unsigned char* getFinalizedData() { return &_compressed[0]; }
/// get size of the finalized data (it may be compressed or rewritten into optimal form)
int getFinalizedSize() const { return _bytesInUse; }
int getFinalizedSize() const { return _compressedBytes; }
/// get pointer to the start of uncompressed stream buffer
const unsigned char* getUncompressedData() { return &_buffer[0]; }
const unsigned char* getUncompressedData() { return &_uncompressed[0]; }
/// the size of the packet in uncompressed form
int getUncompressedSize() { return _bytesInUse; }
/// has some content been written to the packet
bool hasContent() const { return (_bytesInUse > 0); }
/// load compressed content to allow access to decoded content for parsing
void loadCompressedContent(const unsigned char* data, int length);
void debugContent();
private:
/// appends raw bytes, might fail if byte would cause packet to be too large
bool append(const unsigned char* data, int length);
@ -85,10 +92,15 @@ private:
/// append a single byte, might fail if byte would cause packet to be too large
bool append(unsigned char byte);
unsigned char _buffer[MAX_VOXEL_PACKET_SIZE];
unsigned char _uncompressed[MAX_VOXEL_UNCOMRESSED_PACKET_SIZE];
int _bytesInUse;
int _bytesAvailable;
int _subTreeAt;
bool checkCompress();
unsigned char _compressed[MAX_VOXEL_PACKET_SIZE];
int _compressedBytes;
};
#endif /* defined(__hifi__VoxelPacket__) */

2
libraries/voxels/src/VoxelTree.cpp
View file

@ -1576,7 +1576,7 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node,
// reshuffle here...
if (continueThisLevel && params.wantOcclusionCulling) {
unsigned char tempReshuffleBuffer[MAX_VOXEL_PACKET_SIZE];
unsigned char tempReshuffleBuffer[MAX_VOXEL_UNCOMRESSED_PACKET_SIZE];
unsigned char* tempBufferTo = &tempReshuffleBuffer[0]; // this is our temporary destination