Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-23 06:53:46 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

pack more compressed sections into wire packets if there is room

Browse source
This commit is contained in:
ZappoMan 2013-11-26 22:18:24 -08:00
parent b7ee2ea2db
commit dac211cebd
6 changed files with 133 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
interface/src/VoxelSystem.cpp
View file

@ -619,26 +619,41 @@ int VoxelSystem::parseData(unsigned char* sourceBuffer, int numBytes) {
VOXEL_PACKET_INTERNAL_SECTION_SIZE sectionLength = 0;
int dataBytes = numBytes - VOXEL_PACKET_HEADER_SIZE;
if (packetIsCompressed && dataBytes > sizeof(VOXEL_PACKET_INTERNAL_SECTION_SIZE)) {
sectionLength = (*(VOXEL_PACKET_INTERNAL_SECTION_SIZE*)dataAt);
dataAt += sizeof(VOXEL_PACKET_INTERNAL_SECTION_SIZE);
} else {
sectionLength = dataBytes;
int subsection = 1;
while (dataBytes > 0) {
if (packetIsCompressed) {
if (dataBytes > sizeof(VOXEL_PACKET_INTERNAL_SECTION_SIZE)) {
sectionLength = (*(VOXEL_PACKET_INTERNAL_SECTION_SIZE*)dataAt);
dataAt += sizeof(VOXEL_PACKET_INTERNAL_SECTION_SIZE);
dataBytes -= sizeof(VOXEL_PACKET_INTERNAL_SECTION_SIZE);
} else {
sectionLength = 0;
dataBytes = 0; // stop looping something is wrong
}
} else {
sectionLength = dataBytes;
}
if (sectionLength) {
// ask the VoxelTree to read the bitstream into the tree
ReadBitstreamToTreeParams args(packetIsColored ? WANT_COLOR : NO_COLOR, WANT_EXISTS_BITS, NULL, getDataSourceUUID());
lockTree();
VoxelPacketData packetData(packetIsCompressed);
packetData.loadFinalizedContent(dataAt, sectionLength);
if (Menu::getInstance()->isOptionChecked(MenuOption::ExtraDebugging)) {
qDebug("Got Packet color:%s compressed:%s sequence: %u flight:%d usec size:%d data:%d"
" subsection:%d sectionLength:%d uncompressed:%d\n",
debug::valueOf(packetIsColored), debug::valueOf(packetIsCompressed),
sequence, flightTime, numBytes, dataBytes, subsection, sectionLength, packetData.getUncompressedSize());
}
_tree->readBitstreamToTree(packetData.getUncompressedData(), packetData.getUncompressedSize(), args);
unlockTree();
dataBytes -= sectionLength;
dataAt += sectionLength;
}
}
unsigned char* voxelData = dataAt;
// ask the VoxelTree to read the bitstream into the tree
ReadBitstreamToTreeParams args(packetIsColored ? WANT_COLOR : NO_COLOR, WANT_EXISTS_BITS, NULL, getDataSourceUUID());
lockTree();
VoxelPacketData packetData(packetIsCompressed);
packetData.loadFinalizedContent(voxelData, sectionLength);
if (true || Menu::getInstance()->isOptionChecked(MenuOption::ExtraDebugging)) {
qDebug("Got Packet color:%s compressed:%s sequence: %u flight:%d usec size:%d data:%d sectionLength:%d uncompressed:%d\n",
debug::valueOf(packetIsColored), debug::valueOf(packetIsCompressed),
sequence, flightTime, numBytes, dataBytes, sectionLength, packetData.getUncompressedSize());
}
_tree->readBitstreamToTree(packetData.getUncompressedData(), packetData.getUncompressedSize(), args);
unlockTree();
subsection++;
}
break;
}

14
libraries/voxel-server-library/src/VoxelNodeData.cpp
View file

@ -16,7 +16,7 @@
VoxelNodeData::VoxelNodeData(Node* owningNode) :
VoxelQuery(owningNode),
_viewSent(false),
_voxelPacketAvailableBytes(MAX_VOXEL_PACKET_SIZE),
_voxelPacketAvailableBytes(MAX_PACKET_SIZE),
_maxSearchLevel(1),
_maxLevelReachedInLastSearch(1),
_lastTimeBagEmpty(0),
@ -30,9 +30,9 @@ VoxelNodeData::VoxelNodeData(Node* owningNode) :
_lodChanged(false),
_lodInitialized(false)
{
_voxelPacket = new unsigned char[_voxelPacketAvailableBytes];
_voxelPacket = new unsigned char[MAX_PACKET_SIZE];
_voxelPacketAt = _voxelPacket;
_lastVoxelPacket = new unsigned char[_voxelPacketAvailableBytes];
_lastVoxelPacket = new unsigned char[MAX_PACKET_SIZE];
_lastVoxelPacketLength = 0;
_duplicatePacketCount = 0;
_sequenceNumber = 0;
@ -108,18 +108,22 @@ void VoxelNodeData::resetVoxelPacket() {
setAtBit(flags,PACKET_IS_COMPRESSED_BIT);
}
_voxelPacketAvailableBytes = MAX_PACKET_SIZE;
int numBytesPacketHeader = populateTypeAndVersion(_voxelPacket, PACKET_TYPE_VOXEL_DATA);
_voxelPacketAt = _voxelPacket + numBytesPacketHeader;
_voxelPacketAvailableBytes -= numBytesPacketHeader;
// pack in flags
VOXEL_PACKET_FLAGS* flagsAt = (VOXEL_PACKET_FLAGS*)_voxelPacketAt;
*flagsAt = flags;
_voxelPacketAt += sizeof(VOXEL_PACKET_FLAGS);
_voxelPacketAvailableBytes -= sizeof(VOXEL_PACKET_FLAGS);
// pack in sequence number
VOXEL_PACKET_SEQUENCE* sequenceAt = (VOXEL_PACKET_SEQUENCE*)_voxelPacketAt;
*sequenceAt = _sequenceNumber;
_voxelPacketAt += sizeof(VOXEL_PACKET_SEQUENCE);
_voxelPacketAvailableBytes -= sizeof(VOXEL_PACKET_SEQUENCE);
_sequenceNumber++;
// pack in timestamp
@ -127,8 +131,8 @@ void VoxelNodeData::resetVoxelPacket() {
VOXEL_PACKET_SENT_TIME* timeAt = (VOXEL_PACKET_SENT_TIME*)_voxelPacketAt;
*timeAt = now;
_voxelPacketAt += sizeof(VOXEL_PACKET_SENT_TIME);
_voxelPacketAvailableBytes -= sizeof(VOXEL_PACKET_SENT_TIME);
_voxelPacketAvailableBytes = MAX_VOXEL_PACKET_DATA_SIZE;
_voxelPacketWaiting = false;
}
@ -144,6 +148,8 @@ void VoxelNodeData::writeToPacket(const unsigned char* buffer, int bytes) {
_voxelPacketAvailableBytes -= bytes;
_voxelPacketAt += bytes;
_voxelPacketWaiting = true;
assert(_voxelPacketAvailableBytes >= 0);
}
VoxelNodeData::~VoxelNodeData() {

2
libraries/voxel-server-library/src/VoxelNodeData.h
View file

@ -32,7 +32,7 @@ public:
void writeToPacket(const unsigned char* buffer, int bytes); // writes to end of packet
const unsigned char* getPacket() const { return _voxelPacket; }
int getPacketLength() const { return (MAX_VOXEL_PACKET_SIZE - _voxelPacketAvailableBytes); }
int getPacketLength() const { return (MAX_PACKET_SIZE - _voxelPacketAvailableBytes); }
bool isPacketWaiting() const { return _voxelPacketWaiting; }
bool packetIsDuplicate() const;

96
libraries/voxel-server-library/src/VoxelSendThread.cpp
View file

@ -251,9 +251,14 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod
}
nodeData->resetVoxelPacket();
}
int uncompressedSize = !wantCompression ? MAX_VOXEL_PACKET_DATA_SIZE
: MAX_VOXEL_PACKET_DATA_SIZE - sizeof(VOXEL_PACKET_INTERNAL_SECTION_SIZE);
_packetData.changeSettings(wantCompression, uncompressedSize);
int targetSize = MAX_VOXEL_PACKET_DATA_SIZE;
if (wantCompression) {
targetSize = nodeData->getAvailable() - sizeof(VOXEL_PACKET_INTERNAL_SECTION_SIZE);
}
printf("line:%d _packetData.changeSettings() wantCompression=%s targetSize=%d\n",__LINE__,
debug::valueOf(wantCompression), targetSize);
_packetData.changeSettings(wantCompression, targetSize);
}
if (_myServer->wantsDebugVoxelSending() && _myServer->wantsVerboseDebug()) {
@ -315,7 +320,7 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod
unsigned long elapsedTime = nodeData->stats.getElapsedTime();
packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);
if (true || _myServer->wantsDebugVoxelSending()) {
if (_myServer->wantsDebugVoxelSending()) {
qDebug("Scene completed at %llu encodeTime:%lu sleepTime:%lu elapsed:%lu Packets:%llu Bytes:%llu Wasted:%llu\n",
usecTimestampNow(), encodeTime, sleepTime, elapsedTime, _totalPackets, _totalBytes, _totalWastedBytes);
}
@ -373,6 +378,7 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod
nodeData->getMaxVoxelPacketsPerSecond(), clientMaxPacketsPerInterval);
}
int extraPackingAttempts = 0;
while (somethingToSend && packetsSentThisInterval < maxPacketsPerInterval - (shouldSendEnvironments ? 1 : 0)) {
if (_myServer->wantsDebugVoxelSending() && _myServer->wantsVerboseDebug()) {
printf("truePacketsSent=%d packetsSentThisInterval=%d maxPacketsPerInterval=%d server PPI=%d nodePPS=%d nodePPI=%d\n",
@ -407,8 +413,20 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod
nodeData->stats.encodeStarted();
bytesWritten = _myServer->getServerTree().encodeTreeBitstream(subTree, &_packetData, nodeData->nodeBag, params);
if (_packetData.hasContent() && bytesWritten == 0 && params.stopReason == EncodeBitstreamParams::DIDNT_FIT) {
lastNodeDidntFit = true;
// if we're trying to fill a full size packet, then we use this logic to determine if we have a DIDNT_FIT case.
if (_packetData.getTargetSize() == MAX_VOXEL_PACKET_DATA_SIZE) {
if (_packetData.hasContent() && bytesWritten == 0 &&
params.stopReason == EncodeBitstreamParams::DIDNT_FIT) {
lastNodeDidntFit = true;
}
} else {
// in compressed mode and we are trying to pack more... and we don't care if the _packetData has
// content or not... because in this case even if we were unable to pack any data, we want to drop
// below to our sendNow logic, but we do want to track that we attempted to pack extra
extraPackingAttempts++;
if (bytesWritten == 0 && params.stopReason == EncodeBitstreamParams::DIDNT_FIT) {
lastNodeDidntFit = true;
}
}
if (bytesWritten > 0) {
@ -422,18 +440,68 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod
somethingToSend = false; // this will cause us to drop out of the loop...
}
// If the last node didn't fit, but we're in compressed mode, then we actually want to see if we can fit a
// little bit more in this packet. To do this we
// We only consider sending anything if there is something in the _packetData to send... But
// if bytesWritten == 0 it means either the subTree couldn't fit or we had an empty bag... Both cases
// mean we should send the previous packet contents and reset it.
bool sendNow = lastNodeDidntFit;
if (_packetData.hasContent() && sendNow) {
if (_myServer->wantsDebugVoxelSending() && _myServer->wantsVerboseDebug()) {
printf("calling writeToPacket() compressedSize=%d uncompressedSize=%d\n",
_packetData.getFinalizedSize(), _packetData.getUncompressedSize());
if (lastNodeDidntFit) {
if (_packetData.hasContent()) {
// if for some reason the finalized size is greater than our available size, then probably the "compressed"
// form actually inflated beyond our padding, and in this case we will send the current packet, then
// write to out new packet...
int writtenSize = _packetData.getFinalizedSize()
+ (nodeData->getCurrentPacketIsCompressed() ? sizeof(VOXEL_PACKET_INTERNAL_SECTION_SIZE) : 0);
if (writtenSize > nodeData->getAvailable()) {
if (_myServer->wantsDebugVoxelSending() && _myServer->wantsVerboseDebug()) {
printf("writtenSize[%d] > available[%d] too big, sending packet as is.\n",
writtenSize, nodeData->getAvailable());
}
packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);
}
if (_myServer->wantsDebugVoxelSending() && _myServer->wantsVerboseDebug()) {
printf("calling writeToPacket() available=%d compressedSize=%d uncompressedSize=%d target=%d\n",
nodeData->getAvailable(), _packetData.getFinalizedSize(),
_packetData.getUncompressedSize(), _packetData.getTargetSize());
}
nodeData->writeToPacket(_packetData.getFinalizedData(), _packetData.getFinalizedSize());
extraPackingAttempts = 0;
}
nodeData->writeToPacket(_packetData.getFinalizedData(), _packetData.getFinalizedSize());
packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);
_packetData.reset();
// If we're not running compressed, the we know we can just send now. Or if we're running compressed, but
// the packet doesn't have enough space to bother attempting to pack more...
bool sendNow = true;
if (nodeData->getCurrentPacketIsCompressed() &&
nodeData->getAvailable() >= MINIMUM_ATTEMPT_MORE_PACKING &&
extraPackingAttempts <= REASONABLE_NUMBER_OF_PACKING_ATTEMPTS) {
sendNow = false; // try to pack more
}
int targetSize = MAX_VOXEL_PACKET_DATA_SIZE;
if (sendNow) {
packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);
if (wantCompression) {
targetSize = nodeData->getAvailable() - sizeof(VOXEL_PACKET_INTERNAL_SECTION_SIZE);
}
} else {
// If we're in compressed mode, then we want to see if we have room for more in this wire packet.
// but we've finalized the _packetData, so we want to start a new section, we will do that by
// resetting the packet settings with the max uncompressed size of our current available space
// in the wire packet. We also include room for our section header, and a little bit of padding
// to account for the fact that whenc compressing small amounts of data, we sometimes end up with
// a larger compressed size then uncompressed size
targetSize = nodeData->getAvailable() - sizeof(VOXEL_PACKET_INTERNAL_SECTION_SIZE) - COMPRESS_PADDING;
}
if (_myServer->wantsDebugVoxelSending() && _myServer->wantsVerboseDebug()) {
printf("line:%d _packetData.changeSettings() wantCompression=%s targetSize=%d\n",__LINE__,
debug::valueOf(nodeData->getWantCompression()), targetSize);
}
_packetData.changeSettings(nodeData->getWantCompression(), targetSize); // will do reset
}
}

6
libraries/voxels/src/VoxelPacketData.cpp
View file

@ -28,11 +28,7 @@ void VoxelPacketData::changeSettings(bool enableCompression, int targetSize) {
void VoxelPacketData::reset() {
_bytesInUse = 0;
if (_enableCompression) {
_bytesAvailable = MAX_VOXEL_UNCOMRESSED_PACKET_SIZE;
} else {
_bytesAvailable = _targetSize;
}
_bytesAvailable = _targetSize;
_subTreeAt = 0;
_compressedBytes = 0;
_bytesInUseLastCheck = 0;

5
libraries/voxels/src/VoxelPacketData.h
View file

@ -35,6 +35,10 @@ const int MAX_VOXEL_PACKET_DATA_SIZE = MAX_PACKET_SIZE - VOXEL_PACKET_HEADER_SIZ
const int MAX_VOXEL_UNCOMRESSED_PACKET_SIZE = MAX_VOXEL_PACKET_DATA_SIZE;
const int MINIMUM_ATTEMPT_MORE_PACKING = sizeof(VOXEL_PACKET_INTERNAL_SECTION_SIZE) + 40;
const int COMPRESS_PADDING = 15;
const int REASONABLE_NUMBER_OF_PACKING_ATTEMPTS = 5;
const int PACKET_IS_COLOR_BIT = 0;
const int PACKET_IS_COMPRESSED_BIT = 1;
@ -121,6 +125,7 @@ public:
void loadFinalizedContent(const unsigned char* data, int length);
bool isCompressed() const { return _enableCompression; }
int getTargetSize() const { return _targetSize; }
void debugContent();