more DIDNT_FIT logic improvements/fixes

assignment-client/src/octree/OctreeSendThread.cpp

@@ -400,6 +400,9 @@ int OctreeSendThread::packetDistributor(OctreeQueryNode* nodeData, bool viewFrus
 
         int extraPackingAttempts = 0;
         bool completedScene = false;
+        
+        OctreeElement* lastAttemptedSubTree = NULL;
+        
         while (somethingToSend && packetsSentThisInterval < maxPacketsPerInterval && !nodeData->isShuttingDown()) {
             float lockWaitElapsedUsec = OctreeServer::SKIP_TIME;
             float encodeElapsedUsec = OctreeServer::SKIP_TIME;
@@ -419,6 +422,14 @@ if (subTree == _myServer->getOctree()->getRoot()) {
     qDebug() << "===============================================================";
     qDebug() << "OctreeSendThread::packetDistributor() subTree=" << subTree;
 }
+
+                if (lastAttemptedSubTree == subTree) {
+                    qDebug() << "SAME SUBTREE AS LAST TIME subTree=" << subTree << "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%";
+                } else {
+                    qDebug() << "NEW subTree=" << subTree;
+                }
+                lastAttemptedSubTree = subTree;
+
     
                 
                 /* TODO: Looking for a way to prevent locking and encoding a tree that is not
@@ -469,6 +480,19 @@ if (subTree == _myServer->getOctree()->getRoot()) {
                 lockWaitElapsedUsec = (float)(lockWaitEnd - lockWaitStart);
 
                 quint64 encodeStart = usecTimestampNow();
+
+qDebug() << "+++++++++++++++ BEFORE _myServer->getOctree()->encodeTreeBitstream(subTree...) +++++++++++++++";
+qDebug() << "OctreeSendThread::packetDistributor()... line:" << __LINE__;
+qDebug() << "    _packetData.getTargetSize()=" << _packetData.getTargetSize();
+qDebug() << "    MAX_OCTREE_PACKET_DATA_SIZE=" << MAX_OCTREE_PACKET_DATA_SIZE;
+qDebug() << "    _packetData.hasContent()=" << _packetData.hasContent();
+qDebug() << "    _packetData.getUncompressedSize()=" << _packetData.getUncompressedSize();
+qDebug() << "    _packetData.getFinalizedSize()=" << _packetData.getFinalizedSize();
+qDebug() << "    _packetData.isCompressed()=" << _packetData.isCompressed();
+qDebug() << "    _packetData.getReservedBytes()=" << _packetData.getReservedBytes();
+qDebug() << "    extraPackingAttempts=" << extraPackingAttempts;
+qDebug() << "    REASONABLE_NUMBER_OF_PACKING_ATTEMPTS=" << REASONABLE_NUMBER_OF_PACKING_ATTEMPTS;
+
                 bytesWritten = _myServer->getOctree()->encodeTreeBitstream(subTree, &_packetData, nodeData->elementBag, params);
                 
 qDebug() << "OctreeSendThread::packetDistributor()...";
@@ -481,7 +505,15 @@ qDebug() << "    bytesWritten=" << bytesWritten;
                 // If after calling encodeTreeBitstream() there are no nodes left to send, then we know we've
                 // sent the entire scene. We want to know this below so we'll actually write this content into
                 // the packet and send it
+qDebug() << "OctreeSendThread::packetDistributor()... line:" << __LINE__;
+qDebug() << "    nodeData->elementBag.isEmpty()=" << nodeData->elementBag.isEmpty();
                 completedScene = nodeData->elementBag.isEmpty();
+qDebug() << "    completedScene=" << completedScene;
+if (completedScene) {
+qDebug() << "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^";
+qDebug() << "HELLO!!!! DID YOU EXPECT THE SCENE TO COMPLETE!!!!";
+qDebug() << "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^";
+}
 
 
 qDebug() << "+++++++++++++++ lastNodeDidntFit logic +++++++++++++++";
@@ -492,6 +524,7 @@ qDebug() << "    _packetData.hasContent()=" << _packetData.hasContent();
 qDebug() << "    _packetData.getUncompressedSize()=" << _packetData.getUncompressedSize();
 qDebug() << "    _packetData.getFinalizedSize()=" << _packetData.getFinalizedSize();
 qDebug() << "    _packetData.isCompressed()=" << _packetData.isCompressed();
+qDebug() << "    _packetData.getReservedBytes()=" << _packetData.getReservedBytes();
 qDebug() << "    bytesWritten=" << bytesWritten;
 qDebug() << "    extraPackingAttempts=" << extraPackingAttempts;
 qDebug() << "    REASONABLE_NUMBER_OF_PACKING_ATTEMPTS=" << REASONABLE_NUMBER_OF_PACKING_ATTEMPTS;
@@ -569,14 +602,17 @@ qDebug() << "    writtenSize=" << writtenSize;
 qDebug() << "    nodeData->getAvailable()=" << nodeData->getAvailable();
 */
                     if (writtenSize > nodeData->getAvailable()) {
-//qDebug() << "OctreeSendThread::packetDistributor()... line:" << __LINE__;
+qDebug() << "OctreeSendThread::packetDistributor()... line:" << __LINE__;
-//qDebug() << "   calling handlePacketSend()...";
+qDebug() << "   compressed form expanded packet send it...";
+qDebug() << "   calling handlePacketSend()...";
                         packetsSentThisInterval += handlePacketSend(nodeData, trueBytesSent, truePacketsSent);
                     }
 
-//qDebug() << "OctreeSendThread::packetDistributor()... line:" << __LINE__;
+qDebug() << "OctreeSendThread::packetDistributor()... line:" << __LINE__;
-//qDebug() << "    _packetData.getFinalizedSize()=" << _packetData.getFinalizedSize();
+qDebug() << "    WRITING the packetData to the node's packet...!!!";
-//qDebug() << "    called nodeData->writeToPacket(...  _packetData.getFinalizedSize()... )";
+qDebug() << "    _packetData.getFinalizedSize()=" << _packetData.getFinalizedSize();
+qDebug() << "    called nodeData->writeToPacket(...  _packetData.getFinalizedSize()... )";
+                    lastAttemptedSubTree = NULL; // reset this
 
                     nodeData->writeToPacket(_packetData.getFinalizedData(), _packetData.getFinalizedSize());
                     extraPackingAttempts = 0;

libraries/entities/src/EntityItem.cpp

@@ -133,21 +133,38 @@ OctreeElement::AppendState EntityItem::appendEntityData(OctreePacketData* packet
     
     LevelDetails modelLevel = packetData->startLevel();
 
-    bool successIDFits = packetData->appendValue(encodedID);
-    bool successTypeFits = packetData->appendValue(encodedType);
-    
     quint64 lastEdited = getLastEdited();
     //qDebug() << "EntityItem::appendEntityData() ... lastEdited=" << lastEdited;
     
-    bool successLastEditedFits = packetData->appendValue(lastEdited);
+    bool successIDFits = false;
-    bool successLastUpdatedFits = packetData->appendValue(encodedUpdateDelta);
+    bool successTypeFits = false;
-    
+    bool successLastEditedFits = false;
-    int propertyFlagsOffset = packetData->getUncompressedByteOffset();
+    bool successLastUpdatedFits = false;
-    QByteArray encodedPropertyFlags = propertyFlags;
+    bool successPropertyFlagsFits = false;
-    int oldPropertyFlagsLength = encodedPropertyFlags.length();
+    int propertyFlagsOffset = 0;
-    bool successPropertyFlagsFits = packetData->appendValue(encodedPropertyFlags);
+    int oldPropertyFlagsLength = 0;
+    QByteArray encodedPropertyFlags;
     int propertyCount = 0;
 
+    successIDFits = packetData->appendValue(encodedID);
+    if (successIDFits) {
+        successTypeFits = packetData->appendValue(encodedType);
+    }
+    
+    if (successTypeFits) {
+        successLastEditedFits = packetData->appendValue(lastEdited);
+    }
+    if (successLastEditedFits) {
+        successLastUpdatedFits = packetData->appendValue(encodedUpdateDelta);
+    }
+    
+    if (successLastUpdatedFits) {
+        propertyFlagsOffset = packetData->getUncompressedByteOffset();
+        encodedPropertyFlags = propertyFlags;
+        oldPropertyFlagsLength = encodedPropertyFlags.length();
+        successPropertyFlagsFits = packetData->appendValue(encodedPropertyFlags);
+    }
+
     bool headerFits = successIDFits && successTypeFits && successLastEditedFits 
                               && successLastUpdatedFits && successPropertyFlagsFits;
 
@@ -354,6 +371,7 @@ OctreeElement::AppendState EntityItem::appendEntityData(OctreePacketData* packet
             packetData->updatePriorBytes(newEntityItemDataStart, modelItemData, modelItemDataLength);
 
             int newSize = oldSize - (oldPropertyFlagsLength - newPropertyFlagsLength);
+qDebug() << "EntityItem::appendEntityData()... SHRINKING CASE??? DID WE TEST THIS!!!! <<<<<<<<<<<<<<<<<<<<<<<<<<<<<";
             packetData->setUncompressedSize(newSize);
 
         } else {

libraries/entities/src/EntityTreeElement.cpp

@@ -117,6 +117,8 @@ OctreeElement::AppendState EntityTreeElement::appendElementData(OctreePacketData
     qDebug() << "    numberOfEntities=" << numberOfEntities;
     qDebug() << "    successAppendEntityCount=" << successAppendEntityCount;
     qDebug() << "--- before child loop ---";
+    qDebug() << "    packetData->getUncompressedSize()=" << packetData->getUncompressedSize() << "line:" << __LINE__;
+    qDebug() << "    packetData->getReservedBytes()=" << packetData->getReservedBytes();
 
     if (successAppendEntityCount) {
     
@@ -134,9 +136,15 @@ OctreeElement::AppendState EntityTreeElement::appendElementData(OctreePacketData
             
             LevelDetails entityLevel = packetData->startLevel();
 
-            qDebug() << "BEFORE entity packetData->uncompressed size:" << packetData->getUncompressedSize();
+            qDebug() << "--- BEFORE entity ---";
+            qDebug() << "    packetData->getUncompressedSize()=" << packetData->getUncompressedSize() << "line:" << __LINE__;
+            qDebug() << "    packetData->getReservedBytes()=" << packetData->getReservedBytes();
+
             OctreeElement::AppendState appendEntityState = entity->appendEntityData(packetData, params, entityTreeElementExtraEncodeData);
-            qDebug() << "AFTER entity packetData->uncompressed size:" << packetData->getUncompressedSize();
+
+            qDebug() << "--- AFTER entity ---";
+            qDebug() << "    packetData->getUncompressedSize()=" << packetData->getUncompressedSize() << "line:" << __LINE__;
+            qDebug() << "    packetData->getReservedBytes()=" << packetData->getReservedBytes();
 
             // If none of this entity data was able to be appended, then discard it
             // and don't include it in our entity count

libraries/octree/src/Octree.cpp

@@ -1007,6 +1007,10 @@ int Octree::encodeTreeBitstream(OctreeElement* element,
     int childBytesWritten = encodeTreeBitstreamRecursion(element, packetData, bag, params,
                                                             currentEncodeLevel, parentLocationThisView);
 
+
+qDebug() << "Octree::encodeTreeBitstream() AFTER encodeTreeBitstreamRecursion()....";
+qDebug() << "    bag.isEmpty()=" << bag.isEmpty();
+
     // if childBytesWritten == 1 then something went wrong... that's not possible
     assert(childBytesWritten != 1);
 
@@ -1026,10 +1030,14 @@ qDebug() << "STOP REASON.... if (params.includeColor && childBytesWritten == 2).
         // otherwise... if we didn't write any child bytes, then pretend like we also didn't write our octal code
         bytesWritten = 0;
         //params.stopReason = EncodeBitstreamParams::DIDNT_FIT;
-qDebug() << "STOP REASON.... childBytesWritten == 0??????";
+qDebug() << "STOP REASON.... childBytesWritten == 0??????  params.stopReason=" << params.getStopReason();
     }
 
     if (bytesWritten == 0) {
+qDebug() << "Octree::encodeTreeBitstream()... calling packetData->discardSubTree()....";
+qDebug() << "    bytesWritten == 0";
+qDebug() << "    params.stopReason=" << params.getStopReason();
+qDebug() << "    packetData->getReservedBytes()=" << packetData->getReservedBytes();
         packetData->discardSubTree();
     } else {
         packetData->endSubTree();
@@ -1046,6 +1054,12 @@ int Octree::encodeTreeBitstreamRecursion(OctreeElement* element,
                                             EncodeBitstreamParams& params, int& currentEncodeLevel,
                                             const ViewFrustum::location& parentLocationThisView) const {
 
+
+qDebug() << "ENTERING Octree::encodeTreeBitstreamRecursion()...";
+qDebug() << "    params.stopReason=" << params.getStopReason();
+qDebug() << "    packetData->getReservedBytes()=" << packetData->getReservedBytes();
+
+
     // The append state of this level/element.
     OctreeElement::AppendState elementAppendState = OctreeElement::COMPLETED; // assume the best
 
@@ -1212,10 +1226,22 @@ int Octree::encodeTreeBitstreamRecursion(OctreeElement* element,
 
     // If we can't reserve our minimum bytes then we can discard this level and return as if none of this level fits
     if (!continueThisLevel) {
+qDebug() << "Octree::encodeTreeBitstream() AFTER attempt to reserve bytes for bitmasks....";
+qDebug() << "attempt to reserve bytes for bitmasks failed... ";
+qDebug() << "    discarding level";
+qDebug() << "    returning EncodeBitstreamParams::DIDNT_FIT...";
+qDebug() << "    insert our element back in bag, since if we came here, we need to be encoded, but we were removed by the caller...";
+
         packetData->discardLevel(thisLevelKey);
         params.stopReason = EncodeBitstreamParams::DIDNT_FIT;
+        bag.insert(element);
+
+qDebug() << "    bag.isEmpty()=" << bag.isEmpty();
         return bytesAtThisLevel;
     }
+
+qDebug() << "after reserving bytes for bitmasks...";
+qDebug() << "    packetData->getReservedBytes()=" << packetData->getReservedBytes();
     
     int inViewCount = 0;
     int inViewNotLeafCount = 0;
@@ -1401,6 +1427,8 @@ int Octree::encodeTreeBitstreamRecursion(OctreeElement* element,
     // but we may come back later and update the bits that are actually included
     packetData->releaseReservedBytes(sizeof(childrenDataBits));
     continueThisLevel = packetData->appendBitMask(childrenDataBits);
+qDebug() << " packetData->appendBitMask(childrenDataBits) line:" << __LINE__;
+qDebug() << "    continueThisLevel=" << continueThisLevel;
 
     // we know the last thing we wrote to the packet was our childrenDataBits. Let's remember where that was!
     int childDataBitsPlaceHolder = packetData->getUncompressedByteOffset(sizeof(childrenDataBits));
@@ -1417,6 +1445,7 @@ qDebug() << "    childDataBitsPlaceHolder=" << childDataBitsPlaceHolder << "line
     
 qDebug() << "--- BEFORE CHILD LOOP --- line:" << __LINE__;
 qDebug() << "    packetData->getUncompressedSize()=" << packetData->getUncompressedSize() << "line:" << __LINE__;
+qDebug() << "    packetData->getReservedBytes()=" << packetData->getReservedBytes();
 
 
     // write the child element data...
@@ -1437,7 +1466,17 @@ qDebug() << "    packetData->getUncompressedSize()=" << packetData->getUncompres
 
                     // Make our local buffer large enough to handle writing at this level in case we need to.
                     LevelDetails childDataLevelKey = packetData->startLevel();
+                    
+
+qDebug() << "--- BEFORE childElement->appendElementData() --- line:" << __LINE__;
+qDebug() << "    packetData->getUncompressedSize()=" << packetData->getUncompressedSize() << "line:" << __LINE__;
+qDebug() << "    packetData->getReservedBytes()=" << packetData->getReservedBytes();
+                    
                     OctreeElement::AppendState childAppendState = childElement->appendElementData(packetData, params);
+
+qDebug() << "--- AFTER childElement->appendElementData() --- line:" << __LINE__;
+qDebug() << "    packetData->getUncompressedSize()=" << packetData->getUncompressedSize() << "line:" << __LINE__;
+qDebug() << "    packetData->getReservedBytes()=" << packetData->getReservedBytes();
                     
                     // Continue this level so long as some part of this child element was appended.
                     // TODO: consider if we want to also keep going in the child append state was NONE... to do this
@@ -1461,6 +1500,8 @@ qDebug() << "    packetData->getUncompressedSize()=" << packetData->getUncompres
                         packetData->discardLevel(childDataLevelKey);
                         qDebug() << "    since child data didn't fit, this element (not the child) is a partial element....";
                         elementAppendState = OctreeElement::PARTIAL;
+                        qDebug() << "    also set params.stopReason = EncodeBitstreamParams::DIDNT_FIT";
+                        params.stopReason = EncodeBitstreamParams::DIDNT_FIT;
                     }
 
 qDebug() << "Octree::encodeTreeBitstreamRecursion()...";
@@ -1518,7 +1559,7 @@ qDebug() << "    packetData->getUncompressedSize()=" << packetData->getUncompres
     if (continueThisLevel && params.includeExistsBits) {
         packetData->releaseReservedBytes(sizeof(childrenExistInTreeBits));
         continueThisLevel = packetData->appendBitMask(childrenExistInTreeBits);
-qDebug() << " packetData->appendBitMask() line:" << __LINE__;
+qDebug() << " packetData->appendBitMask(childrenExistInTreeBits) line:" << __LINE__;
 qDebug() << "    continueThisLevel=" << continueThisLevel;
         if (continueThisLevel) {
             bytesAtThisLevel += sizeof(childrenExistInTreeBits); // keep track of byte count
@@ -1532,7 +1573,7 @@ qDebug() << "    continueThisLevel=" << continueThisLevel;
     if (continueThisLevel) {
         packetData->releaseReservedBytes(sizeof(childrenExistInPacketBits));
         continueThisLevel = packetData->appendBitMask(childrenExistInPacketBits);
-qDebug() << " packetData->appendBitMask() line:" << __LINE__;
+qDebug() << " packetData->appendBitMask(childrenExistInPacketBits) line:" << __LINE__;
 qDebug() << "    continueThisLevel=" << continueThisLevel;
         if (continueThisLevel) {
             bytesAtThisLevel += sizeof(childrenExistInPacketBits); // keep track of byte count
@@ -1547,6 +1588,8 @@ qDebug() << "    continueThisLevel=" << continueThisLevel;
 
 qDebug() << " --- ABOUT TO DIG DEEPER --- line:" << __LINE__;
 qDebug() << "    continueThisLevel=" << continueThisLevel;
+qDebug() << "    packetData->getUncompressedSize()=" << packetData->getUncompressedSize() << "line:" << __LINE__;
+qDebug() << "    packetData->getReservedBytes()=" << packetData->getReservedBytes();
 
     if (continueThisLevel && keepDiggingDeeper) {
 
@@ -1599,11 +1642,27 @@ qDebug() << "    continueThisLevel=" << continueThisLevel;
                 // elements. In this case, if we stop recursion when we include any data (the colorbits should really be
                 // called databits), then we wouldn't send the children. So those types of Octree's should tell us to keep
                 // recursing, by returning TRUE in recurseChildrenWithData().
+
+
+qDebug() << "--- BEFORE encodeTreeBitstreamRecursion(childElement) --- line:" << __LINE__;
+qDebug() << "    packetData->getUncompressedSize()=" << packetData->getUncompressedSize() << "line:" << __LINE__;
+qDebug() << "    packetData->getReservedBytes()=" << packetData->getReservedBytes();
+qDebug() << "    recurseChildrenWithData()=" << recurseChildrenWithData();
+qDebug() << "    params.viewFrustum=" << params.viewFrustum;
+qDebug() << "    oneAtBit(childrenDataBits, originalIndex)=" << oneAtBit(childrenDataBits, originalIndex);
+
                 if (recurseChildrenWithData() || !params.viewFrustum || !oneAtBit(childrenDataBits, originalIndex)) {
+                
+qDebug() << "    CALLING RECURSION....";
+                
                     childTreeBytesOut = encodeTreeBitstreamRecursion(childElement, packetData, bag, params,
                                                                             thisLevel, nodeLocationThisView);
                 }
 
+qDebug() << "--- AFTER encodeTreeBitstreamRecursion(childElement) --- line:" << __LINE__;
+qDebug() << "    packetData->getUncompressedSize()=" << packetData->getUncompressedSize() << "line:" << __LINE__;
+qDebug() << "    packetData->getReservedBytes()=" << packetData->getReservedBytes();
+
                 // remember this for reshuffling
                 recursiveSliceSizes[originalIndex] = childTreeBytesOut;
                 allSlicesSize += childTreeBytesOut;
@@ -1625,7 +1684,20 @@ qDebug() << "    continueThisLevel=" << continueThisLevel;
                 //
                 // we can make this act like no bytes out, by just resetting the bytes out in this case
                 if (params.includeColor && !params.includeExistsBits && childTreeBytesOut == 2) {
+                
+                    // TODO: this might be wrong for non-voxel cases... we might want to add a virtual function
+                    // to override this case... can it happen with entities? and if it happens is is correct to
+                    // remove it????
+
+qDebug() << "    >>>>>>>>>>> SPECIAL CASE FOR EMPTY TREES <<<<<<<<<<<<<< line:" << __LINE__;
+qDebug() << "        params.includeColor=" << params.includeColor;
+qDebug() << "        params.includeExistsBits=" << params.includeExistsBits;
+qDebug() << "        childTreeBytesOut=" << childTreeBytesOut;
+
                     childTreeBytesOut = 0; // this is the degenerate case of a tree with no colors and no child trees
+
+qDebug() << "        SETTING.... childTreeBytesOut=" << childTreeBytesOut;
+
                 }
                 // We used to try to collapse trees that didn't contain any data, but this does appear to create a problem
                 // in detecting element deletion. So, I've commented this out but left it in here as a warning to anyone else
@@ -1640,20 +1712,29 @@ qDebug() << "    continueThisLevel=" << continueThisLevel;
                 // If we had previously started writing, and if the child DIDN'T write any bytes,
                 // then we want to remove their bit from the childExistsPlaceHolder bitmask
                 if (childTreeBytesOut == 0) {
+
                     // remove this child's bit...
+qDebug() << " AFTER CHILD TREE RECURSION --- removing child bits --- line:" << __LINE__;
+qDebug() << "    OLD childrenExistInPacketBits=" << childrenExistInPacketBits;
                     childrenExistInPacketBits -= (1 << (7 - originalIndex));
+qDebug() << "    NEW childrenExistInPacketBits=" << childrenExistInPacketBits;
 
                     // repair the child exists mask
                     continueThisLevel = packetData->updatePriorBitMask(childExistsPlaceHolder, childrenExistInPacketBits);
-qDebug() << " packetData->updatePriorBitMask() line:" << __LINE__;
+qDebug() << " packetData->updatePriorBitMask(childExistsPlaceHolder, childrenExistInPacketBits) line:" << __LINE__;
 qDebug() << "    continueThisLevel=" << continueThisLevel;
 
+
                     // If this is the last of the child exists bits, then we're actually be rolling out the entire tree
                     if (params.stats && childrenExistInPacketBits == 0) {
                         params.stats->childBitsRemoved(params.includeExistsBits, params.includeColor);
                     }
 
                     if (!continueThisLevel) {
+qDebug() << "    WARNING ************************************************* line:" << __LINE__;
+qDebug() << "       breaking the child recursion loop with continueThisLevel=false!!!";
+qDebug() << "       AFTER attempting to updatePriorBitMask() for empty sub tree....";
+qDebug() << "       IS THIS ACCEPTABLE!!!!";
                         break; // can't continue...
                     }