Merge pull request #11749 from ZappoMan/fixImportLargeEntities

allow splitting of edit or add entity packets across multiple edit packets when property list is larger than MTU
This commit is contained in:
Brad Hefta-Gaub 2017-11-06 10:06:07 -08:00 committed by GitHub
commit 26c77ba72d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 71 additions and 57 deletions

View file

@ -93,27 +93,41 @@ void EntityEditPacketSender::queueEditEntityMessage(PacketType type,
QByteArray bufferOut(NLPacket::maxPayloadSize(type), 0); QByteArray bufferOut(NLPacket::maxPayloadSize(type), 0);
bool success; OctreeElement::AppendState encodeResult = OctreeElement::PARTIAL; // start the loop assuming there's more to send
auto nodeList = DependencyManager::get<NodeList>(); auto nodeList = DependencyManager::get<NodeList>();
EntityPropertyFlags didntFitProperties;
EntityItemProperties propertiesCopy = properties;
if (properties.parentIDChanged() && properties.getParentID() == AVATAR_SELF_ID) { if (properties.parentIDChanged() && properties.getParentID() == AVATAR_SELF_ID) {
EntityItemProperties propertiesCopy = properties;
const QUuid myNodeID = nodeList->getSessionUUID(); const QUuid myNodeID = nodeList->getSessionUUID();
propertiesCopy.setParentID(myNodeID); propertiesCopy.setParentID(myNodeID);
success = EntityItemProperties::encodeEntityEditPacket(type, entityItemID, propertiesCopy, bufferOut);
} else {
success = EntityItemProperties::encodeEntityEditPacket(type, entityItemID, properties, bufferOut);
} }
if (success) { EntityPropertyFlags requestedProperties = propertiesCopy.getChangedProperties();
#ifdef WANT_DEBUG
qCDebug(entities) << "calling queueOctreeEditMessage()..."; while (encodeResult == OctreeElement::PARTIAL) {
qCDebug(entities) << " id:" << entityItemID; encodeResult = EntityItemProperties::encodeEntityEditPacket(type, entityItemID, propertiesCopy, bufferOut, requestedProperties, didntFitProperties);
qCDebug(entities) << " properties:" << properties;
#endif if (encodeResult != OctreeElement::NONE) {
queueOctreeEditMessage(type, bufferOut); #ifdef WANT_DEBUG
if (type == PacketType::EntityAdd && !properties.getCertificateID().isEmpty()) { qCDebug(entities) << "calling queueOctreeEditMessage()...";
emit addingEntityWithCertificate(properties.getCertificateID(), DependencyManager::get<AddressManager>()->getPlaceName()); qCDebug(entities) << " id:" << entityItemID;
qCDebug(entities) << " properties:" << properties;
#endif
queueOctreeEditMessage(type, bufferOut);
if (type == PacketType::EntityAdd && !properties.getCertificateID().isEmpty()) {
emit addingEntityWithCertificate(properties.getCertificateID(), DependencyManager::get<AddressManager>()->getPlaceName());
}
} }
// if we still have properties to send, switch the message type to edit, and request only the packets that didn't fit
if (encodeResult != OctreeElement::COMPLETED) {
type = PacketType::EntityEdit;
requestedProperties = didntFitProperties;
}
bufferOut.resize(NLPacket::maxPayloadSize(type)); // resize our output buffer for the next packet
} }
} }

View file

@ -83,7 +83,7 @@ EntityPropertyFlags EntityItem::getEntityProperties(EncodeBitstreamParams& param
requestedProperties += PROP_ANGULAR_VELOCITY; requestedProperties += PROP_ANGULAR_VELOCITY;
requestedProperties += PROP_ACCELERATION; requestedProperties += PROP_ACCELERATION;
requestedProperties += PROP_DIMENSIONS; // NOTE: PROP_RADIUS obsolete requestedProperties += PROP_DIMENSIONS;
requestedProperties += PROP_DENSITY; requestedProperties += PROP_DENSITY;
requestedProperties += PROP_GRAVITY; requestedProperties += PROP_GRAVITY;
requestedProperties += PROP_DAMPING; requestedProperties += PROP_DAMPING;
@ -241,7 +241,7 @@ OctreeElement::AppendState EntityItem::appendEntityData(OctreePacketData* packet
APPEND_ENTITY_PROPERTY(PROP_ANGULAR_VELOCITY, getLocalAngularVelocity()); APPEND_ENTITY_PROPERTY(PROP_ANGULAR_VELOCITY, getLocalAngularVelocity());
APPEND_ENTITY_PROPERTY(PROP_ACCELERATION, getAcceleration()); APPEND_ENTITY_PROPERTY(PROP_ACCELERATION, getAcceleration());
APPEND_ENTITY_PROPERTY(PROP_DIMENSIONS, getDimensions()); // NOTE: PROP_RADIUS obsolete APPEND_ENTITY_PROPERTY(PROP_DIMENSIONS, getDimensions());
APPEND_ENTITY_PROPERTY(PROP_DENSITY, getDensity()); APPEND_ENTITY_PROPERTY(PROP_DENSITY, getDensity());
APPEND_ENTITY_PROPERTY(PROP_GRAVITY, getGravity()); APPEND_ENTITY_PROPERTY(PROP_GRAVITY, getGravity());
APPEND_ENTITY_PROPERTY(PROP_DAMPING, getDamping()); APPEND_ENTITY_PROPERTY(PROP_DAMPING, getDamping());

View file

@ -1221,8 +1221,9 @@ void EntityItemProperties::entityPropertyFlagsFromScriptValue(const QScriptValue
// //
// TODO: Implement support for script and visible properties. // TODO: Implement support for script and visible properties.
// //
bool EntityItemProperties::encodeEntityEditPacket(PacketType command, EntityItemID id, const EntityItemProperties& properties, OctreeElement::AppendState EntityItemProperties::encodeEntityEditPacket(PacketType command, EntityItemID id, const EntityItemProperties& properties,
QByteArray& buffer) { QByteArray& buffer, EntityPropertyFlags requestedProperties, EntityPropertyFlags& didntFitProperties) {
OctreePacketData ourDataPacket(false, buffer.size()); // create a packetData object to add out packet details too. OctreePacketData ourDataPacket(false, buffer.size()); // create a packetData object to add out packet details too.
OctreePacketData* packetData = &ourDataPacket; // we want a pointer to this so we can use our APPEND_ENTITY_PROPERTY macro OctreePacketData* packetData = &ourDataPacket; // we want a pointer to this so we can use our APPEND_ENTITY_PROPERTY macro
@ -1264,17 +1265,8 @@ bool EntityItemProperties::encodeEntityEditPacket(PacketType command, EntityItem
QByteArray encodedUpdateDelta = updateDeltaCoder; QByteArray encodedUpdateDelta = updateDeltaCoder;
EntityPropertyFlags propertyFlags(PROP_LAST_ITEM); EntityPropertyFlags propertyFlags(PROP_LAST_ITEM);
EntityPropertyFlags requestedProperties = properties.getChangedProperties();
EntityPropertyFlags propertiesDidntFit = requestedProperties; EntityPropertyFlags propertiesDidntFit = requestedProperties;
// TODO: we need to handle the multi-pass form of this, similar to how we handle entity data
//
// If we are being called for a subsequent pass at appendEntityData() that failed to completely encode this item,
// then our modelTreeElementExtraEncodeData should include data about which properties we need to append.
//if (modelTreeElementExtraEncodeData && modelTreeElementExtraEncodeData->includedItems.contains(getEntityItemID())) {
// requestedProperties = modelTreeElementExtraEncodeData->includedItems.value(getEntityItemID());
//}
LevelDetails entityLevel = packetData->startLevel(); LevelDetails entityLevel = packetData->startLevel();
// Last Edited quint64 always first, before any other details, which allows us easy access to adjusting this // Last Edited quint64 always first, before any other details, which allows us easy access to adjusting this
@ -1302,7 +1294,7 @@ bool EntityItemProperties::encodeEntityEditPacket(PacketType command, EntityItem
int propertyCount = 0; int propertyCount = 0;
bool headerFits = successIDFits && successTypeFits && successLastEditedFits bool headerFits = successIDFits && successTypeFits && successLastEditedFits
&& successLastUpdatedFits && successPropertyFlagsFits; && successLastUpdatedFits && successPropertyFlagsFits;
int startOfEntityItemData = packetData->getUncompressedByteOffset(); int startOfEntityItemData = packetData->getUncompressedByteOffset();
@ -1316,7 +1308,7 @@ bool EntityItemProperties::encodeEntityEditPacket(PacketType command, EntityItem
APPEND_ENTITY_PROPERTY(PROP_SIMULATION_OWNER, properties._simulationOwner.toByteArray()); APPEND_ENTITY_PROPERTY(PROP_SIMULATION_OWNER, properties._simulationOwner.toByteArray());
APPEND_ENTITY_PROPERTY(PROP_POSITION, properties.getPosition()); APPEND_ENTITY_PROPERTY(PROP_POSITION, properties.getPosition());
APPEND_ENTITY_PROPERTY(PROP_DIMENSIONS, properties.getDimensions()); // NOTE: PROP_RADIUS obsolete APPEND_ENTITY_PROPERTY(PROP_DIMENSIONS, properties.getDimensions());
APPEND_ENTITY_PROPERTY(PROP_ROTATION, properties.getRotation()); APPEND_ENTITY_PROPERTY(PROP_ROTATION, properties.getRotation());
APPEND_ENTITY_PROPERTY(PROP_DENSITY, properties.getDensity()); APPEND_ENTITY_PROPERTY(PROP_DENSITY, properties.getDensity());
APPEND_ENTITY_PROPERTY(PROP_VELOCITY, properties.getVelocity()); APPEND_ENTITY_PROPERTY(PROP_VELOCITY, properties.getVelocity());
@ -1472,6 +1464,7 @@ bool EntityItemProperties::encodeEntityEditPacket(PacketType command, EntityItem
properties.getType() == EntityTypes::Sphere) { properties.getType() == EntityTypes::Sphere) {
APPEND_ENTITY_PROPERTY(PROP_SHAPE, properties.getShape()); APPEND_ENTITY_PROPERTY(PROP_SHAPE, properties.getShape());
} }
APPEND_ENTITY_PROPERTY(PROP_NAME, properties.getName()); APPEND_ENTITY_PROPERTY(PROP_NAME, properties.getName());
APPEND_ENTITY_PROPERTY(PROP_COLLISION_SOUND_URL, properties.getCollisionSoundURL()); APPEND_ENTITY_PROPERTY(PROP_COLLISION_SOUND_URL, properties.getCollisionSoundURL());
APPEND_ENTITY_PROPERTY(PROP_ACTION_DATA, properties.getActionData()); APPEND_ENTITY_PROPERTY(PROP_ACTION_DATA, properties.getActionData());
@ -1522,12 +1515,7 @@ bool EntityItemProperties::encodeEntityEditPacket(PacketType command, EntityItem
// If any part of the model items didn't fit, then the element is considered partial // If any part of the model items didn't fit, then the element is considered partial
if (appendState != OctreeElement::COMPLETED) { if (appendState != OctreeElement::COMPLETED) {
// TODO: handle mechanism for handling partial fitting data! didntFitProperties = propertiesDidntFit;
// add this item into our list for the next appendElementData() pass
//modelTreeElementExtraEncodeData->includedItems.insert(getEntityItemID(), propertiesDidntFit);
// for now, if it's not complete, it's not successful
success = false;
} }
} }
@ -1543,11 +1531,15 @@ bool EntityItemProperties::encodeEntityEditPacket(PacketType command, EntityItem
} else { } else {
qCDebug(entities) << "ERROR - encoded edit message doesn't fit in output buffer."; qCDebug(entities) << "ERROR - encoded edit message doesn't fit in output buffer.";
success = false; success = false;
appendState = OctreeElement::NONE; // if we got here, then we didn't include the item
// maybe we should assert!!!
} }
} else { } else {
packetData->discardSubTree(); packetData->discardSubTree();
} }
return success;
return appendState;
} }
QByteArray EntityItemProperties::getPackedNormals() const { QByteArray EntityItemProperties::getPackedNormals() const {
@ -1673,7 +1665,7 @@ bool EntityItemProperties::decodeEntityEditPacket(const unsigned char* data, int
READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_SIMULATION_OWNER, QByteArray, setSimulationOwner); READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_SIMULATION_OWNER, QByteArray, setSimulationOwner);
READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_POSITION, glm::vec3, setPosition); READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_POSITION, glm::vec3, setPosition);
READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_DIMENSIONS, glm::vec3, setDimensions); // NOTE: PROP_RADIUS obsolete READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_DIMENSIONS, glm::vec3, setDimensions);
READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_ROTATION, glm::quat, setRotation); READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_ROTATION, glm::quat, setRotation);
READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_DENSITY, float, setDensity); READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_DENSITY, float, setDensity);
READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_VELOCITY, glm::vec3, setVelocity); READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_VELOCITY, glm::vec3, setVelocity);

View file

@ -262,8 +262,8 @@ public:
float getLocalRenderAlpha() const { return _localRenderAlpha; } float getLocalRenderAlpha() const { return _localRenderAlpha; }
void setLocalRenderAlpha(float value) { _localRenderAlpha = value; _localRenderAlphaChanged = true; } void setLocalRenderAlpha(float value) { _localRenderAlpha = value; _localRenderAlphaChanged = true; }
static bool encodeEntityEditPacket(PacketType command, EntityItemID id, const EntityItemProperties& properties, static OctreeElement::AppendState encodeEntityEditPacket(PacketType command, EntityItemID id, const EntityItemProperties& properties,
QByteArray& buffer); QByteArray& buffer, EntityPropertyFlags requestedProperties, EntityPropertyFlags& didntFitProperties);
static bool encodeEraseEntityMessage(const EntityItemID& entityItemID, QByteArray& buffer); static bool encodeEraseEntityMessage(const EntityItemID& entityItemID, QByteArray& buffer);

View file

@ -21,8 +21,7 @@ enum EntityPropertyList {
// these properties are supported by the EntityItem base class // these properties are supported by the EntityItem base class
PROP_VISIBLE, PROP_VISIBLE,
PROP_POSITION, PROP_POSITION,
PROP_RADIUS, // NOTE: PROP_RADIUS is obsolete and only included in old format streams PROP_DIMENSIONS,
PROP_DIMENSIONS = PROP_RADIUS,
PROP_ROTATION, PROP_ROTATION,
PROP_DENSITY, PROP_DENSITY,
PROP_VELOCITY, PROP_VELOCITY,
@ -47,13 +46,13 @@ enum EntityPropertyList {
PROP_ANGULAR_VELOCITY, PROP_ANGULAR_VELOCITY,
PROP_ANGULAR_DAMPING, PROP_ANGULAR_DAMPING,
PROP_COLLISIONLESS, PROP_COLLISIONLESS,
PROP_DYNAMIC, PROP_DYNAMIC, // 24
// property used by Light entity // property used by Light entity
PROP_IS_SPOTLIGHT, PROP_IS_SPOTLIGHT,
PROP_DIFFUSE_COLOR, PROP_DIFFUSE_COLOR,
PROP_AMBIENT_COLOR_UNUSED, PROP_AMBIENT_COLOR_UNUSED, // FIXME - No longer used, can remove and bump protocol
PROP_SPECULAR_COLOR_UNUSED, PROP_SPECULAR_COLOR_UNUSED, // FIXME - No longer used, can remove and bump protocol
PROP_INTENSITY, // Previously PROP_CONSTANT_ATTENUATION PROP_INTENSITY, // Previously PROP_CONSTANT_ATTENUATION
PROP_LINEAR_ATTENUATION_UNUSED, PROP_LINEAR_ATTENUATION_UNUSED,
PROP_QUADRATIC_ATTENUATION_UNUSED, PROP_QUADRATIC_ATTENUATION_UNUSED,
@ -61,30 +60,30 @@ enum EntityPropertyList {
PROP_CUTOFF, PROP_CUTOFF,
// available to all entities // available to all entities
PROP_LOCKED, PROP_LOCKED, // 34
PROP_TEXTURES, // used by Model entities PROP_TEXTURES, // used by Model entities
PROP_ANIMATION_SETTINGS, // used by Model entities PROP_ANIMATION_SETTINGS_UNUSED, // FIXME - No longer used, can remove and bump protocol
PROP_USER_DATA, // all entities PROP_USER_DATA, // all entities -- 37
PROP_SHAPE_TYPE, // used by Model + zones entities PROP_SHAPE_TYPE, // used by Model + zones entities
// used by ParticleEffect entities // used by ParticleEffect entities
PROP_MAX_PARTICLES, PROP_MAX_PARTICLES, // 39
PROP_LIFESPAN, PROP_LIFESPAN, // 40 -- used by all entities
PROP_EMIT_RATE, PROP_EMIT_RATE,
PROP_EMIT_SPEED, PROP_EMIT_SPEED,
PROP_EMIT_STRENGTH, PROP_EMIT_STRENGTH,
PROP_EMIT_ACCELERATION, PROP_EMIT_ACCELERATION, // FIXME - doesn't seem to get set in mark all changed????
PROP_PARTICLE_RADIUS, PROP_PARTICLE_RADIUS, // 45!!
PROP_COMPOUND_SHAPE_URL, // used by Model + zones entities PROP_COMPOUND_SHAPE_URL, // used by Model + zones entities
PROP_MARKETPLACE_ID, // all entities PROP_MARKETPLACE_ID, // all entities
PROP_ACCELERATION, // all entities PROP_ACCELERATION, // all entities
PROP_SIMULATION_OWNER, // formerly known as PROP_SIMULATOR_ID PROP_SIMULATION_OWNER, // formerly known as PROP_SIMULATOR_ID
PROP_NAME, // all entities PROP_NAME, // all entities -- 50
PROP_COLLISION_SOUND_URL, PROP_COLLISION_SOUND_URL,
PROP_RESTITUTION, PROP_RESTITUTION,
PROP_FRICTION, PROP_FRICTION, // 53
PROP_VOXEL_VOLUME_SIZE, PROP_VOXEL_VOLUME_SIZE,
PROP_VOXEL_DATA, PROP_VOXEL_DATA,
@ -96,7 +95,7 @@ enum EntityPropertyList {
// used by hyperlinks // used by hyperlinks
PROP_HREF, PROP_HREF,
PROP_DESCRIPTION, PROP_DESCRIPTION, // 61
PROP_FACE_CAMERA, PROP_FACE_CAMERA,
PROP_SCRIPT_TIMESTAMP, PROP_SCRIPT_TIMESTAMP,

View file

@ -68,8 +68,8 @@ bool OctreePacketData::append(const unsigned char* data, int length) {
_dirty = true; _dirty = true;
} }
const bool wantDebug = false; #ifdef WANT_DEBUG
if (wantDebug && !success) { if (!success) {
qCDebug(octree) << "OctreePacketData::append(const unsigned char* data, int length) FAILING...."; qCDebug(octree) << "OctreePacketData::append(const unsigned char* data, int length) FAILING....";
qCDebug(octree) << " length=" << length; qCDebug(octree) << " length=" << length;
qCDebug(octree) << " _bytesAvailable=" << _bytesAvailable; qCDebug(octree) << " _bytesAvailable=" << _bytesAvailable;
@ -77,6 +77,7 @@ bool OctreePacketData::append(const unsigned char* data, int length) {
qCDebug(octree) << " _targetSize=" << _targetSize; qCDebug(octree) << " _targetSize=" << _targetSize;
qCDebug(octree) << " _bytesReserved=" << _bytesReserved; qCDebug(octree) << " _bytesReserved=" << _bytesReserved;
} }
#endif
return success; return success;
} }
@ -647,6 +648,13 @@ void OctreePacketData::debugContent() {
printf("\n"); printf("\n");
} }
void OctreePacketData::debugBytes() {
qCDebug(octree) << " _bytesAvailable=" << _bytesAvailable;
qCDebug(octree) << " _bytesInUse=" << _bytesInUse;
qCDebug(octree) << " _targetSize=" << _targetSize;
qCDebug(octree) << " _bytesReserved=" << _bytesReserved;
}
int OctreePacketData::unpackDataFromBytes(const unsigned char* dataBytes, QString& result) { int OctreePacketData::unpackDataFromBytes(const unsigned char* dataBytes, QString& result) {
uint16_t length; uint16_t length;
memcpy(&length, dataBytes, sizeof(length)); memcpy(&length, dataBytes, sizeof(length));

View file

@ -240,6 +240,7 @@ public:
/// displays contents for debugging /// displays contents for debugging
void debugContent(); void debugContent();
void debugBytes();
static quint64 getCompressContentTime() { return _compressContentTime; } /// total time spent compressing content static quint64 getCompressContentTime() { return _compressContentTime; } /// total time spent compressing content
static quint64 getCompressContentCalls() { return _compressContentCalls; } /// total calls to compress content static quint64 getCompressContentCalls() { return _compressContentCalls; } /// total calls to compress content

View file

@ -74,7 +74,7 @@ void OctreeTests::propertyFlagsTests() {
EntityPropertyFlags props; EntityPropertyFlags props;
props.setHasProperty(PROP_VISIBLE); props.setHasProperty(PROP_VISIBLE);
props.setHasProperty(PROP_POSITION); props.setHasProperty(PROP_POSITION);
props.setHasProperty(PROP_RADIUS); props.setHasProperty(PROP_DIMENSIONS);
props.setHasProperty(PROP_MODEL_URL); props.setHasProperty(PROP_MODEL_URL);
props.setHasProperty(PROP_COMPOUND_SHAPE_URL); props.setHasProperty(PROP_COMPOUND_SHAPE_URL);
props.setHasProperty(PROP_ROTATION); props.setHasProperty(PROP_ROTATION);