add the basic structure for inclusion of ancestors/descendants in ESS queries

assignment-client/src/entities/EntityServer.cpp

@@ -17,10 +17,11 @@
 #include <ScriptCache.h>
 #include <EntityEditFilters.h>
 
+#include "AssignmentParentFinder.h"
+#include "EntityNodeData.h"
 #include "EntityServer.h"
 #include "EntityServerConsts.h"
-#include "EntityNodeData.h"
-#include "AssignmentParentFinder.h"
+#include "EntityTreeSendThread.h"
 
 const char* MODEL_SERVER_NAME = "Entity";
 const char* MODEL_SERVER_LOGGING_TARGET_NAME = "entity-server";
@@ -77,6 +78,10 @@ OctreePointer EntityServer::createTree() {
     return tree;
 }
 
+OctreeServer::UniqueSendThread EntityServer::newSendThread(const SharedNodePointer& node) {
+    return std::unique_ptr<EntityTreeSendThread>(new EntityTreeSendThread(this, node));
+}
+
 void EntityServer::beforeRun() {
     _pruneDeletedEntitiesTimer = new QTimer();
     connect(_pruneDeletedEntitiesTimer, SIGNAL(timeout()), this, SLOT(pruneDeletedEntities()));

assignment-client/src/entities/EntityServer.h

@@ -67,6 +67,7 @@ public slots:
 
 protected:
     virtual OctreePointer createTree() override;
+    virtual UniqueSendThread newSendThread(const SharedNodePointer& node) override;
 
 private slots:
     void handleEntityPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);

assignment-client/src/entities/EntityTreeSendThread.cpp

@@ -0,0 +1,107 @@
+//
+//  EntityTreeSendThread.cpp
+//  assignment-client/src/entities
+//
+//  Created by Stephen Birarda on 2/15/17.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "EntityTreeSendThread.h"
+
+#include <EntityNodeData.h>
+#include <EntityTypes.h>
+
+#include "EntityServer.h"
+
+void EntityTreeSendThread::preDistributionProcessing() {
+    auto node = _node.toStrongRef();
+    auto nodeData = static_cast<EntityNodeData*>(node->getLinkedData());
+
+    if (nodeData) {
+        auto jsonQuery = nodeData->getJSONParameters();
+
+        // check if we have a JSON query with flags
+        auto flags = jsonQuery[EntityJSONQueryProperties::FLAGS_PROPERTY].toObject();
+        if (!flags.isEmpty()) {
+            // check the flags object for specific flags that require special pre-processing
+
+            bool includeAncestors = flags[EntityJSONQueryProperties::INCLUDE_ANCESTORS_PROPERTY].toBool();
+            bool includeDescendants = flags[EntityJSONQueryProperties::INCLUDE_DESCENDANTS_PROPERTY].toBool();
+
+            if (includeAncestors || includeDescendants) {
+                // we need to either include the ancestors, descendants, or both for entities matching the filter
+                // included in the JSON query
+
+                auto entityTree = std::static_pointer_cast<EntityTree>(_myServer->getOctree());
+
+                // enumerate the set of entity IDs we know currently match the filter
+                foreach(const QUuid& entityID, nodeData->getSentFilteredEntities()) {
+                    if (includeAncestors) {
+                        // we need to include ancestors - recurse up to reach them all and add their IDs
+                        // to the set of extra entities to include for this node
+                        entityTree->withReadLock([&]{
+                            auto filteredEntity = entityTree->findEntityByID(entityID);
+                            if (filteredEntity) {
+                                addAncestorsToExtraFlaggedEntities(entityID, *filteredEntity, *nodeData);
+                            }
+                        });
+                    }
+
+                    if (includeDescendants) {
+                        // we need to include descendants - recurse down to reach them all and add their IDs
+                        // to the set of extra entities to include for this node
+                        entityTree->withReadLock([&]{
+                            auto filteredEntity = entityTree->findEntityByID(entityID);
+                            if (filteredEntity) {
+                                addDescendantsToExtraFlaggedEntities(entityID, *filteredEntity, *nodeData);
+                            }
+                        });
+                    }
+                }
+            }
+
+        }
+    }
+}
+
+void EntityTreeSendThread::addAncestorsToExtraFlaggedEntities(const QUuid& filteredEntityID,
+                                                              EntityItem& entityItem, EntityNodeData& nodeData) {
+    // check if this entity has a parent that is also an entity
+    bool success = false;
+    auto entityParent = entityItem.getParentPointer(success);
+
+    if (success && entityParent && entityParent->getNestableType() == NestableType::Entity) {
+        // we found a parent that is an entity item
+
+        // first add it to the extra list of things we need to send
+        nodeData.insertFlaggedExtraEntity(filteredEntityID, entityParent->getID());
+
+//        qDebug() << "Adding" << entityParent->getID() << "which is an ancestor of" << filteredEntityID;
+
+        // now recursively call ourselves to get its ancestors added too
+        auto parentEntityItem = std::static_pointer_cast<EntityItem>(entityParent);
+        addAncestorsToExtraFlaggedEntities(filteredEntityID, *parentEntityItem, nodeData);
+    }
+}
+
+void EntityTreeSendThread::addDescendantsToExtraFlaggedEntities(const QUuid& filteredEntityID,
+                                                                EntityItem& entityItem, EntityNodeData& nodeData) {
+    // enumerate the immediate children of this entity
+    foreach (SpatiallyNestablePointer child, entityItem.getChildren()) {
+        if (child && child->getNestableType() == NestableType::Entity) {
+            // this is a child that is an entity
+
+            // first add it to the extra list of things we need to send
+            nodeData.insertFlaggedExtraEntity(filteredEntityID, child->getID());
+
+//            qDebug() << "Adding" << child->getID() <<  "which is a descendant of" << filteredEntityID;
+
+            // now recursively call ourselves to get its descendants added too
+            auto childEntityItem = std::static_pointer_cast<EntityItem>(child);
+            addDescendantsToExtraFlaggedEntities(filteredEntityID, *childEntityItem, nodeData);
+        }
+    }
+}

assignment-client/src/entities/EntityTreeSendThread.h

@@ -0,0 +1,34 @@
+//
+//  EntityTreeSendThread.h
+//  assignment-client/src/entities
+//
+//  Created by Stephen Birarda on 2/15/17.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_EntityTreeSendThread_h
+#define hifi_EntityTreeSendThread_h
+
+#include "../octree/OctreeSendThread.h"
+
+class EntityNodeData;
+class EntityItem;
+
+class EntityTreeSendThread : public OctreeSendThread {
+
+public:
+    EntityTreeSendThread(OctreeServer* myServer, const SharedNodePointer& node) : OctreeSendThread(myServer, node) {};
+
+protected:
+    virtual void preDistributionProcessing() override;
+
+private:
+    void addAncestorsToExtraFlaggedEntities(const QUuid& filteredEntityID, EntityItem& entityItem, EntityNodeData& nodeData);
+    void addDescendantsToExtraFlaggedEntities(const QUuid& filteredEntityID, EntityItem& entityItem, EntityNodeData& nodeData);
+
+};
+
+#endif // hifi_EntityTreeSendThread_h

assignment-client/src/octree/OctreeSendThread.cpp

@@ -309,6 +309,9 @@ int OctreeSendThread::packetDistributor(SharedNodePointer node, OctreeQueryNode*
         return 0;
     }
 
+    // give our pre-distribution processing a chance to do what it needs
+    preDistributionProcessing();
+
     // calculate max number of packets that can be sent during this interval
     int clientMaxPacketsPerInterval = std::max(1, (nodeData->getMaxQueryPacketsPerSecond() / INTERVALS_PER_SECOND));
     int maxPacketsPerInterval = std::min(clientMaxPacketsPerInterval, _myServer->getPacketsPerClientPerInterval());

assignment-client/src/octree/OctreeSendThread.h

@@ -17,6 +17,8 @@
 #include <atomic>
 
 #include <GenericThread.h>
+#include <Node.h>
+#include <OctreePacketData.h>
 
 class OctreeQueryNode;
 class OctreeServer;
@@ -49,13 +51,17 @@ protected:
     /// Implements generic processing behavior for this thread.
     virtual bool process() override;
 
+    /// Called before a packetDistributor pass to allow for pre-distribution processing
+    virtual void preDistributionProcessing() {};
+
+    OctreeServer* _myServer { nullptr };
+    QWeakPointer<Node> _node;
+
 private:
     int handlePacketSend(SharedNodePointer node, OctreeQueryNode* nodeData, int& trueBytesSent, int& truePacketsSent, bool dontSuppressDuplicate = false);
     int packetDistributor(SharedNodePointer node, OctreeQueryNode* nodeData, bool viewFrustumChanged);
     
-    
-    OctreeServer* _myServer { nullptr };
-    QWeakPointer<Node> _node;
+
     QUuid _nodeUuid;
 
     OctreePacketData _packetData;

assignment-client/src/octree/OctreeServer.cpp

@@ -872,8 +872,12 @@ void OctreeServer::parsePayload() {
     }
 }
 
+OctreeServer::UniqueSendThread OctreeServer::newSendThread(const SharedNodePointer& node) {
+    return std::unique_ptr<OctreeSendThread>(new OctreeSendThread(this, node));
+}
+
 OctreeServer::UniqueSendThread OctreeServer::createSendThread(const SharedNodePointer& node) {
-    auto sendThread = std::unique_ptr<OctreeSendThread>(new OctreeSendThread(this, node));
+    auto sendThread = newSendThread(node);
     
     // we want to be notified when the thread finishes
     connect(sendThread.get(), &GenericThread::finished, this, &OctreeServer::removeSendThread);

assignment-client/src/octree/OctreeServer.h

@@ -158,6 +158,7 @@ protected:
     QString getStatusLink();
     
     UniqueSendThread createSendThread(const SharedNodePointer& node);
+    virtual UniqueSendThread newSendThread(const SharedNodePointer& node);
 
     int _argc;
     const char** _argv;

assignment-client/src/scripts/EntityScriptServer.cpp

@@ -16,6 +16,7 @@
 #include <AudioConstants.h>
 #include <AudioInjectorManager.h>
 #include <ClientServerUtils.h>
+#include <EntityNodeData.h>
 #include <EntityScriptingInterface.h>
 #include <LogHandler.h>
 #include <MessagesClient.h>
@@ -264,16 +265,14 @@ void EntityScriptServer::run() {
     
     // setup the JSON filter that asks for entities with a non-default serverScripts property
     QJsonObject queryJSONParameters;
-    static const QString SERVER_SCRIPTS_PROPERTY = "serverScripts";
-    queryJSONParameters[SERVER_SCRIPTS_PROPERTY] = EntityQueryFilterSymbol::NonDefault;
+    queryJSONParameters[EntityJSONQueryProperties::SERVER_SCRIPTS_PROPERTY] = EntityQueryFilterSymbol::NonDefault;
 
     QJsonObject queryFlags;
-    static const QString INCLUDE_DESCENDANTS_PROPERTY = "includeDescendants";
-    static const QString INCLUDE_PARENTS_PROPERTY = "includeParents";
-    queryFlags[INCLUDE_DESCENDANTS_PROPERTY] = true;
-    queryFlags[INCLUDE_PARENTS_PROPERTY] = true;
 
-    queryJSONParameters["flags"] = queryFlags;
+    queryFlags[EntityJSONQueryProperties::INCLUDE_ANCESTORS_PROPERTY] = true;
+    queryFlags[EntityJSONQueryProperties::INCLUDE_DESCENDANTS_PROPERTY] = true;
+
+    queryJSONParameters[EntityJSONQueryProperties::FLAGS_PROPERTY] = queryFlags;
     
     // setup the JSON parameters so that OctreeQuery does not use a frustum and uses our JSON filter
     _entityViewer.getOctreeQuery().setUsesFrustum(false);

libraries/entities/src/EntityNodeData.cpp

@@ -0,0 +1,25 @@
+//
+//  EntityNodeData.cpp
+//  libraries/entities/src
+//
+//  Created by Stephen Birarda on 2/15/17
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "EntityNodeData.h"
+
+bool EntityNodeData::isEntityFlaggedAsExtra(const QUuid& entityID) const {
+
+    // enumerate each of the sets for the entities that matched our filter
+    // and immediately return true if any of them contain this entity ID
+    foreach(QSet<QUuid> entitySet, _flaggedExtraEntities) {
+        if (entitySet.contains(entityID)) {
+            return true;
+        }
+    }
+
+    return false;
+}

libraries/entities/src/EntityNodeData.h

@@ -1,6 +1,6 @@
 //
 //  EntityNodeData.h
-//  assignment-client/src/entities
+//  libraries/entities/src
 //
 //  Created by Brad Hefta-Gaub on 4/29/14
 //  Copyright 2014 High Fidelity, Inc.
@@ -16,6 +16,13 @@
 
 #include <OctreeQueryNode.h>
 
+namespace EntityJSONQueryProperties {
+    static const QString SERVER_SCRIPTS_PROPERTY = "serverScripts";
+    static const QString FLAGS_PROPERTY = "flags";
+    static const QString INCLUDE_ANCESTORS_PROPERTY = "includeAncestors";
+    static const QString INCLUDE_DESCENDANTS_PROPERTY = "includeDescendants";
+}
+
 class EntityNodeData : public OctreeQueryNode {
 public:
     virtual PacketType getMyPacketType() const override { return PacketType::EntityData; }
@@ -24,13 +31,20 @@ public:
     void setLastDeletedEntitiesSentAt(quint64 sentAt) { _lastDeletedEntitiesSentAt = sentAt; }
     
     // these can only be called from the OctreeSendThread for the given Node
-    void insertEntitySentLastFrame(const QUuid& entityID) { _entitiesSentLastFrame.insert(entityID); }
-    void removeEntitySentLastFrame(const QUuid& entityID) { _entitiesSentLastFrame.remove(entityID); }
-    bool sentEntityLastFrame(const QUuid& entityID) { return _entitiesSentLastFrame.contains(entityID); }
+    void insertSentFilteredEntity(const QUuid& entityID) { _sentFilteredEntities.insert(entityID); }
+    void removeSentFilteredEntity(const QUuid& entityID) { _sentFilteredEntities.remove(entityID); }
+    bool sentFilteredEntity(const QUuid& entityID) { return _sentFilteredEntities.contains(entityID); }
+    QSet<QUuid> getSentFilteredEntities() { return _sentFilteredEntities; }
+
+    // these can only be called from the OctreeSendThread for the given Node
+    void insertFlaggedExtraEntity(const QUuid& filteredEntityID, const QUuid& extraEntityID)
+        { _flaggedExtraEntities[filteredEntityID].insert(extraEntityID); }
+    bool isEntityFlaggedAsExtra(const QUuid& entityID) const;
 
 private:
     quint64 _lastDeletedEntitiesSentAt { usecTimestampNow() };
-    QSet<QUuid> _entitiesSentLastFrame;
+    QSet<QUuid> _sentFilteredEntities;
+    QHash<QUuid, QSet<QUuid>> _flaggedExtraEntities;
 };
 
 #endif // hifi_EntityNodeData_h

libraries/entities/src/EntityTreeElement.cpp

@@ -310,16 +310,17 @@ OctreeElement::AppendState EntityTreeElement::appendElementData(OctreePacketData
 
                     if (entityMatchesFilters) {
                         // make sure this entity is in the set of entities sent last frame
-                        entityNodeData->insertEntitySentLastFrame(entity->getID());
-
-                    } else {
-                        // we might include this entity if it matched in the previous frame
-                        if (entityNodeData->sentEntityLastFrame(entity->getID())) {
-
-                            entityNodeData->removeEntitySentLastFrame(entity->getID());
-                        } else {
-                            includeThisEntity = false;
-                        }
+                        entityNodeData->insertSentFilteredEntity(entity->getID());
+                    } else if (entityNodeData->sentFilteredEntity(entity->getID())) {
+                        // this entity matched in the previous frame - we send it still so the client realizes it just
+                        // fell outside of their filter
+                        entityNodeData->removeSentFilteredEntity(entity->getID());
+                    } else if (!entityNodeData->isEntityFlaggedAsExtra(entity->getID())) {
+                        // we don't send this entity because
+                        // (1) it didn't match our filter
+                        // (2) it didn't match our filter last frame
+                        // (3) it isn't one the JSON query flags told us we should still include
+                        includeThisEntity = false;
                     }
                 }