Cleanup entity -> ds persist

assignment-client/CMakeLists.txt

@@ -6,7 +6,6 @@ setup_hifi_project(Core Gui Network Script Quick WebSockets)
 if (APPLE)
   set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "@executable_path/../Frameworks")
 endif ()
-set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "/testing/")
 
 setup_memory_debugger()
 

assignment-client/src/octree/OctreeServer.cpp

@@ -1044,12 +1044,13 @@ void OctreeServer::readConfiguration() {
         // If persist filename does not exist, let's see if there is one beside the application binary
         // If there is, let's copy it over to our target persist directory
         QDir persistPath { _persistFilePath };
-        _persistAbsoluteFilePath = persistPath.absolutePath();
 
         if (persistPath.isRelative()) {
             // if the domain settings passed us a relative path, make an absolute path that is relative to the
             // default data directory
             _persistAbsoluteFilePath = QDir(PathUtils::getAppDataFilePath("entities/")).absoluteFilePath(_persistFilePath);
+        } else {
+            _persistAbsoluteFilePath = persistPath.absolutePath();
         }
 
         qDebug() << "persistFilePath=" << _persistFilePath;
@@ -1174,6 +1175,11 @@ void OctreeServer::domainSettingsRequestComplete() {
 } 
 
 void OctreeServer::handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message) {
+    if (_state != OctreeServerState::WaitingForOctreeDataNegotation) {
+        qCWarning(octree_server) << "Server received ocree data file reply but is not currently negotiating.";
+        return;
+    }
+
     bool includesNewData;
     message->readPrimitive(&includesNewData);
     QByteArray replaceData;
@@ -1188,8 +1194,7 @@ void OctreeServer::handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> mes
         if (OctreeUtils::readOctreeDataInfoFromFile(_persistAbsoluteFilePath, &data)) {
             if (data.id.isNull()) {
                 qCDebug(octree_server) << "Current octree data has a null id, updating";
-                data.id = QUuid::createUuid();
+                data.resetIdAndVersion();
-                data.version = 0;
 
                 QFile file(_persistAbsoluteFilePath);
                 if (file.open(QIODevice::WriteOnly)) {
@@ -1202,17 +1207,17 @@ void OctreeServer::handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> mes
             }
         }
     }
+
+    _state = OctreeServerState::Running;
     beginRunning(replaceData);
 }
 
 void OctreeServer::beginRunning(QByteArray replaceData) {
-    if (_state == OctreeServerState::Running) {
+    if (_state != OctreeServerState::Running) {
-        qCWarning(octree_server) << "Server is already running";
+        qCWarning(octree_server) << "Server is not running";
         return;
     }
 
-    _state = OctreeServerState::Running;
-
     auto nodeList = DependencyManager::get<NodeList>();
 
     // we need to ask the DS about agents so we can ping/reply with them

assignment-client/src/octree/OctreeServer.h

@@ -149,8 +149,6 @@ private slots:
     void domainSettingsRequestComplete();
     void handleOctreeQueryPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleOctreeDataNackPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
-    //void handleOctreeFileReplacement(QSharedPointer<ReceivedMessage> message);
-    //void handleOctreeFileReplacementFromURL(QSharedPointer<ReceivedMessage> message);
     void handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message);
     void removeSendThread();
 

domain-server/src/DomainContentBackupManager.cpp

@@ -279,23 +279,9 @@ void DomainContentBackupManager::backup() {
 
             qDebug() << "Created backup: " << fileName;
 
-            removeOldBackupVersions(rule);
+            rule.lastBackupSeconds = nowSeconds;
 
-            if (rule.maxBackupVersions > 0) {
+            removeOldBackupVersions(rule);
-                // Execute backup
-                auto result = true;
-                if (result) {
-                    qCDebug(domain_server) << "DONE backing up persist file...";
-                    rule.lastBackupSeconds = nowSeconds;
-                } else {
-                    qCDebug(domain_server) << "ERROR in backing up persist file...";
-                    perror("ERROR in backing up persist file");
-                }
-            } else {
-                qCDebug(domain_server) << "This backup rule" << rule.name << " has Max Rolled Backup Versions less than 1 ["
-                                        << rule.maxBackupVersions << "]."
-                                        << " There are no backups to be done...";
-            }
         } else {
             qCDebug(domain_server) << "Backup not needed for this rule [" << rule.name << "]...";
         }

domain-server/src/DomainServer.cpp

@@ -289,7 +289,6 @@ DomainServer::DomainServer(int argc, char* argv[]) :
         }
     }
 
-    qDebug() << "Starting persist thread";
     if (QDir(getEntitiesDirPath()).mkpath(".")) {
         qCDebug(domain_server) << "Created entities data directory";
     }
@@ -1785,7 +1784,8 @@ void DomainServer::processOctreeDataRequestMessage(QSharedPointer<ReceivedMessag
     int version;
     message->readPrimitive(&remoteHasExistingData);
     if (remoteHasExistingData) {
-        auto idData = message->read(16);
+        constexpr size_t UUID_SIZE_BYTES = 16;
+        auto idData = message->read(UUID_SIZE_BYTES);
         id = QUuid::fromRfc4122(idData);
         message->readPrimitive(&version);
         qCDebug(domain_server) << "Entity server does have existing data: ID(" << id << ") DataVersion(" << version << ")";
@@ -1794,7 +1794,6 @@ void DomainServer::processOctreeDataRequestMessage(QSharedPointer<ReceivedMessag
     }
     auto entityFilePath = getEntitiesFilePath();
 
-    //QFile file(entityFilePath);
     auto reply = NLPacketList::create(PacketType::OctreeDataFileReply, QByteArray(), true, true);
     OctreeUtils::RawOctreeData data;
     if (OctreeUtils::readOctreeDataInfoFromFile(entityFilePath, &data)) {
@@ -3228,21 +3227,22 @@ void DomainServer::setupGroupCacheRefresh() {
 }
 
 void DomainServer::maybeHandleReplacementEntityFile() {
-    QFile replacementFile(getEntitiesReplacementFilePath());
+    const auto replacementFilePath = getEntitiesReplacementFilePath();
-    if (replacementFile.exists()) {
+    OctreeUtils::RawOctreeData data;
+    if (!OctreeUtils::readOctreeDataInfoFromFile(replacementFilePath, &data)) {
+        qCWarning(domain_server) << "Replacement file could not be read, it either doesn't exist or is invalid.";
+    } else {
         qCDebug(domain_server) << "Replacing existing entity date with replacement file";
+
+        data.resetIdAndVersion();
+        auto gzippedData = data.toGzippedByteArray();
+
         QFile currentFile(getEntitiesFilePath());
-        if (currentFile.exists()) {
+        if (!currentFile.open(QIODevice::WriteOnly)) {
-            if (currentFile.remove()) {
+            qCWarning(domain_server)
-                qCDebug(domain_server) << "Removed existing entity file";
+                << "Failed to update entities data file with replacement file, unable to open entities file for writing";
-            } else {
-                qCWarning(domain_server) << "Failled to remove existing entity file";
-            }
-        }
-        if (replacementFile.rename(getEntitiesFilePath())) {
-            qCDebug(domain_server) << "Successfully updated entities data file with replacement file";
         } else {
-            qCWarning(domain_server) << "Failed to update entities data file with replacement file";
+            currentFile.write(gzippedData);
         }
     }
 }
@@ -3262,8 +3262,7 @@ void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) {
 
     OctreeUtils::RawOctreeData data;
     if (OctreeUtils::readOctreeDataInfoFromData(jsonOctree, &data)) {
-        data.id = QUuid::createUuid();
+        data.resetIdAndVersion();
-        data.version = 0;
 
         gzip(data.toByteArray(), compressedOctree);
 
@@ -3282,21 +3281,6 @@ void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) {
     } else {
         qDebug() << "Received replacement octree file that is invalid - refusing to process";
     }
-
-
-    return;
-    auto limitedNodeList = DependencyManager::get<LimitedNodeList>();
-    limitedNodeList->eachMatchingNode([](const SharedNodePointer& node) {
-        return node->getType() == NodeType::EntityServer && node->getActiveSocket();
-    }, [&octreeFile, limitedNodeList](const SharedNodePointer& octreeNode) {
-        // setup a packet to send to this octree server with the new octree file data
-        auto octreeFilePacketList = NLPacketList::create(PacketType::OctreeFileReplacement, QByteArray(), true, true);
-        octreeFilePacketList->write(octreeFile);
-
-        qDebug() << "Sending an octree file replacement of" << octreeFile.size() << "bytes to" << octreeNode;
-
-        limitedNodeList->sendPacketList(std::move(octreeFilePacketList), *octreeNode);
-    });
 }
 
 void DomainServer::handleOctreeFileReplacementFromURLRequest(QSharedPointer<ReceivedMessage> message) {

interface/src/Application.cpp

@@ -6247,7 +6247,6 @@ bool Application::askToReplaceDomainContent(const QString& url) {
                         }, [&urlData, limitedNodeList, &domainHandler](const SharedNodePointer& octreeNode) {
                             auto octreeFilePacket = NLPacket::create(PacketType::OctreeFileReplacementFromUrl, urlData.size(), true);
                             octreeFilePacket->write(urlData);
-                            qDebug() << "WRiting url data: " << urlData;
                             limitedNodeList->sendPacket(std::move(octreeFilePacket), domainHandler.getSockAddr());
                         });
                     auto addressManager = DependencyManager::get<AddressManager>();

libraries/entities/src/EntityTree.cpp

@@ -2244,7 +2244,7 @@ bool EntityTree::writeToMap(QVariantMap& entityDescription, OctreeElementPointer
     if (! entityDescription.contains("Entities")) {
         entityDescription["Entities"] = QVariantList();
     }
-    entityDescription["DataVersion"] = ++_persistDataVersion;
+    entityDescription["DataVersion"] = _persistDataVersion;
     entityDescription["Id"] = _persistID;
     QScriptEngine scriptEngine;
     RecurseOctreeToMapOperator theOperator(entityDescription, element, &scriptEngine, skipDefaultValues,

libraries/octree/CMakeLists.txt

@@ -1,4 +1,3 @@
 set(TARGET_NAME octree)
-include_directories(system /usr/local/Cellar/qt5/5.9.1/include)
 setup_hifi_library()
 link_hifi_libraries(shared networking)

libraries/octree/src/Octree.cpp

@@ -1757,19 +1757,6 @@ bool Octree::readJSONFromStream(uint64_t streamLength, QDataStream& inputStream,
     QVariant asVariant = asDocument.toVariant();
     QVariantMap asMap = asVariant.toMap();
     bool success = readFromMap(asMap);
-    /*
-    if (success) {
-        if (asMap.contains("DataVersion") && asMap.contains("Id")) {
-            bool versionOk;
-            auto dataVersion = asMap["DataVersion"].toLongLong(&versionOk);
-            if (versionOk) {
-                auto id = asMap["Id"].toUuid();
-                _persistDataVersion = dataVersion;
-                _persistID = id;
-            }
-        }
-    }
-    */
     delete[] rawData;
     return success;
 }

libraries/octree/src/Octree.h

@@ -341,6 +341,8 @@ public:
     virtual quint64 getAverageLoggingTime() const { return 0;  }
     virtual quint64 getAverageFilterTime() const { return 0; }
 
+    void incrementPersistDataVersion() { _persistDataVersion++; }
+
 signals:
     void importSize(float x, float y, float z);
     void importProgress(int progress);

libraries/octree/src/OctreePersistThread.cpp

@@ -311,6 +311,7 @@ void OctreePersistThread::persist() {
         backup(); // handle backup if requested        
         qCDebug(octree) << "persist operation DONE with backup...";
 
+        _tree->incrementPersistDataVersion();
 
         // create our "lock" file to indicate we're saving.
         QString lockFileName = _filename + ".lock";

libraries/octree/src/OctreeUtils.cpp

@@ -80,6 +80,9 @@ float getOrthographicAccuracySize(float octreeSizeScale, int boundaryLevelAdjust
     return (smallestSize * MAX_VISIBILITY_DISTANCE_FOR_UNIT_ELEMENT) / boundaryDistanceForRenderLevel(boundaryLevelAdjust, octreeSizeScale);
 }
 
+// Reads octree file and parses it into a QJsonDocument. Handles both gzipped and non-gzipped files.
+// Returns true if the file was successfully opened and parsed, otherwise false.
+// Example failures: file does not exist, gzipped file cannot be unzipped, invalid JSON.
 bool OctreeUtils::readOctreeFile(QString path, QJsonDocument* doc) {
     QFile file(path);
     if (!file.open(QIODevice::ReadOnly)) {
@@ -129,6 +132,8 @@ bool OctreeUtils::readOctreeDataInfoFromData(QByteArray data, OctreeUtils::RawOc
     return readOctreeDataInfoFromJSON(root, octreeData);
 }
 
+// Reads octree file and parses it into a RawOctreeData object.
+// Returns false if readOctreeFile fails.
 bool OctreeUtils::readOctreeDataInfoFromFile(QString path, OctreeUtils::RawOctreeData* octreeData) {
     QJsonDocument doc;
     if (!OctreeUtils::readOctreeFile(path, &doc)) {
@@ -164,3 +169,8 @@ QByteArray OctreeUtils::RawOctreeData::toGzippedByteArray() {
 
     return gzData;
 }
+
+void OctreeUtils::RawOctreeData::resetIdAndVersion() {
+    id = QUuid::createUuid();
+    version = OctreeUtils::INITIAL_VERSION;
+}

libraries/octree/src/OctreeUtils.h

@@ -22,14 +22,18 @@ class QJsonDocument;
 
 namespace OctreeUtils {
 
+using Version = int64_t;
+constexpr Version INITIAL_VERSION = 0;
+
 // RawOctreeData is an intermediate format between JSON and a fully deserialized Octree.
 class RawOctreeData {
 public:
     QUuid id { QUuid() };
-    int64_t version { -1 };
+    Version version { -1 };
 
     QJsonArray octreeData;
 
+    void resetIdAndVersion();
     QByteArray toByteArray();
     QByteArray toGzippedByteArray();
 };