added persist-as-json checkbox to domain settings. File extension of persist file is automatically adjusted when entity server saves. When loading, if both a json and svo file exist, the newer one is used. This is regardless of the persistAsJson setting

assignment-client/src/octree/OctreeServer.cpp

@@ -1036,6 +1036,13 @@ void OctreeServer::readConfiguration() {
         strcpy(_persistFilename, qPrintable(persistFilename));
         qDebug("persistFilename=%s", _persistFilename);
 
+        bool persistAsJson;
+        if (!readOptionBool(QString("persistAsJson"), settingsSectionObject, persistAsJson)) {
+            persistAsJson = false;
+        }
+        _persistAsJson = persistAsJson;
+        qDebug() << "persistAsJson=" << _persistAsJson;
+
         _persistInterval = OctreePersistThread::DEFAULT_PERSIST_INTERVAL;
         readOptionInt(QString("persistInterval"), settingsSectionObject, _persistInterval);
         qDebug() << "persistInterval=" << _persistInterval;
@@ -1131,7 +1138,7 @@ void OctreeServer::run() {
 
         // now set up PersistThread
         _persistThread = new OctreePersistThread(_tree, _persistFilename, _persistInterval,
-                                    _wantBackup, _settings, _debugTimestampNow);
+                                                 _wantBackup, _settings, _debugTimestampNow, _persistAsJson);
         if (_persistThread) {
             _persistThread->initialize(true);
         }

assignment-client/src/octree/OctreeServer.h

@@ -157,6 +157,7 @@ protected:
     QString _statusHost;
 
     char _persistFilename[MAX_FILENAME_LENGTH];
+    bool _persistAsJson;
     int _packetsPerClientPerInterval;
     int _packetsTotalPerInterval;
     Octree* _tree; // this IS a reaveraging tree

domain-server/resources/describe-settings.json

@@ -325,6 +325,14 @@
         "default": "resources/models.svo",
         "advanced": true
       },
+      {
+        "name": "persistAsJson",
+        "type": "checkbox",
+        "label": "Encode persist file as JSON:",
+        "help": "Entity server will save entities as JSON rather than SVO",
+        "default": false,
+        "advanced": true
+      },
       {
         "name": "persistInterval",
         "label": "Save Check Interval",

interface/src/Application.cpp

@@ -1786,7 +1786,7 @@ bool Application::importEntities(const QString& urlOrFilename) {
         url = QUrl::fromLocalFile(urlOrFilename);
     }
     
-    bool success = _entityClipboard.readFromSVOURL(url.toString());
+    bool success = _entityClipboard.readFromURL(url.toString());
     if (success) {
         _entityClipboard.reaverageOctreeElements();
     }

libraries/entities/src/BoxEntityItem.cpp

@@ -114,6 +114,6 @@ void BoxEntityItem::writeSubTypeToMap(QVariantMap& map) {
 }
 
 
-void BoxEntityItem::readSubTypeFromMap(QVariantMap& map) {
+void BoxEntityItem::readFromMap(QVariantMap& map) {
     qListtoRgbColor(map["color"], _color);
 }

libraries/entities/src/BoxEntityItem.h

@@ -56,7 +56,7 @@ public:
     virtual void debugDump() const;
 
     void writeSubTypeToMap(QVariantMap& map);
-    void readSubTypeFromMap(QVariantMap& map);
+    void readFromMap(QVariantMap& map);
 
 protected:
     rgbColor _color;

libraries/entities/src/EntityItem.cpp

@@ -1174,28 +1174,3 @@ QVariantMap EntityItem::writeToMap() {
     writeSubTypeToMap(result);
     return result;
 }
-
-
-void EntityItem::readFromMap(QVariantMap map) {
-    // _type = EntityTypes::getEntityTypeFromName(map["type"].toString());
-    // _created = map["created"].toULongLong();
-    // _lastEdited = map["last-edited"].toULongLong();
-    // _lastUpdated = map["last-updated"].toULongLong();
-    // _dimensions = qListToGlmVec3(map["dimensions"]);
-    // _rotation = qListToGlmQuat(map["rotation"]);
-    // _density = map["density"].toFloat();
-    // _gravity = qListToGlmVec3(map["gravity"]);
-    // _damping = map["damping"].toFloat();
-    // _lifetime = map["lifetime"].toFloat();
-    // _script = map["script"].toString();
-    // _registrationPoint = qListToGlmVec3(map["registration-point"]);
-    // _angularVelocity = qListToGlmVec3(map["angular-velocity"]);
-    // _angularDamping = map["angular-damping"].toFloat();
-    // _visible = map["visible"].toBool();
-    // _ignoreForCollisions = map["ignore-for-collisions"].toBool();
-    // _collisionsWillMove = map["collisions-will-move"].toBool();
-    // _locked = map["locked"].toBool();
-    // _userData = map["userData"].toString();
-
-    readSubTypeFromMap(map);
-}

libraries/entities/src/EntityItem.h

@@ -298,8 +298,7 @@ public:
     QVariantMap writeToMap();
     virtual void writeSubTypeToMap(QVariantMap& map) = 0;
 
-    void readFromMap(QVariantMap map);
-    virtual void readSubTypeFromMap(QVariantMap& map) = 0;
+    virtual void readFromMap(QVariantMap& map) = 0;
 
 
 protected:

libraries/entities/src/EntityTree.cpp

@@ -1060,7 +1060,9 @@ public:
 
 
 
-bool EntityTree::writeToMap(QVariantMap& entityDescription) {
+bool EntityTree::writeToMap(QVariantMap& entityDescription, OctreeElement* element) {
+
+    // XXX how can I make the RecurseOctreeOperator start with element?
     entityDescription["Entities"] = QVariantList();
     ToMapOperator theOperator(entityDescription);
     recurseTreeWithOperator(&theOperator);
@@ -1078,16 +1080,6 @@ bool EntityTree::readFromMap(QVariantMap& map) {
     foreach (QVariant entityQ, entitiesQList) {
         QVariantMap entityMap = entityQ.toMap();
 
-        // EntityTypes::EntityType entityType = EntityTypes::getEntityTypeFromName(entityMap["type"].toString());
-
-        // qDebug() << "found entity of type" << entityType;
-
-        // if (entityType == EntityTypes::Unknown) {
-        //     qDebug() << "unknown entity type" << entityMap["type"];
-        //     continue;
-        // }
-
-
         EntityItemProperties properties;
 
         EntityItemID entityItemID;
@@ -1101,6 +1093,7 @@ bool EntityTree::readFromMap(QVariantMap& map) {
         QString typeString = entityMap["type"].toString();
         QByteArray typeByteArray = typeString.toLocal8Bit();
         const char *typeCString = typeByteArray.data();
+
         properties.setType(EntityTypes::getEntityTypeFromName(typeCString));
 
         properties.setPosition(qListToGlmVec3(entityMap["position"]));
@@ -1124,13 +1117,8 @@ bool EntityTree::readFromMap(QVariantMap& map) {
         properties.setUserData(entityMap["userData"].toString());
 
         EntityItem* entity = addEntity(entityItemID, properties);
-        // EntityItem* entity = getOrCreateEntityItem(entityItemID, properties);
+
         entity->readFromMap(entityMap);
-        // postAddEntity(entity);
-        // update();
-        // qDebug() << "\n\n\n";
-        // dumpTree();
-        // qDebug() << "\n\n\n";
     }
 
 

libraries/entities/src/EntityTree.h

@@ -158,7 +158,7 @@ public:
     bool wantEditLogging() const { return _wantEditLogging; }
     void setWantEditLogging(bool value) { _wantEditLogging = value; }
 
-    bool writeToMap(QVariantMap& entityDescription);
+    bool writeToMap(QVariantMap& entityDescription, OctreeElement* element);
     bool readFromMap(QVariantMap& entityDescription);
 
 signals:

libraries/entities/src/LightEntityItem.cpp

@@ -160,7 +160,7 @@ void LightEntityItem::writeSubTypeToMap(QVariantMap& map) {
     map["cutoff"] = _cutoff;
 }
 
-void LightEntityItem::readSubTypeFromMap(QVariantMap& map) {
+void LightEntityItem::readFromMap(QVariantMap& map) {
     qListtoRgbColor(map["color"], _color);
     _intensity = map["intensity"].toFloat();
     _exponent = map["exponent"].toFloat();

libraries/entities/src/LightEntityItem.h

@@ -71,7 +71,7 @@ public:
     static void setLightsArePickable(bool value) { _lightsArePickable = value; }
 
     void writeSubTypeToMap(QVariantMap& map);
-    void readSubTypeFromMap(QVariantMap& map);
+    void readFromMap(QVariantMap& map);
     
 protected:
 

libraries/entities/src/ModelEntityItem.cpp

@@ -427,7 +427,7 @@ void ModelEntityItem::writeSubTypeToMap(QVariantMap& map) {
 }
 
 
-void ModelEntityItem::readSubTypeFromMap(QVariantMap& map) {
+void ModelEntityItem::readFromMap(QVariantMap& map) {
     qListtoRgbColor(map["color"], _color);
 
     _modelURL = map["model-url"].toString();

libraries/entities/src/ModelEntityItem.h

@@ -121,7 +121,7 @@ public:
     static void cleanupLoadedAnimations();
 
     void writeSubTypeToMap(QVariantMap& map);
-    void readSubTypeFromMap(QVariantMap& map);
+    void readFromMap(QVariantMap& map);
     
 protected:
 

libraries/entities/src/ParticleEffectEntityItem.cpp

@@ -529,7 +529,7 @@ void ParticleEffectEntityItem::writeSubTypeToMap(QVariantMap& map) {
 }
 
 
-void ParticleEffectEntityItem::readSubTypeFromMap(QVariantMap& map) {
+void ParticleEffectEntityItem::readFromMap(QVariantMap& map) {
     qListtoRgbColor(map["color"], _color);
 
     _maxParticles = map["max-particles"].toFloat();

libraries/entities/src/ParticleEffectEntityItem.h

@@ -142,7 +142,7 @@ public:
     QString getAnimationSettings() const;
 
     void writeSubTypeToMap(QVariantMap& map);
-    void readSubTypeFromMap(QVariantMap& map);
+    void readFromMap(QVariantMap& map);
 
 protected:
 

libraries/entities/src/SphereEntityItem.cpp

@@ -135,6 +135,6 @@ void SphereEntityItem::writeSubTypeToMap(QVariantMap& map) {
 }
 
 
-void SphereEntityItem::readSubTypeFromMap(QVariantMap& map) {
+void SphereEntityItem::readFromMap(QVariantMap& map) {
     qListtoRgbColor(map["color"], _color);
 }

libraries/entities/src/SphereEntityItem.h

@@ -63,7 +63,7 @@ public:
     virtual void debugDump() const;
 
     void writeSubTypeToMap(QVariantMap& map);
-    void readSubTypeFromMap(QVariantMap& map);
+    void readFromMap(QVariantMap& map);
 
 protected:
 

libraries/entities/src/TextEntityItem.cpp

@@ -181,7 +181,7 @@ void TextEntityItem::writeSubTypeToMap(QVariantMap& map) {
 }
 
 
-void TextEntityItem::readSubTypeFromMap(QVariantMap& map) {
+void TextEntityItem::readFromMap(QVariantMap& map) {
     _text = map["text"].toString();
     _lineHeight = map["line-height"].toFloat();
     qListtoRgbColor(map["text-color"], _textColor);

libraries/entities/src/TextEntityItem.h

@@ -82,7 +82,7 @@ public:
     }
 
     void writeSubTypeToMap(QVariantMap& map);
-    void readSubTypeFromMap(QVariantMap& map);
+    void readFromMap(QVariantMap& map);
 
 protected:
     QString _text;

libraries/octree/src/Octree.cpp

@@ -29,6 +29,8 @@
 #include <QVector>
 #include <QFile>
 #include <QJsonDocument>
+#include <QFileInfo>
+#include <QString>
 
 #include <GeometryUtil.h>
 #include <LogHandler.h>
@@ -1844,37 +1846,52 @@ int Octree::encodeTreeBitstreamRecursion(OctreeElement* element,
 }
 
 
-bool Octree::readFromJSONFile(const char* fileName) {
-    QFile file;
-    file.setFileName("/tmp/ok.json");
-    file.open(QIODevice::ReadOnly | QIODevice::Text);
-    QString val = file.readAll();
-    file.close();
-    qWarning() << val;
-    QJsonDocument d = QJsonDocument::fromJson(val.toUtf8());
-    QVariant v = d.toVariant();
-    QVariantMap m = v.toMap();
-
-    readFromMap(m);
-    return true;
+QString withoutFileExtension(const QString& fileName) {
+    return fileName.left(fileName.lastIndexOf("."));
 }
 
 
-bool Octree::readFromSVOFile(const char* fileName) {
+bool Octree::readFromFile(const char* fileName) {
     bool fileOk = false;
 
-    QFile file(fileName);
+    // try to ease migration from svo to json or back
+    QString svoFileName = withoutFileExtension(QString(fileName)) + ".svo";
+    QFileInfo svoFileInfo(svoFileName);
+    QString jsonFileName = withoutFileExtension(QString(fileName)) + ".json";
+    QFileInfo jsonFileInfo(jsonFileName);
+    QString qFileName;
+    QFileInfo fileInfo;
+
+    if (jsonFileInfo.exists() && svoFileInfo.exists()) {
+        if (jsonFileInfo.lastModified() >= svoFileInfo.lastModified()) {
+            qFileName = jsonFileName;
+            fileInfo = jsonFileInfo;
+        } else {
+            qFileName = svoFileName;
+            fileInfo = svoFileInfo;
+        }
+    } else if (jsonFileInfo.exists()) {
+        qFileName = jsonFileName;
+        fileInfo = jsonFileInfo;
+    } else if (svoFileInfo.exists()) {
+        qFileName = svoFileName;
+        fileInfo = svoFileInfo;
+    } else {
+        qDebug() << "failed to read Octree from nonexistant file:" << fileName;
+        return false;
+    }
+
+    QFile file(qFileName);
     fileOk = file.open(QIODevice::ReadOnly);
 
     if(fileOk) {
         QDataStream fileInputStream(&file);
-        QFileInfo fileInfo(fileName);
         unsigned long fileLength = fileInfo.size();
 
         emit importSize(1.0f, 1.0f, 1.0f);
         emit importProgress(0);
 
-        qDebug("Loading file %s...", fileName);
+        qDebug() << "Loading file" << qFileName << "...";
     
         fileOk = readFromStream(fileLength, fileInputStream);
 
@@ -1885,14 +1902,14 @@ bool Octree::readFromSVOFile(const char* fileName) {
     return fileOk;
 }
 
-bool Octree::readFromSVOURL(const QString& urlString) {
+bool Octree::readFromURL(const QString& urlString) {
     bool readOk = false;
 
     // determine if this is a local file or a network resource
     QUrl url(urlString);
     
     if (url.isLocalFile()) {
-        readOk = readFromSVOFile(qPrintable(url.toLocalFile()));
+        readOk = readFromFile(qPrintable(url.toLocalFile()));
     } else {
         QNetworkRequest request;
         request.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
@@ -1918,6 +1935,23 @@ bool Octree::readFromSVOURL(const QString& urlString) {
 
 
 bool Octree::readFromStream(unsigned long streamLength, QDataStream& inputStream) {
+
+    // decide if this is SVO or JSON
+    QIODevice *device = inputStream.device();
+    char firstChar;
+    device->getChar(&firstChar);
+    device->ungetChar(firstChar);
+
+    if (firstChar == (char) PacketTypeEntityData) {
+        return readSVOFromStream(streamLength, inputStream);
+    } else {
+        return readJSONFromStream(streamLength, inputStream);
+    }
+}
+
+
+bool Octree::readSVOFromStream(unsigned long streamLength, QDataStream& inputStream) {
+
     bool fileOk = false;
 
     PacketVersion gotVersion = 0;
@@ -2045,6 +2079,61 @@ bool Octree::readFromStream(unsigned long streamLength, QDataStream& inputStream
     return fileOk;
 }
 
+bool Octree::readJSONFromStream(unsigned long streamLength, QDataStream& inputStream) {
+    // QFile file;
+    // file.setFileName("/tmp/ok.json");
+    // file.open(QIODevice::ReadOnly | QIODevice::Text);
+    // QString val = file.readAll();
+    // file.close();
+
+    char rawData[streamLength];
+    inputStream.readRawData(rawData, streamLength);
+
+    // QJsonDocument d = QJsonDocument::fromJson(val.toUtf8());
+    QJsonDocument d = QJsonDocument::fromJson(rawData);
+    QVariant v = d.toVariant();
+    QVariantMap m = v.toMap();
+
+    readFromMap(m);
+    return true;
+}
+
+void Octree::writeToFile(const char* fileName, OctreeElement* element, bool persistAsJson) {
+    if (persistAsJson) {
+        // make the sure file extension is correct.  This isn't great, but it allows a user with
+        // an existing .svo save to migrate.
+        QString qFileName = withoutFileExtension(QString(fileName)) + ".json";
+        QByteArray byteArray = qFileName.toUtf8();
+        const char* cFileName = byteArray.constData();
+        writeToJSONFile(cFileName, element);
+    } else {
+        writeToSVOFile(fileName, element);
+    }
+}
+
+
+void Octree::writeToJSONFile(const char* fileName, OctreeElement* element) {
+    QFile persistFile(fileName);
+    QVariantMap entityDescription;
+
+    qDebug("Saving to file %s...", fileName);
+
+    OctreeElement* top;
+    if (element) {
+        top = element;
+    } else {
+        top = _rootElement;
+    }
+
+    bool entityDescriptionSuccess = writeToMap(entityDescription, top);
+    if (entityDescriptionSuccess && persistFile.open(QIODevice::WriteOnly)) {
+        persistFile.write(QJsonDocument::fromVariant(entityDescription).toJson());
+    } else {
+        qCritical("Could not write to JSON description of entities.");
+    }
+}
+
+
 void Octree::writeToSVOFile(const char* fileName, OctreeElement* element) {
     std::ofstream file(fileName, std::ios::out|std::ios::binary);
 

libraries/octree/src/Octree.h

@@ -326,12 +326,20 @@ public:
     void loadOctreeFile(const char* fileName, bool wantColorRandomizer);
 
     // these will read/write files that match the wireformat, excluding the 'V' leading
+    void writeToFile(const char* filename, OctreeElement* element = NULL, bool persistAsJson = false);
+    void writeToJSONFile(const char* filename, OctreeElement* element = NULL);
     void writeToSVOFile(const char* filename, OctreeElement* element = NULL);
-    bool readFromSVOFile(const char* filename);
-    bool readFromSVOURL(const QString& url); // will support file urls as well...
+    virtual bool writeToMap(QVariantMap& entityDescription, OctreeElement* element) = 0;
+
+
+    bool readFromFile(const char* filename);
+    bool readFromURL(const QString& url); // will support file urls as well...
+
     bool readFromStream(unsigned long streamLength, QDataStream& inputStream);
-    bool readFromJSONFile(const char* fileName);
-    virtual bool writeToMap(QVariantMap& entityDescription) { return true; }
+    bool readSVOFromStream(unsigned long streamLength, QDataStream& inputStream);
+    bool readJSONFromStream(unsigned long streamLength, QDataStream& inputStream);
+
+
 
     unsigned long getOctreeElementsCount();
 

libraries/octree/src/OctreeElement.h

@@ -260,8 +260,6 @@ public:
     int getMyChildContaining(const AABox& box) const;
     int getMyChildContainingPoint(const glm::vec3& point) const;
 
-    virtual bool writeToMap(QVariantMap& entityDescription) { return true; }
-
 protected:
 
     void deleteAllChildren();

libraries/octree/src/OctreePersistThread.cpp

@@ -30,7 +30,8 @@
 const int OctreePersistThread::DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
 
 OctreePersistThread::OctreePersistThread(Octree* tree, const QString& filename, int persistInterval, 
-                                                bool wantBackup, const QJsonObject& settings, bool debugTimestampNow) :
+                                         bool wantBackup, const QJsonObject& settings, bool debugTimestampNow,
+                                         bool persistAsJson) :
     _tree(tree),
     _filename(filename),
     _persistInterval(persistInterval),
@@ -39,7 +40,8 @@ OctreePersistThread::OctreePersistThread(Octree* tree, const QString& filename,
     _lastCheck(0),
     _wantBackup(wantBackup),
     _debugTimestampNow(debugTimestampNow),
-    _lastTimeDebug(0)
+    _lastTimeDebug(0),
+    _persistAsJson(persistAsJson)
 {
     parseSettings(settings);
 }
@@ -141,8 +143,7 @@ bool OctreePersistThread::process() {
                 qDebug() << "Loading Octree... lock file removed:" << lockFileName;
             }
 
-            // persistantFileRead = _tree->readFromSVOFile(_filename.toLocal8Bit().constData());
-            persistantFileRead = _tree->readFromJSONFile(_filename.toLocal8Bit().constData());
+            persistantFileRead = _tree->readFromFile(_filename.toLocal8Bit().constData());
             _tree->pruneTree();
         }
         _tree->unlock();
@@ -244,19 +245,7 @@ void OctreePersistThread::persist() {
         if(lockFile.is_open()) {
             qDebug() << "saving Octree lock file created at:" << lockFileName;
 
-            qDebug() << "saving Octree to file " << _filename << "...";
-
-
-            QFile persistFile("/tmp/ok.json");
-            QVariantMap entityDescription;
-            bool entityDescriptionSuccess = _tree->writeToMap(entityDescription);
-            if (entityDescriptionSuccess && persistFile.open(QIODevice::WriteOnly)) {
-                persistFile.write(QJsonDocument::fromVariant(entityDescription).toJson());
-            } else {
-                qCritical("Could not write to JSON description of entities.");
-            }
-
-            _tree->writeToSVOFile(qPrintable(_filename));
+            _tree->writeToFile(qPrintable(_filename), NULL, _persistAsJson);
             time(&_lastPersistTime);
             _tree->clearDirtyBit(); // tree is clean after saving
             qDebug() << "DONE saving Octree to file...";

libraries/octree/src/OctreePersistThread.h

@@ -34,8 +34,8 @@ public:
     static const int DEFAULT_PERSIST_INTERVAL;
 
     OctreePersistThread(Octree* tree, const QString& filename, int persistInterval = DEFAULT_PERSIST_INTERVAL, 
-                                bool wantBackup = false, const QJsonObject& settings = QJsonObject(), 
-                                bool debugTimestampNow = false);
+                        bool wantBackup = false, const QJsonObject& settings = QJsonObject(), 
+                        bool debugTimestampNow = false, bool persistAsJson=false);
 
     bool isInitialLoadComplete() const { return _initialLoadComplete; }
     quint64 getLoadElapsedTime() const { return _loadTimeUSecs; }
@@ -72,6 +72,8 @@ private:
     
     bool _debugTimestampNow;
     quint64 _lastTimeDebug;
+
+    bool _persistAsJson;
 };
 
 #endif // hifi_OctreePersistThread_h