From c938c595c017147b9210b6a0b03d0e891cfb05bb Mon Sep 17 00:00:00 2001
From: Simon Walton <simon@highfidelity.io>
Date: Wed, 10 Oct 2018 17:11:42 -0700
Subject: [PATCH 01/14] Detect failed conversion from QVariant to JSON better

---
 libraries/octree/src/Octree.cpp | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp
index 3d387e0956..37dd4e410f 100644
--- a/libraries/octree/src/Octree.cpp
+++ b/libraries/octree/src/Octree.cpp
@@ -878,14 +878,24 @@ bool Octree::toJSONDocument(QJsonDocument* doc, const OctreeElementPointer& elem
         return false;
     }
 
-    *doc = QJsonDocument::fromVariant(entityDescription);
+
+    bool noEntities = entityDescription["Entities"].toList().empty();
+    QJsonDocument jsonDocTree = QJsonDocument::fromVariant(entityDescription);
+    QJsonValue entitiesJson = jsonDocTree["Entities"];
+    if (entitiesJson.isNull() || (entitiesJson.toArray().empty() && !noEntities)) {
+        // Json version of entities too large.
+        return false;
+    } else {
+        *doc = jsonDocTree;
+    }
+
     return true;
 }
 
 bool Octree::toJSON(QByteArray* data, const OctreeElementPointer& element, bool doGzip) {
     QJsonDocument doc;
     if (!toJSONDocument(&doc, element)) {
-        qCritical("Failed to convert Entities to QVariantMap while converting to json.");
+        qCritical("Failed to convert Entities to JSON document.");
         return false;
     }
 

From 0eec0e376c8102ebc931f2cddbf9bdede0b082da Mon Sep 17 00:00:00 2001
From: Simon Walton <simon@highfidelity.io>
Date: Thu, 11 Oct 2018 18:13:26 -0700
Subject: [PATCH 02/14] Convert EntityTree direct to JSON, not via
 QJsonDocument

---
 libraries/entities/src/EntityTree.cpp         | 12 +++++
 libraries/entities/src/EntityTree.h           |  2 +
 .../src/RecurseOctreeToJSONOperator.cpp       | 45 +++++++++++++++++
 .../src/RecurseOctreeToJSONOperator.h         | 31 ++++++++++++
 libraries/octree/src/Octree.cpp               | 50 ++++++++++++++++++-
 libraries/octree/src/Octree.h                 |  2 +
 6 files changed, 141 insertions(+), 1 deletion(-)
 create mode 100644 libraries/entities/src/RecurseOctreeToJSONOperator.cpp
 create mode 100644 libraries/entities/src/RecurseOctreeToJSONOperator.h

diff --git a/libraries/entities/src/EntityTree.cpp b/libraries/entities/src/EntityTree.cpp
index a7c88ddc7d..aa2167029d 100644
--- a/libraries/entities/src/EntityTree.cpp
+++ b/libraries/entities/src/EntityTree.cpp
@@ -35,6 +35,7 @@
 #include "QVariantGLM.h"
 #include "EntitiesLogging.h"
 #include "RecurseOctreeToMapOperator.h"
+#include "RecurseOctreeToJsonOperator.h"
 #include "LogHandler.h"
 #include "EntityEditFilters.h"
 #include "EntityDynamicFactoryInterface.h"
@@ -2658,6 +2659,17 @@ bool EntityTree::readFromMap(QVariantMap& map) {
     return success;
 }
 
+bool EntityTree::writeToJSON(QString & jsonString, const OctreeElementPointer & element) {
+    QScriptEngine scriptEngine;
+    RecurseOctreeToJSONOperator theOperator(element, &scriptEngine, jsonString);
+    withReadLock([&] {
+        recurseTreeWithOperator(&theOperator);
+    });
+
+    jsonString = theOperator.getJson();
+    return true;
+}
+
 void EntityTree::resetClientEditStats() {
     _treeResetTime = usecTimestampNow();
     _maxEditDelta = 0;
diff --git a/libraries/entities/src/EntityTree.h b/libraries/entities/src/EntityTree.h
index 2f971b8566..b852a695d8 100644
--- a/libraries/entities/src/EntityTree.h
+++ b/libraries/entities/src/EntityTree.h
@@ -224,6 +224,8 @@ public:
     virtual bool writeToMap(QVariantMap& entityDescription, OctreeElementPointer element, bool skipDefaultValues,
                             bool skipThoseWithBadParents) override;
     virtual bool readFromMap(QVariantMap& entityDescription) override;
+    virtual bool writeToJSON(QString& jsonString, const OctreeElementPointer& element) override;
+
 
     glm::vec3 getContentsDimensions();
     float getContentsLargestDimension();
diff --git a/libraries/entities/src/RecurseOctreeToJSONOperator.cpp b/libraries/entities/src/RecurseOctreeToJSONOperator.cpp
new file mode 100644
index 0000000000..2db1a2b228
--- /dev/null
+++ b/libraries/entities/src/RecurseOctreeToJSONOperator.cpp
@@ -0,0 +1,45 @@
+//
+//  RecurseOctreeToJSONOperator.cpp
+//  libraries/entities/src
+//
+//  Created by Simon Walton on Oct 11, 2018.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "RecurseOctreeToJSONOperator.h"
+
+#include "EntityItemProperties.h"
+
+RecurseOctreeToJSONOperator::RecurseOctreeToJSONOperator(const OctreeElementPointer& top, QScriptEngine* engine,
+    QString jsonPrefix /* = QString() */)
+    : _top(top)
+    , _engine(engine)
+    , _json(jsonPrefix)
+{
+    _toStringMethod = _engine->evaluate("(function() { return JSON.stringify(this, null, '    ') })");
+}
+
+bool RecurseOctreeToJSONOperator::postRecursion(const OctreeElementPointer& element) {
+    EntityTreeElementPointer entityTreeElement = std::static_pointer_cast<EntityTreeElement>(element);
+
+    entityTreeElement->forEachEntity([&](const EntityItemPointer& entity) { processEntity(entity); } );
+    return true;
+}
+
+void RecurseOctreeToJSONOperator::processEntity(const EntityItemPointer& entity) {
+    QScriptValue qScriptValues = EntityItemNonDefaultPropertiesToScriptValue(_engine, entity->getProperties());
+    if (comma) {
+        _json += ',';
+    };
+    comma = true;
+    _json += "\n    ";
+
+    // Override default toString():
+    qScriptValues.setProperty("toString", _toStringMethod);
+    QString jsonResult = qScriptValues.toString();
+    //auto exceptionString2 = _engine->uncaughtException().toString();
+    _json += jsonResult;
+}
diff --git a/libraries/entities/src/RecurseOctreeToJSONOperator.h b/libraries/entities/src/RecurseOctreeToJSONOperator.h
new file mode 100644
index 0000000000..92ce6bfdca
--- /dev/null
+++ b/libraries/entities/src/RecurseOctreeToJSONOperator.h
@@ -0,0 +1,31 @@
+//
+//  RecurseOctreeToJSONOperator.h
+//  libraries/entities/src
+//
+//  Created by Simon Walton on Oct 11, 2018.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "EntityTree.h"
+
+class RecurseOctreeToJSONOperator : public RecurseOctreeOperator {
+public:
+    RecurseOctreeToJSONOperator(const OctreeElementPointer& top, QScriptEngine* engine, QString jsonPrefix = QString());
+    virtual bool preRecursion(const OctreeElementPointer& element) override { return true; };
+    virtual bool postRecursion(const OctreeElementPointer& element) override;
+
+    QString getJson() const { return _json; }
+
+private:
+    void processEntity(const EntityItemPointer& entity);
+
+    const OctreeElementPointer& _top;
+    QScriptEngine* _engine;
+    QScriptValue _toStringMethod;
+
+    QString _json;
+    bool comma { false };
+};
diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp
index 37dd4e410f..40d2449a38 100644
--- a/libraries/octree/src/Octree.cpp
+++ b/libraries/octree/src/Octree.cpp
@@ -51,7 +51,6 @@
 #include "OctreeQueryNode.h"
 #include "OctreeUtils.h"
 
-
 QVector<QString> PERSIST_EXTENSIONS = {"json", "json.gz"};
 
 Octree::Octree(bool shouldReaverage) :
@@ -892,13 +891,61 @@ bool Octree::toJSONDocument(QJsonDocument* doc, const OctreeElementPointer& elem
     return true;
 }
 
+bool Octree::toJSONString(QString& jsonString, const OctreeElementPointer& element) {
+    OctreeElementPointer top;
+    if (element) {
+        top = element;
+    } else {
+        top = _rootElement;
+    }
+
+    jsonString += QString(R"({
+  "DataVersion": %1,
+  "Entities": [)").arg(_persistDataVersion);
+
+    writeToJSON(jsonString, top);
+
+    // include the "bitstream" version
+    PacketType expectedType = expectedDataPacketType();
+    PacketVersion expectedVersion = versionForPacketType(expectedType);
+
+    jsonString += QString(R"(
+    ],
+  "Id": "%1",
+  "Version": %2
+}
+)").arg(_persistID.toString()).arg((int)expectedVersion);
+
+    return true;
+}
+
 bool Octree::toJSON(QByteArray* data, const OctreeElementPointer& element, bool doGzip) {
+#define HIFI_USE_DIRECT_TO_JSON
+#ifdef HIFI_USE_DIRECT_TO_JSON
+
+    QString jsonString;
+    toJSONString(jsonString);
+
+    if (doGzip) {
+        if (!gzip(jsonString.toUtf8(), *data, -1)) {
+            qCritical("Unable to gzip data while saving to json.");
+            return false;
+        }
+    } else {
+        *data = jsonString.toUtf8();
+    }
+
+#else
+
     QJsonDocument doc;
     if (!toJSONDocument(&doc, element)) {
         qCritical("Failed to convert Entities to JSON document.");
         return false;
     }
 
+    QString jsonString;
+    toJSONString(jsonString);
+
     if (doGzip) {
         QByteArray jsonData = doc.toJson();
 
@@ -909,6 +956,7 @@ bool Octree::toJSON(QByteArray* data, const OctreeElementPointer& element, bool
     } else {
         *data = doc.toJson();
     }
+#endif  // HIFI_USE_DIRECT_TO_JSON
 
     return true;
 }
diff --git a/libraries/octree/src/Octree.h b/libraries/octree/src/Octree.h
index a2b2f227cb..678d8aa5de 100644
--- a/libraries/octree/src/Octree.h
+++ b/libraries/octree/src/Octree.h
@@ -202,11 +202,13 @@ public:
 
     // Octree exporters
     bool toJSONDocument(QJsonDocument* doc, const OctreeElementPointer& element = nullptr);
+    bool toJSONString(QString& jsonString, const OctreeElementPointer& element = nullptr);
     bool toJSON(QByteArray* data, const OctreeElementPointer& element = nullptr, bool doGzip = false);
     bool writeToFile(const char* filename, const OctreeElementPointer& element = nullptr, QString persistAsFileType = "json.gz");
     bool writeToJSONFile(const char* filename, const OctreeElementPointer& element = nullptr, bool doGzip = false);
     virtual bool writeToMap(QVariantMap& entityDescription, OctreeElementPointer element, bool skipDefaultValues,
                             bool skipThoseWithBadParents) = 0;
+    virtual bool writeToJSON(QString& jsonString, const OctreeElementPointer& element) = 0;
 
     // Octree importers
     bool readFromFile(const char* filename);

From ad9f7f3a1d97d1a3fe766fcdb2b8190764d5e44b Mon Sep 17 00:00:00 2001
From: Simon Walton <simon@highfidelity.io>
Date: Fri, 12 Oct 2018 11:18:38 -0700
Subject: [PATCH 03/14] Include the skipDefaults option in case we ever need it

Also print errors from parsing entities file; other tweaks
---
 libraries/entities/src/EntityTree.cpp                |  2 +-
 .../entities/src/RecurseOctreeToJSONOperator.cpp     | 12 +++++++-----
 libraries/entities/src/RecurseOctreeToJSONOperator.h |  3 ++-
 libraries/octree/src/Octree.cpp                      |  2 +-
 libraries/octree/src/OctreeDataUtils.cpp             |  6 +++++-
 5 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/libraries/entities/src/EntityTree.cpp b/libraries/entities/src/EntityTree.cpp
index aa2167029d..19182d8b22 100644
--- a/libraries/entities/src/EntityTree.cpp
+++ b/libraries/entities/src/EntityTree.cpp
@@ -2659,7 +2659,7 @@ bool EntityTree::readFromMap(QVariantMap& map) {
     return success;
 }
 
-bool EntityTree::writeToJSON(QString & jsonString, const OctreeElementPointer & element) {
+bool EntityTree::writeToJSON(QString& jsonString, const OctreeElementPointer& element) {
     QScriptEngine scriptEngine;
     RecurseOctreeToJSONOperator theOperator(element, &scriptEngine, jsonString);
     withReadLock([&] {
diff --git a/libraries/entities/src/RecurseOctreeToJSONOperator.cpp b/libraries/entities/src/RecurseOctreeToJSONOperator.cpp
index 2db1a2b228..d67e7176a3 100644
--- a/libraries/entities/src/RecurseOctreeToJSONOperator.cpp
+++ b/libraries/entities/src/RecurseOctreeToJSONOperator.cpp
@@ -10,14 +10,14 @@
 //
 
 #include "RecurseOctreeToJSONOperator.h"
-
 #include "EntityItemProperties.h"
 
 RecurseOctreeToJSONOperator::RecurseOctreeToJSONOperator(const OctreeElementPointer& top, QScriptEngine* engine,
-    QString jsonPrefix /* = QString() */)
+    QString jsonPrefix /* = QString() */, bool skipDefaults /* = true */)
     : _top(top)
     , _engine(engine)
     , _json(jsonPrefix)
+    , _skipDefaults(skipDefaults)
 {
     _toStringMethod = _engine->evaluate("(function() { return JSON.stringify(this, null, '    ') })");
 }
@@ -30,7 +30,10 @@ bool RecurseOctreeToJSONOperator::postRecursion(const OctreeElementPointer& elem
 }
 
 void RecurseOctreeToJSONOperator::processEntity(const EntityItemPointer& entity) {
-    QScriptValue qScriptValues = EntityItemNonDefaultPropertiesToScriptValue(_engine, entity->getProperties());
+    QScriptValue qScriptValues = _skipDefaults
+        ? EntityItemNonDefaultPropertiesToScriptValue(_engine, entity->getProperties())
+        : EntityItemPropertiesToScriptValue(_engine, entity->getProperties());
+
     if (comma) {
         _json += ',';
     };
@@ -39,7 +42,6 @@ void RecurseOctreeToJSONOperator::processEntity(const EntityItemPointer& entity)
 
     // Override default toString():
     qScriptValues.setProperty("toString", _toStringMethod);
-    QString jsonResult = qScriptValues.toString();
+    _json += qScriptValues.toString();
     //auto exceptionString2 = _engine->uncaughtException().toString();
-    _json += jsonResult;
 }
diff --git a/libraries/entities/src/RecurseOctreeToJSONOperator.h b/libraries/entities/src/RecurseOctreeToJSONOperator.h
index 92ce6bfdca..f661e08d5b 100644
--- a/libraries/entities/src/RecurseOctreeToJSONOperator.h
+++ b/libraries/entities/src/RecurseOctreeToJSONOperator.h
@@ -13,7 +13,7 @@
 
 class RecurseOctreeToJSONOperator : public RecurseOctreeOperator {
 public:
-    RecurseOctreeToJSONOperator(const OctreeElementPointer& top, QScriptEngine* engine, QString jsonPrefix = QString());
+    RecurseOctreeToJSONOperator(const OctreeElementPointer& top, QScriptEngine* engine, QString jsonPrefix = QString(), bool skipDefaults = true);
     virtual bool preRecursion(const OctreeElementPointer& element) override { return true; };
     virtual bool postRecursion(const OctreeElementPointer& element) override;
 
@@ -27,5 +27,6 @@ private:
     QScriptValue _toStringMethod;
 
     QString _json;
+    const bool _skipDefaults;
     bool comma { false };
 };
diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp
index 40d2449a38..d583901b41 100644
--- a/libraries/octree/src/Octree.cpp
+++ b/libraries/octree/src/Octree.cpp
@@ -956,8 +956,8 @@ bool Octree::toJSON(QByteArray* data, const OctreeElementPointer& element, bool
     } else {
         *data = doc.toJson();
     }
-#endif  // HIFI_USE_DIRECT_TO_JSON
 
+#endif  // HIFI_USE_DIRECT_TO_JSON
     return true;
 }
 
diff --git a/libraries/octree/src/OctreeDataUtils.cpp b/libraries/octree/src/OctreeDataUtils.cpp
index 44a56fe97a..4b1700d593 100644
--- a/libraries/octree/src/OctreeDataUtils.cpp
+++ b/libraries/octree/src/OctreeDataUtils.cpp
@@ -35,7 +35,11 @@ bool readOctreeFile(QString path, QJsonDocument* doc) {
         jsonData = data;
     }
 
-    *doc = QJsonDocument::fromJson(jsonData);
+    QJsonParseError parserError;
+    *doc = QJsonDocument::fromJson(jsonData, &parserError);
+    if (parserError.error != QJsonParseError::NoError) {
+        qWarning() << "Error reading JSON file" << path << "-" << parserError.errorString();
+    }
     return !doc->isNull();
 }
 

From e86d1691ce00bc99cdfa556a15bbda51c59bfef9 Mon Sep 17 00:00:00 2001
From: Simon Walton <simon@highfidelity.io>
Date: Fri, 12 Oct 2018 17:26:46 -0700
Subject: [PATCH 04/14] Cache the local Entities file to avoid reading 2 or 3
 times

---
 libraries/octree/src/OctreeDataUtils.cpp     | 42 ++++++++++----------
 libraries/octree/src/OctreeDataUtils.h       |  1 +
 libraries/octree/src/OctreePersistThread.cpp | 39 +++++++++++++-----
 libraries/octree/src/OctreePersistThread.h   |  1 +
 4 files changed, 54 insertions(+), 29 deletions(-)

diff --git a/libraries/octree/src/OctreeDataUtils.cpp b/libraries/octree/src/OctreeDataUtils.cpp
index 4b1700d593..a2b0d78209 100644
--- a/libraries/octree/src/OctreeDataUtils.cpp
+++ b/libraries/octree/src/OctreeDataUtils.cpp
@@ -18,30 +18,32 @@
 #include <QJsonDocument>
 #include <QFile>
 
-// Reads octree file and parses it into a QJsonDocument. Handles both gzipped and non-gzipped files.
-// Returns true if the file was successfully opened and parsed, otherwise false.
-// Example failures: file does not exist, gzipped file cannot be unzipped, invalid JSON.
-bool readOctreeFile(QString path, QJsonDocument* doc) {
-    QFile file(path);
-    if (!file.open(QIODevice::ReadOnly)) {
-        qCritical() << "Cannot open json file for reading: " << path;
-        return false;
-    }
+namespace {
+    // Reads octree file and parses it into a QJsonDocument. Handles both gzipped and non-gzipped files.
+    // Returns true if the file was successfully opened and parsed, otherwise false.
+    // Example failures: file does not exist, gzipped file cannot be unzipped, invalid JSON.
+    bool readOctreeFile(QString path, QJsonDocument* doc) {
+        QFile file(path);
+        if (!file.open(QIODevice::ReadOnly)) {
+            qCritical() << "Cannot open json file for reading: " << path;
+            return false;
+        }
 
-    QByteArray data = file.readAll();
-    QByteArray jsonData;
+        QByteArray data = file.readAll();
+        QByteArray jsonData;
 
-    if (!gunzip(data, jsonData)) {
-        jsonData = data;
-    }
+        if (!gunzip(data, jsonData)) {
+            jsonData = data;
+        }
 
-    QJsonParseError parserError;
-    *doc = QJsonDocument::fromJson(jsonData, &parserError);
-    if (parserError.error != QJsonParseError::NoError) {
-        qWarning() << "Error reading JSON file" << path << "-" << parserError.errorString();
+        QJsonParseError parserError;
+        *doc = QJsonDocument::fromJson(jsonData, &parserError);
+        if (parserError.error != QJsonParseError::NoError) {
+            qWarning() << "Error reading JSON file" << path << "-" << parserError.errorString();
+        }
+        return !doc->isNull();
     }
-    return !doc->isNull();
-}
+}  // Anon namespace.
 
 bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromJSON(QJsonObject root) {
     if (root.contains("Id") && root.contains("DataVersion") && root.contains("Version")) {
diff --git a/libraries/octree/src/OctreeDataUtils.h b/libraries/octree/src/OctreeDataUtils.h
index 9060e7b460..ef96415044 100644
--- a/libraries/octree/src/OctreeDataUtils.h
+++ b/libraries/octree/src/OctreeDataUtils.h
@@ -46,6 +46,7 @@ public:
 };
 
 class RawEntityData : public RawOctreeData {
+public:
     PacketType dataPacketType() const override;
     void readSubclassData(const QJsonObject& root) override;
     void writeSubclassData(QJsonObject& root) const override;
diff --git a/libraries/octree/src/OctreePersistThread.cpp b/libraries/octree/src/OctreePersistThread.cpp
index 8b1d766418..0c78a1a797 100644
--- a/libraries/octree/src/OctreePersistThread.cpp
+++ b/libraries/octree/src/OctreePersistThread.cpp
@@ -31,6 +31,7 @@
 #include <NumericalConstants.h>
 #include <PerfStat.h>
 #include <PathUtils.h>
+#include <Gzip.h>
 
 #include "OctreeLogging.h"
 #include "OctreeUtils.h"
@@ -72,14 +73,27 @@ void OctreePersistThread::start() {
 
     OctreeUtils::RawOctreeData data;
     qCDebug(octree) << "Reading octree data from" << _filename;
-    if (data.readOctreeDataInfoFromFile(_filename)) {
-        qCDebug(octree) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")";
-        packet->writePrimitive(true);
-        auto id = data.id.toRfc4122();
-        packet->write(id);
-        packet->writePrimitive(data.version);
+    QFile file(_filename);
+    if (file.open(QIODevice::ReadOnly)) {
+        QByteArray jsonData(file.readAll());
+        file.close();
+        if (!gunzip(jsonData, _cachedJSONData)) {
+            _cachedJSONData = jsonData;
+        }
+
+        if (data.readOctreeDataInfoFromData(_cachedJSONData)) {
+            qCDebug(octree) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")";
+            packet->writePrimitive(true);
+            auto id = data.id.toRfc4122();
+            packet->write(id);
+            packet->writePrimitive(data.version);
+        } else {
+            _cachedJSONData.clear();
+            qCWarning(octree) << "No octree data found";
+            packet->writePrimitive(false);
+        }
     } else {
-        qCWarning(octree) << "No octree data found";
+        qCWarning(octree) << "Couldn't access file" << _filename << file.errorString();
         packet->writePrimitive(false);
     }
 
@@ -99,6 +113,7 @@ void OctreePersistThread::handleOctreeDataFileReply(QSharedPointer<ReceivedMessa
     OctreeUtils::RawOctreeData data;
     bool hasValidOctreeData { false };
     if (includesNewData) {
+        _cachedJSONData.clear();
         replacementData = message->readAll();
         replaceData(replacementData);
         hasValidOctreeData = data.readOctreeDataInfoFromFile(_filename);
@@ -108,7 +123,7 @@ void OctreePersistThread::handleOctreeDataFileReply(QSharedPointer<ReceivedMessa
         
         OctreeUtils::RawEntityData data;
         qCDebug(octree) << "Reading octree data from" << _filename;
-        if (data.readOctreeDataInfoFromFile(_filename)) {
+        if (data.readOctreeDataInfoFromData(_cachedJSONData)) {
             hasValidOctreeData = true;
             if (data.id.isNull()) {
                 qCDebug(octree) << "Current octree data has a null id, updating";
@@ -139,10 +154,16 @@ void OctreePersistThread::handleOctreeDataFileReply(QSharedPointer<ReceivedMessa
     _tree->withWriteLock([&] {
         PerformanceWarning warn(true, "Loading Octree File", true);
 
-        persistentFileRead = _tree->readFromFile(_filename.toLocal8Bit().constData());
+        if (_cachedJSONData.isEmpty()) {
+            persistentFileRead = _tree->readFromFile(_filename.toLocal8Bit().constData());
+        } else {
+            QDataStream jsonStream(_cachedJSONData);
+            persistentFileRead = _tree->readFromStream(-1, QDataStream(_cachedJSONData));
+        }
         _tree->pruneTree();
     });
 
+    _cachedJSONData.clear();
     quint64 loadDone = usecTimestampNow();
     _loadTimeUSecs = loadDone - loadStarted;
 
diff --git a/libraries/octree/src/OctreePersistThread.h b/libraries/octree/src/OctreePersistThread.h
index 0044a8fa5a..6a38a21a84 100644
--- a/libraries/octree/src/OctreePersistThread.h
+++ b/libraries/octree/src/OctreePersistThread.h
@@ -78,6 +78,7 @@ private:
     quint64 _lastTimeDebug;
 
     QString _persistAsFileType;
+    QByteArray _cachedJSONData;
 };
 
 #endif // hifi_OctreePersistThread_h

From f5f34e8e7de96dc49766c98af48abd92a1435429 Mon Sep 17 00:00:00 2001
From: Simon Walton <simon@highfidelity.io>
Date: Mon, 15 Oct 2018 18:14:40 -0700
Subject: [PATCH 05/14] Pasre Entities JSON with our own top-level parser

Use QJsonDocument for each individual entity.
---
 libraries/octree/src/Octree.cpp               |   8 +
 .../octree/src/OctreeEntitiesFileParser.cpp   | 218 ++++++++++++++++++
 .../octree/src/OctreeEntitiesFileParser.h     |  38 +++
 3 files changed, 264 insertions(+)
 create mode 100644 libraries/octree/src/OctreeEntitiesFileParser.cpp
 create mode 100644 libraries/octree/src/OctreeEntitiesFileParser.h

diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp
index d583901b41..30108d973e 100644
--- a/libraries/octree/src/Octree.cpp
+++ b/libraries/octree/src/Octree.cpp
@@ -50,6 +50,7 @@
 #include "OctreeLogging.h"
 #include "OctreeQueryNode.h"
 #include "OctreeUtils.h"
+#include "OctreeEntitiesFileParser.h"
 
 QVector<QString> PERSIST_EXTENSIONS = {"json", "json.gz"};
 
@@ -827,12 +828,19 @@ bool Octree::readJSONFromStream(uint64_t streamLength, QDataStream& inputStream,
         jsonBuffer += QByteArray(rawData, got);
     }
 
+    OctreeEntitiesFileParser octreeParser;
+    octreeParser.setEntitiesString(jsonBuffer);
+    QVariantMap asMap;
+    bool parseSuccess = octreeParser.parseEntities(asMap);
+
+    /*
     QJsonDocument asDocument = QJsonDocument::fromJson(jsonBuffer);
     if (!marketplaceID.isEmpty()) {
         asDocument = addMarketplaceIDToDocumentEntities(asDocument, marketplaceID);
     }
     QVariant asVariant = asDocument.toVariant();
     QVariantMap asMap = asVariant.toMap();
+    */
     bool success = readFromMap(asMap);
     delete[] rawData;
     return success;
diff --git a/libraries/octree/src/OctreeEntitiesFileParser.cpp b/libraries/octree/src/OctreeEntitiesFileParser.cpp
new file mode 100644
index 0000000000..14380de835
--- /dev/null
+++ b/libraries/octree/src/OctreeEntitiesFileParser.cpp
@@ -0,0 +1,218 @@
+//
+//  OctreeEntititesFileParser.cpp
+//  libraries/octree/src
+//
+//  Created by Simon Walton on Oct 15, 2018.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <sstream>
+#include <QUuid>
+#include <QJsonDocument>
+#include <QJsonObject>
+#include <QVariantList>
+
+#include "OctreeEntitiesFileParser.h"
+
+using std::string;
+
+OctreeEntitiesFileParser::OctreeEntitiesFileParser() {
+}
+
+std::string OctreeEntitiesFileParser::getErrorString() const {
+    std::ostringstream err;
+    if (_errorString.size() != 0) {
+        err << "Error: Line " << _line << ", byte position " << _position << ": " << _errorString;
+    };
+
+    return err.str();
+}
+
+void OctreeEntitiesFileParser::setEntitiesString(const QByteArray& entitiesContents) {
+    _entitiesContents = entitiesContents;
+    _entitiesLength = _entitiesContents.length();
+    _position = 0;
+    _line = 1;
+}
+
+bool OctreeEntitiesFileParser::parseEntities(QVariantMap& parsedEntities) {
+    if (nextToken() != '{') {
+        _errorString = "Text before start of object";
+        return false;
+    }
+
+    bool gotDataVersion = false;
+    bool gotEntities = false;
+    bool gotId = false;
+    bool gotVersion = false;
+
+    while (!(gotDataVersion && gotEntities && gotId && gotVersion)) {
+        if (nextToken() != '"') {
+            _errorString = "Incorrect key string";
+            return false;
+        }
+
+        string key = readString();
+        if (key.size() == 0) {
+            _errorString = "Missing object key";
+            return false;
+        }
+
+        if (nextToken() != ':') {
+            _errorString = "Ill-formed id/value entry";
+            return false;
+        }
+
+        if (key == "DataVersion") {
+            if (gotDataVersion) {
+                _errorString = "Duplicate DataVersion entries";
+                return false;
+            }
+
+            int dataVersionValue = readInteger();
+            parsedEntities["DataVersion"] = dataVersionValue;
+            gotDataVersion = true;
+        } else if (key == "Entities") {
+            if (gotEntities) {
+                _errorString = "Duplicate Entities entries";
+                return false;
+            }
+
+            QVariantList entitiesValue;
+            if (!readEntitiesArray(entitiesValue)) {
+                return false;
+            }
+
+            parsedEntities["Entities"] = entitiesValue;
+            gotEntities = true;
+        } else if (key == "Id") {
+            if (gotId) {
+                _errorString = "Duplicate Id entries";
+                return false;
+            }
+
+            if (nextToken() != '"') {
+                _errorString = "Invalid Id value";
+                return false;
+            };
+            string idString = readString();
+            if (idString.size() == 0) {
+                _errorString = "Invalid Id string";
+                return false;
+            }
+            QUuid idValue = QUuid::fromString(QLatin1String(idString.c_str()) );
+            if (idValue.isNull()) {
+                _errorString = "Id value invalid UUID string: " + idString;
+                return false;
+            }
+
+            parsedEntities["Id"] = idValue;
+            gotId = true;
+        } else if (key == "Version") {
+            if (gotVersion) {
+                _errorString = "Duplicate Version entries";
+                return false;
+            }
+
+            int versionValue = readInteger();
+            parsedEntities["Version"] = versionValue;
+            gotVersion = true;
+        } else {
+            _errorString = "Unrecognized key name: " + key;
+            return false;
+        }
+
+        if (gotDataVersion && gotEntities && gotId && gotVersion) {
+            break;
+        } else if (nextToken() != ',') {
+            _errorString = "Id/value incorrectly terminated";
+            return false;
+        }
+    }
+
+    if (nextToken() != '}' || nextToken() != -1) {
+        _errorString = "Ill-formed end of object";
+        return false;
+    }
+
+    return true;
+}
+
+int OctreeEntitiesFileParser::nextToken() {
+    while (_position < _entitiesLength) {
+        char c = _entitiesContents[_position++];
+        if (c != ' ' && c != '\t' && c != '\n' && c != '\r') {
+            return c;
+        }
+        if (c == '\n') {
+            ++_line;
+        }
+    }
+
+    return -1;
+}
+
+string OctreeEntitiesFileParser::readString() {
+    string returnString;
+    while (_position < _entitiesLength) {
+        char c = _entitiesContents[_position++];
+        if (c == '"') {
+            break;
+        } else {
+            returnString.push_back(c);
+        }
+    }
+
+    return returnString;
+}
+
+int OctreeEntitiesFileParser::readInteger() {
+    const char* currentPosition = _entitiesContents.constData() + _position;
+    int i = atoi(currentPosition);
+
+    int token;
+    do {
+        token = nextToken();
+    } while (token == '-' || token == '+' || (token >= '0' && token <= '9'));
+
+    --_position;
+    return i;
+}
+
+bool OctreeEntitiesFileParser::readEntitiesArray(QVariantList& entitiesArray) {
+    if (nextToken() != '[') {
+        _errorString = "Entities entry is not an array";
+        return false;
+    }
+
+    while (true) {
+        QJsonParseError parseError;
+        QByteArray entitiesJson(_entitiesContents.right(_entitiesLength - _position));
+        QJsonDocument entity = QJsonDocument::fromJson(entitiesJson, &parseError);
+        if (parseError.error != QJsonParseError::GarbageAtEnd) {
+            _errorString = "Ill-formed entity array";
+            return false;
+        }
+        int entityLength = parseError.offset;
+        entitiesJson.truncate(entityLength);
+        _position += entityLength;
+
+        entity = QJsonDocument::fromJson(entitiesJson, &parseError);
+        if (parseError.error != QJsonParseError::NoError) {
+            _errorString = "Entity item parse error";
+            return false;
+        }
+        entitiesArray.append(entity.object());
+        char c = nextToken();
+        if (c == ']') {
+            return true;
+        } else if (c != ',') {
+            _errorString = "Entity array item incorrectly terminated";
+            return false;
+        }
+    }
+    return true;
+}
diff --git a/libraries/octree/src/OctreeEntitiesFileParser.h b/libraries/octree/src/OctreeEntitiesFileParser.h
new file mode 100644
index 0000000000..562a18e833
--- /dev/null
+++ b/libraries/octree/src/OctreeEntitiesFileParser.h
@@ -0,0 +1,38 @@
+//
+//  OctreeEntititesFileParser.h
+//  libraries/octree/src
+//
+//  Created by Simon Walton on Oct 15, 2018.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_OctreeEntitiesFileParser_h
+#define hifi_OctreeEntitiesFileParser_h
+
+#include <QVariantMap>
+#include <QByteArray>
+
+class OctreeEntitiesFileParser {
+public:
+    OctreeEntitiesFileParser();
+    void setEntitiesString(const QByteArray& entitiesContents);
+    bool parseEntities(QVariantMap& parsedEntities);
+    std::string getErrorString() const;
+
+private:
+    int nextToken();
+    std::string readString();
+    int readInteger();
+    bool readEntitiesArray(QVariantList& entitiesArray);
+
+    QByteArray _entitiesContents;
+    int _position { 0 };
+    int _line { 1 };
+    int _entitiesLength { 0 };
+    std::string _errorString;
+};
+
+#endif  // hifi_OctreeEntitiesFileParser_h

From c031769c67d0c5391b7fd13cd937a8576e8d20e7 Mon Sep 17 00:00:00 2001
From: Simon Walton <simon@highfidelity.io>
Date: Tue, 16 Oct 2018 18:08:07 -0700
Subject: [PATCH 06/14] Faster JSON entities parser

---
 libraries/octree/src/OctreeDataUtils.cpp      | 20 +++++--
 libraries/octree/src/OctreeDataUtils.h        |  1 +
 .../octree/src/OctreeEntitiesFileParser.cpp   | 58 +++++++++++++++----
 .../octree/src/OctreeEntitiesFileParser.h     |  3 +
 4 files changed, 67 insertions(+), 15 deletions(-)

diff --git a/libraries/octree/src/OctreeDataUtils.cpp b/libraries/octree/src/OctreeDataUtils.cpp
index a2b0d78209..7bbad652f8 100644
--- a/libraries/octree/src/OctreeDataUtils.cpp
+++ b/libraries/octree/src/OctreeDataUtils.cpp
@@ -9,6 +9,7 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 
 #include "OctreeDataUtils.h"
+#include "OctreeEntitiesFileParser.h"
 
 #include <Gzip.h>
 #include <udt/PacketHeaders.h>
@@ -55,19 +56,30 @@ bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromJSON(QJsonObject root) {
     return true;
 }
 
+bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromMap(QVariantMap map) {
+    if (map.contains("Id") && map.contains("DataVersion") && map.contains("Version")) {
+        id = map["Id"].toUuid();
+        dataVersion = map["DataVersion"].toInt();
+        version = map["Version"].toInt();
+    }
+    return true;
+}
+
 bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromData(QByteArray data) {
     QByteArray jsonData;
     if (gunzip(data, jsonData)) {
         data = jsonData;
     }
 
-    auto doc = QJsonDocument::fromJson(data);
-    if (doc.isNull()) {
+    OctreeEntitiesFileParser jsonParser;
+    jsonParser.setEntitiesString(data);
+    QVariantMap entitiesMap;
+    if (!jsonParser.parseEntities(entitiesMap)) {
+        qCritical() << "Can't parse Entities JSON: " << jsonParser.getErrorString().c_str();
         return false;
     }
 
-    auto root = doc.object();
-    return readOctreeDataInfoFromJSON(root);
+    return readOctreeDataInfoFromMap(entitiesMap);
 }
 
 // Reads octree file and parses it into a RawOctreeData object.
diff --git a/libraries/octree/src/OctreeDataUtils.h b/libraries/octree/src/OctreeDataUtils.h
index ef96415044..236d280bbf 100644
--- a/libraries/octree/src/OctreeDataUtils.h
+++ b/libraries/octree/src/OctreeDataUtils.h
@@ -43,6 +43,7 @@ public:
     bool readOctreeDataInfoFromData(QByteArray data);
     bool readOctreeDataInfoFromFile(QString path);
     bool readOctreeDataInfoFromJSON(QJsonObject root);
+    bool readOctreeDataInfoFromMap(QVariantMap map);
 };
 
 class RawEntityData : public RawOctreeData {
diff --git a/libraries/octree/src/OctreeEntitiesFileParser.cpp b/libraries/octree/src/OctreeEntitiesFileParser.cpp
index 14380de835..a0e43d0810 100644
--- a/libraries/octree/src/OctreeEntitiesFileParser.cpp
+++ b/libraries/octree/src/OctreeEntitiesFileParser.cpp
@@ -189,23 +189,25 @@ bool OctreeEntitiesFileParser::readEntitiesArray(QVariantList& entitiesArray) {
     }
 
     while (true) {
-        QJsonParseError parseError;
-        QByteArray entitiesJson(_entitiesContents.right(_entitiesLength - _position));
-        QJsonDocument entity = QJsonDocument::fromJson(entitiesJson, &parseError);
-        if (parseError.error != QJsonParseError::GarbageAtEnd) {
-            _errorString = "Ill-formed entity array";
+        if (nextToken() != '{') {
+            _errorString = "Entity array item is not an object";
+            return false;
+        }
+        int matchingBrace = findMatchingBrace();
+        if (matchingBrace < 0) {
+            _errorString = "Unterminated entity object";
             return false;
         }
-        int entityLength = parseError.offset;
-        entitiesJson.truncate(entityLength);
-        _position += entityLength;
 
-        entity = QJsonDocument::fromJson(entitiesJson, &parseError);
-        if (parseError.error != QJsonParseError::NoError) {
-            _errorString = "Entity item parse error";
+        QByteArray jsonEntity = _entitiesContents.mid(_position - 1, matchingBrace - _position + 1);
+        QJsonDocument entity = QJsonDocument::fromJson(jsonEntity);
+        if (entity.isNull()) {
+            _errorString = "Ill-formed entity";
             return false;
         }
+
         entitiesArray.append(entity.object());
+        _position = matchingBrace;
         char c = nextToken();
         if (c == ']') {
             return true;
@@ -216,3 +218,37 @@ bool OctreeEntitiesFileParser::readEntitiesArray(QVariantList& entitiesArray) {
     }
     return true;
 }
+
+int OctreeEntitiesFileParser::findMatchingBrace() const {
+    int index = _position;
+    int nestCount = 1;
+    while (index < _entitiesLength && nestCount != 0) {
+        switch (_entitiesContents[index++]) {
+        case '{':
+            ++nestCount;
+            break;
+
+        case '}':
+            --nestCount;
+            break;
+
+        case '"':
+            // Skip string
+            while (index < _entitiesLength) {
+                if (_entitiesContents[index] == '"') {
+                    ++index;
+                    break;
+                } else if (_entitiesContents[index] == '\\' && _entitiesContents[++index] == 'u') {
+                    index += 4;
+                }
+                ++index;
+            }
+            break;
+
+        default:
+            break;
+        }
+    }
+
+    return nestCount == 0 ? index : -1;
+}
diff --git a/libraries/octree/src/OctreeEntitiesFileParser.h b/libraries/octree/src/OctreeEntitiesFileParser.h
index 562a18e833..2daeb7e561 100644
--- a/libraries/octree/src/OctreeEntitiesFileParser.h
+++ b/libraries/octree/src/OctreeEntitiesFileParser.h
@@ -9,6 +9,8 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
+// Parse the top-level of the Models object ourselves - use QJsonDocument for each Entity object.
+
 #ifndef hifi_OctreeEntitiesFileParser_h
 #define hifi_OctreeEntitiesFileParser_h
 
@@ -27,6 +29,7 @@ private:
     std::string readString();
     int readInteger();
     bool readEntitiesArray(QVariantList& entitiesArray);
+    int findMatchingBrace() const;
 
     QByteArray _entitiesContents;
     int _position { 0 };

From be279f08dc3d8819002125bb5e36eae9c121e0e8 Mon Sep 17 00:00:00 2001
From: Simon Walton <simon@highfidelity.io>
Date: Wed, 17 Oct 2018 15:43:38 -0700
Subject: [PATCH 07/14] Restore the addMarketplaceIDToDocumentEntities() hack

Fixes for Linux compiles; other improvements
---
 libraries/entities/src/EntityTree.cpp         |  2 +-
 libraries/octree/src/Octree.cpp               | 35 +++++++++----------
 .../octree/src/OctreeEntitiesFileParser.cpp   |  6 ++--
 .../octree/src/OctreeEntitiesFileParser.h     |  2 +-
 libraries/octree/src/OctreePersistThread.cpp  |  2 +-
 5 files changed, 22 insertions(+), 25 deletions(-)

diff --git a/libraries/entities/src/EntityTree.cpp b/libraries/entities/src/EntityTree.cpp
index 19182d8b22..4cac2205a4 100644
--- a/libraries/entities/src/EntityTree.cpp
+++ b/libraries/entities/src/EntityTree.cpp
@@ -35,7 +35,7 @@
 #include "QVariantGLM.h"
 #include "EntitiesLogging.h"
 #include "RecurseOctreeToMapOperator.h"
-#include "RecurseOctreeToJsonOperator.h"
+#include "RecurseOctreeToJSONOperator.h"
 #include "LogHandler.h"
 #include "EntityEditFilters.h"
 #include "EntityDynamicFactoryInterface.h"
diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp
index 30108d973e..bbf1e44d08 100644
--- a/libraries/octree/src/Octree.cpp
+++ b/libraries/octree/src/Octree.cpp
@@ -785,28 +785,26 @@ bool Octree::readFromStream(uint64_t streamLength, QDataStream& inputStream, con
 }
 
 
+namespace {
 // hack to get the marketplace id into the entities.  We will create a way to get this from a hash of
 // the entity later, but this helps us move things along for now
-QJsonDocument addMarketplaceIDToDocumentEntities(QJsonDocument& doc, const QString& marketplaceID) {
+QVariantMap addMarketplaceIDToDocumentEntities(QVariantMap& doc, const QString& marketplaceID) {
     if (!marketplaceID.isEmpty()) {
-        QJsonDocument newDoc;
-        QJsonObject rootObj = doc.object();
-        QJsonArray newEntitiesArray;
+        QVariantList newEntitiesArray;
 
         // build a new entities array
-        auto entitiesArray = rootObj["Entities"].toArray();
-        for(auto it = entitiesArray.begin(); it != entitiesArray.end(); it++) {
-            auto entity = (*it).toObject();
+        auto entitiesArray = doc["Entities"].toList();
+        for (auto it = entitiesArray.begin(); it != entitiesArray.end(); it++) {
+            auto entity = (*it).toMap();
             entity["marketplaceID"] = marketplaceID;
             newEntitiesArray.append(entity);
         }
-        rootObj["Entities"] = newEntitiesArray;
-        newDoc.setObject(rootObj);
-        return newDoc;
+        doc["Entities"] = newEntitiesArray;
     }
     return doc;
 }
 
+}  // Unnamed namepsace
 const int READ_JSON_BUFFER_SIZE = 2048;
 
 bool Octree::readJSONFromStream(uint64_t streamLength, QDataStream& inputStream, const QString& marketplaceID /*=""*/) {
@@ -831,16 +829,15 @@ bool Octree::readJSONFromStream(uint64_t streamLength, QDataStream& inputStream,
     OctreeEntitiesFileParser octreeParser;
     octreeParser.setEntitiesString(jsonBuffer);
     QVariantMap asMap;
-    bool parseSuccess = octreeParser.parseEntities(asMap);
-
-    /*
-    QJsonDocument asDocument = QJsonDocument::fromJson(jsonBuffer);
-    if (!marketplaceID.isEmpty()) {
-        asDocument = addMarketplaceIDToDocumentEntities(asDocument, marketplaceID);
+    if (!octreeParser.parseEntities(asMap)) {
+        qCritical() << "Couldn't parse Entities JSON:" << octreeParser.getErrorString().c_str();
+        return false;
     }
-    QVariant asVariant = asDocument.toVariant();
-    QVariantMap asMap = asVariant.toMap();
-    */
+
+    if (!marketplaceID.isEmpty()) {
+        addMarketplaceIDToDocumentEntities(asMap, marketplaceID);
+    }
+
     bool success = readFromMap(asMap);
     delete[] rawData;
     return success;
diff --git a/libraries/octree/src/OctreeEntitiesFileParser.cpp b/libraries/octree/src/OctreeEntitiesFileParser.cpp
index a0e43d0810..0b75991fd2 100644
--- a/libraries/octree/src/OctreeEntitiesFileParser.cpp
+++ b/libraries/octree/src/OctreeEntitiesFileParser.cpp
@@ -10,10 +10,10 @@
 //
 
 #include <sstream>
+#include <cctype>
 #include <QUuid>
 #include <QJsonDocument>
 #include <QJsonObject>
-#include <QVariantList>
 
 #include "OctreeEntitiesFileParser.h"
 
@@ -171,12 +171,12 @@ string OctreeEntitiesFileParser::readString() {
 
 int OctreeEntitiesFileParser::readInteger() {
     const char* currentPosition = _entitiesContents.constData() + _position;
-    int i = atoi(currentPosition);
+    int i = std::atoi(currentPosition);
 
     int token;
     do {
         token = nextToken();
-    } while (token == '-' || token == '+' || (token >= '0' && token <= '9'));
+    } while (token == '-' || token == '+' || std::isdigit(token));
 
     --_position;
     return i;
diff --git a/libraries/octree/src/OctreeEntitiesFileParser.h b/libraries/octree/src/OctreeEntitiesFileParser.h
index 2daeb7e561..e4e82f0e66 100644
--- a/libraries/octree/src/OctreeEntitiesFileParser.h
+++ b/libraries/octree/src/OctreeEntitiesFileParser.h
@@ -14,8 +14,8 @@
 #ifndef hifi_OctreeEntitiesFileParser_h
 #define hifi_OctreeEntitiesFileParser_h
 
-#include <QVariantMap>
 #include <QByteArray>
+#include <QVariant>
 
 class OctreeEntitiesFileParser {
 public:
diff --git a/libraries/octree/src/OctreePersistThread.cpp b/libraries/octree/src/OctreePersistThread.cpp
index 0c78a1a797..2717be593b 100644
--- a/libraries/octree/src/OctreePersistThread.cpp
+++ b/libraries/octree/src/OctreePersistThread.cpp
@@ -158,7 +158,7 @@ void OctreePersistThread::handleOctreeDataFileReply(QSharedPointer<ReceivedMessa
             persistentFileRead = _tree->readFromFile(_filename.toLocal8Bit().constData());
         } else {
             QDataStream jsonStream(_cachedJSONData);
-            persistentFileRead = _tree->readFromStream(-1, QDataStream(_cachedJSONData));
+            persistentFileRead = _tree->readFromStream(-1, jsonStream);
         }
         _tree->pruneTree();
     });

From 6f6c92e6475503ae6fd05fd87f440b04700e5321 Mon Sep 17 00:00:00 2001
From: Simon Walton <simon@highfidelity.io>
Date: Tue, 30 Oct 2018 09:40:08 -0700
Subject: [PATCH 08/14] Remove unused variable

---
 libraries/entities/src/RecurseOctreeToJSONOperator.cpp | 5 ++---
 libraries/entities/src/RecurseOctreeToJSONOperator.h   | 3 +--
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/libraries/entities/src/RecurseOctreeToJSONOperator.cpp b/libraries/entities/src/RecurseOctreeToJSONOperator.cpp
index d67e7176a3..a865009566 100644
--- a/libraries/entities/src/RecurseOctreeToJSONOperator.cpp
+++ b/libraries/entities/src/RecurseOctreeToJSONOperator.cpp
@@ -12,10 +12,9 @@
 #include "RecurseOctreeToJSONOperator.h"
 #include "EntityItemProperties.h"
 
-RecurseOctreeToJSONOperator::RecurseOctreeToJSONOperator(const OctreeElementPointer& top, QScriptEngine* engine,
+RecurseOctreeToJSONOperator::RecurseOctreeToJSONOperator(const OctreeElementPointer&, QScriptEngine* engine,
     QString jsonPrefix /* = QString() */, bool skipDefaults /* = true */)
-    : _top(top)
-    , _engine(engine)
+    : _engine(engine)
     , _json(jsonPrefix)
     , _skipDefaults(skipDefaults)
 {
diff --git a/libraries/entities/src/RecurseOctreeToJSONOperator.h b/libraries/entities/src/RecurseOctreeToJSONOperator.h
index f661e08d5b..d0f03d02eb 100644
--- a/libraries/entities/src/RecurseOctreeToJSONOperator.h
+++ b/libraries/entities/src/RecurseOctreeToJSONOperator.h
@@ -13,7 +13,7 @@
 
 class RecurseOctreeToJSONOperator : public RecurseOctreeOperator {
 public:
-    RecurseOctreeToJSONOperator(const OctreeElementPointer& top, QScriptEngine* engine, QString jsonPrefix = QString(), bool skipDefaults = true);
+    RecurseOctreeToJSONOperator(const OctreeElementPointer&, QScriptEngine* engine, QString jsonPrefix = QString(), bool skipDefaults = true);
     virtual bool preRecursion(const OctreeElementPointer& element) override { return true; };
     virtual bool postRecursion(const OctreeElementPointer& element) override;
 
@@ -22,7 +22,6 @@ public:
 private:
     void processEntity(const EntityItemPointer& entity);
 
-    const OctreeElementPointer& _top;
     QScriptEngine* _engine;
     QScriptValue _toStringMethod;
 

From d183968175a98782385a511ebd0a17791943a6c4 Mon Sep 17 00:00:00 2001
From: Simon Walton <simon@highfidelity.io>
Date: Thu, 8 Nov 2018 18:20:20 -0800
Subject: [PATCH 09/14] Add skipThoseWithBadParents functionality to octree
 visitor; reviewer suggestions

---
 .../src/RecurseOctreeToJSONOperator.cpp       | 18 ++++++++-----
 .../src/RecurseOctreeToJSONOperator.h         |  6 +++--
 libraries/octree/src/Octree.cpp               | 26 -------------------
 libraries/octree/src/OctreeDataUtils.cpp      |  2 +-
 libraries/octree/src/OctreeDataUtils.h        |  2 +-
 .../octree/src/OctreeEntitiesFileParser.cpp   |  5 +---
 .../octree/src/OctreeEntitiesFileParser.h     |  1 -
 7 files changed, 18 insertions(+), 42 deletions(-)

diff --git a/libraries/entities/src/RecurseOctreeToJSONOperator.cpp b/libraries/entities/src/RecurseOctreeToJSONOperator.cpp
index a865009566..b64a700abc 100644
--- a/libraries/entities/src/RecurseOctreeToJSONOperator.cpp
+++ b/libraries/entities/src/RecurseOctreeToJSONOperator.cpp
@@ -13,10 +13,11 @@
 #include "EntityItemProperties.h"
 
 RecurseOctreeToJSONOperator::RecurseOctreeToJSONOperator(const OctreeElementPointer&, QScriptEngine* engine,
-    QString jsonPrefix /* = QString() */, bool skipDefaults /* = true */)
-    : _engine(engine)
-    , _json(jsonPrefix)
-    , _skipDefaults(skipDefaults)
+    QString jsonPrefix, bool skipDefaults, bool skipThoseWithBadParents):
+    _engine(engine),
+    _json(jsonPrefix),
+    _skipDefaults(skipDefaults),
+    _skipThoseWithBadParents(skipThoseWithBadParents)
 {
     _toStringMethod = _engine->evaluate("(function() { return JSON.stringify(this, null, '    ') })");
 }
@@ -29,18 +30,21 @@ bool RecurseOctreeToJSONOperator::postRecursion(const OctreeElementPointer& elem
 }
 
 void RecurseOctreeToJSONOperator::processEntity(const EntityItemPointer& entity) {
+    if (_skipThoseWithBadParents && !entity->isParentIDValid()) {
+        return;  // we weren't able to resolve a parent from _parentID, so don't save this entity.
+    }
+
     QScriptValue qScriptValues = _skipDefaults
         ? EntityItemNonDefaultPropertiesToScriptValue(_engine, entity->getProperties())
         : EntityItemPropertiesToScriptValue(_engine, entity->getProperties());
 
-    if (comma) {
+    if (_comma) {
         _json += ',';
     };
-    comma = true;
+    _comma = true;
     _json += "\n    ";
 
     // Override default toString():
     qScriptValues.setProperty("toString", _toStringMethod);
     _json += qScriptValues.toString();
-    //auto exceptionString2 = _engine->uncaughtException().toString();
 }
diff --git a/libraries/entities/src/RecurseOctreeToJSONOperator.h b/libraries/entities/src/RecurseOctreeToJSONOperator.h
index d0f03d02eb..a1d388ed22 100644
--- a/libraries/entities/src/RecurseOctreeToJSONOperator.h
+++ b/libraries/entities/src/RecurseOctreeToJSONOperator.h
@@ -13,7 +13,8 @@
 
 class RecurseOctreeToJSONOperator : public RecurseOctreeOperator {
 public:
-    RecurseOctreeToJSONOperator(const OctreeElementPointer&, QScriptEngine* engine, QString jsonPrefix = QString(), bool skipDefaults = true);
+    RecurseOctreeToJSONOperator(const OctreeElementPointer&, QScriptEngine* engine, QString jsonPrefix = QString(), bool skipDefaults = true,
+        bool skipThoseWithBadParents = false);
     virtual bool preRecursion(const OctreeElementPointer& element) override { return true; };
     virtual bool postRecursion(const OctreeElementPointer& element) override;
 
@@ -27,5 +28,6 @@ private:
 
     QString _json;
     const bool _skipDefaults;
-    bool comma { false };
+    bool _skipThoseWithBadParents;
+    bool _comma { false };
 };
diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp
index bbf1e44d08..cfa207dbf8 100644
--- a/libraries/octree/src/Octree.cpp
+++ b/libraries/octree/src/Octree.cpp
@@ -925,9 +925,6 @@ bool Octree::toJSONString(QString& jsonString, const OctreeElementPointer& eleme
 }
 
 bool Octree::toJSON(QByteArray* data, const OctreeElementPointer& element, bool doGzip) {
-#define HIFI_USE_DIRECT_TO_JSON
-#ifdef HIFI_USE_DIRECT_TO_JSON
-
     QString jsonString;
     toJSONString(jsonString);
 
@@ -940,29 +937,6 @@ bool Octree::toJSON(QByteArray* data, const OctreeElementPointer& element, bool
         *data = jsonString.toUtf8();
     }
 
-#else
-
-    QJsonDocument doc;
-    if (!toJSONDocument(&doc, element)) {
-        qCritical("Failed to convert Entities to JSON document.");
-        return false;
-    }
-
-    QString jsonString;
-    toJSONString(jsonString);
-
-    if (doGzip) {
-        QByteArray jsonData = doc.toJson();
-
-        if (!gzip(jsonData, *data, -1)) {
-            qCritical("Unable to gzip data while saving to json.");
-            return false;
-        }
-    } else {
-        *data = doc.toJson();
-    }
-
-#endif  // HIFI_USE_DIRECT_TO_JSON
     return true;
 }
 
diff --git a/libraries/octree/src/OctreeDataUtils.cpp b/libraries/octree/src/OctreeDataUtils.cpp
index 7bbad652f8..a94fa31800 100644
--- a/libraries/octree/src/OctreeDataUtils.cpp
+++ b/libraries/octree/src/OctreeDataUtils.cpp
@@ -56,7 +56,7 @@ bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromJSON(QJsonObject root) {
     return true;
 }
 
-bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromMap(QVariantMap map) {
+bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromMap(const QVariantMap& map) {
     if (map.contains("Id") && map.contains("DataVersion") && map.contains("Version")) {
         id = map["Id"].toUuid();
         dataVersion = map["DataVersion"].toInt();
diff --git a/libraries/octree/src/OctreeDataUtils.h b/libraries/octree/src/OctreeDataUtils.h
index 236d280bbf..0dfa99a9a0 100644
--- a/libraries/octree/src/OctreeDataUtils.h
+++ b/libraries/octree/src/OctreeDataUtils.h
@@ -43,7 +43,7 @@ public:
     bool readOctreeDataInfoFromData(QByteArray data);
     bool readOctreeDataInfoFromFile(QString path);
     bool readOctreeDataInfoFromJSON(QJsonObject root);
-    bool readOctreeDataInfoFromMap(QVariantMap map);
+    bool readOctreeDataInfoFromMap(const QVariantMap& map);
 };
 
 class RawEntityData : public RawOctreeData {
diff --git a/libraries/octree/src/OctreeEntitiesFileParser.cpp b/libraries/octree/src/OctreeEntitiesFileParser.cpp
index 0b75991fd2..72853947c6 100644
--- a/libraries/octree/src/OctreeEntitiesFileParser.cpp
+++ b/libraries/octree/src/OctreeEntitiesFileParser.cpp
@@ -19,9 +19,6 @@
 
 using std::string;
 
-OctreeEntitiesFileParser::OctreeEntitiesFileParser() {
-}
-
 std::string OctreeEntitiesFileParser::getErrorString() const {
     std::ostringstream err;
     if (_errorString.size() != 0) {
@@ -86,7 +83,7 @@ bool OctreeEntitiesFileParser::parseEntities(QVariantMap& parsedEntities) {
                 return false;
             }
 
-            parsedEntities["Entities"] = entitiesValue;
+            parsedEntities["Entities"] = std::move(entitiesValue);
             gotEntities = true;
         } else if (key == "Id") {
             if (gotId) {
diff --git a/libraries/octree/src/OctreeEntitiesFileParser.h b/libraries/octree/src/OctreeEntitiesFileParser.h
index e4e82f0e66..bc51896b18 100644
--- a/libraries/octree/src/OctreeEntitiesFileParser.h
+++ b/libraries/octree/src/OctreeEntitiesFileParser.h
@@ -19,7 +19,6 @@
 
 class OctreeEntitiesFileParser {
 public:
-    OctreeEntitiesFileParser();
     void setEntitiesString(const QByteArray& entitiesContents);
     bool parseEntities(QVariantMap& parsedEntities);
     std::string getErrorString() const;

From 8c347fae7082a746d594f3a3b5fb743618addcc8 Mon Sep 17 00:00:00 2001
From: Simon Walton <simon@highfidelity.io>
Date: Mon, 12 Nov 2018 12:45:01 -0800
Subject: [PATCH 10/14] Fixes for domain server use of changed OctreeDataUtils

---
 libraries/octree/src/OctreeDataUtils.cpp | 33 +++++++++++++++++++-----
 libraries/octree/src/OctreeDataUtils.h   |  3 +++
 2 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/libraries/octree/src/OctreeDataUtils.cpp b/libraries/octree/src/OctreeDataUtils.cpp
index a94fa31800..255933a783 100644
--- a/libraries/octree/src/OctreeDataUtils.cpp
+++ b/libraries/octree/src/OctreeDataUtils.cpp
@@ -44,6 +44,7 @@ namespace {
         }
         return !doc->isNull();
     }
+
 }  // Anon namespace.
 
 bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromJSON(QJsonObject root) {
@@ -62,6 +63,7 @@ bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromMap(const QVariantMap& ma
         dataVersion = map["DataVersion"].toInt();
         version = map["Version"].toInt();
     }
+    readSubclassData(map);
     return true;
 }
 
@@ -85,13 +87,20 @@ bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromData(QByteArray data) {
 // Reads octree file and parses it into a RawOctreeData object.
 // Returns false if readOctreeFile fails.
 bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromFile(QString path) {
-    QJsonDocument doc;
-    if (!readOctreeFile(path, &doc)) {
+    QFile file(path);
+    if (!file.open(QIODevice::ReadOnly)) {
+        qCritical() << "Cannot open json file for reading: " << path;
         return false;
     }
 
-    auto root = doc.object();
-    return readOctreeDataInfoFromJSON(root);
+    QByteArray data = file.readAll();
+    QByteArray jsonData;
+
+    if (!gunzip(data, jsonData)) {
+        jsonData = data;
+    }
+
+    return readOctreeDataInfoFromData(jsonData);
 }
 
 QByteArray OctreeUtils::RawOctreeData::toByteArray() {
@@ -139,8 +148,18 @@ void OctreeUtils::RawEntityData::readSubclassData(const QJsonObject& root) {
     }
 }
 
-void OctreeUtils::RawEntityData::writeSubclassData(QJsonObject& root) const {
-    root["Entities"] = entityData;
+void OctreeUtils::RawEntityData::readSubclassData(const QVariantMap& root) {
+    variantEntityData = root["Entities"].toList();
 }
 
-PacketType OctreeUtils::RawEntityData::dataPacketType() const { return PacketType::EntityData; }
\ No newline at end of file
+void OctreeUtils::RawEntityData::writeSubclassData(QJsonObject& root) const {
+    //root["Entities"] = entityData;
+    QJsonArray entitiesJsonArray;
+    for (auto entityIter = variantEntityData.begin(); entityIter != variantEntityData.end(); ++entityIter) {
+        entitiesJsonArray += entityIter->toJsonObject();
+    }
+
+    root["Entities"] = entitiesJsonArray;
+}
+
+PacketType OctreeUtils::RawEntityData::dataPacketType() const { return PacketType::EntityData; }
diff --git a/libraries/octree/src/OctreeDataUtils.h b/libraries/octree/src/OctreeDataUtils.h
index 0dfa99a9a0..20092cd451 100644
--- a/libraries/octree/src/OctreeDataUtils.h
+++ b/libraries/octree/src/OctreeDataUtils.h
@@ -34,6 +34,7 @@ public:
     virtual PacketType dataPacketType() const;
 
     virtual void readSubclassData(const QJsonObject& root) { }
+    virtual void readSubclassData(const QVariantMap& root) { }
     virtual void writeSubclassData(QJsonObject& root) const { }
 
     void resetIdAndVersion();
@@ -50,9 +51,11 @@ class RawEntityData : public RawOctreeData {
 public:
     PacketType dataPacketType() const override;
     void readSubclassData(const QJsonObject& root) override;
+    void readSubclassData(const QVariantMap& root) override;
     void writeSubclassData(QJsonObject& root) const override;
 
     QJsonArray entityData;
+    QVariantList variantEntityData;
 };
 
 }

From 6ee837d47c2f198e2284828baefff5687cd86534 Mon Sep 17 00:00:00 2001
From: Simon Walton <simon@highfidelity.io>
Date: Mon, 12 Nov 2018 15:43:09 -0800
Subject: [PATCH 11/14] OctreeDataUtils - use new json from DS also; allow
 upload of large domains

---
 domain-server/src/DomainServer.cpp       | 13 +-----
 libraries/octree/src/OctreeDataUtils.cpp | 58 +++++++++---------------
 libraries/octree/src/OctreeDataUtils.h   |  8 +---
 3 files changed, 25 insertions(+), 54 deletions(-)

diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp
index 8136bf3b53..f90155c786 100644
--- a/domain-server/src/DomainServer.cpp
+++ b/domain-server/src/DomainServer.cpp
@@ -3411,20 +3411,11 @@ void DomainServer::maybeHandleReplacementEntityFile() {
 }
 
 void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) {
-    //Assume we have compressed data
-    auto compressedOctree = octreeFile;
-    QByteArray jsonOctree;
-
-    bool wasCompressed = gunzip(compressedOctree, jsonOctree);
-    if (!wasCompressed) {
-        // the source was not compressed, assume we were sent regular JSON data
-        jsonOctree = compressedOctree;
-    }
-
     OctreeUtils::RawEntityData data;
-    if (data.readOctreeDataInfoFromData(jsonOctree)) {
+    if (data.readOctreeDataInfoFromData(octreeFile)) {
         data.resetIdAndVersion();
 
+        QByteArray compressedOctree;
         gzip(data.toByteArray(), compressedOctree);
 
         // write the compressed octree data to a special file
diff --git a/libraries/octree/src/OctreeDataUtils.cpp b/libraries/octree/src/OctreeDataUtils.cpp
index 255933a783..8c6e45ae66 100644
--- a/libraries/octree/src/OctreeDataUtils.cpp
+++ b/libraries/octree/src/OctreeDataUtils.cpp
@@ -47,16 +47,6 @@ namespace {
 
 }  // Anon namespace.
 
-bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromJSON(QJsonObject root) {
-    if (root.contains("Id") && root.contains("DataVersion") && root.contains("Version")) {
-        id = root["Id"].toVariant().toUuid();
-        dataVersion = root["DataVersion"].toInt();
-        version = root["Version"].toInt();
-    }
-    readSubclassData(root);
-    return true;
-}
-
 bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromMap(const QVariantMap& map) {
     if (map.contains("Id") && map.contains("DataVersion") && map.contains("Version")) {
         id = map["Id"].toUuid();
@@ -94,28 +84,25 @@ bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromFile(QString path) {
     }
 
     QByteArray data = file.readAll();
-    QByteArray jsonData;
 
-    if (!gunzip(data, jsonData)) {
-        jsonData = data;
-    }
-
-    return readOctreeDataInfoFromData(jsonData);
+    return readOctreeDataInfoFromData(data);
 }
 
 QByteArray OctreeUtils::RawOctreeData::toByteArray() {
-    QJsonObject obj {
-        { "DataVersion", QJsonValue((qint64)dataVersion) },
-        { "Id", QJsonValue(id.toString()) },
-        { "Version", QJsonValue((qint64)version) },
-    };
+    QByteArray jsonString;
 
-    writeSubclassData(obj);
+    jsonString += QString(R"({
+  "DataVersion": %1,
+)").arg(dataVersion);
 
-    QJsonDocument doc;
-    doc.setObject(obj);
+    writeSubclassData(jsonString);
 
-    return doc.toJson();
+    jsonString += QString(R"(,
+  "Id": "%1",
+  "Version": %2
+})").arg(id.toString()).arg(version);
+
+    return jsonString;
 }
 
 QByteArray OctreeUtils::RawOctreeData::toGzippedByteArray() {
@@ -142,24 +129,21 @@ void OctreeUtils::RawOctreeData::resetIdAndVersion() {
     qDebug() << "Reset octree data to: " << id << dataVersion;
 }
 
-void OctreeUtils::RawEntityData::readSubclassData(const QJsonObject& root) {
-    if (root.contains("Entities")) {
-        entityData = root["Entities"].toArray();
-    }
-}
-
 void OctreeUtils::RawEntityData::readSubclassData(const QVariantMap& root) {
     variantEntityData = root["Entities"].toList();
 }
 
-void OctreeUtils::RawEntityData::writeSubclassData(QJsonObject& root) const {
-    //root["Entities"] = entityData;
-    QJsonArray entitiesJsonArray;
+void OctreeUtils::RawEntityData::writeSubclassData(QByteArray& root) const {
+    root += "  \"Entities\": [";
     for (auto entityIter = variantEntityData.begin(); entityIter != variantEntityData.end(); ++entityIter) {
-        entitiesJsonArray += entityIter->toJsonObject();
+        if (entityIter != variantEntityData.begin()) {
+            root += ",";
+        }
+        root += "\n    ";
+        // Convert to string and remove trailing LF.
+        root += QJsonDocument(entityIter->toJsonObject()).toJson().chopped(1);
     }
-
-    root["Entities"] = entitiesJsonArray;
+    root += "]";
 }
 
 PacketType OctreeUtils::RawEntityData::dataPacketType() const { return PacketType::EntityData; }
diff --git a/libraries/octree/src/OctreeDataUtils.h b/libraries/octree/src/OctreeDataUtils.h
index 20092cd451..642c00a675 100644
--- a/libraries/octree/src/OctreeDataUtils.h
+++ b/libraries/octree/src/OctreeDataUtils.h
@@ -33,9 +33,8 @@ public:
 
     virtual PacketType dataPacketType() const;
 
-    virtual void readSubclassData(const QJsonObject& root) { }
     virtual void readSubclassData(const QVariantMap& root) { }
-    virtual void writeSubclassData(QJsonObject& root) const { }
+    virtual void writeSubclassData(QByteArray& root) const { }
 
     void resetIdAndVersion();
     QByteArray toByteArray();
@@ -43,18 +42,15 @@ public:
 
     bool readOctreeDataInfoFromData(QByteArray data);
     bool readOctreeDataInfoFromFile(QString path);
-    bool readOctreeDataInfoFromJSON(QJsonObject root);
     bool readOctreeDataInfoFromMap(const QVariantMap& map);
 };
 
 class RawEntityData : public RawOctreeData {
 public:
     PacketType dataPacketType() const override;
-    void readSubclassData(const QJsonObject& root) override;
     void readSubclassData(const QVariantMap& root) override;
-    void writeSubclassData(QJsonObject& root) const override;
+    void writeSubclassData(QByteArray& root) const override;
 
-    QJsonArray entityData;
     QVariantList variantEntityData;
 };
 

From df2bc6ba5cbf1236f6ba89a2085eadca35df88de Mon Sep 17 00:00:00 2001
From: Simon Walton <simon@highfidelity.io>
Date: Tue, 13 Nov 2018 10:18:30 -0800
Subject: [PATCH 12/14] Remove unused function

---
 libraries/octree/src/OctreeDataUtils.cpp | 28 ------------------------
 1 file changed, 28 deletions(-)

diff --git a/libraries/octree/src/OctreeDataUtils.cpp b/libraries/octree/src/OctreeDataUtils.cpp
index 8c6e45ae66..f3eed084e0 100644
--- a/libraries/octree/src/OctreeDataUtils.cpp
+++ b/libraries/octree/src/OctreeDataUtils.cpp
@@ -19,34 +19,6 @@
 #include <QJsonDocument>
 #include <QFile>
 
-namespace {
-    // Reads octree file and parses it into a QJsonDocument. Handles both gzipped and non-gzipped files.
-    // Returns true if the file was successfully opened and parsed, otherwise false.
-    // Example failures: file does not exist, gzipped file cannot be unzipped, invalid JSON.
-    bool readOctreeFile(QString path, QJsonDocument* doc) {
-        QFile file(path);
-        if (!file.open(QIODevice::ReadOnly)) {
-            qCritical() << "Cannot open json file for reading: " << path;
-            return false;
-        }
-
-        QByteArray data = file.readAll();
-        QByteArray jsonData;
-
-        if (!gunzip(data, jsonData)) {
-            jsonData = data;
-        }
-
-        QJsonParseError parserError;
-        *doc = QJsonDocument::fromJson(jsonData, &parserError);
-        if (parserError.error != QJsonParseError::NoError) {
-            qWarning() << "Error reading JSON file" << path << "-" << parserError.errorString();
-        }
-        return !doc->isNull();
-    }
-
-}  // Anon namespace.
-
 bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromMap(const QVariantMap& map) {
     if (map.contains("Id") && map.contains("DataVersion") && map.contains("Version")) {
         id = map["Id"].toUuid();

From a65466f1546ff42a35d11e8fcc0188dfe1ebb169 Mon Sep 17 00:00:00 2001
From: Simon Walton <simon@highfidelity.io>
Date: Tue, 13 Nov 2018 13:33:36 -0800
Subject: [PATCH 13/14] Use traditional C strings rather than raw strings

---
 libraries/octree/src/Octree.cpp          | 11 ++---------
 libraries/octree/src/OctreeDataUtils.cpp |  9 ++-------
 2 files changed, 4 insertions(+), 16 deletions(-)

diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp
index cfa207dbf8..c184fe122d 100644
--- a/libraries/octree/src/Octree.cpp
+++ b/libraries/octree/src/Octree.cpp
@@ -904,9 +904,7 @@ bool Octree::toJSONString(QString& jsonString, const OctreeElementPointer& eleme
         top = _rootElement;
     }
 
-    jsonString += QString(R"({
-  "DataVersion": %1,
-  "Entities": [)").arg(_persistDataVersion);
+    jsonString += QString("{\n  \"DataVersion\": %1,\n  \"Entities\": [").arg(_persistDataVersion);
 
     writeToJSON(jsonString, top);
 
@@ -914,12 +912,7 @@ bool Octree::toJSONString(QString& jsonString, const OctreeElementPointer& eleme
     PacketType expectedType = expectedDataPacketType();
     PacketVersion expectedVersion = versionForPacketType(expectedType);
 
-    jsonString += QString(R"(
-    ],
-  "Id": "%1",
-  "Version": %2
-}
-)").arg(_persistID.toString()).arg((int)expectedVersion);
+    jsonString += QString("\n    ],\n  \"Id\": \"%1\",\n  \"Version\": %2\n}\n").arg(_persistID.toString()).arg((int)expectedVersion);
 
     return true;
 }
diff --git a/libraries/octree/src/OctreeDataUtils.cpp b/libraries/octree/src/OctreeDataUtils.cpp
index f3eed084e0..b861904255 100644
--- a/libraries/octree/src/OctreeDataUtils.cpp
+++ b/libraries/octree/src/OctreeDataUtils.cpp
@@ -63,16 +63,11 @@ bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromFile(QString path) {
 QByteArray OctreeUtils::RawOctreeData::toByteArray() {
     QByteArray jsonString;
 
-    jsonString += QString(R"({
-  "DataVersion": %1,
-)").arg(dataVersion);
+    jsonString += QString("{\n  \"DataVersion\": %1,\n").arg(dataVersion);
 
     writeSubclassData(jsonString);
 
-    jsonString += QString(R"(,
-  "Id": "%1",
-  "Version": %2
-})").arg(id.toString()).arg(version);
+    jsonString += QString(",\n  \"Id\": \"%1\",\n  \"Version\": %2\n}").arg(id.toString()).arg(version);
 
     return jsonString;
 }

From 452c212c4f43fd912606b6da6c144308bcb52633 Mon Sep 17 00:00:00 2001
From: Simon Walton <simon@highfidelity.io>
Date: Tue, 13 Nov 2018 17:58:12 -0800
Subject: [PATCH 14/14] Add Paths entry parsing to json parser

---
 .../octree/src/OctreeEntitiesFileParser.cpp   | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/libraries/octree/src/OctreeEntitiesFileParser.cpp b/libraries/octree/src/OctreeEntitiesFileParser.cpp
index 72853947c6..873eaff0e1 100644
--- a/libraries/octree/src/OctreeEntitiesFileParser.cpp
+++ b/libraries/octree/src/OctreeEntitiesFileParser.cpp
@@ -117,6 +117,28 @@ bool OctreeEntitiesFileParser::parseEntities(QVariantMap& parsedEntities) {
             int versionValue = readInteger();
             parsedEntities["Version"] = versionValue;
             gotVersion = true;
+        } else if (key == "Paths") {
+            // Serverless JSON has optional Paths entry.
+            if (nextToken() != '{') {
+                _errorString = "Paths item is not an object";
+                return false;
+            }
+
+            int matchingBrace = findMatchingBrace();
+            if (matchingBrace < 0) {
+                _errorString = "Unterminated entity object";
+                return false;
+            }
+
+            QByteArray jsonObject = _entitiesContents.mid(_position - 1, matchingBrace - _position + 1);
+            QJsonDocument pathsObject = QJsonDocument::fromJson(jsonObject);
+            if (pathsObject.isNull()) {
+                _errorString = "Ill-formed paths entry";
+                return false;
+            }
+
+            parsedEntities["Paths"] = pathsObject.object();
+            _position = matchingBrace;
         } else {
             _errorString = "Unrecognized key name: " + key;
             return false;