Merge pull request #14294 from SimonWalton-HiFi/max-json-size

Allow for large Models.json file

domain-server/src/DomainServer.cpp

@@ -3411,20 +3411,11 @@ void DomainServer::maybeHandleReplacementEntityFile() {
 }
 
 void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) {
-    //Assume we have compressed data
-    auto compressedOctree = octreeFile;
-    QByteArray jsonOctree;
-
-    bool wasCompressed = gunzip(compressedOctree, jsonOctree);
-    if (!wasCompressed) {
-        // the source was not compressed, assume we were sent regular JSON data
-        jsonOctree = compressedOctree;
-    }
-
     OctreeUtils::RawEntityData data;
-    if (data.readOctreeDataInfoFromData(jsonOctree)) {
+    if (data.readOctreeDataInfoFromData(octreeFile)) {
         data.resetIdAndVersion();
 
+        QByteArray compressedOctree;
         gzip(data.toByteArray(), compressedOctree);
 
         // write the compressed octree data to a special file

libraries/entities/src/EntityTree.cpp

@@ -35,6 +35,7 @@
 #include "QVariantGLM.h"
 #include "EntitiesLogging.h"
 #include "RecurseOctreeToMapOperator.h"
+#include "RecurseOctreeToJSONOperator.h"
 #include "LogHandler.h"
 #include "EntityEditFilters.h"
 #include "EntityDynamicFactoryInterface.h"
@@ -2785,6 +2786,17 @@ bool EntityTree::readFromMap(QVariantMap& map) {
     return success;
 }
 
+bool EntityTree::writeToJSON(QString& jsonString, const OctreeElementPointer& element) {
+    QScriptEngine scriptEngine;
+    RecurseOctreeToJSONOperator theOperator(element, &scriptEngine, jsonString);
+    withReadLock([&] {
+        recurseTreeWithOperator(&theOperator);
+    });
+
+    jsonString = theOperator.getJson();
+    return true;
+}
+
 void EntityTree::resetClientEditStats() {
     _treeResetTime = usecTimestampNow();
     _maxEditDelta = 0;

libraries/entities/src/EntityTree.h

@@ -224,6 +224,8 @@ public:
     virtual bool writeToMap(QVariantMap& entityDescription, OctreeElementPointer element, bool skipDefaultValues,
                             bool skipThoseWithBadParents) override;
     virtual bool readFromMap(QVariantMap& entityDescription) override;
+    virtual bool writeToJSON(QString& jsonString, const OctreeElementPointer& element) override;
+
 
     glm::vec3 getContentsDimensions();
     float getContentsLargestDimension();

libraries/entities/src/RecurseOctreeToJSONOperator.cpp

@@ -0,0 +1,50 @@
+//
+//  RecurseOctreeToJSONOperator.cpp
+//  libraries/entities/src
+//
+//  Created by Simon Walton on Oct 11, 2018.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "RecurseOctreeToJSONOperator.h"
+#include "EntityItemProperties.h"
+
+RecurseOctreeToJSONOperator::RecurseOctreeToJSONOperator(const OctreeElementPointer&, QScriptEngine* engine,
+    QString jsonPrefix, bool skipDefaults, bool skipThoseWithBadParents):
+    _engine(engine),
+    _json(jsonPrefix),
+    _skipDefaults(skipDefaults),
+    _skipThoseWithBadParents(skipThoseWithBadParents)
+{
+    _toStringMethod = _engine->evaluate("(function() { return JSON.stringify(this, null, '    ') })");
+}
+
+bool RecurseOctreeToJSONOperator::postRecursion(const OctreeElementPointer& element) {
+    EntityTreeElementPointer entityTreeElement = std::static_pointer_cast<EntityTreeElement>(element);
+
+    entityTreeElement->forEachEntity([&](const EntityItemPointer& entity) { processEntity(entity); } );
+    return true;
+}
+
+void RecurseOctreeToJSONOperator::processEntity(const EntityItemPointer& entity) {
+    if (_skipThoseWithBadParents && !entity->isParentIDValid()) {
+        return;  // we weren't able to resolve a parent from _parentID, so don't save this entity.
+    }
+
+    QScriptValue qScriptValues = _skipDefaults
+        ? EntityItemNonDefaultPropertiesToScriptValue(_engine, entity->getProperties())
+        : EntityItemPropertiesToScriptValue(_engine, entity->getProperties());
+
+    if (_comma) {
+        _json += ',';
+    };
+    _comma = true;
+    _json += "\n    ";
+
+    // Override default toString():
+    qScriptValues.setProperty("toString", _toStringMethod);
+    _json += qScriptValues.toString();
+}

libraries/entities/src/RecurseOctreeToJSONOperator.h

@@ -0,0 +1,33 @@
+//
+//  RecurseOctreeToJSONOperator.h
+//  libraries/entities/src
+//
+//  Created by Simon Walton on Oct 11, 2018.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "EntityTree.h"
+
+class RecurseOctreeToJSONOperator : public RecurseOctreeOperator {
+public:
+    RecurseOctreeToJSONOperator(const OctreeElementPointer&, QScriptEngine* engine, QString jsonPrefix = QString(), bool skipDefaults = true,
+        bool skipThoseWithBadParents = false);
+    virtual bool preRecursion(const OctreeElementPointer& element) override { return true; };
+    virtual bool postRecursion(const OctreeElementPointer& element) override;
+
+    QString getJson() const { return _json; }
+
+private:
+    void processEntity(const EntityItemPointer& entity);
+
+    QScriptEngine* _engine;
+    QScriptValue _toStringMethod;
+
+    QString _json;
+    const bool _skipDefaults;
+    bool _skipThoseWithBadParents;
+    bool _comma { false };
+};

libraries/octree/src/Octree.cpp

@@ -50,7 +50,7 @@
 #include "OctreeLogging.h"
 #include "OctreeQueryNode.h"
 #include "OctreeUtils.h"
-
+#include "OctreeEntitiesFileParser.h"
 
 QVector<QString> PERSIST_EXTENSIONS = {"json", "json.gz"};
 
@@ -792,28 +792,26 @@ bool Octree::readFromStream(
 }
 
 
+namespace {
 // hack to get the marketplace id into the entities.  We will create a way to get this from a hash of
 // the entity later, but this helps us move things along for now
-QJsonDocument addMarketplaceIDToDocumentEntities(QJsonDocument& doc, const QString& marketplaceID) {
+QVariantMap addMarketplaceIDToDocumentEntities(QVariantMap& doc, const QString& marketplaceID) {
     if (!marketplaceID.isEmpty()) {
-        QJsonDocument newDoc;
-        QJsonObject rootObj = doc.object();
-        QJsonArray newEntitiesArray;
+        QVariantList newEntitiesArray;
 
         // build a new entities array
-        auto entitiesArray = rootObj["Entities"].toArray();
-        for(auto it = entitiesArray.begin(); it != entitiesArray.end(); it++) {
-            auto entity = (*it).toObject();
+        auto entitiesArray = doc["Entities"].toList();
+        for (auto it = entitiesArray.begin(); it != entitiesArray.end(); it++) {
+            auto entity = (*it).toMap();
             entity["marketplaceID"] = marketplaceID;
             newEntitiesArray.append(entity);
         }
-        rootObj["Entities"] = newEntitiesArray;
-        newDoc.setObject(rootObj);
-        return newDoc;
+        doc["Entities"] = newEntitiesArray;
     }
     return doc;
 }
 
+}  // Unnamed namepsace
 const int READ_JSON_BUFFER_SIZE = 2048;
 
 bool Octree::readJSONFromStream(
@@ -839,12 +837,18 @@ bool Octree::readJSONFromStream(
         jsonBuffer += QByteArray(rawData, got);
     }
 
-    QJsonDocument asDocument = QJsonDocument::fromJson(jsonBuffer);
-    if (!marketplaceID.isEmpty()) {
-        asDocument = addMarketplaceIDToDocumentEntities(asDocument, marketplaceID);
+    OctreeEntitiesFileParser octreeParser;
+    octreeParser.setEntitiesString(jsonBuffer);
+    QVariantMap asMap;
+    if (!octreeParser.parseEntities(asMap)) {
+        qCritical() << "Couldn't parse Entities JSON:" << octreeParser.getErrorString().c_str();
+        return false;
     }
-    QVariant asVariant = asDocument.toVariant();
-    QVariantMap asMap = asVariant.toMap();
+
+    if (!marketplaceID.isEmpty()) {
+        addMarketplaceIDToDocumentEntities(asMap, marketplaceID);
+    }
+
     bool success = readFromMap(asMap);
     delete[] rawData;
     return success;
@@ -889,26 +893,52 @@ bool Octree::toJSONDocument(QJsonDocument* doc, const OctreeElementPointer& elem
         return false;
     }
 
-    *doc = QJsonDocument::fromVariant(entityDescription);
+
+    bool noEntities = entityDescription["Entities"].toList().empty();
+    QJsonDocument jsonDocTree = QJsonDocument::fromVariant(entityDescription);
+    QJsonValue entitiesJson = jsonDocTree["Entities"];
+    if (entitiesJson.isNull() || (entitiesJson.toArray().empty() && !noEntities)) {
+        // Json version of entities too large.
+        return false;
+    } else {
+        *doc = jsonDocTree;
+    }
+
+    return true;
+}
+
+bool Octree::toJSONString(QString& jsonString, const OctreeElementPointer& element) {
+    OctreeElementPointer top;
+    if (element) {
+        top = element;
+    } else {
+        top = _rootElement;
+    }
+
+    jsonString += QString("{\n  \"DataVersion\": %1,\n  \"Entities\": [").arg(_persistDataVersion);
+
+    writeToJSON(jsonString, top);
+
+    // include the "bitstream" version
+    PacketType expectedType = expectedDataPacketType();
+    PacketVersion expectedVersion = versionForPacketType(expectedType);
+
+    jsonString += QString("\n    ],\n  \"Id\": \"%1\",\n  \"Version\": %2\n}\n").arg(_persistID.toString()).arg((int)expectedVersion);
+
     return true;
 }
 
 bool Octree::toJSON(QByteArray* data, const OctreeElementPointer& element, bool doGzip) {
-    QJsonDocument doc;
-    if (!toJSONDocument(&doc, element)) {
-        qCritical("Failed to convert Entities to QVariantMap while converting to json.");
-        return false;
-    }
+    QString jsonString;
+    toJSONString(jsonString);
 
     if (doGzip) {
-        QByteArray jsonData = doc.toJson();
-
-        if (!gzip(jsonData, *data, -1)) {
+        if (!gzip(jsonString.toUtf8(), *data, -1)) {
             qCritical("Unable to gzip data while saving to json.");
             return false;
         }
     } else {
-        *data = doc.toJson();
+        *data = jsonString.toUtf8();
     }
 
     return true;

libraries/octree/src/Octree.h

@@ -202,11 +202,13 @@ public:
 
     // Octree exporters
     bool toJSONDocument(QJsonDocument* doc, const OctreeElementPointer& element = nullptr);
+    bool toJSONString(QString& jsonString, const OctreeElementPointer& element = nullptr);
     bool toJSON(QByteArray* data, const OctreeElementPointer& element = nullptr, bool doGzip = false);
     bool writeToFile(const char* filename, const OctreeElementPointer& element = nullptr, QString persistAsFileType = "json.gz");
     bool writeToJSONFile(const char* filename, const OctreeElementPointer& element = nullptr, bool doGzip = false);
     virtual bool writeToMap(QVariantMap& entityDescription, OctreeElementPointer element, bool skipDefaultValues,
                             bool skipThoseWithBadParents) = 0;
+    virtual bool writeToJSON(QString& jsonString, const OctreeElementPointer& element) = 0;
 
     // Octree importers
     bool readFromFile(const char* filename);

libraries/octree/src/OctreeDataUtils.cpp

@@ -9,6 +9,7 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 
 #include "OctreeDataUtils.h"
+#include "OctreeEntitiesFileParser.h"
 
 #include <Gzip.h>
 #include <udt/PacketHeaders.h>
@@ -18,33 +19,13 @@
 #include <QJsonDocument>
 #include <QFile>
 
-// Reads octree file and parses it into a QJsonDocument. Handles both gzipped and non-gzipped files.
-// Returns true if the file was successfully opened and parsed, otherwise false.
-// Example failures: file does not exist, gzipped file cannot be unzipped, invalid JSON.
-bool readOctreeFile(QString path, QJsonDocument* doc) {
-    QFile file(path);
-    if (!file.open(QIODevice::ReadOnly)) {
-        return false;
+bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromMap(const QVariantMap& map) {
+    if (map.contains("Id") && map.contains("DataVersion") && map.contains("Version")) {
+        id = map["Id"].toUuid();
+        dataVersion = map["DataVersion"].toInt();
+        version = map["Version"].toInt();
     }
-
-    QByteArray data = file.readAll();
-    QByteArray jsonData;
-
-    if (!gunzip(data, jsonData)) {
-        jsonData = data;
-    }
-
-    *doc = QJsonDocument::fromJson(jsonData);
-    return !doc->isNull();
-}
-
-bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromJSON(QJsonObject root) {
-    if (root.contains("Id") && root.contains("DataVersion") && root.contains("Version")) {
-        id = root["Id"].toVariant().toUuid();
-        dataVersion = root["DataVersion"].toInt();
-        version = root["Version"].toInt();
-    }
-    readSubclassData(root);
+    readSubclassData(map);
     return true;
 }
 
@@ -54,40 +35,41 @@ bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromData(QByteArray data) {
         data = jsonData;
     }
 
-    auto doc = QJsonDocument::fromJson(data);
-    if (doc.isNull()) {
+    OctreeEntitiesFileParser jsonParser;
+    jsonParser.setEntitiesString(data);
+    QVariantMap entitiesMap;
+    if (!jsonParser.parseEntities(entitiesMap)) {
+        qCritical() << "Can't parse Entities JSON: " << jsonParser.getErrorString().c_str();
         return false;
     }
 
-    auto root = doc.object();
-    return readOctreeDataInfoFromJSON(root);
+    return readOctreeDataInfoFromMap(entitiesMap);
 }
 
 // Reads octree file and parses it into a RawOctreeData object.
 // Returns false if readOctreeFile fails.
 bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromFile(QString path) {
-    QJsonDocument doc;
-    if (!readOctreeFile(path, &doc)) {
+    QFile file(path);
+    if (!file.open(QIODevice::ReadOnly)) {
+        qCritical() << "Cannot open json file for reading: " << path;
         return false;
     }
 
-    auto root = doc.object();
-    return readOctreeDataInfoFromJSON(root);
+    QByteArray data = file.readAll();
+
+    return readOctreeDataInfoFromData(data);
 }
 
 QByteArray OctreeUtils::RawOctreeData::toByteArray() {
-    QJsonObject obj {
-        { "DataVersion", QJsonValue((qint64)dataVersion) },
-        { "Id", QJsonValue(id.toString()) },
-        { "Version", QJsonValue((qint64)version) },
-    };
+    QByteArray jsonString;
 
-    writeSubclassData(obj);
+    jsonString += QString("{\n  \"DataVersion\": %1,\n").arg(dataVersion);
 
-    QJsonDocument doc;
-    doc.setObject(obj);
+    writeSubclassData(jsonString);
 
-    return doc.toJson();
+    jsonString += QString(",\n  \"Id\": \"%1\",\n  \"Version\": %2\n}").arg(id.toString()).arg(version);
+
+    return jsonString;
 }
 
 QByteArray OctreeUtils::RawOctreeData::toGzippedByteArray() {
@@ -114,14 +96,21 @@ void OctreeUtils::RawOctreeData::resetIdAndVersion() {
     qDebug() << "Reset octree data to: " << id << dataVersion;
 }
 
-void OctreeUtils::RawEntityData::readSubclassData(const QJsonObject& root) {
-    if (root.contains("Entities")) {
-        entityData = root["Entities"].toArray();
+void OctreeUtils::RawEntityData::readSubclassData(const QVariantMap& root) {
+    variantEntityData = root["Entities"].toList();
+}
+
+void OctreeUtils::RawEntityData::writeSubclassData(QByteArray& root) const {
+    root += "  \"Entities\": [";
+    for (auto entityIter = variantEntityData.begin(); entityIter != variantEntityData.end(); ++entityIter) {
+        if (entityIter != variantEntityData.begin()) {
+            root += ",";
+        }
+        root += "\n    ";
+        // Convert to string and remove trailing LF.
+        root += QJsonDocument(entityIter->toJsonObject()).toJson().chopped(1);
     }
+    root += "]";
 }
 
-void OctreeUtils::RawEntityData::writeSubclassData(QJsonObject& root) const {
-    root["Entities"] = entityData;
-}
-
-PacketType OctreeUtils::RawEntityData::dataPacketType() const { return PacketType::EntityData; }
+PacketType OctreeUtils::RawEntityData::dataPacketType() const { return PacketType::EntityData; }

libraries/octree/src/OctreeDataUtils.h

@@ -33,8 +33,8 @@ public:
 
     virtual PacketType dataPacketType() const;
 
-    virtual void readSubclassData(const QJsonObject& root) { }
-    virtual void writeSubclassData(QJsonObject& root) const { }
+    virtual void readSubclassData(const QVariantMap& root) { }
+    virtual void writeSubclassData(QByteArray& root) const { }
 
     void resetIdAndVersion();
     QByteArray toByteArray();
@@ -42,15 +42,16 @@ public:
 
     bool readOctreeDataInfoFromData(QByteArray data);
     bool readOctreeDataInfoFromFile(QString path);
-    bool readOctreeDataInfoFromJSON(QJsonObject root);
+    bool readOctreeDataInfoFromMap(const QVariantMap& map);
 };
 
 class RawEntityData : public RawOctreeData {
+public:
     PacketType dataPacketType() const override;
-    void readSubclassData(const QJsonObject& root) override;
-    void writeSubclassData(QJsonObject& root) const override;
+    void readSubclassData(const QVariantMap& root) override;
+    void writeSubclassData(QByteArray& root) const override;
 
-    QJsonArray entityData;
+    QVariantList variantEntityData;
 };
 
 }

libraries/octree/src/OctreeEntitiesFileParser.cpp

@@ -0,0 +1,273 @@
+//
+//  OctreeEntititesFileParser.cpp
+//  libraries/octree/src
+//
+//  Created by Simon Walton on Oct 15, 2018.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <sstream>
+#include <cctype>
+#include <QUuid>
+#include <QJsonDocument>
+#include <QJsonObject>
+
+#include "OctreeEntitiesFileParser.h"
+
+using std::string;
+
+std::string OctreeEntitiesFileParser::getErrorString() const {
+    std::ostringstream err;
+    if (_errorString.size() != 0) {
+        err << "Error: Line " << _line << ", byte position " << _position << ": " << _errorString;
+    };
+
+    return err.str();
+}
+
+void OctreeEntitiesFileParser::setEntitiesString(const QByteArray& entitiesContents) {
+    _entitiesContents = entitiesContents;
+    _entitiesLength = _entitiesContents.length();
+    _position = 0;
+    _line = 1;
+}
+
+bool OctreeEntitiesFileParser::parseEntities(QVariantMap& parsedEntities) {
+    if (nextToken() != '{') {
+        _errorString = "Text before start of object";
+        return false;
+    }
+
+    bool gotDataVersion = false;
+    bool gotEntities = false;
+    bool gotId = false;
+    bool gotVersion = false;
+
+    while (!(gotDataVersion && gotEntities && gotId && gotVersion)) {
+        if (nextToken() != '"') {
+            _errorString = "Incorrect key string";
+            return false;
+        }
+
+        string key = readString();
+        if (key.size() == 0) {
+            _errorString = "Missing object key";
+            return false;
+        }
+
+        if (nextToken() != ':') {
+            _errorString = "Ill-formed id/value entry";
+            return false;
+        }
+
+        if (key == "DataVersion") {
+            if (gotDataVersion) {
+                _errorString = "Duplicate DataVersion entries";
+                return false;
+            }
+
+            int dataVersionValue = readInteger();
+            parsedEntities["DataVersion"] = dataVersionValue;
+            gotDataVersion = true;
+        } else if (key == "Entities") {
+            if (gotEntities) {
+                _errorString = "Duplicate Entities entries";
+                return false;
+            }
+
+            QVariantList entitiesValue;
+            if (!readEntitiesArray(entitiesValue)) {
+                return false;
+            }
+
+            parsedEntities["Entities"] = std::move(entitiesValue);
+            gotEntities = true;
+        } else if (key == "Id") {
+            if (gotId) {
+                _errorString = "Duplicate Id entries";
+                return false;
+            }
+
+            if (nextToken() != '"') {
+                _errorString = "Invalid Id value";
+                return false;
+            };
+            string idString = readString();
+            if (idString.size() == 0) {
+                _errorString = "Invalid Id string";
+                return false;
+            }
+            QUuid idValue = QUuid::fromString(QLatin1String(idString.c_str()) );
+            if (idValue.isNull()) {
+                _errorString = "Id value invalid UUID string: " + idString;
+                return false;
+            }
+
+            parsedEntities["Id"] = idValue;
+            gotId = true;
+        } else if (key == "Version") {
+            if (gotVersion) {
+                _errorString = "Duplicate Version entries";
+                return false;
+            }
+
+            int versionValue = readInteger();
+            parsedEntities["Version"] = versionValue;
+            gotVersion = true;
+        } else if (key == "Paths") {
+            // Serverless JSON has optional Paths entry.
+            if (nextToken() != '{') {
+                _errorString = "Paths item is not an object";
+                return false;
+            }
+
+            int matchingBrace = findMatchingBrace();
+            if (matchingBrace < 0) {
+                _errorString = "Unterminated entity object";
+                return false;
+            }
+
+            QByteArray jsonObject = _entitiesContents.mid(_position - 1, matchingBrace - _position + 1);
+            QJsonDocument pathsObject = QJsonDocument::fromJson(jsonObject);
+            if (pathsObject.isNull()) {
+                _errorString = "Ill-formed paths entry";
+                return false;
+            }
+
+            parsedEntities["Paths"] = pathsObject.object();
+            _position = matchingBrace;
+        } else {
+            _errorString = "Unrecognized key name: " + key;
+            return false;
+        }
+
+        if (gotDataVersion && gotEntities && gotId && gotVersion) {
+            break;
+        } else if (nextToken() != ',') {
+            _errorString = "Id/value incorrectly terminated";
+            return false;
+        }
+    }
+
+    if (nextToken() != '}' || nextToken() != -1) {
+        _errorString = "Ill-formed end of object";
+        return false;
+    }
+
+    return true;
+}
+
+int OctreeEntitiesFileParser::nextToken() {
+    while (_position < _entitiesLength) {
+        char c = _entitiesContents[_position++];
+        if (c != ' ' && c != '\t' && c != '\n' && c != '\r') {
+            return c;
+        }
+        if (c == '\n') {
+            ++_line;
+        }
+    }
+
+    return -1;
+}
+
+string OctreeEntitiesFileParser::readString() {
+    string returnString;
+    while (_position < _entitiesLength) {
+        char c = _entitiesContents[_position++];
+        if (c == '"') {
+            break;
+        } else {
+            returnString.push_back(c);
+        }
+    }
+
+    return returnString;
+}
+
+int OctreeEntitiesFileParser::readInteger() {
+    const char* currentPosition = _entitiesContents.constData() + _position;
+    int i = std::atoi(currentPosition);
+
+    int token;
+    do {
+        token = nextToken();
+    } while (token == '-' || token == '+' || std::isdigit(token));
+
+    --_position;
+    return i;
+}
+
+bool OctreeEntitiesFileParser::readEntitiesArray(QVariantList& entitiesArray) {
+    if (nextToken() != '[') {
+        _errorString = "Entities entry is not an array";
+        return false;
+    }
+
+    while (true) {
+        if (nextToken() != '{') {
+            _errorString = "Entity array item is not an object";
+            return false;
+        }
+        int matchingBrace = findMatchingBrace();
+        if (matchingBrace < 0) {
+            _errorString = "Unterminated entity object";
+            return false;
+        }
+
+        QByteArray jsonEntity = _entitiesContents.mid(_position - 1, matchingBrace - _position + 1);
+        QJsonDocument entity = QJsonDocument::fromJson(jsonEntity);
+        if (entity.isNull()) {
+            _errorString = "Ill-formed entity";
+            return false;
+        }
+
+        entitiesArray.append(entity.object());
+        _position = matchingBrace;
+        char c = nextToken();
+        if (c == ']') {
+            return true;
+        } else if (c != ',') {
+            _errorString = "Entity array item incorrectly terminated";
+            return false;
+        }
+    }
+    return true;
+}
+
+int OctreeEntitiesFileParser::findMatchingBrace() const {
+    int index = _position;
+    int nestCount = 1;
+    while (index < _entitiesLength && nestCount != 0) {
+        switch (_entitiesContents[index++]) {
+        case '{':
+            ++nestCount;
+            break;
+
+        case '}':
+            --nestCount;
+            break;
+
+        case '"':
+            // Skip string
+            while (index < _entitiesLength) {
+                if (_entitiesContents[index] == '"') {
+                    ++index;
+                    break;
+                } else if (_entitiesContents[index] == '\\' && _entitiesContents[++index] == 'u') {
+                    index += 4;
+                }
+                ++index;
+            }
+            break;
+
+        default:
+            break;
+        }
+    }
+
+    return nestCount == 0 ? index : -1;
+}

libraries/octree/src/OctreeEntitiesFileParser.h

@@ -0,0 +1,40 @@
+//
+//  OctreeEntititesFileParser.h
+//  libraries/octree/src
+//
+//  Created by Simon Walton on Oct 15, 2018.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+// Parse the top-level of the Models object ourselves - use QJsonDocument for each Entity object.
+
+#ifndef hifi_OctreeEntitiesFileParser_h
+#define hifi_OctreeEntitiesFileParser_h
+
+#include <QByteArray>
+#include <QVariant>
+
+class OctreeEntitiesFileParser {
+public:
+    void setEntitiesString(const QByteArray& entitiesContents);
+    bool parseEntities(QVariantMap& parsedEntities);
+    std::string getErrorString() const;
+
+private:
+    int nextToken();
+    std::string readString();
+    int readInteger();
+    bool readEntitiesArray(QVariantList& entitiesArray);
+    int findMatchingBrace() const;
+
+    QByteArray _entitiesContents;
+    int _position { 0 };
+    int _line { 1 };
+    int _entitiesLength { 0 };
+    std::string _errorString;
+};
+
+#endif  // hifi_OctreeEntitiesFileParser_h

libraries/octree/src/OctreePersistThread.cpp

@@ -31,6 +31,7 @@
 #include <NumericalConstants.h>
 #include <PerfStat.h>
 #include <PathUtils.h>
+#include <Gzip.h>
 
 #include "OctreeLogging.h"
 #include "OctreeUtils.h"
@@ -72,14 +73,27 @@ void OctreePersistThread::start() {
 
     OctreeUtils::RawOctreeData data;
     qCDebug(octree) << "Reading octree data from" << _filename;
-    if (data.readOctreeDataInfoFromFile(_filename)) {
-        qCDebug(octree) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")";
-        packet->writePrimitive(true);
-        auto id = data.id.toRfc4122();
-        packet->write(id);
-        packet->writePrimitive(data.version);
+    QFile file(_filename);
+    if (file.open(QIODevice::ReadOnly)) {
+        QByteArray jsonData(file.readAll());
+        file.close();
+        if (!gunzip(jsonData, _cachedJSONData)) {
+            _cachedJSONData = jsonData;
+        }
+
+        if (data.readOctreeDataInfoFromData(_cachedJSONData)) {
+            qCDebug(octree) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")";
+            packet->writePrimitive(true);
+            auto id = data.id.toRfc4122();
+            packet->write(id);
+            packet->writePrimitive(data.version);
+        } else {
+            _cachedJSONData.clear();
+            qCWarning(octree) << "No octree data found";
+            packet->writePrimitive(false);
+        }
     } else {
-        qCWarning(octree) << "No octree data found";
+        qCWarning(octree) << "Couldn't access file" << _filename << file.errorString();
         packet->writePrimitive(false);
     }
 
@@ -99,6 +113,7 @@ void OctreePersistThread::handleOctreeDataFileReply(QSharedPointer<ReceivedMessa
     OctreeUtils::RawOctreeData data;
     bool hasValidOctreeData { false };
     if (includesNewData) {
+        _cachedJSONData.clear();
         replacementData = message->readAll();
         replaceData(replacementData);
         hasValidOctreeData = data.readOctreeDataInfoFromFile(_filename);
@@ -108,7 +123,7 @@ void OctreePersistThread::handleOctreeDataFileReply(QSharedPointer<ReceivedMessa
         
         OctreeUtils::RawEntityData data;
         qCDebug(octree) << "Reading octree data from" << _filename;
-        if (data.readOctreeDataInfoFromFile(_filename)) {
+        if (data.readOctreeDataInfoFromData(_cachedJSONData)) {
             hasValidOctreeData = true;
             if (data.id.isNull()) {
                 qCDebug(octree) << "Current octree data has a null id, updating";
@@ -138,10 +153,16 @@ void OctreePersistThread::handleOctreeDataFileReply(QSharedPointer<ReceivedMessa
     _tree->withWriteLock([&] {
         PerformanceWarning warn(true, "Loading Octree File", true);
 
-        persistentFileRead = _tree->readFromFile(_filename.toLocal8Bit().constData());
+        if (_cachedJSONData.isEmpty()) {
+            persistentFileRead = _tree->readFromFile(_filename.toLocal8Bit().constData());
+        } else {
+            QDataStream jsonStream(_cachedJSONData);
+            persistentFileRead = _tree->readFromStream(-1, jsonStream);
+        }
         _tree->pruneTree();
     });
 
+    _cachedJSONData.clear();
     quint64 loadDone = usecTimestampNow();
     _loadTimeUSecs = loadDone - loadStarted;
 

libraries/octree/src/OctreePersistThread.h

@@ -78,6 +78,7 @@ private:
     quint64 _lastTimeDebug;
 
     QString _persistAsFileType;
+    QByteArray _cachedJSONData;
 };
 
 #endif // hifi_OctreePersistThread_h