Faster JSON entities parser

libraries/octree/src/OctreeDataUtils.cpp

@@ -9,6 +9,7 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 
 #include "OctreeDataUtils.h"
+#include "OctreeEntitiesFileParser.h"
 
 #include <Gzip.h>
 #include <udt/PacketHeaders.h>
@@ -55,19 +56,30 @@ bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromJSON(QJsonObject root) {
     return true;
 }
 
+bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromMap(QVariantMap map) {
+    if (map.contains("Id") && map.contains("DataVersion") && map.contains("Version")) {
+        id = map["Id"].toUuid();
+        dataVersion = map["DataVersion"].toInt();
+        version = map["Version"].toInt();
+    }
+    return true;
+}
+
 bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromData(QByteArray data) {
     QByteArray jsonData;
     if (gunzip(data, jsonData)) {
         data = jsonData;
     }
 
-    auto doc = QJsonDocument::fromJson(data);
+    OctreeEntitiesFileParser jsonParser;
-    if (doc.isNull()) {
+    jsonParser.setEntitiesString(data);
+    QVariantMap entitiesMap;
+    if (!jsonParser.parseEntities(entitiesMap)) {
+        qCritical() << "Can't parse Entities JSON: " << jsonParser.getErrorString().c_str();
         return false;
     }
 
-    auto root = doc.object();
+    return readOctreeDataInfoFromMap(entitiesMap);
-    return readOctreeDataInfoFromJSON(root);
 }
 
 // Reads octree file and parses it into a RawOctreeData object.

libraries/octree/src/OctreeDataUtils.h

@@ -43,6 +43,7 @@ public:
     bool readOctreeDataInfoFromData(QByteArray data);
     bool readOctreeDataInfoFromFile(QString path);
     bool readOctreeDataInfoFromJSON(QJsonObject root);
+    bool readOctreeDataInfoFromMap(QVariantMap map);
 };
 
 class RawEntityData : public RawOctreeData {

libraries/octree/src/OctreeEntitiesFileParser.cpp

@@ -189,23 +189,25 @@ bool OctreeEntitiesFileParser::readEntitiesArray(QVariantList& entitiesArray) {
     }
 
     while (true) {
-        QJsonParseError parseError;
+        if (nextToken() != '{') {
-        QByteArray entitiesJson(_entitiesContents.right(_entitiesLength - _position));
+            _errorString = "Entity array item is not an object";
-        QJsonDocument entity = QJsonDocument::fromJson(entitiesJson, &parseError);
+            return false;
-        if (parseError.error != QJsonParseError::GarbageAtEnd) {
+        }
-            _errorString = "Ill-formed entity array";
+        int matchingBrace = findMatchingBrace();
+        if (matchingBrace < 0) {
+            _errorString = "Unterminated entity object";
             return false;
         }
-        int entityLength = parseError.offset;
-        entitiesJson.truncate(entityLength);
-        _position += entityLength;
 
-        entity = QJsonDocument::fromJson(entitiesJson, &parseError);
+        QByteArray jsonEntity = _entitiesContents.mid(_position - 1, matchingBrace - _position + 1);
-        if (parseError.error != QJsonParseError::NoError) {
+        QJsonDocument entity = QJsonDocument::fromJson(jsonEntity);
-            _errorString = "Entity item parse error";
+        if (entity.isNull()) {
+            _errorString = "Ill-formed entity";
             return false;
         }
+
         entitiesArray.append(entity.object());
+        _position = matchingBrace;
         char c = nextToken();
         if (c == ']') {
             return true;
@@ -216,3 +218,37 @@ bool OctreeEntitiesFileParser::readEntitiesArray(QVariantList& entitiesArray) {
     }
     return true;
 }
+
+int OctreeEntitiesFileParser::findMatchingBrace() const {
+    int index = _position;
+    int nestCount = 1;
+    while (index < _entitiesLength && nestCount != 0) {
+        switch (_entitiesContents[index++]) {
+        case '{':
+            ++nestCount;
+            break;
+
+        case '}':
+            --nestCount;
+            break;
+
+        case '"':
+            // Skip string
+            while (index < _entitiesLength) {
+                if (_entitiesContents[index] == '"') {
+                    ++index;
+                    break;
+                } else if (_entitiesContents[index] == '\\' && _entitiesContents[++index] == 'u') {
+                    index += 4;
+                }
+                ++index;
+            }
+            break;
+
+        default:
+            break;
+        }
+    }
+
+    return nestCount == 0 ? index : -1;
+}

libraries/octree/src/OctreeEntitiesFileParser.h

@@ -9,6 +9,8 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
+// Parse the top-level of the Models object ourselves - use QJsonDocument for each Entity object.
+
 #ifndef hifi_OctreeEntitiesFileParser_h
 #define hifi_OctreeEntitiesFileParser_h
 
@@ -27,6 +29,7 @@ private:
     std::string readString();
     int readInteger();
     bool readEntitiesArray(QVariantList& entitiesArray);
+    int findMatchingBrace() const;
 
     QByteArray _entitiesContents;
     int _position { 0 };