diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index b540592d7e..8cf033c130 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -3411,20 +3411,11 @@ void DomainServer::maybeHandleReplacementEntityFile() { } void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) { - //Assume we have compressed data - auto compressedOctree = octreeFile; - QByteArray jsonOctree; - - bool wasCompressed = gunzip(compressedOctree, jsonOctree); - if (!wasCompressed) { - // the source was not compressed, assume we were sent regular JSON data - jsonOctree = compressedOctree; - } - OctreeUtils::RawEntityData data; - if (data.readOctreeDataInfoFromData(jsonOctree)) { + if (data.readOctreeDataInfoFromData(octreeFile)) { data.resetIdAndVersion(); + QByteArray compressedOctree; gzip(data.toByteArray(), compressedOctree); // write the compressed octree data to a special file diff --git a/libraries/entities/src/EntityTree.cpp b/libraries/entities/src/EntityTree.cpp index 16d7e74703..fee8d72fe7 100644 --- a/libraries/entities/src/EntityTree.cpp +++ b/libraries/entities/src/EntityTree.cpp @@ -35,6 +35,7 @@ #include "QVariantGLM.h" #include "EntitiesLogging.h" #include "RecurseOctreeToMapOperator.h" +#include "RecurseOctreeToJSONOperator.h" #include "LogHandler.h" #include "EntityEditFilters.h" #include "EntityDynamicFactoryInterface.h" @@ -2785,6 +2786,17 @@ bool EntityTree::readFromMap(QVariantMap& map) { return success; } +bool EntityTree::writeToJSON(QString& jsonString, const OctreeElementPointer& element) { + QScriptEngine scriptEngine; + RecurseOctreeToJSONOperator theOperator(element, &scriptEngine, jsonString); + withReadLock([&] { + recurseTreeWithOperator(&theOperator); + }); + + jsonString = theOperator.getJson(); + return true; +} + void EntityTree::resetClientEditStats() { _treeResetTime = usecTimestampNow(); _maxEditDelta = 0; diff --git a/libraries/entities/src/EntityTree.h b/libraries/entities/src/EntityTree.h index 634ffcc1f3..c6a590ec71 100644 --- a/libraries/entities/src/EntityTree.h +++ b/libraries/entities/src/EntityTree.h @@ -224,6 +224,8 @@ public: virtual bool writeToMap(QVariantMap& entityDescription, OctreeElementPointer element, bool skipDefaultValues, bool skipThoseWithBadParents) override; virtual bool readFromMap(QVariantMap& entityDescription) override; + virtual bool writeToJSON(QString& jsonString, const OctreeElementPointer& element) override; + glm::vec3 getContentsDimensions(); float getContentsLargestDimension(); diff --git a/libraries/entities/src/RecurseOctreeToJSONOperator.cpp b/libraries/entities/src/RecurseOctreeToJSONOperator.cpp new file mode 100644 index 0000000000..b64a700abc --- /dev/null +++ b/libraries/entities/src/RecurseOctreeToJSONOperator.cpp @@ -0,0 +1,50 @@ +// +// RecurseOctreeToJSONOperator.cpp +// libraries/entities/src +// +// Created by Simon Walton on Oct 11, 2018. +// Copyright 2018 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "RecurseOctreeToJSONOperator.h" +#include "EntityItemProperties.h" + +RecurseOctreeToJSONOperator::RecurseOctreeToJSONOperator(const OctreeElementPointer&, QScriptEngine* engine, + QString jsonPrefix, bool skipDefaults, bool skipThoseWithBadParents): + _engine(engine), + _json(jsonPrefix), + _skipDefaults(skipDefaults), + _skipThoseWithBadParents(skipThoseWithBadParents) +{ + _toStringMethod = _engine->evaluate("(function() { return JSON.stringify(this, null, ' ') })"); +} + +bool RecurseOctreeToJSONOperator::postRecursion(const OctreeElementPointer& element) { + EntityTreeElementPointer entityTreeElement = std::static_pointer_cast(element); + + entityTreeElement->forEachEntity([&](const EntityItemPointer& entity) { processEntity(entity); } ); + return true; +} + +void RecurseOctreeToJSONOperator::processEntity(const EntityItemPointer& entity) { + if (_skipThoseWithBadParents && !entity->isParentIDValid()) { + return; // we weren't able to resolve a parent from _parentID, so don't save this entity. + } + + QScriptValue qScriptValues = _skipDefaults + ? EntityItemNonDefaultPropertiesToScriptValue(_engine, entity->getProperties()) + : EntityItemPropertiesToScriptValue(_engine, entity->getProperties()); + + if (_comma) { + _json += ','; + }; + _comma = true; + _json += "\n "; + + // Override default toString(): + qScriptValues.setProperty("toString", _toStringMethod); + _json += qScriptValues.toString(); +} diff --git a/libraries/entities/src/RecurseOctreeToJSONOperator.h b/libraries/entities/src/RecurseOctreeToJSONOperator.h new file mode 100644 index 0000000000..a1d388ed22 --- /dev/null +++ b/libraries/entities/src/RecurseOctreeToJSONOperator.h @@ -0,0 +1,33 @@ +// +// RecurseOctreeToJSONOperator.h +// libraries/entities/src +// +// Created by Simon Walton on Oct 11, 2018. +// Copyright 2018 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "EntityTree.h" + +class RecurseOctreeToJSONOperator : public RecurseOctreeOperator { +public: + RecurseOctreeToJSONOperator(const OctreeElementPointer&, QScriptEngine* engine, QString jsonPrefix = QString(), bool skipDefaults = true, + bool skipThoseWithBadParents = false); + virtual bool preRecursion(const OctreeElementPointer& element) override { return true; }; + virtual bool postRecursion(const OctreeElementPointer& element) override; + + QString getJson() const { return _json; } + +private: + void processEntity(const EntityItemPointer& entity); + + QScriptEngine* _engine; + QScriptValue _toStringMethod; + + QString _json; + const bool _skipDefaults; + bool _skipThoseWithBadParents; + bool _comma { false }; +}; diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp index 9e77968384..df02b54856 100644 --- a/libraries/octree/src/Octree.cpp +++ b/libraries/octree/src/Octree.cpp @@ -50,7 +50,7 @@ #include "OctreeLogging.h" #include "OctreeQueryNode.h" #include "OctreeUtils.h" - +#include "OctreeEntitiesFileParser.h" QVector PERSIST_EXTENSIONS = {"json", "json.gz"}; @@ -792,28 +792,26 @@ bool Octree::readFromStream( } +namespace { // hack to get the marketplace id into the entities. We will create a way to get this from a hash of // the entity later, but this helps us move things along for now -QJsonDocument addMarketplaceIDToDocumentEntities(QJsonDocument& doc, const QString& marketplaceID) { +QVariantMap addMarketplaceIDToDocumentEntities(QVariantMap& doc, const QString& marketplaceID) { if (!marketplaceID.isEmpty()) { - QJsonDocument newDoc; - QJsonObject rootObj = doc.object(); - QJsonArray newEntitiesArray; + QVariantList newEntitiesArray; // build a new entities array - auto entitiesArray = rootObj["Entities"].toArray(); - for(auto it = entitiesArray.begin(); it != entitiesArray.end(); it++) { - auto entity = (*it).toObject(); + auto entitiesArray = doc["Entities"].toList(); + for (auto it = entitiesArray.begin(); it != entitiesArray.end(); it++) { + auto entity = (*it).toMap(); entity["marketplaceID"] = marketplaceID; newEntitiesArray.append(entity); } - rootObj["Entities"] = newEntitiesArray; - newDoc.setObject(rootObj); - return newDoc; + doc["Entities"] = newEntitiesArray; } return doc; } +} // Unnamed namepsace const int READ_JSON_BUFFER_SIZE = 2048; bool Octree::readJSONFromStream( @@ -839,12 +837,18 @@ bool Octree::readJSONFromStream( jsonBuffer += QByteArray(rawData, got); } - QJsonDocument asDocument = QJsonDocument::fromJson(jsonBuffer); - if (!marketplaceID.isEmpty()) { - asDocument = addMarketplaceIDToDocumentEntities(asDocument, marketplaceID); + OctreeEntitiesFileParser octreeParser; + octreeParser.setEntitiesString(jsonBuffer); + QVariantMap asMap; + if (!octreeParser.parseEntities(asMap)) { + qCritical() << "Couldn't parse Entities JSON:" << octreeParser.getErrorString().c_str(); + return false; } - QVariant asVariant = asDocument.toVariant(); - QVariantMap asMap = asVariant.toMap(); + + if (!marketplaceID.isEmpty()) { + addMarketplaceIDToDocumentEntities(asMap, marketplaceID); + } + bool success = readFromMap(asMap); delete[] rawData; return success; @@ -889,26 +893,52 @@ bool Octree::toJSONDocument(QJsonDocument* doc, const OctreeElementPointer& elem return false; } - *doc = QJsonDocument::fromVariant(entityDescription); + + bool noEntities = entityDescription["Entities"].toList().empty(); + QJsonDocument jsonDocTree = QJsonDocument::fromVariant(entityDescription); + QJsonValue entitiesJson = jsonDocTree["Entities"]; + if (entitiesJson.isNull() || (entitiesJson.toArray().empty() && !noEntities)) { + // Json version of entities too large. + return false; + } else { + *doc = jsonDocTree; + } + + return true; +} + +bool Octree::toJSONString(QString& jsonString, const OctreeElementPointer& element) { + OctreeElementPointer top; + if (element) { + top = element; + } else { + top = _rootElement; + } + + jsonString += QString("{\n \"DataVersion\": %1,\n \"Entities\": [").arg(_persistDataVersion); + + writeToJSON(jsonString, top); + + // include the "bitstream" version + PacketType expectedType = expectedDataPacketType(); + PacketVersion expectedVersion = versionForPacketType(expectedType); + + jsonString += QString("\n ],\n \"Id\": \"%1\",\n \"Version\": %2\n}\n").arg(_persistID.toString()).arg((int)expectedVersion); + return true; } bool Octree::toJSON(QByteArray* data, const OctreeElementPointer& element, bool doGzip) { - QJsonDocument doc; - if (!toJSONDocument(&doc, element)) { - qCritical("Failed to convert Entities to QVariantMap while converting to json."); - return false; - } + QString jsonString; + toJSONString(jsonString); if (doGzip) { - QByteArray jsonData = doc.toJson(); - - if (!gzip(jsonData, *data, -1)) { + if (!gzip(jsonString.toUtf8(), *data, -1)) { qCritical("Unable to gzip data while saving to json."); return false; } } else { - *data = doc.toJson(); + *data = jsonString.toUtf8(); } return true; diff --git a/libraries/octree/src/Octree.h b/libraries/octree/src/Octree.h index 44b429582a..eef23493f6 100644 --- a/libraries/octree/src/Octree.h +++ b/libraries/octree/src/Octree.h @@ -202,11 +202,13 @@ public: // Octree exporters bool toJSONDocument(QJsonDocument* doc, const OctreeElementPointer& element = nullptr); + bool toJSONString(QString& jsonString, const OctreeElementPointer& element = nullptr); bool toJSON(QByteArray* data, const OctreeElementPointer& element = nullptr, bool doGzip = false); bool writeToFile(const char* filename, const OctreeElementPointer& element = nullptr, QString persistAsFileType = "json.gz"); bool writeToJSONFile(const char* filename, const OctreeElementPointer& element = nullptr, bool doGzip = false); virtual bool writeToMap(QVariantMap& entityDescription, OctreeElementPointer element, bool skipDefaultValues, bool skipThoseWithBadParents) = 0; + virtual bool writeToJSON(QString& jsonString, const OctreeElementPointer& element) = 0; // Octree importers bool readFromFile(const char* filename); diff --git a/libraries/octree/src/OctreeDataUtils.cpp b/libraries/octree/src/OctreeDataUtils.cpp index 3da498d7f3..b861904255 100644 --- a/libraries/octree/src/OctreeDataUtils.cpp +++ b/libraries/octree/src/OctreeDataUtils.cpp @@ -9,6 +9,7 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html #include "OctreeDataUtils.h" +#include "OctreeEntitiesFileParser.h" #include #include @@ -18,33 +19,13 @@ #include #include -// Reads octree file and parses it into a QJsonDocument. Handles both gzipped and non-gzipped files. -// Returns true if the file was successfully opened and parsed, otherwise false. -// Example failures: file does not exist, gzipped file cannot be unzipped, invalid JSON. -bool readOctreeFile(QString path, QJsonDocument* doc) { - QFile file(path); - if (!file.open(QIODevice::ReadOnly)) { - return false; +bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromMap(const QVariantMap& map) { + if (map.contains("Id") && map.contains("DataVersion") && map.contains("Version")) { + id = map["Id"].toUuid(); + dataVersion = map["DataVersion"].toInt(); + version = map["Version"].toInt(); } - - QByteArray data = file.readAll(); - QByteArray jsonData; - - if (!gunzip(data, jsonData)) { - jsonData = data; - } - - *doc = QJsonDocument::fromJson(jsonData); - return !doc->isNull(); -} - -bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromJSON(QJsonObject root) { - if (root.contains("Id") && root.contains("DataVersion") && root.contains("Version")) { - id = root["Id"].toVariant().toUuid(); - dataVersion = root["DataVersion"].toInt(); - version = root["Version"].toInt(); - } - readSubclassData(root); + readSubclassData(map); return true; } @@ -54,40 +35,41 @@ bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromData(QByteArray data) { data = jsonData; } - auto doc = QJsonDocument::fromJson(data); - if (doc.isNull()) { + OctreeEntitiesFileParser jsonParser; + jsonParser.setEntitiesString(data); + QVariantMap entitiesMap; + if (!jsonParser.parseEntities(entitiesMap)) { + qCritical() << "Can't parse Entities JSON: " << jsonParser.getErrorString().c_str(); return false; } - auto root = doc.object(); - return readOctreeDataInfoFromJSON(root); + return readOctreeDataInfoFromMap(entitiesMap); } // Reads octree file and parses it into a RawOctreeData object. // Returns false if readOctreeFile fails. bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromFile(QString path) { - QJsonDocument doc; - if (!readOctreeFile(path, &doc)) { + QFile file(path); + if (!file.open(QIODevice::ReadOnly)) { + qCritical() << "Cannot open json file for reading: " << path; return false; } - auto root = doc.object(); - return readOctreeDataInfoFromJSON(root); + QByteArray data = file.readAll(); + + return readOctreeDataInfoFromData(data); } QByteArray OctreeUtils::RawOctreeData::toByteArray() { - QJsonObject obj { - { "DataVersion", QJsonValue((qint64)dataVersion) }, - { "Id", QJsonValue(id.toString()) }, - { "Version", QJsonValue((qint64)version) }, - }; + QByteArray jsonString; - writeSubclassData(obj); + jsonString += QString("{\n \"DataVersion\": %1,\n").arg(dataVersion); - QJsonDocument doc; - doc.setObject(obj); + writeSubclassData(jsonString); - return doc.toJson(); + jsonString += QString(",\n \"Id\": \"%1\",\n \"Version\": %2\n}").arg(id.toString()).arg(version); + + return jsonString; } QByteArray OctreeUtils::RawOctreeData::toGzippedByteArray() { @@ -114,14 +96,21 @@ void OctreeUtils::RawOctreeData::resetIdAndVersion() { qDebug() << "Reset octree data to: " << id << dataVersion; } -void OctreeUtils::RawEntityData::readSubclassData(const QJsonObject& root) { - if (root.contains("Entities")) { - entityData = root["Entities"].toArray(); +void OctreeUtils::RawEntityData::readSubclassData(const QVariantMap& root) { + variantEntityData = root["Entities"].toList(); +} + +void OctreeUtils::RawEntityData::writeSubclassData(QByteArray& root) const { + root += " \"Entities\": ["; + for (auto entityIter = variantEntityData.begin(); entityIter != variantEntityData.end(); ++entityIter) { + if (entityIter != variantEntityData.begin()) { + root += ","; + } + root += "\n "; + // Convert to string and remove trailing LF. + root += QJsonDocument(entityIter->toJsonObject()).toJson().chopped(1); } + root += "]"; } -void OctreeUtils::RawEntityData::writeSubclassData(QJsonObject& root) const { - root["Entities"] = entityData; -} - -PacketType OctreeUtils::RawEntityData::dataPacketType() const { return PacketType::EntityData; } \ No newline at end of file +PacketType OctreeUtils::RawEntityData::dataPacketType() const { return PacketType::EntityData; } diff --git a/libraries/octree/src/OctreeDataUtils.h b/libraries/octree/src/OctreeDataUtils.h index 9060e7b460..642c00a675 100644 --- a/libraries/octree/src/OctreeDataUtils.h +++ b/libraries/octree/src/OctreeDataUtils.h @@ -33,8 +33,8 @@ public: virtual PacketType dataPacketType() const; - virtual void readSubclassData(const QJsonObject& root) { } - virtual void writeSubclassData(QJsonObject& root) const { } + virtual void readSubclassData(const QVariantMap& root) { } + virtual void writeSubclassData(QByteArray& root) const { } void resetIdAndVersion(); QByteArray toByteArray(); @@ -42,15 +42,16 @@ public: bool readOctreeDataInfoFromData(QByteArray data); bool readOctreeDataInfoFromFile(QString path); - bool readOctreeDataInfoFromJSON(QJsonObject root); + bool readOctreeDataInfoFromMap(const QVariantMap& map); }; class RawEntityData : public RawOctreeData { +public: PacketType dataPacketType() const override; - void readSubclassData(const QJsonObject& root) override; - void writeSubclassData(QJsonObject& root) const override; + void readSubclassData(const QVariantMap& root) override; + void writeSubclassData(QByteArray& root) const override; - QJsonArray entityData; + QVariantList variantEntityData; }; } diff --git a/libraries/octree/src/OctreeEntitiesFileParser.cpp b/libraries/octree/src/OctreeEntitiesFileParser.cpp new file mode 100644 index 0000000000..873eaff0e1 --- /dev/null +++ b/libraries/octree/src/OctreeEntitiesFileParser.cpp @@ -0,0 +1,273 @@ +// +// OctreeEntititesFileParser.cpp +// libraries/octree/src +// +// Created by Simon Walton on Oct 15, 2018. +// Copyright 2018 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include +#include +#include +#include +#include + +#include "OctreeEntitiesFileParser.h" + +using std::string; + +std::string OctreeEntitiesFileParser::getErrorString() const { + std::ostringstream err; + if (_errorString.size() != 0) { + err << "Error: Line " << _line << ", byte position " << _position << ": " << _errorString; + }; + + return err.str(); +} + +void OctreeEntitiesFileParser::setEntitiesString(const QByteArray& entitiesContents) { + _entitiesContents = entitiesContents; + _entitiesLength = _entitiesContents.length(); + _position = 0; + _line = 1; +} + +bool OctreeEntitiesFileParser::parseEntities(QVariantMap& parsedEntities) { + if (nextToken() != '{') { + _errorString = "Text before start of object"; + return false; + } + + bool gotDataVersion = false; + bool gotEntities = false; + bool gotId = false; + bool gotVersion = false; + + while (!(gotDataVersion && gotEntities && gotId && gotVersion)) { + if (nextToken() != '"') { + _errorString = "Incorrect key string"; + return false; + } + + string key = readString(); + if (key.size() == 0) { + _errorString = "Missing object key"; + return false; + } + + if (nextToken() != ':') { + _errorString = "Ill-formed id/value entry"; + return false; + } + + if (key == "DataVersion") { + if (gotDataVersion) { + _errorString = "Duplicate DataVersion entries"; + return false; + } + + int dataVersionValue = readInteger(); + parsedEntities["DataVersion"] = dataVersionValue; + gotDataVersion = true; + } else if (key == "Entities") { + if (gotEntities) { + _errorString = "Duplicate Entities entries"; + return false; + } + + QVariantList entitiesValue; + if (!readEntitiesArray(entitiesValue)) { + return false; + } + + parsedEntities["Entities"] = std::move(entitiesValue); + gotEntities = true; + } else if (key == "Id") { + if (gotId) { + _errorString = "Duplicate Id entries"; + return false; + } + + if (nextToken() != '"') { + _errorString = "Invalid Id value"; + return false; + }; + string idString = readString(); + if (idString.size() == 0) { + _errorString = "Invalid Id string"; + return false; + } + QUuid idValue = QUuid::fromString(QLatin1String(idString.c_str()) ); + if (idValue.isNull()) { + _errorString = "Id value invalid UUID string: " + idString; + return false; + } + + parsedEntities["Id"] = idValue; + gotId = true; + } else if (key == "Version") { + if (gotVersion) { + _errorString = "Duplicate Version entries"; + return false; + } + + int versionValue = readInteger(); + parsedEntities["Version"] = versionValue; + gotVersion = true; + } else if (key == "Paths") { + // Serverless JSON has optional Paths entry. + if (nextToken() != '{') { + _errorString = "Paths item is not an object"; + return false; + } + + int matchingBrace = findMatchingBrace(); + if (matchingBrace < 0) { + _errorString = "Unterminated entity object"; + return false; + } + + QByteArray jsonObject = _entitiesContents.mid(_position - 1, matchingBrace - _position + 1); + QJsonDocument pathsObject = QJsonDocument::fromJson(jsonObject); + if (pathsObject.isNull()) { + _errorString = "Ill-formed paths entry"; + return false; + } + + parsedEntities["Paths"] = pathsObject.object(); + _position = matchingBrace; + } else { + _errorString = "Unrecognized key name: " + key; + return false; + } + + if (gotDataVersion && gotEntities && gotId && gotVersion) { + break; + } else if (nextToken() != ',') { + _errorString = "Id/value incorrectly terminated"; + return false; + } + } + + if (nextToken() != '}' || nextToken() != -1) { + _errorString = "Ill-formed end of object"; + return false; + } + + return true; +} + +int OctreeEntitiesFileParser::nextToken() { + while (_position < _entitiesLength) { + char c = _entitiesContents[_position++]; + if (c != ' ' && c != '\t' && c != '\n' && c != '\r') { + return c; + } + if (c == '\n') { + ++_line; + } + } + + return -1; +} + +string OctreeEntitiesFileParser::readString() { + string returnString; + while (_position < _entitiesLength) { + char c = _entitiesContents[_position++]; + if (c == '"') { + break; + } else { + returnString.push_back(c); + } + } + + return returnString; +} + +int OctreeEntitiesFileParser::readInteger() { + const char* currentPosition = _entitiesContents.constData() + _position; + int i = std::atoi(currentPosition); + + int token; + do { + token = nextToken(); + } while (token == '-' || token == '+' || std::isdigit(token)); + + --_position; + return i; +} + +bool OctreeEntitiesFileParser::readEntitiesArray(QVariantList& entitiesArray) { + if (nextToken() != '[') { + _errorString = "Entities entry is not an array"; + return false; + } + + while (true) { + if (nextToken() != '{') { + _errorString = "Entity array item is not an object"; + return false; + } + int matchingBrace = findMatchingBrace(); + if (matchingBrace < 0) { + _errorString = "Unterminated entity object"; + return false; + } + + QByteArray jsonEntity = _entitiesContents.mid(_position - 1, matchingBrace - _position + 1); + QJsonDocument entity = QJsonDocument::fromJson(jsonEntity); + if (entity.isNull()) { + _errorString = "Ill-formed entity"; + return false; + } + + entitiesArray.append(entity.object()); + _position = matchingBrace; + char c = nextToken(); + if (c == ']') { + return true; + } else if (c != ',') { + _errorString = "Entity array item incorrectly terminated"; + return false; + } + } + return true; +} + +int OctreeEntitiesFileParser::findMatchingBrace() const { + int index = _position; + int nestCount = 1; + while (index < _entitiesLength && nestCount != 0) { + switch (_entitiesContents[index++]) { + case '{': + ++nestCount; + break; + + case '}': + --nestCount; + break; + + case '"': + // Skip string + while (index < _entitiesLength) { + if (_entitiesContents[index] == '"') { + ++index; + break; + } else if (_entitiesContents[index] == '\\' && _entitiesContents[++index] == 'u') { + index += 4; + } + ++index; + } + break; + + default: + break; + } + } + + return nestCount == 0 ? index : -1; +} diff --git a/libraries/octree/src/OctreeEntitiesFileParser.h b/libraries/octree/src/OctreeEntitiesFileParser.h new file mode 100644 index 0000000000..bc51896b18 --- /dev/null +++ b/libraries/octree/src/OctreeEntitiesFileParser.h @@ -0,0 +1,40 @@ +// +// OctreeEntititesFileParser.h +// libraries/octree/src +// +// Created by Simon Walton on Oct 15, 2018. +// Copyright 2018 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +// Parse the top-level of the Models object ourselves - use QJsonDocument for each Entity object. + +#ifndef hifi_OctreeEntitiesFileParser_h +#define hifi_OctreeEntitiesFileParser_h + +#include +#include + +class OctreeEntitiesFileParser { +public: + void setEntitiesString(const QByteArray& entitiesContents); + bool parseEntities(QVariantMap& parsedEntities); + std::string getErrorString() const; + +private: + int nextToken(); + std::string readString(); + int readInteger(); + bool readEntitiesArray(QVariantList& entitiesArray); + int findMatchingBrace() const; + + QByteArray _entitiesContents; + int _position { 0 }; + int _line { 1 }; + int _entitiesLength { 0 }; + std::string _errorString; +}; + +#endif // hifi_OctreeEntitiesFileParser_h diff --git a/libraries/octree/src/OctreePersistThread.cpp b/libraries/octree/src/OctreePersistThread.cpp index 5a20f55a76..32ee72ea1c 100644 --- a/libraries/octree/src/OctreePersistThread.cpp +++ b/libraries/octree/src/OctreePersistThread.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include "OctreeLogging.h" #include "OctreeUtils.h" @@ -72,14 +73,27 @@ void OctreePersistThread::start() { OctreeUtils::RawOctreeData data; qCDebug(octree) << "Reading octree data from" << _filename; - if (data.readOctreeDataInfoFromFile(_filename)) { - qCDebug(octree) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")"; - packet->writePrimitive(true); - auto id = data.id.toRfc4122(); - packet->write(id); - packet->writePrimitive(data.version); + QFile file(_filename); + if (file.open(QIODevice::ReadOnly)) { + QByteArray jsonData(file.readAll()); + file.close(); + if (!gunzip(jsonData, _cachedJSONData)) { + _cachedJSONData = jsonData; + } + + if (data.readOctreeDataInfoFromData(_cachedJSONData)) { + qCDebug(octree) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")"; + packet->writePrimitive(true); + auto id = data.id.toRfc4122(); + packet->write(id); + packet->writePrimitive(data.version); + } else { + _cachedJSONData.clear(); + qCWarning(octree) << "No octree data found"; + packet->writePrimitive(false); + } } else { - qCWarning(octree) << "No octree data found"; + qCWarning(octree) << "Couldn't access file" << _filename << file.errorString(); packet->writePrimitive(false); } @@ -99,6 +113,7 @@ void OctreePersistThread::handleOctreeDataFileReply(QSharedPointerreadAll(); replaceData(replacementData); hasValidOctreeData = data.readOctreeDataInfoFromFile(_filename); @@ -108,7 +123,7 @@ void OctreePersistThread::handleOctreeDataFileReply(QSharedPointerwithWriteLock([&] { PerformanceWarning warn(true, "Loading Octree File", true); - persistentFileRead = _tree->readFromFile(_filename.toLocal8Bit().constData()); + if (_cachedJSONData.isEmpty()) { + persistentFileRead = _tree->readFromFile(_filename.toLocal8Bit().constData()); + } else { + QDataStream jsonStream(_cachedJSONData); + persistentFileRead = _tree->readFromStream(-1, jsonStream); + } _tree->pruneTree(); }); + _cachedJSONData.clear(); quint64 loadDone = usecTimestampNow(); _loadTimeUSecs = loadDone - loadStarted; diff --git a/libraries/octree/src/OctreePersistThread.h b/libraries/octree/src/OctreePersistThread.h index 0044a8fa5a..6a38a21a84 100644 --- a/libraries/octree/src/OctreePersistThread.h +++ b/libraries/octree/src/OctreePersistThread.h @@ -78,6 +78,7 @@ private: quint64 _lastTimeDebug; QString _persistAsFileType; + QByteArray _cachedJSONData; }; #endif // hifi_OctreePersistThread_h diff --git a/tools/oven/src/Oven.cpp b/tools/oven/src/Oven.cpp index 52b6db1aa5..af98376034 100644 --- a/tools/oven/src/Oven.cpp +++ b/tools/oven/src/Oven.cpp @@ -19,6 +19,7 @@ #include #include #include +#include Oven* Oven::_staticInstance { nullptr }; @@ -31,6 +32,7 @@ Oven::Oven() { // Initialize dependencies for OBJ Baker DependencyManager::set(); DependencyManager::set(false); + DependencyManager::set(); } Oven::~Oven() {