Merge branch 'master' of https://github.com/highfidelity/hifi into atp

domain-server/resources/describe-settings.json

@@ -389,6 +389,10 @@
             {
               "value": "json",
               "label": "Entity server persists data as JSON"
+            },
+            {
+              "value": "json.gz",
+              "label": "Entity server persists data as gzipped JSON"
             }
           ],
           "advanced": true

libraries/entities/src/EntityItemProperties.cpp

@@ -71,7 +71,8 @@ CONSTRUCT_PROPERTY(exponent, 0.0f),
 CONSTRUCT_PROPERTY(cutoff, ENTITY_ITEM_DEFAULT_CUTOFF),
 CONSTRUCT_PROPERTY(locked, ENTITY_ITEM_DEFAULT_LOCKED),
 CONSTRUCT_PROPERTY(textures, ""),
-CONSTRUCT_PROPERTY(animationSettings, ""),
+CONSTRUCT_PROPERTY(animationSettings, "{\"firstFrame\":0,\"fps\":30,\"frameIndex\":0,\"hold\":false,"
+                   "\"lastFrame\":100000,\"loop\":false,\"running\":false,\"startAutomatically\":false}"),
 CONSTRUCT_PROPERTY(userData, ENTITY_ITEM_DEFAULT_USER_DATA),
 CONSTRUCT_PROPERTY(simulationOwner, SimulationOwner()),
 CONSTRUCT_PROPERTY(text, TextEntityItem::DEFAULT_TEXT),

libraries/octree/src/Octree.cpp

@@ -39,6 +39,7 @@
 #include <udt/PacketHeaders.h>
 #include <SharedUtil.h>
 #include <PathUtils.h>
+#include <Gzip.h>
 
 #include "CoverageMap.h"
 #include "OctreeConstants.h"
@@ -48,7 +49,7 @@
 #include "OctreeLogging.h"
 
 
-QVector<QString> PERSIST_EXTENSIONS = {"svo", "json"};
+QVector<QString> PERSIST_EXTENSIONS = {"svo", "json", "json.gz"};
 
 float boundaryDistanceForRenderLevel(unsigned int renderLevel, float voxelSizeScale) {
     return voxelSizeScale / powf(2, renderLevel);
@@ -1801,29 +1802,52 @@ int Octree::encodeTreeBitstreamRecursion(OctreeElement* element,
 }
 
 bool Octree::readFromFile(const char* fileName) {
-    bool fileOk = false;
-
     QString qFileName = findMostRecentFileExtension(fileName, PERSIST_EXTENSIONS);
-    QFile file(qFileName);
-    fileOk = file.open(QIODevice::ReadOnly);
 
-    if(fileOk) {
-        QDataStream fileInputStream(&file);
-        QFileInfo fileInfo(qFileName);
-        unsigned long fileLength = fileInfo.size();
-
-        emit importSize(1.0f, 1.0f, 1.0f);
-        emit importProgress(0);
-
-        qCDebug(octree) << "Loading file" << qFileName << "...";
-
-        fileOk = readFromStream(fileLength, fileInputStream);
-
-        emit importProgress(100);
-        file.close();
+    if (qFileName.endsWith(".json.gz")) {
+        return readJSONFromGzippedFile(qFileName);
     }
 
-    return fileOk;
+    QFile file(qFileName);
+
+    if (!file.open(QIODevice::ReadOnly)) {
+        qCritical() << "unable to open for reading: " << fileName;
+        return false;
+    }
+
+    QDataStream fileInputStream(&file);
+    QFileInfo fileInfo(qFileName);
+    unsigned long fileLength = fileInfo.size();
+
+    emit importSize(1.0f, 1.0f, 1.0f);
+    emit importProgress(0);
+
+    qCDebug(octree) << "Loading file" << qFileName << "...";
+
+    bool success = readFromStream(fileLength, fileInputStream);
+
+    emit importProgress(100);
+    file.close();
+
+    return success;
+}
+
+bool Octree::readJSONFromGzippedFile(QString qFileName) {
+    QFile file(qFileName);
+    if (!file.open(QIODevice::ReadOnly)) {
+        qCritical() << "Cannot open gzipped json file for reading: " << qFileName;
+        return false;
+    }
+    QByteArray compressedJsonData = file.readAll();
+    QByteArray jsonData;
+
+    if (!gunzip(compressedJsonData, jsonData)) {
+        qCritical() << "json File not in gzip format: " << qFileName;
+        return false;
+    }
+
+    QDataStream jsonStream(jsonData);
+    return readJSONFromStream(-1, jsonStream);
 }
 
 bool Octree::readFromURL(const QString& urlString) {
@@ -1859,18 +1883,17 @@ bool Octree::readFromURL(const QString& urlString) {
 
 
 bool Octree::readFromStream(unsigned long streamLength, QDataStream& inputStream) {
-
-    // decide if this is SVO or JSON
+    // decide if this is binary SVO or JSON-formatted SVO
     QIODevice *device = inputStream.device();
     char firstChar;
     device->getChar(&firstChar);
     device->ungetChar(firstChar);
 
     if (firstChar == (char) PacketType::EntityData) {
-        qCDebug(octree) << "Reading from SVO Stream length:" << streamLength;
+        qCDebug(octree) << "Reading from binary SVO Stream length:" << streamLength;
         return readSVOFromStream(streamLength, inputStream);
     } else {
-        qCDebug(octree) << "Reading from JSON Stream length:" << streamLength;
+        qCDebug(octree) << "Reading from JSON SVO Stream length:" << streamLength;
         return readJSONFromStream(streamLength, inputStream);
     }
 }
@@ -2006,12 +2029,28 @@ bool Octree::readSVOFromStream(unsigned long streamLength, QDataStream& inputStr
     return fileOk;
 }
 
-bool Octree::readJSONFromStream(unsigned long streamLength, QDataStream& inputStream) {
-    char* rawData = new char[streamLength + 1]; // allocate enough room to null terminate
-    inputStream.readRawData(rawData, streamLength);
-    rawData[streamLength] = 0; // make sure we null terminate this string
+const int READ_JSON_BUFFER_SIZE = 2048;
 
-    QJsonDocument asDocument = QJsonDocument::fromJson(rawData);
+bool Octree::readJSONFromStream(unsigned long streamLength, QDataStream& inputStream) {
+    // if the data is gzipped we may not have a useful bytesAvailable() result, so just keep reading until
+    // we get an eof.  Leave streamLength parameter for consistency.
+
+    QByteArray jsonBuffer;
+    char* rawData = new char[READ_JSON_BUFFER_SIZE];
+    while (true) {
+        int got = inputStream.readRawData(rawData, READ_JSON_BUFFER_SIZE - 1);
+        if (got < 0) {
+            qCritical() << "error while reading from json stream";
+            delete[] rawData;
+            return false;
+        }
+        if (got == 0) {
+            break;
+        }
+        jsonBuffer += QByteArray(rawData, got);
+    }
+
+    QJsonDocument asDocument = QJsonDocument::fromJson(jsonBuffer);
     QVariant asVariant = asDocument.toVariant();
     QVariantMap asMap = asVariant.toMap();
     readFromMap(asMap);
@@ -2029,13 +2068,14 @@ void Octree::writeToFile(const char* fileName, OctreeElement* element, QString p
         writeToSVOFile(fileName, element);
     } else if (persistAsFileType == "json") {
         writeToJSONFile(cFileName, element);
+    } else if (persistAsFileType == "json.gz") {
+        writeToJSONFile(cFileName, element, true);
     } else {
         qCDebug(octree) << "unable to write octree to file of type" << persistAsFileType;
     }
 }
 
-void Octree::writeToJSONFile(const char* fileName, OctreeElement* element) {
-    QFile persistFile(fileName);
+void Octree::writeToJSONFile(const char* fileName, OctreeElement* element, bool doGzip) {
     QVariantMap entityDescription;
 
     qCDebug(octree, "Saving JSON SVO to file %s...", fileName);
@@ -2054,10 +2094,27 @@ void Octree::writeToJSONFile(const char* fileName, OctreeElement* element) {
 
     // store the entity data
     bool entityDescriptionSuccess = writeToMap(entityDescription, top, true);
+    if (!entityDescriptionSuccess) {
+        qCritical("Failed to convert Entities to QVariantMap while saving to json.");
+        return;
+    }
 
     // convert the QVariantMap to JSON
-    if (entityDescriptionSuccess && persistFile.open(QIODevice::WriteOnly)) {
-        persistFile.write(QJsonDocument::fromVariant(entityDescription).toJson());
+    QByteArray jsonData = QJsonDocument::fromVariant(entityDescription).toJson();
+    QByteArray jsonDataForFile;
+
+    if (doGzip) {
+        if (!gzip(jsonData, jsonDataForFile, -1)) {
+            qCritical("unable to gzip data while saving to json.");
+            return;
+        }
+    } else {
+        jsonDataForFile = jsonData;
+    }
+
+    QFile persistFile(fileName);
+    if (persistFile.open(QIODevice::WriteOnly)) {
+        persistFile.write(jsonDataForFile);
     } else {
         qCritical("Could not write to JSON description of entities.");
     }

libraries/octree/src/Octree.h

@@ -330,7 +330,7 @@ public:
 
     // Octree exporters
     void writeToFile(const char* filename, OctreeElement* element = NULL, QString persistAsFileType = "svo");
-    void writeToJSONFile(const char* filename, OctreeElement* element = NULL);
+    void writeToJSONFile(const char* filename, OctreeElement* element = NULL, bool doGzip = false);
     void writeToSVOFile(const char* filename, OctreeElement* element = NULL);
     virtual bool writeToMap(QVariantMap& entityDescription, OctreeElement* element, bool skipDefaultValues) = 0;
 
@@ -340,6 +340,7 @@ public:
     bool readFromStream(unsigned long streamLength, QDataStream& inputStream);
     bool readSVOFromStream(unsigned long streamLength, QDataStream& inputStream);
     bool readJSONFromStream(unsigned long streamLength, QDataStream& inputStream);
+    bool readJSONFromGzippedFile(QString qFileName);
     virtual bool readFromMap(QVariantMap& entityDescription) = 0;
 
     unsigned long getOctreeElementsCount();

libraries/shared/CMakeLists.txt

@@ -4,7 +4,17 @@ set(TARGET_NAME shared)
 # TODO: there isn't really a good reason to have Script linked here - let's get what is requiring it out (RegisteredMetaTypes.cpp)
 setup_hifi_library(Gui Network Script Widgets)
 
+find_package(ZLIB REQUIRED)
+target_link_libraries(${TARGET_NAME} ${ZLIB_LIBRARIES})
+
 add_dependency_external_projects(glm)
 find_package(GLM REQUIRED)
-target_include_directories(${TARGET_NAME} PUBLIC ${GLM_INCLUDE_DIRS})
+target_include_directories(${TARGET_NAME} PUBLIC ${GLM_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS})
 
+if (WIN32)
+  # Birarda will fix this when he finds it.
+  get_filename_component(ZLIB_LIB_DIR "${ZLIB_LIBRARIES}" DIRECTORY)
+  get_filename_component(ZLIB_DIR "${ZLIB_LIB_DIR}" DIRECTORY)
+  set(ZLIB_BIN_DIR "${ZLIB_DIR}/bin")
+  add_paths_to_fixup_libs(${ZLIB_BIN_DIR})
+endif ()

libraries/shared/src/Gzip.cpp

@@ -0,0 +1,153 @@
+//
+//  Gzip.cpp
+//  libraries/shared/src
+//
+//  Created by Seth Alves on 2015-08-03.
+//  Copyright 2015 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <zlib.h>
+#include "Gzip.h"
+
+const int GZIP_WINDOWS_BIT = 31;
+const int GZIP_CHUNK_SIZE = 4096;
+const int DEFAULT_MEM_LEVEL = 8;
+
+bool gunzip(QByteArray source, QByteArray &destination) {
+    destination.clear();
+    if (source.length() == 0) {
+        return true;
+    }
+
+    z_stream strm;
+    strm.zalloc = Z_NULL;
+    strm.zfree = Z_NULL;
+    strm.opaque = Z_NULL;
+    strm.avail_in = 0;
+    strm.next_in = Z_NULL;
+
+    int status = inflateInit2(&strm, GZIP_WINDOWS_BIT);
+
+    if (status != Z_OK) {
+        return false;
+    }
+
+    char *sourceData = source.data();
+    int sourceDataLength = source.length();
+
+    for (;;) {
+        int chunkSize = qMin(GZIP_CHUNK_SIZE, sourceDataLength);
+        if (chunkSize <= 0) {
+            break;
+        }
+
+        strm.next_in = (unsigned char*)sourceData;
+        strm.avail_in = chunkSize;
+        sourceData += chunkSize;
+        sourceDataLength -= chunkSize;
+
+        for (;;) {
+            char out[GZIP_CHUNK_SIZE];
+
+            strm.next_out = (unsigned char*)out;
+            strm.avail_out = GZIP_CHUNK_SIZE;
+
+            status = inflate(&strm, Z_NO_FLUSH);
+
+            switch (status) {
+                case Z_NEED_DICT:
+                    status = Z_DATA_ERROR;
+                case Z_DATA_ERROR:
+                case Z_MEM_ERROR:
+                case Z_STREAM_ERROR:
+                    inflateEnd(&strm);
+                    return false;
+            }
+
+            int available = (GZIP_CHUNK_SIZE - strm.avail_out);
+            if (available > 0) {
+                destination.append((char*)out, available);
+            }
+
+            if (strm.avail_out != 0) {
+                break;
+            }
+        }
+
+        if (status == Z_STREAM_END) {
+            break;
+        }
+    }
+
+    inflateEnd(&strm);
+    return status == Z_STREAM_END;
+}
+
+bool gzip(QByteArray source, QByteArray &destination, int compressionLevel) {
+    destination.clear();
+    if (source.length() == 0) {
+        return true;
+    }
+
+    int flushOrFinish = 0;
+    z_stream strm;
+    strm.zalloc = Z_NULL;
+    strm.zfree = Z_NULL;
+    strm.opaque = Z_NULL;
+    strm.next_in = Z_NULL;
+    strm.avail_in = 0;
+
+    int status = deflateInit2(&strm,
+                              qMax(Z_DEFAULT_COMPRESSION, qMin(9, compressionLevel)),
+                              Z_DEFLATED,
+                              GZIP_WINDOWS_BIT,
+                              DEFAULT_MEM_LEVEL,
+                              Z_DEFAULT_STRATEGY);
+    if (status != Z_OK) {
+        return false;
+    }
+    char *sourceData = source.data();
+    int sourceDataLength = source.length();
+
+    for (;;) {
+        int chunkSize = qMin(GZIP_CHUNK_SIZE, sourceDataLength);
+        strm.next_in = (unsigned char*)sourceData;
+        strm.avail_in = chunkSize;
+        sourceData += chunkSize;
+        sourceDataLength -= chunkSize;
+
+        if (sourceDataLength <= 0) {
+            flushOrFinish = Z_FINISH;
+        } else {
+            flushOrFinish = Z_NO_FLUSH;
+        }
+
+        for (;;) {
+            char out[GZIP_CHUNK_SIZE];
+            strm.next_out = (unsigned char*)out;
+            strm.avail_out = GZIP_CHUNK_SIZE;
+            status = deflate(&strm, flushOrFinish);
+            if (status == Z_STREAM_ERROR) {
+                deflateEnd(&strm);
+                return false;
+            }
+            int available = (GZIP_CHUNK_SIZE - strm.avail_out);
+            if (available > 0) {
+                destination.append((char*)out, available);
+            }
+            if (strm.avail_out != 0) {
+                break;
+            }
+        }
+
+        if (flushOrFinish == Z_FINISH) {
+            break;
+        }
+    }
+
+    deflateEnd(&strm);
+    return status == Z_STREAM_END;
+}

libraries/shared/src/Gzip.h

@@ -0,0 +1,27 @@
+//
+//  Gzip.h
+//  libraries/shared/src
+//
+//  Created by Seth Alves on 2015-08-03.
+//  Copyright 2015 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef GZIP_H
+#define GZIP_H
+
+#include <QByteArray>
+
+// The compression level must be Z_DEFAULT_COMPRESSION (-1), or between 0 and
+// 9: 1 gives best speed, 9 gives best compression, 0 gives no
+// compression at all (the input data is simply copied a block at a
+// time).  Z_DEFAULT_COMPRESSION requests a default compromise between
+// speed and compression (currently equivalent to level 6).
+
+bool gzip(QByteArray source, QByteArray &destination, int compressionLevel = -1); // -1 is Z_DEFAULT_COMPRESSION
+
+bool gunzip(QByteArray source, QByteArray &destination);
+
+#endif