Added an option to domain-server settings to persist entities as gzipped json. With this setting enabled, persist files take much less space on disk than with the other two options.

domain-server/resources/describe-settings.json

@@ -389,6 +389,10 @@
             {
               "value": "json",
               "label": "Entity server persists data as JSON"
+            },
+            {
+              "value": "json.gz",
+              "label": "Entity server persists data as gzipped JSON"
             }
           ],
           "advanced": true

interface/CMakeLists.txt

@@ -13,6 +13,7 @@ endforeach()
 
 find_package(Qt5LinguistTools REQUIRED)
 find_package(Qt5LinguistToolsMacros)
+find_package(ZLIB REQUIRED)
 
 if (DEFINED ENV{JOB_ID})
   set(BUILD_SEQ $ENV{JOB_ID})
@@ -23,8 +24,8 @@ else ()
 endif ()
 
 if (WIN32)
-  add_definitions(-D_USE_MATH_DEFINES) # apparently needed to get M_PI and other defines from cmath/math.h		
+  add_definitions(-D_USE_MATH_DEFINES) # apparently needed to get M_PI and other defines from cmath/math.h
-  add_definitions(-DWINDOWS_LEAN_AND_MEAN) # needed to make sure windows doesn't go to crazy with its defines		
+  add_definitions(-DWINDOWS_LEAN_AND_MEAN) # needed to make sure windows doesn't go to crazy with its defines
 endif()
 
 configure_file(InterfaceVersion.h.in "${PROJECT_BINARY_DIR}/includes/InterfaceVersion.h")
@@ -82,19 +83,19 @@ if (APPLE)
 
   set(MACOSX_BUNDLE_BUNDLE_NAME Interface)
   set(MACOSX_BUNDLE_GUI_IDENTIFIER io.highfidelity.Interface)
-  
+
   if (UPPER_CMAKE_BUILD_TYPE MATCHES RELEASE OR UPPER_CMAKE_BUILD_TYPE MATCHES RELWITHDEBINFO)
     set(ICON_FILENAME "interface.icns")
   else ()
     set(ICON_FILENAME "interface-beta.icns")
   endif ()
-  
+
   # set how the icon shows up in the Info.plist file
   SET(MACOSX_BUNDLE_ICON_FILE "${ICON_FILENAME}")
 
   # set where in the bundle to put the resources file
   SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_SOURCE_DIR}/icon/${ICON_FILENAME} PROPERTIES MACOSX_PACKAGE_LOCATION Resources)
-  
+
   set(DISCOVERED_RESOURCES "")
 
   # use the add_resources_to_os_x_bundle macro to recurse into resources
@@ -102,7 +103,7 @@ if (APPLE)
 
   # append the discovered resources to our list of interface sources
   list(APPEND INTERFACE_SRCS ${DISCOVERED_RESOURCES})
-  
+
   set(INTERFACE_SRCS ${INTERFACE_SRCS} "${CMAKE_CURRENT_SOURCE_DIR}/icon/${ICON_FILENAME}")
 endif()
 
@@ -145,34 +146,34 @@ add_dependency_external_projects(sdl2)
 
 # perform standard include and linking for found externals
 foreach(EXTERNAL ${OPTIONAL_EXTERNALS})
-  
+
   if (${${EXTERNAL}_UPPERCASE}_REQUIRED)
     find_package(${EXTERNAL} REQUIRED)
   else ()
     find_package(${EXTERNAL})
   endif ()
-  
+
   if (${${EXTERNAL}_UPPERCASE}_FOUND AND NOT DISABLE_${${EXTERNAL}_UPPERCASE})
     add_definitions(-DHAVE_${${EXTERNAL}_UPPERCASE})
-    
+
     # include the library directories (ignoring warnings)
     if (NOT ${${EXTERNAL}_UPPERCASE}_INCLUDE_DIRS)
       set(${${EXTERNAL}_UPPERCASE}_INCLUDE_DIRS ${${${EXTERNAL}_UPPERCASE}_INCLUDE_DIR})
     endif ()
-    
+
     include_directories(SYSTEM ${${${EXTERNAL}_UPPERCASE}_INCLUDE_DIRS})
-    
+
     # perform the system include hack for OS X to ignore warnings
     if (APPLE)
       foreach(EXTERNAL_INCLUDE_DIR  ${${${EXTERNAL}_UPPERCASE}_INCLUDE_DIRS})
         SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem ${EXTERNAL_INCLUDE_DIR}")
       endforeach()
     endif ()
-    
+
     if (NOT ${${EXTERNAL}_UPPERCASE}_LIBRARIES)
       set(${${EXTERNAL}_UPPERCASE}_LIBRARIES ${${${EXTERNAL}_UPPERCASE}_LIBRARY})
     endif ()
-    
+
     if (NOT APPLE OR NOT ${${EXTERNAL}_UPPERCASE} MATCHES "SIXENSE")
       target_link_libraries(${TARGET_NAME} ${${${EXTERNAL}_UPPERCASE}_LIBRARIES})
     elseif (APPLE AND NOT INSTALLER_BUILD)
@@ -199,13 +200,12 @@ target_link_libraries(
 # assume we are using a Qt build without bearer management
 add_definitions(-DQT_NO_BEARERMANAGEMENT)
 
+
 if (APPLE)
   # link in required OS X frameworks and include the right GL headers
   find_library(OpenGL OpenGL)
   find_library(AppKit AppKit)
 
-  target_link_libraries(${TARGET_NAME} ${OpenGL} ${AppKit})
-    
   # install command for OS X bundle
   INSTALL(TARGETS ${TARGET_NAME}
     BUNDLE DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/install" COMPONENT Runtime
@@ -224,7 +224,6 @@ else (APPLE)
   #  target_link_libraries(${TARGET_NAME} wsock32.lib Winmm.lib)
     target_link_libraries(${TARGET_NAME} wsock32.lib Winmm.lib)
   else (WIN32)
-    # Nothing else required on linux apparently
   endif()
 endif (APPLE)
 

libraries/entities/src/EntityItemProperties.cpp

@@ -71,7 +71,8 @@ CONSTRUCT_PROPERTY(exponent, 0.0f),
 CONSTRUCT_PROPERTY(cutoff, ENTITY_ITEM_DEFAULT_CUTOFF),
 CONSTRUCT_PROPERTY(locked, ENTITY_ITEM_DEFAULT_LOCKED),
 CONSTRUCT_PROPERTY(textures, ""),
-CONSTRUCT_PROPERTY(animationSettings, ""),
+CONSTRUCT_PROPERTY(animationSettings, "{\"firstFrame\":0,\"fps\":30,\"frameIndex\":0,\"hold\":false,"
+                   "\"lastFrame\":100000,\"loop\":false,\"running\":false,\"startAutomatically\":false}"),
 CONSTRUCT_PROPERTY(userData, ENTITY_ITEM_DEFAULT_USER_DATA),
 CONSTRUCT_PROPERTY(simulationOwner, SimulationOwner()),
 CONSTRUCT_PROPERTY(text, TextEntityItem::DEFAULT_TEXT),

libraries/octree/src/Octree.cpp

@@ -40,6 +40,7 @@
 #include <SharedUtil.h>
 #include <Shape.h>
 #include <PathUtils.h>
+#include <Gzip.h>
 
 #include "CoverageMap.h"
 #include "OctreeConstants.h"
@@ -49,7 +50,7 @@
 #include "OctreeLogging.h"
 
 
-QVector<QString> PERSIST_EXTENSIONS = {"svo", "json"};
+QVector<QString> PERSIST_EXTENSIONS = {"svo", "json", "json.gz"};
 
 float boundaryDistanceForRenderLevel(unsigned int renderLevel, float voxelSizeScale) {
     return voxelSizeScale / powf(2, renderLevel);
@@ -1809,29 +1810,52 @@ int Octree::encodeTreeBitstreamRecursion(OctreeElement* element,
 }
 
 bool Octree::readFromFile(const char* fileName) {
-    bool fileOk = false;
-
     QString qFileName = findMostRecentFileExtension(fileName, PERSIST_EXTENSIONS);
-    QFile file(qFileName);
-    fileOk = file.open(QIODevice::ReadOnly);
 
-    if(fileOk) {
+    if (qFileName.endsWith(".json.gz")) {
-        QDataStream fileInputStream(&file);
+        return readJSONFromGzippedFile(qFileName);
-        QFileInfo fileInfo(qFileName);
-        unsigned long fileLength = fileInfo.size();
-
-        emit importSize(1.0f, 1.0f, 1.0f);
-        emit importProgress(0);
-
-        qCDebug(octree) << "Loading file" << qFileName << "...";
-
-        fileOk = readFromStream(fileLength, fileInputStream);
-
-        emit importProgress(100);
-        file.close();
     }
 
-    return fileOk;
+    QFile file(qFileName);
+
+    if (!file.open(QIODevice::ReadOnly)) {
+        qCritical() << "unable to open for reading: " << fileName;
+        return false;
+    }
+
+    QDataStream fileInputStream(&file);
+    QFileInfo fileInfo(qFileName);
+    unsigned long fileLength = fileInfo.size();
+
+    emit importSize(1.0f, 1.0f, 1.0f);
+    emit importProgress(0);
+
+    qCDebug(octree) << "Loading file" << qFileName << "...";
+
+    bool success = readFromStream(fileLength, fileInputStream);
+
+    emit importProgress(100);
+    file.close();
+
+    return success;
+}
+
+bool Octree::readJSONFromGzippedFile(QString qFileName) {
+    QFile file(qFileName);
+    if (!file.open(QIODevice::ReadOnly)) {
+        qCritical() << "Cannot open gzipped json file for reading: " << qFileName;
+        return false;
+    }
+    QByteArray compressedJsonData = file.readAll();
+    QByteArray jsonData;
+
+    if (!gunzip(compressedJsonData, jsonData)) {
+        qCritical() << "json File not in gzip format: " << qFileName;
+        return false;
+    }
+
+    QDataStream jsonStream(jsonData);
+    return readJSONFromStream(-1, jsonStream);
 }
 
 bool Octree::readFromURL(const QString& urlString) {
@@ -1868,17 +1892,17 @@ bool Octree::readFromURL(const QString& urlString) {
 
 bool Octree::readFromStream(unsigned long streamLength, QDataStream& inputStream) {
 
-    // decide if this is SVO or JSON
+    // decide if this is binary SVO or JSON encoded SVO
     QIODevice *device = inputStream.device();
     char firstChar;
     device->getChar(&firstChar);
     device->ungetChar(firstChar);
 
     if (firstChar == (char) PacketType::EntityData) {
-        qCDebug(octree) << "Reading from SVO Stream length:" << streamLength;
+        qCDebug(octree) << "Reading from binary SVO Stream length:" << streamLength;
         return readSVOFromStream(streamLength, inputStream);
     } else {
-        qCDebug(octree) << "Reading from JSON Stream length:" << streamLength;
+        qCDebug(octree) << "Reading from JSON SVO Stream length:" << streamLength;
         return readJSONFromStream(streamLength, inputStream);
     }
 }
@@ -2013,12 +2037,28 @@ bool Octree::readSVOFromStream(unsigned long streamLength, QDataStream& inputStr
     return fileOk;
 }
 
-bool Octree::readJSONFromStream(unsigned long streamLength, QDataStream& inputStream) {
+const int READ_JSON_BUFFER_SIZE = 2048;
-    char* rawData = new char[streamLength + 1]; // allocate enough room to null terminate
-    inputStream.readRawData(rawData, streamLength);
-    rawData[streamLength] = 0; // make sure we null terminate this string
 
-    QJsonDocument asDocument = QJsonDocument::fromJson(rawData);
+bool Octree::readJSONFromStream(unsigned long streamLength, QDataStream& inputStream) {
+    // QuaGzipFile doesn't appear to give a useful bytesAvailable() result, so just keep reading until
+    // we get an eof.  Leave streamLength parameter for consistency.
+
+    QByteArray jsonBuffer;
+    char* rawData = new char[READ_JSON_BUFFER_SIZE];
+    while (true) {
+        int got = inputStream.readRawData(rawData, READ_JSON_BUFFER_SIZE - 1);
+        if (got < 0) {
+            qCritical() << "error while reading from json stream";
+            delete[] rawData;
+            return false;
+        }
+        if (got == 0) {
+            break;
+        }
+        jsonBuffer += QByteArray(rawData, got);
+    }
+
+    QJsonDocument asDocument = QJsonDocument::fromJson(jsonBuffer);
     QVariant asVariant = asDocument.toVariant();
     QVariantMap asMap = asVariant.toMap();
     readFromMap(asMap);
@@ -2036,13 +2076,14 @@ void Octree::writeToFile(const char* fileName, OctreeElement* element, QString p
         writeToSVOFile(fileName, element);
     } else if (persistAsFileType == "json") {
         writeToJSONFile(cFileName, element);
+    } else if (persistAsFileType == "json.gz") {
+        writeToJSONFile(cFileName, element, true);
     } else {
         qCDebug(octree) << "unable to write octree to file of type" << persistAsFileType;
     }
 }
 
-void Octree::writeToJSONFile(const char* fileName, OctreeElement* element) {
+void Octree::writeToJSONFile(const char* fileName, OctreeElement* element, bool doGzip) {
-    QFile persistFile(fileName);
     QVariantMap entityDescription;
 
     qCDebug(octree, "Saving JSON SVO to file %s...", fileName);
@@ -2061,10 +2102,27 @@ void Octree::writeToJSONFile(const char* fileName, OctreeElement* element) {
 
     // store the entity data
     bool entityDescriptionSuccess = writeToMap(entityDescription, top, true);
+    if (!entityDescriptionSuccess) {
+        qCritical("Failed to convert Entities to QVariantMap while saving to json.");
+        return;
+    }
 
     // convert the QVariantMap to JSON
-    if (entityDescriptionSuccess && persistFile.open(QIODevice::WriteOnly)) {
+    QByteArray jsonData = QJsonDocument::fromVariant(entityDescription).toJson();
-        persistFile.write(QJsonDocument::fromVariant(entityDescription).toJson());
+    QByteArray jsonDataForFile;
+
+    if (doGzip) {
+        if (!gzip(jsonData, jsonDataForFile, -1)) {
+            qCritical("unable to gzip data while saving to json.");
+            return;
+        }
+    } else {
+        jsonDataForFile = jsonData;
+    }
+
+    QFile persistFile(fileName);
+    if (persistFile.open(QIODevice::WriteOnly)) {
+        persistFile.write(jsonDataForFile);
     } else {
         qCritical("Could not write to JSON description of entities.");
     }

libraries/octree/src/Octree.h

@@ -330,7 +330,7 @@ public:
 
     // Octree exporters
     void writeToFile(const char* filename, OctreeElement* element = NULL, QString persistAsFileType = "svo");
-    void writeToJSONFile(const char* filename, OctreeElement* element = NULL);
+    void writeToJSONFile(const char* filename, OctreeElement* element = NULL, bool doGzip = false);
     void writeToSVOFile(const char* filename, OctreeElement* element = NULL);
     virtual bool writeToMap(QVariantMap& entityDescription, OctreeElement* element, bool skipDefaultValues) = 0;
 
@@ -340,6 +340,7 @@ public:
     bool readFromStream(unsigned long streamLength, QDataStream& inputStream);
     bool readSVOFromStream(unsigned long streamLength, QDataStream& inputStream);
     bool readJSONFromStream(unsigned long streamLength, QDataStream& inputStream);
+    bool readJSONFromGzippedFile(QString qFileName);
     virtual bool readFromMap(QVariantMap& entityDescription) = 0;
 
     unsigned long getOctreeElementsCount();

libraries/shared/CMakeLists.txt

@@ -4,7 +4,9 @@ set(TARGET_NAME shared)
 # TODO: there isn't really a good reason to have Script linked here - let's get what is requiring it out (RegisteredMetaTypes.cpp)
 setup_hifi_library(Gui Network Script Widgets)
 
+find_package(ZLIB REQUIRED)
+target_link_libraries(${TARGET_NAME} ${ZLIB_LIBRARIES})
+
 add_dependency_external_projects(glm)
 find_package(GLM REQUIRED)
 target_include_directories(${TARGET_NAME} PUBLIC ${GLM_INCLUDE_DIRS})
-

libraries/shared/src/Gzip.cpp

@@ -0,0 +1,152 @@
+//
+//  Gzip.cpp
+//  libraries/shared/src
+//
+//  Created by Seth Alves on 2015-08-03.
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <zlib.h>
+#include "Gzip.h"
+
+const int GZIP_WINDOWS_BIT = 31;
+const int GZIP_CHUNK_SIZE = 4096;
+
+bool gunzip(QByteArray source, QByteArray &destination) {
+    destination.clear();
+    if (source.length() == 0) {
+        return true;
+    }
+
+    z_stream strm;
+    strm.zalloc = Z_NULL;
+    strm.zfree = Z_NULL;
+    strm.opaque = Z_NULL;
+    strm.avail_in = 0;
+    strm.next_in = Z_NULL;
+
+    int status = inflateInit2(&strm, GZIP_WINDOWS_BIT);
+
+    if (status != Z_OK) {
+        return false;
+    }
+
+    char *sourceData = source.data();
+    int sourceDataLength = source.length();
+
+    for (;;) {
+        int chunkSize = qMin(GZIP_CHUNK_SIZE, sourceDataLength);
+        if (chunkSize <= 0) {
+            break;
+        }
+
+        strm.next_in = (unsigned char*)sourceData;
+        strm.avail_in = chunkSize;
+        sourceData += chunkSize;
+        sourceDataLength -= chunkSize;
+
+        for (;;) {
+            char out[GZIP_CHUNK_SIZE];
+
+            strm.next_out = (unsigned char*)out;
+            strm.avail_out = GZIP_CHUNK_SIZE;
+
+            status = inflate(&strm, Z_NO_FLUSH);
+
+            switch (status) {
+                case Z_NEED_DICT:
+                    status = Z_DATA_ERROR;
+                case Z_DATA_ERROR:
+                case Z_MEM_ERROR:
+                case Z_STREAM_ERROR:
+                    inflateEnd(&strm);
+                    return false;
+            }
+
+            int available = (GZIP_CHUNK_SIZE - strm.avail_out);
+            if (available > 0) {
+                destination.append((char*)out, available);
+            }
+
+            if (strm.avail_out != 0) {
+                break;
+            }
+        }
+
+        if (status == Z_STREAM_END) {
+            break;
+        }
+    }
+
+    inflateEnd(&strm);
+    return status == Z_STREAM_END;
+}
+
+bool gzip(QByteArray source, QByteArray &destination, int compressionLevel) {
+    destination.clear();
+    if (source.length() == 0) {
+        return true;
+    }
+
+    int flushOrFinish = 0;
+    z_stream strm;
+    strm.zalloc = Z_NULL;
+    strm.zfree = Z_NULL;
+    strm.opaque = Z_NULL;
+    strm.next_in = Z_NULL;
+    strm.avail_in = 0;
+
+    int status = deflateInit2(&strm,
+                              qMax(-1, qMin(9, compressionLevel)),
+                              Z_DEFLATED,
+                              GZIP_WINDOWS_BIT,
+                              8,
+                              Z_DEFAULT_STRATEGY);
+    if (status != Z_OK) {
+        return false;
+    }
+    char *sourceData = source.data();
+    int sourceDataLength = source.length();
+
+    for (;;) {
+        int chunkSize = qMin(GZIP_CHUNK_SIZE, sourceDataLength);
+        strm.next_in = (unsigned char*)sourceData;
+        strm.avail_in = chunkSize;
+        sourceData += chunkSize;
+        sourceDataLength -= chunkSize;
+
+        if (sourceDataLength <= 0) {
+            flushOrFinish = Z_FINISH;
+        } else {
+            flushOrFinish = Z_NO_FLUSH;
+        }
+
+        for (;;) {
+            char out[GZIP_CHUNK_SIZE];
+            strm.next_out = (unsigned char*)out;
+            strm.avail_out = GZIP_CHUNK_SIZE;
+            status = deflate(&strm, flushOrFinish);
+            if (status == Z_STREAM_ERROR) {
+                deflateEnd(&strm);
+                return false;
+            }
+            int available = (GZIP_CHUNK_SIZE - strm.avail_out);
+            if (available > 0) {
+                destination.append((char*)out, available);
+            }
+            if (strm.avail_out != 0) {
+                break;
+            }
+        }
+
+        if (flushOrFinish == Z_FINISH) {
+            break;
+        }
+    }
+
+    deflateEnd(&strm);
+    return status == Z_STREAM_END;
+}

libraries/shared/src/Gzip.h

@@ -0,0 +1,20 @@
+//
+//  Gzip.h
+//  libraries/shared/src
+//
+//  Created by Seth Alves on 2015-08-03.
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef GZIP_H
+#define GZIP_H
+
+#include <QByteArray>
+
+bool gzip(QByteArray source, QByteArray &destination, int compressionLevel = -1);
+bool gunzip(QByteArray source, QByteArray &destination);
+
+#endif