diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index ff2f4cf683..696a87d2b8 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -389,6 +389,10 @@ { "value": "json", "label": "Entity server persists data as JSON" + }, + { + "value": "json.gz", + "label": "Entity server persists data as gzipped JSON" } ], "advanced": true diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index f1ef38ade9..11af6e70a0 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -13,6 +13,7 @@ endforeach() find_package(Qt5LinguistTools REQUIRED) find_package(Qt5LinguistToolsMacros) +find_package(ZLIB REQUIRED) if (DEFINED ENV{JOB_ID}) set(BUILD_SEQ $ENV{JOB_ID}) @@ -23,8 +24,8 @@ else () endif () if (WIN32) - add_definitions(-D_USE_MATH_DEFINES) # apparently needed to get M_PI and other defines from cmath/math.h - add_definitions(-DWINDOWS_LEAN_AND_MEAN) # needed to make sure windows doesn't go to crazy with its defines + add_definitions(-D_USE_MATH_DEFINES) # apparently needed to get M_PI and other defines from cmath/math.h + add_definitions(-DWINDOWS_LEAN_AND_MEAN) # needed to make sure windows doesn't go to crazy with its defines endif() configure_file(InterfaceVersion.h.in "${PROJECT_BINARY_DIR}/includes/InterfaceVersion.h") @@ -82,19 +83,19 @@ if (APPLE) set(MACOSX_BUNDLE_BUNDLE_NAME Interface) set(MACOSX_BUNDLE_GUI_IDENTIFIER io.highfidelity.Interface) - + if (UPPER_CMAKE_BUILD_TYPE MATCHES RELEASE OR UPPER_CMAKE_BUILD_TYPE MATCHES RELWITHDEBINFO) set(ICON_FILENAME "interface.icns") else () set(ICON_FILENAME "interface-beta.icns") endif () - + # set how the icon shows up in the Info.plist file SET(MACOSX_BUNDLE_ICON_FILE "${ICON_FILENAME}") # set where in the bundle to put the resources file SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_SOURCE_DIR}/icon/${ICON_FILENAME} PROPERTIES MACOSX_PACKAGE_LOCATION Resources) - + set(DISCOVERED_RESOURCES "") # use the add_resources_to_os_x_bundle macro to recurse into resources @@ -102,7 +103,7 @@ if (APPLE) # append the discovered resources to our list of interface sources list(APPEND INTERFACE_SRCS ${DISCOVERED_RESOURCES}) - + set(INTERFACE_SRCS ${INTERFACE_SRCS} "${CMAKE_CURRENT_SOURCE_DIR}/icon/${ICON_FILENAME}") endif() @@ -145,34 +146,34 @@ add_dependency_external_projects(sdl2) # perform standard include and linking for found externals foreach(EXTERNAL ${OPTIONAL_EXTERNALS}) - + if (${${EXTERNAL}_UPPERCASE}_REQUIRED) find_package(${EXTERNAL} REQUIRED) else () find_package(${EXTERNAL}) endif () - + if (${${EXTERNAL}_UPPERCASE}_FOUND AND NOT DISABLE_${${EXTERNAL}_UPPERCASE}) add_definitions(-DHAVE_${${EXTERNAL}_UPPERCASE}) - + # include the library directories (ignoring warnings) if (NOT ${${EXTERNAL}_UPPERCASE}_INCLUDE_DIRS) set(${${EXTERNAL}_UPPERCASE}_INCLUDE_DIRS ${${${EXTERNAL}_UPPERCASE}_INCLUDE_DIR}) endif () - + include_directories(SYSTEM ${${${EXTERNAL}_UPPERCASE}_INCLUDE_DIRS}) - + # perform the system include hack for OS X to ignore warnings if (APPLE) foreach(EXTERNAL_INCLUDE_DIR ${${${EXTERNAL}_UPPERCASE}_INCLUDE_DIRS}) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem ${EXTERNAL_INCLUDE_DIR}") endforeach() endif () - + if (NOT ${${EXTERNAL}_UPPERCASE}_LIBRARIES) set(${${EXTERNAL}_UPPERCASE}_LIBRARIES ${${${EXTERNAL}_UPPERCASE}_LIBRARY}) endif () - + if (NOT APPLE OR NOT ${${EXTERNAL}_UPPERCASE} MATCHES "SIXENSE") target_link_libraries(${TARGET_NAME} ${${${EXTERNAL}_UPPERCASE}_LIBRARIES}) elseif (APPLE AND NOT INSTALLER_BUILD) @@ -199,13 +200,12 @@ target_link_libraries( # assume we are using a Qt build without bearer management add_definitions(-DQT_NO_BEARERMANAGEMENT) + if (APPLE) # link in required OS X frameworks and include the right GL headers find_library(OpenGL OpenGL) find_library(AppKit AppKit) - target_link_libraries(${TARGET_NAME} ${OpenGL} ${AppKit}) - # install command for OS X bundle INSTALL(TARGETS ${TARGET_NAME} BUNDLE DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/install" COMPONENT Runtime @@ -224,7 +224,6 @@ else (APPLE) # target_link_libraries(${TARGET_NAME} wsock32.lib Winmm.lib) target_link_libraries(${TARGET_NAME} wsock32.lib Winmm.lib) else (WIN32) - # Nothing else required on linux apparently endif() endif (APPLE) diff --git a/libraries/entities/src/EntityItemProperties.cpp b/libraries/entities/src/EntityItemProperties.cpp index ebe4ac6014..4e7228e573 100644 --- a/libraries/entities/src/EntityItemProperties.cpp +++ b/libraries/entities/src/EntityItemProperties.cpp @@ -71,7 +71,8 @@ CONSTRUCT_PROPERTY(exponent, 0.0f), CONSTRUCT_PROPERTY(cutoff, ENTITY_ITEM_DEFAULT_CUTOFF), CONSTRUCT_PROPERTY(locked, ENTITY_ITEM_DEFAULT_LOCKED), CONSTRUCT_PROPERTY(textures, ""), -CONSTRUCT_PROPERTY(animationSettings, ""), +CONSTRUCT_PROPERTY(animationSettings, "{\"firstFrame\":0,\"fps\":30,\"frameIndex\":0,\"hold\":false," + "\"lastFrame\":100000,\"loop\":false,\"running\":false,\"startAutomatically\":false}"), CONSTRUCT_PROPERTY(userData, ENTITY_ITEM_DEFAULT_USER_DATA), CONSTRUCT_PROPERTY(simulationOwner, SimulationOwner()), CONSTRUCT_PROPERTY(text, TextEntityItem::DEFAULT_TEXT), diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp index 203ff2b072..25970d66d5 100644 --- a/libraries/octree/src/Octree.cpp +++ b/libraries/octree/src/Octree.cpp @@ -40,6 +40,7 @@ #include #include #include +#include #include "CoverageMap.h" #include "OctreeConstants.h" @@ -49,7 +50,7 @@ #include "OctreeLogging.h" -QVector PERSIST_EXTENSIONS = {"svo", "json"}; +QVector PERSIST_EXTENSIONS = {"svo", "json", "json.gz"}; float boundaryDistanceForRenderLevel(unsigned int renderLevel, float voxelSizeScale) { return voxelSizeScale / powf(2, renderLevel); @@ -1809,29 +1810,52 @@ int Octree::encodeTreeBitstreamRecursion(OctreeElement* element, } bool Octree::readFromFile(const char* fileName) { - bool fileOk = false; - QString qFileName = findMostRecentFileExtension(fileName, PERSIST_EXTENSIONS); - QFile file(qFileName); - fileOk = file.open(QIODevice::ReadOnly); - if(fileOk) { - QDataStream fileInputStream(&file); - QFileInfo fileInfo(qFileName); - unsigned long fileLength = fileInfo.size(); - - emit importSize(1.0f, 1.0f, 1.0f); - emit importProgress(0); - - qCDebug(octree) << "Loading file" << qFileName << "..."; - - fileOk = readFromStream(fileLength, fileInputStream); - - emit importProgress(100); - file.close(); + if (qFileName.endsWith(".json.gz")) { + return readJSONFromGzippedFile(qFileName); } - return fileOk; + QFile file(qFileName); + + if (!file.open(QIODevice::ReadOnly)) { + qCritical() << "unable to open for reading: " << fileName; + return false; + } + + QDataStream fileInputStream(&file); + QFileInfo fileInfo(qFileName); + unsigned long fileLength = fileInfo.size(); + + emit importSize(1.0f, 1.0f, 1.0f); + emit importProgress(0); + + qCDebug(octree) << "Loading file" << qFileName << "..."; + + bool success = readFromStream(fileLength, fileInputStream); + + emit importProgress(100); + file.close(); + + return success; +} + +bool Octree::readJSONFromGzippedFile(QString qFileName) { + QFile file(qFileName); + if (!file.open(QIODevice::ReadOnly)) { + qCritical() << "Cannot open gzipped json file for reading: " << qFileName; + return false; + } + QByteArray compressedJsonData = file.readAll(); + QByteArray jsonData; + + if (!gunzip(compressedJsonData, jsonData)) { + qCritical() << "json File not in gzip format: " << qFileName; + return false; + } + + QDataStream jsonStream(jsonData); + return readJSONFromStream(-1, jsonStream); } bool Octree::readFromURL(const QString& urlString) { @@ -1868,17 +1892,17 @@ bool Octree::readFromURL(const QString& urlString) { bool Octree::readFromStream(unsigned long streamLength, QDataStream& inputStream) { - // decide if this is SVO or JSON + // decide if this is binary SVO or JSON encoded SVO QIODevice *device = inputStream.device(); char firstChar; device->getChar(&firstChar); device->ungetChar(firstChar); if (firstChar == (char) PacketType::EntityData) { - qCDebug(octree) << "Reading from SVO Stream length:" << streamLength; + qCDebug(octree) << "Reading from binary SVO Stream length:" << streamLength; return readSVOFromStream(streamLength, inputStream); } else { - qCDebug(octree) << "Reading from JSON Stream length:" << streamLength; + qCDebug(octree) << "Reading from JSON SVO Stream length:" << streamLength; return readJSONFromStream(streamLength, inputStream); } } @@ -2013,12 +2037,28 @@ bool Octree::readSVOFromStream(unsigned long streamLength, QDataStream& inputStr return fileOk; } -bool Octree::readJSONFromStream(unsigned long streamLength, QDataStream& inputStream) { - char* rawData = new char[streamLength + 1]; // allocate enough room to null terminate - inputStream.readRawData(rawData, streamLength); - rawData[streamLength] = 0; // make sure we null terminate this string +const int READ_JSON_BUFFER_SIZE = 2048; - QJsonDocument asDocument = QJsonDocument::fromJson(rawData); +bool Octree::readJSONFromStream(unsigned long streamLength, QDataStream& inputStream) { + // QuaGzipFile doesn't appear to give a useful bytesAvailable() result, so just keep reading until + // we get an eof. Leave streamLength parameter for consistency. + + QByteArray jsonBuffer; + char* rawData = new char[READ_JSON_BUFFER_SIZE]; + while (true) { + int got = inputStream.readRawData(rawData, READ_JSON_BUFFER_SIZE - 1); + if (got < 0) { + qCritical() << "error while reading from json stream"; + delete[] rawData; + return false; + } + if (got == 0) { + break; + } + jsonBuffer += QByteArray(rawData, got); + } + + QJsonDocument asDocument = QJsonDocument::fromJson(jsonBuffer); QVariant asVariant = asDocument.toVariant(); QVariantMap asMap = asVariant.toMap(); readFromMap(asMap); @@ -2036,13 +2076,14 @@ void Octree::writeToFile(const char* fileName, OctreeElement* element, QString p writeToSVOFile(fileName, element); } else if (persistAsFileType == "json") { writeToJSONFile(cFileName, element); + } else if (persistAsFileType == "json.gz") { + writeToJSONFile(cFileName, element, true); } else { qCDebug(octree) << "unable to write octree to file of type" << persistAsFileType; } } -void Octree::writeToJSONFile(const char* fileName, OctreeElement* element) { - QFile persistFile(fileName); +void Octree::writeToJSONFile(const char* fileName, OctreeElement* element, bool doGzip) { QVariantMap entityDescription; qCDebug(octree, "Saving JSON SVO to file %s...", fileName); @@ -2061,10 +2102,27 @@ void Octree::writeToJSONFile(const char* fileName, OctreeElement* element) { // store the entity data bool entityDescriptionSuccess = writeToMap(entityDescription, top, true); + if (!entityDescriptionSuccess) { + qCritical("Failed to convert Entities to QVariantMap while saving to json."); + return; + } // convert the QVariantMap to JSON - if (entityDescriptionSuccess && persistFile.open(QIODevice::WriteOnly)) { - persistFile.write(QJsonDocument::fromVariant(entityDescription).toJson()); + QByteArray jsonData = QJsonDocument::fromVariant(entityDescription).toJson(); + QByteArray jsonDataForFile; + + if (doGzip) { + if (!gzip(jsonData, jsonDataForFile, -1)) { + qCritical("unable to gzip data while saving to json."); + return; + } + } else { + jsonDataForFile = jsonData; + } + + QFile persistFile(fileName); + if (persistFile.open(QIODevice::WriteOnly)) { + persistFile.write(jsonDataForFile); } else { qCritical("Could not write to JSON description of entities."); } diff --git a/libraries/octree/src/Octree.h b/libraries/octree/src/Octree.h index e00434be80..244359e394 100644 --- a/libraries/octree/src/Octree.h +++ b/libraries/octree/src/Octree.h @@ -330,7 +330,7 @@ public: // Octree exporters void writeToFile(const char* filename, OctreeElement* element = NULL, QString persistAsFileType = "svo"); - void writeToJSONFile(const char* filename, OctreeElement* element = NULL); + void writeToJSONFile(const char* filename, OctreeElement* element = NULL, bool doGzip = false); void writeToSVOFile(const char* filename, OctreeElement* element = NULL); virtual bool writeToMap(QVariantMap& entityDescription, OctreeElement* element, bool skipDefaultValues) = 0; @@ -340,6 +340,7 @@ public: bool readFromStream(unsigned long streamLength, QDataStream& inputStream); bool readSVOFromStream(unsigned long streamLength, QDataStream& inputStream); bool readJSONFromStream(unsigned long streamLength, QDataStream& inputStream); + bool readJSONFromGzippedFile(QString qFileName); virtual bool readFromMap(QVariantMap& entityDescription) = 0; unsigned long getOctreeElementsCount(); diff --git a/libraries/shared/CMakeLists.txt b/libraries/shared/CMakeLists.txt index 8deda7f4b1..298381689d 100644 --- a/libraries/shared/CMakeLists.txt +++ b/libraries/shared/CMakeLists.txt @@ -4,7 +4,9 @@ set(TARGET_NAME shared) # TODO: there isn't really a good reason to have Script linked here - let's get what is requiring it out (RegisteredMetaTypes.cpp) setup_hifi_library(Gui Network Script Widgets) +find_package(ZLIB REQUIRED) +target_link_libraries(${TARGET_NAME} ${ZLIB_LIBRARIES}) + add_dependency_external_projects(glm) find_package(GLM REQUIRED) target_include_directories(${TARGET_NAME} PUBLIC ${GLM_INCLUDE_DIRS}) - diff --git a/libraries/shared/src/Gzip.cpp b/libraries/shared/src/Gzip.cpp new file mode 100644 index 0000000000..ae4504cd24 --- /dev/null +++ b/libraries/shared/src/Gzip.cpp @@ -0,0 +1,152 @@ +// +// Gzip.cpp +// libraries/shared/src +// +// Created by Seth Alves on 2015-08-03. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include +#include "Gzip.h" + +const int GZIP_WINDOWS_BIT = 31; +const int GZIP_CHUNK_SIZE = 4096; + +bool gunzip(QByteArray source, QByteArray &destination) { + destination.clear(); + if (source.length() == 0) { + return true; + } + + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = Z_NULL; + + int status = inflateInit2(&strm, GZIP_WINDOWS_BIT); + + if (status != Z_OK) { + return false; + } + + char *sourceData = source.data(); + int sourceDataLength = source.length(); + + for (;;) { + int chunkSize = qMin(GZIP_CHUNK_SIZE, sourceDataLength); + if (chunkSize <= 0) { + break; + } + + strm.next_in = (unsigned char*)sourceData; + strm.avail_in = chunkSize; + sourceData += chunkSize; + sourceDataLength -= chunkSize; + + for (;;) { + char out[GZIP_CHUNK_SIZE]; + + strm.next_out = (unsigned char*)out; + strm.avail_out = GZIP_CHUNK_SIZE; + + status = inflate(&strm, Z_NO_FLUSH); + + switch (status) { + case Z_NEED_DICT: + status = Z_DATA_ERROR; + case Z_DATA_ERROR: + case Z_MEM_ERROR: + case Z_STREAM_ERROR: + inflateEnd(&strm); + return false; + } + + int available = (GZIP_CHUNK_SIZE - strm.avail_out); + if (available > 0) { + destination.append((char*)out, available); + } + + if (strm.avail_out != 0) { + break; + } + } + + if (status == Z_STREAM_END) { + break; + } + } + + inflateEnd(&strm); + return status == Z_STREAM_END; +} + +bool gzip(QByteArray source, QByteArray &destination, int compressionLevel) { + destination.clear(); + if (source.length() == 0) { + return true; + } + + int flushOrFinish = 0; + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.next_in = Z_NULL; + strm.avail_in = 0; + + int status = deflateInit2(&strm, + qMax(-1, qMin(9, compressionLevel)), + Z_DEFLATED, + GZIP_WINDOWS_BIT, + 8, + Z_DEFAULT_STRATEGY); + if (status != Z_OK) { + return false; + } + char *sourceData = source.data(); + int sourceDataLength = source.length(); + + for (;;) { + int chunkSize = qMin(GZIP_CHUNK_SIZE, sourceDataLength); + strm.next_in = (unsigned char*)sourceData; + strm.avail_in = chunkSize; + sourceData += chunkSize; + sourceDataLength -= chunkSize; + + if (sourceDataLength <= 0) { + flushOrFinish = Z_FINISH; + } else { + flushOrFinish = Z_NO_FLUSH; + } + + for (;;) { + char out[GZIP_CHUNK_SIZE]; + strm.next_out = (unsigned char*)out; + strm.avail_out = GZIP_CHUNK_SIZE; + status = deflate(&strm, flushOrFinish); + if (status == Z_STREAM_ERROR) { + deflateEnd(&strm); + return false; + } + int available = (GZIP_CHUNK_SIZE - strm.avail_out); + if (available > 0) { + destination.append((char*)out, available); + } + if (strm.avail_out != 0) { + break; + } + } + + if (flushOrFinish == Z_FINISH) { + break; + } + } + + deflateEnd(&strm); + return status == Z_STREAM_END; +} diff --git a/libraries/shared/src/Gzip.h b/libraries/shared/src/Gzip.h new file mode 100644 index 0000000000..5ebbf97589 --- /dev/null +++ b/libraries/shared/src/Gzip.h @@ -0,0 +1,20 @@ +// +// Gzip.h +// libraries/shared/src +// +// Created by Seth Alves on 2015-08-03. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef GZIP_H +#define GZIP_H + +#include + +bool gzip(QByteArray source, QByteArray &destination, int compressionLevel = -1); +bool gunzip(QByteArray source, QByteArray &destination); + +#endif