New format for captured GPU frames

interface/src/Application.cpp

@@ -4441,7 +4441,7 @@ void Application::keyPressEvent(QKeyEvent* event) {
                     static const QString GPU_FRAME_FOLDER = QProcessEnvironment::systemEnvironment().contains(HIFI_FRAMES_FOLDER_VAR)
                         ? QProcessEnvironment::systemEnvironment().value(HIFI_FRAMES_FOLDER_VAR)
                         : "hifiFrames";
-                    static QString GPU_FRAME_TEMPLATE = GPU_FRAME_FOLDER + "/{DATE}_{TIME}";
+                    static QString GPU_FRAME_TEMPLATE = GPU_FRAME_FOLDER + "/{DATE}_{TIME}.hfb";
                     QString fullPath = FileUtils::computeDocumentPath(FileUtils::replaceDateTimeTokens(GPU_FRAME_TEMPLATE));
                     if (FileUtils::canCreateFile(fullPath)) {
                         getActiveDisplayPlugin()->captureFrame(fullPath.toStdString());

libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.cpp

@@ -13,6 +13,7 @@
 #include <gl/Config.h>
 
 #include <QtCore/QCoreApplication>
+#include <QtCore/QBuffer>
 #include <QtCore/QThread>
 #include <QtCore/QTimer>
 #include <QtCore/QFileInfo>
@@ -480,7 +481,7 @@ void OpenGLDisplayPlugin::captureFrame(const std::string& filename) const {
         using namespace gpu;
         auto glBackend = const_cast<OpenGLDisplayPlugin&>(*this).getGLBackend();
         FramebufferPointer framebuffer{ Framebuffer::create("captureFramebuffer") };
-        TextureCapturer captureLambda = [&](const std::string& filename, const gpu::TexturePointer& texture, uint16 layer) {
+        TextureCapturer captureLambda = [&](std::vector<uint8_t>& outputBuffer, const gpu::TexturePointer& texture, uint16 layer) {
             QImage image;
             if (texture->getUsageType() == TextureUsageType::STRICT_RESOURCE) {
                 image = QImage{ 1, 1, QImage::Format_ARGB32 };
@@ -498,7 +499,11 @@ void OpenGLDisplayPlugin::captureFrame(const std::string& filename) const {
                 image = QImage{ rect.z, rect.w, QImage::Format_ARGB32 };
                 glBackend->downloadFramebuffer(framebuffer, rect, image);
             }
-            QImageWriter(filename.c_str()).write(image);
+            QBuffer buffer;
+            QImageWriter(&buffer, "png").write(image);
+            const auto& data = buffer.data();
+            outputBuffer.resize(data.size());
+            memcpy(outputBuffer.data(), data.constData(), data.size());
         };
 
         if (_currentFrame) {

libraries/gpu/src/gpu/FrameIO.cpp

@@ -0,0 +1,129 @@
+//
+//  Created by Bradley Austin Davis on 2019/10/03
+//  Copyright 2013-2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "FrameIO.h"
+#include <shared/Storage.h>
+
+using namespace gpu::hfb;
+
+static bool skip(const uint8_t*& ptr, size_t& remaining, uint32_t size) {
+    if (remaining < size) {
+        return false;
+    }
+    ptr += size;
+    remaining -= size;
+    return true;
+}
+
+template <typename T>
+static bool read(const uint8_t*& ptr, size_t& remaining, T& output) {
+    uint32_t readSize = (uint32_t)sizeof(T);
+    if (remaining < readSize) {
+        return false;
+    }
+    memcpy(&output, ptr, readSize);
+    return skip(ptr, remaining, readSize);
+}
+
+Descriptor Descriptor::parse(const uint8_t* const data, size_t size) {
+    const auto* ptr = data;
+    auto remaining = size;
+    Descriptor result;
+    if (!read(ptr, remaining, result.header)) {
+        return {};
+    }
+    if (!result.header.length == size) {
+        return {};
+    }
+
+    while (remaining != 0) {
+        result.chunks.emplace_back();
+        auto& chunk = result.chunks.back();
+        ChunkHeader& chunkHeader = chunk;
+        if (!read(ptr, remaining, chunkHeader)) {
+            return {};
+        }
+        chunk.offset = ptr - data;
+        if (!skip(ptr, remaining, chunk.length)) {
+            return {};
+        }
+    }
+    return result;
+}
+
+size_t Chunk::end() const {
+    size_t result = offset;
+    result += length;
+    return result;
+}
+
+
+bool Descriptor::getChunkString(std::string& output, size_t chunkIndex, const uint8_t* const data, size_t size) {
+    if (chunkIndex >= chunks.size()) {
+        return false;
+    }
+    const auto& chunk = chunks[chunkIndex];
+    if (chunk.end() > size) {
+        return false;
+    }
+    output = std::string{ (const char*)(data + chunk.offset), chunk.length };
+    return true;
+}
+
+bool Descriptor::getChunkBuffer(Buffer& output, size_t chunkIndex, const uint8_t* const data, size_t size) {
+    if (chunkIndex >= chunks.size()) {
+        return false;
+    }
+    const auto& chunk = chunks[chunkIndex];
+    if (chunk.end() > size) {
+        return false;
+    }
+    output.resize(chunk.length);
+    memcpy(output.data(), data + chunk.offset, chunk.length);
+    return true;
+}
+
+static void writeUint(uint8_t*& dest, uint32_t value) {
+    memcpy(dest, &value, sizeof(uint32_t));
+    dest += sizeof(uint32_t);
+}
+
+template <typename T>
+static void writeChunk(uint8_t*& dest, uint32_t chunkType, const T& chunkData) {
+    uint32_t chunkSize = static_cast<uint32_t>(chunkData.size());
+    writeUint(dest, chunkSize);
+    writeUint(dest, chunkType);
+    memcpy(dest, chunkData.data(), chunkSize);
+    dest += chunkSize;
+}
+
+void gpu::hfb::writeFrame(const std::string& filename,
+                          const std::string& json,
+                          const Buffer& binaryBuffer,
+                          const Buffers& pngBuffers) {
+    uint32_t strLen = (uint32_t)json.size();
+    uint32_t size = gpu::hfb::HEADER_SIZE + gpu::hfb::CHUNK_HEADER_SIZE + strLen;
+    size += gpu::hfb::CHUNK_HEADER_SIZE + (uint32_t)binaryBuffer.size();
+    for (const auto& pngBuffer : pngBuffers) {
+        size += gpu::hfb::CHUNK_HEADER_SIZE + (uint32_t)pngBuffer.size();
+    }
+
+    auto outputConst = storage::FileStorage::create(filename.c_str(), size, nullptr);
+    auto output = std::const_pointer_cast<storage::Storage>(outputConst);
+    auto ptr = output->mutableData();
+    writeUint(ptr, gpu::hfb::MAGIC);
+    writeUint(ptr, gpu::hfb::VERSION);
+    writeUint(ptr, size);
+    writeChunk(ptr, gpu::hfb::CHUNK_TYPE_JSON, json);
+    writeChunk(ptr, gpu::hfb::CHUNK_TYPE_BIN, binaryBuffer);
+    for (const auto& png : pngBuffers) {
+        writeChunk(ptr, gpu::hfb::CHUNK_TYPE_PNG, png);
+    }
+    auto writeSize = ptr - output->data();
+    assert(writeSize == size);
+}

libraries/gpu/src/gpu/FrameIO.h

@@ -16,13 +16,57 @@
 
 namespace gpu {
 
-using TextureCapturer = std::function<void(const std::string&, const TexturePointer&, uint16 layer)>;
+using TextureCapturer = std::function<void(std::vector<uint8_t>&, const TexturePointer&, uint16 layer)>;
-using TextureLoader = std::function<void(const std::string&, const TexturePointer&, uint16 layer)>;
+using TextureLoader = std::function<void(const std::vector<uint8_t>&, const TexturePointer&, uint16 layer)>;
 void writeFrame(const std::string& filename, const FramePointer& frame, const TextureCapturer& capturer = nullptr);
 FramePointer readFrame(const std::string& filename, uint32_t externalTexture, const TextureLoader& loader = nullptr);
 
-using IndexOptimizer = std::function<void(Primitive, uint32_t faceCount, uint32_t indexCount, uint32_t* indices )>;
+namespace hfb {
-void optimizeFrame(const std::string& filename, const IndexOptimizer& optimizer);
+
+constexpr char* EXTENSION{ ".hfb" };
+constexpr uint32_t HEADER_SIZE{ sizeof(uint32_t) * 3 };
+constexpr uint32_t CHUNK_HEADER_SIZE = sizeof(uint32_t) * 2;
+constexpr uint32_t MAGIC{ 0x49464948 };
+constexpr uint32_t VERSION{ 0x01 };
+constexpr uint32_t CHUNK_TYPE_JSON{ 0x4E4F534A };
+constexpr uint32_t CHUNK_TYPE_BIN{ 0x004E4942 };
+constexpr uint32_t CHUNK_TYPE_PNG{ 0x00474E50 };
+
+using Buffer = std::vector<uint8_t>;
+using Buffers = std::vector<Buffer>;
+
+struct Header {
+    uint32_t magic{ 0 };
+    uint32_t version{ 0 };
+    uint32_t length{ 0 };
+};
+
+struct ChunkHeader {
+    uint32_t length{ 0 };
+    uint32_t type{ 0 };
+};
+
+struct Chunk : public ChunkHeader {
+    uint32_t offset{ 0 };
+
+    size_t end() const;
+};
+
+using Chunks = std::vector<Chunk>;
+
+struct Descriptor {
+    Header header;
+    Chunks chunks;
+
+    operator bool() const { return header.magic == MAGIC; }
+    bool getChunkString(std::string& output, size_t chunk, const uint8_t* const data, size_t size);
+    bool getChunkBuffer(Buffer& output, size_t chunk, const uint8_t* const data, size_t size);
+    static Descriptor parse(const uint8_t* const data, size_t size);
+};
+
+void writeFrame(const std::string& filename, const std::string& json, const Buffer& binaryBuffer, const Buffers& pngBuffers);
+
+}  // namespace hfb
 
 }  // namespace gpu
 

libraries/gpu/src/gpu/FrameIOKeys.h

@@ -11,128 +11,130 @@
 
 namespace gpu { namespace keys {
 
-static const char* binary = "binary";
-static const char* L00 = "L00";
-static const char* L1m1 = "L1m1";
-static const char* L10 = "L10";
-static const char* L11 = "L11";
-static const char* L2m2 = "L2m2";
-static const char* L2m1 = "L2m1";
-static const char* L20 = "L20";
-static const char* L21 = "L21";
-static const char* L22 = "L22";
 
-static const char* eyeProjections = "eyeProjections";
+constexpr char* binary = "binary";
-static const char* eyeViews = "eyeViews";
+constexpr char* L00 = "L00";
-static const char* alphaToCoverageEnable = "alphaToCoverageEnable";
+constexpr char* L1m1 = "L1m1";
-static const char* antialisedLineEnable = "antialisedLineEnable";
+constexpr char* L10 = "L10";
-static const char* attributes = "attributes";
+constexpr char* L11 = "L11";
-static const char* batches = "batches";
+constexpr char* L2m2 = "L2m2";
-static const char* blendFunction = "blendFunction";
+constexpr char* L2m1 = "L2m1";
-static const char* borderColor = "borderColor";
+constexpr char* L20 = "L20";
-static const char* bufferMask = "bufferMask";
+constexpr char* L21 = "L21";
-static const char* buffers = "buffers";
+constexpr char* L22 = "L22";
-static const char* capturedTextures = "capturedTextures";
+
-static const char* channel = "channel";
+constexpr char* eyeProjections = "eyeProjections";
-static const char* colorAttachments = "colorAttachments";
+constexpr char* eyeViews = "eyeViews";
-static const char* colorWriteMask = "colorWriteMask";
+constexpr char* alphaToCoverageEnable = "alphaToCoverageEnable";
-static const char* commands = "commands";
+constexpr char* antialisedLineEnable = "antialisedLineEnable";
-static const char* comparisonFunction = "comparisonFunction";
+constexpr char* attributes = "attributes";
-static const char* cullMode = "cullMode";
+constexpr char* batches = "batches";
-static const char* data = "data";
+constexpr char* blendFunction = "blendFunction";
-static const char* depth = "depth";
+constexpr char* borderColor = "borderColor";
-static const char* depthBias = "depthBias";
+constexpr char* bufferMask = "bufferMask";
-static const char* depthBiasSlopeScale = "depthBiasSlopeScale";
+constexpr char* buffers = "buffers";
-static const char* depthClampEnable = "depthClampEnable";
+constexpr char* capturedTextures = "capturedTextures";
-static const char* depthStencilAttachment = "depthStencilAttachment";
+constexpr char* channel = "channel";
-static const char* depthTest = "depthTest";
+constexpr char* chunk = "chunk";
-static const char* drawCallInfos = "drawCallInfos";
+constexpr char* colorAttachments = "colorAttachments";
-static const char* drawcallUniform = "drawcallUniform";
+constexpr char* colorWriteMask = "colorWriteMask";
-static const char* drawcallUniformReset = "drawcallUniformReset";
+constexpr char* commands = "commands";
-static const char* element = "element";
+constexpr char* comparisonFunction = "comparisonFunction";
-static const char* fillMode = "fillMode";
+constexpr char* cullMode = "cullMode";
-static const char* filter = "filter";
+constexpr char* data = "data";
-static const char* formats = "formats";
+constexpr char* depth = "depth";
-static const char* frameIndex = "frameIndex";
+constexpr char* depthBias = "depthBias";
-static const char* framebuffer = "framebuffer";
+constexpr char* depthBiasSlopeScale = "depthBiasSlopeScale";
-static const char* framebuffers = "framebuffers";
+constexpr char* depthClampEnable = "depthClampEnable";
-static const char* frequency = "frequency";
+constexpr char* depthStencilAttachment = "depthStencilAttachment";
-static const char* frontFaceClockwise = "frontFaceClockwise";
+constexpr char* depthTest = "depthTest";
-static const char* height = "height";
+constexpr char* drawCallInfos = "drawCallInfos";
-static const char* id = "id";
+constexpr char* drawcallUniform = "drawcallUniform";
-static const char* ktxFile = "ktxFile";
+constexpr char* drawcallUniformReset = "drawcallUniformReset";
-static const char* layers = "layers";
+constexpr char* element = "element";
-static const char* maxAnisotropy = "maxAnisotropy";
+constexpr char* fillMode = "fillMode";
-static const char* maxMip = "maxMip";
+constexpr char* filter = "filter";
-static const char* minMip = "minMip";
+constexpr char* formats = "formats";
-static const char* mipOffset = "mipOffset";
+constexpr char* frameIndex = "frameIndex";
-static const char* mips = "mips";
+constexpr char* framebuffer = "framebuffer";
-static const char* multisampleEnable = "multisampleEnable";
+constexpr char* framebuffers = "framebuffers";
-static const char* name = "name";
+constexpr char* frequency = "frequency";
-static const char* namedData = "namedData";
+constexpr char* frontFaceClockwise = "frontFaceClockwise";
-static const char* names = "names";
+constexpr char* height = "height";
-static const char* objects = "objects";
+constexpr char* id = "id";
-static const char* offset = "offset";
+constexpr char* ktxFile = "ktxFile";
-static const char* pipelines = "pipelines";
+constexpr char* layers = "layers";
-static const char* pose = "pose";
+constexpr char* maxAnisotropy = "maxAnisotropy";
-static const char* profileRanges = "profileRanges";
+constexpr char* maxMip = "maxMip";
-static const char* program = "program";
+constexpr char* minMip = "minMip";
-static const char* programs = "programs";
+constexpr char* mipOffset = "mipOffset";
-static const char* projectionJitter = "projectionJitter";
+constexpr char* mips = "mips";
-static const char* queries = "queries";
+constexpr char* multisampleEnable = "multisampleEnable";
-static const char* sampleCount = "sampleCount";
+constexpr char* name = "name";
-static const char* sampleMask = "sampleMask";
+constexpr char* namedData = "namedData";
-static const char* sampler = "sampler";
+constexpr char* names = "names";
-static const char* samples = "samples";
+constexpr char* objects = "objects";
-static const char* scissorEnable = "scissorEnable";
+constexpr char* offset = "offset";
-static const char* shaders = "shaders";
+constexpr char* pipelines = "pipelines";
-static const char* size = "size";
+constexpr char* pose = "pose";
-static const char* skybox = "skybox";
+constexpr char* profileRanges = "profileRanges";
-static const char* slot = "slot";
+constexpr char* program = "program";
-static const char* source = "source";
+constexpr char* programs = "programs";
-static const char* state = "state";
+constexpr char* projectionJitter = "projectionJitter";
-static const char* stencilActivation = "stencilActivation";
+constexpr char* queries = "queries";
-static const char* stencilTestBack = "stencilTestBack";
+constexpr char* sampleCount = "sampleCount";
-static const char* stencilTestFront = "stencilTestFront";
+constexpr char* sampleMask = "sampleMask";
-static const char* stereo = "stereo";
+constexpr char* sampler = "sampler";
-static const char* subresource = "subresource";
+constexpr char* samples = "samples";
-static const char* swapchains = "swapchains";
+constexpr char* scissorEnable = "scissorEnable";
-static const char* texelFormat = "texelFormat";
+constexpr char* shaders = "shaders";
-static const char* texture = "texture";
+constexpr char* size = "size";
-static const char* textureTables = "textureTables";
+constexpr char* skybox = "skybox";
-static const char* textures = "textures";
+constexpr char* slot = "slot";
-static const char* transforms = "transforms";
+constexpr char* source = "source";
-static const char* type = "type";
+constexpr char* state = "state";
-static const char* usageType = "usageType";
+constexpr char* stencilActivation = "stencilActivation";
-static const char* view = "view";
+constexpr char* stencilTestBack = "stencilTestBack";
-static const char* width = "width";
+constexpr char* stencilTestFront = "stencilTestFront";
-static const char* wrapModeU = "wrapModeU";
+constexpr char* stereo = "stereo";
-static const char* wrapModeV = "wrapModeV";
+constexpr char* subresource = "subresource";
-static const char* wrapModeW = "wrapModeW";
+constexpr char* swapchains = "swapchains";
+constexpr char* texelFormat = "texelFormat";
+constexpr char* texture = "texture";
+constexpr char* textureTables = "textureTables";
+constexpr char* textures = "textures";
+constexpr char* transforms = "transforms";
+constexpr char* type = "type";
+constexpr char* usageType = "usageType";
+constexpr char* view = "view";
+constexpr char* width = "width";
+constexpr char* wrapModeU = "wrapModeU";
+constexpr char* wrapModeV = "wrapModeV";
+constexpr char* wrapModeW = "wrapModeW";
 
 
-static const char* backWriteMask = "backWriteMask";
+constexpr char* backWriteMask = "backWriteMask";
-static const char* frontWriteMask = "frontWriteMask";
+constexpr char* frontWriteMask = "frontWriteMask";
-static const char* reference = "reference";
+constexpr char* reference = "reference";
-static const char* readMask = "readMask";
+constexpr char* readMask = "readMask";
-static const char* failOp = "failOp";
+constexpr char* failOp = "failOp";
-static const char* depthFailOp = "depthFailOp";
+constexpr char* depthFailOp = "depthFailOp";
-static const char* passOp = "passOp";
+constexpr char* passOp = "passOp";
-static const char* enabled = "enabled";
+constexpr char* enabled = "enabled";
-static const char* blend = "blend";
+constexpr char* blend = "blend";
-static const char* flags = "flags";
+constexpr char* flags = "flags";
-static const char* writeMask = "writeMask";
+constexpr char* writeMask = "writeMask";
-static const char* function = "function";
+constexpr char* function = "function";
-static const char* sourceColor = "sourceColor";
+constexpr char* sourceColor = "sourceColor";
-static const char* sourceAlpha = "sourceAlpha";
+constexpr char* sourceAlpha = "sourceAlpha";
-static const char* destColor = "destColor";
+constexpr char* destColor = "destColor";
-static const char* destAlpha = "destAlpha";
+constexpr char* destAlpha = "destAlpha";
-static const char* opColor = "opColor";
+constexpr char* opColor = "opColor";
-static const char* opAlpha = "opAlpha";
+constexpr char* opAlpha = "opAlpha";
-static const char* enable = "enable";
+constexpr char* enable = "enable";
-static const char* contextDisable = "contextDisable";
+constexpr char* contextDisable = "contextDisable";
 
-static const char* COMMAND_NAMES[] = {
+constexpr char* COMMAND_NAMES[] = {
     "draw",
     "drawIndexed",
     "drawInstanced",

libraries/gpu/src/gpu/FrameReader.cpp

@@ -22,17 +22,11 @@
 
 namespace gpu {
 using json = nlohmann::json;
+using StoragePointer = storage::StoragePointer;
+using FileStorage = storage::FileStorage;
 
 class Deserializer {
 public:
-    static std::string getBaseName(const std::string& filename) {
-        static const std::string ext{ ".json" };
-        if (std::string::npos != filename.rfind(ext)) {
-            return filename.substr(0, filename.size() - ext.size());
-        }
-        return filename;
-    }
-
     static std::string getBaseDir(const std::string& filename) {
         std::string result;
         if (0 == filename.find("assets:")) {
@@ -47,14 +41,17 @@ public:
         return result;
     }
 
-    Deserializer(const std::string& filename, uint32_t externalTexture, const TextureLoader& loader) :
+    Deserializer(const std::string& filename_, uint32_t externalTexture = 0, const TextureLoader& loader = {}) :
-        basename(getBaseName(filename)), basedir(getBaseDir(filename)), externalTexture(externalTexture), textureLoader(loader) {
+        filename(filename_), basedir(getBaseDir(filename_)), mappedFile(std::make_shared<FileStorage>(filename.c_str())),
+        externalTexture(externalTexture), textureLoader(loader) {
+        descriptor = hfb::Descriptor::parse(mappedFile->data(), (uint32_t)mappedFile->size());
     }
 
-    const std::string basename;
+    const std::string filename;
     const std::string basedir;
-    std::string binaryFile;
+    const StoragePointer mappedFile;
     const uint32_t externalTexture;
+    hfb::Descriptor descriptor;
     TextureLoader textureLoader;
     std::vector<ShaderPointer> shaders;
     std::vector<ShaderPointer> programs;
@@ -69,10 +66,24 @@ public:
     std::vector<QueryPointer> queries;
     json frameNode;
     FramePointer readFrame();
-    void optimizeFrame(const IndexOptimizer& optimizer);
 
     FramePointer deserializeFrame();
 
+    std::string getStringChunk(size_t chunkIndex) {
+        std::string result;
+        if (!descriptor.getChunkString(result, chunkIndex, mappedFile->data(), mappedFile->size())) {
+            return {};
+        }
+        return result;
+    }
+
+    hfb::Buffer getBufferChunk(size_t chunkIndex) {
+        hfb::Buffer result;
+        if (!descriptor.getChunkBuffer(result, chunkIndex, mappedFile->data(), mappedFile->size())) {
+            return {};
+        }
+        return result;
+    }
 
     void readBuffers(const json& node);
 
@@ -148,12 +159,11 @@ public:
     }
 
     template <typename T, typename TT = T>
-    static bool readBatchCacheTransformed(typename Batch::Cache<T>::Vector& dest,
+    static bool readBatchCacheTransformed(
-                                          const json& node,
+        typename Batch::Cache<T>::Vector& dest,
-                                          const std::string& name,
+        const json& node,
-                                          std::function<TT(const json&)> f = [](const json& node) -> TT {
+        const std::string& name,
-                                              return node.get<TT>();
+        std::function<TT(const json&)> f = [](const json& node) -> TT { return node.get<TT>(); }) {
-                                          }) {
         if (node.count(name)) {
             const auto& arrayNode = node[name];
             for (const auto& entry : arrayNode) {
@@ -234,18 +244,14 @@ FramePointer readFrame(const std::string& filename, uint32_t externalTexture, co
     return Deserializer(filename, externalTexture, loader).readFrame();
 }
 
-void optimizeFrame(const std::string& filename, const IndexOptimizer& optimizer) {
-    return Deserializer(filename, 0, {}).optimizeFrame(optimizer);
-}
-
 }  // namespace gpu
 
 using namespace gpu;
 
 void Deserializer::readBuffers(const json& buffersNode) {
-    storage::FileStorage mappedFile(binaryFile.c_str());
+    const auto& binaryChunk = descriptor.chunks[1];
-    const auto mappedSize = mappedFile.size();
+    const auto* mapped = mappedFile->data() + binaryChunk.offset;
-    const auto* mapped = mappedFile.data();
+    const auto mappedSize = binaryChunk.length;
     size_t bufferCount = buffersNode.size();
     buffers.reserve(buffersNode.size());
     size_t offset = 0;
@@ -311,6 +317,8 @@ Sampler Deserializer::readSampler(const json& node) {
     return result;
 }
 
+constexpr uint32_t INVALID_CHUNK_INDEX{ (uint32_t)-1 };
+
 TexturePointer Deserializer::readTexture(const json& node, uint32_t external) {
     if (node.is_null()) {
         return nullptr;
@@ -319,8 +327,17 @@ TexturePointer Deserializer::readTexture(const json& node, uint32_t external) {
     std::string source;
     readOptional(source, node, keys::source);
 
+    uint32_t chunkIndex = INVALID_CHUNK_INDEX;
+    readOptional(chunkIndex, node, keys::chunk);
+
     std::string ktxFile;
     readOptional(ktxFile, node, keys::ktxFile);
+    if (!ktxFile.empty()) {
+        if (!QFileInfo(ktxFile.c_str()).exists()) {
+            qDebug() << "Warning" << ktxFile.c_str() << " not found, ignoring";
+            ktxFile = {};
+        }
+    }
     Element ktxTexelFormat, ktxMipFormat;
     if (!ktxFile.empty()) {
         // If we get a texture that starts with ":" we need to re-route it to the resources directory
@@ -368,8 +385,15 @@ TexturePointer Deserializer::readTexture(const json& node, uint32_t external) {
         if (QFileInfo(ktxFile.c_str()).isRelative()) {
             ktxFile = basedir + "/" + ktxFile;
         }
+        texture.setSource(ktxFile);
         texture.setKtxBacking(ktxFile);
+    } else if (chunkIndex != INVALID_CHUNK_INDEX) {
+        if (textureLoader) {
+            texture.setSource("Chunk " + std::to_string(chunkIndex));
+            textureLoader(getBufferChunk(chunkIndex), result, 0);
+        }
     }
+
     return result;
 }
 
@@ -405,11 +429,11 @@ ShaderPointer Deserializer::readShader(const json& node) {
     // FIXME support procedural shaders
     Shader::Type type = node[keys::type];
     std::string name = node[keys::name];
-    // Using the serialized ID is bad, because it's generated at 
+    // Using the serialized ID is bad, because it's generated at
-    // cmake time, and can change across platforms or when 
+    // cmake time, and can change across platforms or when
     // shaders are added or removed
     // uint32_t id = node[keys::id];
-    
+
     uint32_t id = shadersIdsByName[name];
     ShaderPointer result;
     switch (type) {
@@ -555,11 +579,15 @@ StatePointer readState(const json& node) {
 
     State::Data data;
     Deserializer::readOptionalTransformed<State::Flags>(data.flags, node, keys::flags, &readStateFlags);
-    Deserializer::readOptionalTransformed<State::BlendFunction>(data.blendFunction, node, keys::blendFunction, &readBlendFunction);
+    Deserializer::readOptionalTransformed<State::BlendFunction>(data.blendFunction, node, keys::blendFunction,
+                                                                &readBlendFunction);
     Deserializer::readOptionalTransformed<State::DepthTest>(data.depthTest, node, keys::depthTest, &readDepthTest);
-    Deserializer::readOptionalTransformed<State::StencilActivation>(data.stencilActivation, node, keys::stencilActivation, &readStencilActivation);
+    Deserializer::readOptionalTransformed<State::StencilActivation>(data.stencilActivation, node, keys::stencilActivation,
-    Deserializer::readOptionalTransformed<State::StencilTest>(data.stencilTestFront, node, keys::stencilTestFront, &readStencilTest);
+                                                                    &readStencilActivation);
-    Deserializer::readOptionalTransformed<State::StencilTest>(data.stencilTestBack, node, keys::stencilTestBack, &readStencilTest);
+    Deserializer::readOptionalTransformed<State::StencilTest>(data.stencilTestFront, node, keys::stencilTestFront,
+                                                              &readStencilTest);
+    Deserializer::readOptionalTransformed<State::StencilTest>(data.stencilTestBack, node, keys::stencilTestBack,
+                                                              &readStencilTest);
     Deserializer::readOptional(data.colorWriteMask, node, keys::colorWriteMask);
     Deserializer::readOptional(data.cullMode, node, keys::cullMode);
     Deserializer::readOptional(data.depthBias, node, keys::depthBias);
@@ -799,25 +827,15 @@ StereoState readStereoState(const json& node) {
 
 
 FramePointer Deserializer::deserializeFrame() {
-    {
+    if (!descriptor.operator bool()) {
-        std::string filename{ basename + ".json" };
+        return {};
-        storage::FileStorage mappedFile(filename.c_str());
-        frameNode = json::parse(std::string((const char*)mappedFile.data(), mappedFile.size()));
     }
 
+    frameNode = json::parse(getStringChunk(0));
+
     FramePointer result = std::make_shared<Frame>();
     auto& frame = *result;
 
-    if (frameNode[keys::binary].is_string()) {
-        binaryFile = frameNode[keys::binary];
-        if (QFileInfo(binaryFile.c_str()).isRelative()) {
-            binaryFile = basedir + "/" + binaryFile;
-        }
-    } else {
-        binaryFile = basename + ".bin";
-    }
-
-
     if (frameNode.count(keys::buffers)) {
         readBuffers(frameNode[keys::buffers]);
     }
@@ -830,19 +848,7 @@ FramePointer Deserializer::deserializeFrame() {
 
     formats = readArray<Stream::FormatPointer>(frameNode, keys::formats, [](const json& node) { return readFormat(node); });
 
-    auto textureReader = [this](const json& node) { return readTexture(node, externalTexture); };
+    textures = readArray<TexturePointer>(frameNode, keys::textures, [this](const json& node) { return readTexture(node, externalTexture); });
-    textures = readArray<TexturePointer>(frameNode, keys::textures, textureReader);
-    if (textureLoader) {
-        std::vector<uint32_t> capturedTextures = readNumericVector<uint32_t>(frameNode[keys::capturedTextures]);
-        for (const auto& index : capturedTextures) {
-            const auto& texturePointer = textures[index];
-            uint16 layers = std::max<uint16>(texturePointer->getNumSlices(), 1);
-            for (uint16 layer = 0; layer < layers; ++layer) {
-                std::string filename = basename + "." + std::to_string(index) + "." + std::to_string(layer) + ".png";
-                textureLoader(filename, texturePointer, layer);
-            }
-        }
-    }
 
     // Must come after textures
     auto textureTableReader = [this](const json& node) { return readTextureTable(node); };
@@ -868,87 +874,22 @@ FramePointer Deserializer::deserializeFrame() {
         }
     }
 
+    for (uint32_t i = 0; i < textures.size(); ++i) {
+        const auto& texturePtr = textures[i];
+        if (!texturePtr) {
+            continue;
+        }
+        const auto& texture = *texturePtr;
+        if (texture.getUsageType() == gpu::TextureUsageType::RESOURCE && texture.source().empty()) {
+            qDebug() << "Empty source ";
+        }
+    }
+
     return result;
 }
 
-
-
 FramePointer Deserializer::readFrame() {
     auto result = deserializeFrame();
     result->finish();
     return result;
 }
-
-void Deserializer::optimizeFrame(const IndexOptimizer& optimizer) {
-    auto result = deserializeFrame();
-    auto& frame = *result;
-
-
-        // optimize the index buffers?
-    struct CurrentIndexBuffer {
-        Offset offset{ 0 };
-        BufferPointer buffer;
-        Type type{ gpu::Type::INT32 };
-        Primitive primitve{ Primitive::TRIANGLES };
-        uint32_t numIndices{ 0 };
-        uint32_t startIndex{ 0 };
-    };
-
-    std::vector<CurrentIndexBuffer> captured;
-    for (auto& batch : frame.batches) {
-
-        CurrentIndexBuffer currentIndexBuffer;
-        batch->forEachCommand([&](Batch::Command cmd, const Batch::Param* params){
-            switch(cmd) {
-                case Batch::Command::COMMAND_setIndexBuffer:
-                    currentIndexBuffer.offset = params[0]._size;
-                    currentIndexBuffer.buffer = batch->_buffers.get(params[1]._int);
-                    currentIndexBuffer.type = (Type)params[2]._int;
-                    break;
-
-                case Batch::Command::COMMAND_drawIndexed:
-                    currentIndexBuffer.startIndex = params[0]._int;
-                    currentIndexBuffer.numIndices = params[1]._int;
-                    currentIndexBuffer.primitve = (Primitive)params[2]._int;
-                    captured.emplace_back(currentIndexBuffer);
-                    break;
-
-                case Batch::Command::COMMAND_drawIndexedInstanced:
-                    currentIndexBuffer.startIndex = params[1]._int;
-                    currentIndexBuffer.numIndices = params[2]._int;
-                    currentIndexBuffer.primitve = (Primitive)params[3]._int;
-                    captured.emplace_back(currentIndexBuffer);
-                    break;
-
-                default: 
-                    break;
-            }
-        });
-    }
-
-
-    std::string optimizedBinaryFile = basename + "_optimized.bin";
-    QFile(binaryFile.c_str()).copy(optimizedBinaryFile.c_str());
-    {
-        storage::FileStorage mappedFile(optimizedBinaryFile.c_str());
-        std::set<BufferPointer> uniqueBuffers;
-        for (const auto& capturedIndexData : captured) {
-            if (uniqueBuffers.count(capturedIndexData.buffer)) {
-                continue;
-            }
-            uniqueBuffers.insert(capturedIndexData.buffer);
-            auto bufferOffset = bufferOffsets[capturedIndexData.buffer];
-            auto& buffer = *capturedIndexData.buffer;
-            const auto& count = capturedIndexData.numIndices;
-            auto indices = (uint32_t*)buffer.editData();
-            optimizer(capturedIndexData.primitve, count / 3, count, indices);
-            memcpy(mappedFile.mutableData() + bufferOffset, indices, sizeof(uint32_t) * count);
-        }
-    }
-    frameNode[keys::binary] = optimizedBinaryFile;
-    {
-        std::string frameJson = frameNode.dump();
-        std::string filename = basename + "_optimized.json";
-        storage::FileStorage::create(filename.c_str(), frameJson.size(), (const uint8_t*)frameJson.data());
-    }
-}

libraries/gpu/src/gpu/FrameWriter.cpp

@@ -20,7 +20,7 @@ using json = nlohmann::json;
 
 class Serializer {
 public:
-    const std::string basename;
+    const std::string filename;
     const TextureCapturer textureCapturer;
     std::unordered_map<ShaderPointer, uint32_t> shaderMap;
     std::unordered_map<ShaderPointer, uint32_t> programMap;
@@ -32,8 +32,11 @@ public:
     std::unordered_map<FramebufferPointer, uint32_t> framebufferMap;
     std::unordered_map<SwapChainPointer, uint32_t> swapchainMap;
     std::unordered_map<QueryPointer, uint32_t> queryMap;
+    std::unordered_set<TexturePointer> captureTextures;
+    hfb::Buffer binaryBuffer;
+    hfb::Buffers pngBuffers;
 
-    Serializer(const std::string& basename, const TextureCapturer& capturer) : basename(basename), textureCapturer(capturer) {}
+    Serializer(const std::string& basename, const TextureCapturer& capturer) : filename(basename + hfb::EXTENSION), textureCapturer(capturer) {}
 
     template <typename T>
     static uint32_t getGlobalIndex(const T& value, std::unordered_map<T, uint32_t>& map) {
@@ -129,7 +132,7 @@ public:
     json writeProgram(const ShaderPointer& program);
     json writeNamedBatchData(const Batch::NamedBatchData& namedData);
 
-    json writeCapturableTextures(const Frame& frame);
+    void findCapturableTextures(const Frame& frame);
     void writeBinaryBlob();
     static std::string toBase64(const std::vector<uint8_t>& v);
     static json writeIrradiance(const SHPointer& irradiance);
@@ -146,7 +149,7 @@ public:
     static json writeTransform(const Transform& t) { return writeMat4(t.getMatrix()); }
     static json writeCommand(size_t index, const Batch& batch);
     static json writeSampler(const Sampler& sampler);
-    static json writeTexture(const TexturePointer& texture);
+    json writeTexture(const TexturePointer& texture);
     static json writeFormat(const Stream::FormatPointer& format);
     static json writeQuery(const QueryPointer& query);
     static json writeShader(const ShaderPointer& shader);
@@ -390,8 +393,12 @@ json Serializer::writeTexture(const TexturePointer& texturePointer) {
         const auto* ktxStorage = dynamic_cast<const Texture::KtxStorage*>(storage);
         if (ktxStorage) {
             result[keys::ktxFile] = ktxStorage->_filename;
-        } else {
+        } else if (textureCapturer && captureTextures.count(texturePointer) != 0) {
-            // TODO serialize the backing storage
+            auto layers = std::max<uint16>(texture.getNumSlices(), 1);
+            result[keys::chunk] = 2 + pngBuffers.size();
+            pngBuffers.push_back({});
+            hfb::Buffer& pngBuffer = pngBuffers.back();
+            textureCapturer(pngBuffer, texturePointer, 0);
         }
     }
     return result;
@@ -673,14 +680,8 @@ json Serializer::writeQuery(const QueryPointer& queryPointer) {
     return result;
 }
 
-json Serializer::writeCapturableTextures(const Frame& frame) {
+void Serializer::findCapturableTextures(const Frame& frame) {
-    if (!textureCapturer) {
-        return json::array();
-    }
-
     std::unordered_set<TexturePointer> writtenRenderbuffers;
-    std::unordered_set<TexturePointer> captureTextures;
-
     auto maybeCaptureTexture = [&](const TexturePointer& texture) {
         // Not a valid texture
         if (!texture) {
@@ -755,20 +756,6 @@ json Serializer::writeCapturableTextures(const Frame& frame) {
             }
         }
     }
-
-    json result = json::array();
-    for (const auto& texture : captureTextures) {
-        if (textureCapturer) {
-            auto index = textureMap[texture];
-            auto layers = std::max<uint16>(texture->getNumSlices(), 1);
-            for (uint16 layer = 0; layer < layers; ++layer) {
-                std::string textureFilename = basename + "." + std::to_string(index) + "." + std::to_string(layer) + ".png";
-                textureCapturer(textureFilename, texture, layer);
-            }
-            result.push_back(index);
-        }
-    }
-    return result;
 }
 
 void Serializer::writeFrame(const Frame& frame) {
@@ -780,7 +767,7 @@ void Serializer::writeFrame(const Frame& frame) {
     }
 
     frameNode[keys::stereo] = writeStereoState(frame.stereoState);
-    frameNode[keys::capturedTextures] = writeCapturableTextures(frame);
+    findCapturableTextures(frame);
     frameNode[keys::frameIndex] = frame.frameIndex;
     frameNode[keys::view] = writeMat4(frame.view);
     frameNode[keys::pose] = writeMat4(frame.pose);
@@ -797,35 +784,21 @@ void Serializer::writeFrame(const Frame& frame) {
 
     // Serialize textures and buffers last, since the maps they use can be populated by some of the above code
     // Serialize textures
-    serializeMap(frameNode, keys::textures, textureMap, writeTexture);
+    serializeMap(frameNode, keys::textures, textureMap, std::bind(&Serializer::writeTexture, this, _1));
     // Serialize buffers
     serializeMap(frameNode, keys::buffers, bufferMap, writeBuffer);
 
-    {
-        std::string frameJson = frameNode.dump();
-        std::string filename = basename + ".json";
-        storage::FileStorage::create(filename.c_str(), frameJson.size(), (const uint8_t*)frameJson.data());
-    }
-
     writeBinaryBlob();
-    frameNode[keys::binary] = basename + ".bin";
+
+    hfb::writeFrame(filename, frameNode.dump(), binaryBuffer, pngBuffers);
 }
 
 void Serializer::writeBinaryBlob() {
     const auto buffers = mapToVector(bufferMap);
     auto accumulator = [](size_t total, const BufferPointer& buffer) { return total + (buffer ? buffer->getSize() : 0); };
     size_t totalSize = std::accumulate(buffers.begin(), buffers.end(), (size_t)0, accumulator);
-
+    binaryBuffer.resize(totalSize);
-    const auto blobFilename = basename + ".bin";
+    auto mapped = binaryBuffer.data();
-    QFile file(blobFilename.c_str());
-    if (!file.open(QFile::ReadWrite | QIODevice::Truncate)) {
-        throw std::runtime_error("Unable to open file for writing");
-    }
-    if (!file.resize(totalSize)) {
-        throw std::runtime_error("Unable to resize file");
-    }
-
-    auto mapped = file.map(0, totalSize);
     size_t offset = 0;
 
     for (const auto& bufferPointer : buffers) {
@@ -838,7 +811,4 @@ void Serializer::writeBinaryBlob() {
         memcpy(mapped + offset, bufferData, bufferSize);
         offset += bufferSize;
     }
-    if (!file.unmap(mapped)) {
-        throw std::runtime_error("Unable to unmap file");
-    }
 }

tools/CMakeLists.txt

@@ -19,34 +19,24 @@ function(check_test name)
 endfunction()
 
 if (BUILD_TOOLS)
+    set(ALL_TOOLS 
+        udt-test
+        vhacd-util
+        frame-optimizer
+        gpu-frame-player
+        gpu-frame-converter
+        ice-client
+        ktx-tool
+        ac-client
+        skeleton-dump
+        atp-client
+        oven
+    )
+
     # Allow different tools for stable builds
     if (STABLE_BUILD)
-        set(ALL_TOOLS
-            udt-test
-            vhacd-util
-            frame-optimizer
-            gpu-frame-player
-            ice-client
-            ktx-tool
-            ac-client
-            skeleton-dump
-            atp-client
-            oven
-        )
     else()
-        set(ALL_TOOLS
+        list(APPEND ALL_TOOLS nitpick)
-            udt-test
-            vhacd-util
-            frame-optimizer
-            gpu-frame-player
-            ice-client
-            ktx-tool
-            ac-client
-            skeleton-dump
-            atp-client
-            oven
-            nitpick
-        )
     endif()
 
     foreach(TOOL ${ALL_TOOLS})

tools/gpu-frame-converter/CMakeLists.txt

@@ -0,0 +1,9 @@
+
+set(TARGET_NAME gpu-frame-converter)
+setup_memory_debugger()
+setup_hifi_project()
+set_property(TARGET ${TARGET_NAME} PROPERTY CXX_STANDARD 17)
+# link in the shared libraries
+link_hifi_libraries( shared gpu shaders )
+
+package_libraries_for_deployment()

tools/gpu-frame-converter/src/main.cpp

@@ -0,0 +1,104 @@
+//
+//  Created by Bradley Austin Davis on 2019/10/03
+//  Copyright 2013-2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <gpu/FrameIO.h>
+#include <gpu/Batch.h>
+#include <gpu/FrameIOKeys.h>
+#include <shared/Storage.h>
+#include <nlohmann/json.hpp>
+#include <shared/FileUtils.h>
+
+#include <iostream>
+#include <filesystem>
+#include <fstream>
+
+namespace fs = std::filesystem;
+using Json = nlohmann::json;
+using Paths = std::vector<std::filesystem::path>;
+
+const char* DEFAULT_SOURCE_PATH{ "D:/Frames" };
+static const std::string OLD_FRAME_EXTENSION{ ".json" };
+static const std::string OLD_BINARY_EXTENSION{ ".bin" };
+static const std::string NEW_EXTENSION{ gpu::hfb::EXTENSION };
+
+inline std::string readFileToString(const fs::path& path) {
+    std::ifstream file(path);
+    return std::string((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
+}
+
+inline gpu::hfb::Buffer readFile(const fs::path& path) {
+    std::ifstream file(path, std::ios::binary | std::ios::ate);
+    size_t size = file.tellg();
+    if (!size) {
+        return {};
+    }
+    file.seekg(0, std::ios::beg);
+
+    gpu::hfb::Buffer result;
+    result.resize(size);
+    if (!file.read((char*)result.data(), size)) {
+        throw std::runtime_error("Failed to read file");
+    }
+    return result;
+}
+
+Paths getFrames(const std::string& sourcePath) {
+    Paths result;
+    for (auto& p : fs::directory_iterator(sourcePath)) {
+        if (!p.is_regular_file()) {
+            continue;
+        }
+        const auto& path = p.path();
+        if (path.string().find(".hfb.json") != std::string::npos) {
+            continue;
+        }
+        if (path.extension().string() == OLD_FRAME_EXTENSION) {
+            result.push_back(path);
+        }
+    }
+    return result;
+}
+
+void convertFrame(const fs::path& path) {
+    auto name = path.filename().string();
+    name = name.substr(0, name.length() - OLD_FRAME_EXTENSION.length());
+
+    auto frameNode = Json::parse(readFileToString(path));
+    auto capturedTexturesNode = frameNode[gpu::keys::capturedTextures];
+
+    gpu::hfb::Buffer binary = readFile(path.parent_path() / (name + OLD_BINARY_EXTENSION));
+    gpu::hfb::Buffers pngs;
+    for (const auto& capturedTextureIndexNode : capturedTexturesNode) {
+        int index = capturedTextureIndexNode;
+        auto imageFile = path.parent_path() / (name + "." + std::to_string(index) + ".0.png");
+        frameNode[gpu::keys::textures][index][gpu::keys::chunk] = 2 + pngs.size();
+        pngs.push_back(readFile(imageFile));
+    }
+    frameNode.erase(gpu::keys::capturedTextures);
+    auto outputPath = path.parent_path() / (name + NEW_EXTENSION);
+    {
+        auto jsonOutputPath = path.parent_path() / (name + ".hfb.json");
+        std::ofstream of(jsonOutputPath);
+        auto str = frameNode.dump(2);
+        of.write(str.data(), str.size());
+    }
+    gpu::hfb::writeFrame(outputPath.string(), frameNode.dump(), binary, pngs);
+    {
+        auto frameBuffer = readFile(outputPath.string());
+        auto descriptor = gpu::hfb::Descriptor::parse(frameBuffer.data(), frameBuffer.size());
+        std::cout << descriptor.header.magic << std::endl;
+    }
+}
+
+int main(int argc, char** argv) {
+    for (const auto& framePath : getFrames(DEFAULT_SOURCE_PATH)) {
+        std::cout << framePath << std::endl;
+        convertFrame(framePath);
+    }
+    return 0;
+}

tools/gpu-frame-player/src/PlayerWindow.cpp

@@ -8,6 +8,8 @@
 
 #include "PlayerWindow.h"
 
+#include <QtCore/QByteArray>
+#include <QtCore/QBuffer>
 #include <QtGui/QResizeEvent>
 #include <QtGui/QImageReader>
 #include <QtGui/QScreen>
@@ -55,7 +57,7 @@ void PlayerWindow::loadFrame() {
         }
     }
 
-    QString fileName = QFileDialog::getOpenFileName(nullptr, tr("Open File"), openDir, tr("GPU Frames (*.json)"));
+    QString fileName = QFileDialog::getOpenFileName(nullptr, tr("Open File"), openDir, tr("GPU Frames (*.hfb)"));
     if (fileName.isNull()) {
         return;
     }
@@ -104,9 +106,11 @@ void PlayerWindow::resizeEvent(QResizeEvent* ev) {
     _renderThread.resize(ev->size());
 }
 
-void PlayerWindow::textureLoader(const std::string& filename, const gpu::TexturePointer& texture, uint16_t layer) {
+void PlayerWindow::textureLoader(const std::vector<uint8_t>& imageBytes, const gpu::TexturePointer& texture, uint16_t layer) {
     QImage image;
-    QImageReader(filename.c_str()).read(&image);
+    QByteArray bytes{ (const char*)imageBytes.data(), (int)imageBytes.size() };
+    QBuffer bytesBuffer(&bytes);
+    QImageReader(&bytesBuffer).read(&image);
     if (layer > 0) {
         return;
     }

tools/gpu-frame-player/src/PlayerWindow.h

@@ -28,7 +28,7 @@ protected:
     void loadFrame(const QString& path);
 
 private:
-    static void textureLoader(const std::string& filename, const gpu::TexturePointer& texture, uint16_t layer);
+    static void textureLoader(const std::vector<uint8_t>& filename, const gpu::TexturePointer& texture, uint16_t layer);
     QSettings _settings;
     RenderThread _renderThread;
 };