Merge branch 'master' into multiSphereAvatar04

CMakeLists.txt

@@ -108,8 +108,10 @@ set(PLATFORM_QT_GL OpenGL)
 
 if (USE_GLES)
   add_definitions(-DUSE_GLES)
+  add_definitions(-DGPU_POINTER_STORAGE_SHARED)
   set(PLATFORM_GL_BACKEND gpu-gl-common gpu-gles)
 else()
+  add_definitions(-DGPU_POINTER_STORAGE_RAW)
   set(PLATFORM_GL_BACKEND gpu-gl-common gpu-gl)
 endif()
 

android/apps/framePlayer/CMakeLists.txt

@@ -0,0 +1,5 @@
+set(TARGET_NAME framePlayer)
+setup_hifi_library(AndroidExtras)
+link_hifi_libraries(shared ktx shaders qml gpu gl ${PLATFORM_GL_BACKEND})
+target_link_libraries(${TARGET_NAME} android log m)
+target_opengl()

android/apps/framePlayer/build.gradle

@@ -0,0 +1,50 @@
+apply plugin: 'com.android.application'
+
+android {
+    signingConfigs {
+        release {
+            keyAlias 'key0'
+            keyPassword 'password'
+            storeFile file('C:/android/keystore.jks')
+            storePassword 'password'
+        }
+    }
+
+    compileSdkVersion 28
+    defaultConfig {
+        applicationId "io.highfidelity.frameplayer"
+        minSdkVersion 25
+        targetSdkVersion 28
+        ndk { abiFilters 'arm64-v8a' }
+        externalNativeBuild {
+            cmake {
+                arguments '-DHIFI_ANDROID=1',
+                    '-DHIFI_ANDROID_APP=framePlayer',
+                    '-DANDROID_TOOLCHAIN=clang',
+                    '-DANDROID_STL=c++_shared',
+                    '-DCMAKE_VERBOSE_MAKEFILE=ON'
+                targets = ['framePlayer']
+            }
+        }
+    }
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
+
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
+            signingConfig signingConfigs.release
+        }
+    }
+
+    externalNativeBuild.cmake.path '../../../CMakeLists.txt'
+}
+
+dependencies {
+    implementation fileTree(include: ['*.jar'], dir: '../../libraries/qt/libs')
+    //implementation project(':oculus')
+    implementation project(':qt')
+}

android/apps/framePlayer/proguard-rules.pro

@@ -0,0 +1,25 @@
+# Add project specific ProGuard rules here.
+# By default, the flags in this file are appended to flags specified
+# in C:\Android\SDK/tools/proguard/proguard-android.txt
+# You can edit the include path and order by changing the proguardFiles
+# directive in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# Add any project specific keep options here:
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

android/apps/framePlayer/src/main/AndroidManifest.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools"
+    package="io.highfidelity.frameplayer"
+    android:versionCode="1"
+    android:versionName="1.0"
+    android:installLocation="auto">
+    <uses-feature android:glEsVersion="0x00030002" android:required="true" />
+    <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
+    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+    <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
+    <application android:label="Frame Viewer"
+        android:allowBackup="false"
+        android:name="org.qtproject.qt5.android.bindings.QtApplication"
+        tools:ignore="GoogleAppIndexingWarning,MissingApplicationIcon">
+        <meta-data android:name="com.samsung.android.vr.application.mode" android:value="vr_only"/>
+        <activity
+            android:name="org.qtproject.qt5.android.bindings.QtActivity"
+            android:theme="@android:style/Theme.Black.NoTitleBar.Fullscreen"
+            android:launchMode="singleTask"
+            android:screenOrientation="landscape"
+            android:excludeFromRecents="false"
+            android:configChanges="screenSize|screenLayout|orientation|keyboardHidden|keyboard|navigation|uiMode">
+            <!-- JNI nonsense -->
+            <meta-data android:name="android.app.lib_name" android:value="framePlayer"/>
+            <!-- Qt nonsense -->
+            <meta-data android:name="android.app.qt_libs_resource_id" android:resource="@array/qt_libs"/>
+            <meta-data android:name="android.app.bundled_in_lib_resource_id" android:resource="@array/bundled_in_lib"/>
+            <meta-data android:name="android.app.bundled_in_assets_resource_id" android:resource="@array/bundled_in_assets"/>
+            <meta-data android:name="android.app.load_local_libs" android:value="plugins/platforms/android/libqtforandroid.so:plugins/bearer/libqandroidbearer.so"/>
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+    </application>
+</manifest>

android/apps/framePlayer/src/main/cpp/FramePlayer.qrc

@@ -0,0 +1,6 @@
+<!DOCTYPE RCC>
+<RCC version="1.0">
+  <qresource prefix="/">
+    <file>qml/main.qml</file>
+  </qresource>
+</RCC>

android/apps/framePlayer/src/main/cpp/PlayerWindow.cpp

@@ -0,0 +1,91 @@
+//
+//  Created by Bradley Austin Davis on 2018/10/21
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "PlayerWindow.h"
+
+#include <QtCore/QFileInfo>
+#include <QtGui/QImageReader>
+#include <QtQml/QQmlContext>
+#include <QtQuick/QQuickItem>
+
+#include <gpu/Frame.h>
+#include <gpu/FrameIO.h>
+
+PlayerWindow::PlayerWindow() {
+    setFlags(Qt::MSWindowsOwnDC | Qt::Window | Qt::Dialog | Qt::WindowMinMaxButtonsHint | Qt::WindowTitleHint);
+    setSurfaceType(QSurface::OpenGLSurface);
+    create();
+    showFullScreen();
+
+    // Make sure the window has been created by processing events
+    QCoreApplication::processEvents();
+
+    // Start the rendering thread
+    _renderThread.initialize(this, &_surface);
+
+    // Start the UI
+    _surface.resize(size());
+    connect(&_surface, &hifi::qml::OffscreenSurface::rootContextCreated, this, [](QQmlContext* context){
+        context->setContextProperty("FRAMES_FOLDER", "file:assets:/frames");
+    });
+    _surface.load("qrc:///qml/main.qml");
+
+    // Connect the UI handler
+    QObject::connect(_surface.getRootItem(), SIGNAL(loadFile(QString)),
+        this, SLOT(loadFile(QString))
+    );
+
+    // Turn on UI input events
+    installEventFilter(&_surface);
+}
+
+PlayerWindow::~PlayerWindow() {
+}
+
+// static const char* FRAME_FILE = "assets:/frames/20190110_1635.json";
+
+static void textureLoader(const std::string& filename, const gpu::TexturePointer& texture, uint16_t layer) {
+    QImage image;
+    QImageReader(filename.c_str()).read(&image);
+    if (layer > 0) {
+        return;
+    }
+    texture->assignStoredMip(0, image.byteCount(), image.constBits());
+}
+
+void PlayerWindow::loadFile(QString filename) {
+    QString realFilename = QUrl(filename).toLocalFile();
+    if (QFileInfo(realFilename).exists()) {
+        auto frame = gpu::readFrame(realFilename.toStdString(), _renderThread._externalTexture, &textureLoader);
+        _surface.pause();
+        _renderThread.submitFrame(frame);
+    }
+}
+
+void PlayerWindow::touchEvent(QTouchEvent* event) {
+    // Super basic input handling when the 3D scene is active.... tap with two finders to return to the
+    // QML UI
+    static size_t touches = 0;
+    switch (event->type()) {
+        case QEvent::TouchBegin:
+        case QEvent::TouchUpdate:
+            touches = std::max<size_t>(touches, event->touchPoints().size());
+            break;
+
+        case QEvent::TouchEnd:
+            if (touches >= 2) {
+                _renderThread.submitFrame(nullptr);
+                _surface.resume();
+            }
+            touches = 0;
+            break;
+
+        default:
+            break;
+    }
+}

android/apps/framePlayer/src/main/cpp/PlayerWindow.h

@@ -0,0 +1,35 @@
+//
+//  Created by Bradley Austin Davis on 2018/10/21
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#pragma once
+#include <QtGui/QWindow>
+#include <QtCore/QSettings>
+
+#include <qml/OffscreenSurface.h>
+#include <gpu/Forward.h>
+#include "RenderThread.h"
+
+// Create a simple OpenGL window that renders text in various ways
+class PlayerWindow : public QWindow {
+    Q_OBJECT
+
+public:
+    PlayerWindow();
+    virtual ~PlayerWindow();
+
+protected:
+    void touchEvent(QTouchEvent *ev) override;
+
+public slots:
+    void loadFile(QString filename);
+
+private:
+    hifi::qml::OffscreenSurface _surface;
+    QSettings _settings;
+    RenderThread _renderThread;
+};

android/apps/framePlayer/src/main/cpp/RenderThread.cpp

@@ -0,0 +1,162 @@
+//
+//  Created by Bradley Austin Davis on 2018/10/21
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "RenderThread.h"
+
+#include <QtGui/QWindow>
+
+void RenderThread::submitFrame(const gpu::FramePointer& frame) {
+    std::unique_lock<std::mutex> lock(_frameLock);
+    _pendingFrames.push(frame);
+}
+
+void RenderThread::move(const glm::vec3& v) {
+    std::unique_lock<std::mutex> lock(_frameLock);
+    _correction = glm::inverse(glm::translate(mat4(), v)) * _correction;
+}
+
+void RenderThread::setup() {
+    // Wait until the context has been moved to this thread
+    { std::unique_lock<std::mutex> lock(_frameLock); }
+
+    makeCurrent();
+    // Disable vsync for profiling
+    ::gl::setSwapInterval(0);
+
+    glClearColor(1, 1, 0, 1);
+    glClear(GL_COLOR_BUFFER_BIT);
+    _glContext.swapBuffers();
+
+    // GPU library init
+    gpu::Context::init<gpu::gl::GLBackend>();
+    _gpuContext = std::make_shared<gpu::Context>();
+    _backend = _gpuContext->getBackend();
+    _gpuContext->beginFrame();
+    _gpuContext->endFrame();
+    makeCurrent();
+
+
+    glGenFramebuffers(1, &_uiFbo);
+    glGenTextures(1, &_externalTexture);
+    glBindTexture(GL_TEXTURE_2D, _externalTexture);
+    static const glm::u8vec4 color{ 0 };
+    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, 1, 1, 0, GL_RGBA, GL_UNSIGNED_BYTE, &color);
+
+    glClearColor(0, 1, 1, 1);
+    glClear(GL_COLOR_BUFFER_BIT);
+    _glContext.swapBuffers();
+}
+
+void RenderThread::initialize(QWindow* window, hifi::qml::OffscreenSurface* offscreen) {
+    std::unique_lock<std::mutex> lock(_frameLock);
+    setObjectName("RenderThread");
+    Parent::initialize();
+
+    _offscreen = offscreen;
+    _window = window;
+    _glContext.setWindow(_window);
+    _glContext.create();
+    _glContext.makeCurrent();
+
+    hifi::qml::OffscreenSurface::setSharedContext(_glContext.qglContext());
+    glClearColor(1, 0, 0, 1);
+    glClear(GL_COLOR_BUFFER_BIT);
+    _glContext.swapBuffers();
+    _glContext.doneCurrent();
+    _glContext.moveToThread(_thread);
+    _thread->setObjectName("RenderThread");
+}
+
+void RenderThread::shutdown() {
+    _activeFrame.reset();
+    while (!_pendingFrames.empty()) {
+        _gpuContext->consumeFrameUpdates(_pendingFrames.front());
+        _pendingFrames.pop();
+    }
+    _gpuContext->shutdown();
+    _gpuContext.reset();
+}
+
+void RenderThread::renderFrame() {
+    auto windowSize = _window->geometry().size();
+    uvec2 readFboSize;
+    uint32_t readFbo{ 0 };
+
+    if (_activeFrame) {
+        const auto &frame = _activeFrame;
+        _backend->recycle();
+        _backend->syncCache();
+        _gpuContext->enableStereo(frame->stereoState._enable);
+        if (frame && !frame->batches.empty()) {
+            _gpuContext->executeFrame(frame);
+        }
+        auto &glBackend = static_cast<gpu::gl::GLBackend&>(*_backend);
+        readFbo = glBackend.getFramebufferID(frame->framebuffer);
+        readFboSize = frame->framebuffer->getSize();
+        CHECK_GL_ERROR();
+    } else {
+        hifi::qml::OffscreenSurface::TextureAndFence newTextureAndFence;
+        if (_offscreen->fetchTexture(newTextureAndFence)) {
+            if (_uiTexture != 0) {
+                auto readFence = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
+                glFlush();
+                _offscreen->getDiscardLambda()(_uiTexture, readFence);
+                _uiTexture = 0;
+            }
+
+            glWaitSync((GLsync)newTextureAndFence.second, 0, GL_TIMEOUT_IGNORED);
+            glDeleteSync((GLsync)newTextureAndFence.second);
+            _uiTexture = newTextureAndFence.first;
+            glBindFramebuffer(GL_READ_FRAMEBUFFER, _uiFbo);
+            glFramebufferTexture2D(GL_READ_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, _uiTexture, 0);
+        }
+
+        if (_uiTexture != 0) {
+            readFbo = _uiFbo;
+            readFboSize = { windowSize.width(), windowSize.height() };
+        }
+    }
+
+    if (readFbo) {
+        glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
+        glBindFramebuffer(GL_READ_FRAMEBUFFER, readFbo);
+        glBlitFramebuffer(
+            0, 0, readFboSize.x, readFboSize.y,
+            0, 0, windowSize.width(), windowSize.height(),
+            GL_COLOR_BUFFER_BIT, GL_NEAREST);
+        glBindFramebuffer(GL_READ_FRAMEBUFFER, 0);
+    } else {
+        glClearColor(1, 0, 0, 1);
+        glClear(GL_COLOR_BUFFER_BIT);
+    }
+
+    _glContext.swapBuffers();
+}
+
+void RenderThread::updateFrame() {
+    std::queue<gpu::FramePointer> pendingFrames;
+    {
+        std::unique_lock<std::mutex> lock(_frameLock);
+        pendingFrames.swap(_pendingFrames);
+    }
+
+    while (!pendingFrames.empty()) {
+        _activeFrame = pendingFrames.front();
+        pendingFrames.pop();
+        if (_activeFrame) {
+            _gpuContext->consumeFrameUpdates(_activeFrame);
+        }
+    }
+}
+
+bool RenderThread::process() {
+    updateFrame();
+    makeCurrent();
+    renderFrame();
+    return true;
+}

android/apps/framePlayer/src/main/cpp/RenderThread.h

@@ -0,0 +1,54 @@
+//
+//  Created by Bradley Austin Davis on 2018/10/21
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#pragma once
+
+#include <GenericThread.h>
+
+#include <gl/Context.h>
+#include <gpu/gl/GLBackend.h>
+#include <qml/OffscreenSurface.h>
+
+class RenderThread : public GenericThread {
+    using Parent = GenericThread;
+public:
+    QWindow* _window{ nullptr };
+    std::mutex _mutex;
+    gpu::ContextPointer _gpuContext;  // initialized during window creation
+    std::shared_ptr<gpu::Backend> _backend;
+    std::atomic<size_t> _presentCount{ 0 };
+    std::mutex _frameLock;
+    std::queue<gpu::FramePointer> _pendingFrames;
+    gpu::FramePointer _activeFrame;
+    uint32_t _externalTexture{ 0 };
+    glm::mat4 _correction;
+    hifi::qml::OffscreenSurface* _offscreen{ nullptr };
+
+    gl::Context _glContext;
+    uint32_t _uiTexture{ 0 };
+    uint32_t _uiFbo{ 0 };
+
+    void move(const glm::vec3& v);
+    void setup() override;
+    bool process() override;
+    void shutdown() override;
+
+    void initialize(QWindow* window, hifi::qml::OffscreenSurface* offscreen);
+
+    void submitFrame(const gpu::FramePointer& frame);
+    void updateFrame();
+    void renderFrame();
+
+    bool makeCurrent() {
+        return _glContext.makeCurrent();
+    }
+
+    void doneCurrent() {
+        _glContext.doneCurrent();
+    }
+};

android/apps/framePlayer/src/main/cpp/main.cpp

@@ -0,0 +1,54 @@
+//
+//  Created by Bradley Austin Davis on 2018/11/22
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <android/log.h>
+
+#include <QtGui/QGuiApplication>
+#include <QtCore/QTimer>
+#include <QtCore/QFileInfo>
+
+#include <Trace.h>
+
+#include "PlayerWindow.h"
+
+void messageHandler(QtMsgType type, const QMessageLogContext& context, const QString& message) {
+    if (!message.isEmpty()) {
+        const char * local=message.toStdString().c_str();
+        switch (type) {
+            case QtDebugMsg:
+                __android_log_write(ANDROID_LOG_DEBUG,"Interface",local);
+                break;
+            case QtInfoMsg:
+                __android_log_write(ANDROID_LOG_INFO,"Interface",local);
+                break;
+            case QtWarningMsg:
+                __android_log_write(ANDROID_LOG_WARN,"Interface",local);
+                break;
+            case QtCriticalMsg:
+                __android_log_write(ANDROID_LOG_ERROR,"Interface",local);
+                break;
+            case QtFatalMsg:
+            default:
+                __android_log_write(ANDROID_LOG_FATAL,"Interface",local);
+                abort();
+        }
+    }
+}
+
+int main(int argc, char** argv) {
+    setupHifiApplication("gpuFramePlayer");
+    QGuiApplication app(argc, argv);
+    auto oldMessageHandler = qInstallMessageHandler(messageHandler);
+    DependencyManager::set<tracing::Tracer>();
+    PlayerWindow window;
+    app.exec();
+    qInstallMessageHandler(oldMessageHandler);
+    return 0;
+}
+
+

android/apps/framePlayer/src/main/cpp/qml/main.qml

@@ -0,0 +1,36 @@
+import QtQuick 2.2
+import QtQuick.Dialogs 1.1
+import Qt.labs.folderlistmodel 2.11
+
+Item {
+    id: root
+    width: 640
+    height: 480
+    
+    ListView {
+        anchors.fill: parent
+
+        FolderListModel {
+            id: folderModel
+            folder: FRAMES_FOLDER
+            nameFilters: ["*.json"]
+        }
+
+        Component {
+            id: fileDelegate
+            Text { 
+                text: fileName 
+                font.pointSize: 36
+                MouseArea {
+                    anchors.fill: parent
+                    onClicked: root.loadFile(folderModel.folder + "/" + fileName);
+                }
+            }
+        }
+
+        model: folderModel
+        delegate: fileDelegate
+    }
+
+    signal loadFile(string filename);
+}

android/apps/framePlayer/src/main/res/drawable/ic_launcher.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--suppress AndroidUnknownAttribute -->
+<vector xmlns:api24="http://schemas.android.com/apk/res/android" xmlns:android="http://schemas.android.com/apk/res/android"
+    android:viewportWidth="192"
+    android:viewportHeight="192"
+    android:width="192dp"
+    android:height="192dp">
+    <path
+        android:pathData="M189.5 96.5A93.5 93.5 0 0 1 96 190 93.5 93.5 0 0 1 2.5 96.5 93.5 93.5 0 0 1 96 3 93.5 93.5 0 0 1 189.5 96.5Z"
+        android:fillColor="#333333" />
+    <path
+        android:pathData="M96.2 173.1c-10.3 0 -20.4 -2.1 -29.8 -6 -9.2 -3.8 -17.3 -9.4 -24.3 -16.4 -7 -7 -12.6 -15.2 -16.4 -24.3 -4.1 -9.6 -6.2 -19.6 -6.2 -30 0 -10.3 2.1 -20.4 6 -29.8 3.8 -9.2 9.4 -17.3 16.4 -24.3 7 -7 15.2 -12.6 24.3 -16.4 9.5 -4 19.5 -6 29.8 -6 10.3 0 20.4 2.1 29.8 6 9.2 3.8 17.3 9.4 24.3 16.4 7 7 12.6 15.2 16.4 24.3 4 9.5 6 19.5 6 29.8 0 10.3 -2.1 20.4 -6 29.8 -3.8 9.2 -9.4 17.3 -16.4 24.3 -7 7 -15.2 12.6 -24.3 16.4 -9.2 4.1 -19.3 6.2 -29.6 6.2zm0 -145.3c-37.8 0 -68.6 30.8 -68.6 68.6 0 37.8 30.8 68.6 68.6 68.6 37.8 0 68.6 -30.8 68.6 -68.6 0 -37.8 -30.8 -68.6 -68.6 -68.6z"
+        android:fillColor="#00b4f0" />
+    <path
+        android:pathData="M119.6 129l0 -53.8c3.4 -1.1 5.8 -4.3 5.8 -8 0 -4.6 -3.8 -8.4 -8.4 -8.4 -4.6 0 -8.4 3.8 -8.4 8.4 0 3.6 2.2 6.6 5.4 7.9l0 25L79 83.8 79 64c3.4 -1.1 5.8 -4.3 5.8 -8 0 -4.6 -3.8 -8.4 -8.4 -8.4 -4.6 0 -8.4 3.8 -8.4 8.4 0 3.6 2.2 6.6 5.4 7.9l0 54.1c-3.1 1.2 -5.4 4.3 -5.4 7.9 0 4.6 3.8 8.4 8.4 8.4 4.6 0 8.4 -3.8 8.4 -8.4 0 -3.7 -2.4 -6.9 -5.8 -8l0 -27.3 35 16.3 0 22.2c-3.1 1.2 -5.4 4.3 -5.4 7.9 0 4.6 3.8 8.4 8.4 8.4 4.6 0 8.4 -3.8 8.4 -8.4 0 -3.8 -2.4 -6.9 -5.8 -8z"
+        android:fillColor="#00b4f0" />
+</vector>

android/apps/framePlayer/src/main/res/values/strings.xml

@@ -0,0 +1,3 @@
+<resources>
+    <string name="app_name" translatable="false">GPU Frame Player</string>
+</resources>

android/settings.gradle

@@ -3,3 +3,6 @@ project(':qt').projectDir = new File(settingsDir, 'libraries/qt')
 
 include ':interface'
 project(':interface').projectDir = new File(settingsDir, 'apps/interface')
+
+//include ':framePlayer'
+//project(':framePlayer').projectDir = new File(settingsDir, 'apps/framePlayer')

cmake/macros/TargetGlad.cmake

@@ -7,6 +7,7 @@
 # 
 macro(TARGET_GLAD)
     if (ANDROID)
+        include(SelectLibraryConfigurations)
         set(INSTALL_DIR ${HIFI_ANDROID_PRECOMPILED}/glad)
         set(GLAD_INCLUDE_DIRS "${INSTALL_DIR}/include")
         set(GLAD_LIBRARY_DEBUG ${INSTALL_DIR}/lib/libglad_d.a)
@@ -31,8 +32,8 @@ macro(TARGET_GLAD)
         set(GLAD_INCLUDE_DIRS ${${GLAD_UPPER}_INCLUDE_DIRS})
         set(GLAD_LIBRARY ${${GLAD_UPPER}_LIBRARY})
     endif()
-            
+
     target_include_directories(${TARGET_NAME} PUBLIC ${GLAD_INCLUDE_DIRS})
     target_link_libraries(${TARGET_NAME} ${GLAD_LIBRARY})
-    target_link_libraries(${TARGET_NAME} ${GLAD_EXTRA_LIBRARIES})
+    target_link_libraries(${TARGET_NAME} ${GLAD_EXTRA_LIBRARIES})       
 endmacro()

cmake/macros/TargetZlib.cmake

@@ -6,8 +6,13 @@
 #  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 #
 macro(TARGET_ZLIB)
-   	# using VCPKG for zlib
-    find_package(ZLIB REQUIRED)
-    target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${ZLIB_INCLUDE_DIRS})
-    target_link_libraries(${TARGET_NAME} ${ZLIB_LIBRARIES})
+    if (ANDROID)
+        # zlib is part of the NDK
+        target_link_libraries(${TARGET_NAME} z)
+    else()
+        # using VCPKG for zlib
+        find_package(ZLIB REQUIRED)
+        target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${ZLIB_INCLUDE_DIRS})
+        target_link_libraries(${TARGET_NAME} ${ZLIB_LIBRARIES})
+    endif()
 endmacro()

cmake/ports/nvtt/CONTROL

@@ -1,3 +1,3 @@
 Source: nvtt
-Version: 8c7e6b40ee5095f227b75880fabd89c99d6f34c0
+Version: 330c4d56274a0f602a5c70596e2eb670a4ed56c2
 Description: Texture processing tools with support for Direct3D 10 and 11 formats.

cmake/ports/nvtt/portfile.cmake

@@ -10,8 +10,8 @@ include(vcpkg_common_functions)
 vcpkg_from_github(
     OUT_SOURCE_PATH SOURCE_PATH
     REPO highfidelity/nvidia-texture-tools
-    REF 8c7e6b40ee5095f227b75880fabd89c99d6f34c0
-    SHA512 f107d19dbbd6651ef2126b1422a5db8db291bf70311ac4fb1dbacb5ceaa8752fee38becbd32964f57596f0b84e1223bb2c3ff9d9c4fdc65c3e77a47836657cef
+    REF 330c4d56274a0f602a5c70596e2eb670a4ed56c2
+    SHA512 4c0bc2f369120d696cc27710b6d33086b27eef55f537ec66b9a5c8b1839bc2426c0413670b0f65be52c5d353468f0126dfe024be1f0690611d4d7e33ac530127
     HEAD_REF master
 )
 

interface/resources/qml/androidAppForce/ForFolderListModel.qml

@@ -0,0 +1,26 @@
+import QtQuick 2.2
+import QtQuick.Dialogs 1.1
+import Qt.labs.folderlistmodel 2.11
+
+Item {
+    width: 640
+    height: 480
+    
+    ListView {
+        width: 200; height: 400
+
+        FolderListModel {
+            id: folderModel
+            folder: "assets:/frames/"
+            nameFilters: ["*.json"]
+        }
+
+        Component {
+            id: fileDelegate
+            Text { text: fileName }
+        }
+
+        model: folderModel
+        delegate: fileDelegate
+    }
+}

interface/src/Application.cpp

@@ -3551,8 +3551,10 @@ void Application::resizeGL() {
         auto renderConfig = _graphicsEngine.getRenderEngine()->getConfiguration();
         assert(renderConfig);
         auto mainView = renderConfig->getConfig("RenderMainView.RenderDeferredTask");
-        assert(mainView);
-        mainView->setProperty("resolutionScale", renderResolutionScale);
+        // mainView can be null if we're rendering in forward mode
+        if (mainView) {
+            mainView->setProperty("resolutionScale", renderResolutionScale);
+        }
         displayPlugin->setRenderResolutionScale(renderResolutionScale);
     }
 
@@ -4100,6 +4102,19 @@ void Application::keyPressEvent(QKeyEvent* event) {
                 }
                 break;
 
+            case Qt::Key_G:
+                if (isShifted && isMeta && Menu::getInstance() && Menu::getInstance()->getMenu("Developer")->isVisible()) {
+                    static const QString HIFI_FRAMES_FOLDER_VAR = "HIFI_FRAMES_FOLDER";
+                    static const QString GPU_FRAME_FOLDER = QProcessEnvironment::systemEnvironment().contains(HIFI_FRAMES_FOLDER_VAR)
+                        ? QProcessEnvironment::systemEnvironment().value(HIFI_FRAMES_FOLDER_VAR)
+                        : "hifiFrames";
+                    static QString GPU_FRAME_TEMPLATE = GPU_FRAME_FOLDER + "/{DATE}_{TIME}";
+                    QString fullPath = FileUtils::computeDocumentPath(FileUtils::replaceDateTimeTokens(GPU_FRAME_TEMPLATE));
+                    if (FileUtils::canCreateFile(fullPath)) {
+                        getActiveDisplayPlugin()->captureFrame(fullPath.toStdString());
+                    }
+                }
+                break;
             case Qt::Key_X:
                 if (isShifted && isMeta) {
                     auto offscreenUi = getOffscreenUI();

interface/src/avatar/AvatarActionHold.cpp

@@ -288,39 +288,37 @@ void AvatarActionHold::doKinematicUpdate(float deltaTimeStep) {
             glm::vec3 oneFrameVelocity = (_positionalTarget - _previousPositionalTarget) / deltaTimeStep;
 
             _measuredLinearVelocities[_measuredLinearVelocitiesIndex++] = oneFrameVelocity;
-            if (_measuredLinearVelocitiesIndex >= AvatarActionHold::velocitySmoothFrames) {
-                _measuredLinearVelocitiesIndex = 0;
+            _measuredLinearVelocitiesIndex %= AvatarActionHold::velocitySmoothFrames;
+        }
+
+        if (_kinematicSetVelocity) {
+            glm::vec3 measuredLinearVelocity = _measuredLinearVelocities[0];
+            for (int i = 1; i < AvatarActionHold::velocitySmoothFrames; i++) {
+                // there is a bit of lag between when someone releases the trigger and when the software reacts to
+                // the release.  we calculate the velocity from previous frames but we don't include several
+                // of the most recent.
+                //
+                // if _measuredLinearVelocitiesIndex is
+                //     0 -- ignore i of 3 4 5
+                //     1 -- ignore i of 4 5 0
+                //     2 -- ignore i of 5 0 1
+                //     3 -- ignore i of 0 1 2
+                //     4 -- ignore i of 1 2 3
+                //     5 -- ignore i of 2 3 4
+
+                // This code is now disabled, but I'm leaving it commented-out because I suspect it will come back.
+                // if ((i + 1) % AvatarActionHold::velocitySmoothFrames == _measuredLinearVelocitiesIndex ||
+                //     (i + 2) % AvatarActionHold::velocitySmoothFrames == _measuredLinearVelocitiesIndex ||
+                //     (i + 3) % AvatarActionHold::velocitySmoothFrames == _measuredLinearVelocitiesIndex) {
+                //     continue;
+                // }
+
+                measuredLinearVelocity += _measuredLinearVelocities[i];
             }
-        }
-
-        glm::vec3 measuredLinearVelocity;
-        for (int i = 0; i < AvatarActionHold::velocitySmoothFrames; i++) {
-            // there is a bit of lag between when someone releases the trigger and when the software reacts to
-            // the release.  we calculate the velocity from previous frames but we don't include several
-            // of the most recent.
-            //
-            // if _measuredLinearVelocitiesIndex is
-            //     0 -- ignore i of 3 4 5
-            //     1 -- ignore i of 4 5 0
-            //     2 -- ignore i of 5 0 1
-            //     3 -- ignore i of 0 1 2
-            //     4 -- ignore i of 1 2 3
-            //     5 -- ignore i of 2 3 4
-
-            // This code is now disabled, but I'm leaving it commented-out because I suspect it will come back.
-            // if ((i + 1) % AvatarActionHold::velocitySmoothFrames == _measuredLinearVelocitiesIndex ||
-            //     (i + 2) % AvatarActionHold::velocitySmoothFrames == _measuredLinearVelocitiesIndex ||
-            //     (i + 3) % AvatarActionHold::velocitySmoothFrames == _measuredLinearVelocitiesIndex) {
-            //     continue;
-            // }
-
-            measuredLinearVelocity += _measuredLinearVelocities[i];
-        }
-        measuredLinearVelocity /= (float)(AvatarActionHold::velocitySmoothFrames
+            measuredLinearVelocity /= (float)(AvatarActionHold::velocitySmoothFrames
                                           // - 3  // 3 because of the 3 we skipped, above
                                           );
 
-        if (_kinematicSetVelocity) {
             rigidBody->setLinearVelocity(glmToBullet(measuredLinearVelocity));
             rigidBody->setAngularVelocity(glmToBullet(_angularVelocityTarget));
         }

interface/src/avatar/AvatarManager.cpp

@@ -475,6 +475,7 @@ void AvatarManager::handleRemovedAvatar(const AvatarSharedPointer& removedAvatar
         _spaceProxiesToDelete.push_back(avatar->getSpaceIndex());
     }
     AvatarHashMap::handleRemovedAvatar(avatar, removalReason);
+    avatar->tearDownGrabs();
 
     avatar->die();
     queuePhysicsChange(avatar);

interface/src/avatar/MyAvatar.cpp

@@ -866,7 +866,7 @@ void MyAvatar::simulate(float deltaTime, bool inView) {
     // and all of its joints, now update our attachements.
     Avatar::simulateAttachments(deltaTime);
     relayJointDataToChildren();
-    if (updateGrabs()) {
+    if (applyGrabChanges()) {
         _cauterizationNeedsUpdate = true;
     }
 
@@ -5301,7 +5301,7 @@ void MyAvatar::releaseGrab(const QUuid& grabID) {
 
     _avatarGrabsLock.withWriteLock([&] {
         if (_avatarGrabData.remove(grabID)) {
-            _deletedAvatarGrabs.insert(grabID);
+            _grabsToDelete.push_back(grabID);
             tellHandler = true;
         }
     });

interface/src/avatar/OtherAvatar.cpp

@@ -363,7 +363,7 @@ void OtherAvatar::simulate(float deltaTime, bool inView) {
 
     {
         PROFILE_RANGE(simulation, "grabs");
-        updateGrabs();
+        applyGrabChanges();
     }
 
     updateFadingStatus();

libraries/avatars-renderer/src/avatars-renderer/Avatar.cpp

@@ -326,88 +326,79 @@ void Avatar::removeAvatarEntitiesFromTree() {
     }
 }
 
-bool Avatar::updateGrabs() {
+bool Avatar::applyGrabChanges() {
+    if (!_avatarGrabDataChanged && _grabsToChange.empty() && _grabsToDelete.empty()) {
+        // early exit for most common case: nothing to do
+        return false;
+    }
+
     bool grabAddedOrRemoved = false;
-    // update the Grabs according to any changes in _avatarGrabData
     _avatarGrabsLock.withWriteLock([&] {
         if (_avatarGrabDataChanged) {
+            // collect changes in _avatarGrabData
             foreach (auto grabID, _avatarGrabData.keys()) {
-                AvatarGrabMap::iterator grabItr = _avatarGrabs.find(grabID);
-                if (grabItr == _avatarGrabs.end()) {
+                MapOfGrabs::iterator itr = _avatarGrabs.find(grabID);
+                if (itr == _avatarGrabs.end()) {
                     GrabPointer grab = std::make_shared<Grab>();
                     grab->fromByteArray(_avatarGrabData.value(grabID));
                     _avatarGrabs[grabID] = grab;
-                    _changedAvatarGrabs.insert(grabID);
+                    _grabsToChange.insert(grabID);
                 } else {
-                    GrabPointer grab = grabItr.value();
-                    bool changed = grab->fromByteArray(_avatarGrabData.value(grabID));
+                    bool changed = itr->second->fromByteArray(_avatarGrabData.value(grabID));
                     if (changed) {
-                        _changedAvatarGrabs.insert(grabID);
+                        _grabsToChange.insert(grabID);
                     }
                 }
             }
             _avatarGrabDataChanged = false;
         }
 
-        auto treeRenderer = DependencyManager::get<EntityTreeRenderer>();
-        auto entityTree = treeRenderer ? treeRenderer->getTree() : nullptr;
-        EntityEditPacketSender* packetSender = treeRenderer ? treeRenderer->getPacketSender() : nullptr;
-        auto sessionID = DependencyManager::get<NodeList>()->getSessionUUID();
-
-        QMutableSetIterator<QUuid> delItr(_deletedAvatarGrabs);
-        while (delItr.hasNext()) {
-            QUuid grabID = delItr.next();
-            GrabPointer grab = _avatarGrabs[grabID];
-            if (!grab) {
-                delItr.remove();
+        // delete _avatarGrabs
+        VectorOfIDs undeleted;
+        for (const auto& id : _grabsToDelete) {
+            MapOfGrabs::iterator itr = _avatarGrabs.find(id);
+            if (itr == _avatarGrabs.end()) {
                 continue;
             }
 
             bool success;
+            const GrabPointer& grab = itr->second;
             SpatiallyNestablePointer target = SpatiallyNestable::findByID(grab->getTargetID(), success);
-
-            // only clear this entry from the _deletedAvatarGrabs if we found the entity.
             if (success && target) {
-                bool iShouldTellServer = target->getEditSenderID() == sessionID;
-
-                EntityItemPointer entity = std::dynamic_pointer_cast<EntityItem>(target);
-                if (entity && entity->isAvatarEntity() && (entity->getOwningAvatarID() == sessionID ||
-                                                           entity->getOwningAvatarID() == AVATAR_SELF_ID)) {
-                    // this is our own avatar-entity, so we always tell the server about the release
-                    iShouldTellServer = true;
-                }
-
                 target->removeGrab(grab);
-                delItr.remove();
-                // in case this is the last grab on an entity, we need to shrink the queryAACube and tell the server
-                // about the final position.
-                if (entityTree) {
-                    bool force = true;
-                    entityTree->withWriteLock([&] {
-                        entityTree->updateEntityQueryAACube(target, packetSender, force, iShouldTellServer);
-                    });
-                }
+                _avatarGrabs.erase(itr);
                 grabAddedOrRemoved = true;
+            } else {
+                undeleted.push_back(id);
             }
-            _avatarGrabs.remove(grabID);
-            _changedAvatarGrabs.remove(grabID);
         }
+        _grabsToDelete = std::move(undeleted);
 
-        QMutableSetIterator<QUuid> changeItr(_changedAvatarGrabs);
-        while (changeItr.hasNext()) {
-            QUuid grabID = changeItr.next();
-            GrabPointer& grab = _avatarGrabs[grabID];
+        // change _avatarGrabs and add Actions to target
+        SetOfIDs unchanged;
+        for (const auto& id : _grabsToChange) {
+            MapOfGrabs::iterator itr = _avatarGrabs.find(id);
+            if (itr == _avatarGrabs.end()) {
+                continue;
+            }
 
             bool success;
+            const GrabPointer& grab = itr->second;
             SpatiallyNestablePointer target = SpatiallyNestable::findByID(grab->getTargetID(), success);
-
             if (success && target) {
                 target->addGrab(grab);
-                // only clear this entry from the _changedAvatarGrabs if we found the entity.
-                changeItr.remove();
+                if (isMyAvatar()) {
+                    EntityItemPointer entity = std::dynamic_pointer_cast<EntityItem>(target);
+                    if (entity) {
+                        entity->upgradeScriptSimulationPriority(PERSONAL_SIMULATION_PRIORITY);
+                    }
+                }
                 grabAddedOrRemoved = true;
+            } else {
+                unchanged.insert(id);
             }
         }
+        _grabsToChange = std::move(unchanged);
     });
     return grabAddedOrRemoved;
 }
@@ -415,8 +406,8 @@ bool Avatar::updateGrabs() {
 void Avatar::accumulateGrabPositions(std::map<QUuid, GrabLocationAccumulator>& grabAccumulators) {
     // relay avatar's joint position to grabbed target in a way that allows for averaging
     _avatarGrabsLock.withReadLock([&] {
-        foreach (auto grabID, _avatarGrabs.keys()) {
-            const GrabPointer& grab = _avatarGrabs.value(grabID);
+        for (const auto& entry : _avatarGrabs) {
+            const GrabPointer& grab = entry.second;
 
             if (!grab || !grab->getActionID().isNull()) {
                 continue; // the accumulated value isn't used, in this case.
@@ -434,6 +425,20 @@ void Avatar::accumulateGrabPositions(std::map<QUuid, GrabLocationAccumulator>& g
     });
 }
 
+void Avatar::tearDownGrabs() {
+    _avatarGrabsLock.withWriteLock([&] {
+        for (const auto& entry : _avatarGrabs) {
+            _grabsToDelete.push_back(entry.first);
+        }
+        _grabsToChange.clear();
+    });
+    applyGrabChanges();
+    if (!_grabsToDelete.empty()) {
+        // some grabs failed to delete, which is a possible "leak", so we log about it
+        qWarning() << "Failed to tearDown" << _grabsToDelete.size() << "grabs for Avatar" << getID();
+    }
+}
+
 void Avatar::relayJointDataToChildren() {
     forEachChild([&](SpatiallyNestablePointer child) {
         if (child->getNestableType() == NestableType::Entity) {
@@ -2063,3 +2068,12 @@ scriptable::ScriptableModelBase Avatar::getScriptableModel() {
     }
     return result;
 }
+
+void Avatar::clearAvatarGrabData(const QUuid& id) {
+    AvatarData::clearAvatarGrabData(id);
+    _avatarGrabsLock.withWriteLock([&] {
+        if (_avatarGrabs.find(id) != _avatarGrabs.end()) {
+            _grabsToDelete.push_back(id);
+        }
+    });
+}

libraries/avatars-renderer/src/avatars-renderer/Avatar.h

@@ -14,6 +14,9 @@
 #include <functional>
 #include <glm/glm.hpp>
 #include <glm/gtc/quaternion.hpp>
+#include <map>
+#include <set>
+#include <vector>
 
 #include <QtCore/QUuid>
 
@@ -23,10 +26,12 @@
 #include <graphics-scripting/Forward.h>
 #include <GLMHelpers.h>
 
+#include <Grab.h>
+#include <ThreadSafeValueCache.h>
+
 #include "Head.h"
 #include "SkeletonModel.h"
 #include "Rig.h"
-#include <ThreadSafeValueCache.h>
 
 #include "MetaModelPayload.h"
 #include "MultiSphereShape.h"
@@ -492,6 +497,7 @@ public:
     void accumulateGrabPositions(std::map<QUuid, GrabLocationAccumulator>& grabAccumulators);
 
     const std::vector<MultiSphereShape>& getMultiSphereShapes() const { return _multiSphereShapes; }
+    void tearDownGrabs();
 
 signals:
     void targetScaleChanged(float targetScale);
@@ -597,7 +603,7 @@ protected:
 
     // protected methods...
     bool isLookingAtMe(AvatarSharedPointer avatar) const;
-    bool updateGrabs();
+    bool applyGrabChanges();
     void relayJointDataToChildren();
 
     void fade(render::Transaction& transaction, render::Transition::Type type);
@@ -687,8 +693,15 @@ protected:
     
     std::vector<MultiSphereShape> _multiSphereShapes;
     AABox _fitBoundingBox;
+    void clearAvatarGrabData(const QUuid& grabID) override;
 
-    AvatarGrabMap _avatarGrabs;
+    using SetOfIDs = std::set<QUuid>;
+    using VectorOfIDs = std::vector<QUuid>;
+    using MapOfGrabs = std::map<QUuid, GrabPointer>;
+
+    MapOfGrabs _avatarGrabs;
+    SetOfIDs _grabsToChange; // updated grab IDs -- changes needed to entities or physics
+    VectorOfIDs _grabsToDelete; // deleted grab IDs -- changes needed to entities or physics
 };
 
 #endif // hifi_Avatar_h

libraries/avatars/src/AvatarData.cpp

@@ -3012,7 +3012,6 @@ void AvatarData::clearAvatarGrabData(const QUuid& grabID) {
     _avatarGrabsLock.withWriteLock([&] {
         if (_avatarGrabData.remove(grabID)) {
             _avatarGrabDataChanged = true;
-            _deletedAvatarGrabs.insert(grabID);
         }
     });
 }

libraries/avatars/src/AvatarData.h

@@ -54,7 +54,6 @@
 #include "AvatarTraits.h"
 #include "HeadData.h"
 #include "PathUtils.h"
-#include "Grab.h"
 
 #include <graphics/Material.h>
 
@@ -67,8 +66,6 @@ using PackedAvatarEntityMap = QMap<QUuid, QByteArray>; // similar to AvatarEntit
 using AvatarEntityIDs = QSet<QUuid>;
 
 using AvatarGrabDataMap = QMap<QUuid, QByteArray>;
-using AvatarGrabIDs = QSet<QUuid>;
-using AvatarGrabMap = QMap<QUuid, GrabPointer>;
 
 using AvatarDataSequenceNumber = uint16_t;
 
@@ -1475,8 +1472,6 @@ protected:
     mutable ReadWriteLockable _avatarGrabsLock;
     AvatarGrabDataMap _avatarGrabData;
     bool _avatarGrabDataChanged { false }; // by network
-    AvatarGrabIDs _changedAvatarGrabs; // updated grab IDs -- changes needed to entities or physics
-    AvatarGrabIDs _deletedAvatarGrabs; // deleted grab IDs -- changes needed to entities or physics
 
     // used to transform any sensor into world space, including the _hmdSensorMat, or hand controllers.
     ThreadSafeValueCache<glm::mat4> _sensorToWorldMatrixCache { glm::mat4() };
@@ -1539,7 +1534,7 @@ protected:
     }
 
     bool updateAvatarGrabData(const QUuid& grabID, const QByteArray& grabData);
-    void clearAvatarGrabData(const QUuid& grabID);
+    virtual void clearAvatarGrabData(const QUuid& grabID);
 
 private:
     friend void avatarStateFromFrame(const QByteArray& frameData, AvatarData* _avatar);

libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.cpp

@@ -15,8 +15,10 @@
 #include <QtCore/QCoreApplication>
 #include <QtCore/QThread>
 #include <QtCore/QTimer>
+#include <QtCore/QFileInfo>
 
 #include <QtGui/QImage>
+#include <QtGui/QImageWriter>
 #include <QtGui/QOpenGLFramebufferObject>
 
 #include <NumericalConstants.h>
@@ -30,6 +32,7 @@
 #include <gl/OffscreenGLCanvas.h>
 
 #include <gpu/Texture.h>
+#include <gpu/FrameIO.h>
 #include <shaders/Shaders.h>
 #include <gpu/gl/GLShared.h>
 #include <gpu/gl/GLBackend.h>
@@ -465,11 +468,43 @@ void OpenGLDisplayPlugin::submitFrame(const gpu::FramePointer& newFrame) {
     });
 }
 
-void OpenGLDisplayPlugin::renderFromTexture(gpu::Batch& batch, const gpu::TexturePointer texture, glm::ivec4 viewport, const glm::ivec4 scissor) {
-    renderFromTexture(batch, texture, viewport, scissor, gpu::FramebufferPointer());
+void OpenGLDisplayPlugin::captureFrame(const std::string& filename) const {
+    withOtherThreadContext([&] {
+        using namespace gpu;
+        auto glBackend = const_cast<OpenGLDisplayPlugin&>(*this).getGLBackend();
+        FramebufferPointer framebuffer{ Framebuffer::create("captureFramebuffer") };
+        TextureCapturer captureLambda = [&](const std::string& filename, const gpu::TexturePointer& texture, uint16 layer) {
+            QImage image;
+            if (texture->getUsageType() == TextureUsageType::STRICT_RESOURCE) {
+                image = QImage{ 1, 1, QImage::Format_ARGB32 };
+                auto storedImage = texture->accessStoredMipFace(0, 0);
+                memcpy(image.bits(), storedImage->data(), image.sizeInBytes());
+            //if (texture == textureCache->getWhiteTexture()) {
+            //} else if (texture == textureCache->getBlackTexture()) {
+            //} else if (texture == textureCache->getBlueTexture()) {
+            //} else if (texture == textureCache->getGrayTexture()) {
+            } else {
+                ivec4 rect = { 0, 0, texture->getWidth(), texture->getHeight() };
+                framebuffer->setRenderBuffer(0, texture, layer);
+                glBackend->syncGPUObject(*framebuffer);
+
+                image = QImage{ rect.z, rect.w, QImage::Format_ARGB32 };
+                glBackend->downloadFramebuffer(framebuffer, rect, image);
+            }
+            QImageWriter(filename.c_str()).write(image);
+        };
+
+        if (_currentFrame) {
+            gpu::writeFrame(filename, _currentFrame, captureLambda);
+        }
+    });
 }
 
-void OpenGLDisplayPlugin::renderFromTexture(gpu::Batch& batch, const gpu::TexturePointer texture, glm::ivec4 viewport, const glm::ivec4 scissor, gpu::FramebufferPointer copyFbo /*=gpu::FramebufferPointer()*/) {
+void OpenGLDisplayPlugin::renderFromTexture(gpu::Batch& batch, const gpu::TexturePointer& texture, const glm::ivec4& viewport, const glm::ivec4& scissor) {
+    renderFromTexture(batch, texture, viewport, scissor, nullptr);
+}
+
+void OpenGLDisplayPlugin::renderFromTexture(gpu::Batch& batch, const gpu::TexturePointer& texture, const glm::ivec4& viewport, const glm::ivec4& scissor, const gpu::FramebufferPointer& copyFbo /*=gpu::FramebufferPointer()*/) {
     auto fbo = gpu::FramebufferPointer();
     batch.enableStereo(false);
     batch.resetViewTransform();

libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.h

@@ -47,7 +47,7 @@ public:
     void endSession() override final;
     bool eventFilter(QObject* receiver, QEvent* event) override;
     bool isDisplayVisible() const override { return true; }
-
+    void captureFrame(const std::string& outputName) const override;
     void submitFrame(const gpu::FramePointer& newFrame) override;
 
     glm::uvec2 getRecommendedRenderSize() const override {
@@ -113,8 +113,8 @@ protected:
     // Plugin specific functionality to send the composed scene to the output window or device
     virtual void internalPresent();
 
-    void renderFromTexture(gpu::Batch& batch, const gpu::TexturePointer texture, glm::ivec4 viewport, const glm::ivec4 scissor, gpu::FramebufferPointer fbo);
-    void renderFromTexture(gpu::Batch& batch, const gpu::TexturePointer texture, glm::ivec4 viewport, const glm::ivec4 scissor);
+    void renderFromTexture(gpu::Batch& batch, const gpu::TexturePointer& texture, const glm::ivec4& viewport, const glm::ivec4& scissor, const gpu::FramebufferPointer& fbo);
+    void renderFromTexture(gpu::Batch& batch, const gpu::TexturePointer& texture, const glm::ivec4& viewport, const glm::ivec4& scissor);
     virtual void updateFrameData();
     virtual glm::mat4 getViewCorrection() { return glm::mat4(); }
 

libraries/display-plugins/src/display-plugins/hmd/DebugHmdDisplayPlugin.cpp

@@ -69,7 +69,10 @@ bool DebugHmdDisplayPlugin::internalActivate() {
     _eyeInverseProjections[1] = glm::inverse(_eyeProjections[1]);
     _eyeOffsets[0][3] = vec4{ -0.0327499993, 0.0, -0.0149999997, 1.0 };
     _eyeOffsets[1][3] = vec4{ 0.0327499993, 0.0, -0.0149999997, 1.0 };
-    _renderTargetSize = { 3024, 1680 };
+    // Test HMD per-eye resolution
+    _renderTargetSize = uvec2{ 1214 * 2 , 1344 };
+    // uncomment to capture a quarter size frame
+    //_renderTargetSize /= 2;
     _cullingProjection = _eyeProjections[0];
     // This must come after the initialization, so that the values calculated
     // above are available during the customizeContext call (when not running

libraries/entities/src/EntityItem.cpp

@@ -3425,7 +3425,19 @@ void EntityItem::addGrab(GrabPointer grab) {
     enableNoBootstrap();
     SpatiallyNestable::addGrab(grab);
 
-    if (getDynamic() && getParentID().isNull()) {
+    if (!getParentID().isNull()) {
+        return;
+    }
+
+    int jointIndex = grab->getParentJointIndex();
+    bool isFarGrab = jointIndex == FARGRAB_RIGHTHAND_INDEX
+        || jointIndex == FARGRAB_LEFTHAND_INDEX
+        || jointIndex == FARGRAB_MOUSE_INDEX;
+
+    // GRAB HACK: FarGrab doesn't work on non-dynamic things yet, but we really really want NearGrab
+    // (aka Hold) to work for such ojects, hence we filter the useAction case like so:
+    bool useAction = getDynamic() || (_physicsInfo && !isFarGrab);
+    if (useAction) {
         EntityTreePointer entityTree = getTree();
         assert(entityTree);
         EntitySimulationPointer simulation = entityTree ? entityTree->getSimulation() : nullptr;
@@ -3436,13 +3448,11 @@ void EntityItem::addGrab(GrabPointer grab) {
 
         EntityDynamicType dynamicType;
         QVariantMap arguments;
-        int grabParentJointIndex =grab->getParentJointIndex();
-        if (grabParentJointIndex == FARGRAB_RIGHTHAND_INDEX || grabParentJointIndex == FARGRAB_LEFTHAND_INDEX ||
-            grabParentJointIndex == FARGRAB_MOUSE_INDEX) {
+        if (isFarGrab) {
             // add a far-grab action
             dynamicType = DYNAMIC_TYPE_FAR_GRAB;
             arguments["otherID"] = grab->getOwnerID();
-            arguments["otherJointIndex"] = grabParentJointIndex;
+            arguments["otherJointIndex"] = jointIndex;
             arguments["targetPosition"] = vec3ToQMap(grab->getPositionalOffset());
             arguments["targetRotation"] = quatToQMap(grab->getRotationalOffset());
             arguments["linearTimeScale"] = 0.05;
@@ -3463,11 +3473,23 @@ void EntityItem::addGrab(GrabPointer grab) {
         grab->setActionID(actionID);
         _grabActions[actionID] = action;
         simulation->addDynamic(action);
+        markDirtyFlags(Simulation::DIRTY_MOTION_TYPE);
+        simulation->changeEntity(getThisPointer());
     }
 }
 
 void EntityItem::removeGrab(GrabPointer grab) {
+    int oldNumGrabs = _grabs.size();
     SpatiallyNestable::removeGrab(grab);
+    if (!getDynamic() && _grabs.size() != oldNumGrabs) {
+        // GRAB HACK: the expected behavior is for non-dynamic grabbed things to NOT be throwable
+        // so we slam the velocities to zero here whenever the number of grabs change.
+        // NOTE: if there is still another grab in effect this shouldn't interfere with the object's motion
+        // because that grab will slam the position+velocities next frame.
+        setLocalVelocity(glm::vec3(0.0f));
+        setAngularVelocity(glm::vec3(0.0f));
+    }
+    markDirtyFlags(Simulation::DIRTY_MOTION_TYPE);
 
     QUuid actionID = grab->getActionID();
     if (!actionID.isNull()) {

libraries/entities/src/SimulationOwner.h

@@ -96,11 +96,11 @@ const uint8_t RECRUIT_SIMULATION_PRIORITY = VOLUNTEER_SIMULATION_PRIORITY + 1;
 // When poking objects with scripts an observer will bid at SCRIPT_EDIT priority.
 const uint8_t SCRIPT_GRAB_SIMULATION_PRIORITY = 128;
 const uint8_t SCRIPT_POKE_SIMULATION_PRIORITY = SCRIPT_GRAB_SIMULATION_PRIORITY - 1;
-const uint8_t AVATAR_ENTITY_SIMULATION_PRIORITY = 255;
 
 // PERSONAL priority (needs a better name) is the level at which a simulation observer owns its own avatar
 // which really just means: things that collide with it will be bid at a priority level one lower
 const uint8_t PERSONAL_SIMULATION_PRIORITY = SCRIPT_GRAB_SIMULATION_PRIORITY;
+const uint8_t AVATAR_ENTITY_SIMULATION_PRIORITY = PERSONAL_SIMULATION_PRIORITY;
 
 
 class SimulationOwner {

libraries/fbx/src/FBXSerializer.cpp

@@ -1480,9 +1480,6 @@ HFMModel* FBXSerializer::extractHFMModel(const QVariantHash& mapping, const QStr
             }
         }
 
-        extracted.mesh.createMeshTangents(generateTangents);
-        extracted.mesh.createBlendShapeTangents(generateTangents);
-
         // find the clusters with which the mesh is associated
         QVector<QString> clusterIDs;
         foreach (const QString& childID, _connectionChildMap.values(it.key())) {

libraries/gl/src/gl/Config.cpp

@@ -118,6 +118,8 @@ void gl::setSwapInterval(int interval) {
     wglSwapIntervalEXT(interval);
 #elif defined(Q_OS_MAC)
     CGLSetParameter(CGLGetCurrentContext(), kCGLCPSwapInterval, &interval);
+#elif defined(Q_OS_ANDROID)
+    eglSwapInterval(eglGetCurrentDisplay(), interval);
 #else
     Q_UNUSED(interval);
 #endif

libraries/gl/src/gl/Context.cpp

@@ -34,22 +34,9 @@ bool Context::USE_CUSTOM_CONTEXT { true };
 #endif
 
 bool Context::enableDebugLogger() {
-#if defined(Q_OS_MAC)
-    // OSX does not support GL_KHR_debug or GL_ARB_debug_output
-    return false;
-#else
-#if defined(DEBUG) || defined(USE_GLES)
-    static bool enableDebugLogger = true;
-#else
-    static const QString DEBUG_FLAG("HIFI_DEBUG_OPENGL");
-    static bool enableDebugLogger = QProcessEnvironment::systemEnvironment().contains(DEBUG_FLAG);
-#endif
-    return enableDebugLogger;
-#endif
+    return gl::debugContextEnabled();
 }
 
-
-
 std::atomic<size_t> Context::_totalSwapchainMemoryUsage { 0 };
 
 size_t Context::getSwapchainMemoryUsage() { return _totalSwapchainMemoryUsage.load(); }

libraries/gl/src/gl/ContextQt.cpp

@@ -52,6 +52,11 @@ void Context::moveToThread(QThread* thread) {
 }
 
 void Context::debugMessageHandler(const QOpenGLDebugMessage& debugMessage) {
+    auto type = debugMessage.type();
+    if (type == QOpenGLDebugMessage::PerformanceType) {
+        return;
+    }
+
     auto severity = debugMessage.severity();
     switch (severity) {
         case QOpenGLDebugMessage::NotificationSeverity:
@@ -60,13 +65,13 @@ void Context::debugMessageHandler(const QOpenGLDebugMessage& debugMessage) {
         default:
             break;
     }
-    qDebug(glLogging) << debugMessage;
+    qWarning(glLogging) << debugMessage;
     return;
 }
 
 void Context::setupDebugLogging(QOpenGLContext *context) {
     QOpenGLDebugLogger *logger = new QOpenGLDebugLogger(context);
-    QObject::connect(logger, &QOpenGLDebugLogger::messageLogged, nullptr, [](const QOpenGLDebugMessage& message){
+    QObject::connect(logger, &QOpenGLDebugLogger::messageLogged, context, [](const QOpenGLDebugMessage& message){
         Context::debugMessageHandler(message);
     });
     if (logger->initialize()) {

libraries/gl/src/gl/GLHelpers.cpp

@@ -198,11 +198,48 @@ namespace gl {
 
 
     bool checkGLErrorDebug(const char* name) {
-#ifdef DEBUG
+        // Disabling error checking macro on Android debug builds for now, 
+        // as it throws off performance testing, which must be done on 
+        // Debug builds
+#if defined(DEBUG) && !defined(Q_OS_ANDROID)
         return checkGLError(name);
 #else
         Q_UNUSED(name);
         return false;
 #endif
     }
+
+    // Enables annotation of captures made by tools like renderdoc
+    bool khrDebugEnabled() {
+        static std::once_flag once;
+        static bool khrDebug = false;
+        std::call_once(once, [&] {
+            khrDebug = nullptr != glPushDebugGroupKHR;
+        });
+        return khrDebug;
+    }
+
+    // Enables annotation of captures made by tools like renderdoc
+    bool extDebugMarkerEnabled() {
+        static std::once_flag once;
+        static bool extMarker = false;
+        std::call_once(once, [&] {
+            extMarker = nullptr != glPushGroupMarkerEXT;
+        });
+        return extMarker;
+    }
+
+    bool debugContextEnabled() {
+#if defined(Q_OS_MAC)
+        // OSX does not support GL_KHR_debug or GL_ARB_debug_output
+        static bool enableDebugLogger = false;
+#elif defined(DEBUG) || defined(USE_GLES)
+        //static bool enableDebugLogger = true;
+        static bool enableDebugLogger = false;
+#else
+        static const QString DEBUG_FLAG("HIFI_DEBUG_OPENGL");
+        static bool enableDebugLogger = QProcessEnvironment::systemEnvironment().contains(DEBUG_FLAG);
+#endif
+        return enableDebugLogger;
+    }
 }

libraries/gl/src/gl/GLHelpers.h

@@ -37,7 +37,13 @@ bool isRenderThread();
 namespace gl {
     void globalLock();
     void globalRelease(bool finish = true);
-    
+
+    bool debugContextEnabled();
+
+    bool khrDebugEnabled();
+
+    bool extDebugMarkerEnabled();
+
     void withSavedContext(const std::function<void()>& f);
 
     bool checkGLError(const char* name);

libraries/gpu-gl-common/src/gpu/gl/GLBackend.cpp

@@ -392,8 +392,38 @@ void GLBackend::renderPassDraw(const Batch& batch) {
     }
 }
 
+// Support annotating captures in tools like Renderdoc
+class GlDuration {
+public:
+#ifdef USE_GLES
+    GlDuration(const char* name) {
+        // We need to use strlen here instead of -1, because the Snapdragon profiler
+        // will crash otherwise 
+        glPushDebugGroup(GL_DEBUG_SOURCE_APPLICATION, 0, strlen(name), name);
+    }
+    ~GlDuration() {
+        glPopDebugGroup();
+    }
+#else
+    GlDuration(const char* name) {
+        if (::gl::khrDebugEnabled()) {
+            glPushDebugGroupKHR(GL_DEBUG_SOURCE_APPLICATION_KHR, 0, -1, name);
+        }
+    }
+    ~GlDuration() {
+        if (::gl::khrDebugEnabled()) {
+            glPopDebugGroupKHR();
+        }
+    }
+#endif
+};
+
+#define GL_PROFILE_RANGE(category, name) \
+    PROFILE_RANGE(category, name); \
+    GlDuration glProfileRangeThis(name);
+
 void GLBackend::render(const Batch& batch) {
-    PROFILE_RANGE(render_gpu_gl, batch.getName());
+    GL_PROFILE_RANGE(render_gpu_gl, batch.getName().c_str());
 
     _transform._skybox = _stereo._skybox = batch.isSkyboxEnabled();
     // Allow the batch to override the rendering stereo settings
@@ -406,7 +436,7 @@ void GLBackend::render(const Batch& batch) {
     _transform._projectionJitter = Vec2(0.0f, 0.0f);
     
     {
-        PROFILE_RANGE(render_gpu_gl_detail, "Transfer");
+        GL_PROFILE_RANGE(render_gpu_gl_detail, "Transfer");
         renderPassTransfer(batch);
     }
 
@@ -416,7 +446,7 @@ void GLBackend::render(const Batch& batch) {
     }
 #endif
     {
-        PROFILE_RANGE(render_gpu_gl_detail, _stereo.isStereo() ? "Render Stereo" : "Render");
+        GL_PROFILE_RANGE(render_gpu_gl_detail, _stereo.isStereo() ? "Render Stereo" : "Render");
         renderPassDraw(batch);
     }
 #ifdef GPU_STEREO_DRAWCALL_INSTANCED

libraries/gpu-gl-common/src/gpu/gl/GLBackend.h

@@ -54,153 +54,8 @@
 #define GPU_STEREO_CAMERA_BUFFER
 #endif
 
-//
-// GL Backend pointer storage mechanism
-// One of the following three defines must be defined.
-// GPU_POINTER_STORAGE_SHARED
-
-// The platonic ideal, use references to smart pointers.
-// However, this produces artifacts because there are too many places in the code right now that
-// create temporary values (undesirable smart pointer duplications) and then those temp variables
-// get passed on and have their reference taken, and then invalidated
-// GPU_POINTER_STORAGE_REF
-
-// Raw pointer manipulation.  Seems more dangerous than the reference wrappers,
-// but in practice, the danger of grabbing a reference to a temporary variable
-// is causing issues
-// GPU_POINTER_STORAGE_RAW
-
-#if defined(USE_GLES)
-#define GPU_POINTER_STORAGE_SHARED
-#else
-#define GPU_POINTER_STORAGE_RAW
-#endif
-
 namespace gpu { namespace gl {
 
-#if defined(GPU_POINTER_STORAGE_SHARED)
-template <typename T>
-static inline bool compare(const std::shared_ptr<T>& a, const std::shared_ptr<T>& b) {
-    return a == b;
-}
-
-template <typename T>
-static inline T* acquire(const std::shared_ptr<T>& pointer) {
-    return pointer.get();
-}
-
-template <typename T>
-static inline void reset(std::shared_ptr<T>& pointer) {
-    return pointer.reset();
-}
-
-template <typename T>
-static inline bool valid(const std::shared_ptr<T>& pointer) {
-    return pointer.operator bool();
-}
-
-template <typename T>
-static inline void assign(std::shared_ptr<T>& pointer, const std::shared_ptr<T>& source) {
-    pointer = source;
-}
-
-using BufferReference = BufferPointer;
-using TextureReference = TexturePointer;
-using FramebufferReference = FramebufferPointer;
-using FormatReference = Stream::FormatPointer;
-using PipelineReference = PipelinePointer;
-
-#define GPU_REFERENCE_INIT_VALUE nullptr
-
-#elif defined(GPU_POINTER_STORAGE_REF)
-
-template <typename T>
-class PointerReferenceWrapper : public std::reference_wrapper<const std::shared_ptr<T>> {
-    using Parent = std::reference_wrapper<const std::shared_ptr<T>>;
-
-public:
-    using Pointer = std::shared_ptr<T>;
-    PointerReferenceWrapper() : Parent(EMPTY()) {}
-    PointerReferenceWrapper(const Pointer& pointer) : Parent(pointer) {}
-    void clear() { *this = EMPTY(); }
-
-private:
-    static const Pointer& EMPTY() {
-        static const Pointer EMPTY_VALUE;
-        return EMPTY_VALUE;
-    };
-};
-
-template <typename T>
-static bool compare(const PointerReferenceWrapper<T>& reference, const std::shared_ptr<T>& pointer) {
-    return reference.get() == pointer;
-}
-
-template <typename T>
-static inline T* acquire(const PointerReferenceWrapper<T>& reference) {
-    return reference.get().get();
-}
-
-template <typename T>
-static void assign(PointerReferenceWrapper<T>& reference, const std::shared_ptr<T>& pointer) {
-    reference = pointer;
-}
-
-template <typename T>
-static bool valid(const PointerReferenceWrapper<T>& reference) {
-    return reference.get().operator bool();
-}
-
-template <typename T>
-static inline void reset(PointerReferenceWrapper<T>& reference) {
-    return reference.clear();
-}
-
-using BufferReference = PointerReferenceWrapper<Buffer>;
-using TextureReference = PointerReferenceWrapper<Texture>;
-using FramebufferReference = PointerReferenceWrapper<Framebuffer>;
-using FormatReference = PointerReferenceWrapper<Stream::Format>;
-using PipelineReference = PointerReferenceWrapper<Pipeline>;
-
-#define GPU_REFERENCE_INIT_VALUE
-
-#elif defined(GPU_POINTER_STORAGE_RAW)
-
-template <typename T>
-static bool compare(const T* const& rawPointer, const std::shared_ptr<T>& pointer) {
-    return rawPointer == pointer.get();
-}
-
-template <typename T>
-static inline T* acquire(T*& rawPointer) {
-    return rawPointer;
-}
-
-template <typename T>
-static inline bool valid(const T* const& rawPointer) {
-    return rawPointer;
-}
-
-template <typename T>
-static inline void reset(T*& rawPointer) {
-    rawPointer = nullptr;
-}
-
-template <typename T>
-static inline void assign(T*& rawPointer, const std::shared_ptr<T>& pointer) {
-    rawPointer = pointer.get();
-}
-
-using BufferReference = Buffer*;
-using TextureReference = Texture*;
-using FramebufferReference = Framebuffer*;
-using FormatReference = Stream::Format*;
-using PipelineReference = Pipeline*;
-
-#define GPU_REFERENCE_INIT_VALUE nullptr
-
-#endif
-
 class GLBackend : public Backend, public std::enable_shared_from_this<GLBackend> {
     // Context Backend static interface required
     friend class gpu::Context;
@@ -583,13 +438,13 @@ protected:
 
             BufferState& operator=(const BufferState& other) = delete;
             void reset() {
-                gpu::gl::reset(buffer);
+                gpu::reset(buffer);
                 offset = 0;
                 size = 0;
             }
             bool compare(const BufferPointer& buffer, GLintptr offset, GLsizeiptr size) {
                 const auto& self = *this;
-                return (self.offset == offset && self.size == size && gpu::gl::compare(self.buffer, buffer));
+                return (self.offset == offset && self.size == size && gpu::compare(self.buffer, buffer));
             }
         };
 

libraries/gpu-gl-common/src/gpu/gl/GLBackendOutput.cpp

@@ -73,7 +73,7 @@ void GLBackend::do_advance(const Batch& batch, size_t paramOffset) {
 }
 
 void GLBackend::do_clearFramebuffer(const Batch& batch, size_t paramOffset) {
-    if (_stereo.isStereo() && !_pipeline._stateCache.scissorEnable) {
+    if (_stereo.isStereo() && !_pipeline._stateCache.flags.scissorEnable) {
         qWarning("Clear without scissor in stereo mode");
     }
 
@@ -140,7 +140,7 @@ void GLBackend::do_clearFramebuffer(const Batch& batch, size_t paramOffset) {
     }
 
     // Apply scissor if needed and if not already on
-    bool doEnableScissor = (useScissor && (!_pipeline._stateCache.scissorEnable));
+    bool doEnableScissor = (useScissor && (!_pipeline._stateCache.flags.scissorEnable));
     if (doEnableScissor) {
         glEnable(GL_SCISSOR_TEST);
     }

libraries/gpu-gl-common/src/gpu/gl/GLBackendState.cpp

@@ -100,17 +100,17 @@ void GLBackend::do_setStateCullMode(int32 mode) {
 }
 
 void GLBackend::do_setStateFrontFaceClockwise(bool isClockwise) {
-    if (_pipeline._stateCache.frontFaceClockwise != isClockwise) {
+    if (_pipeline._stateCache.flags.frontFaceClockwise != isClockwise) {
         static GLenum  GL_FRONT_FACES[] = { GL_CCW, GL_CW };
         glFrontFace(GL_FRONT_FACES[isClockwise]);
         (void)CHECK_GL_ERROR();
 
-        _pipeline._stateCache.frontFaceClockwise = isClockwise;
+        _pipeline._stateCache.flags.frontFaceClockwise = isClockwise;
     }
 }
 
 void GLBackend::do_setStateDepthClampEnable(bool enable) {
-    if (_pipeline._stateCache.depthClampEnable != enable) {
+    if (_pipeline._stateCache.flags.depthClampEnable != enable) {
 #if !defined(USE_GLES)
         if (enable) {
             glEnable(GL_DEPTH_CLAMP);
@@ -118,13 +118,13 @@ void GLBackend::do_setStateDepthClampEnable(bool enable) {
             glDisable(GL_DEPTH_CLAMP);
         }
         (void)CHECK_GL_ERROR();
-        _pipeline._stateCache.depthClampEnable = enable;
+        _pipeline._stateCache.flags.depthClampEnable = enable;
 #endif
     }
 }
 
 void GLBackend::do_setStateScissorEnable(bool enable) {
-    if (_pipeline._stateCache.scissorEnable != enable) {
+    if (_pipeline._stateCache.flags.scissorEnable != enable) {
         if (enable) {
             glEnable(GL_SCISSOR_TEST);
         } else {
@@ -132,12 +132,12 @@ void GLBackend::do_setStateScissorEnable(bool enable) {
         }
         (void)CHECK_GL_ERROR();
 
-        _pipeline._stateCache.scissorEnable = enable;
+        _pipeline._stateCache.flags.scissorEnable = enable;
     }
 }
 
 void GLBackend::do_setStateMultisampleEnable(bool enable) {
-    if (_pipeline._stateCache.multisampleEnable != enable) {
+    if (_pipeline._stateCache.flags.multisampleEnable != enable) {
 #if !defined(USE_GLES)
         if (enable) {
             glEnable(GL_MULTISAMPLE);
@@ -146,13 +146,13 @@ void GLBackend::do_setStateMultisampleEnable(bool enable) {
         }
         (void)CHECK_GL_ERROR();
 
-        _pipeline._stateCache.multisampleEnable = enable;
+        _pipeline._stateCache.flags.multisampleEnable = enable;
 #endif
     }
 }
 
 void GLBackend::do_setStateAntialiasedLineEnable(bool enable) {
-    if (_pipeline._stateCache.antialisedLineEnable != enable) {
+    if (_pipeline._stateCache.flags.antialisedLineEnable != enable) {
 #if !defined(USE_GLES)
         if (enable) {
             glEnable(GL_LINE_SMOOTH);
@@ -161,7 +161,7 @@ void GLBackend::do_setStateAntialiasedLineEnable(bool enable) {
         }
         (void)CHECK_GL_ERROR();
 
-        _pipeline._stateCache.antialisedLineEnable = enable;
+        _pipeline._stateCache.flags.antialisedLineEnable = enable;
 #endif
     }
 }
@@ -206,7 +206,7 @@ void GLBackend::do_setStateDepthTest(State::DepthTest test) {
         if (CHECK_GL_ERROR()) {
             qCDebug(gpulogging) << "DepthTest" << (test.isEnabled() ? "Enabled" : "Disabled")
                 << "Mask=" << (test.getWriteMask() ? "Write" : "no Write")
-                << "Func=" << test.getFunction()
+                << "Func=" << (uint16_t)test.getFunction()
                 << "Raw=" << test.getRaw();
         }
         _pipeline._stateCache.depthTest = test;
@@ -264,7 +264,7 @@ void GLBackend::do_setStateStencil(State::StencilActivation activation, State::S
 }
 
 void GLBackend::do_setStateAlphaToCoverageEnable(bool enable) {
-    if (_pipeline._stateCache.alphaToCoverageEnable != enable) {
+    if (_pipeline._stateCache.flags.alphaToCoverageEnable != enable) {
         if (enable) {
             glEnable(GL_SAMPLE_ALPHA_TO_COVERAGE);
         } else {
@@ -272,7 +272,7 @@ void GLBackend::do_setStateAlphaToCoverageEnable(bool enable) {
         }
         (void)CHECK_GL_ERROR();
 
-        _pipeline._stateCache.alphaToCoverageEnable = enable;
+        _pipeline._stateCache.flags.alphaToCoverageEnable = enable;
     }
 }
 
@@ -317,7 +317,7 @@ void GLBackend::do_setStateColorWriteMask(uint32 mask) {
             mask & State::ColorMask::WRITE_ALPHA);
         (void)CHECK_GL_ERROR();
 
-        _pipeline._stateCache.colorWriteMask = mask;
+        _pipeline._stateCache.colorWriteMask = (State::ColorMask)mask;
     }
 }
 

libraries/gpu-gl-common/src/gpu/gl/GLShared.cpp

@@ -174,17 +174,17 @@ void getCurrentGLState(State::Data& state) {
     {
         GLint winding;
         glGetIntegerv(GL_FRONT_FACE, &winding);
-        state.frontFaceClockwise = (winding == GL_CW);
+        state.flags.frontFaceClockwise = (winding == GL_CW);
 #if defined(USE_GLES)
-        state.multisampleEnable = glIsEnabled(GL_MULTISAMPLE_EXT);
-        state.antialisedLineEnable = false;
-        state.depthClampEnable = false;
+        state.flags.multisampleEnable = glIsEnabled(GL_MULTISAMPLE_EXT);
+        state.flags.antialisedLineEnable = false;
+        state.flags.depthClampEnable = false;
 #else
-        state.multisampleEnable = glIsEnabled(GL_MULTISAMPLE);
-        state.antialisedLineEnable = glIsEnabled(GL_LINE_SMOOTH);
-        state.depthClampEnable = glIsEnabled(GL_DEPTH_CLAMP);
+        state.flags.multisampleEnable = glIsEnabled(GL_MULTISAMPLE);
+        state.flags.antialisedLineEnable = glIsEnabled(GL_LINE_SMOOTH);
+        state.flags.depthClampEnable = glIsEnabled(GL_DEPTH_CLAMP);
 #endif
-        state.scissorEnable = glIsEnabled(GL_SCISSOR_TEST);
+        state.flags.scissorEnable = glIsEnabled(GL_SCISSOR_TEST);
     }
     {
         if (glIsEnabled(GL_POLYGON_OFFSET_FILL)) {
@@ -247,7 +247,7 @@ void getCurrentGLState(State::Data& state) {
         state.sampleMask = mask;
     }
     {
-        state.alphaToCoverageEnable = glIsEnabled(GL_SAMPLE_ALPHA_TO_COVERAGE);
+        state.flags.alphaToCoverageEnable = glIsEnabled(GL_SAMPLE_ALPHA_TO_COVERAGE);
     }
     {
         GLboolean isEnabled = glIsEnabled(GL_BLEND);
@@ -272,10 +272,10 @@ void getCurrentGLState(State::Data& state) {
     {
         GLboolean mask[4];
         glGetBooleanv(GL_COLOR_WRITEMASK, mask);
-        state.colorWriteMask = (mask[0] ? State::WRITE_RED : 0)
+        state.colorWriteMask = (State::ColorMask)((mask[0] ? State::WRITE_RED : 0)
             | (mask[1] ? State::WRITE_GREEN : 0)
             | (mask[2] ? State::WRITE_BLUE : 0)
-            | (mask[3] ? State::WRITE_ALPHA : 0);
+            | (mask[3] ? State::WRITE_ALPHA : 0));
     }
 
     (void)CHECK_GL_ERROR();

libraries/gpu-gl-common/src/gpu/gl/GLState.cpp

@@ -56,11 +56,11 @@ const GLState::Commands makeResetStateCommands() {
     return {
         std::make_shared<Command1I>(&GLBackend::do_setStateFillMode, DEFAULT.fillMode),
         std::make_shared<Command1I>(&GLBackend::do_setStateCullMode, DEFAULT.cullMode),
-        std::make_shared<Command1B>(&GLBackend::do_setStateFrontFaceClockwise, DEFAULT.frontFaceClockwise),
-        std::make_shared<Command1B>(&GLBackend::do_setStateDepthClampEnable, DEFAULT.depthClampEnable),
-        std::make_shared<Command1B>(&GLBackend::do_setStateScissorEnable, DEFAULT.scissorEnable),
-        std::make_shared<Command1B>(&GLBackend::do_setStateMultisampleEnable, DEFAULT.multisampleEnable),
-        std::make_shared<Command1B>(&GLBackend::do_setStateAntialiasedLineEnable, DEFAULT.antialisedLineEnable),
+        std::make_shared<Command1B>(&GLBackend::do_setStateFrontFaceClockwise, DEFAULT.flags.frontFaceClockwise),
+        std::make_shared<Command1B>(&GLBackend::do_setStateDepthClampEnable, DEFAULT.flags.depthClampEnable),
+        std::make_shared<Command1B>(&GLBackend::do_setStateScissorEnable, DEFAULT.flags.scissorEnable),
+        std::make_shared<Command1B>(&GLBackend::do_setStateMultisampleEnable, DEFAULT.flags.multisampleEnable),
+        std::make_shared<Command1B>(&GLBackend::do_setStateAntialiasedLineEnable, DEFAULT.flags.antialisedLineEnable),
 
         // Depth bias has 2 fields in State but really one call in GLBackend
         CommandPointer(depthBiasCommand),
@@ -75,7 +75,7 @@ const GLState::Commands makeResetStateCommands() {
 
         std::make_shared<Command1U>(&GLBackend::do_setStateSampleMask, DEFAULT.sampleMask),
 
-        std::make_shared<Command1B>(&GLBackend::do_setStateAlphaToCoverageEnable, DEFAULT.alphaToCoverageEnable),
+        std::make_shared<Command1B>(&GLBackend::do_setStateAlphaToCoverageEnable, DEFAULT.flags.alphaToCoverageEnable),
 
         std::make_shared<CommandBlend>(&GLBackend::do_setStateBlend, DEFAULT.blendFunction),
 

libraries/gpu/CMakeLists.txt

@@ -4,3 +4,4 @@ setup_hifi_library()
 link_hifi_libraries(shared ktx shaders)
 
 target_nsight()
+target_json()

libraries/gpu/src/gpu/Batch.cpp

@@ -45,7 +45,7 @@ size_t Batch::_dataMax{ BATCH_PREALLOCATE_MIN };
 size_t Batch::_objectsMax{ BATCH_PREALLOCATE_MIN };
 size_t Batch::_drawCallInfosMax{ BATCH_PREALLOCATE_MIN };
 
-Batch::Batch(const char* name) {
+Batch::Batch(const std::string& name) {
     _name = name;
     _commands.reserve(_commandsMax);
     _commandOffsets.reserve(_commandOffsetsMax);
@@ -64,7 +64,7 @@ Batch::~Batch() {
     _drawCallInfosMax = std::max(_drawCallInfos.size(), _drawCallInfosMax);
 }
 
-void Batch::setName(const char* name) {
+void Batch::setName(const std::string& name) {
     _name = name;
 }
 
@@ -96,7 +96,7 @@ void Batch::clear() {
     _textureTables.clear();
     _transforms.clear();
 
-    _name = nullptr;
+    _name.clear();
     _invalidModel = true;
     _currentModel = Transform();
     _drawcallUniform = 0;

libraries/gpu/src/gpu/Batch.h

@@ -89,14 +89,14 @@ public:
     void captureDrawCallInfo();
     void captureNamedDrawCallInfo(std::string name);
 
-    Batch(const char* name = nullptr);
+    Batch(const std::string& name = "");
     // Disallow copy construction and assignement of batches
     Batch(const Batch& batch) = delete;
     Batch& operator=(const Batch& batch) = delete;
     ~Batch();
 
-    void setName(const char* name);
-    const char* getName() const { return _name; }
+    void setName(const std::string& name);
+    const std::string& getName() const { return _name; }
     void clear();
 
     // Batches may need to override the context level stereo settings
@@ -440,6 +440,18 @@ public:
         };
     };
 
+    using CommandHandler = std::function<void(Command, const Param*)>;
+
+    void forEachCommand(const CommandHandler& handler) const {
+        size_t count = _commands.size();
+        for (size_t i = 0; i < count; ++i) {
+            const auto command = _commands[i];
+            const auto offset = _commandOffsets[i];
+            const Param* params = _params.data() + offset;
+            handler(command, params);
+        }
+    }
+
     typedef Cache<BufferPointer>::Vector BufferCaches;
     typedef Cache<TexturePointer>::Vector TextureCaches;
     typedef Cache<TextureTablePointer>::Vector TextureTableCaches;
@@ -519,7 +531,7 @@ public:
     bool _enableSkybox { false };
 
 protected:
-    const char* _name;
+    std::string _name;
 
     friend class Context;
     friend class Frame;

libraries/gpu/src/gpu/Buffer.h

@@ -149,7 +149,8 @@ protected:
     Size _end{ 0 };
     Sysmem _sysmem;
 
-
+    friend class Serializer;
+    friend class Deserializer;
     friend class BufferView;
     friend class Frame;
     friend class Batch;

libraries/gpu/src/gpu/Context.cpp

@@ -47,10 +47,7 @@ Context::Context(const Context& context) {
 }
 
 Context::~Context() {
-    for (auto batch : _batchPool) {
-        delete batch;
-    }
-    _batchPool.clear();
+    clearBatches();
     _syncedPrograms.clear();
 }
 
@@ -97,6 +94,12 @@ FramePointer Context::endFrame() {
     return result;
 }
 
+void Context::executeBatch(const char* name, std::function<void(Batch&)> lambda) const {
+    auto batch = acquireBatch(name);
+    lambda(*batch);
+    executeBatch(*batch);
+}
+
 void Context::executeBatch(Batch& batch) const {
     PROFILE_RANGE(render_gpu, __FUNCTION__);
     batch.flush();
@@ -117,28 +120,27 @@ void Context::executeFrame(const FramePointer& frame) const {
     PROFILE_RANGE(render_gpu, __FUNCTION__);
 
     // Grab the stats at the around the frame and delta to have a consistent sampling
-    ContextStats beginStats;
+    static ContextStats beginStats;
     getStats(beginStats);
 
     // FIXME? probably not necessary, but safe
     consumeFrameUpdates(frame);
     _backend->setStereoState(frame->stereoState);
-    {
-        Batch beginBatch("Context::executeFrame::begin");
-        _frameRangeTimer->begin(beginBatch);
-        _backend->render(beginBatch);
 
-        // Execute the frame rendering commands
-        for (auto& batch : frame->batches) {
-            _backend->render(*batch);
-        }
-
-        Batch endBatch("Context::executeFrame::end");
-        _frameRangeTimer->end(endBatch);
-        _backend->render(endBatch);
+    executeBatch("Context::executeFrame::begin", [&](Batch& batch){
+        batch.pushProfileRange("Frame");
+        _frameRangeTimer->begin(batch);
+    });
+    // Execute the frame rendering commands
+    for (auto& batch : frame->batches) {
+        _backend->render(*batch);
     }
+    executeBatch("Context::executeFrame::end", [&](Batch& batch){
+        batch.popProfileRange();
+        _frameRangeTimer->end(batch);
+    });
 
-    ContextStats endStats;
+    static ContextStats endStats;
     getStats(endStats);
     _frameStats.evalDelta(beginStats, endStats);
 }
@@ -381,6 +383,16 @@ void Context::processProgramsToSync() {
     }
 }
 
+std::mutex Context::_batchPoolMutex;
+std::list<Batch*> Context::_batchPool;
+
+void Context::clearBatches() {
+    for (auto batch : _batchPool) {
+        delete batch;
+    }
+    _batchPool.clear();
+}
+
 BatchPointer Context::acquireBatch(const char* name) {
     Batch* rawBatch = nullptr;
     {
@@ -393,8 +405,10 @@ BatchPointer Context::acquireBatch(const char* name) {
     if (!rawBatch) {
         rawBatch = new Batch();
     }
-    rawBatch->setName(name);
-    return BatchPointer(rawBatch, [this](Batch* batch) { releaseBatch(batch); });
+    if (name) {
+        rawBatch->setName(name);
+    }
+    return BatchPointer(rawBatch, [](Batch* batch) { releaseBatch(batch); });
 }
 
 void Context::releaseBatch(Batch* batch) {
@@ -406,7 +420,7 @@ void Context::releaseBatch(Batch* batch) {
 void gpu::doInBatch(const char* name,
                     const std::shared_ptr<gpu::Context>& context,
                     const std::function<void(Batch& batch)>& f) {
-    auto batch = context->acquireBatch(name);
+    auto batch = Context::acquireBatch(name);
     f(*batch);
     context->appendFrameBatch(batch);
 }

libraries/gpu/src/gpu/Context.h

@@ -24,6 +24,7 @@
 #include "Pipeline.h"
 #include "Framebuffer.h"
 #include "Frame.h"
+#include "PointerStorage.h"
 
 class QImage;
 
@@ -162,8 +163,8 @@ public:
     void appendFrameBatch(const BatchPointer& batch);
     FramePointer endFrame();
 
-    BatchPointer acquireBatch(const char* name = nullptr);
-    void releaseBatch(Batch* batch);
+    static BatchPointer acquireBatch(const char* name = nullptr);
+    static void releaseBatch(Batch* batch);
 
     // MUST only be called on the rendering thread
     //
@@ -175,6 +176,11 @@ public:
     // Execute a batch immediately, rather than as part of a frame
     void executeBatch(Batch& batch) const;
 
+    // MUST only be called on the rendering thread
+    //
+    // Execute a batch immediately, rather than as part of a frame
+    void executeBatch(const char* name, std::function<void(Batch&)> lambda) const;
+
     // MUST only be called on the rendering thread
     //
     // Executes a frame, applying any updates contained in the frame batches to the rendering
@@ -267,8 +273,6 @@ protected:
     Context(const Context& context);
 
     std::shared_ptr<Backend> _backend;
-    std::mutex _batchPoolMutex;
-    std::list<Batch*> _batchPool;
     bool _frameActive{ false };
     FramePointer _currentFrame;
     RangeTimerPointer _frameRangeTimer;
@@ -285,6 +289,11 @@ protected:
     static CreateBackend _createBackendCallback;
     static std::once_flag _initialized;
 
+    // Should probably move this functionality to Batch
+    static void clearBatches();
+    static std::mutex _batchPoolMutex;
+    static std::list<Batch*> _batchPool;
+
     friend class Shader;
     friend class Backend;
 };

libraries/gpu/src/gpu/Format.h

@@ -322,7 +322,7 @@ public:
     uint8 getLocationScalarCount() const { return DIMENSION_SCALAR_COUNT_PER_LOCATION[(Dimension)_dimension]; }
     uint32 getLocationSize() const { return DIMENSION_SCALAR_COUNT_PER_LOCATION[_dimension] * TYPE_SIZE[_type]; }
 
-    uint16 getRaw() const { return *((uint16*) (this)); }
+    uint16 getRaw() const { return *((const uint16*) (this)); }
 
     
     bool operator ==(const Element& right) const {
@@ -376,7 +376,7 @@ public:
 };
 
   
-enum ComparisonFunction {
+enum ComparisonFunction : uint16 {
     NEVER = 0,
     LESS,
     EQUAL,

libraries/gpu/src/gpu/Forward.h

@@ -96,6 +96,7 @@ namespace gpu {
     using TextureTablePointer = std::shared_ptr<TextureTable>;
 
     struct StereoState {
+        StereoState() {}
         bool isStereo() const {
             return _enable && !_contextDisable;
         }
@@ -108,6 +109,9 @@ namespace gpu {
         Mat4 _eyeProjections[2];
     };
 
+    class Serializer;
+    class Deserializer;
+
     class GPUObject {
     public:
         virtual ~GPUObject() = default;

libraries/gpu/src/gpu/Frame.h

@@ -42,6 +42,8 @@ namespace gpu {
         FramebufferRecycler framebufferRecycler;
 
     protected:
+        friend class Deserializer;
+
         // Should be called once per frame, on the recording thred
         void finish();
         void preRender();

libraries/gpu/src/gpu/FrameIO.h

@@ -0,0 +1,29 @@
+//
+//  Created by Bradley Austin Davis on 2018/10/14
+//  Copyright 2013-2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#pragma once
+#ifndef hifi_gpu_FrameIO_h
+#define hifi_gpu_FrameIO_h
+
+#include "Forward.h"
+#include "Format.h"
+
+#include <functional>
+
+namespace gpu {
+
+using TextureCapturer = std::function<void(const std::string&, const TexturePointer&, uint16 layer)>;
+using TextureLoader = std::function<void(const std::string&, const TexturePointer&, uint16 layer)>;
+void writeFrame(const std::string& filename, const FramePointer& frame, const TextureCapturer& capturer = nullptr);
+FramePointer readFrame(const std::string& filename, uint32_t externalTexture, const TextureLoader& loader = nullptr);
+
+using IndexOptimizer = std::function<void(Primitive, uint32_t faceCount, uint32_t indexCount, uint32_t* indices )>;
+void optimizeFrame(const std::string& filename, const IndexOptimizer& optimizer);
+
+}  // namespace gpu
+
+#endif

libraries/gpu/src/gpu/FrameIOKeys.h

@@ -0,0 +1,214 @@
+//
+//  Created by Bradley Austin Davis on 2018/10/14
+//  Copyright 2013-2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#pragma once
+#ifndef hifi_gpu_FrameIOKeys_h
+#define hifi_gpu_FrameIOKeys_h
+
+namespace gpu { namespace keys {
+
+static const char* binary = "binary";
+static const char* L00 = "L00";
+static const char* L1m1 = "L1m1";
+static const char* L10 = "L10";
+static const char* L11 = "L11";
+static const char* L2m2 = "L2m2";
+static const char* L2m1 = "L2m1";
+static const char* L20 = "L20";
+static const char* L21 = "L21";
+static const char* L22 = "L22";
+
+static const char* eyeProjections = "eyeProjections";
+static const char* eyeViews = "eyeViews";
+static const char* alphaToCoverageEnable = "alphaToCoverageEnable";
+static const char* antialisedLineEnable = "antialisedLineEnable";
+static const char* attributes = "attributes";
+static const char* batches = "batches";
+static const char* blendFunction = "blendFunction";
+static const char* borderColor = "borderColor";
+static const char* bufferMask = "bufferMask";
+static const char* buffers = "buffers";
+static const char* capturedTextures = "capturedTextures";
+static const char* channel = "channel";
+static const char* colorAttachments = "colorAttachments";
+static const char* colorWriteMask = "colorWriteMask";
+static const char* commands = "commands";
+static const char* comparisonFunction = "comparisonFunction";
+static const char* cullMode = "cullMode";
+static const char* data = "data";
+static const char* depth = "depth";
+static const char* depthBias = "depthBias";
+static const char* depthBiasSlopeScale = "depthBiasSlopeScale";
+static const char* depthClampEnable = "depthClampEnable";
+static const char* depthStencilAttachment = "depthStencilAttachment";
+static const char* depthTest = "depthTest";
+static const char* drawCallInfos = "drawCallInfos";
+static const char* drawcallUniform = "drawcallUniform";
+static const char* drawcallUniformReset = "drawcallUniformReset";
+static const char* element = "element";
+static const char* fillMode = "fillMode";
+static const char* filter = "filter";
+static const char* formats = "formats";
+static const char* frameIndex = "frameIndex";
+static const char* framebuffer = "framebuffer";
+static const char* framebuffers = "framebuffers";
+static const char* frequency = "frequency";
+static const char* frontFaceClockwise = "frontFaceClockwise";
+static const char* height = "height";
+static const char* id = "id";
+static const char* ktxFile = "ktxFile";
+static const char* layers = "layers";
+static const char* maxAnisotropy = "maxAnisotropy";
+static const char* maxMip = "maxMip";
+static const char* minMip = "minMip";
+static const char* mipOffset = "mipOffset";
+static const char* mips = "mips";
+static const char* multisampleEnable = "multisampleEnable";
+static const char* name = "name";
+static const char* namedData = "namedData";
+static const char* names = "names";
+static const char* objects = "objects";
+static const char* offset = "offset";
+static const char* pipelines = "pipelines";
+static const char* pose = "pose";
+static const char* profileRanges = "profileRanges";
+static const char* program = "program";
+static const char* programs = "programs";
+static const char* projectionJitter = "projectionJitter";
+static const char* queries = "queries";
+static const char* sampleCount = "sampleCount";
+static const char* sampleMask = "sampleMask";
+static const char* sampler = "sampler";
+static const char* samples = "samples";
+static const char* scissorEnable = "scissorEnable";
+static const char* shaders = "shaders";
+static const char* size = "size";
+static const char* skybox = "skybox";
+static const char* slot = "slot";
+static const char* source = "source";
+static const char* state = "state";
+static const char* stencilActivation = "stencilActivation";
+static const char* stencilTestBack = "stencilTestBack";
+static const char* stencilTestFront = "stencilTestFront";
+static const char* stereo = "stereo";
+static const char* subresource = "subresource";
+static const char* swapchains = "swapchains";
+static const char* texelFormat = "texelFormat";
+static const char* texture = "texture";
+static const char* textureTables = "textureTables";
+static const char* textures = "textures";
+static const char* transforms = "transforms";
+static const char* type = "type";
+static const char* usageType = "usageType";
+static const char* view = "view";
+static const char* width = "width";
+static const char* wrapModeU = "wrapModeU";
+static const char* wrapModeV = "wrapModeV";
+static const char* wrapModeW = "wrapModeW";
+
+
+static const char* backWriteMask = "backWriteMask";
+static const char* frontWriteMask = "frontWriteMask";
+static const char* reference = "reference";
+static const char* readMask = "readMask";
+static const char* failOp = "failOp";
+static const char* depthFailOp = "depthFailOp";
+static const char* passOp = "passOp";
+static const char* enabled = "enabled";
+static const char* blend = "blend";
+static const char* flags = "flags";
+static const char* writeMask = "writeMask";
+static const char* function = "function";
+static const char* sourceColor = "sourceColor";
+static const char* sourceAlpha = "sourceAlpha";
+static const char* destColor = "destColor";
+static const char* destAlpha = "destAlpha";
+static const char* opColor = "opColor";
+static const char* opAlpha = "opAlpha";
+static const char* enable = "enable";
+static const char* contextDisable = "contextDisable";
+
+static const char* COMMAND_NAMES[] = {
+    "draw",
+    "drawIndexed",
+    "drawInstanced",
+    "drawIndexedInstanced",
+    "multiDrawIndirect",
+    "multiDrawIndexedIndirect",
+
+    "setInputFormat",
+    "setInputBuffer",
+    "setIndexBuffer",
+    "setIndirectBuffer",
+
+    "setModelTransform",
+    "setViewTransform",
+    "setProjectionTransform",
+    "setProjectionJitter",
+    "setViewportTransform",
+    "setDepthRangeTransform",
+
+    "setPipeline",
+    "setStateBlendFactor",
+    "setStateScissorRect",
+
+    "setUniformBuffer",
+    "setResourceBuffer",
+    "setResourceTexture",
+    "setResourceTextureTable",
+    "setResourceFramebufferSwapChainTexture",
+
+    "setFramebuffer",
+    "setFramebufferSwapChain",
+    "clearFramebuffer",
+    "blit",
+    "generateTextureMips",
+    "generateTextureMipsWithPipeline",
+
+    "advance",
+
+    "beginQuery",
+    "endQuery",
+    "getQuery",
+
+    "resetStages",
+
+    "disableContextViewCorrection",
+    "restoreContextViewCorrection",
+
+    "disableContextStereo",
+    "restoreContextStereo",
+
+    "runLambda",
+
+    "startNamedCall",
+    "stopNamedCall",
+
+    "glUniform1i",
+    "glUniform1f",
+    "glUniform2f",
+    "glUniform3f",
+    "glUniform4f",
+    "glUniform3fv",
+    "glUniform4fv",
+    "glUniform4iv",
+    "glUniformMatrix3fv",
+    "glUniformMatrix4fv",
+
+    "glColor4f",
+
+    "pushProfileRange",
+    "popProfileRange",
+};
+
+template<class T, size_t N>
+constexpr size_t array_size(T (&)[N]) { return N; }
+
+static_assert(array_size(COMMAND_NAMES) == Batch::Command::NUM_COMMANDS, "Command array sizes must match");
+}}  // namespace gpu::keys
+
+#endif

libraries/gpu/src/gpu/FrameReader.cpp

@@ -0,0 +1,906 @@
+//
+//  Created by Bradley Austin Davis on 2018/10/14
+//  Copyright 2013-2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#include "FrameIO.h"
+
+#include <nlohmann/json.hpp>
+#include <unordered_map>
+
+#include <QtCore/QFileInfo>
+#include <QtCore/QDir>
+
+#include <ktx/KTX.h>
+#include "Frame.h"
+#include "Batch.h"
+#include "TextureTable.h"
+
+
+#include "FrameIOKeys.h"
+
+namespace gpu {
+using json = nlohmann::json;
+
+class Deserializer {
+public:
+    static std::string getBaseName(const std::string& filename) {
+        static const std::string ext{ ".json" };
+        if (std::string::npos != filename.rfind(ext)) {
+            return filename.substr(0, filename.size() - ext.size());
+        }
+        return filename;
+    }
+
+    Deserializer(const std::string& filename, uint32_t externalTexture, const TextureLoader& loader) :
+        basename(getBaseName(filename)), externalTexture(externalTexture), textureLoader(loader) {}
+
+    const std::string basename;
+    std::string basedir;
+    std::string binaryFile;
+    const uint32_t externalTexture;
+    TextureLoader textureLoader;
+    std::vector<ShaderPointer> shaders;
+    std::vector<ShaderPointer> programs;
+    std::vector<TexturePointer> textures;
+    std::vector<TextureTablePointer> textureTables;
+    std::vector<BufferPointer> buffers;
+    std::unordered_map<BufferPointer, size_t> bufferOffsets;
+    std::vector<Stream::FormatPointer> formats;
+    std::vector<PipelinePointer> pipelines;
+    std::vector<FramebufferPointer> framebuffers;
+    std::vector<SwapChainPointer> swapchains;
+    std::vector<QueryPointer> queries;
+    json frameNode;
+    FramePointer readFrame();
+    void optimizeFrame(const IndexOptimizer& optimizer);
+
+    FramePointer deserializeFrame();
+
+
+    void readBuffers(const json& node);
+
+    template <typename T>
+    static std::vector<T> readArray(const json& node, const std::string& name, std::function<T(const json& node)> parser) {
+        std::vector<T> result;
+        if (node.count(name)) {
+            const auto& sourceArrayNode = node[name];
+            result.reserve(sourceArrayNode.size());
+            for (const auto& sourceNode : sourceArrayNode) {
+                if (sourceNode.is_null()) {
+                    result.push_back(nullptr);
+                    continue;
+                }
+                result.push_back(parser(sourceNode));
+            }
+        }
+        return result;
+    }
+
+    template <typename T>
+    static std::vector<T> readNumericVector(const json& node) {
+        auto count = node.size();
+        std::vector<T> result;
+        result.resize(count);
+        for (size_t i = 0; i < count; ++i) {
+            result[i] = node[i];
+        }
+        return result;
+    }
+
+    template <size_t N>
+    static void readFloatArray(const json& node, float* out) {
+        for (size_t i = 0; i < N; ++i) {
+            out[i] = node[i].operator float();
+        }
+    }
+
+    template <typename T>
+    static bool readOptionalTransformed(T& dest, const json& node, const std::string& name, std::function<T(const json&)> f) {
+        if (node.count(name)) {
+            dest = f(node[name]);
+            return true;
+        }
+        return false;
+    }
+
+    template <typename T>
+    static bool readOptionalVectorTransformed(std::vector<T>& dest,
+                                              const json& node,
+                                              const std::string& name,
+                                              std::function<T(const json&)> f) {
+        if (node.count(name)) {
+            const auto& arrayNode = node[name];
+            const auto count = arrayNode.size();
+            dest.reserve(count);
+            for (size_t i = 0; i < count; ++i) {
+                dest.emplace_back(f(arrayNode[i]));
+            }
+            return true;
+        }
+        return false;
+    }
+
+    template <typename T>
+    static bool readOptionalVector(std::vector<T>& dest, const json& node, const std::string& name) {
+        return readOptionalVectorTransformed(dest, node, name, [](const json& node) { return node.get<T>(); });
+    }
+
+    template <typename T>
+    static T defaultNodeTransform(const json& node) {
+        return node.get<T>();
+    }
+
+    template <typename T, typename TT = T>
+    static bool readBatchCacheTransformed(typename Batch::Cache<T>::Vector& dest,
+                                          const json& node,
+                                          const std::string& name,
+                                          std::function<TT(const json&)> f = [](const json& node) -> TT {
+                                              return node.get<TT>();
+                                          }) {
+        if (node.count(name)) {
+            const auto& arrayNode = node[name];
+            for (const auto& entry : arrayNode) {
+                dest.cache(f(entry));
+            }
+            return true;
+        }
+        return false;
+    }
+
+    template <typename T>
+    static bool readPointerCache(typename Batch::Cache<T>::Vector& dest,
+                                 const json& node,
+                                 const std::string& name,
+                                 std::vector<T>& global) {
+        auto transform = [&](const json& node) -> const T& { return global[node.get<uint32_t>()]; };
+        return readBatchCacheTransformed<T, const T&>(dest, node, name, transform);
+    }
+
+    template <typename T>
+    static bool readOptional(T& dest, const json& node, const std::string& name) {
+        return readOptionalTransformed<T>(dest, node, name, [](const json& child) {
+            T result = child;
+            return result;
+        });
+    }
+
+    SwapChainPointer readSwapchain(const json& node);
+    ShaderPointer readProgram(const json& node);
+    PipelinePointer readPipeline(const json& node);
+    TextureTablePointer readTextureTable(const json& node);
+    TextureView readTextureView(const json& node);
+    FramebufferPointer readFramebuffer(const json& node);
+    BatchPointer readBatch(const json& node);
+    Batch::NamedBatchData readNamedBatchData(const json& node);
+
+    //static StatePointer readState(const json& node);
+    static QueryPointer readQuery(const json& node);
+    TexturePointer readTexture(const json& node, uint32_t externalTexture);
+    static ShaderPointer readShader(const json& node);
+    static Stream::FormatPointer readFormat(const json& node);
+    static Element readElement(const json& node);
+    static Sampler readSampler(const json& node);
+    static glm::mat4 readMat4(const json& node) {
+        glm::mat4 m;
+        if (!node.is_null()) {
+            readFloatArray<16>(node, &m[0][0]);
+        }
+        return m;
+    }
+    static glm::vec4 readVec4(const json& node) {
+        glm::vec4 v;
+        if (!node.is_null()) {
+            readFloatArray<4>(node, &v[0]);
+        }
+        return v;
+    }
+    static glm::vec3 readVec3(const json& node) {
+        glm::vec3 v;
+        if (!node.is_null()) {
+            readFloatArray<3>(node, &v[0]);
+        }
+        return v;
+    }
+    static glm::vec2 readVec2(const json& node) {
+        glm::vec2 v;
+        if (!node.is_null()) {
+            readFloatArray<2>(node, &v[0]);
+        }
+        return v;
+    }
+    static Transform readTransform(const json& node) { return Transform{ readMat4(node) }; }
+    static std::vector<uint8_t> fromBase64(const json& node);
+    static void readCommand(const json& node, Batch& batch);
+};
+
+FramePointer readFrame(const std::string& filename, uint32_t externalTexture, const TextureLoader& loader) {
+    return Deserializer(filename, externalTexture, loader).readFrame();
+}
+
+void optimizeFrame(const std::string& filename, const IndexOptimizer& optimizer) {
+    return Deserializer(filename, 0, {}).optimizeFrame(optimizer);
+}
+
+}  // namespace gpu
+
+using namespace gpu;
+
+void Deserializer::readBuffers(const json& buffersNode) {
+    storage::FileStorage mappedFile(binaryFile.c_str());
+    const auto mappedSize = mappedFile.size();
+    const auto* mapped = mappedFile.data();
+    size_t bufferCount = buffersNode.size();
+    buffers.reserve(buffersNode.size());
+    size_t offset = 0;
+    for (size_t i = 0; i < bufferCount; ++i) {
+        const auto& bufferNode = buffersNode[i];
+        if (bufferNode.is_null()) {
+            buffers.push_back(nullptr);
+            continue;
+        }
+
+        size_t size = bufferNode;
+        if (offset + size > mappedSize) {
+            throw std::runtime_error("read buffer error");
+        }
+        buffers.push_back(std::make_shared<Buffer>(size, mapped + offset));
+        bufferOffsets[buffers.back()] = offset;
+        offset += size;
+    }
+}
+
+Element Deserializer::readElement(const json& node) {
+    Element result;
+    if (!node.is_null()) {
+        *((uint16*)&result) = node;
+    }
+    return result;
+}
+
+Sampler Deserializer::readSampler(const json& node) {
+    Sampler result;
+    if (!node.is_null()) {
+        if (node.count(keys::borderColor)) {
+            result._desc._borderColor = readVec4(node[keys::borderColor]);
+        }
+        if (node.count(keys::maxAnisotropy)) {
+            result._desc._maxAnisotropy = node[keys::maxAnisotropy];
+        }
+        if (node.count(keys::wrapModeU)) {
+            result._desc._wrapModeU = node[keys::wrapModeU];
+        }
+        if (node.count(keys::wrapModeV)) {
+            result._desc._wrapModeV = node[keys::wrapModeV];
+        }
+        if (node.count(keys::wrapModeW)) {
+            result._desc._wrapModeW = node[keys::wrapModeW];
+        }
+        if (node.count(keys::filter)) {
+            result._desc._filter = node[keys::filter];
+        }
+        if (node.count(keys::comparisonFunction)) {
+            result._desc._comparisonFunc = node[keys::comparisonFunction];
+        }
+        if (node.count(keys::minMip)) {
+            result._desc._minMip = node[keys::minMip];
+        }
+        if (node.count(keys::maxMip)) {
+            result._desc._maxMip = node[keys::maxMip];
+        }
+        if (node.count(keys::mipOffset)) {
+            result._desc._mipOffset = node[keys::mipOffset];
+        }
+    }
+    return result;
+}
+
+TexturePointer Deserializer::readTexture(const json& node, uint32_t external) {
+    if (node.is_null()) {
+        return nullptr;
+    }
+
+    TextureUsageType usageType = node[keys::usageType];
+    Texture::Type type = node[keys::type];
+    glm::u16vec4 dims;
+    dims.x = node[keys::width];
+    dims.y = node[keys::height];
+    dims.z = node[keys::depth];
+    dims.w = node[keys::layers];
+    uint16 mips = node[keys::mips];
+    uint16 samples = node[keys::samples];
+    Element texelFormat = readElement(node[keys::texelFormat]);
+    Sampler sampler;
+    readOptionalTransformed<Sampler>(sampler, node, keys::sampler, [](const json& node) { return readSampler(node); });
+    TexturePointer result;
+    if (usageType == TextureUsageType::EXTERNAL) {
+        result = Texture::createExternal([](uint32_t, void*) {});
+        result->setExternalTexture(external, nullptr);
+    } else {
+        result = Texture::create(usageType, type, texelFormat, dims.x, dims.y, dims.z, samples, dims.w, mips, sampler);
+    }
+
+    auto& texture = *result;
+    readOptional(texture._source, node, keys::source);
+
+    std::string ktxFile;
+    readOptional(ktxFile, node, keys::ktxFile);
+    if (!ktxFile.empty()) {
+        if (QFileInfo(ktxFile.c_str()).isRelative()) {
+            ktxFile = basedir + "/" + ktxFile;
+        }
+        texture.setKtxBacking(ktxFile);
+    }
+    return result;
+}
+
+SwapChainPointer Deserializer::readSwapchain(const json& node) {
+    if (node.is_null()) {
+        return nullptr;
+    }
+
+    uint8_t swapChainSize = node[keys::size];
+    std::vector<FramebufferPointer> swapChainFramebuffers;
+    const auto& framebuffersNode = node[keys::framebuffers];
+    swapChainFramebuffers.resize(swapChainSize);
+    for (uint8_t i = 0; i < swapChainSize; ++i) {
+        auto index = framebuffersNode[i].get<uint32_t>();
+        swapChainFramebuffers[i] = framebuffers[index];
+    }
+    return std::make_shared<FramebufferSwapChain>(swapChainFramebuffers);
+}
+
+ShaderPointer Deserializer::readShader(const json& node) {
+    if (node.is_null()) {
+        return nullptr;
+    }
+
+    // FIXME support procedural shaders
+    Shader::Type type = node[keys::type];
+    uint32_t id = node[keys::id];
+    ShaderPointer result;
+    switch (type) {
+        //case Shader::Type::GEOMETRY:
+        //    result = Shader::createGeometry(id);
+        //    break;
+        case Shader::Type::VERTEX:
+            result = Shader::createVertex(id);
+            break;
+        case Shader::Type::FRAGMENT:
+            result = Shader::createPixel(id);
+            break;
+        default:
+            throw std::runtime_error("not implemented");
+    }
+    return result;
+}
+
+ShaderPointer Deserializer::readProgram(const json& node) {
+    if (node.is_null()) {
+        return nullptr;
+    }
+
+    std::vector<ShaderPointer> programShaders;
+    programShaders.reserve(node.size());
+    for (const auto& shaderRef : node) {
+        uint32_t shaderIndex = shaderRef;
+        programShaders.push_back(this->shaders[shaderIndex]);
+    }
+
+    // FIXME account for geometry and compute shaders?
+    return Shader::createProgram(programShaders[0], programShaders[1]);
+}
+
+static State::Flags readStateFlags(const json& node) {
+    State::Flags result;
+    // Hacky implementation because you can't pass boolean bitfields as references
+    bool value;
+    if (Deserializer::readOptional(value, node, keys::alphaToCoverageEnable)) {
+        result.alphaToCoverageEnable = value;
+    }
+    if (Deserializer::readOptional(value, node, keys::frontFaceClockwise)) {
+        result.frontFaceClockwise = value;
+    }
+    if (Deserializer::readOptional(value, node, keys::depthClampEnable)) {
+        result.depthClampEnable = value;
+    }
+    if (Deserializer::readOptional(value, node, keys::scissorEnable)) {
+        result.scissorEnable = value;
+    }
+    if (Deserializer::readOptional(value, node, keys::multisampleEnable)) {
+        result.multisampleEnable = value;
+    }
+    if (Deserializer::readOptional(value, node, keys::antialisedLineEnable)) {
+        result.antialisedLineEnable = value;
+    }
+    if (Deserializer::readOptional(value, node, keys::alphaToCoverageEnable)) {
+        result.alphaToCoverageEnable = value;
+    }
+    return result;
+}
+
+static State::BlendFunction readBlendFunction(const json& node) {
+    State::BlendFunction result;
+    uint16 enabled;
+    State::BlendArg blendArg;
+    State::BlendOp blendOp;
+    if (Deserializer::readOptional(enabled, node, keys::enabled)) {
+        result.enabled = enabled;
+    }
+    if (Deserializer::readOptional(blendArg, node, keys::sourceColor)) {
+        result.sourceColor = blendArg;
+    }
+    if (Deserializer::readOptional(blendArg, node, keys::sourceAlpha)) {
+        result.sourceAlpha = blendArg;
+    }
+    if (Deserializer::readOptional(blendArg, node, keys::destColor)) {
+        result.destColor = blendArg;
+    }
+    if (Deserializer::readOptional(blendArg, node, keys::destAlpha)) {
+        result.destAlpha = blendArg;
+    }
+    if (Deserializer::readOptional(blendOp, node, keys::opAlpha)) {
+        result.opAlpha = blendOp;
+    }
+    if (Deserializer::readOptional(blendOp, node, keys::opColor)) {
+        result.opColor = blendOp;
+    }
+    return result;
+}
+
+static State::DepthTest readDepthTest(const json& node) {
+    State::DepthTest result;
+    Deserializer::readOptional(result.writeMask, node, keys::writeMask);
+    Deserializer::readOptional(result.enabled, node, keys::enabled);
+    Deserializer::readOptional(result.function, node, keys::function);
+    return result;
+}
+
+static State::StencilTest readStencilTest(const json& node) {
+    State::StencilTest result;
+    State::ComparisonFunction compareOp;
+    State::StencilOp stencilOp;
+    if (Deserializer::readOptional(compareOp, node, keys::function)) {
+        result.function = compareOp;
+    }
+    if (Deserializer::readOptional(stencilOp, node, keys::failOp)) {
+        result.failOp = stencilOp;
+    }
+    if (Deserializer::readOptional(stencilOp, node, keys::depthFailOp)) {
+        result.depthFailOp = stencilOp;
+    }
+    if (Deserializer::readOptional(stencilOp, node, keys::passOp)) {
+        result.passOp = stencilOp;
+    }
+    if (Deserializer::readOptional(compareOp, node, keys::function)) {
+        result.function = compareOp;
+    }
+    Deserializer::readOptional(result.reference, node, keys::reference);
+    Deserializer::readOptional(result.readMask, node, keys::readMask);
+    return result;
+}
+
+static State::StencilActivation readStencilActivation(const json& node) {
+    auto jsonString = node.dump(2);
+    State::StencilActivation result;
+    bool enabled;
+    if (Deserializer::readOptional(enabled, node, keys::enabled)) {
+        result.enabled = enabled;
+    }
+    Deserializer::readOptional(result.frontWriteMask, node, keys::frontWriteMask);
+    Deserializer::readOptional(result.backWriteMask, node, keys::backWriteMask);
+    return result;
+}
+
+StatePointer readState(const json& node) {
+    if (node.is_null()) {
+        return nullptr;
+    }
+
+    State::Data data;
+    Deserializer::readOptionalTransformed<State::Flags>(data.flags, node, keys::flags, &readStateFlags);
+    Deserializer::readOptionalTransformed<State::BlendFunction>(data.blendFunction, node, keys::blendFunction, &readBlendFunction);
+    Deserializer::readOptionalTransformed<State::DepthTest>(data.depthTest, node, keys::depthTest, &readDepthTest);
+    Deserializer::readOptionalTransformed<State::StencilActivation>(data.stencilActivation, node, keys::stencilActivation, &readStencilActivation);
+    Deserializer::readOptionalTransformed<State::StencilTest>(data.stencilTestFront, node, keys::stencilTestFront, &readStencilTest);
+    Deserializer::readOptionalTransformed<State::StencilTest>(data.stencilTestBack, node, keys::stencilTestBack, &readStencilTest);
+    Deserializer::readOptional(data.colorWriteMask, node, keys::colorWriteMask);
+    Deserializer::readOptional(data.cullMode, node, keys::cullMode);
+    Deserializer::readOptional(data.depthBias, node, keys::depthBias);
+    Deserializer::readOptional(data.depthBiasSlopeScale, node, keys::depthBiasSlopeScale);
+    Deserializer::readOptional(data.fillMode, node, keys::fillMode);
+    Deserializer::readOptional(data.sampleMask, node, keys::sampleMask);
+    return std::make_shared<State>(data);
+}
+
+PipelinePointer Deserializer::readPipeline(const json& node) {
+    if (node.is_null()) {
+        return nullptr;
+    }
+
+    auto state = readState(node[keys::state]);
+    uint32_t programIndex = node[keys::program];
+    auto program = programs[programIndex];
+    return Pipeline::create(program, state);
+}
+
+Stream::FormatPointer Deserializer::readFormat(const json& node) {
+    if (node.is_null()) {
+        return nullptr;
+    }
+
+    auto result = std::make_shared<Stream::Format>();
+    auto& format = *result;
+    const auto& attributesNode = node[keys::attributes];
+    for (const auto& attributeNode : attributesNode) {
+        uint8_t slot = attributeNode[keys::slot];
+        auto& attribute = format._attributes[slot];
+        attribute._slot = slot;
+        attribute._channel = attributeNode[keys::channel];
+        readOptionalTransformed<Element>(attribute._element, attributeNode, keys::element,
+                                         [](const json& node) { return readElement(node); });
+        readOptional(attribute._frequency, attributeNode, keys::frequency);
+        readOptional(attribute._offset, attributeNode, keys::offset);
+    }
+    format.evaluateCache();
+    return result;
+}
+
+TextureTablePointer Deserializer::readTextureTable(const json& node) {
+    if (node.is_null()) {
+        return nullptr;
+    }
+    TextureTablePointer result = std::make_shared<TextureTable>();
+    auto& table = *result;
+    auto count = node.size();
+    for (size_t i = 0; i < count; ++i) {
+        uint32_t index = node[i];
+        table.setTexture(i, textures[index]);
+    }
+    return result;
+}
+
+TextureView Deserializer::readTextureView(const json& node) {
+    TextureView result;
+    auto texturePointerReader = [this](const json& node) {
+        uint32_t textureIndex = node;
+        return textures[textureIndex];
+    };
+    readOptionalTransformed<TexturePointer>(result._texture, node, keys::texture, texturePointerReader);
+    readOptionalTransformed<Element>(result._element, node, keys::element, &readElement);
+    readOptional(result._subresource, node, keys::subresource);
+    return result;
+}
+
+FramebufferPointer Deserializer::readFramebuffer(const json& node) {
+    if (node.is_null()) {
+        return nullptr;
+    }
+
+    FramebufferPointer result;
+    {
+        std::string name;
+        readOptional(name, node, keys::name);
+        result.reset(Framebuffer::create(name));
+    }
+    auto& framebuffer = *result;
+    readOptional(framebuffer._bufferMask, node, keys::bufferMask);
+    readOptional(framebuffer._height, node, keys::height);
+    readOptional(framebuffer._width, node, keys::width);
+    readOptional(framebuffer._numSamples, node, keys::sampleCount);
+    auto textureViewReader = [this](const json& node) -> TextureView { return readTextureView(node); };
+    readOptionalTransformed<TextureView>(framebuffer._depthStencilBuffer, node, keys::depthStencilAttachment,
+                                         textureViewReader);
+    if (framebuffer._depthStencilBuffer) {
+        framebuffer._depthStamp++;
+    }
+    if (node.count(keys::colorAttachments)) {
+        const auto& colorAttachmentsNode = node[keys::colorAttachments];
+        size_t count = colorAttachmentsNode.size();
+        for (size_t i = 0; i < count; ++i) {
+            const auto& colorAttachmentNode = colorAttachmentsNode[i];
+            if (colorAttachmentNode.is_null()) {
+                continue;
+            }
+            framebuffer._renderBuffers[i] = readTextureView(colorAttachmentNode);
+            framebuffer._colorStamps[i]++;
+        }
+    }
+
+    return result;
+}
+
+QueryPointer Deserializer::readQuery(const json& node) {
+    if (node.is_null()) {
+        return nullptr;
+    }
+    std::string name = node[keys::name];
+    return std::make_shared<Query>([](const Query&) {}, name);
+}
+
+std::vector<uint8_t> Deserializer::fromBase64(const json& node) {
+    std::vector<uint8_t> result;
+    auto decoded = QByteArray::fromBase64(QByteArray{ node.get<std::string>().c_str() });
+    result.resize(decoded.size());
+    memcpy(result.data(), decoded.data(), decoded.size());
+    return result;
+}
+
+static std::unordered_map<std::string, Batch::Command> getCommandNameMap() {
+    static std::unordered_map<std::string, Batch::Command> result;
+    if (result.empty()) {
+        for (Batch::Command i = Batch::COMMAND_draw; i < Batch::NUM_COMMANDS; i = (Batch::Command)(i + 1)) {
+            result[keys::COMMAND_NAMES[i]] = i;
+        }
+    }
+    return result;
+}
+
+void Deserializer::readCommand(const json& commandNode, Batch& batch) {
+    size_t count = commandNode.size();
+    std::string commandName = commandNode[0];
+    Batch::Command command = getCommandNameMap()[commandName];
+    batch._commands.push_back(command);
+    batch._commandOffsets.push_back(batch._params.size());
+    for (size_t i = 1; i < count; ++i) {
+        batch._params.emplace_back(commandNode[i].get<size_t>());
+    }
+}
+
+Batch::NamedBatchData Deserializer::readNamedBatchData(const json& node) {
+    Batch::NamedBatchData result;
+    readOptionalVectorTransformed<BufferPointer>(result.buffers, node, keys::buffers, [this](const json& node) {
+        uint32_t index = node;
+        return buffers[index];
+    });
+    readOptionalVectorTransformed<Batch::DrawCallInfo>(result.drawCallInfos, node, keys::drawCallInfos,
+                                                       [](const json& node) -> Batch::DrawCallInfo {
+                                                           Batch::DrawCallInfo result{ 0 };
+                                                           *((uint32_t*)&result) = node;
+                                                           return result;
+                                                       });
+    return result;
+}
+
+BatchPointer Deserializer::readBatch(const json& node) {
+    if (node.is_null()) {
+        return nullptr;
+    }
+
+    std::string batchName;
+    if (node.count(keys::name)) {
+        batchName = node[keys::name];
+    }
+    BatchPointer result = std::make_shared<Batch>(batchName);
+    auto& batch = *result;
+    readOptional(batch._enableStereo, node, keys::stereo);
+    readOptional(batch._enableSkybox, node, keys::skybox);
+    readOptionalTransformed<glm::vec2>(batch._projectionJitter, node, keys::projectionJitter, &readVec2);
+    readOptional(batch._drawcallUniform, node, keys::drawcallUniform);
+    readOptional(batch._drawcallUniformReset, node, keys::drawcallUniformReset);
+    readPointerCache(batch._textures, node, keys::textures, textures);
+    readPointerCache(batch._textureTables, node, keys::textureTables, textureTables);
+    readPointerCache(batch._buffers, node, keys::buffers, buffers);
+    readPointerCache(batch._pipelines, node, keys::pipelines, pipelines);
+    readPointerCache(batch._streamFormats, node, keys::formats, formats);
+    readPointerCache(batch._framebuffers, node, keys::framebuffers, framebuffers);
+    readPointerCache(batch._swapChains, node, keys::swapchains, swapchains);
+    readPointerCache(batch._queries, node, keys::queries, queries);
+
+    readOptionalVectorTransformed<Batch::DrawCallInfo>(batch._drawCallInfos, node, keys::drawCallInfos,
+                                                       [](const json& node) -> Batch::DrawCallInfo {
+                                                           Batch::DrawCallInfo result{ 0 };
+                                                           *((uint32_t*)&result) = node;
+                                                           return result;
+                                                       });
+
+    readOptionalTransformed<std::vector<uint8_t>>(batch._data, node, keys::data,
+                                                  [](const json& node) { return fromBase64(node); });
+
+    for (const auto& commandNode : node[keys::commands]) {
+        readCommand(commandNode, batch);
+    }
+    readBatchCacheTransformed<Transform, Transform>(batch._transforms, node, keys::transforms, &readTransform);
+    readBatchCacheTransformed<std::string>(batch._profileRanges, node, keys::profileRanges);
+    readBatchCacheTransformed<std::string>(batch._names, node, keys::names);
+
+    auto objectTransformReader = [](const json& node) -> Batch::TransformObject {
+        Batch::TransformObject result;
+        result._model = readMat4(node);
+        result._modelInverse = glm::inverse(result._model);
+        return result;
+    };
+    readOptionalVectorTransformed<Batch::TransformObject>(batch._objects, node, keys::objects, objectTransformReader);
+
+    if (node.count(keys::namedData)) {
+        const auto& namedDataNode = node[keys::namedData];
+        for (auto itr = namedDataNode.begin(); itr != namedDataNode.end(); ++itr) {
+            auto name = itr.key();
+            batch._namedData[name] = readNamedBatchData(itr.value());
+        }
+    }
+
+    return result;
+}
+
+StereoState readStereoState(const json& node) {
+    StereoState result;
+    Deserializer::readOptional(result._enable, node, keys::enable);
+    Deserializer::readOptional(result._contextDisable, node, keys::contextDisable);
+    Deserializer::readOptional(result._skybox, node, keys::skybox);
+    if (node.count(keys::eyeProjections)) {
+        auto projections = node[keys::eyeProjections];
+        result._eyeProjections[0] = Deserializer::readMat4(projections[0]);
+        result._eyeProjections[1] = Deserializer::readMat4(projections[1]);
+    }
+    if (node.count(keys::eyeViews)) {
+        auto views = node[keys::eyeViews];
+        result._eyeViews[0] = Deserializer::readMat4(views[0]);
+        result._eyeViews[1] = Deserializer::readMat4(views[1]);
+    }
+    return result;
+}
+
+
+FramePointer Deserializer::deserializeFrame() {
+    {
+        std::string filename{ basename + ".json" };
+        if (0 == basename.find("assets:")) {
+            auto lastSlash = basename.rfind('/');
+            basedir = basename.substr(0, lastSlash);
+        } else {
+            basedir = QFileInfo(basename.c_str()).absolutePath().toStdString();
+        }
+        storage::FileStorage mappedFile(filename.c_str());
+        frameNode = json::parse(std::string((const char*)mappedFile.data(), mappedFile.size()));
+    }
+
+    FramePointer result = std::make_shared<Frame>();
+    auto& frame = *result;
+
+    if (frameNode[keys::binary].is_string()) {
+        binaryFile = frameNode[keys::binary];
+        if (QFileInfo(binaryFile.c_str()).isRelative()) {
+            binaryFile = basedir + "/" + binaryFile;
+        }
+    } else {
+        binaryFile = basename + ".bin";
+    }
+
+
+    if (frameNode.count(keys::buffers)) {
+        readBuffers(frameNode[keys::buffers]);
+    }
+
+    shaders = readArray<ShaderPointer>(frameNode, keys::shaders, [](const json& node) { return readShader(node); });
+    // Must come after shaders
+    programs = readArray<ShaderPointer>(frameNode, keys::programs, [this](const json& node) { return readProgram(node); });
+    // Must come after programs
+    pipelines = readArray<PipelinePointer>(frameNode, keys::pipelines, [this](const json& node) { return readPipeline(node); });
+
+    formats = readArray<Stream::FormatPointer>(frameNode, keys::formats, [](const json& node) { return readFormat(node); });
+
+    auto textureReader = [this](const json& node) { return readTexture(node, externalTexture); };
+    textures = readArray<TexturePointer>(frameNode, keys::textures, textureReader);
+    if (textureLoader) {
+        std::vector<uint32_t> capturedTextures = readNumericVector<uint32_t>(frameNode[keys::capturedTextures]);
+        for (const auto& index : capturedTextures) {
+            const auto& texturePointer = textures[index];
+            uint16 layers = std::max<uint16>(texturePointer->getNumSlices(), 1);
+            for (uint16 layer = 0; layer < layers; ++layer) {
+                std::string filename = basename + "." + std::to_string(index) + "." + std::to_string(layer) + ".png";
+                textureLoader(filename, texturePointer, layer);
+            }
+        }
+    }
+
+    // Must come after textures
+    auto textureTableReader = [this](const json& node) { return readTextureTable(node); };
+    textureTables = readArray<TextureTablePointer>(frameNode, keys::textureTables, textureTableReader);
+
+    // Must come after textures
+    auto framebufferReader = [this](const json& node) { return readFramebuffer(node); };
+    framebuffers = readArray<FramebufferPointer>(frameNode, keys::framebuffers, framebufferReader);
+
+    // Must come after textures & framebuffers
+    swapchains =
+        readArray<SwapChainPointer>(frameNode, keys::swapchains, [this](const json& node) { return readSwapchain(node); });
+
+    queries = readArray<QueryPointer>(frameNode, keys::queries, [this](const json& node) { return readQuery(node); });
+    frame.framebuffer = framebuffers[frameNode[keys::framebuffer].get<uint32_t>()];
+    frame.view = readMat4(frameNode[keys::view]);
+    frame.pose = readMat4(frameNode[keys::pose]);
+    frame.frameIndex = frameNode[keys::frameIndex];
+    frame.stereoState = readStereoState(frameNode[keys::stereo]);
+    if (frameNode.count(keys::batches)) {
+        for (const auto& batchNode : frameNode[keys::batches]) {
+            frame.batches.push_back(readBatch(batchNode));
+        }
+    }
+
+    return result;
+}
+
+
+
+FramePointer Deserializer::readFrame() {
+    auto result = deserializeFrame();
+    result->finish();
+    return result;
+}
+
+void Deserializer::optimizeFrame(const IndexOptimizer& optimizer) {
+    auto result = deserializeFrame();
+    auto& frame = *result;
+
+
+        // optimize the index buffers?
+    struct CurrentIndexBuffer {
+        Offset offset{ 0 };
+        BufferPointer buffer;
+        Type type{ gpu::Type::INT32 };
+        Primitive primitve{ Primitive::TRIANGLES };
+        uint32_t numIndices{ 0 };
+        uint32_t startIndex{ 0 };
+    };
+
+    std::vector<CurrentIndexBuffer> captured;
+    for (auto& batch : frame.batches) {
+
+        CurrentIndexBuffer currentIndexBuffer;
+        batch->forEachCommand([&](Batch::Command cmd, const Batch::Param* params){
+            switch(cmd) {
+                case Batch::Command::COMMAND_setIndexBuffer:
+                    currentIndexBuffer.offset = params[0]._size;
+                    currentIndexBuffer.buffer = batch->_buffers.get(params[1]._int);
+                    currentIndexBuffer.type = (Type)params[2]._int;
+                    break;
+
+                case Batch::Command::COMMAND_drawIndexed:
+                    currentIndexBuffer.startIndex = params[0]._int;
+                    currentIndexBuffer.numIndices = params[1]._int;
+                    currentIndexBuffer.primitve = (Primitive)params[2]._int;
+                    captured.emplace_back(currentIndexBuffer);
+                    break;
+
+                case Batch::Command::COMMAND_drawIndexedInstanced:
+                    currentIndexBuffer.startIndex = params[1]._int;
+                    currentIndexBuffer.numIndices = params[2]._int;
+                    currentIndexBuffer.primitve = (Primitive)params[3]._int;
+                    captured.emplace_back(currentIndexBuffer);
+                    break;
+
+                default: 
+                    break;
+            }
+        });
+    }
+
+
+    std::string optimizedBinaryFile = basename + "_optimized.bin";
+    QFile(binaryFile.c_str()).copy(optimizedBinaryFile.c_str());
+    {
+        storage::FileStorage mappedFile(optimizedBinaryFile.c_str());
+        std::set<BufferPointer> uniqueBuffers;
+        for (const auto& capturedIndexData : captured) {
+            if (uniqueBuffers.count(capturedIndexData.buffer)) {
+                continue;
+            }
+            uniqueBuffers.insert(capturedIndexData.buffer);
+            auto bufferOffset = bufferOffsets[capturedIndexData.buffer];
+            auto& buffer = *capturedIndexData.buffer;
+            const auto& count = capturedIndexData.numIndices;
+            auto indices = (uint32_t*)buffer.editData();
+            optimizer(capturedIndexData.primitve, count / 3, count, indices);
+            memcpy(mappedFile.mutableData() + bufferOffset, indices, sizeof(uint32_t) * count);
+        }
+    }
+    frameNode[keys::binary] = optimizedBinaryFile;
+    {
+        std::string frameJson = frameNode.dump();
+        std::string filename = basename + "_optimized.json";
+        storage::FileStorage::create(filename.c_str(), frameJson.size(), (const uint8_t*)frameJson.data());
+    }
+}

libraries/gpu/src/gpu/FrameWriter.cpp

@@ -0,0 +1,844 @@
+//
+//  Created by Bradley Austin Davis on 2018/10/14
+//  Copyright 2013-2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#include "FrameIO.h"
+#include "Frame.h"
+#include "Batch.h"
+#include "TextureTable.h"
+#include <nlohmann/json.hpp>
+#include <unordered_map>
+
+#include "FrameIOKeys.h"
+
+namespace gpu {
+
+using json = nlohmann::json;
+
+class Serializer {
+public:
+    const std::string basename;
+    const TextureCapturer textureCapturer;
+    std::unordered_map<ShaderPointer, uint32_t> shaderMap;
+    std::unordered_map<ShaderPointer, uint32_t> programMap;
+    std::unordered_map<TexturePointer, uint32_t> textureMap;
+    std::unordered_map<TextureTablePointer, uint32_t> textureTableMap;
+    std::unordered_map<BufferPointer, uint32_t> bufferMap;
+    std::unordered_map<Stream::FormatPointer, uint32_t> formatMap;
+    std::unordered_map<PipelinePointer, uint32_t> pipelineMap;
+    std::unordered_map<FramebufferPointer, uint32_t> framebufferMap;
+    std::unordered_map<SwapChainPointer, uint32_t> swapchainMap;
+    std::unordered_map<QueryPointer, uint32_t> queryMap;
+
+    Serializer(const std::string& basename, const TextureCapturer& capturer) : basename(basename), textureCapturer(capturer) {}
+
+    template <typename T>
+    static uint32_t getGlobalIndex(const T& value, std::unordered_map<T, uint32_t>& map) {
+        if (map.count(value) == 0) {
+            uint32_t result = (uint32_t)map.size();
+            map[value] = result;
+            return result;
+        }
+        return map[value];
+    }
+
+    template <typename T>
+    static json serializePointerCache(const typename Batch::Cache<T>::Vector& cache, std::unordered_map<T, uint32_t>& map) {
+        json result = json::array();
+        const auto count = cache._items.size();
+        for (uint32_t i = 0; i < count; ++i) {
+            const auto& cacheEntry = cache._items[i];
+            const auto& val = cacheEntry._data;
+            result.push_back(getGlobalIndex(val, map));
+        }
+        return result;
+    }
+
+    template <typename T, typename TT = const T&>
+    static json serializeDataCache(const typename Batch::Cache<T>::Vector& cache,
+                                   std::function<TT(const T&)> f = [](const T& t) -> TT { return t; }) {
+        json result = json::array();
+        const auto count = cache._items.size();
+        for (uint32_t i = 0; i < count; ++i) {
+            const auto& cacheEntry = cache._items[i];
+            const auto& val = cacheEntry._data;
+            result.push_back(f(val));
+        }
+        return result;
+    }
+
+    template <typename T, typename TT = const T&>
+    static json writeVector(const std::vector<T>& v,
+                            const std::function<TT(const T&)>& f = [](const T& t) -> TT { return t; }) {
+        auto node = json::array();
+        for (const auto& e : v) {
+            node.push_back(f(e));
+        }
+        return node;
+    }
+
+    template <typename T>
+    static json writeNumericVector(const std::vector<T>& v) {
+        return writeVector<T, const T&>(v);
+    }
+
+    template <typename T>
+    static json writeUintVector(const std::vector<T>& v) {
+        return writeVector<T, const uint32_t&>(v, [](const T& t) -> const uint32_t& {
+            return reinterpret_cast<const uint32_t&>(t);
+        });
+    }
+
+    template <size_t N = 1>
+    static json writeFloatArray(const float* f) {
+        json result = json::array();
+        for (size_t i = 0; i < N; ++i) {
+            result.push_back(f[i]);
+        }
+        return result;
+    }
+
+    template <typename T>
+    static std::vector<T> mapToVector(const std::unordered_map<T, uint32_t>& map) {
+        std::vector<T> result;
+        result.resize(map.size());
+        for (const auto& entry : map) {
+            if (result[entry.second]) {
+                throw std::runtime_error("Invalid map");
+            }
+            result[entry.second] = entry.first;
+        }
+        return result;
+    }
+
+    template <typename T, typename F>
+    std::function<json(const T&)> memberWriter(F f) {
+        return std::bind(f, this, std::placeholders::_1);
+    }
+
+    void writeFrame(const Frame& frame);
+    json writeBatch(const Batch& batch);
+    json writeTextureTable(const TextureTablePointer& textureTable);
+    json writeTextureView(const TextureView& textureView);
+    json writeFramebuffer(const FramebufferPointer& texture);
+    json writePipeline(const PipelinePointer& pipeline);
+    json writeSwapchain(const SwapChainPointer& swapchain);
+    json writeProgram(const ShaderPointer& program);
+    json writeNamedBatchData(const Batch::NamedBatchData& namedData);
+
+    json writeCapturableTextures(const Frame& frame);
+    void writeBinaryBlob();
+    static std::string toBase64(const std::vector<uint8_t>& v);
+    static json writeIrradiance(const SHPointer& irradiance);
+    static json writeMat4(const glm::mat4& m) {
+        static const glm::mat4 IDENTITY;
+        if (m == IDENTITY) {
+            return json();
+        }
+        return writeFloatArray<16>(&m[0][0]);
+    }
+    static json writeVec4(const glm::vec4& v) { return writeFloatArray<4>(&v[0]); }
+    static json writeVec3(const glm::vec3& v) { return writeFloatArray<3>(&v[0]); }
+    static json writeVec2(const glm::vec2& v) { return writeFloatArray<2>(&v[0]); }
+    static json writeTransform(const Transform& t) { return writeMat4(t.getMatrix()); }
+    static json writeCommand(size_t index, const Batch& batch);
+    static json writeSampler(const Sampler& sampler);
+    static json writeTexture(const TexturePointer& texture);
+    static json writeFormat(const Stream::FormatPointer& format);
+    static json writeQuery(const QueryPointer& query);
+    static json writeShader(const ShaderPointer& shader);
+    static json writeBuffer(const BufferPointer& bufferPointer);
+
+    static const TextureView DEFAULT_TEXTURE_VIEW;
+    static const Sampler DEFAULT_SAMPLER;
+
+    template <typename T, typename F>
+    void serializeMap(json& frameNode, const char* key, const std::unordered_map<T, uint32_t>& map, F f) {
+        auto& node = frameNode[key] = json::array();
+        for (const auto& item : mapToVector(map)) {
+            node.push_back(f(item));
+        }
+    }
+};
+
+void writeFrame(const std::string& filename, const FramePointer& frame, const TextureCapturer& capturer) {
+    Serializer(filename, capturer).writeFrame(*frame);
+}
+
+}  // namespace gpu
+
+using namespace gpu;
+
+const TextureView Serializer::DEFAULT_TEXTURE_VIEW = TextureView();
+const Sampler Serializer::DEFAULT_SAMPLER = Sampler();
+
+std::string Serializer::toBase64(const std::vector<uint8_t>& v) {
+    return QByteArray((const char*)v.data(), (int)v.size()).toBase64().toStdString();
+}
+
+json Serializer::writeCommand(size_t index, const Batch& batch) {
+    const auto& command = batch._commands[index];
+    auto offset = batch._commandOffsets[index];
+    auto endOffset = batch._params.size();
+    if ((index + 1) < batch._commands.size()) {
+        endOffset = batch._commandOffsets[index + 1];
+    }
+
+    json result = json::array();
+    result.push_back(keys::COMMAND_NAMES[command]);
+    while (offset < endOffset) {
+        result.push_back(batch._params[offset]._size);
+        ++offset;
+    }
+    return result;
+}
+
+json Serializer::writeNamedBatchData(const Batch::NamedBatchData& namedData) {
+    json result = json::object();
+    auto& buffersNode = result[keys::buffers] = json::array();
+    for (const auto& buffer : namedData.buffers) {
+        buffersNode.push_back(getGlobalIndex(buffer, bufferMap));
+    }
+    result[keys::drawCallInfos] = writeUintVector(namedData.drawCallInfos);
+    return result;
+}
+
+json Serializer::writeBatch(const Batch& batch) {
+    json batchNode;
+    static const Batch DEFAULT_BATCH;
+
+    batchNode[keys::name] = batch.getName();
+    if (batch._enableSkybox != DEFAULT_BATCH._enableSkybox) {
+        batchNode[keys::skybox] = batch._enableSkybox;
+    }
+    if (batch._enableStereo != DEFAULT_BATCH._enableStereo) {
+        batchNode[keys::stereo] = batch._enableStereo;
+    }
+    if (batch._projectionJitter != DEFAULT_BATCH._projectionJitter) {
+        batchNode[keys::projectionJitter] = writeVec2(batch._projectionJitter);
+    }
+    if (batch._drawcallUniform != DEFAULT_BATCH._drawcallUniform) {
+        batchNode[keys::drawcallUniform] = batch._drawcallUniform;
+    }
+    if (batch._drawcallUniformReset != DEFAULT_BATCH._drawcallUniformReset) {
+        batchNode[keys::drawcallUniformReset] = batch._drawcallUniformReset;
+    }
+    if (0 != batch._textures.size()) {
+        batchNode[keys::textures] = serializePointerCache(batch._textures, textureMap);
+    }
+    if (0 != batch._textureTables.size()) {
+        batchNode[keys::textureTables] = serializePointerCache(batch._textureTables, textureTableMap);
+    }
+    if (0 != batch._buffers.size()) {
+        batchNode[keys::buffers] = serializePointerCache(batch._buffers, bufferMap);
+    }
+    if (0 != batch._pipelines.size()) {
+        batchNode[keys::pipelines] = serializePointerCache(batch._pipelines, pipelineMap);
+    }
+    if (0 != batch._streamFormats.size()) {
+        batchNode[keys::formats] = serializePointerCache(batch._streamFormats, formatMap);
+    }
+    if (0 != batch._framebuffers.size()) {
+        batchNode[keys::framebuffers] = serializePointerCache(batch._framebuffers, framebufferMap);
+    }
+    if (0 != batch._swapChains.size()) {
+        batchNode[keys::swapchains] = serializePointerCache(batch._swapChains, swapchainMap);
+    }
+    if (0 != batch._queries.size()) {
+        batchNode[keys::queries] = serializePointerCache(batch._queries, queryMap);
+    }
+    if (!batch._drawCallInfos.empty()) {
+        batchNode[keys::drawCallInfos] = writeUintVector(batch._drawCallInfos);
+    }
+    if (!batch._data.empty()) {
+        batchNode[keys::data] = toBase64(batch._data);
+    }
+
+    {
+        auto& node = batchNode[keys::commands] = json::array();
+        size_t commandCount = batch._commands.size();
+        for (size_t i = 0; i < commandCount; ++i) {
+            node.push_back(writeCommand(i, batch));
+        }
+    }
+
+    if (0 != batch._transforms.size()) {
+        batchNode[keys::transforms] =
+            serializeDataCache<Transform, json>(batch._transforms, [](const Transform& t) { return writeTransform(t); });
+    }
+    if (0 != batch._profileRanges.size()) {
+        batchNode[keys::profileRanges] = serializeDataCache<std::string>(batch._profileRanges);
+    }
+    if (0 != batch._names.size()) {
+        batchNode[keys::names] = serializeDataCache<std::string>(batch._names);
+    }
+    if (0 != batch._objects.size()) {
+        auto transform = [](const Batch::TransformObject& object) -> json { return writeMat4(object._model); };
+        batchNode[keys::objects] = writeVector<Batch::TransformObject, json>(batch._objects, transform);
+    }
+
+    if (!batch._namedData.empty()) {
+        auto& namedDataNode = batchNode[keys::namedData] = json::object();
+        for (const auto& entry : batch._namedData) {
+            namedDataNode[entry.first] = writeNamedBatchData(entry.second);
+        }
+    }
+
+    //    LambdaCache _lambdas;
+
+    return batchNode;
+}
+
+json Serializer::writeSampler(const Sampler& sampler) {
+    json result = json::object();
+    if (sampler.getBorderColor() != DEFAULT_SAMPLER.getBorderColor()) {
+        result[keys::borderColor] = writeVec4(sampler.getBorderColor());
+    }
+    if (sampler.getMaxAnisotropy() != DEFAULT_SAMPLER.getMaxAnisotropy()) {
+        result[keys::maxAnisotropy] = sampler.getMaxAnisotropy();
+    }
+    if (sampler.getWrapModeU() != DEFAULT_SAMPLER.getWrapModeU()) {
+        result[keys::wrapModeU] = sampler.getWrapModeU();
+    }
+    if (sampler.getWrapModeV() != DEFAULT_SAMPLER.getWrapModeV()) {
+        result[keys::wrapModeV] = sampler.getWrapModeV();
+    }
+    if (sampler.getWrapModeW() != DEFAULT_SAMPLER.getWrapModeW()) {
+        result[keys::wrapModeW] = sampler.getWrapModeW();
+    }
+    if (sampler.getFilter() != DEFAULT_SAMPLER.getFilter()) {
+        result[keys::filter] = sampler.getFilter();
+    }
+    if (sampler.getComparisonFunction() != DEFAULT_SAMPLER.getComparisonFunction()) {
+        result[keys::comparisonFunction] = sampler.getComparisonFunction();
+    }
+    if (sampler.getMinMip() != DEFAULT_SAMPLER.getMinMip()) {
+        result[keys::minMip] = sampler.getMinMip();
+    }
+    if (sampler.getMaxMip() != DEFAULT_SAMPLER.getMaxMip()) {
+        result[keys::maxMip] = sampler.getMaxMip();
+    }
+    if (sampler.getMipOffset() != DEFAULT_SAMPLER.getMipOffset()) {
+        result[keys::mipOffset] = sampler.getMipOffset();
+    }
+    return result;
+}
+
+json Serializer::writeBuffer(const BufferPointer& bufferPointer) {
+    if (!bufferPointer) {
+        return json();
+    }
+
+    return json(bufferPointer->getSize());
+}
+
+json Serializer::writeIrradiance(const SHPointer& irradiancePointer) {
+    if (!irradiancePointer) {
+        return json();
+    }
+
+    json result = json::object();
+    const auto& irradiance = *irradiancePointer;
+    result[keys::L00] = writeVec3(irradiance.L00);
+    result[keys::L1m1] = writeVec3(irradiance.L1m1);
+    result[keys::L10] = writeVec3(irradiance.L10);
+    result[keys::L11] = writeVec3(irradiance.L11);
+    result[keys::L2m2] = writeVec3(irradiance.L2m2);
+    result[keys::L2m1] = writeVec3(irradiance.L2m1);
+    result[keys::L20] = writeVec3(irradiance.L20);
+    result[keys::L21] = writeVec3(irradiance.L21);
+    result[keys::L22] = writeVec3(irradiance.L22);
+    return result;
+}
+
+json Serializer::writeTexture(const TexturePointer& texturePointer) {
+    if (!texturePointer) {
+        return json();
+    }
+
+    const auto& texture = *texturePointer;
+    json result = json::object();
+    if (!texture.source().empty()) {
+        result[keys::source] = texture.source();
+    }
+    const auto usageType = texture.getUsageType();
+    result[keys::usageType] = usageType;
+    result[keys::type] = texture.getType();
+    result[keys::width] = texture._width;
+    result[keys::height] = texture._height;
+    result[keys::depth] = texture._depth;
+    result[keys::mips] = texture._maxMipLevel + 1;
+    result[keys::samples] = texture._numSamples;
+    result[keys::layers] = texture._numSlices;
+    result[keys::texelFormat] = texture.getTexelFormat().getRaw();
+    if (texture.isIrradianceValid()) {
+        result["irradiance"] = writeIrradiance(texture._irradiance);
+    }
+    if (texture._sampler != DEFAULT_SAMPLER) {
+        result[keys::sampler] = writeSampler(texture._sampler);
+    }
+    if (usageType == TextureUsageType::RENDERBUFFER) {
+        // TODO figure out if the buffer contents need to be preserved (if it's used as an input before it's ever written to)
+        // This might be the case for things like the TAA output attachments from the previous frame
+    } else if (usageType == TextureUsageType::EXTERNAL) {
+        // TODO serialize the current GL contents (if any) to the JSON
+    } else {
+        const auto* storage = texture._storage.get();
+        const auto* ktxStorage = dynamic_cast<const Texture::KtxStorage*>(storage);
+        if (ktxStorage) {
+            result[keys::ktxFile] = ktxStorage->_filename;
+        } else {
+            // TODO serialize the backing storage
+        }
+    }
+    return result;
+}
+
+json Serializer::writeTextureView(const TextureView& textureView) {
+    static const auto DEFAULT_TEXTURE_VIEW = TextureView();
+    json result = json::object();
+    if (textureView._texture) {
+        result[keys::texture] = getGlobalIndex(textureView._texture, textureMap);
+    }
+    if (textureView._subresource != DEFAULT_TEXTURE_VIEW._subresource) {
+        result[keys::subresource] = textureView._subresource;
+    }
+    if (textureView._element != DEFAULT_TEXTURE_VIEW._element) {
+        result[keys::element] = textureView._element.getRaw();
+    }
+    return result;
+}
+
+json Serializer::writeFramebuffer(const FramebufferPointer& framebufferPointer) {
+    if (!framebufferPointer) {
+        return json();
+    }
+
+    auto result = json::object();
+    const auto& framebuffer = *framebufferPointer;
+    if (!framebuffer._name.empty()) {
+        result[keys::name] = framebuffer._name;
+    }
+    if (framebuffer._bufferMask != 0) {
+        result[keys::bufferMask] = framebuffer._bufferMask;
+    }
+    if (framebuffer._height != 0) {
+        result[keys::height] = framebuffer._height;
+    }
+    if (framebuffer._width != 0) {
+        result[keys::width] = framebuffer._width;
+    }
+    if (framebuffer._numSamples != 0 && framebuffer._numSamples != 1) {
+        result[keys::sampleCount] = framebuffer._numSamples;
+    }
+
+    if (framebuffer._depthStencilBuffer.isValid()) {
+        result[keys::depthStencilAttachment] = writeTextureView(framebuffer._depthStencilBuffer);
+    }
+
+    if (!framebuffer._renderBuffers.empty()) {
+        size_t rendereBufferCount = 0;
+        for (size_t i = 0; i < framebuffer._renderBuffers.size(); ++i) {
+            if (framebuffer._renderBuffers[i].isValid()) {
+                rendereBufferCount = i + 1;
+            }
+        }
+        if (rendereBufferCount > 0) {
+            auto& node = result[keys::colorAttachments] = json::array();
+            for (size_t i = 0; i < rendereBufferCount; ++i) {
+                node.push_back(writeTextureView(framebuffer._renderBuffers[i]));
+            }
+        }
+    }
+
+    //SwapchainPointer _swapchain;
+    return result;
+}
+
+json Serializer::writeTextureTable(const TextureTablePointer& textureTablePointer) {
+    auto tableNode = json::array();
+    const auto& textureTable = *textureTablePointer;
+    for (const auto& texture : textureTable.getTextures()) {
+        tableNode.push_back(getGlobalIndex(texture, textureMap));
+    }
+    return tableNode;
+}
+
+json Serializer::writeFormat(const Stream::FormatPointer& formatPointer) {
+    if (!formatPointer) {
+        return json();
+    }
+
+    const auto& format = *formatPointer;
+    json result = json::object();
+    auto& attributesNode = result[keys::attributes] = json::array();
+    static const auto DEFAULT_ATTRIBUTE = Stream::Attribute();
+    for (const auto& entry : format._attributes) {
+        const auto& attribute = entry.second;
+        auto attributeNode = json::object();
+        attributeNode[keys::slot] = attribute._slot;
+        attributeNode[keys::channel] = attribute._channel;
+        if (DEFAULT_ATTRIBUTE._element.getRaw() != attribute._element.getRaw()) {
+            attributeNode[keys::element] = attribute._element.getRaw();
+        }
+        if (DEFAULT_ATTRIBUTE._frequency != attribute._frequency) {
+            attributeNode[keys::frequency] = attribute._frequency;
+        }
+        if (DEFAULT_ATTRIBUTE._offset != attribute._offset) {
+            attributeNode[keys::offset] = attribute._offset;
+        }
+        attributesNode.push_back(attributeNode);
+    }
+    return result;
+}
+
+#define SET_IF_NOT_DEFAULT(FIELD) \
+    if (value.FIELD != DEFAULT.FIELD) { \
+        result[keys::FIELD] = value.FIELD; \
+    }
+
+#define SET_IF_NOT_DEFAULT_(FIELD) \
+    if (value._##FIELD != DEFAULT._##FIELD) { \
+        result[keys::FIELD] = value._##FIELD; \
+    }
+
+#define SET_IF_NOT_DEFAULT_TRANSFORM(FIELD, TRANSFORM) \
+    if (value.FIELD != DEFAULT.FIELD) { \
+        result[keys::FIELD] = TRANSFORM(value.FIELD); \
+    }
+
+static json writeBlendFunction(const State::BlendFunction& value) {
+    static const State::BlendFunction DEFAULT;
+    json result = json::object();
+    SET_IF_NOT_DEFAULT(enabled);
+    SET_IF_NOT_DEFAULT(sourceColor);
+    SET_IF_NOT_DEFAULT(sourceAlpha);
+    SET_IF_NOT_DEFAULT(destColor);
+    SET_IF_NOT_DEFAULT(destAlpha);
+    SET_IF_NOT_DEFAULT(destColor);
+    SET_IF_NOT_DEFAULT(opAlpha);
+    SET_IF_NOT_DEFAULT(opColor);
+    return result;
+}
+
+static json writeStateFlags(const State::Flags& value) {
+    static const State::Flags DEFAULT;
+    json result = json::object();
+    SET_IF_NOT_DEFAULT(frontFaceClockwise);
+    SET_IF_NOT_DEFAULT(depthClampEnable);
+    SET_IF_NOT_DEFAULT(scissorEnable);
+    SET_IF_NOT_DEFAULT(multisampleEnable);
+    SET_IF_NOT_DEFAULT(antialisedLineEnable);
+    SET_IF_NOT_DEFAULT(alphaToCoverageEnable);
+    return result;
+}
+
+static json writeDepthTest(const State::DepthTest& value) {
+    static const State::DepthTest DEFAULT;
+    json result = json::object();
+    SET_IF_NOT_DEFAULT(writeMask);
+    SET_IF_NOT_DEFAULT(enabled);
+    SET_IF_NOT_DEFAULT(function);
+    return result;
+}
+
+
+static json writeStereoState(const StereoState& value) {
+    static const StereoState DEFAULT;
+    json result = json::object();
+    SET_IF_NOT_DEFAULT_(enable);
+    SET_IF_NOT_DEFAULT_(contextDisable);
+    SET_IF_NOT_DEFAULT_(skybox);
+    if ((value._eyeProjections[0] != DEFAULT._eyeProjections[0]) || (value._eyeProjections[1] != DEFAULT._eyeProjections[1])) {
+        json projections = json::array();
+        projections.push_back(Serializer::writeMat4(value._eyeProjections[0]));
+        projections.push_back(Serializer::writeMat4(value._eyeProjections[1]));
+        result[keys::eyeProjections] = projections;
+    }
+    if ((value._eyeViews[0] != DEFAULT._eyeViews[0]) || (value._eyeViews[1] != DEFAULT._eyeViews[1])) {
+        json views = json::array();
+        views.push_back(Serializer::writeMat4(value._eyeViews[0]));
+        views.push_back(Serializer::writeMat4(value._eyeViews[1]));
+        result[keys::eyeViews] = views;
+    }
+    return result;
+}
+
+static json writeStencilTest(const State::StencilTest& value) {
+    static const State::StencilTest DEFAULT;
+    json result = json::object();
+    SET_IF_NOT_DEFAULT(function);
+    SET_IF_NOT_DEFAULT(failOp);
+    SET_IF_NOT_DEFAULT(depthFailOp);
+    SET_IF_NOT_DEFAULT(passOp);
+    SET_IF_NOT_DEFAULT(reference);
+    SET_IF_NOT_DEFAULT(readMask);
+    return result;
+}
+
+static json writeStencilActivation(const State::StencilActivation& value) {
+    static const State::StencilActivation DEFAULT;
+    json result = json::object();
+    SET_IF_NOT_DEFAULT(frontWriteMask);
+    SET_IF_NOT_DEFAULT(backWriteMask);
+    SET_IF_NOT_DEFAULT(enabled);
+    return result;
+}
+
+json writeState(const StatePointer& statePointer) {
+    if (!statePointer) {
+        return json();
+    }
+    const auto& state = *statePointer;
+    const auto& value = state.getValues();
+    const auto& DEFAULT = State::DEFAULT;
+    auto result = json::object();
+    SET_IF_NOT_DEFAULT(colorWriteMask);
+    SET_IF_NOT_DEFAULT(cullMode);
+    SET_IF_NOT_DEFAULT(depthBias);
+    SET_IF_NOT_DEFAULT(depthBiasSlopeScale);
+    SET_IF_NOT_DEFAULT(fillMode);
+    SET_IF_NOT_DEFAULT(sampleMask);
+    SET_IF_NOT_DEFAULT_TRANSFORM(blendFunction, writeBlendFunction);
+    SET_IF_NOT_DEFAULT_TRANSFORM(flags, writeStateFlags);
+    SET_IF_NOT_DEFAULT_TRANSFORM(depthTest, writeDepthTest);
+    SET_IF_NOT_DEFAULT_TRANSFORM(stencilActivation, writeStencilActivation);
+    SET_IF_NOT_DEFAULT_TRANSFORM(stencilTestFront, writeStencilTest);
+    SET_IF_NOT_DEFAULT_TRANSFORM(stencilTestBack, writeStencilTest);
+    return result;
+}
+
+json Serializer::writePipeline(const PipelinePointer& pipelinePointer) {
+    if (!pipelinePointer) {
+        return json();
+    }
+    const auto& pipeline = *pipelinePointer;
+    auto result = json::object();
+    result[keys::state] = writeState(pipeline.getState());
+    result[keys::program] = getGlobalIndex(pipeline.getProgram(), programMap);
+    return result;
+}
+
+json Serializer::writeProgram(const ShaderPointer& programPointer) {
+    if (!programPointer) {
+        return json();
+    }
+    const auto& program = *programPointer;
+    auto result = json::array();
+    for (const auto& shader : program._shaders) {
+        result.push_back(getGlobalIndex(shader, shaderMap));
+    }
+    return result;
+}
+
+json Serializer::writeShader(const ShaderPointer& shaderPointer) {
+    if (!shaderPointer) {
+        return json();
+    }
+    auto result = json::object();
+    const auto& shader = *shaderPointer;
+    result[keys::id] = shader._source.id;
+    result[keys::name] = shader._source.name;
+    result[keys::type] = shader._type;
+    return result;
+}
+
+json Serializer::writeSwapchain(const SwapChainPointer& swapchainPointer) {
+    auto framebufferSwapchainPointer = std::static_pointer_cast<FramebufferSwapChain>(swapchainPointer);
+    if (!framebufferSwapchainPointer) {
+        return json();
+    }
+    const FramebufferSwapChain& swapchain = *framebufferSwapchainPointer;
+    auto result = json::object();
+    result[keys::size] = swapchain.getSize();
+    auto& framebuffersNode = result[keys::framebuffers] = json::array();
+    for (uint32_t i = 0; i < swapchain.getSize(); ++i) {
+        uint32_t index = getGlobalIndex(swapchain.get(i), framebufferMap);
+        framebuffersNode.push_back(index);
+    }
+    return result;
+}
+
+json Serializer::writeQuery(const QueryPointer& queryPointer) {
+    if (!queryPointer) {
+        return json();
+    }
+
+    const auto& query = *queryPointer;
+    auto result = json::object();
+    result[keys::name] = query._name;
+    return result;
+}
+
+json Serializer::writeCapturableTextures(const Frame& frame) {
+    if (!textureCapturer) {
+        return json::array();
+    }
+
+    std::unordered_set<TexturePointer> writtenRenderbuffers;
+    std::unordered_set<TexturePointer> captureTextures;
+
+    auto maybeCaptureTexture = [&](const TexturePointer& texture) {
+        // Not a valid texture
+        if (!texture) {
+            return;
+        }
+
+        // Not a renderbuffer
+        if (texture->getUsageType() != TextureUsageType::RENDERBUFFER) {
+            return;
+        }
+
+        // Already used in a target framebuffer
+        if (writtenRenderbuffers.count(texture) > 0) {
+            return;
+        }
+
+        captureTextures.insert(texture);
+    };
+
+    for (const auto& batchPtr : frame.batches) {
+        const auto& batch = *batchPtr;
+        batch.forEachCommand([&](Batch::Command command, const Batch::Param* params) {
+            switch (command) {
+                case Batch::COMMAND_setResourceTexture: {
+                    const auto& texture = batch._textures.get(params[0]._uint);
+                    maybeCaptureTexture(texture);
+                } break;
+
+                case Batch::COMMAND_setResourceTextureTable: {
+                    const auto& textureTablePointer = batch._textureTables.get(params[0]._uint);
+                    if (textureTablePointer) {
+                        for (const auto& texture : textureTablePointer->getTextures()) {
+                            maybeCaptureTexture(texture);
+                        }
+                    }
+                } break;
+
+                case Batch::COMMAND_setFramebuffer: {
+                    const auto& framebufferPointer = batch._framebuffers.get(params[0]._uint);
+                    if (framebufferPointer) {
+                        const auto& framebuffer = *framebufferPointer;
+                        for (const auto& colorAttachment : framebuffer._renderBuffers) {
+                            if (colorAttachment._texture) {
+                                writtenRenderbuffers.insert(colorAttachment._texture);
+                            }
+                        }
+                        if (framebuffer._depthStencilBuffer._texture) {
+                            writtenRenderbuffers.insert(framebuffer._depthStencilBuffer._texture);
+                        }
+                    }
+                }
+
+                case Batch::COMMAND_setResourceFramebufferSwapChainTexture:
+                default:
+                    break;
+            }
+        });  // for each command
+    }        // for each batch
+
+    for (const auto& entry : textureMap) {
+        const auto& texturePointer = entry.first;
+        if (!texturePointer) {
+            continue;
+        }
+        const auto& texture = *texturePointer;
+        auto usageType = texture.getUsageType();
+        if (usageType == TextureUsageType::RESOURCE || usageType == TextureUsageType::STRICT_RESOURCE) {
+            const auto* storage = texture._storage.get();
+            const auto* ktxStorage = dynamic_cast<const Texture::KtxStorage*>(storage);
+            if (!ktxStorage) {
+                captureTextures.insert(texturePointer);
+            }
+        }
+    }
+
+    json result = json::array();
+    for (const auto& texture : captureTextures) {
+        if (textureCapturer) {
+            auto index = textureMap[texture];
+            auto layers = std::max<uint16>(texture->getNumSlices(), 1);
+            for (uint16 layer = 0; layer < layers; ++layer) {
+                std::string textureFilename = basename + "." + std::to_string(index) + "." + std::to_string(layer) + ".png";
+                textureCapturer(textureFilename, texture, layer);
+            }
+            result.push_back(index);
+        }
+    }
+    return result;
+}
+
+void Serializer::writeFrame(const Frame& frame) {
+    json frameNode = json::object();
+
+    frameNode[keys::batches] = json::array();
+    for (const auto& batchPointer : frame.batches) {
+        frameNode[keys::batches].push_back(writeBatch(*batchPointer));
+    }
+
+    frameNode[keys::stereo] = writeStereoState(frame.stereoState);
+    frameNode[keys::capturedTextures] = writeCapturableTextures(frame);
+    frameNode[keys::frameIndex] = frame.frameIndex;
+    frameNode[keys::view] = writeMat4(frame.view);
+    frameNode[keys::pose] = writeMat4(frame.pose);
+    frameNode[keys::framebuffer] = getGlobalIndex(frame.framebuffer, framebufferMap);
+    using namespace std::placeholders;
+    serializeMap(frameNode, keys::swapchains, swapchainMap, std::bind(&Serializer::writeSwapchain, this, _1));
+    serializeMap(frameNode, keys::framebuffers, framebufferMap, [this](const auto& t) { return writeFramebuffer(t); });
+    serializeMap(frameNode, keys::textureTables, textureTableMap, [this](const auto& t) { return writeTextureTable(t); });
+    serializeMap(frameNode, keys::pipelines, pipelineMap, [this](const auto& t) { return writePipeline(t); });
+    serializeMap(frameNode, keys::programs, programMap, [this](const auto& t) { return writeProgram(t); });
+    serializeMap(frameNode, keys::shaders, shaderMap, writeShader);
+    serializeMap(frameNode, keys::queries, queryMap, writeQuery);
+    serializeMap(frameNode, keys::formats, formatMap, writeFormat);
+
+    // Serialize textures and buffers last, since the maps they use can be populated by some of the above code
+    // Serialize textures
+    serializeMap(frameNode, keys::textures, textureMap, writeTexture);
+    // Serialize buffers
+    serializeMap(frameNode, keys::buffers, bufferMap, writeBuffer);
+
+    {
+        std::string frameJson = frameNode.dump();
+        std::string filename = basename + ".json";
+        storage::FileStorage::create(filename.c_str(), frameJson.size(), (const uint8_t*)frameJson.data());
+    }
+
+    writeBinaryBlob();
+    frameNode[keys::binary] = basename + ".bin";
+}
+
+void Serializer::writeBinaryBlob() {
+    const auto buffers = mapToVector(bufferMap);
+    auto accumulator = [](size_t total, const BufferPointer& buffer) { return total + (buffer ? buffer->getSize() : 0); };
+    size_t totalSize = std::accumulate(buffers.begin(), buffers.end(), (size_t)0, accumulator);
+
+    const auto blobFilename = basename + ".bin";
+    QFile file(blobFilename.c_str());
+    if (!file.open(QFile::ReadWrite | QIODevice::Truncate)) {
+        throw std::runtime_error("Unable to open file for writing");
+    }
+    if (!file.resize(totalSize)) {
+        throw std::runtime_error("Unable to resize file");
+    }
+
+    auto mapped = file.map(0, totalSize);
+    size_t offset = 0;
+
+    for (const auto& bufferPointer : buffers) {
+        if (!bufferPointer) {
+            continue;
+        }
+        const auto& buffer = *bufferPointer;
+        const auto bufferSize = buffer.getSize();
+        const auto& bufferData = buffer._renderSysmem.readData();
+        memcpy(mapped + offset, bufferData, bufferSize);
+        offset += bufferSize;
+    }
+    if (!file.unmap(mapped)) {
+        throw std::runtime_error("Unable to unmap file");
+    }
+}

libraries/gpu/src/gpu/Framebuffer.h

@@ -173,6 +173,8 @@ protected:
     void updateSize(const TexturePointer& texture);
     bool assignDepthStencilBuffer(const TexturePointer& texture, const Format& format, uint32 subresource);
 
+    friend class Serializer;
+    friend class Deserializer;
     // Non exposed
     Framebuffer(const Framebuffer& framebuffer) = delete;
     Framebuffer() {}

libraries/gpu/src/gpu/PointerStorage.h

@@ -0,0 +1,159 @@
+//
+//  Created by Bradley Austin Davis on 2019/01/25
+//  Copyright 2013-2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#pragma once
+#ifndef hifi_gpu_PointerStorage_h
+#define hifi_gpu_PointerStorage_h
+
+namespace gpu {
+
+//
+// GL Backend pointer storage mechanism
+// One of the following three defines must be defined.
+// GPU_POINTER_STORAGE_SHARED
+
+// The platonic ideal, use references to smart pointers.
+// However, this produces artifacts because there are too many places in the code right now that
+// create temporary values (undesirable smart pointer duplications) and then those temp variables
+// get passed on and have their reference taken, and then invalidated
+// GPU_POINTER_STORAGE_REF
+
+// Raw pointer manipulation.  Seems more dangerous than the reference wrappers,
+// but in practice, the danger of grabbing a reference to a temporary variable
+// is causing issues
+// GPU_POINTER_STORAGE_RAW
+
+#if defined(GPU_POINTER_STORAGE_SHARED)
+template <typename T>
+static inline bool compare(const std::shared_ptr<T>& a, const std::shared_ptr<T>& b) {
+    return a == b;
+}
+
+template <typename T>
+static inline T* acquire(const std::shared_ptr<T>& pointer) {
+    return pointer.get();
+}
+
+template <typename T>
+static inline void reset(std::shared_ptr<T>& pointer) {
+    return pointer.reset();
+}
+
+template <typename T>
+static inline bool valid(const std::shared_ptr<T>& pointer) {
+    return pointer.operator bool();
+}
+
+template <typename T>
+static inline void assign(std::shared_ptr<T>& pointer, const std::shared_ptr<T>& source) {
+    pointer = source;
+}
+
+using BufferReference = BufferPointer;
+using TextureReference = TexturePointer;
+using FramebufferReference = FramebufferPointer;
+using FormatReference = Stream::FormatPointer;
+using PipelineReference = PipelinePointer;
+
+#define GPU_REFERENCE_INIT_VALUE nullptr
+
+#elif defined(GPU_POINTER_STORAGE_REF)
+
+template <typename T>
+class PointerReferenceWrapper : public std::reference_wrapper<const std::shared_ptr<T>> {
+    using Parent = std::reference_wrapper<const std::shared_ptr<T>>;
+
+public:
+    using Pointer = std::shared_ptr<T>;
+    PointerReferenceWrapper() : Parent(EMPTY()) {}
+    PointerReferenceWrapper(const Pointer& pointer) : Parent(pointer) {}
+    void clear() { *this = EMPTY(); }
+
+private:
+    static const Pointer& EMPTY() {
+        static const Pointer EMPTY_VALUE;
+        return EMPTY_VALUE;
+    };
+};
+
+template <typename T>
+static bool compare(const PointerReferenceWrapper<T>& reference, const std::shared_ptr<T>& pointer) {
+    return reference.get() == pointer;
+}
+
+template <typename T>
+static inline T* acquire(const PointerReferenceWrapper<T>& reference) {
+    return reference.get().get();
+}
+
+template <typename T>
+static void assign(PointerReferenceWrapper<T>& reference, const std::shared_ptr<T>& pointer) {
+    reference = pointer;
+}
+
+template <typename T>
+static bool valid(const PointerReferenceWrapper<T>& reference) {
+    return reference.get().operator bool();
+}
+
+template <typename T>
+static inline void reset(PointerReferenceWrapper<T>& reference) {
+    return reference.clear();
+}
+
+using BufferReference = PointerReferenceWrapper<Buffer>;
+using TextureReference = PointerReferenceWrapper<Texture>;
+using FramebufferReference = PointerReferenceWrapper<Framebuffer>;
+using FormatReference = PointerReferenceWrapper<Stream::Format>;
+using PipelineReference = PointerReferenceWrapper<Pipeline>;
+
+#define GPU_REFERENCE_INIT_VALUE
+
+#elif defined(GPU_POINTER_STORAGE_RAW)
+
+template <typename T>
+static bool compare(const T* const& rawPointer, const std::shared_ptr<T>& pointer) {
+    return rawPointer == pointer.get();
+}
+
+template <typename T>
+static inline T* acquire(T* const& rawPointer) {
+    return rawPointer;
+}
+
+template <typename T>
+static inline bool valid(const T* const& rawPointer) {
+    return rawPointer;
+}
+
+template <typename T>
+static inline void reset(T*& rawPointer) {
+    rawPointer = nullptr;
+}
+
+template <typename T>
+static inline void assign(T*& rawPointer, const std::shared_ptr<T>& pointer) {
+    rawPointer = pointer.get();
+}
+
+using BufferReference = Buffer*;
+using TextureReference = Texture*;
+using FramebufferReference = Framebuffer*;
+using FormatReference = Stream::Format*;
+using PipelineReference = Pipeline*;
+
+#define GPU_REFERENCE_INIT_VALUE nullptr
+
+#else
+
+#error "No GPU pointer storage scheme defined"
+
+#endif
+
+}
+
+#endif // hifi_gpu_PointerStorage_h

libraries/gpu/src/gpu/Query.h

@@ -45,6 +45,9 @@ namespace gpu {
         const std::string _name;
         uint64_t _queryResult { 0 };
         uint64_t _usecBatchElapsedTime { 0 };
+
+        friend class Serializer;
+        friend class Deserializer;
     };
     
     typedef std::shared_ptr<Query> QueryPointer;

libraries/gpu/src/gpu/Shader.h

@@ -167,6 +167,8 @@ protected:
                                                     const Pointer& vertexShader,
                                                     const Pointer& geometryShader,
                                                     const Pointer& pixelShader);
+    friend class Serializer;
+    friend class Deserializer;
 };
 
 typedef Shader::Pointer ShaderPointer;

libraries/gpu/src/gpu/State.cpp

@@ -33,19 +33,19 @@ State::Signature State::evalSignature(const Data& state) {
     if (state.cullMode != State::DEFAULT.cullMode) {
         signature.set(State::CULL_MODE);
     }
-    if (state.frontFaceClockwise != State::DEFAULT.frontFaceClockwise) {
+    if (state.flags.frontFaceClockwise != State::DEFAULT.flags.frontFaceClockwise) {
         signature.set(State::FRONT_FACE_CLOCKWISE);
     }
-    if (state.depthClampEnable != State::DEFAULT.depthClampEnable) {
+    if (state.flags.depthClampEnable != State::DEFAULT.flags.depthClampEnable) {
         signature.set(State::DEPTH_CLAMP_ENABLE);
     }
-    if (state.scissorEnable != State::DEFAULT.scissorEnable) {
+    if (state.flags.scissorEnable != State::DEFAULT.flags.scissorEnable) {
         signature.set(State::SCISSOR_ENABLE);
     }
-    if (state.multisampleEnable != State::DEFAULT.multisampleEnable) {
+    if (state.flags.multisampleEnable != State::DEFAULT.flags.multisampleEnable) {
         signature.set(State::MULTISAMPLE_ENABLE);
     }
-    if (state.antialisedLineEnable != State::DEFAULT.antialisedLineEnable) {
+    if (state.flags.antialisedLineEnable != State::DEFAULT.flags.antialisedLineEnable) {
         signature.set(State::ANTIALISED_LINE_ENABLE);
     }
     if (state.depthBias != State::DEFAULT.depthBias) {
@@ -69,7 +69,7 @@ State::Signature State::evalSignature(const Data& state) {
     if (state.sampleMask != State::DEFAULT.sampleMask) {
         signature.set(State::SAMPLE_MASK);
     }
-    if (state.alphaToCoverageEnable != State::DEFAULT.alphaToCoverageEnable) {
+    if (state.flags.alphaToCoverageEnable != State::DEFAULT.flags.alphaToCoverageEnable) {
         signature.set(State::ALPHA_TO_COVERAGE_ENABLE); 
     }
     if (state.blendFunction != State::DEFAULT.blendFunction) {
@@ -86,3 +86,28 @@ State::State(const Data& values) :
     _values(values) {
     _signature = evalSignature(_values);
 }
+
+
+template <typename T>
+static std::string hex(T t) {
+    std::stringstream stream;
+    stream << std::hex << t;
+    return stream.str();
+}
+
+std::string State::getKey() const {
+    std::string key;
+    key = hex(*(int*)&_values.depthBias);
+    key += ":" + hex(*(int*)&_values.depthBiasSlopeScale);
+    key += ":" + hex(_values.depthTest.getRaw());
+    key += ":" + hex(_values.stencilActivation.getRaw());
+    key += ":" + hex(_values.stencilTestFront.getRaw());
+    key += ":" + hex(_values.stencilTestBack.getRaw());
+    key += ":" + hex(_values.blendFunction.getRaw());
+    key += ":" + hex(_values.sampleMask);
+    // fillMode, cullMode, colorMaskWrite and the flags consume 32 bits alltogether
+    static_assert(0 == offsetof(State::Data, fillMode) % 4, "Validate fillMode offset");
+    key += ":" + hex(*(int*)&_values.fillMode);
+    return key;
+}
+

libraries/gpu/src/gpu/State.h

@@ -14,6 +14,7 @@
 #include "Format.h"
 
 #include <memory>
+#include <sstream>
 #include <vector>
 #include <unordered_map>
 #include <bitset>
@@ -21,16 +22,16 @@
 // Why a macro and not a fancy template you will ask me ?
 // Because some of the fields are bool packed tightly in the State::Cache class
 // and it s just not good anymore for template T& variable manipulation...
-#define SET_FIELD(field, defaultValue, value, dest) {\
-    dest = value;\
-    if (value == defaultValue) {\
-        _signature.reset(field);\
-    } else {\
-        _signature.set(field);\
-    }\
-    _stamp++;\
-}\
-
+#define SET_FIELD(FIELD, PATH, value) \
+    {                                 \
+        _values.PATH = value;         \
+        if (value == DEFAULT.PATH) {  \
+            _signature.reset(FIELD);  \
+        } else {                      \
+            _signature.set(FIELD);    \
+        }                             \
+        _stamp++;                     \
+    }
 
 namespace gpu {
 
@@ -45,7 +46,8 @@ public:
 
     typedef ::gpu::ComparisonFunction ComparisonFunction;
 
-    enum FillMode {
+    enum FillMode : uint8
+    {
         FILL_POINT = 0,
         FILL_LINE,
         FILL_FACE,
@@ -53,7 +55,8 @@ public:
         NUM_FILL_MODES,
     };
 
-    enum CullMode {
+    enum CullMode : uint8
+    {
         CULL_NONE = 0,
         CULL_FRONT,
         CULL_BACK,
@@ -61,7 +64,8 @@ public:
         NUM_CULL_MODES,
     };
 
-    enum StencilOp {
+    enum StencilOp : uint16
+    {
         STENCIL_OP_KEEP = 0,
         STENCIL_OP_ZERO,
         STENCIL_OP_REPLACE,
@@ -74,7 +78,8 @@ public:
         NUM_STENCIL_OPS,
     };
 
-    enum BlendArg {
+    enum BlendArg : uint16
+    {
         ZERO = 0,
         ONE,
         SRC_COLOR,
@@ -94,7 +99,8 @@ public:
         NUM_BLEND_ARGS,
     };
 
-    enum BlendOp {
+    enum BlendOp : uint16
+    {
         BLEND_OP_ADD = 0,
         BLEND_OP_SUBTRACT,
         BLEND_OP_REV_SUBTRACT,
@@ -104,132 +110,158 @@ public:
         NUM_BLEND_OPS,
     };
 
-    enum ColorMask
+    enum ColorMask : uint8
     {
         WRITE_NONE = 0,
         WRITE_RED = 1,
         WRITE_GREEN = 2,
         WRITE_BLUE = 4,
         WRITE_ALPHA = 8,
-        WRITE_ALL = (WRITE_RED | WRITE_GREEN | WRITE_BLUE | WRITE_ALPHA ),
+        WRITE_ALL = (WRITE_RED | WRITE_GREEN | WRITE_BLUE | WRITE_ALPHA),
     };
 
     class DepthTest {
-        uint8 _function = LESS;
-        uint8 _writeMask = true;
-        uint8 _enabled = false;
-#if defined(__clang__)
-        __attribute__((unused))
-#endif
-        uint8 _spare = 0; // Padding
+    public:
+        uint8 writeMask{ true };
+        uint8 enabled{ false };
+        ComparisonFunction function{ LESS };
+
     public:
         DepthTest(bool enabled = false, bool writeMask = true, ComparisonFunction func = LESS) :
-            _function(func), _writeMask(writeMask), _enabled(enabled) {
-            }
+            writeMask(writeMask), enabled(enabled), function(func) {}
 
-        bool isEnabled() const { return _enabled != 0; }
-        ComparisonFunction getFunction() const { return ComparisonFunction(_function); }
-        uint8 getWriteMask() const { return _writeMask; }
+        bool isEnabled() const { return enabled != 0; }
+        ComparisonFunction getFunction() const { return function; }
+        uint8 getWriteMask() const { return writeMask; }
 
         int32 getRaw() const { return *(reinterpret_cast<const int32*>(this)); }
         DepthTest(int32 raw) { *(reinterpret_cast<int32*>(this)) = raw; }
-        bool operator== (const DepthTest& right) const { return getRaw() == right.getRaw(); }
-        bool operator!= (const DepthTest& right) const { return getRaw() != right.getRaw(); }
-   };
+        bool operator==(const DepthTest& right) const { return getRaw() == right.getRaw(); }
+        bool operator!=(const DepthTest& right) const { return getRaw() != right.getRaw(); }
+    };
 
-     class StencilTest {
-        static const int FUNC_MASK = 0x000f;
-        static const int FAIL_OP_MASK = 0x00f0;
-        static const int DEPTH_FAIL_OP_MASK = 0x0f00;
-        static const int PASS_OP_MASK = 0xf000;
-        static const int FAIL_OP_OFFSET = 4;
-        static const int DEPTH_FAIL_OP_OFFSET = 8;
-        static const int PASS_OP_OFFSET = 12;
+    static_assert(sizeof(DepthTest) == sizeof(uint32_t), "DepthTest size check");
 
-        uint16 _functionAndOperations;
-        int8 _reference = 0;
-        uint8 _readMask = 0xff;
-     public:
+    struct StencilTest {
+        ComparisonFunction function : 4;
+        StencilOp failOp : 4;
+        StencilOp depthFailOp : 4;
+        StencilOp passOp : 4;
+        int8 reference{ 0 };
+        uint8 readMask{ 0xff };
 
-        StencilTest(int8 reference = 0, uint8 readMask =0xFF, ComparisonFunction func = ALWAYS, StencilOp failOp = STENCIL_OP_KEEP, StencilOp depthFailOp = STENCIL_OP_KEEP, StencilOp passOp = STENCIL_OP_KEEP) :
-             _functionAndOperations(func | (failOp << FAIL_OP_OFFSET) | (depthFailOp << DEPTH_FAIL_OP_OFFSET) | (passOp << PASS_OP_OFFSET)),
-            _reference(reference), _readMask(readMask)
-            {}
+    public:
+        StencilTest(int8 reference = 0,
+                    uint8 readMask = 0xFF,
+                    ComparisonFunction func = ALWAYS,
+                    StencilOp failOp = STENCIL_OP_KEEP,
+                    StencilOp depthFailOp = STENCIL_OP_KEEP,
+                    StencilOp passOp = STENCIL_OP_KEEP) :
+            function(func),
+            failOp(failOp), depthFailOp(depthFailOp), passOp(passOp), reference(reference), readMask(readMask) {}
 
-        ComparisonFunction getFunction() const { return ComparisonFunction(_functionAndOperations & FUNC_MASK); }
-        StencilOp getFailOp() const { return StencilOp((_functionAndOperations & FAIL_OP_MASK) >> FAIL_OP_OFFSET); }
-        StencilOp getDepthFailOp() const { return StencilOp((_functionAndOperations & DEPTH_FAIL_OP_MASK) >> DEPTH_FAIL_OP_OFFSET); }
-        StencilOp getPassOp() const { return StencilOp((_functionAndOperations & PASS_OP_MASK) >> PASS_OP_OFFSET); }
+        ComparisonFunction getFunction() const { return function; }
+        StencilOp getFailOp() const { return failOp; }
+        StencilOp getDepthFailOp() const { return depthFailOp; }
+        StencilOp getPassOp() const { return passOp; }
 
-        int8 getReference() const { return _reference; }
-        uint8 getReadMask() const { return _readMask; }
+        int8 getReference() const { return reference; }
+        uint8 getReadMask() const { return readMask; }
 
         int32 getRaw() const { return *(reinterpret_cast<const int32*>(this)); }
         StencilTest(int32 raw) { *(reinterpret_cast<int32*>(this)) = raw; }
-        bool operator== (const StencilTest& right) const { return getRaw() == right.getRaw(); }
-        bool operator!= (const StencilTest& right) const { return getRaw() != right.getRaw(); }
-     };
+        bool operator==(const StencilTest& right) const { return getRaw() == right.getRaw(); }
+        bool operator!=(const StencilTest& right) const { return getRaw() != right.getRaw(); }
+    };
+    static_assert(sizeof(StencilTest) == sizeof(uint32_t), "StencilTest size check");
 
-     class StencilActivation {
-        uint8 _frontWriteMask = 0xFF;
-        uint8 _backWriteMask = 0xFF;
-        uint16 _enabled = 0;
-     public:
+    StencilTest stencilTestFront;
 
-        StencilActivation(bool enabled, uint8 frontWriteMask = 0xFF, uint8 backWriteMask = 0xFF) :
-            _frontWriteMask(frontWriteMask), _backWriteMask(backWriteMask), _enabled(enabled) {}
+    struct StencilActivation {
+        uint8 frontWriteMask = 0xFF;
+        uint8 backWriteMask = 0xFF;
+        bool enabled : 1;
+        uint8 _spare1 : 7;
+        uint8 _spare2{ 0 };
 
-        bool isEnabled() const { return (_enabled != 0); }
-        uint8 getWriteMaskFront() const { return _frontWriteMask; }
-        uint8 getWriteMaskBack() const { return _backWriteMask; }
+    public:
+        StencilActivation(bool enabled = false, uint8 frontWriteMask = 0xFF, uint8 backWriteMask = 0xFF) :
+            frontWriteMask(frontWriteMask), backWriteMask(backWriteMask), enabled(enabled), _spare1{ 0 } {}
+
+        bool isEnabled() const { return enabled; }
+        uint8 getWriteMaskFront() const { return frontWriteMask; }
+        uint8 getWriteMaskBack() const { return backWriteMask; }
 
         int32 getRaw() const { return *(reinterpret_cast<const int32*>(this)); }
         StencilActivation(int32 raw) { *(reinterpret_cast<int32*>(this)) = raw; }
-        bool operator== (const StencilActivation& right) const { return getRaw() == right.getRaw(); }
-        bool operator!= (const StencilActivation& right) const { return getRaw() != right.getRaw(); }
+        bool operator==(const StencilActivation& right) const { return getRaw() == right.getRaw(); }
+        bool operator!=(const StencilActivation& right) const { return getRaw() != right.getRaw(); }
     };
 
-    class BlendFunction {
-        static const int COLOR_MASK = 0x0f;
-        static const int ALPHA_MASK = 0xf0;
-        static const int ALPHA_OFFSET = 4;
+    static_assert(sizeof(StencilActivation) == sizeof(uint32_t), "StencilActivation size check");
+
+    struct BlendFunction {
+        // Using uint8 here will make the structure as a whole not align to 32 bits
+        uint16 enabled : 8;
+        BlendArg sourceColor : 4;
+        BlendArg sourceAlpha : 4;
+        BlendArg destColor : 4;
+        BlendArg destAlpha : 4;
+        BlendOp opColor : 4;
+        BlendOp opAlpha : 4;
 
-        uint8 _enabled;
-        uint8 _source;
-        uint8 _destination;
-        uint8 _operation;
     public:
-
         BlendFunction(bool enabled,
-            BlendArg sourceColor, BlendOp operationColor, BlendArg destinationColor,
-            BlendArg sourceAlpha, BlendOp operationAlpha, BlendArg destinationAlpha) :
-            _enabled(enabled),
-            _source(sourceColor | (sourceAlpha << ALPHA_OFFSET)),
-            _destination(destinationColor | (destinationAlpha << ALPHA_OFFSET)),
-            _operation(operationColor | (operationAlpha << ALPHA_OFFSET)) {}
+                      BlendArg sourceColor,
+                      BlendOp operationColor,
+                      BlendArg destinationColor,
+                      BlendArg sourceAlpha,
+                      BlendOp operationAlpha,
+                      BlendArg destinationAlpha) :
+            enabled(enabled),
+            sourceColor(sourceColor), sourceAlpha(sourceAlpha), 
+            destColor(destinationColor), destAlpha(destinationAlpha),
+            opColor(operationColor), opAlpha(operationAlpha) {}
 
-        BlendFunction(bool enabled, BlendArg source = ONE, BlendOp operation = BLEND_OP_ADD, BlendArg destination = ZERO) :
-            _enabled(enabled),
-            _source(source | (source << ALPHA_OFFSET)),
-            _destination(destination | (destination << ALPHA_OFFSET)),
-            _operation(operation | (operation << ALPHA_OFFSET)) {}
+        BlendFunction(bool enabled = false, BlendArg source = ONE, BlendOp operation = BLEND_OP_ADD, BlendArg destination = ZERO) :
+            BlendFunction(enabled, source, operation, destination, source, operation, destination) {}
 
-        bool isEnabled() const { return (_enabled != 0); }
+        bool isEnabled() const { return (enabled != 0); }
 
-        BlendArg getSourceColor() const { return BlendArg(_source & COLOR_MASK); }
-        BlendArg getDestinationColor() const { return BlendArg(_destination & COLOR_MASK); }
-        BlendOp getOperationColor() const { return BlendOp(_operation & COLOR_MASK); }
+        BlendArg getSourceColor() const { return sourceColor; }
+        BlendArg getDestinationColor() const { return destColor; }
+        BlendOp getOperationColor() const { return opColor; }
 
-        BlendArg getSourceAlpha() const { return BlendArg((_source & ALPHA_MASK) >> ALPHA_OFFSET); }
-        BlendArg getDestinationAlpha() const { return BlendArg((_destination & ALPHA_MASK) >> ALPHA_OFFSET); }
-        BlendOp getOperationAlpha() const { return BlendOp((_operation & ALPHA_MASK) >> ALPHA_OFFSET); }
+        BlendArg getSourceAlpha() const { return sourceAlpha; }
+        BlendArg getDestinationAlpha() const { return destAlpha; }
+        BlendOp getOperationAlpha() const { return opAlpha; }
 
         int32 getRaw() const { return *(reinterpret_cast<const int32*>(this)); }
         BlendFunction(int32 raw) { *(reinterpret_cast<int32*>(this)) = raw; }
-        bool operator== (const BlendFunction& right) const { return getRaw() == right.getRaw(); }
-        bool operator!= (const BlendFunction& right) const { return getRaw() != right.getRaw(); }
+        bool operator==(const BlendFunction& right) const { return getRaw() == right.getRaw(); }
+        bool operator!=(const BlendFunction& right) const { return getRaw() != right.getRaw(); }
     };
 
+    static_assert(sizeof(BlendFunction) == sizeof(uint32_t), "BlendFunction size check");
+
+    struct Flags {
+        Flags() :
+            frontFaceClockwise(false), depthClampEnable(false), scissorEnable(false), multisampleEnable(false),
+            antialisedLineEnable(true), alphaToCoverageEnable(false), _spare1(0) {}
+        bool frontFaceClockwise : 1;
+        bool depthClampEnable : 1;
+        bool scissorEnable : 1;
+        bool multisampleEnable : 1;
+        bool antialisedLineEnable : 1;
+        bool alphaToCoverageEnable : 1;
+        uint8 _spare1 : 2;
+
+        bool operator==(const Flags& right) const { return *(uint8*)this == *(uint8*)&right; }
+        bool operator!=(const Flags& right) const { return *(uint8*)this != *(uint8*)&right; }
+    };
+
+    static_assert(sizeof(Flags) == sizeof(uint8), "Flags size check");
+
     // The Data class is the full explicit description of the State class fields value.
     // Useful for having one const static called Default for reference or for the gpu::Backend to keep track of the current value
     class Data {
@@ -237,71 +269,72 @@ public:
         float depthBias = 0.0f;
         float depthBiasSlopeScale = 0.0f;
 
-        DepthTest depthTest = DepthTest(false, true, LESS);
-
-        StencilActivation stencilActivation = StencilActivation(false);
-        StencilTest stencilTestFront = StencilTest(0, 0xff, ALWAYS, STENCIL_OP_KEEP, STENCIL_OP_KEEP, STENCIL_OP_KEEP);
-        StencilTest stencilTestBack = StencilTest(0, 0xff, ALWAYS, STENCIL_OP_KEEP, STENCIL_OP_KEEP, STENCIL_OP_KEEP);
-
+        DepthTest depthTest;
+        StencilActivation stencilActivation;
+        StencilTest stencilTestFront;
+        StencilTest stencilTestBack;
         uint32 sampleMask = 0xFFFFFFFF;
+        BlendFunction blendFunction;
+        FillMode fillMode{ FILL_FACE };
+        CullMode cullMode{ CULL_NONE };
+        ColorMask colorWriteMask{ WRITE_ALL };
 
-        BlendFunction blendFunction = BlendFunction(false);
-
-        uint8 fillMode = FILL_FACE;
-        uint8 cullMode = CULL_NONE;
-
-        uint8 colorWriteMask = WRITE_ALL;
-
-        bool frontFaceClockwise : 1;
-        bool depthClampEnable : 1;
-        bool scissorEnable : 1;
-        bool multisampleEnable : 1;
-        bool antialisedLineEnable : 1;
-        bool alphaToCoverageEnable : 1;
-
-        Data() :
-            frontFaceClockwise(false),
-            depthClampEnable(false),
-            scissorEnable(false),
-            multisampleEnable(false),
-            antialisedLineEnable(true),
-            alphaToCoverageEnable(false)
-        {}
+        Flags flags;
     };
 
+    static_assert(offsetof(Data, depthBias) == 0, "Data offsets");
+    static_assert(offsetof(Data, depthBiasSlopeScale) == 4, "Data offsets");
+    static_assert(offsetof(Data, depthTest) == 8, "Data offsets");
+    static_assert(offsetof(Data, stencilActivation) == 12, "Data offsets");
+    static_assert(offsetof(Data, stencilTestFront) == 16, "Data offsets");
+    static_assert(offsetof(Data, stencilTestBack) == 20, "Data offsets");
+    static_assert(offsetof(Data, sampleMask) == 24, "Data offsets");
+    static_assert(offsetof(Data, blendFunction) == 28, "Data offsets");
+    static_assert(offsetof(Data, fillMode) == 32, "Data offsets");
+    static_assert(offsetof(Data, cullMode) == 33, "Data offsets");
+    static_assert(offsetof(Data, colorWriteMask) == 34, "Data offsets");
+    static_assert(offsetof(Data, flags) == 35, "Data offsets");
+    static_assert(sizeof(Data) == 36, "Data Size Check");
+
+    std::string getKey() const;
+
     // The unique default values for all the fields
     static const Data DEFAULT;
-    void setFillMode(FillMode fill) { SET_FIELD(FILL_MODE, DEFAULT.fillMode, fill, _values.fillMode); }
-    FillMode getFillMode() const { return FillMode(_values.fillMode); }
+    void setFillMode(FillMode fill) { SET_FIELD(FILL_MODE, fillMode, fill); }
+    FillMode getFillMode() const { return _values.fillMode; }
 
-    void setCullMode(CullMode cull)  { SET_FIELD(CULL_MODE, DEFAULT.cullMode, cull, _values.cullMode); }
-    CullMode getCullMode() const { return CullMode(_values.cullMode); }
+    void setCullMode(CullMode cull) { SET_FIELD(CULL_MODE, cullMode, cull); }
+    CullMode getCullMode() const { return _values.cullMode; }
 
-    void setFrontFaceClockwise(bool isClockwise) { SET_FIELD(FRONT_FACE_CLOCKWISE, DEFAULT.frontFaceClockwise, isClockwise, _values.frontFaceClockwise); }
-    bool isFrontFaceClockwise() const { return _values.frontFaceClockwise; }
+    const Flags& getFlags() const { return _values.flags; }
 
-    void setDepthClampEnable(bool enable) { SET_FIELD(DEPTH_CLAMP_ENABLE, DEFAULT.depthClampEnable, enable, _values.depthClampEnable); }
-    bool isDepthClampEnable() const { return _values.depthClampEnable; }
+    void setFrontFaceClockwise(bool isClockwise) { SET_FIELD(FRONT_FACE_CLOCKWISE, flags.frontFaceClockwise, isClockwise); }
+    bool isFrontFaceClockwise() const { return _values.flags.frontFaceClockwise; }
 
-    void setScissorEnable(bool enable) { SET_FIELD(SCISSOR_ENABLE, DEFAULT.scissorEnable, enable, _values.scissorEnable); }
-    bool isScissorEnable() const { return _values.scissorEnable; }
+    void setDepthClampEnable(bool enable) { SET_FIELD(DEPTH_CLAMP_ENABLE, flags.depthClampEnable, enable); }
+    bool isDepthClampEnable() const { return _values.flags.depthClampEnable; }
 
-    void setMultisampleEnable(bool enable) { SET_FIELD(MULTISAMPLE_ENABLE, DEFAULT.multisampleEnable, enable, _values.multisampleEnable); }
-    bool isMultisampleEnable() const { return _values.multisampleEnable; }
+    void setScissorEnable(bool enable) { SET_FIELD(SCISSOR_ENABLE, flags.scissorEnable, enable); }
+    bool isScissorEnable() const { return _values.flags.scissorEnable; }
 
-    void setAntialiasedLineEnable(bool enable) { SET_FIELD(ANTIALISED_LINE_ENABLE, DEFAULT.antialisedLineEnable, enable, _values.antialisedLineEnable); }
-    bool isAntialiasedLineEnable() const { return _values.antialisedLineEnable; }
+    void setMultisampleEnable(bool enable) { SET_FIELD(MULTISAMPLE_ENABLE, flags.multisampleEnable, enable); }
+    bool isMultisampleEnable() const { return _values.flags.multisampleEnable; }
+
+    void setAntialiasedLineEnable(bool enable) { SET_FIELD(ANTIALISED_LINE_ENABLE, flags.antialisedLineEnable, enable); }
+    bool isAntialiasedLineEnable() const { return _values.flags.antialisedLineEnable; }
 
     // Depth Bias
-    void setDepthBias(float bias) { SET_FIELD(DEPTH_BIAS, DEFAULT.depthBias, bias, _values.depthBias); }
+    void setDepthBias(float bias) { SET_FIELD(DEPTH_BIAS, depthBias, bias); }
     float getDepthBias() const { return _values.depthBias; }
 
-    void setDepthBiasSlopeScale(float scale) { SET_FIELD(DEPTH_BIAS_SLOPE_SCALE, DEFAULT.depthBiasSlopeScale, scale, _values.depthBiasSlopeScale); }
+    void setDepthBiasSlopeScale(float scale) { SET_FIELD(DEPTH_BIAS_SLOPE_SCALE, depthBiasSlopeScale, scale); }
     float getDepthBiasSlopeScale() const { return _values.depthBiasSlopeScale; }
 
     // Depth Test
-    void setDepthTest(DepthTest depthTest) { SET_FIELD(DEPTH_TEST, DEFAULT.depthTest, depthTest, _values.depthTest); }
-    void setDepthTest(bool enable, bool writeMask, ComparisonFunction func) { setDepthTest(DepthTest(enable, writeMask, func)); }
+    void setDepthTest(DepthTest newDepthTest) { SET_FIELD(DEPTH_TEST, depthTest, newDepthTest); }
+    void setDepthTest(bool enable, bool writeMask, ComparisonFunction func) {
+        setDepthTest(DepthTest(enable, writeMask, func));
+    }
     DepthTest getDepthTest() const { return _values.depthTest; }
 
     bool isDepthTestEnabled() const { return getDepthTest().isEnabled(); }
@@ -310,11 +343,14 @@ public:
 
     // Stencil test
     void setStencilTest(bool enabled, uint8 frontWriteMask, StencilTest frontTest, uint8 backWriteMask, StencilTest backTest) {
-        SET_FIELD(STENCIL_ACTIVATION, DEFAULT.stencilActivation, StencilActivation(enabled, frontWriteMask, backWriteMask), _values.stencilActivation);
-        SET_FIELD(STENCIL_TEST_FRONT, DEFAULT.stencilTestFront, frontTest, _values.stencilTestFront);
-        SET_FIELD(STENCIL_TEST_BACK, DEFAULT.stencilTestBack, backTest, _values.stencilTestBack); }
+        SET_FIELD(STENCIL_ACTIVATION, stencilActivation, StencilActivation(enabled, frontWriteMask, backWriteMask));
+        SET_FIELD(STENCIL_TEST_FRONT, stencilTestFront, frontTest);
+        SET_FIELD(STENCIL_TEST_BACK, stencilTestBack, backTest);
+    }
+
     void setStencilTest(bool enabled, uint8 frontWriteMask, StencilTest frontTest) {
-        setStencilTest(enabled, frontWriteMask, frontTest, frontWriteMask, frontTest); }
+        setStencilTest(enabled, frontWriteMask, frontTest, frontWriteMask, frontTest);
+    }
 
     StencilActivation getStencilActivation() const { return _values.stencilActivation; }
     StencilTest getStencilTestFront() const { return _values.stencilTestFront; }
@@ -325,32 +361,45 @@ public:
     uint8 getStencilWriteMaskBack() const { return getStencilActivation().getWriteMaskBack(); }
 
     // Alpha to coverage
-    void setAlphaToCoverageEnable(bool enable) { SET_FIELD(ALPHA_TO_COVERAGE_ENABLE, DEFAULT.alphaToCoverageEnable, enable, _values.alphaToCoverageEnable); }
-    bool isAlphaToCoverageEnabled() const { return _values.alphaToCoverageEnable; }
+    void setAlphaToCoverageEnable(bool enable) { SET_FIELD(ALPHA_TO_COVERAGE_ENABLE, flags.alphaToCoverageEnable, enable); }
+    bool isAlphaToCoverageEnabled() const { return _values.flags.alphaToCoverageEnable; }
 
     // Sample mask
-    void setSampleMask(uint32 mask) { SET_FIELD(SAMPLE_MASK, DEFAULT.sampleMask, mask, _values.sampleMask); }
+    void setSampleMask(uint32 mask) { SET_FIELD(SAMPLE_MASK, sampleMask, mask); }
     uint32 getSampleMask() const { return _values.sampleMask; }
 
     // Blend Function
-    void setBlendFunction(BlendFunction function) { SET_FIELD(BLEND_FUNCTION, DEFAULT.blendFunction, function, _values.blendFunction); }
-    BlendFunction getBlendFunction() const { return _values.blendFunction; }
+    void setBlendFunction(BlendFunction function) { SET_FIELD(BLEND_FUNCTION, blendFunction, function); }
+    const BlendFunction& getBlendFunction() const { return _values.blendFunction; }
 
-    void setBlendFunction(bool enabled, BlendArg sourceColor, BlendOp operationColor, BlendArg destinationColor, BlendArg sourceAlpha, BlendOp operationAlpha, BlendArg destinationAlpha) {
-        setBlendFunction(BlendFunction(enabled, sourceColor, operationColor, destinationColor, sourceAlpha, operationAlpha, destinationAlpha)); }
+    void setBlendFunction(bool enabled,
+                          BlendArg sourceColor,
+                          BlendOp operationColor,
+                          BlendArg destinationColor,
+                          BlendArg sourceAlpha,
+                          BlendOp operationAlpha,
+                          BlendArg destinationAlpha) {
+        setBlendFunction(BlendFunction(enabled, sourceColor, operationColor, destinationColor, sourceAlpha, operationAlpha,
+                                       destinationAlpha));
+    }
     void setBlendFunction(bool enabled, BlendArg source, BlendOp operation, BlendArg destination) {
-        setBlendFunction(BlendFunction(enabled, source, operation, destination)); }
+        setBlendFunction(BlendFunction(enabled, source, operation, destination));
+    }
 
     bool isBlendEnabled() const { return getBlendFunction().isEnabled(); }
 
     // Color write mask
-    void setColorWriteMask(uint8 mask) { SET_FIELD(COLOR_WRITE_MASK, DEFAULT.colorWriteMask, mask, _values.colorWriteMask); }
-    void setColorWriteMask(bool red, bool green, bool blue, bool alpha) { uint32 value = ((WRITE_RED * red) | (WRITE_GREEN * green) | (WRITE_BLUE * blue) | (WRITE_ALPHA * alpha)); SET_FIELD(COLOR_WRITE_MASK, DEFAULT.colorWriteMask, value, _values.colorWriteMask); }
-    uint8 getColorWriteMask() const { return _values.colorWriteMask; }
+    void setColorWriteMask(ColorMask mask) { SET_FIELD(COLOR_WRITE_MASK, colorWriteMask, mask); }
+    void setColorWriteMask(bool red, bool green, bool blue, bool alpha) {
+        ColorMask value = (ColorMask)((WRITE_RED * red) | (WRITE_GREEN * green) | (WRITE_BLUE * blue) | (WRITE_ALPHA * alpha));
+        SET_FIELD(COLOR_WRITE_MASK, colorWriteMask, value);
+    }
+    ColorMask getColorWriteMask() const { return _values.colorWriteMask; }
 
     // All the possible fields
     // NOTE: If you change this, you must update GLBackend::GLState::_resetStateCommands
-    enum Field {
+    enum Field
+    {
         FILL_MODE,
         CULL_MODE,
         FRONT_FACE_CLOCKWISE,
@@ -376,7 +425,7 @@ public:
 
         COLOR_WRITE_MASK,
 
-        NUM_FIELDS, // not a valid field, just the count
+        NUM_FIELDS,  // not a valid field, just the count
     };
 
     // The signature of the state tells which fields of the state are not default
@@ -391,20 +440,20 @@ public:
     State(const Data& values);
     const Data& getValues() const { return _values; }
 
-    const GPUObjectPointer gpuObject {};
+    const GPUObjectPointer gpuObject{};
 
 protected:
     State(const State& state);
     State& operator=(const State& state);
 
     Data _values;
-    Signature _signature{0};
-    Stamp _stamp{0};
+    Signature _signature{ 0 };
+    Stamp _stamp{ 0 };
 };
 
-typedef std::shared_ptr< State > StatePointer;
-typedef std::vector< StatePointer > States;
+typedef std::shared_ptr<State> StatePointer;
+typedef std::vector<StatePointer> States;
 
-};
+};  // namespace gpu
 
 #endif

libraries/gpu/src/gpu/Stream.h

@@ -128,6 +128,8 @@ public:
         uint32 _elementTotalSize { 0 };
         std::string _key;
 
+        friend class Serializer;
+        friend class Deserializer;
         void evaluateCache();
     };
 

libraries/gpu/src/gpu/Texture.h

@@ -189,6 +189,8 @@ public:
     }
 protected:
     Desc _desc;
+
+    friend class Deserializer;
 };
 
 enum class TextureUsageType : uint8 {
@@ -371,6 +373,8 @@ public:
 
         ktx::KTXDescriptorPointer _ktxDescriptor;
         friend class Texture;
+        friend class Serializer;
+        friend class Deserializer;
     };
 
     uint16 minAvailableMipLevel() const { return _storage->minAvailableMipLevel(); };
@@ -628,6 +632,9 @@ protected:
     static TexturePointer create(TextureUsageType usageType, Type type, const Element& texelFormat, uint16 width, uint16 height, uint16 depth, uint16 numSamples, uint16 numSlices, uint16 numMips, const Sampler& sampler);
 
     Size resize(Type type, const Element& texelFormat, uint16 width, uint16 height, uint16 depth, uint16 numSamples, uint16 numSlices, uint16 numMips);
+
+    friend class Serializer;
+    friend class Deserializer;
 };
 
 typedef std::shared_ptr<Texture> TexturePointer;

libraries/hfm/src/hfm/HFM.cpp

@@ -67,108 +67,6 @@ bool HFMMaterial::needTangentSpace() const {
     return !normalTexture.isNull();
 }
 
-static void _createBlendShapeTangents(HFMMesh& mesh, bool generateFromTexCoords, HFMBlendshape& blendShape);
-
-void HFMMesh::createBlendShapeTangents(bool generateTangents) {
-    for (auto& blendShape : blendshapes) {
-        _createBlendShapeTangents(*this, generateTangents, blendShape);
-    }
-}
-
-using IndexAccessor = std::function<glm::vec3*(const HFMMesh&, int, int, glm::vec3*, glm::vec3&)>;
-
-static void setTangents(const HFMMesh& mesh, const IndexAccessor& vertexAccessor, int firstIndex, int secondIndex,
-    const QVector<glm::vec3>& vertices, const QVector<glm::vec3>& normals, QVector<glm::vec3>& tangents) {
-    glm::vec3 vertex[2];
-    glm::vec3 normal;
-    glm::vec3* tangent = vertexAccessor(mesh, firstIndex, secondIndex, vertex, normal);
-    if (tangent) {
-        glm::vec3 bitangent = glm::cross(normal, vertex[1] - vertex[0]);
-        if (glm::length(bitangent) < EPSILON) {
-            return;
-        }
-        glm::vec2 texCoordDelta = mesh.texCoords.at(secondIndex) - mesh.texCoords.at(firstIndex);
-        glm::vec3 normalizedNormal = glm::normalize(normal);
-        *tangent += glm::cross(glm::angleAxis(-atan2f(-texCoordDelta.t, texCoordDelta.s), normalizedNormal) *
-            glm::normalize(bitangent), normalizedNormal);
-    }
-}
-
-static void createTangents(const HFMMesh& mesh, bool generateFromTexCoords,
-    const QVector<glm::vec3>& vertices, const QVector<glm::vec3>& normals, QVector<glm::vec3>& tangents,
-    IndexAccessor accessor) {
-    // if we have a normal map (and texture coordinates), we must compute tangents
-    if (generateFromTexCoords && !mesh.texCoords.isEmpty()) {
-        tangents.resize(vertices.size());
-
-        foreach(const HFMMeshPart& part, mesh.parts) {
-            for (int i = 0; i < part.quadIndices.size(); i += 4) {
-                setTangents(mesh, accessor, part.quadIndices.at(i), part.quadIndices.at(i + 1), vertices, normals, tangents);
-                setTangents(mesh, accessor, part.quadIndices.at(i + 1), part.quadIndices.at(i + 2), vertices, normals, tangents);
-                setTangents(mesh, accessor, part.quadIndices.at(i + 2), part.quadIndices.at(i + 3), vertices, normals, tangents);
-                setTangents(mesh, accessor, part.quadIndices.at(i + 3), part.quadIndices.at(i), vertices, normals, tangents);
-            }
-            // <= size - 3 in order to prevent overflowing triangleIndices when (i % 3) != 0
-            // This is most likely evidence of a further problem in extractMesh()
-            for (int i = 0; i <= part.triangleIndices.size() - 3; i += 3) {
-                setTangents(mesh, accessor, part.triangleIndices.at(i), part.triangleIndices.at(i + 1), vertices, normals, tangents);
-                setTangents(mesh, accessor, part.triangleIndices.at(i + 1), part.triangleIndices.at(i + 2), vertices, normals, tangents);
-                setTangents(mesh, accessor, part.triangleIndices.at(i + 2), part.triangleIndices.at(i), vertices, normals, tangents);
-            }
-            if ((part.triangleIndices.size() % 3) != 0) {
-                qCDebug(modelformat) << "Error in extractHFMModel part.triangleIndices.size() is not divisible by three ";
-            }
-        }
-    }
-}
-
-void HFMMesh::createMeshTangents(bool generateFromTexCoords) {
-    HFMMesh& mesh = *this;
-    // This is the only workaround I've found to trick the compiler into understanding that mesh.tangents isn't
-    // const in the lambda function.
-    auto& tangents = mesh.tangents;
-    createTangents(mesh, generateFromTexCoords, mesh.vertices, mesh.normals, mesh.tangents, 
-        [&](const HFMMesh& mesh, int firstIndex, int secondIndex, glm::vec3* outVertices, glm::vec3& outNormal) {
-        outVertices[0] = mesh.vertices[firstIndex];
-        outVertices[1] = mesh.vertices[secondIndex];
-        outNormal = mesh.normals[firstIndex];
-        return &(tangents[firstIndex]);
-    });
-}
-
-static void _createBlendShapeTangents(HFMMesh& mesh, bool generateFromTexCoords, HFMBlendshape& blendShape) {
-    // Create lookup to get index in blend shape from vertex index in mesh
-    std::vector<int> reverseIndices;
-    reverseIndices.resize(mesh.vertices.size());
-    std::iota(reverseIndices.begin(), reverseIndices.end(), 0);
-
-    for (int indexInBlendShape = 0; indexInBlendShape < blendShape.indices.size(); ++indexInBlendShape) {
-        auto indexInMesh = blendShape.indices[indexInBlendShape];
-        reverseIndices[indexInMesh] = indexInBlendShape;
-    }
-
-    createTangents(mesh, generateFromTexCoords, blendShape.vertices, blendShape.normals, blendShape.tangents,
-        [&](const HFMMesh& mesh, int firstIndex, int secondIndex, glm::vec3* outVertices, glm::vec3& outNormal) {
-        const auto index1 = reverseIndices[firstIndex];
-        const auto index2 = reverseIndices[secondIndex];
-
-        if (index1 < blendShape.vertices.size()) {
-            outVertices[0] = blendShape.vertices[index1];
-            if (index2 < blendShape.vertices.size()) {
-                outVertices[1] = blendShape.vertices[index2];
-            } else {
-                // Index isn't in the blend shape so return vertex from mesh
-                outVertices[1] = mesh.vertices[secondIndex];
-            }
-            outNormal = blendShape.normals[index1];
-            return &blendShape.tangents[index1];
-        } else {
-            // Index isn't in blend shape so return nullptr
-            return (glm::vec3*)nullptr;
-        }
-    });
-}
-
 QStringList HFMModel::getJointNames() const {
     QStringList names;
     foreach (const HFMJoint& joint, joints) {

libraries/hfm/src/hfm/HFM.h

@@ -239,9 +239,6 @@ public:
 
     graphics::MeshPointer _mesh;
     bool wasCompressed { false };
-
-    void createMeshTangents(bool generateFromTexCoords);
-    void createBlendShapeTangents(bool generateTangents);
 };
 
 /**jsdoc

libraries/model-baker/CMakeLists.txt

@@ -1,6 +1,4 @@
 set(TARGET_NAME model-baker)
 setup_hifi_library()
 
-link_hifi_libraries(shared task gpu graphics)
-
-include_hifi_library_headers(hfm)
+link_hifi_libraries(shared task gpu graphics hfm)

libraries/model-baker/src/model-baker/Baker.cpp

@@ -15,26 +15,65 @@
 
 #include "BakerTypes.h"
 #include "BuildGraphicsMeshTask.h"
+#include "CalculateMeshNormalsTask.h"
+#include "CalculateMeshTangentsTask.h"
+#include "CalculateBlendshapeNormalsTask.h"
+#include "CalculateBlendshapeTangentsTask.h"
 
 namespace baker {
 
     class GetModelPartsTask {
     public:
         using Input = hfm::Model::Pointer;
-        using Output = VaryingSet3<std::vector<hfm::Mesh>, hifi::URL, MeshIndicesToModelNames>;
+        using Output = VaryingSet5<std::vector<hfm::Mesh>, hifi::URL, baker::MeshIndicesToModelNames, baker::BlendshapesPerMesh, QHash<QString, hfm::Material>>;
         using JobModel = Job::ModelIO<GetModelPartsTask, Input, Output>;
 
         void run(const BakeContextPointer& context, const Input& input, Output& output) {
-            auto& hfmModelIn = input;
+            const auto& hfmModelIn = input;
             output.edit0() = hfmModelIn->meshes.toStdVector();
             output.edit1() = hfmModelIn->originalURL;
             output.edit2() = hfmModelIn->meshIndicesToModelNames;
+            auto& blendshapesPerMesh = output.edit3();
+            blendshapesPerMesh.reserve(hfmModelIn->meshes.size());
+            for (int i = 0; i < hfmModelIn->meshes.size(); i++) {
+                blendshapesPerMesh.push_back(hfmModelIn->meshes[i].blendshapes.toStdVector());
+            }
+            output.edit4() = hfmModelIn->materials;
+        }
+    };
+
+    class BuildBlendshapesTask {
+    public:
+        using Input = VaryingSet3<BlendshapesPerMesh, std::vector<NormalsPerBlendshape>, std::vector<TangentsPerBlendshape>>;
+        using Output = BlendshapesPerMesh;
+        using JobModel = Job::ModelIO<BuildBlendshapesTask, Input, Output>;
+
+        void run(const BakeContextPointer& context, const Input& input, Output& output) {
+            const auto& blendshapesPerMeshIn = input.get0();
+            const auto& normalsPerBlendshapePerMesh = input.get1();
+            const auto& tangentsPerBlendshapePerMesh = input.get2();
+            auto& blendshapesPerMeshOut = output;
+
+            blendshapesPerMeshOut = blendshapesPerMeshIn;
+
+            for (int i = 0; i < (int)blendshapesPerMeshOut.size(); i++) {
+                const auto& normalsPerBlendshape = normalsPerBlendshapePerMesh[i];
+                const auto& tangentsPerBlendshape = tangentsPerBlendshapePerMesh[i];
+                auto& blendshapesOut = blendshapesPerMeshOut[i];
+                for (int j = 0; j < (int)blendshapesOut.size(); j++) {
+                    const auto& normals = normalsPerBlendshape[j];
+                    const auto& tangents = tangentsPerBlendshape[j];
+                    auto& blendshape = blendshapesOut[j];
+                    blendshape.normals = QVector<glm::vec3>::fromStdVector(normals);
+                    blendshape.tangents = QVector<glm::vec3>::fromStdVector(tangents);
+                }
+            }
         }
     };
 
     class BuildMeshesTask {
     public:
-        using Input = VaryingSet4<std::vector<hfm::Mesh>, std::vector<graphics::MeshPointer>, TangentsPerMesh, BlendshapesPerMesh>;
+        using Input = VaryingSet5<std::vector<hfm::Mesh>, std::vector<graphics::MeshPointer>, NormalsPerMesh, TangentsPerMesh, BlendshapesPerMesh>;
         using Output = std::vector<hfm::Mesh>;
         using JobModel = Job::ModelIO<BuildMeshesTask, Input, Output>;
 
@@ -42,13 +81,15 @@ namespace baker {
             auto& meshesIn = input.get0();
             int numMeshes = (int)meshesIn.size();
             auto& graphicsMeshesIn = input.get1();
-            auto& tangentsPerMeshIn = input.get2();
-            auto& blendshapesPerMeshIn = input.get3();
+            auto& normalsPerMeshIn = input.get2();
+            auto& tangentsPerMeshIn = input.get3();
+            auto& blendshapesPerMeshIn = input.get4();
 
             auto meshesOut = meshesIn;
             for (int i = 0; i < numMeshes; i++) {
                 auto& meshOut = meshesOut[i];
                 meshOut._mesh = graphicsMeshesIn[i];
+                meshOut.normals = QVector<glm::vec3>::fromStdVector(normalsPerMeshIn[i]);
                 meshOut.tangents = QVector<glm::vec3>::fromStdVector(tangentsPerMeshIn[i]);
                 meshOut.blendshapes = QVector<hfm::Blendshape>::fromStdVector(blendshapesPerMeshIn[i]);
             }
@@ -80,17 +121,27 @@ namespace baker {
             const auto meshesIn = modelPartsIn.getN<GetModelPartsTask::Output>(0);
             const auto url = modelPartsIn.getN<GetModelPartsTask::Output>(1);
             const auto meshIndicesToModelNames = modelPartsIn.getN<GetModelPartsTask::Output>(2);
+            const auto blendshapesPerMeshIn = modelPartsIn.getN<GetModelPartsTask::Output>(3);
+            const auto materials = modelPartsIn.getN<GetModelPartsTask::Output>(4);
+
+            // Calculate normals and tangents for meshes and blendshapes if they do not exist
+            // Note: Normals are never calculated here for OBJ models. OBJ files optionally define normals on a per-face basis, so for consistency normals are calculated beforehand in OBJSerializer.
+            const auto normalsPerMesh = model.addJob<CalculateMeshNormalsTask>("CalculateMeshNormals", meshesIn);
+            const auto calculateMeshTangentsInputs = CalculateMeshTangentsTask::Input(normalsPerMesh, meshesIn, materials).asVarying();
+            const auto tangentsPerMesh = model.addJob<CalculateMeshTangentsTask>("CalculateMeshTangents", calculateMeshTangentsInputs);
+            const auto calculateBlendshapeNormalsInputs = CalculateBlendshapeNormalsTask::Input(blendshapesPerMeshIn, meshesIn).asVarying();
+            const auto normalsPerBlendshapePerMesh = model.addJob<CalculateBlendshapeNormalsTask>("CalculateBlendshapeNormals", calculateBlendshapeNormalsInputs);
+            const auto calculateBlendshapeTangentsInputs = CalculateBlendshapeTangentsTask::Input(normalsPerBlendshapePerMesh, blendshapesPerMeshIn, meshesIn, materials).asVarying();
+            const auto tangentsPerBlendshapePerMesh = model.addJob<CalculateBlendshapeTangentsTask>("CalculateBlendshapeTangents", calculateBlendshapeTangentsInputs);
 
             // Build the graphics::MeshPointer for each hfm::Mesh
-            const auto buildGraphicsMeshInputs = BuildGraphicsMeshTask::Input(meshesIn, url, meshIndicesToModelNames).asVarying();
-            const auto buildGraphicsMeshOutputs = model.addJob<BuildGraphicsMeshTask>("BuildGraphicsMesh", buildGraphicsMeshInputs);
-            const auto graphicsMeshes = buildGraphicsMeshOutputs.getN<BuildGraphicsMeshTask::Output>(0);
-            // TODO: Move tangent/blendshape validation/calculation to an earlier step
-            const auto tangentsPerMesh = buildGraphicsMeshOutputs.getN<BuildGraphicsMeshTask::Output>(1);
-            const auto blendshapesPerMesh = buildGraphicsMeshOutputs.getN<BuildGraphicsMeshTask::Output>(2);
+            const auto buildGraphicsMeshInputs = BuildGraphicsMeshTask::Input(meshesIn, url, meshIndicesToModelNames, normalsPerMesh, tangentsPerMesh).asVarying();
+            const auto graphicsMeshes = model.addJob<BuildGraphicsMeshTask>("BuildGraphicsMesh", buildGraphicsMeshInputs);
 
             // Combine the outputs into a new hfm::Model
-            const auto buildMeshesInputs = BuildMeshesTask::Input(meshesIn, graphicsMeshes, tangentsPerMesh, blendshapesPerMesh).asVarying();
+            const auto buildBlendshapesInputs = BuildBlendshapesTask::Input(blendshapesPerMeshIn, normalsPerBlendshapePerMesh, tangentsPerBlendshapePerMesh).asVarying();
+            const auto blendshapesPerMeshOut = model.addJob<BuildBlendshapesTask>("BuildBlendshapes", buildBlendshapesInputs);
+            const auto buildMeshesInputs = BuildMeshesTask::Input(meshesIn, graphicsMeshes, normalsPerMesh, tangentsPerMesh, blendshapesPerMeshOut).asVarying();
             const auto meshesOut = model.addJob<BuildMeshesTask>("BuildMeshes", buildMeshesInputs);
             const auto buildModelInputs = BuildModelTask::Input(hfmModelIn, meshesOut).asVarying();
             hfmModelOut = model.addJob<BuildModelTask>("BuildModel", buildModelInputs);

libraries/model-baker/src/model-baker/BakerTypes.h

@@ -15,10 +15,25 @@
 #include <hfm/HFM.h>
 
 namespace baker {
+    using MeshIndices = std::vector<int>;
+    using IndicesPerMesh = std::vector<std::vector<int>>;
+    using VerticesPerMesh = std::vector<std::vector<glm::vec3>>;
+    using MeshNormals = std::vector<glm::vec3>;
+    using NormalsPerMesh = std::vector<std::vector<glm::vec3>>;
     using MeshTangents = std::vector<glm::vec3>;
     using TangentsPerMesh = std::vector<std::vector<glm::vec3>>;
+
     using Blendshapes = std::vector<hfm::Blendshape>;
     using BlendshapesPerMesh = std::vector<std::vector<hfm::Blendshape>>;
+    using BlendshapeVertices = std::vector<glm::vec3>;
+    using BlendshapeNormals = std::vector<glm::vec3>;
+    using BlendshapeIndices = std::vector<int>;
+    using VerticesPerBlendshape = std::vector<std::vector<glm::vec3>>;
+    using NormalsPerBlendshape = std::vector<std::vector<glm::vec3>>;
+    using IndicesPerBlendshape = std::vector<std::vector<int>>;
+    using BlendshapeTangents = std::vector<glm::vec3>;
+    using TangentsPerBlendshape = std::vector<std::vector<glm::vec3>>;
+
     using MeshIndicesToModelNames = QHash<int, QString>;
 };
 

libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.cpp

@@ -26,9 +26,18 @@ glm::vec3 normalizeDirForPacking(const glm::vec3& dir) {
     return dir;
 }
 
-void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphicsMeshPointer, baker::MeshTangents& meshTangents, baker::Blendshapes& blendshapes) {
+void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphicsMeshPointer, const baker::MeshNormals& meshNormals, const baker::MeshTangents& meshTangentsIn) {
     auto graphicsMesh = std::make_shared<graphics::Mesh>();
 
+    // Fill tangents with a dummy value to force tangents to be present if there are normals
+    baker::MeshTangents meshTangents;
+    if (!meshTangentsIn.empty()) {
+        meshTangents = meshTangentsIn;
+    } else {
+        meshTangents.reserve(meshNormals.size());
+        std::fill_n(std::back_inserter(meshTangents), meshNormals.size(), Vectors::UNIT_X);
+    }
+
     unsigned int totalSourceIndices = 0;
     foreach(const HFMMeshPart& part, hfmMesh.parts) {
         totalSourceIndices += (part.quadTrianglesIndices.size() + part.triangleIndices.size());
@@ -48,23 +57,6 @@ void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphics
 
     int numVerts = hfmMesh.vertices.size();
 
-    if (!hfmMesh.normals.empty() && hfmMesh.tangents.empty()) {
-        // Fill with a dummy value to force tangents to be present if there are normals
-        meshTangents.reserve(hfmMesh.normals.size());
-        std::fill_n(std::back_inserter(meshTangents), hfmMesh.normals.size(), Vectors::UNIT_X);
-    } else {
-        meshTangents = hfmMesh.tangents.toStdVector();
-    }
-    // Same thing with blend shapes
-    blendshapes = hfmMesh.blendshapes.toStdVector();
-    for (auto& blendShape : blendshapes) {
-        if (!blendShape.normals.empty() && blendShape.tangents.empty()) {
-            // Fill with a dummy value to force tangents to be present if there are normals
-            blendShape.tangents.reserve(blendShape.normals.size());
-            std::fill_n(std::back_inserter(blendShape.tangents), blendShape.normals.size(), Vectors::UNIT_X);
-        }
-    }
-
     // evaluate all attribute elements and data sizes
 
     // Position is a vec3
@@ -73,12 +65,12 @@ void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphics
 
     // Normal and tangent are always there together packed in normalized xyz32bits word (times 2)
     const auto normalElement = HFM_NORMAL_ELEMENT;
-    const int normalsSize = hfmMesh.normals.size() * normalElement.getSize();
+    const int normalsSize = (int)meshNormals.size() * normalElement.getSize();
     const int tangentsSize = (int)meshTangents.size() * normalElement.getSize();
     // If there are normals then there should be tangents
     assert(normalsSize <= tangentsSize);
     if (tangentsSize > normalsSize) {
-        HIFI_FCDEBUG_ID(model_baker(), repeatMessageID, "BuildGraphicsMeshTask -- Unexpected tangents in file");
+        HIFI_FCDEBUG_ID(model_baker(), repeatMessageID, "BuildGraphicsMeshTask -- Unexpected tangents in mesh");
     }
     const auto normalsAndTangentsSize = normalsSize + tangentsSize;
 
@@ -124,11 +116,11 @@ void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphics
     if (normalsSize > 0) {
         std::vector<NormalType> normalsAndTangents;
 
-        normalsAndTangents.reserve(hfmMesh.normals.size() + (int)meshTangents.size());
-        auto normalIt = hfmMesh.normals.constBegin();
+        normalsAndTangents.reserve(meshNormals.size() + (int)meshTangents.size());
+        auto normalIt = meshNormals.cbegin();
         auto tangentIt = meshTangents.cbegin();
         for (;
-            normalIt != hfmMesh.normals.constEnd();
+            normalIt != meshNormals.cend();
             ++normalIt, ++tangentIt) {
 #if HFM_PACK_NORMALS
             const auto normal = normalizeDirForPacking(*normalIt);
@@ -212,11 +204,6 @@ void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphics
     auto vertexFormat = std::make_shared<gpu::Stream::Format>();
     auto vertexBufferStream = std::make_shared<gpu::BufferStream>();
 
-    // Decision time:
-    // if blendshapes then keep position and normals/tangents as separated channel buffers from interleaved attributes
-    // else everything is interleaved in one buffer
-
-    // Default case is no blend shapes
     gpu::BufferPointer attribBuffer;
     int totalAttribBufferSize = totalVertsSize;
     gpu::uint8 posChannel = 0;
@@ -244,7 +231,7 @@ void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphics
         }
     }
 
-    // Pack normal and Tangent with the rest of atributes if no blend shapes
+    // Pack normal and Tangent with the rest of atributes
     if (colorsSize) {
         vertexFormat->setAttribute(gpu::Stream::COLOR, attribChannel, colorElement, bufOffset);
         bufOffset += colorElement.getSize();
@@ -384,22 +371,21 @@ void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphics
 }
 
 void BuildGraphicsMeshTask::run(const baker::BakeContextPointer& context, const Input& input, Output& output) {
-    auto& meshes = input.get0();
-    auto& url = input.get1();
-    auto& meshIndicesToModelNames = input.get2();
+    const auto& meshes = input.get0();
+    const auto& url = input.get1();
+    const auto& meshIndicesToModelNames = input.get2();
+    const auto& normalsPerMesh = input.get3();
+    const auto& tangentsPerMesh = input.get4();
+
+    auto& graphicsMeshes = output;
 
-    auto& graphicsMeshes = output.edit0();
-    auto& tangentsPerMesh = output.edit1();
-    auto& blendshapesPerMesh = output.edit2();
     int n = (int)meshes.size();
     for (int i = 0; i < n; i++) {
         graphicsMeshes.emplace_back();
         auto& graphicsMesh = graphicsMeshes[i];
-        tangentsPerMesh.emplace_back();
-        blendshapesPerMesh.emplace_back();
         
         // Try to create the graphics::Mesh
-        buildGraphicsMesh(meshes[i], graphicsMesh, tangentsPerMesh[i], blendshapesPerMesh[i]);
+        buildGraphicsMesh(meshes[i], graphicsMesh, normalsPerMesh[i], tangentsPerMesh[i]);
 
         // Choose a name for the mesh
         if (graphicsMesh) {

libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.h

@@ -20,8 +20,8 @@
 
 class BuildGraphicsMeshTask {
 public:
-    using Input = baker::VaryingSet3<std::vector<hfm::Mesh>, hifi::URL, baker::MeshIndicesToModelNames>;
-    using Output = baker::VaryingSet3<std::vector<graphics::MeshPointer>, std::vector<baker::MeshTangents>, std::vector<baker::Blendshapes>>;
+    using Input = baker::VaryingSet5<std::vector<hfm::Mesh>, hifi::URL, baker::MeshIndicesToModelNames, baker::NormalsPerMesh, baker::TangentsPerMesh>;
+    using Output = std::vector<graphics::MeshPointer>;
     using JobModel = baker::Job::ModelIO<BuildGraphicsMeshTask, Input, Output>;
 
     void run(const baker::BakeContextPointer& context, const Input& input, Output& output);

libraries/model-baker/src/model-baker/CalculateBlendshapeNormalsTask.cpp

@@ -0,0 +1,70 @@
+//
+//  CalculateBlendshapeNormalsTask.h
+//  model-baker/src/model-baker
+//
+//  Created by Sabrina Shanman on 2019/01/07.
+//  Copyright 2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "CalculateBlendshapeNormalsTask.h"
+
+#include "ModelMath.h"
+
+void CalculateBlendshapeNormalsTask::run(const baker::BakeContextPointer& context, const Input& input, Output& output) {
+    const auto& blendshapesPerMesh = input.get0();
+    const auto& meshes = input.get1();
+    auto& normalsPerBlendshapePerMeshOut = output;
+
+    normalsPerBlendshapePerMeshOut.reserve(blendshapesPerMesh.size());
+    for (size_t i = 0; i < blendshapesPerMesh.size(); i++) {
+        const auto& mesh = meshes[i];
+        const auto& blendshapes = blendshapesPerMesh[i];
+        normalsPerBlendshapePerMeshOut.emplace_back();
+        auto& normalsPerBlendshapeOut = normalsPerBlendshapePerMeshOut[normalsPerBlendshapePerMeshOut.size()-1];
+
+        normalsPerBlendshapeOut.reserve(blendshapes.size());
+        for (size_t j = 0; j < blendshapes.size(); j++) {
+            const auto& blendshape = blendshapes[j];
+            const auto& normalsIn = blendshape.normals;
+            // Check if normals are already defined. Otherwise, calculate them from existing blendshape vertices.
+            if (!normalsIn.empty()) {
+                normalsPerBlendshapeOut.push_back(normalsIn.toStdVector());
+            } else {
+                // Create lookup to get index in blendshape from vertex index in mesh
+                std::vector<int> reverseIndices;
+                reverseIndices.resize(mesh.vertices.size());
+                std::iota(reverseIndices.begin(), reverseIndices.end(), 0);
+                for (int indexInBlendShape = 0; indexInBlendShape < blendshape.indices.size(); ++indexInBlendShape) {
+                    auto indexInMesh = blendshape.indices[indexInBlendShape];
+                    reverseIndices[indexInMesh] = indexInBlendShape;
+                }
+
+                normalsPerBlendshapeOut.emplace_back();
+                auto& normals = normalsPerBlendshapeOut[normalsPerBlendshapeOut.size()-1];
+                normals.resize(mesh.vertices.size());
+                baker::calculateNormals(mesh,
+                    [&reverseIndices, &blendshape, &normals](int normalIndex) /* NormalAccessor */ {
+                        const auto lookupIndex = reverseIndices[normalIndex];
+                        if (lookupIndex < blendshape.vertices.size()) {
+                            return &normals[lookupIndex];
+                        } else {
+                            // Index isn't in the blendshape. Request that the normal not be calculated.
+                            return (glm::vec3*)nullptr;
+                        }
+                    },
+                    [&mesh, &reverseIndices, &blendshape](int vertexIndex, glm::vec3& outVertex) /* VertexSetter */ {
+                        const auto lookupIndex = reverseIndices[vertexIndex];
+                        if (lookupIndex < blendshape.vertices.size()) {
+                            outVertex = blendshape.vertices[lookupIndex];
+                        } else {
+                            // Index isn't in the blendshape, so return vertex from mesh
+                            outVertex = mesh.vertices[lookupIndex];
+                        }
+                    });
+            }
+        }
+    }
+}

libraries/model-baker/src/model-baker/CalculateBlendshapeNormalsTask.h

@@ -0,0 +1,28 @@
+//
+//  CalculateBlendshapeNormalsTask.h
+//  model-baker/src/model-baker
+//
+//  Created by Sabrina Shanman on 2019/01/07.
+//  Copyright 2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_CalculateBlendshapeNormalsTask_h
+#define hifi_CalculateBlendshapeNormalsTask_h
+
+#include "Engine.h"
+#include "BakerTypes.h"
+
+// Calculate blendshape normals if not already present in the blendshape
+class CalculateBlendshapeNormalsTask {
+public:
+    using Input = baker::VaryingSet2<baker::BlendshapesPerMesh, std::vector<hfm::Mesh>>;
+    using Output = std::vector<baker::NormalsPerBlendshape>;
+    using JobModel = baker::Job::ModelIO<CalculateBlendshapeNormalsTask, Input, Output>;
+
+    void run(const baker::BakeContextPointer& context, const Input& input, Output& output);
+};
+
+#endif // hifi_CalculateBlendshapeNormalsTask_h

libraries/model-baker/src/model-baker/CalculateBlendshapeTangentsTask.cpp

@@ -0,0 +1,95 @@
+//
+//  CalculateBlendshapeTangentsTask.cpp
+//  model-baker/src/model-baker
+//
+//  Created by Sabrina Shanman on 2019/01/08.
+//  Copyright 2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "CalculateBlendshapeTangentsTask.h"
+
+#include <set>
+
+#include "ModelMath.h"
+
+void CalculateBlendshapeTangentsTask::run(const baker::BakeContextPointer& context, const Input& input, Output& output) {
+    const auto& normalsPerBlendshapePerMesh = input.get0();
+    const auto& blendshapesPerMesh = input.get1();
+    const auto& meshes = input.get2();
+    const auto& materials = input.get3();
+    auto& tangentsPerBlendshapePerMeshOut = output;
+    
+    tangentsPerBlendshapePerMeshOut.reserve(normalsPerBlendshapePerMesh.size());
+    for (size_t i = 0; i < blendshapesPerMesh.size(); i++) {
+        const auto& normalsPerBlendshape = normalsPerBlendshapePerMesh[i];
+        const auto& blendshapes = blendshapesPerMesh[i];
+        const auto& mesh = meshes[i];
+        tangentsPerBlendshapePerMeshOut.emplace_back();
+        auto& tangentsPerBlendshapeOut = tangentsPerBlendshapePerMeshOut[tangentsPerBlendshapePerMeshOut.size()-1];
+
+        // Check if we actually need to calculate the tangents, or just append empty arrays
+        bool needTangents = false;
+        for (const auto& meshPart : mesh.parts) {
+            auto materialIt = materials.find(meshPart.materialID);
+            if (materialIt != materials.end() && (*materialIt).needTangentSpace()) {
+                needTangents = true;
+                break;
+            }
+        }
+
+        for (size_t j = 0; j < blendshapes.size(); j++) {
+            const auto& blendshape = blendshapes[j];
+            const auto& tangentsIn = blendshape.tangents;
+            const auto& normals = normalsPerBlendshape[j];
+            tangentsPerBlendshapeOut.emplace_back();
+            auto& tangentsOut = tangentsPerBlendshapeOut[tangentsPerBlendshapeOut.size()-1];
+
+            // Check if we already have tangents
+            if (!tangentsIn.empty()) {
+                tangentsOut = tangentsIn.toStdVector();
+                continue;
+            }
+
+            // Check if we can and should calculate tangents (we need normals to calculate the tangents)
+            if (normals.empty() || !needTangents) {
+                continue;
+            }
+            tangentsOut.resize(normals.size());
+
+            // Create lookup to get index in blend shape from vertex index in mesh
+            std::vector<int> reverseIndices;
+            reverseIndices.resize(mesh.vertices.size());
+            std::iota(reverseIndices.begin(), reverseIndices.end(), 0);
+            for (int indexInBlendShape = 0; indexInBlendShape < blendshape.indices.size(); ++indexInBlendShape) {
+                auto indexInMesh = blendshape.indices[indexInBlendShape];
+                reverseIndices[indexInMesh] = indexInBlendShape;
+            }
+
+            baker::calculateTangents(mesh,
+                [&mesh, &blendshape, &normals, &tangentsOut, &reverseIndices](int firstIndex, int secondIndex, glm::vec3* outVertices, glm::vec2* outTexCoords, glm::vec3& outNormal) {
+                const auto index1 = reverseIndices[firstIndex];
+                const auto index2 = reverseIndices[secondIndex];
+
+                if (index1 < blendshape.vertices.size()) {
+                    outVertices[0] = blendshape.vertices[index1];
+                    outTexCoords[0] = mesh.texCoords[index1];
+                    outTexCoords[1] = mesh.texCoords[index2];
+                    if (index2 < blendshape.vertices.size()) {
+                        outVertices[1] = blendshape.vertices[index2];
+                    } else {
+                        // Index isn't in the blend shape so return vertex from mesh
+                        outVertices[1] = mesh.vertices[secondIndex];
+                    }
+                    outNormal = normals[index1];
+                    return &tangentsOut[index1];
+                } else {
+                    // Index isn't in blend shape so return nullptr
+                    return (glm::vec3*)nullptr;
+                }
+            });
+        }
+    }
+}

libraries/model-baker/src/model-baker/CalculateBlendshapeTangentsTask.h

@@ -0,0 +1,28 @@
+//
+//  CalculateBlendshapeTangentsTask.h
+//  model-baker/src/model-baker
+//
+//  Created by Sabrina Shanman on 2019/01/07.
+//  Copyright 2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_CalculateBlendshapeTangentsTask_h
+#define hifi_CalculateBlendshapeTangentsTask_h
+
+#include "Engine.h"
+#include "BakerTypes.h"
+
+// Calculate blendshape tangents if not already present in the blendshape
+class CalculateBlendshapeTangentsTask {
+public:
+    using Input = baker::VaryingSet4<std::vector<baker::NormalsPerBlendshape>, baker::BlendshapesPerMesh, std::vector<hfm::Mesh>, QHash<QString, hfm::Material>>;
+    using Output = std::vector<baker::TangentsPerBlendshape>;
+    using JobModel = baker::Job::ModelIO<CalculateBlendshapeTangentsTask, Input, Output>;
+
+    void run(const baker::BakeContextPointer& context, const Input& input, Output& output);
+};
+
+#endif // hifi_CalculateBlendshapeTangentsTask_h

libraries/model-baker/src/model-baker/CalculateMeshNormalsTask.cpp

@@ -0,0 +1,40 @@
+//
+//  CalculateMeshNormalsTask.cpp
+//  model-baker/src/model-baker
+//
+//  Created by Sabrina Shanman on 2019/01/22.
+//  Copyright 2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "CalculateMeshNormalsTask.h"
+
+#include "ModelMath.h"
+
+void CalculateMeshNormalsTask::run(const baker::BakeContextPointer& context, const Input& input, Output& output) {
+    const auto& meshes = input;
+    auto& normalsPerMeshOut = output;
+
+    normalsPerMeshOut.reserve(meshes.size());
+    for (int i = 0; i < (int)meshes.size(); i++) {
+        const auto& mesh = meshes[i];
+        normalsPerMeshOut.emplace_back();
+        auto& normalsOut = normalsPerMeshOut[normalsPerMeshOut.size()-1];
+        // Only calculate normals if this mesh doesn't already have them
+        if (!mesh.normals.empty()) {
+            normalsOut = mesh.normals.toStdVector();
+        } else {
+            normalsOut.resize(mesh.vertices.size());
+            baker::calculateNormals(mesh,
+                [&normalsOut](int normalIndex) /* NormalAccessor */ {
+                    return &normalsOut[normalIndex];
+                },
+                [&mesh](int vertexIndex, glm::vec3& outVertex) /* VertexSetter */ {
+                    outVertex = mesh.vertices[vertexIndex];
+                }
+            );
+        }
+    }
+}

libraries/model-baker/src/model-baker/CalculateMeshNormalsTask.h

@@ -0,0 +1,30 @@
+//
+//  CalculateMeshNormalsTask.h
+//  model-baker/src/model-baker
+//
+//  Created by Sabrina Shanman on 2019/01/07.
+//  Copyright 2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_CalculateMeshNormalsTask_h
+#define hifi_CalculateMeshNormalsTask_h
+
+#include <hfm/HFM.h>
+
+#include "Engine.h"
+#include "BakerTypes.h"
+
+// Calculate mesh normals if not already present in the mesh
+class CalculateMeshNormalsTask {
+public:
+    using Input = std::vector<hfm::Mesh>;
+    using Output = baker::NormalsPerMesh;
+    using JobModel = baker::Job::ModelIO<CalculateMeshNormalsTask, Input, Output>;
+
+    void run(const baker::BakeContextPointer& context, const Input& input, Output& output);
+};
+
+#endif // hifi_CalculateMeshNormalsTask_h

libraries/model-baker/src/model-baker/CalculateMeshTangentsTask.cpp

@@ -0,0 +1,65 @@
+//
+//  CalculateMeshTangentsTask.cpp
+//  model-baker/src/model-baker
+//
+//  Created by Sabrina Shanman on 2019/01/22.
+//  Copyright 2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "CalculateMeshTangentsTask.h"
+
+#include "ModelMath.h"
+
+void CalculateMeshTangentsTask::run(const baker::BakeContextPointer& context, const Input& input, Output& output) {
+    const auto& normalsPerMesh = input.get0();
+    const std::vector<hfm::Mesh>& meshes = input.get1();
+    const auto& materials = input.get2();
+    auto& tangentsPerMeshOut = output;
+
+    tangentsPerMeshOut.reserve(meshes.size());
+    for (int i = 0; i < (int)meshes.size(); i++) {
+        const auto& mesh = meshes[i];
+        const auto& tangentsIn = mesh.tangents;
+        const auto& normals = normalsPerMesh[i];
+        tangentsPerMeshOut.emplace_back();
+        auto& tangentsOut = tangentsPerMeshOut[tangentsPerMeshOut.size()-1];
+
+        // Check if we already have tangents and therefore do not need to do any calculation
+        if (!tangentsIn.empty()) {
+            tangentsOut = tangentsIn.toStdVector();
+            continue;
+        }
+
+        // Check if we have normals, and if not then tangents can't be calculated
+        if (normals.empty()) {
+            continue;
+        }
+
+        // Check if we actually need to calculate the tangents
+        bool needTangents = false;
+        for (const auto& meshPart : mesh.parts) {
+            auto materialIt = materials.find(meshPart.materialID);
+            if (materialIt != materials.end() && (*materialIt).needTangentSpace()) {
+                needTangents = true;
+                break;
+            }
+        }
+        if (needTangents) {
+            continue;
+        }
+
+        tangentsOut.resize(normals.size());
+        baker::calculateTangents(mesh,
+        [&mesh, &normals, &tangentsOut](int firstIndex, int secondIndex, glm::vec3* outVertices, glm::vec2* outTexCoords, glm::vec3& outNormal) {
+            outVertices[0] = mesh.vertices[firstIndex];
+            outVertices[1] = mesh.vertices[secondIndex];
+            outNormal = normals[firstIndex];
+            outTexCoords[0] = mesh.texCoords[firstIndex];
+            outTexCoords[1] = mesh.texCoords[secondIndex];
+            return &(tangentsOut[firstIndex]);
+        });
+    }
+}

libraries/model-baker/src/model-baker/CalculateMeshTangentsTask.h

@@ -0,0 +1,32 @@
+//
+//  CalculateMeshTangentsTask.h
+//  model-baker/src/model-baker
+//
+//  Created by Sabrina Shanman on 2019/01/07.
+//  Copyright 2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_CalculateMeshTangentsTask_h
+#define hifi_CalculateMeshTangentsTask_h
+
+#include <hfm/HFM.h>
+
+#include "Engine.h"
+#include "BakerTypes.h"
+
+// Calculate mesh tangents if not already present in the mesh
+class CalculateMeshTangentsTask {
+public:
+    using NormalsPerMesh = std::vector<std::vector<glm::vec3>>;
+
+    using Input = baker::VaryingSet3<baker::NormalsPerMesh, std::vector<hfm::Mesh>, QHash<QString, hfm::Material>>;
+    using Output = baker::TangentsPerMesh;
+    using JobModel = baker::Job::ModelIO<CalculateMeshTangentsTask, Input, Output>;
+
+    void run(const baker::BakeContextPointer& context, const Input& input, Output& output);
+};
+
+#endif // hifi_CalculateMeshTangentsTask_h

libraries/model-baker/src/model-baker/ModelMath.cpp

@@ -0,0 +1,121 @@
+//
+//  ModelMath.cpp
+//  model-baker/src/model-baker
+//
+//  Created by Sabrina Shanman on 2019/01/18.
+//  Copyright 2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "ModelMath.h"
+
+#include <LogHandler.h>
+#include "ModelBakerLogging.h"
+
+namespace baker {
+    template<class T>
+    const T& checkedAt(const QVector<T>& vector, int i) {
+        if (i < 0 || i >= vector.size()) {
+            throw std::out_of_range("baker::checked_at (ModelMath.cpp): index " + std::to_string(i) + " is out of range");
+        }
+        return vector[i];
+    }
+
+    template<class T>
+    const T& checkedAt(const std::vector<T>& vector, int i) {
+        if (i < 0 || i >= vector.size()) {
+            throw std::out_of_range("baker::checked_at (ModelMath.cpp): index " + std::to_string(i) + " is out of range");
+        }
+        return vector[i];
+    }
+
+    template<class T>
+    T& checkedAt(std::vector<T>& vector, int i) {
+        if (i < 0 || i >= vector.size()) {
+            throw std::out_of_range("baker::checked_at (ModelMath.cpp): index " + std::to_string(i) + " is out of range");
+        }
+        return vector[i];
+    }
+
+    void setTangent(const HFMMesh& mesh, const IndexAccessor& vertexAccessor, int firstIndex, int secondIndex) {
+        glm::vec3 vertex[2];
+        glm::vec2 texCoords[2];
+        glm::vec3 normal;
+        glm::vec3* tangent = vertexAccessor(firstIndex, secondIndex, vertex, texCoords, normal);
+        if (tangent) {
+            glm::vec3 bitangent = glm::cross(normal, vertex[1] - vertex[0]);
+            if (glm::length(bitangent) < EPSILON) {
+                return;
+            }
+            glm::vec2 texCoordDelta = texCoords[1] - texCoords[0];
+            glm::vec3 normalizedNormal = glm::normalize(normal);
+            *tangent += glm::cross(glm::angleAxis(-atan2f(-texCoordDelta.t, texCoordDelta.s), normalizedNormal) *
+                glm::normalize(bitangent), normalizedNormal);
+        }
+    }
+
+    void calculateNormals(const hfm::Mesh& mesh, NormalAccessor normalAccessor, VertexSetter vertexSetter) {
+        static int repeatMessageID = LogHandler::getInstance().newRepeatedMessageID();
+        for (const HFMMeshPart& part : mesh.parts) {
+            for (int i = 0; i < part.quadIndices.size(); i += 4) {
+                glm::vec3* n0 = normalAccessor(part.quadIndices[i]);
+                glm::vec3* n1 = normalAccessor(part.quadIndices[i + 1]);
+                glm::vec3* n2 = normalAccessor(part.quadIndices[i + 2]);
+                glm::vec3* n3 = normalAccessor(part.quadIndices[i + 3]);
+                if (!n0 || !n1 || !n2 || !n3) {
+                    // Quad is not in the mesh (can occur with blendshape meshes, which are a subset of the hfm Mesh vertices)
+                    continue;
+                }
+                glm::vec3 vertices[3]; // Assume all vertices in this quad are in the same plane, so only the first three are needed to calculate the normal
+                vertexSetter(part.quadIndices[i], vertices[0]);
+                vertexSetter(part.quadIndices[i + 1], vertices[1]);
+                vertexSetter(part.quadIndices[i + 2], vertices[2]);
+                *n0 = *n1 = *n2 = *n3 = glm::cross(vertices[1] - vertices[0], vertices[2] - vertices[0]);
+            }
+            // <= size - 3 in order to prevent overflowing triangleIndices when (i % 3) != 0
+            // This is most likely evidence of a further problem in extractMesh()
+            for (int i = 0; i <= part.triangleIndices.size() - 3; i += 3) {
+                glm::vec3* n0 = normalAccessor(part.triangleIndices[i]);
+                glm::vec3* n1 = normalAccessor(part.triangleIndices[i + 1]);
+                glm::vec3* n2 = normalAccessor(part.triangleIndices[i + 2]);
+                if (!n0 || !n1 || !n2) {
+                    // Tri is not in the mesh (can occur with blendshape meshes, which are a subset of the hfm Mesh vertices)
+                    continue;
+                }
+                glm::vec3 vertices[3];
+                vertexSetter(part.triangleIndices[i], vertices[0]);
+                vertexSetter(part.triangleIndices[i + 1], vertices[1]);
+                vertexSetter(part.triangleIndices[i + 2], vertices[2]);
+                *n0 = *n1 = *n2 = glm::cross(vertices[1] - vertices[0], vertices[2] - vertices[0]);
+            }
+            if ((part.triangleIndices.size() % 3) != 0) {
+                HIFI_FCDEBUG_ID(model_baker(), repeatMessageID, "Error in baker::calculateNormals: part.triangleIndices.size() is not divisible by three");
+            }
+        }
+    }
+
+    void calculateTangents(const hfm::Mesh& mesh, IndexAccessor accessor) {
+        static int repeatMessageID = LogHandler::getInstance().newRepeatedMessageID();
+        for (const HFMMeshPart& part : mesh.parts) {
+            for (int i = 0; i < part.quadIndices.size(); i += 4) {
+                setTangent(mesh, accessor, part.quadIndices.at(i), part.quadIndices.at(i + 1));
+                setTangent(mesh, accessor, part.quadIndices.at(i + 1), part.quadIndices.at(i + 2));
+                setTangent(mesh, accessor, part.quadIndices.at(i + 2), part.quadIndices.at(i + 3));
+                setTangent(mesh, accessor, part.quadIndices.at(i + 3), part.quadIndices.at(i));
+            }
+            // <= size - 3 in order to prevent overflowing triangleIndices when (i % 3) != 0
+            // This is most likely evidence of a further problem in extractMesh()
+            for (int i = 0; i <= part.triangleIndices.size() - 3; i += 3) {
+                setTangent(mesh, accessor, part.triangleIndices.at(i), part.triangleIndices.at(i + 1));
+                setTangent(mesh, accessor, part.triangleIndices.at(i + 1), part.triangleIndices.at(i + 2));
+                setTangent(mesh, accessor, part.triangleIndices.at(i + 2), part.triangleIndices.at(i));
+            }
+            if ((part.triangleIndices.size() % 3) != 0) {
+                HIFI_FCDEBUG_ID(model_baker(), repeatMessageID, "Error in baker::calculateTangents: part.triangleIndices.size() is not divisible by three");
+            }
+        }
+    }
+}
+

libraries/model-baker/src/model-baker/ModelMath.h

@@ -0,0 +1,34 @@
+//
+//  ModelMath.h
+//  model-baker/src/model-baker
+//
+//  Created by Sabrina Shanman on 2019/01/07.
+//  Copyright 2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <hfm/HFM.h>
+
+#include "BakerTypes.h"
+
+namespace baker {
+    // Returns a reference to the normal at the specified index, or nullptr if it cannot be accessed
+    using NormalAccessor = std::function<glm::vec3*(int index)>;
+
+    // Assigns a vertex to outVertex given the lookup index
+    using VertexSetter = std::function<void(int index, glm::vec3& outVertex)>;
+
+    void calculateNormals(const hfm::Mesh& mesh, NormalAccessor normalAccessor, VertexSetter vertexAccessor);
+
+    // firstIndex, secondIndex: the vertex indices to be used for calculation
+    // outVertices: should be assigned a 2 element array containing the vertices at firstIndex and secondIndex
+    // outTexCoords: same as outVertices but for texture coordinates
+    // outNormal: reference to the normal of this triangle
+    //
+    // Return value: pointer to the tangent you want to be calculated
+    using IndexAccessor = std::function<glm::vec3*(int firstIndex, int secondIndex, glm::vec3* outVertices, glm::vec2* outTexCoords, glm::vec3& outNormal)>;
+
+    void calculateTangents(const hfm::Mesh& mesh, IndexAccessor accessor);
+};

libraries/physics/src/EntityMotionState.cpp

@@ -81,7 +81,7 @@ EntityMotionState::EntityMotionState(btCollisionShape* shape, EntityItemPointer
     setShape(shape);
 
     if (_entity->isAvatarEntity() && _entity->getOwningAvatarID() != Physics::getSessionUUID()) {
-        // avatar entities entities are always thus, so we cache this fact in _ownershipState
+        // avatar entities are always thus, so we cache this fact in _ownershipState
         _ownershipState = EntityMotionState::OwnershipState::Unownable;
     }
 
@@ -211,6 +211,7 @@ PhysicsMotionType EntityMotionState::computePhysicsMotionType() const {
     }
     if (_entity->isMovingRelativeToParent() ||
         _entity->hasActions() ||
+        _entity->hasGrabs() ||
         _entity->hasAncestorOfType(NestableType::Avatar)) {
         return MOTION_TYPE_KINEMATIC;
     }
@@ -235,11 +236,19 @@ void EntityMotionState::getWorldTransform(btTransform& worldTrans) const {
     assert(entityTreeIsLocked());
     if (_motionType == MOTION_TYPE_KINEMATIC) {
         BT_PROFILE("kinematicIntegration");
+        uint32_t thisStep = ObjectMotionState::getWorldSimulationStep();
+        if (hasInternalKinematicChanges()) {
+            // ACTION_CAN_CONTROL_KINEMATIC_OBJECT_HACK: This kinematic body was moved by an Action
+            // and doesn't require transform update because the body is authoritative and its transform
+            // has already been copied out --> do no kinematic integration.
+            clearInternalKinematicChanges();
+            _lastKinematicStep = thisStep;
+            return;
+        }
         // This is physical kinematic motion which steps strictly by the subframe count
         // of the physics simulation and uses full gravity for acceleration.
         _entity->setAcceleration(_entity->getGravity());
 
-        uint32_t thisStep = ObjectMotionState::getWorldSimulationStep();
         float dt = (thisStep - _lastKinematicStep) * PHYSICS_ENGINE_FIXED_SUBSTEP;
         _lastKinematicStep = thisStep;
         _entity->stepKinematicMotion(dt);
@@ -767,6 +776,7 @@ bool EntityMotionState::shouldSendBid() const {
     // NOTE: this method is only ever called when the entity's simulation is NOT locally owned
     return _body->isActive()
         && (_region == workload::Region::R1)
+        && _ownershipState != EntityMotionState::OwnershipState::Unownable
         && glm::max(glm::max(VOLUNTEER_SIMULATION_PRIORITY, _bumpedPriority), _entity->getScriptSimulationPriority()) >= _entity->getSimulationPriority()
         && !_entity->getLocked();
 }

libraries/physics/src/EntityMotionState.h

@@ -88,7 +88,6 @@ public:
     virtual void computeCollisionGroupAndMask(int32_t& group, int32_t& mask) const override;
 
     bool shouldSendBid() const;
-    uint8_t computeFinalBidPriority() const;
 
     bool isLocallyOwned() const override;
     bool isLocallyOwnedOrShouldBe() const override; // aka shouldEmitCollisionEvents()
@@ -100,6 +99,7 @@ public:
     void saveKinematicState(btScalar timeStep) override;
 
 protected:
+    uint8_t computeFinalBidPriority() const;
     void updateSendVelocities();
     uint64_t getNextBidExpiry() const { return _nextBidExpiry; }
     void initForBid();

libraries/physics/src/ObjectMotionState.h

@@ -161,8 +161,9 @@ public:
 
     bool hasInternalKinematicChanges() const { return _hasInternalKinematicChanges; }
 
-    void dirtyInternalKinematicChanges() { _hasInternalKinematicChanges = true; }
-    void clearInternalKinematicChanges() { _hasInternalKinematicChanges = false; }
+    // these methods are declared const so they can be called inside other const methods
+    void dirtyInternalKinematicChanges() const { _hasInternalKinematicChanges = true; }
+    void clearInternalKinematicChanges() const { _hasInternalKinematicChanges = false; }
 
     virtual bool isLocallyOwned() const { return false; }
     virtual bool isLocallyOwnedOrShouldBe() const { return false; } // aka shouldEmitCollisionEvents()
@@ -186,8 +187,11 @@ protected:
     btRigidBody* _body { nullptr };
     float _density { 1.0f };
 
+    // ACTION_CAN_CONTROL_KINEMATIC_OBJECT_HACK: These date members allow an Action
+    // to operate on a kinematic object without screwing up our default kinematic integration
+    // which is done in the MotionState::getWorldTransform().
     mutable uint32_t _lastKinematicStep;
-    bool _hasInternalKinematicChanges { false };
+    mutable bool _hasInternalKinematicChanges { false };
 };
 
 using SetOfMotionStates = QSet<ObjectMotionState*>;

libraries/physics/src/ThreadSafeDynamicsWorld.cpp

@@ -100,9 +100,11 @@ void ThreadSafeDynamicsWorld::synchronizeMotionState(btRigidBody* body) {
     if (body->isKinematicObject()) {
         ObjectMotionState* objectMotionState = static_cast<ObjectMotionState*>(body->getMotionState());
         if (objectMotionState->hasInternalKinematicChanges()) {
-            // this is a special case where the kinematic motion has been updated by an Action
-            // so we supply the body's current transform to the MotionState
-            objectMotionState->clearInternalKinematicChanges();
+            // ACTION_CAN_CONTROL_KINEMATIC_OBJECT_HACK:
+            // This is a special case where the kinematic motion has been updated by an Action
+            // so we supply the body's current transform to the MotionState,
+            // but we DON'T clear the internalKinematicChanges bit here because
+            // objectMotionState.getWorldTransform() will use and clear it later
             body->getMotionState()->setWorldTransform(body->getWorldTransform());
         }
         return;

libraries/plugins/src/plugins/DisplayPlugin.h

@@ -142,6 +142,7 @@ public:
     // Rendering support
     virtual void setContext(const gpu::ContextPointer& context) final { _gpuContext = context; }
     virtual void submitFrame(const gpu::FramePointer& newFrame) = 0;
+    virtual void captureFrame(const std::string& outputName) const { }
 
     virtual float getRenderResolutionScale() const {
         return _renderResolutionScale;

libraries/render-utils/src/StencilMaskPass.cpp

@@ -42,7 +42,7 @@ gpu::PipelinePointer PrepareStencil::getMeshStencilPipeline() {
         auto program = gpu::Shader::createProgram(shader::gpu::program::drawNothing);
         auto state = std::make_shared<gpu::State>();
         drawMask(*state);
-        state->setColorWriteMask(0);
+        state->setColorWriteMask(gpu::State::WRITE_NONE);
 
         _meshStencilPipeline = gpu::Pipeline::create(program, state);
     }
@@ -54,7 +54,7 @@ gpu::PipelinePointer PrepareStencil::getPaintStencilPipeline() {
         auto program = gpu::Shader::createProgram(shader::render_utils::program::stencil_drawMask);
         auto state = std::make_shared<gpu::State>();
         drawMask(*state);
-        state->setColorWriteMask(0);
+        state->setColorWriteMask(gpu::State::WRITE_NONE);
 
         _paintStencilPipeline = gpu::Pipeline::create(program, state);
     }

libraries/shared/src/shared/Storage.cpp

@@ -70,12 +70,12 @@ StoragePointer FileStorage::create(const QString& filename, size_t size, const u
 }
 
 FileStorage::FileStorage(const QString& filename) : _file(filename) {
-    bool opened = _file.open(QFile::ReadWrite);
+    bool opened = _file.open(QFile::ReadWrite | QFile::Unbuffered);
     if (opened) {
         _hasWriteAccess = true;
     } else {
         _hasWriteAccess = false;
-        opened = _file.open(QFile::ReadOnly);
+        opened = _file.open(QFile::ReadOnly| QFile::Unbuffered);
     }
 
     if (opened) {

scripts/system/assets/data/createAppTooltips.json

@@ -370,6 +370,9 @@
     "priority": {
         "tooltip": "The priority of the material, where a larger number means higher priority. Original materials = 0."
     },
+    "materialMappingMode": {
+        "tooltip": "How the material is mapped to the entity. If set to \"UV space\", then the material will be applied with the target entity's UV coordinates. If set to \"3D Projected\", then the 3D transform of the material entity will be used."
+    },
     "materialMappingPos": {
         "tooltip": "The offset position of the bottom left of the material within the parent's UV space."
     },
@@ -379,6 +382,9 @@
     "materialMappingRot": {
         "tooltip": "How much to rotate the material within the parent's UV-space, in degrees."
     },
+    "materialRepeat": {
+        "tooltip": "If enabled, the material will repeat, otherwise it will clamp."
+    },
     "followCamera": {
         "tooltip": "If enabled, the grid is always visible even as the camera moves to another position."
     },

scripts/system/assets/images/icon-zone.svg

@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Generator: Adobe Illustrator 19.2.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   version="1.1"
+   id="Layer_1"
+   x="0px"
+   y="0px"
+   viewBox="0 0 512 512"
+   style="enable-background:new 0 0 512 512;"
+   xml:space="preserve"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="icon-zone.svg"><metadata
+     id="metadata4211"><rdf:RDF><cc:Work
+         rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" /></cc:Work></rdf:RDF></metadata><defs
+     id="defs4209" /><sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="2560"
+     inkscape:window-height="1377"
+     id="namedview4207"
+     showgrid="false"
+     inkscape:zoom="2"
+     inkscape:cx="228.01796"
+     inkscape:cy="376.88605"
+     inkscape:window-x="-8"
+     inkscape:window-y="-8"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="g4191" /><style
+     type="text/css"
+     id="style4189">
+	.st0{fill:none;stroke:#000000;stroke-width:18;stroke-miterlimit:10;}
+	.st1{fill:none;stroke:#000000;stroke-width:18;stroke-linecap:round;stroke-miterlimit:10;}
+	.st2{fill:none;stroke:#000000;stroke-width:19;stroke-linecap:round;stroke-miterlimit:10;}
+</style><g
+     id="g4191"><path
+       d="M 380.7,139.8 C 378,133.2 371.5,129 364.4,129 l -72.4,0 0,34 29.9,0 -162.9,163.1 0,-36.1 -36,0 0,79 0.006,0.024 c 0.1,2 0.5,4.1 1.3,5.9 C 127.00564,381.52397 134.7,386 141.8,386 l 83.2,0 0,-35 -40.8,0 161.8,-161.3 0,25.3 36.125,0.125 -0.19692,-62.65758 C 382.08839,148.14645 382.4,144 380.7,139.8 Z"
+       id="path4193"
+       style="fill:#333333;fill-opacity:1"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="cscccccccccscccccccc" /><path
+       d="M338.4,437.6c0-12,9.7-21.7,21.7-21.7l0,0c12,0,21.7,9.7,21.7,21.7l0,0c0,12-9.7,21.7-21.7,21.7l0,0   C348.1,459.3,338.4,449.6,338.4,437.6z M266.9,437.6c0-12,9.7-21.7,21.7-21.7l0,0c12,0,21.7,9.7,21.7,21.7l0,0   c0,12-9.7,21.7-21.7,21.7l0,0C276.6,459.3,266.9,449.6,266.9,437.6z M195.4,437.6c0-12,9.7-21.7,21.7-21.7l0,0   c12,0,21.7,9.7,21.7,21.7l0,0c0,12-9.7,21.7-21.7,21.7l0,0C205.1,459.3,195.4,449.6,195.4,437.6z M123.9,437.6   c0-12,9.7-21.7,21.7-21.7l0,0c12,0,21.7,9.7,21.7,21.7l0,0c0,12-9.7,21.7-21.7,21.7l0,0C133.6,459.3,123.9,449.6,123.9,437.6z"
+       id="path4195"
+       style="fill:#333333;fill-opacity:1" /><path
+       d="M74.1,459.3c-5.7,0-11.3-2.3-15.4-6.4c-4-4-6.4-9.6-6.4-15.3c0-5.7,2.3-11.3,6.4-15.3c4-4,9.6-6.4,15.4-6.4   c5.7,0,11.3,2.3,15.3,6.4c4,4,6.4,9.6,6.4,15.3c0,5.7-2.3,11.3-6.4,15.3C85.4,457,79.8,459.3,74.1,459.3z"
+       id="path4197"
+       style="fill:#333333;fill-opacity:1" /><path
+       d="M52.4,366.1c0-12,9.7-21.7,21.7-21.7l0,0c12,0,21.7,9.7,21.7,21.7l0,0c0,12-9.7,21.7-21.7,21.7l0,0   C62.1,387.8,52.4,378,52.4,366.1z M52.4,294.6c0-12,9.7-21.7,21.7-21.7l0,0c12,0,21.7,9.7,21.7,21.7l0,0c0,12-9.7,21.7-21.7,21.7   l0,0C62.1,316.3,52.4,306.5,52.4,294.6z M52.4,223.1c0-12,9.7-21.7,21.7-21.7l0,0c12,0,21.7,9.7,21.7,21.7l0,0   c0,12-9.7,21.7-21.7,21.7l0,0C62.1,244.8,52.4,235,52.4,223.1z M52.4,151.5c0-12,9.7-21.7,21.7-21.7l0,0c12,0,21.7,9.7,21.7,21.7   l0,0c0,12-9.7,21.7-21.7,21.7l0,0C62.1,173.2,52.4,163.5,52.4,151.5z"
+       id="path4199"
+       style="fill:#333333;fill-opacity:1" /><path
+       d="M338.4,80c0-12,9.7-21.7,21.7-21.7l0,0c12,0,21.7,9.7,21.7,21.7l0,0c0,12-9.7,21.7-21.7,21.7l0,0   C348.1,101.7,338.4,92,338.4,80z M266.9,80c0-12,9.7-21.7,21.7-21.7l0,0c12,0,21.7,9.7,21.7,21.7l0,0c0,12-9.7,21.7-21.7,21.7l0,0   C276.6,101.7,266.9,92,266.9,80z M195.4,80c0-12,9.7-21.7,21.7-21.7l0,0c12,0,21.7,9.7,21.7,21.7l0,0c0,12-9.7,21.7-21.7,21.7l0,0   C205.1,101.7,195.4,92,195.4,80z M123.9,80c0-12,9.7-21.7,21.7-21.7l0,0c12,0,21.7,9.7,21.7,21.7l0,0c0,12-9.7,21.7-21.7,21.7l0,0   C133.6,101.7,123.9,92,123.9,80z"
+       id="path4201"
+       style="fill:#333333;fill-opacity:1" /><path
+       d="M431.6,101.7c-5.7,0-11.3-2.3-15.3-6.4c-4-4-6.4-9.6-6.4-15.3c0-5.7,2.3-11.3,6.4-15.3c4-4,9.6-6.4,15.3-6.4   s11.3,2.3,15.3,6.4c4,4,6.4,9.6,6.4,15.3c0,5.7-2.3,11.3-6.4,15.3C442.9,99.4,437.4,101.7,431.6,101.7z"
+       id="path4203"
+       style="fill:#333333;fill-opacity:1" /><path
+       d="M409.9,366.1c0-12,9.7-21.7,21.7-21.7l0,0c12,0,21.7,9.7,21.7,21.7l0,0c0,12-9.7,21.7-21.7,21.7l0,0   C419.7,387.8,409.9,378.1,409.9,366.1z M409.9,294.6c0-12,9.7-21.7,21.7-21.7l0,0c12,0,21.7,9.7,21.7,21.7l0,0   c0,12-9.7,21.7-21.7,21.7l0,0C419.7,316.3,409.9,306.5,409.9,294.6z M409.9,223.1c0-12,9.7-21.7,21.7-21.7l0,0   c12,0,21.7,9.7,21.7,21.7l0,0c0,12-9.7,21.7-21.7,21.7l0,0C419.7,244.7,409.9,235,409.9,223.1z M409.9,151.5   c0-12,9.7-21.7,21.7-21.7l0,0c12,0,21.7,9.7,21.7,21.7l0,0c0,12-9.7,21.7-21.7,21.7l0,0C419.7,173.2,409.9,163.5,409.9,151.5z"
+       id="path4205"
+       style="fill:#333333;fill-opacity:1" /></g></svg>

scripts/system/assets/images/materials/GridPattern.json

@@ -0,0 +1,13 @@
+{
+    "materialVersion": 1,
+    "materials": {
+        "albedo": [
+            0.0,
+            0.0,
+            7.0
+        ],
+        "unlit": true,
+        "opacity": 0.4,
+        "albedoMap": "GridPattern.png"
+    }
+}

scripts/system/edit.js

@@ -82,13 +82,18 @@ var selectionManager = SelectionManager;
 var PARTICLE_SYSTEM_URL = Script.resolvePath("assets/images/icon-particles.svg");
 var POINT_LIGHT_URL = Script.resolvePath("assets/images/icon-point-light.svg");
 var SPOT_LIGHT_URL = Script.resolvePath("assets/images/icon-spot-light.svg");
+var ZONE_URL = Script.resolvePath("assets/images/icon-zone.svg");
 
-var entityIconOverlayManager = new EntityIconOverlayManager(['Light', 'ParticleEffect'], function(entityID) {
+var entityIconOverlayManager = new EntityIconOverlayManager(['Light', 'ParticleEffect', 'Zone'], function(entityID) {
     var properties = Entities.getEntityProperties(entityID, ['type', 'isSpotlight']);
     if (properties.type === 'Light') {
         return {
             url: properties.isSpotlight ? SPOT_LIGHT_URL : POINT_LIGHT_URL,
         };
+    } else if (properties.type === 'Zone') {
+        return {
+            url: ZONE_URL,
+        };
     } else {
         return {
             url: PARTICLE_SYSTEM_URL,
@@ -106,11 +111,15 @@ var gridTool = new GridTool({
 });
 gridTool.setVisible(false);
 
+var EntityShapeVisualizer = Script.require('./modules/entityShapeVisualizer.js');
+var entityShapeVisualizer = new EntityShapeVisualizer(["Zone"]);
+
 var entityListTool = new EntityListTool(shouldUseEditTabletApp);
 
 selectionManager.addEventListener(function () {
     selectionDisplay.updateHandles();
     entityIconOverlayManager.updatePositions();
+    entityShapeVisualizer.setEntities(selectionManager.selections);
 });
 
 var DEGREES_TO_RADIANS = Math.PI / 180.0;
@@ -836,7 +845,7 @@ var toolBar = (function () {
                     dialogWindow.fromQml.connect(fromQml);
                 }
             };
-        };
+        }
 
         addButton("newModelButton", createNewEntityDialogButtonCallback("Model"));
 
@@ -1492,6 +1501,7 @@ Script.scriptEnding.connect(function () {
     cleanupModelMenus();
     tooltip.cleanup();
     selectionDisplay.cleanup();
+    entityShapeVisualizer.cleanup();
     Entities.setLightsArePickable(originalLightsArePickable);
 
     Overlays.deleteOverlay(importingSVOImageOverlay);

scripts/system/html/js/entityProperties.js

@@ -3798,6 +3798,11 @@ function loaded() {
             if (FILTERED_NODE_NAMES.includes(keyUpEvent.target.nodeName)) {
                 return;
             }
+
+            if (elUserDataEditor.contains(keyUpEvent.target) || elMaterialDataEditor.contains(keyUpEvent.target)) {
+                return;
+            }
+
             let {code, key, keyCode, altKey, ctrlKey, metaKey, shiftKey} = keyUpEvent;
 
             let controlKey = window.navigator.platform.startsWith("Mac") ? metaKey : ctrlKey;

scripts/system/modules/entityShapeVisualizer.js

@@ -0,0 +1,255 @@
+"use strict";
+
+//  entityShapeVisualizer.js
+//
+//  Created by Thijs Wenker on 1/11/19
+//
+//  Copyright 2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+var SHAPETYPE_TO_SHAPE = {
+    "box": "Cube",
+    "ellipsoid": "Sphere",
+    "cylinder-y": "Cylinder",
+};
+
+var REQUESTED_ENTITY_SHAPE_PROPERTIES = [
+    'type', 'shapeType', 'compoundShapeURL', 'localDimensions'
+];
+
+function getEntityShapePropertiesForType(properties) {
+    switch (properties.type) {
+        case "Zone":
+            if (SHAPETYPE_TO_SHAPE[properties.shapeType]) {
+                return {
+                    type: "Shape",
+                    shape: SHAPETYPE_TO_SHAPE[properties.shapeType],
+                    localDimensions: properties.localDimensions
+                };
+            } else if (properties.shapeType === "compound") {
+                return {
+                    type: "Model",
+                    modelURL: properties.compoundShapeURL,
+                    localDimensions: properties.localDimensions
+                };
+            } else if (properties.shapeType === "sphere") {
+                var sphereDiameter = Math.max(properties.localDimensions.x, properties.localDimensions.y,
+                    properties.localDimensions.z);
+                return {
+                    type: "Sphere",
+                    modelURL: properties.compoundShapeURL,
+                    localDimensions: {x: sphereDiameter, y: sphereDiameter, z: sphereDiameter}
+                };
+            }
+            break;
+    }
+
+    // Default properties
+    return {
+        type: "Shape",
+        shape: "Cube",
+        localDimensions: properties.localDimensions
+    };
+}
+
+function deepEqual(a, b) {
+    if (a === b) {
+        return true;
+    }
+
+    if (typeof(a) !== "object" || typeof(b) !== "object") {
+        return false;
+    }
+
+    if (Object.keys(a).length !== Object.keys(b).length) {
+        return false;
+    }
+
+    for (var property in a) {
+        if (!a.hasOwnProperty(property)) {
+            continue;
+        }
+        if (!b.hasOwnProperty(property)) {
+            return false;
+        }
+        if (!deepEqual(a[property], b[property])) {
+            return false;
+        }
+    }
+    return true;
+}
+
+/**
+ * Returns an array of property names which are different in comparison.
+ * @param propertiesA
+ * @param propertiesB
+ * @returns {Array} - array of different property names
+ */
+function compareEntityProperties(propertiesA, propertiesB) {
+    var differentProperties = [],
+        property;
+
+    for (property in propertiesA) {
+        if (!propertiesA.hasOwnProperty(property)) {
+            continue;
+        }
+        if (!propertiesB.hasOwnProperty(property) || !deepEqual(propertiesA[property], propertiesB[property])) {
+            differentProperties.push(property);
+        }
+    }
+    for (property in propertiesB) {
+        if (!propertiesB.hasOwnProperty(property)) {
+            continue;
+        }
+        if (!propertiesA.hasOwnProperty(property)) {
+            differentProperties.push(property);
+        }
+    }
+
+    return differentProperties;
+}
+
+function deepCopy(v) {
+    return JSON.parse(JSON.stringify(v));
+}
+
+function EntityShape(entityID) {
+    this.entityID = entityID;
+    var propertiesForType = getEntityShapePropertiesForType(Entities.getEntityProperties(entityID, REQUESTED_ENTITY_SHAPE_PROPERTIES));
+
+    this.previousPropertiesForType = propertiesForType;
+
+    this.initialize(propertiesForType);
+}
+
+EntityShape.prototype = {
+    initialize: function(properties) {
+        // Create new instance of JS object:
+        var overlayProperties = deepCopy(properties);
+
+        overlayProperties.localPosition = Vec3.ZERO;
+        overlayProperties.localRotation = Quat.IDENTITY;
+        overlayProperties.canCastShadows = false;
+        overlayProperties.parentID = this.entityID;
+        overlayProperties.collisionless = true;
+        this.entity = Entities.addEntity(overlayProperties, "local");
+        var PROJECTED_MATERIALS = false;
+        this.materialEntity = Entities.addEntity({
+            type: "Material",
+            localPosition: Vec3.ZERO,
+            localRotation: Quat.IDENTITY,
+            localDimensions: properties.localDimensions,
+            parentID: this.entity,
+            priority: 1,
+            materialMappingMode: PROJECTED_MATERIALS ? "projected" : "uv",
+            materialURL: Script.resolvePath("../assets/images/materials/GridPattern.json"),
+        }, "local");
+    },
+    update: function() {
+        var propertiesForType = getEntityShapePropertiesForType(Entities.getEntityProperties(this.entityID, REQUESTED_ENTITY_SHAPE_PROPERTIES));
+
+        var difference = compareEntityProperties(propertiesForType, this.previousPropertiesForType);
+
+        if (deepEqual(difference, ['localDimensions'])) {
+            this.previousPropertiesForType = propertiesForType;
+            Entities.editEntity(this.entity, {
+                localDimensions: propertiesForType.localDimensions,
+            });
+        } else if (difference.length > 0) {
+            this.previousPropertiesForType = propertiesForType;
+            this.clear();
+            this.initialize(propertiesForType);
+        }
+    },
+    clear: function() {
+        Entities.deleteEntity(this.materialEntity);
+        Entities.deleteEntity(this.entity);
+    }
+};
+
+function EntityShapeVisualizer(visualizedTypes) {
+    this.acceptedEntities = [];
+    this.ignoredEntities = [];
+    this.entityShapes = {};
+
+    this.visualizedTypes = visualizedTypes;
+}
+
+EntityShapeVisualizer.prototype = {
+    addEntity: function(entityID) {
+        if (this.entityShapes[entityID]) {
+            return;
+        }
+        this.entityShapes[entityID] = new EntityShape(entityID);
+
+    },
+    updateEntity: function(entityID) {
+        if (!this.entityShapes[entityID]) {
+            return;
+        }
+        this.entityShapes[entityID].update();
+    },
+    removeEntity: function(entityID) {
+        if (!this.entityShapes[entityID]) {
+            return;
+        }
+        this.entityShapes[entityID].clear();
+        delete this.entityShapes[entityID];
+    },
+    cleanup: function() {
+        Object.keys(this.entityShapes).forEach(function(entityID) {
+            this.entityShapes[entityID].clear();
+        }, this);
+        this.entityShapes = {};
+    },
+    setEntities: function(entities) {
+        var qualifiedEntities = entities.filter(function(entityID) {
+            if (this.acceptedEntities.indexOf(entityID) !== -1) {
+                return true;
+            }
+            if (this.ignoredEntities.indexOf(entityID) !== -1) {
+                return false;
+            }
+            if (this.visualizedTypes.indexOf(Entities.getEntityProperties(entityID, "type").type) !== -1) {
+                this.acceptedEntities.push(entityID);
+                return true;
+            }
+            this.ignoredEntities.push(entityID);
+            return false;
+        }, this);
+
+
+        var newEntries = [];
+        var updateEntries = [];
+
+        var currentEntries = Object.keys(this.entityShapes);
+        qualifiedEntities.forEach(function(entityID) {
+            if (currentEntries.indexOf(entityID) !== -1) {
+                updateEntries.push(entityID);
+            } else {
+                newEntries.push(entityID);
+            }
+        });
+
+        var deleteEntries = currentEntries.filter(function(entityID) {
+            return updateEntries.indexOf(entityID) === -1;
+        });
+
+        deleteEntries.forEach(function(entityID) {
+            this.removeEntity(entityID);
+        }, this);
+
+        updateEntries.forEach(function(entityID) {
+            this.updateEntity(entityID);
+        }, this);
+
+        newEntries.forEach(function(entityID) {
+            this.addEntity(entityID);
+        }, this);
+    }
+};
+
+module.exports = EntityShapeVisualizer;

tools/CMakeLists.txt

@@ -6,31 +6,38 @@ if (NPM_EXECUTABLE)
   set_target_properties(jsdoc PROPERTIES FOLDER "Tools")
 endif()
 
+function(check_test name)
+    set(RESULT TRUE)
+    if (BUILD_TOOLS_INCLUDE)
+        unset(RESULT)
+        list(FIND BUILD_TOOLS_INCLUDE ${name} BUILD_TOOL_FIND)
+        if (NOT (${BUILD_TOOL_FIND} EQUAL -1))
+            set(RESULT TRUE)
+        endif()
+    endif()
+    set(BUILD_TOOL_RESULT ${RESULT} PARENT_SCOPE)
+endfunction()
+
 if (BUILD_TOOLS)
-  add_subdirectory(udt-test)
-  set_target_properties(udt-test PROPERTIES FOLDER "Tools")
+    set(ALL_TOOLS 
+        udt-test 
+        vhacd-util
+        frame-optimizer
+        gpu-frame-player
+        ice-client
+        ktx-tool
+        ac-client
+        skeleton-dump
+        atp-client
+        oven
+        nitpick
+    )
 
-  add_subdirectory(vhacd-util)
-  set_target_properties(vhacd-util PROPERTIES FOLDER "Tools")
-
-  add_subdirectory(ice-client)
-  set_target_properties(ice-client PROPERTIES FOLDER "Tools")
-
-  add_subdirectory(ktx-tool)
-  set_target_properties(ktx-tool PROPERTIES FOLDER "Tools")
-
-  add_subdirectory(ac-client)
-  set_target_properties(ac-client PROPERTIES FOLDER "Tools")
-
-  add_subdirectory(skeleton-dump)
-  set_target_properties(skeleton-dump PROPERTIES FOLDER "Tools")
-
-  add_subdirectory(atp-client)
-  set_target_properties(atp-client PROPERTIES FOLDER "Tools")
-
-  add_subdirectory(oven)
-  set_target_properties(oven PROPERTIES FOLDER "Tools")
-
-  add_subdirectory(nitpick)
-  set_target_properties(nitpick PROPERTIES FOLDER "Tools")
+    foreach(TOOL ${ALL_TOOLS})
+        check_test(${TOOL})
+        if (${BUILD_TOOL_RESULT})
+            add_subdirectory(${TOOL})
+            set_target_properties(${TOOL} PROPERTIES FOLDER "Tools")
+        endif()
+    endforeach()
 endif()

tools/frame-optimizer/CMakeLists.txt

@@ -0,0 +1,6 @@
+set(TARGET_NAME frame-optimizer)
+ 
+setup_memory_debugger()
+setup_hifi_project(Gui Widgets)
+link_hifi_libraries(shared ktx shaders gpu )
+package_libraries_for_deployment()