Cached shader binaries and uniform data for shaders

cmake/macros/SetupHifiTestCase.cmake

@@ -61,16 +61,21 @@ macro(SETUP_HIFI_TESTCASE)
       endif()
     endforeach()
 
+
     # Find test classes to build into test executables.
     # Warn about any .cpp files that are *not* test classes (*Test[s].cpp), since those files will not be used.
     foreach (SRC_FILE ${TEST_PROJ_SRC_FILES})   
       string(REGEX MATCH ".+Tests?\\.cpp$"   TEST_CPP_FILE ${SRC_FILE})
       string(REGEX MATCH ".+\\.cpp$"     NON_TEST_CPP_FILE ${SRC_FILE}) 
+      string(REGEX MATCH ".+\\.qrc$"     QRC_FILE ${SRC_FILE}) 
       if (TEST_CPP_FILE)
         list(APPEND TEST_CASE_FILES ${TEST_CPP_FILE})
       elseif (NON_TEST_CPP_FILE)
         message(WARNING "ignoring .cpp file (not a test class -- this will not be linked or compiled!): " ${NON_TEST_CPP_FILE})
       endif ()
+      if (QRC_FILE)
+        list(APPEND EXTRA_FILES ${QRC_FILE}) 
+      endif()
     endforeach ()
 
     if (TEST_CASE_FILES)
@@ -88,7 +93,7 @@ macro(SETUP_HIFI_TESTCASE)
         # grab the implemenation and header files
         set(TARGET_SRCS ${TEST_FILE})  # only one source / .cpp file (the test class)
       
-        add_executable(${TARGET_NAME} ${TEST_FILE})
+        add_executable(${TARGET_NAME} ${TEST_FILE} ${EXTRA_FILES})
         add_test(${TARGET_NAME}-test  ${TARGET_NAME})
         set_target_properties(${TARGET_NAME} PROPERTIES 
           EXCLUDE_FROM_DEFAULT_BUILD TRUE

interface/src/Application.cpp

@@ -2492,6 +2492,7 @@ void Application::cleanupBeforeQuit() {
     }
 
     _window->saveGeometry();
+    _gpuContext->shutdown();
 
     // Destroy third party processes after scripts have finished using them.
 #ifdef HAVE_DDE

libraries/gl/src/gl/GLShaders.cpp

@@ -2,15 +2,64 @@
 
 #include "GLLogging.h"
 
-namespace gl {
+#include <QtCore/QJsonDocument>
+#include <QtCore/QJsonValue>
+#include <QtCore/QJsonObject>
+#include <QtCore/QFileInfo>
+#include <QtCore/QCryptographicHash>
 
+#include <shared/FileUtils.h>
+
+using namespace gl;
+
+void Uniform::load(GLuint glprogram, int index) {
+    const GLint NAME_LENGTH = 256;
+    GLchar glname[NAME_LENGTH];
+    GLint length = 0;
+    glGetActiveUniform(glprogram, index, NAME_LENGTH, &length, &size, &type, glname);
+    name = std::string(glname, length);
+    location = glGetUniformLocation(glprogram, glname);
+}
+
+Uniforms gl::loadUniforms(GLuint glprogram) {
+    GLint uniformsCount = 0;
+    glGetProgramiv(glprogram, GL_ACTIVE_UNIFORMS, &uniformsCount);
+
+    Uniforms result;
+    result.resize(uniformsCount);
+    for (int i = 0; i < uniformsCount; i++) {
+        result[i].load(glprogram, i);
+    }
+    return result;
+}
 
 #ifdef SEPARATE_PROGRAM
-    bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, GLuint &programObject, std::string& message) {
+bool gl::compileShader(GLenum shaderDomain,
+                   const std::string& shaderSource,
+                   GLuint& shaderObject,
+                   GLuint& programObject,
+                   std::string& message) {
+    return compileShader(shaderDomain, std::vector<std::string>{ shaderSource }, shaderObject, programObject, message);
+}
 #else
-    bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, std::string& message) {
+bool gl::compileShader(GLenum shaderDomain, const std::string& shaderSource, GLuint& shaderObject, std::string& message) {
+    return compileShader(shaderDomain, std::vector<std::string>{ shaderSource }, shaderObject, message);
+}
 #endif
-    if (shaderSource.empty()) {
+
+#ifdef SEPARATE_PROGRAM
+bool gl::compileShader(GLenum shaderDomain,
+                   const std::string& shaderSource,
+                   GLuint& shaderObject,
+                   GLuint& programObject,
+                   std::string& message) {
+#else
+bool gl::compileShader(GLenum shaderDomain,
+                   const std::vector<std::string>& shaderSources,
+                   GLuint& shaderObject,
+                   std::string& message) {
+#endif
+    if (shaderSources.empty()) {
         qCDebug(glLogging) << "GLShader::compileShader - no GLSL shader source code ? so failed to create";
         return false;
     }
@@ -23,9 +72,11 @@ namespace gl {
     }
 
     // Assign the source
-    const int NUM_SOURCE_STRINGS = 2;
+    std::vector<const GLchar*> cstrs;
-    const GLchar* srcstr[] = { defines.c_str(), shaderSource.c_str() };
+    for (const auto& str : shaderSources) {
-    glShaderSource(glshader, NUM_SOURCE_STRINGS, srcstr, NULL);
+        cstrs.push_back(str.c_str());
+    }
+    glShaderSource(glshader, static_cast<GLint>(cstrs.size()), cstrs.data(), NULL);
 
     // Compile !
     glCompileShader(glshader);
@@ -66,7 +117,7 @@ namespace gl {
 
             qCCritical(glLogging) << "GLShader::compileShader - failed to compile the gl shader object:";
             int lineNumber = 0;
-            for (auto s : srcstr) {
+            for (const auto& s : cstrs) {
                 QString str(s);
                 QStringList lines = str.split("\n");
                 for (auto& line : lines) {
@@ -142,7 +193,7 @@ namespace gl {
     return true;
 }
 
-GLuint compileProgram(const std::vector<GLuint>& glshaders, std::string& message, std::vector<GLchar>& binary) {
+GLuint gl::compileProgram(const std::vector<GLuint>& glshaders, std::string& message, CachedShader& cachedShader) {
     // A brand new program:
     GLuint glprogram = glCreateProgram();
     if (!glprogram) {
@@ -150,14 +201,21 @@ GLuint compileProgram(const std::vector<GLuint>& glshaders, std::string& message
         return 0;
     }
 
-    // glProgramParameteri(glprogram, GL_PROGRAM_, GL_TRUE);
+    bool binaryLoaded = false;
-    // Create the program from the sub shaders
-    for (auto so : glshaders) {
-        glAttachShader(glprogram, so);
-    }
 
-    // Link!
+    if (glshaders.empty() && cachedShader) {
-    glLinkProgram(glprogram);
+        glProgramBinary(glprogram, cachedShader.format, cachedShader.binary.data(), (GLsizei)cachedShader.binary.size());
+        binaryLoaded = true;
+    } else {
+        // glProgramParameteri(glprogram, GL_PROGRAM_, GL_TRUE);
+        // Create the program from the sub shaders
+        for (auto so : glshaders) {
+            glAttachShader(glprogram, so);
+        }
+
+        // Link!
+        glLinkProgram(glprogram);
+    }
 
     GLint linked = 0;
     glGetProgramiv(glprogram, GL_LINK_STATUS, &linked);
@@ -205,25 +263,73 @@ GLuint compileProgram(const std::vector<GLuint>& glshaders, std::string& message
     }
 
     // If linked get the binaries
-    if (linked) {
+    if (linked && !binaryLoaded) {
         GLint binaryLength = 0;
         glGetProgramiv(glprogram, GL_PROGRAM_BINARY_LENGTH, &binaryLength);
-
         if (binaryLength > 0) {
-            GLint numBinFormats = 0;
+            cachedShader.binary.resize(binaryLength);
-            glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, &numBinFormats);
+            glGetProgramBinary(glprogram, binaryLength, NULL, &cachedShader.format, cachedShader.binary.data());
-            if (numBinFormats > 0) {
-                binary.resize(binaryLength);
-                std::vector<GLint> binFormats(numBinFormats);
-                glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, binFormats.data());
-
-                GLenum programBinFormat;
-                glGetProgramBinary(glprogram, binaryLength, NULL, &programBinFormat, binary.data());
-            }
         }
     }
 
     return glprogram;
 }
 
+static const QString& getShaderCacheFile() {
+    static const QString SHADER_CACHE_FOLDER{ "shaders" };
+    static const QString SHADER_CACHE_FILE_NAME{ "cache.json" };
+    static const QString SHADER_CACHE_FILE = FileUtils::standardPath(SHADER_CACHE_FOLDER) + SHADER_CACHE_FILE_NAME;
+    return SHADER_CACHE_FILE;
+}
+
+static const char* SHADER_JSON_TYPE_KEY = "type";
+static const char* SHADER_JSON_SOURCE_KEY = "source";
+static const char* SHADER_JSON_DATA_KEY = "data";
+
+void gl::loadShaderCache(ShaderCache& cache) {
+    QString shaderCacheFile = getShaderCacheFile();
+    if (QFileInfo(shaderCacheFile).exists()) {
+        QString json = FileUtils::readFile(shaderCacheFile);
+        auto root = QJsonDocument::fromJson(json.toUtf8()).object();
+        for (const auto& qhash : root.keys()) {
+            auto programObject = root[qhash].toObject();
+            QByteArray qbinary = QByteArray::fromBase64(programObject[SHADER_JSON_DATA_KEY].toString().toUtf8());
+            std::string hash = qhash.toStdString();
+            auto& cachedShader = cache[hash];
+            cachedShader.binary.resize(qbinary.size());
+            memcpy(cachedShader.binary.data(), qbinary.data(), qbinary.size());
+            cachedShader.format = (GLenum)programObject[SHADER_JSON_TYPE_KEY].toInt();
+            cachedShader.source = programObject[SHADER_JSON_SOURCE_KEY].toString().toStdString();
+        }
+    }
+}
+
+void gl::saveShaderCache(const ShaderCache& cache) {
+    QByteArray json;
+    {
+        QVariantMap variantMap;
+        for (const auto& entry : cache) {
+            const auto& key = entry.first;
+            const auto& type = entry.second.format;
+            const auto& binary = entry.second.binary;
+            QVariantMap qentry;
+            qentry[SHADER_JSON_TYPE_KEY] = QVariant(type);
+            qentry[SHADER_JSON_SOURCE_KEY] = QString(entry.second.source.c_str());
+            qentry[SHADER_JSON_DATA_KEY] = QByteArray{ binary.data(), (int)binary.size() }.toBase64();
+            variantMap[key.c_str()] = qentry;
+        }
+        json = QJsonDocument::fromVariant(variantMap).toJson(QJsonDocument::Indented);
+    }
+
+    if (!json.isEmpty()) {
+        QString shaderCacheFile = getShaderCacheFile();
+        QFile saveFile(shaderCacheFile);
+        saveFile.open(QFile::WriteOnly | QFile::Text | QFile::Truncate);
+        saveFile.write(json);
+        saveFile.close();
+    }
+}
+
+std::string gl::getShaderHash(const std::string& shaderSource) {
+    return QCryptographicHash::hash(QByteArray(shaderSource.c_str()), QCryptographicHash::Md5).toBase64().toStdString();
 }

libraries/gl/src/gl/GLShaders.h

@@ -14,15 +14,47 @@
 
 #include <vector>
 #include <string>
+#include <unordered_map>
 
 namespace gl {
+
+    struct Uniform {
+        std::string name;
+        GLint size{ -1 };
+        GLenum type{ GL_FLOAT };
+        GLint location{ -1 };
+        void load(GLuint glprogram, int index);
+    };
+
+    using Uniforms = std::vector<Uniform>;
+
+    Uniforms loadUniforms(GLuint glprogram);
+
+    struct CachedShader {
+        GLenum format{ 0 };
+        std::string source;
+        std::vector<char> binary;
+        inline operator bool() const {
+            return format != 0 && !binary.empty();
+        }
+    };
+
+    using ShaderCache = std::unordered_map<std::string, CachedShader>;
+
+    std::string getShaderHash(const std::string& shaderSource);
+    void loadShaderCache(ShaderCache& cache);
+    void saveShaderCache(const ShaderCache& cache);
+
+
 #ifdef SEPARATE_PROGRAM
-    bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, GLuint &programObject, std::string& message);
+    bool compileShader(GLenum shaderDomain, const std::string& shaderSource, GLuint &shaderObject, GLuint &programObject, std::string& message);
+    bool compileShader(GLenum shaderDomain, const std::vector<std::string>& shaderSources, GLuint &shaderObject, GLuint &programObject, std::string& message);
 #else
-    bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, std::string& message);
+    bool compileShader(GLenum shaderDomain, const std::string& shaderSource, GLuint &shaderObject, std::string& message);
+    bool compileShader(GLenum shaderDomain, const std::vector<std::string>& shaderSources, GLuint &shaderObject, std::string& message);
 #endif
 
-    GLuint compileProgram(const std::vector<GLuint>& glshaders, std::string& message, std::vector<GLchar>& binary);
+    GLuint compileProgram(const std::vector<GLuint>& glshaders, std::string& message, CachedShader& binary);
 
 }
 

libraries/gpu-gl-common/src/gpu/gl/GLBackend.cpp

@@ -124,13 +124,16 @@ void GLBackend::init() {
 GLBackend::GLBackend() {
     _pipeline._cameraCorrectionBuffer._buffer->flush();
     glGetIntegerv(GL_UNIFORM_BUFFER_OFFSET_ALIGNMENT, &_uboAlignment);
+    initShaderBinaryCache();
 }
 
+GLBackend::~GLBackend() {}
 
-GLBackend::~GLBackend() {
+void GLBackend::shutdown() {
     killInput();
     killTransform();
     killTextureManagementStage();
+    killShaderBinaryCache();
 }
 
 void GLBackend::renderPassTransfer(const Batch& batch) {

libraries/gpu-gl-common/src/gpu/gl/GLBackend.h

@@ -23,6 +23,7 @@
 #include <QtCore/QLoggingCategory>
 
 #include <gl/Config.h>
+#include <gl/GLShaders.h>
 
 #include <gpu/Forward.h>
 #include <gpu/Context.h>
@@ -71,6 +72,9 @@ public:
 
     virtual ~GLBackend();
 
+    // Shutdown rendering and persist any required resources
+    void shutdown() override;
+
     void setCameraCorrection(const Mat4& correction, const Mat4& prevRenderView, bool reset = false);
     void render(const Batch& batch) final override;
 
@@ -455,6 +459,13 @@ protected:
     virtual GLShader* compileBackendProgram(const Shader& program, const Shader::CompilationHandler& handler);
     virtual GLShader* compileBackendShader(const Shader& shader, const Shader::CompilationHandler& handler);
     virtual std::string getBackendShaderHeader() const = 0;
+    // For a program, this will return a string containing all the source files (without any 
+    // backend headers or defines).  For a vertex, fragment or geometry shader, this will 
+    // return the fully customized shader with all the version and backend specific 
+    // preprocessor directives
+    // The program string returned can be used as a key for a cache of shader binaries
+    // The shader strings can be reliably sent to the low level `compileShader` functions
+    virtual std::string getShaderSource(const Shader& shader, int version) final;
     virtual void makeProgramBindings(ShaderObject& shaderObject);
     class ElementResource {
     public:
@@ -465,12 +476,12 @@ protected:
     ElementResource getFormatFromGLUniform(GLenum gltype);
     static const GLint UNUSED_SLOT {-1};
     static bool isUnusedSlot(GLint binding) { return (binding == UNUSED_SLOT); }
-    virtual int makeUniformSlots(GLuint glprogram, const Shader::BindingSet& slotBindings,
+    virtual int makeUniformSlots(const ShaderObject& program, const Shader::BindingSet& slotBindings,
         Shader::SlotSet& uniforms, Shader::SlotSet& textures, Shader::SlotSet& samplers);
-    virtual int makeUniformBlockSlots(GLuint glprogram, const Shader::BindingSet& slotBindings, Shader::SlotSet& buffers);
+    virtual int makeUniformBlockSlots(const ShaderObject& program, const Shader::BindingSet& slotBindings, Shader::SlotSet& buffers);
-    virtual int makeResourceBufferSlots(GLuint glprogram, const Shader::BindingSet& slotBindings, Shader::SlotSet& resourceBuffers) = 0;
+    virtual int makeResourceBufferSlots(const ShaderObject& program, const Shader::BindingSet& slotBindings, Shader::SlotSet& resourceBuffers) = 0;
-    virtual int makeInputSlots(GLuint glprogram, const Shader::BindingSet& slotBindings, Shader::SlotSet& inputs);
+    virtual int makeInputSlots(const ShaderObject& program, const Shader::BindingSet& slotBindings, Shader::SlotSet& inputs);
-    virtual int makeOutputSlots(GLuint glprogram, const Shader::BindingSet& slotBindings, Shader::SlotSet& outputs);
+    virtual int makeOutputSlots(const ShaderObject& program, const Shader::BindingSet& slotBindings, Shader::SlotSet& outputs);
 
 
     // Synchronize the state cache of this Backend with the actual real state of the GL Context
@@ -489,6 +500,19 @@ protected:
 
     void resetStages();
 
+    // Stores cached binary versions of the shaders for quicker startup on subsequent runs
+    // Note that shaders in the cache can still fail to load due to hardware or driver 
+    // changes that invalidate the cached binary, in which case we fall back on compiling 
+    // the source again
+    struct ShaderBinaryCache {
+        std::mutex _mutex;
+        std::vector<GLint> _formats;
+        std::unordered_map<std::string, ::gl::CachedShader> _binaries;
+    } _shaderBinaryCache;
+
+    virtual void initShaderBinaryCache();
+    virtual void killShaderBinaryCache();
+
     struct TextureManagementStageState {
         bool _sparseCapable { false };
         GLTextureTransferEnginePointer _transferEngine;

libraries/gpu-gl-common/src/gpu/gl/GLBackendShader.cpp

@@ -11,6 +11,8 @@
 
 using namespace gpu;
 using namespace gpu::gl;
+using CachedShader = ::gl::CachedShader;
+
 
 // Shader domain
 static const size_t NUM_SHADER_DOMAINS = 3;
@@ -68,9 +70,45 @@ static const std::array<std::string, GLShader::NumVersions> VERSION_DEFINES { {
     stereoVersion
 } };
 
+static std::string getShaderTypeString(Shader::Type type) {
+    switch (type) {
+    case Shader::Type::VERTEX:
+        return "vertex";
+    case Shader::Type::PIXEL:
+        return "pixel";
+    case Shader::Type::GEOMETRY:
+        return "geometry";
+    case Shader::Type::PROGRAM:
+        return "program";
+    default:
+        qFatal("Unexpected shader type %d", type);
+        Q_UNREACHABLE();
+    }
+}
+
+std::string GLBackend::getShaderSource(const Shader& shader, int version) {
+    if (shader.isProgram()) {
+        std::string result;
+        result.append("// VERSION " + std::to_string(version));
+        for (const auto& subShader : shader.getShaders()) {
+            result.append("//-------- ");
+            result.append(getShaderTypeString(subShader->getType()));
+            result.append("\n");
+            result.append(subShader->getSource().getCode());
+        }
+        return result;
+    } 
+
+    std::string shaderDefines = getBackendShaderHeader() + "\n"
+        + (supportsBindless() ? textureTableVersion : "\n")
+        + DOMAIN_DEFINES[shader.getType()] + "\n"
+        + VERSION_DEFINES[version];
+
+    return shaderDefines + "\n" + shader.getSource().getCode();
+}
+
 GLShader* GLBackend::compileBackendShader(const Shader& shader, const Shader::CompilationHandler& handler) {
     // Any GLSLprogram ? normally yes...
-    const std::string& shaderSource = shader.getSource().getCode();
     GLenum shaderDomain = SHADER_DOMAINS[shader.getType()];
     GLShader::ShaderObjects shaderObjects;
     Shader::CompilationLogs compilationLogs(GLShader::NumVersions);
@@ -78,11 +116,7 @@ GLShader* GLBackend::compileBackendShader(const Shader& shader, const Shader::Co
 
     for (int version = 0; version < GLShader::NumVersions; version++) {
         auto& shaderObject = shaderObjects[version];
-
+        auto shaderSource = getShaderSource(shader, version);
-        std::string shaderDefines = getBackendShaderHeader() + "\n"
-            + (supportsBindless() ? textureTableVersion : "\n")
-            + DOMAIN_DEFINES[shader.getType()] + "\n"
-            + VERSION_DEFINES[version];
         if (handler) {
             bool retest = true;
             std::string currentSrc = shaderSource;
@@ -90,7 +124,7 @@ GLShader* GLBackend::compileBackendShader(const Shader& shader, const Shader::Co
             // The retest bool is set to false as soon as the compilation succeed to wexit the while loop.
             // The handler tells us if we should retry or not while returning a modified version of the source.
             while (retest) {
-                bool result = ::gl::compileShader(shaderDomain, currentSrc, shaderDefines, shaderObject.glshader, compilationLogs[version].message);
+                bool result = ::gl::compileShader(shaderDomain, currentSrc, shaderObject.glshader, compilationLogs[version].message);
                 compilationLogs[version].compiled = result;
                 if (!result) {
                     std::string newSrc;
@@ -101,7 +135,7 @@ GLShader* GLBackend::compileBackendShader(const Shader& shader, const Shader::Co
                 }
             }
         } else {
-            compilationLogs[version].compiled = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, compilationLogs[version].message);
+            compilationLogs[version].compiled = ::gl::compileShader(shaderDomain, shaderSource, shaderObject.glshader, compilationLogs[version].message);
         }
 
         if (!compilationLogs[version].compiled) {
@@ -126,37 +160,69 @@ GLShader* GLBackend::compileBackendProgram(const Shader& program, const Shader::
     }
 
     GLShader::ShaderObjects programObjects;
-
     program.incrementCompilationAttempt();
     Shader::CompilationLogs compilationLogs(GLShader::NumVersions);
 
     for (int version = 0; version < GLShader::NumVersions; version++) {
         auto& programObject = programObjects[version];
+        auto programSource = getShaderSource(program, version);
+        auto hash = ::gl::getShaderHash(programSource);
 
-        // Let's go through every shaders and make sure they are ready to go
+        CachedShader cachedBinary;
-        std::vector< GLuint > shaderGLObjects;
+        {
-        for (auto subShader : program.getShaders()) {
+            Lock shaderCacheLock{ _shaderBinaryCache._mutex };
-            auto object = GLShader::sync((*this), *subShader, handler);
+            if (_shaderBinaryCache._binaries.count(hash) != 0) {
-            if (object) {
+                cachedBinary = _shaderBinaryCache._binaries[hash];
-                shaderGLObjects.push_back(object->_shaderObjects[version].glshader);
+            }
-            } else {
+        }
-                qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - One of the shaders of the program is not compiled?";
+
-                compilationLogs[version].compiled = false;
+
-                compilationLogs[version].message = std::string("Failed to compile, one of the shaders of the program is not compiled ?");
+        GLuint glprogram = 0;
-                program.setCompilationLogs(compilationLogs);
+
-                return nullptr;
+        // If we have a cached binary program, try to load it instead of compiling the individual shaders
+        if (cachedBinary) {
+            glprogram = ::gl::compileProgram({}, compilationLogs[version].message, cachedBinary);
+        }
+
+        // If we have no program, then either no cached binary, or the binary failed to load (perhaps a GPU driver update invalidated the cache)
+        if (0 == glprogram) {
+            cachedBinary = CachedShader();
+            {
+                std::unique_lock<std::mutex> shaderCacheLock{ _shaderBinaryCache._mutex };
+                _shaderBinaryCache._binaries.erase(hash);
+            }
+            // Let's go through every shaders and make sure they are ready to go
+            std::vector<GLuint> shaderGLObjects;
+            shaderGLObjects.reserve(program.getShaders().size());
+            for (auto subShader : program.getShaders()) {
+                auto object = GLShader::sync((*this), *subShader, handler);
+                if (object) {
+                    shaderGLObjects.push_back(object->_shaderObjects[version].glshader);
+                } else {
+                    qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - One of the shaders of the program is not compiled?";
+                    compilationLogs[version].compiled = false;
+                    compilationLogs[version].message = std::string("Failed to compile, one of the shaders of the program is not compiled ?");
+                    program.setCompilationLogs(compilationLogs);
+                    return nullptr;
+                }
+            }
+
+            glprogram = ::gl::compileProgram(shaderGLObjects, compilationLogs[version].message, cachedBinary);
+            if (cachedBinary) {
+                cachedBinary.source = programSource;
+                std::unique_lock<std::mutex> shaderCacheLock{ _shaderBinaryCache._mutex };
+                _shaderBinaryCache._binaries[hash] = cachedBinary;
             }
         }
 
-        GLuint glprogram = ::gl::compileProgram(shaderGLObjects, compilationLogs[version].message, compilationLogs[version].binary);
         if (glprogram == 0) {
             qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Program didn't link:\n" << compilationLogs[version].message.c_str(); 
             program.setCompilationLogs(compilationLogs);
             return nullptr;
         }
+
         compilationLogs[version].compiled = true;
         programObject.glprogram = glprogram;
-
         makeProgramBindings(programObject);
     }
     // Compilation feedback
@@ -338,20 +404,15 @@ GLBackend::ElementResource GLBackend::getFormatFromGLUniform(GLenum gltype) {
 
 };
 
-int GLBackend::makeUniformSlots(GLuint glprogram, const Shader::BindingSet& slotBindings,
+int GLBackend::makeUniformSlots(const ShaderObject& shaderProgram, const Shader::BindingSet& slotBindings,
     Shader::SlotSet& uniforms, Shader::SlotSet& textures, Shader::SlotSet& samplers) {
-    GLint uniformsCount = 0;
+    auto& glprogram = shaderProgram.glprogram;
 
-    glGetProgramiv(glprogram, GL_ACTIVE_UNIFORMS, &uniformsCount);
+    for (const auto& uniform : shaderProgram.uniforms) {
-
+        const auto& type = uniform.type;
-    for (int i = 0; i < uniformsCount; i++) {
+        const auto& location = uniform.location;
-        const GLint NAME_LENGTH = 256;
+        const auto& size = uniform.size;
-        GLchar name[NAME_LENGTH];
+        const auto& name = uniform.name;
-        GLint length = 0;
-        GLint size = 0;
-        GLenum type = 0;
-        glGetActiveUniform(glprogram, i, NAME_LENGTH, &length, &size, &type, name);
-        GLint location = glGetUniformLocation(glprogram, name);
         const GLint INVALID_UNIFORM_LOCATION = -1;
 
         // Try to make sense of the gltype
@@ -359,8 +420,8 @@ int GLBackend::makeUniformSlots(GLuint glprogram, const Shader::BindingSet& slot
 
         // The uniform as a standard var type
         if (location != INVALID_UNIFORM_LOCATION) {
+            auto sname = uniform.name;
             // Let's make sure the name doesn't contains an array element
-            std::string sname(name);
             auto foundBracket = sname.find_first_of('[');
             if (foundBracket != std::string::npos) {
                 //  std::string arrayname = sname.substr(0, foundBracket);
@@ -397,10 +458,11 @@ int GLBackend::makeUniformSlots(GLuint glprogram, const Shader::BindingSet& slot
         }
     }
 
-    return uniformsCount;
+    return static_cast<uint32_t>(shaderProgram.uniforms.size());
 }
 
-int GLBackend::makeUniformBlockSlots(GLuint glprogram, const Shader::BindingSet& slotBindings, Shader::SlotSet& buffers) {
+int GLBackend::makeUniformBlockSlots(const ShaderObject& shaderProgram, const Shader::BindingSet& slotBindings, Shader::SlotSet& buffers) {
+    const auto& glprogram = shaderProgram.glprogram;
     GLint buffersCount = 0;
 
     glGetProgramiv(glprogram, GL_ACTIVE_UNIFORM_BLOCKS, &buffersCount);
@@ -479,7 +541,8 @@ int GLBackend::makeUniformBlockSlots(GLuint glprogram, const Shader::BindingSet&
     return buffersCount;
 }
 
-int GLBackend::makeInputSlots(GLuint glprogram, const Shader::BindingSet& slotBindings, Shader::SlotSet& inputs) {
+int GLBackend::makeInputSlots(const ShaderObject& shaderProgram, const Shader::BindingSet& slotBindings, Shader::SlotSet& inputs) {
+    const auto& glprogram = shaderProgram.glprogram;
     GLint inputsCount = 0;
 
     glGetProgramiv(glprogram, GL_ACTIVE_ATTRIBUTES, &inputsCount);
@@ -501,7 +564,7 @@ int GLBackend::makeInputSlots(GLuint glprogram, const Shader::BindingSet& slotBi
     return inputsCount;
 }
 
-int GLBackend::makeOutputSlots(GLuint glprogram, const Shader::BindingSet& slotBindings, Shader::SlotSet& outputs) {
+int GLBackend::makeOutputSlots(const ShaderObject& shaderProgram, const Shader::BindingSet& slotBindings, Shader::SlotSet& outputs) {
     /*   GLint outputsCount = 0;
 
     glGetProgramiv(glprogram, GL_ACTIVE_, &outputsCount);
@@ -525,67 +588,19 @@ void GLBackend::makeProgramBindings(ShaderObject& shaderObject) {
     if (!shaderObject.glprogram) {
         return;
     }
-    GLuint glprogram = shaderObject.glprogram;
+}
-    GLint loc = -1;
+
-
+
-    //Check for gpu specific attribute slotBindings
+void GLBackend::initShaderBinaryCache() {
-    loc = glGetAttribLocation(glprogram, "inPosition");
+    GLint numBinFormats = 0;
-    if (loc >= 0 && loc != gpu::Stream::POSITION) {
+    glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, &numBinFormats);
-        glBindAttribLocation(glprogram, gpu::Stream::POSITION, "inPosition");
+    if (numBinFormats > 0) {
-    }
+        _shaderBinaryCache._formats.resize(numBinFormats);
-
+        glGetIntegerv(GL_PROGRAM_BINARY_FORMATS, _shaderBinaryCache._formats.data());
-    loc = glGetAttribLocation(glprogram, "inNormal");
+    }
-    if (loc >= 0 && loc != gpu::Stream::NORMAL) {
+    ::gl::loadShaderCache(_shaderBinaryCache._binaries);
-        glBindAttribLocation(glprogram, gpu::Stream::NORMAL, "inNormal");
+}
-    }
+
-
+void GLBackend::killShaderBinaryCache() {
-    loc = glGetAttribLocation(glprogram, "inColor");
+    ::gl::saveShaderCache(_shaderBinaryCache._binaries);
-    if (loc >= 0 && loc != gpu::Stream::COLOR) {
-        glBindAttribLocation(glprogram, gpu::Stream::COLOR, "inColor");
-    }
-
-    loc = glGetAttribLocation(glprogram, "inTexCoord0");
-    if (loc >= 0 && loc != gpu::Stream::TEXCOORD) {
-        glBindAttribLocation(glprogram, gpu::Stream::TEXCOORD, "inTexCoord0");
-    }
-
-    loc = glGetAttribLocation(glprogram, "inTangent");
-    if (loc >= 0 && loc != gpu::Stream::TANGENT) {
-        glBindAttribLocation(glprogram, gpu::Stream::TANGENT, "inTangent");
-    }
-
-    char attribName[] = "inTexCoordn";
-    for (auto i = 0; i < 4; i++) {
-        auto streamId = gpu::Stream::TEXCOORD1 + i;
-
-        attribName[strlen(attribName) - 1] = '1' + i;
-        loc = glGetAttribLocation(glprogram, attribName);
-        if (loc >= 0 && loc != streamId) {
-            glBindAttribLocation(glprogram, streamId, attribName);
-        }
-    }
-
-    loc = glGetAttribLocation(glprogram, "inSkinClusterIndex");
-    if (loc >= 0 && loc != gpu::Stream::SKIN_CLUSTER_INDEX) {
-        glBindAttribLocation(glprogram, gpu::Stream::SKIN_CLUSTER_INDEX, "inSkinClusterIndex");
-    }
-
-    loc = glGetAttribLocation(glprogram, "inSkinClusterWeight");
-    if (loc >= 0 && loc != gpu::Stream::SKIN_CLUSTER_WEIGHT) {
-        glBindAttribLocation(glprogram, gpu::Stream::SKIN_CLUSTER_WEIGHT, "inSkinClusterWeight");
-    }
-
-    loc = glGetAttribLocation(glprogram, "_drawCallInfo");
-    if (loc >= 0 && loc != gpu::Stream::DRAW_CALL_INFO) {
-        glBindAttribLocation(glprogram, gpu::Stream::DRAW_CALL_INFO, "_drawCallInfo");
-    }
-
-    // Link again to take into account the assigned attrib location
-    glLinkProgram(glprogram);
-
-    GLint linked = 0;
-    glGetProgramiv(glprogram, GL_LINK_STATUS, &linked);
-    if (!linked) {
-        qCWarning(gpugllogging) << "GLShader::makeBindings - failed to link after assigning slotBindings?";
-    }
 }

libraries/gpu-gl-common/src/gpu/gl/GLShader.cpp

@@ -68,22 +68,23 @@ bool GLShader::makeProgram(GLBackend& backend, Shader& shader, const Shader::Bin
     for (int version = 0; version < GLShader::NumVersions; version++) {
         auto& shaderObject = object->_shaderObjects[version];
         if (shaderObject.glprogram) {
+            shaderObject.uniforms = ::gl::loadUniforms(shaderObject.glprogram);
             Shader::SlotSet buffers;
-            backend.makeUniformBlockSlots(shaderObject.glprogram, slotBindings, buffers);
+            backend.makeUniformBlockSlots(shaderObject, slotBindings, buffers);
 
             Shader::SlotSet uniforms;
             Shader::SlotSet textures;
             Shader::SlotSet samplers;
-            backend.makeUniformSlots(shaderObject.glprogram, slotBindings, uniforms, textures, samplers);
+            backend.makeUniformSlots(shaderObject, slotBindings, uniforms, textures, samplers);
 
             Shader::SlotSet resourceBuffers;
-            backend.makeResourceBufferSlots(shaderObject.glprogram, slotBindings, resourceBuffers);
+            backend.makeResourceBufferSlots(shaderObject, slotBindings, resourceBuffers);
 
             Shader::SlotSet inputs;
-            backend.makeInputSlots(shaderObject.glprogram, slotBindings, inputs);
+            backend.makeInputSlots(shaderObject, slotBindings, inputs);
 
             Shader::SlotSet outputs;
-            backend.makeOutputSlots(shaderObject.glprogram, slotBindings, outputs);
+            backend.makeOutputSlots(shaderObject, slotBindings, outputs);
 
             // Define the public slots only from the default version
             if (version == 0) {

libraries/gpu-gl-common/src/gpu/gl/GLShader.h

@@ -9,14 +9,17 @@
 #define hifi_gpu_gl_GLShader_h
 
 #include "GLShared.h"
+#include <gl/GLShaders.h>
 
 namespace gpu { namespace gl {
 
 struct ShaderObject {
+    using Uniforms = ::gl::Uniforms;
     GLuint glshader { 0 };
     GLuint glprogram { 0 };
     GLint transformCameraSlot { -1 };
     GLint transformObjectSlot { -1 };
+    Uniforms uniforms;
 };
 
 class GLShader : public GPUObject {

libraries/gpu-gl/src/gpu/gl41/GL41Backend.h

@@ -173,7 +173,7 @@ protected:
 
     std::string getBackendShaderHeader() const override;
     void makeProgramBindings(ShaderObject& shaderObject) override;
-    int makeResourceBufferSlots(GLuint glprogram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) override;
+    int makeResourceBufferSlots(const ShaderObject& shaderProgram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) override;
 
 };
 

libraries/gpu-gl/src/gpu/gl41/GL41BackendShader.cpp

@@ -22,20 +22,13 @@ std::string GL41Backend::getBackendShaderHeader() const {
     return header;
 }
 
-int GL41Backend::makeResourceBufferSlots(GLuint glprogram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) {
+int GL41Backend::makeResourceBufferSlots(const ShaderObject& shaderProgram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) {
     GLint ssboCount = 0;
-       GLint uniformsCount = 0;
+    const auto& glprogram = shaderProgram.glprogram;
-
+    for (const auto& uniform : shaderProgram.uniforms) {
-    glGetProgramiv(glprogram, GL_ACTIVE_UNIFORMS, &uniformsCount);
+        const auto& name = uniform.name;
-
+        const auto& type = uniform.type;
-    for (int i = 0; i < uniformsCount; i++) {
+        const auto& location = uniform.location;
-        const GLint NAME_LENGTH = 256;
-        GLchar name[NAME_LENGTH];
-        GLint length = 0;
-        GLint size = 0;
-        GLenum type = 0;
-        glGetActiveUniform(glprogram, i, NAME_LENGTH, &length, &size, &type, name);
-        GLint location = glGetUniformLocation(glprogram, name);
         const GLint INVALID_UNIFORM_LOCATION = -1;
 
         // Try to make sense of the gltype

libraries/gpu-gl/src/gpu/gl45/GL45Backend.h

@@ -274,7 +274,7 @@ protected:
     // Shader Stage
     std::string getBackendShaderHeader() const override;
     void makeProgramBindings(ShaderObject& shaderObject) override;
-    int makeResourceBufferSlots(GLuint glprogram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) override;
+    int makeResourceBufferSlots(const ShaderObject& shaderProgram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) override;
 
     // Texture Management Stage
     void initTextureManagementStage() override;

libraries/gpu-gl/src/gpu/gl45/GL45BackendShader.cpp

@@ -27,7 +27,8 @@ std::string GL45Backend::getBackendShaderHeader() const {
     return header;
 }
 
-int GL45Backend::makeResourceBufferSlots(GLuint glprogram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) {
+int GL45Backend::makeResourceBufferSlots(const ShaderObject& shaderProgram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) {
+    const auto& glprogram = shaderProgram.glprogram;
     GLint buffersCount = 0;
     glGetProgramInterfaceiv(glprogram, GL_SHADER_STORAGE_BLOCK, GL_ACTIVE_RESOURCES, &buffersCount);
 

libraries/gpu-gles/src/gpu/gles/GLESBackend.h

@@ -164,7 +164,7 @@ protected:
     
     std::string getBackendShaderHeader() const override;
     void makeProgramBindings(ShaderObject& shaderObject) override;
-    int makeResourceBufferSlots(GLuint glprogram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) override;
+    int makeResourceBufferSlots(const ShaderObject& shaderObject, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) override;
 
 };
 

libraries/gpu-gles/src/gpu/gles/GLESBackendShader.cpp

@@ -25,20 +25,15 @@ std::string GLESBackend::getBackendShaderHeader() const {
     return header;
 }
 
-int GLESBackend::makeResourceBufferSlots(GLuint glprogram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) {
+int GLESBackend::makeResourceBufferSlots(const ShaderObject& shaderObject, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) {
     GLint ssboCount = 0;
-       GLint uniformsCount = 0;
+    GLint uniformsCount = 0;
+    const auto& glprogram = shaderObject.glprogram;
 
-    glGetProgramiv(glprogram, GL_ACTIVE_UNIFORMS, &uniformsCount);
+    for (const auto& uniform : shaderObject.uniforms) {
-
+        const auto& type = uniform.type;
-    for (int i = 0; i < uniformsCount; i++) {
+        const auto& location = uniform.location;
-        const GLint NAME_LENGTH = 256;
+        const auto& name = uniform.name;
-        GLchar name[NAME_LENGTH];
-        GLint length = 0;
-        GLint size = 0;
-        GLenum type = 0;
-        glGetActiveUniform(glprogram, i, NAME_LENGTH, &length, &size, &type, name);
-        GLint location = glGetUniformLocation(glprogram, name);
         const GLint INVALID_UNIFORM_LOCATION = -1;
 
         // Try to make sense of the gltype

libraries/gpu/src/gpu/Context.cpp

@@ -53,6 +53,13 @@ Context::~Context() {
     _batchPool.clear();
 }
 
+void Context::shutdown() {
+    if (_backend) {
+        _backend->shutdown();
+        _backend.reset();
+    }
+}
+
 const std::string& Context::getBackendVersion() const {
     return _backend->getVersion();
 }

libraries/gpu/src/gpu/Context.h

@@ -54,6 +54,7 @@ class Backend {
 public:
     virtual ~Backend(){};
 
+    virtual void shutdown() {}
     virtual const std::string& getVersion() const = 0;
 
     void setStereoState(const StereoState& stereo) { _stereo = stereo; }
@@ -154,6 +155,7 @@ public:
     Context();
     ~Context();
 
+    void shutdown();
     const std::string& getBackendVersion() const;
 
     void beginFrame(const glm::mat4& renderView = glm::mat4(), const glm::mat4& renderPose = glm::mat4());

libraries/gpu/src/gpu/Shader.h

@@ -54,13 +54,11 @@ public:
 
     struct CompilationLog {
         std::string message;
-        std::vector<char> binary;
         bool compiled{ false };
 
         CompilationLog() {}
         CompilationLog(const CompilationLog& src) :
             message(src.message),
-            binary(src.binary),
             compiled(src.compiled) {}
     };
     using CompilationLogs = std::vector<CompilationLog>;

libraries/networking/src/ResourceCache.cpp

@@ -218,8 +218,8 @@ ScriptableResource* ResourceCache::prefetch(const QUrl& url, void* extra) {
 }
 
 ResourceCache::ResourceCache(QObject* parent) : QObject(parent) {
-    auto nodeList = DependencyManager::get<NodeList>();
+    if (DependencyManager::isSet<NodeList>()) {
-    if (nodeList) {
+        auto nodeList = DependencyManager::get<NodeList>();
         auto& domainHandler = nodeList->getDomainHandler();
         connect(&domainHandler, &DomainHandler::disconnectedFromDomain,
             this, &ResourceCache::clearATPAssets, Qt::DirectConnection);

libraries/shared/src/SharedUtil.h

@@ -25,6 +25,7 @@
 #include <QtCore/QCoreApplication>
 #include <QUuid>
 
+#include "NumericalConstants.h"
 // When writing out avatarEntities to a QByteArray, if the parentID is the ID of MyAvatar, use this ID instead.  This allows
 // the value to be reset when the sessionID changes.
 const QUuid AVATAR_SELF_ID = QUuid("{00000000-0000-0000-0000-000000000001}");
@@ -122,6 +123,27 @@ const QByteArray HIGH_FIDELITY_USER_AGENT = "Mozilla/5.0 (HighFidelityInterface)
 quint64 usecTimestampNow(bool wantDebug = false);
 void usecTimestampNowForceClockSkew(qint64 clockSkew);
 
+inline bool afterUsecs(quint64& startUsecs, quint64 maxIntervalUecs) {
+    auto now = usecTimestampNow();
+    auto interval = now - startUsecs;
+    if (interval > maxIntervalUecs) {
+        startUsecs = now;
+        return true;
+    }
+    return false;
+}
+
+inline bool afterSecs(quint64& startUsecs, quint64 maxIntervalSecs) {
+    return afterUsecs(startUsecs, maxIntervalSecs * USECS_PER_SECOND);
+}
+
+template <typename F>
+void doEvery(quint64& lastReportUsecs, quint64 secs, F lamdba) {
+    if (afterSecs(lastReportUsecs, secs)) {
+        lamdba();
+    }
+}
+
 // Number of seconds expressed since the first call to this function, expressed as a float
 // Maximum accuracy in msecs
 float secTimestampNow();

libraries/test-utils/CMakeLists.txt

@@ -1,3 +1,3 @@
 set(TARGET_NAME test-utils)
 setup_hifi_library(Network Gui)
-
+link_hifi_libraries(shared)

libraries/test-utils/src/test-utils/QTestExtensions.h

@@ -313,27 +313,6 @@ inline QString getTestResource(const QString& relativePath) {
     return QDir::cleanPath(dir.absoluteFilePath(relativePath));
 }
 
-inline bool afterUsecs(quint64& startUsecs, quint64 maxIntervalUecs) {
-    auto now = usecTimestampNow();
-    auto interval = now - startUsecs;
-    if (interval > maxIntervalUecs) {
-        startUsecs = now;
-        return true;
-    }
-    return false;
-}
-
-inline bool afterSecs(quint64& startUsecs, quint64 maxIntervalSecs) {
-    return afterUsecs(startUsecs, maxIntervalSecs * USECS_PER_SECOND);
-}
-
-template <typename F>
-void doEvery(quint64& lastReportUsecs, quint64 secs, F lamdba) {
-    if (afterSecs(lastReportUsecs, secs)) {
-        lamdba();
-    }
-}
-
 inline void failAfter(quint64 startUsecs, quint64 secs, const char* message) {
     if (afterSecs(startUsecs, secs)) {
         QFAIL(message);

plugins/openvr/src/OpenVrDisplayPlugin.cpp

@@ -198,9 +198,9 @@ public:
             std::string fsSource = HMD_REPROJECTION_FRAG;
             GLuint vertexShader { 0 }, fragmentShader { 0 };
             std::string error;
-            std::vector<char> binary;
+            ::gl::CachedShader binary;
-            ::gl::compileShader(GL_VERTEX_SHADER, vsSource, "", vertexShader, error);
+            ::gl::compileShader(GL_VERTEX_SHADER, vsSource, vertexShader, error);
-            ::gl::compileShader(GL_FRAGMENT_SHADER, fsSource, "", fragmentShader, error);
+            ::gl::compileShader(GL_FRAGMENT_SHADER, fsSource, fragmentShader, error);
             _program = ::gl::compileProgram({ { vertexShader, fragmentShader } }, error, binary);
             glDeleteShader(vertexShader);
             glDeleteShader(fragmentShader);

tests-manual/shaders/src/main.cpp

@@ -134,12 +134,12 @@ const std::string PIXEL_SHADER_DEFINES{ R"GLSL(
 
 void testShaderBuild(const std::string& vs_src, const std::string& fs_src) {
     std::string error;
-    std::vector<char> binary;
     GLuint vs, fs;
-    if (!gl::compileShader(GL_VERTEX_SHADER, vs_src, VERTEX_SHADER_DEFINES, vs, error) || 
+    if (!gl::compileShader(GL_VERTEX_SHADER, VERTEX_SHADER_DEFINES + vs_src, vs, error) ||
-        !gl::compileShader(GL_FRAGMENT_SHADER, fs_src, PIXEL_SHADER_DEFINES, fs, error)) {
+        !gl::compileShader(GL_FRAGMENT_SHADER, PIXEL_SHADER_DEFINES + fs_src, fs, error)) {
         throw std::runtime_error("Failed to compile shader");
     }
+    gl::CachedShader binary;
     auto pr = gl::compileProgram({ vs, fs }, error, binary);
     if (!pr) {
         throw std::runtime_error("Failed to link shader");

tests/animation/CMakeLists.txt

@@ -1,7 +1,7 @@
 # Declare dependencies
 macro (setup_testcase_dependencies)
   # link in the shared libraries
-  link_hifi_libraries(shared animation gpu fbx graphics networking)
+  link_hifi_libraries(shared animation gpu fbx graphics networking test-utils)
 
   package_libraries_for_deployment()
 endmacro ()

tests/animation/src/AnimInverseKinematicsTests.cpp

@@ -16,7 +16,7 @@
 #include <AnimationLogging.h>
 #include <NumericalConstants.h>
 
-#include "../QTestExtensions.h"
+#include <test-utils/QTestExtensions.h>
 
 QTEST_MAIN(AnimInverseKinematicsTests)
 

tests/animation/src/AnimTests.cpp

@@ -20,7 +20,7 @@
 #include <AccountManager.h>
 #include <ResourceManager.h>
 #include <StatTracker.h>
-#include <../QTestExtensions.h>
+#include <test-utils/QTestExtensions.h>
 
 QTEST_MAIN(AnimTests)
 

tests/animation/src/RotationConstraintTests.cpp

@@ -17,7 +17,7 @@
 #include <NumericalConstants.h>
 #include <SwingTwistConstraint.h>
 
-#include "../QTestExtensions.h"
+#include <test-utils/QTestExtensions.h>
 
 
 QTEST_MAIN(RotationConstraintTests)

tests/gpu/src/ShaderLoadTest.cpp

@@ -0,0 +1,103 @@
+//
+//  Created by Bradley Austin Davis on 2018/01/11
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "ShaderLoadTest.h"
+
+#include <iostream>
+#include <QtCore/QTemporaryFile>
+
+#include <NumericalConstants.h>
+#include <gpu/Forward.h>
+#include <gl/Config.h>
+#include <gl/GLHelpers.h>
+#include <gl/GLShaders.h>
+
+#include <gpu/gl/GLBackend.h>
+#include <shared/FileUtils.h>
+#include <SettingManager.h>
+
+#include <test-utils/QTestExtensions.h>
+
+//#pragma optimize("", off)
+
+QTEST_MAIN(ShaderLoadTest)
+
+#define FAIL_AFTER_SECONDS 30
+
+QtMessageHandler originalHandler;
+
+void messageHandler(QtMsgType type, const QMessageLogContext& context, const QString& message) {
+#if defined(Q_OS_WIN)
+    OutputDebugStringA(message.toStdString().c_str());
+    OutputDebugStringA("\n");
+#endif
+    originalHandler(type, context, message);
+}
+
+void ShaderLoadTest::loadProgramSources() {
+    _programSources.clear();
+    QString json = FileUtils::readFile(":cache.json");
+    auto root = QJsonDocument::fromJson(json.toUtf8()).object();
+    _programSources.reserve(root.size());
+    QRegularExpression regex("//-------- \\d");
+    for (auto shaderKey : root.keys()) {
+        auto cacheEntry = root[shaderKey].toObject();
+        auto source = cacheEntry["source"].toString();
+        auto split = source.split(regex, QString::SplitBehavior::SkipEmptyParts);
+        _programSources.emplace_back(split.at(0).trimmed().toStdString(), split.at(1).trimmed().toStdString());
+    }
+}        
+
+
+void ShaderLoadTest::initTestCase() {
+    originalHandler = qInstallMessageHandler(messageHandler);
+    loadProgramSources();
+    getDefaultOpenGLSurfaceFormat();
+    _canvas.create();
+    if (!_canvas.makeCurrent()) {
+        qFatal("Unable to make test GL context current");
+    }
+    gl::initModuleGl();
+    gpu::Context::init<gpu::gl::GLBackend>();
+    _gpuContext = std::make_shared<gpu::Context>();
+    _canvas.makeCurrent();
+    DependencyManager::set<Setting::Manager>();
+}
+
+void ShaderLoadTest::cleanupTestCase() {
+    _gpuContext->recycle();
+    _gpuContext->shutdown();
+    _gpuContext.reset();
+    DependencyManager::destroy<Setting::Manager>();
+}
+
+std::string randomString() {
+    return "\n//" + QUuid::createUuid().toString().toStdString();
+}
+
+std::unordered_map<std::string, gpu::ShaderPointer> cachedShaders;
+
+gpu::ShaderPointer getShader(const std::string& shaderSource, bool pixel) {
+    if (0 != cachedShaders.count(shaderSource)) {
+        return cachedShaders[shaderSource];
+    }
+    auto shader = pixel ? gpu::Shader::createPixel({ shaderSource + randomString() })  : gpu::Shader::createVertex({ shaderSource + randomString() });
+    cachedShaders.insert({shaderSource, shader});
+    return shader;
+}
+
+void ShaderLoadTest::testShaderLoad() {
+    QBENCHMARK {
+        for (const auto& programSource : _programSources) {
+            auto vertexShader = getShader(programSource.first, false);
+            auto pixelShader = getShader(programSource.second, true);
+            auto program = gpu::Shader::createProgram(vertexShader, pixelShader);
+            QVERIFY(gpu::gl::GLBackend::makeProgram(*program, {}, {}));
+        }
+    }
+}

tests/gpu/src/ShaderLoadTest.h

@@ -0,0 +1,38 @@
+//
+//  Created by Bradley Austin Davis on 2018/05/08
+//  Copyright 2013-2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#pragma once
+
+#include <QtTest/QtTest>
+#include <QtCore/QTemporaryDir>
+
+#include <gpu/Forward.h>
+#include <gl/OffscreenGLCanvas.h>
+
+class ShaderLoadTest : public QObject {
+    Q_OBJECT
+
+private:
+    void loadProgramSources();
+
+private slots:
+    void initTestCase();
+    void cleanupTestCase();
+    void testShaderLoad();
+
+
+private:
+    using ProgramSource = std::pair<std::string, std::string>;
+    using ProgramSources = std::vector<ProgramSource>;
+
+    ProgramSources _programSources;
+    QString _resourcesPath;
+    OffscreenGLCanvas _canvas;
+    gpu::ContextPointer _gpuContext;
+    const glm::uvec2 _size{ 640, 480 };
+};

tests/gpu/src/TextureTest.cpp

@@ -21,7 +21,7 @@
 #include <quazip5/quazip.h>
 #include <quazip5/JlCompress.h>
 
-#include "../../QTestExtensions.h"
+#include <test-utils/QTestExtensions.h>
 
 QTEST_MAIN(TextureTest)
 

tests/gpu/src/data.qrc

@@ -0,0 +1,6 @@
+<!DOCTYPE RCC>
+<RCC version="1.0">
+  <qresource>
+      <file>cache.json</file>
+  </qresource>
+</RCC>

tests/networking/CMakeLists.txt

@@ -2,7 +2,7 @@
 # Declare dependencies
 macro (setup_testcase_dependencies)
   # link in the shared libraries
-  link_hifi_libraries(shared networking)
+  link_hifi_libraries(shared test-utils networking)
 
   package_libraries_for_deployment()
 endmacro ()

tests/networking/src/PacketTests.cpp

@@ -10,7 +10,7 @@
 //
 
 #include "PacketTests.h"
-#include "../QTestExtensions.h"
+#include <test-utils/QTestExtensions.h>
 
 #include <NLPacket.h>
 

tests/octree/CMakeLists.txt

@@ -2,7 +2,7 @@
 # Declare dependencies
 macro (setup_testcase_dependencies)
   # link in the shared libraries
-  link_hifi_libraries(shared octree gpu graphics fbx networking entities avatars audio animation script-engine physics)
+  link_hifi_libraries(shared test-utils octree gpu graphics fbx networking entities avatars audio animation script-engine physics)
 
   package_libraries_for_deployment()
 endmacro ()

tests/octree/src/ViewFrustumTests.cpp

@@ -18,8 +18,8 @@
 #include <ViewFrustum.h>
 
 //#include <StreamUtils.h>
-#include <../GLMTestUtils.h>
+#include <test-utils/GLMTestUtils.h>
-#include <../QTestExtensions.h>
+#include <test-utils/QTestExtensions.h>
 
 
 const float ACCEPTABLE_FLOAT_ERROR = 1.0e-6f;

tests/physics/CMakeLists.txt

@@ -2,7 +2,7 @@
 # Declare dependencies
 macro (SETUP_TESTCASE_DEPENDENCIES)
   target_bullet()
-  link_hifi_libraries(shared physics gpu graphics)
+  link_hifi_libraries(shared test-utils physics gpu graphics)
   package_libraries_for_deployment()
 endmacro ()
 

tests/physics/src/BulletUtilTests.cpp

@@ -18,7 +18,7 @@
 #include <GLMHelpers.h>
 
 // Add additional qtest functionality (the include order is important!)
-#include "../QTestExtensions.h"
+#include <test-utils/QTestExtensions.h>
 
 // Constants
 const glm::vec3 origin(0.0f);

tests/physics/src/MeshMassPropertiesTests.cpp

@@ -16,7 +16,7 @@
 
 // Add additional qtest functionality (the include order is important!)
 #include "BulletTestUtils.h"
-#include "../QTestExtensions.h"
+#include <test-utils/QTestExtensions.h>
 
 const btScalar acceptableRelativeError(1.0e-5f);
 const btScalar acceptableAbsoluteError(1.0e-4f);

tests/shared/CMakeLists.txt

@@ -3,7 +3,7 @@
 macro (setup_testcase_dependencies)
 
   # link in the shared libraries
-  link_hifi_libraries(shared)
+  link_hifi_libraries(shared test-utils)
 
   package_libraries_for_deployment()
 endmacro ()

tests/shared/src/AABoxTests.cpp

@@ -17,8 +17,8 @@
 #include <NumericalConstants.h>
 #include <StreamUtils.h>
 
-#include <../GLMTestUtils.h>
+#include <test-utils/GLMTestUtils.h>
-#include <../QTestExtensions.h>
+#include <test-utils/QTestExtensions.h>
 
 
 QTEST_MAIN(AABoxTests)

tests/shared/src/AACubeTests.cpp

@@ -17,8 +17,8 @@
 #include <NumericalConstants.h>
 #include <StreamUtils.h>
 
-#include <../GLMTestUtils.h>
+#include <test-utils/GLMTestUtils.h>
-#include <../QTestExtensions.h>
+#include <test-utils/QTestExtensions.h>
 
 
 QTEST_MAIN(AACubeTests)

tests/shared/src/BitVectorHelperTests.cpp

@@ -12,7 +12,7 @@
 
 #include <SharedLogging.h>
 
-#include "../QTestExtensions.h"
+#include <test-utils/QTestExtensions.h>
 #include <QtCore/QDebug>
 #include <StreamUtils.h>
 #include <glm/glm.hpp>

tests/shared/src/CubicHermiteSplineTests.cpp

@@ -9,7 +9,7 @@
 //
 
 #include "CubicHermiteSplineTests.h"
-#include "../QTestExtensions.h"
+#include <test-utils/QTestExtensions.h>
 #include <QtCore/QDebug>
 #include "CubicHermiteSpline.h"
 

tests/shared/src/DualQuaternionTests.cpp

@@ -17,8 +17,8 @@
 #include <NumericalConstants.h>
 #include <StreamUtils.h>
 
-#include <../GLMTestUtils.h>
+#include <test-utils/GLMTestUtils.h>
-#include <../QTestExtensions.h>
+#include <test-utils/QTestExtensions.h>
 
 QTEST_MAIN(DualQuaternionTests)
 

tests/shared/src/GLMHelpersTests.cpp

@@ -14,7 +14,7 @@
 #include <NumericalConstants.h>
 #include <StreamUtils.h>
 
-#include <../QTestExtensions.h>
+#include <test-utils/QTestExtensions.h>
 #include <glm/gtc/matrix_transform.hpp>
 #include <glm/simd/matrix.h>
 

tests/shared/src/GeometryUtilTests.cpp

@@ -18,7 +18,8 @@
 #include <NumericalConstants.h>
 #include <StreamUtils.h>
 
-#include <../QTestExtensions.h>
+#include <test-utils/GLMTestUtils.h>
+#include <test-utils/QTestExtensions.h>
 
 
 QTEST_MAIN(GeometryUtilTests)

tests/shared/src/MovingMinMaxAvgTests.cpp

@@ -17,7 +17,8 @@
 #include <NumericalConstants.h>
 #include <SharedUtil.h>
 
-#include "../QTestExtensions.h"
+#include <test-utils/GLMTestUtils.h>
+#include <test-utils/QTestExtensions.h>
 
 
 QTEST_MAIN(MovingMinMaxAvgTests)

tests/shared/src/MovingPercentileTests.cpp

@@ -15,8 +15,9 @@
 #include "MovingPercentile.h"
 
 #include <limits>
-#include <qqueue.h>
+#include <QtCore/QQueue>
-#include <../QTestExtensions.h>
+#include <test-utils/GLMTestUtils.h>
+#include <test-utils/QTestExtensions.h>
 
 QTEST_MAIN(MovingPercentileTests)
 

tests/shared/src/TraceTests.cpp

@@ -14,7 +14,7 @@
 #include <Profile.h>
 
 #include <NumericalConstants.h>
-#include <../QTestExtensions.h>
+#include <test-utils/QTestExtensions.h>
 
 QTEST_MAIN(TraceTests)
 Q_LOGGING_CATEGORY(trace_test, "trace.test")

tests/shared/src/TransformTests.cpp

@@ -14,7 +14,7 @@
 
 #include <SharedLogging.h>
 
-#include "../QTestExtensions.h"
+#include <test-utils/QTestExtensions.h>
 #include <QtCore/QDebug>
 #include <Transform.h>
 #include <StreamUtils.h>