Merge pull request #12294 from samcake/pastel

Improve startup time by optimizing shader bookkeeping

interface/src/Application.cpp

@@ -2296,16 +2296,16 @@ void Application::initializeGL() {
 #endif
     _renderEngine->addJob<RenderViewTask>("RenderMainView", cullFunctor, !DISABLE_DEFERRED, render::ItemKey::TAG_BITS_0, render::ItemKey::TAG_BITS_0);
 
-#ifdef Q_OS_OSX
-    DeadlockWatchdogThread::resume();
-#endif
-
     _renderEngine->load();
     _renderEngine->registerScene(_main3DScene);
 
     // Now that OpenGL is initialized, we are sure we have a valid context and can create the various pipeline shaders with success.
     DependencyManager::get<GeometryCache>()->initializeShapePipelines();
 
+#ifdef Q_OS_OSX
+    DeadlockWatchdogThread::resume();
+#endif
+
     _offscreenContext = new OffscreenGLCanvas();
     _offscreenContext->setObjectName("MainThreadContext");
     _offscreenContext->create(_glWidget->qglContext());

libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp

@@ -398,7 +398,7 @@ void HmdDisplayPlugin::HUDRenderer::updatePipeline() {
         auto vs = gpu::Shader::createVertex(std::string(hmd_ui_vert));
         auto ps = gpu::Shader::createPixel(std::string(hmd_ui_frag));
         auto program = gpu::Shader::createProgram(vs, ps);
-        gpu::gl::GLBackend::makeProgram(*program, gpu::Shader::BindingSet());
+        gpu::Shader::makeProgram(*program, gpu::Shader::BindingSet());
         uniformsLocation = program->getUniformBuffers().findLocation("hudBuffer");
 
         gpu::StatePointer state = gpu::StatePointer(new gpu::State());

libraries/gl/src/gl/GLShaders.cpp

@@ -6,9 +6,9 @@ namespace gl {
 
 
 #ifdef SEPARATE_PROGRAM
-    bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, GLuint &programObject, std::string& error) {
+    bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, GLuint &programObject, std::string& message) {
 #else
-    bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, std::string& error) {
+    bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, std::string& message) {
 #endif
     if (shaderSource.empty()) {
         qCDebug(glLogging) << "GLShader::compileShader - no GLSL shader source code ? so failed to create";
@@ -34,9 +34,17 @@ namespace gl {
     GLint compiled = 0;
     glGetShaderiv(glshader, GL_COMPILE_STATUS, &compiled);
 
+    GLint infoLength = 0;
+    glGetShaderiv(glshader, GL_INFO_LOG_LENGTH, &infoLength);
+
+    if ((infoLength > 0) || !compiled) {
+        char* temp = new char[infoLength];
+        glGetShaderInfoLog(glshader, infoLength, NULL, temp);
+
+        message = std::string(temp);
+
         // if compilation fails
         if (!compiled) {
-
             // save the source code to a temp file so we can debug easily
             /*
             std::ofstream filestream;
@@ -48,13 +56,6 @@ namespace gl {
             }
             */
 
-        GLint infoLength = 0;
-        glGetShaderiv(glshader, GL_INFO_LOG_LENGTH, &infoLength);
-
-        char* temp = new char[infoLength];
-        glGetShaderInfoLog(glshader, infoLength, NULL, temp);
-
-
             /*
             filestream.open("debugshader.glsl.info.txt");
             if (filestream.is_open()) {
@@ -75,13 +76,17 @@ namespace gl {
             qCCritical(glLogging) << "GLShader::compileShader - errors:";
             qCCritical(glLogging) << temp;
 
-        error = std::string(temp);
             delete[] temp;
-
             glDeleteShader(glshader);
             return false;
         }
 
+        // Compilation success
+        qCWarning(glLogging) << "GLShader::compileShader - Success:";
+        qCWarning(glLogging) << temp;
+        delete[] temp;
+    }
+
 #ifdef SEPARATE_PROGRAM
     GLuint glprogram = 0;
     // so far so good, program is almost done, need to link:
@@ -137,7 +142,7 @@ namespace gl {
     return true;
 }
 
-GLuint compileProgram(const std::vector<GLuint>& glshaders, std::string& error) {
+GLuint compileProgram(const std::vector<GLuint>& glshaders, std::string& message, std::vector<GLchar>& binary) {
     // A brand new program:
     GLuint glprogram = glCreateProgram();
     if (!glprogram) {
@@ -157,6 +162,15 @@ GLuint compileProgram(const std::vector<GLuint>& glshaders, std::string& error)
     GLint linked = 0;
     glGetProgramiv(glprogram, GL_LINK_STATUS, &linked);
 
+    GLint infoLength = 0;
+    glGetProgramiv(glprogram, GL_INFO_LOG_LENGTH, &infoLength);
+
+    if ((infoLength > 0) || !linked) {
+        char* temp = new char[infoLength];
+        glGetProgramInfoLog(glprogram, infoLength, NULL, temp);
+
+        message = std::string(temp);
+
         if (!linked) {
             /*
             // save the source code to a temp file so we can debug easily
@@ -168,16 +182,9 @@ GLuint compileProgram(const std::vector<GLuint>& glshaders, std::string& error)
             }
             */
 
-        GLint infoLength = 0;
-        glGetProgramiv(glprogram, GL_INFO_LOG_LENGTH, &infoLength);
-
-        char* temp = new char[infoLength];
-        glGetProgramInfoLog(glprogram, infoLength, NULL, temp);
-
             qCDebug(glLogging) << "GLShader::compileProgram -  failed to LINK the gl program object :";
             qCDebug(glLogging) << temp;
 
-        error = std::string(temp);
             delete[] temp;
 
             /*
@@ -190,6 +197,30 @@ GLuint compileProgram(const std::vector<GLuint>& glshaders, std::string& error)
 
             glDeleteProgram(glprogram);
             return 0;
+        } else {
+            qCDebug(glLogging) << "GLShader::compileProgram -  success:";
+            qCDebug(glLogging) << temp;
+            delete[] temp;
+        }
+    }
+
+    // If linked get the binaries
+    if (linked) {
+        GLint binaryLength = 0;
+        glGetProgramiv(glprogram, GL_PROGRAM_BINARY_LENGTH, &binaryLength);
+
+        if (binaryLength > 0) {
+            GLint numBinFormats = 0;
+            glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, &numBinFormats);
+            if (numBinFormats > 0) {
+                binary.resize(binaryLength);
+                std::vector<GLint> binFormats(numBinFormats);
+                glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, binFormats.data());
+
+                GLenum programBinFormat;
+                glGetProgramBinary(glprogram, binaryLength, NULL, &programBinFormat, binary.data());
+            }
+        }
     }
 
     return glprogram;

libraries/gl/src/gl/GLShaders.h

@@ -17,12 +17,12 @@
 
 namespace gl {
 #ifdef SEPARATE_PROGRAM
-    bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, GLuint &programObject, std::string& error);
+    bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, GLuint &programObject, std::string& message);
 #else
-    bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, std::string& error);
+    bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, std::string& message);
 #endif
 
-    GLuint compileProgram(const std::vector<GLuint>& glshaders, std::string& error);
+    GLuint compileProgram(const std::vector<GLuint>& glshaders, std::string& message, std::vector<GLchar>& binary);
 
 }
 

libraries/gpu-gl/src/gpu/gl/GLBackend.cpp

@@ -68,8 +68,8 @@ GLBackend& getBackend() {
     return *INSTANCE;
 }
 
-bool GLBackend::makeProgram(Shader& shader, const Shader::BindingSet& slotBindings) {
+bool GLBackend::makeProgram(Shader& shader, const Shader::BindingSet& slotBindings, const Shader::CompilationHandler& handler) {
-    return GLShader::makeProgram(getBackend(), shader, slotBindings);
+    return GLShader::makeProgram(getBackend(), shader, slotBindings, handler);
 }
 
 GLBackend::CommandCall GLBackend::_commandCalls[Batch::NUM_COMMANDS] = 

libraries/gpu-gl/src/gpu/gl/GLBackend.h

@@ -64,7 +64,7 @@ protected:
     explicit GLBackend(bool syncCache);
     GLBackend();
 public:
-    static bool makeProgram(Shader& shader, const Shader::BindingSet& slotBindings = Shader::BindingSet());
+    static bool makeProgram(Shader& shader, const Shader::BindingSet& slotBindings, const Shader::CompilationHandler& handler);
 
     virtual ~GLBackend();
 
@@ -423,8 +423,8 @@ protected:
     } _pipeline;
 
     // Backend dependant compilation of the shader
-    virtual GLShader* compileBackendProgram(const Shader& program);
+    virtual GLShader* compileBackendProgram(const Shader& program, const Shader::CompilationHandler& handler);
-    virtual GLShader* compileBackendShader(const Shader& shader);
+    virtual GLShader* compileBackendShader(const Shader& shader, const Shader::CompilationHandler& handler);
     virtual std::string getBackendShaderHeader() const;
     virtual void makeProgramBindings(ShaderObject& shaderObject);
     class ElementResource {

libraries/gpu-gl/src/gpu/gl/GLBackendShader.cpp

@@ -56,28 +56,47 @@ static const std::array<std::string, GLShader::NumVersions> VERSION_DEFINES { {
     stereoVersion
 } };
 
-GLShader* GLBackend::compileBackendShader(const Shader& shader) {
+GLShader* GLBackend::compileBackendShader(const Shader& shader, const Shader::CompilationHandler& handler) {
     // Any GLSLprogram ? normally yes...
     const std::string& shaderSource = shader.getSource().getCode();
     GLenum shaderDomain = SHADER_DOMAINS[shader.getType()];
     GLShader::ShaderObjects shaderObjects;
+    Shader::CompilationLogs compilationLogs(GLShader::NumVersions);
+    shader.incrementCompilationAttempt();
 
     for (int version = 0; version < GLShader::NumVersions; version++) {
         auto& shaderObject = shaderObjects[version];
 
         std::string shaderDefines = getBackendShaderHeader() + "\n" + DOMAIN_DEFINES[shader.getType()] + "\n" + VERSION_DEFINES[version];
-        std::string error;
+        if (handler) {
-
+            bool retest = true;
-#ifdef SEPARATE_PROGRAM
+            std::string currentSrc = shaderSource;
-        bool result = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, shaderObject.glprogram, error);
+            // When a Handler is specified, we can try multiple times to build the shader and let the handler change the source if the compilation fails.
-#else
+            // The retest bool is set to false as soon as the compilation succeed to wexit the while loop.
-        bool result = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, error);
+            // The handler tells us if we should retry or not while returning a modified version of the source.
-#endif
+            while (retest) {
+                bool result = ::gl::compileShader(shaderDomain, currentSrc, shaderDefines, shaderObject.glshader, compilationLogs[version].message);
+                compilationLogs[version].compiled = result;
                 if (!result) {
-            qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Shader didn't compile:\n" << error.c_str();
+                    std::string newSrc;
+                    retest = handler(shader, currentSrc, compilationLogs[version], newSrc);
+                    currentSrc = newSrc;
+                } else {
+                    retest = false;
+                }
+            }
+        } else {
+            compilationLogs[version].compiled = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, compilationLogs[version].message);
+        }
+
+         if (!compilationLogs[version].compiled) {
+            qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Shader didn't compile:\n" << compilationLogs[version].message.c_str();
+            shader.setCompilationLogs(compilationLogs);
             return nullptr;
         }
     }
+    // Compilation feedback
+    shader.setCompilationLogs(compilationLogs);
 
     // So far so good, the shader is created successfully
     GLShader* object = new GLShader(this->shared_from_this());
@@ -86,39 +105,47 @@ GLShader* GLBackend::compileBackendShader(const Shader& shader) {
     return object;
 }
 
-GLShader* GLBackend::compileBackendProgram(const Shader& program) {
+GLShader* GLBackend::compileBackendProgram(const Shader& program, const Shader::CompilationHandler& handler) {
     if (!program.isProgram()) {
         return nullptr;
     }
 
     GLShader::ShaderObjects programObjects;
 
+    program.incrementCompilationAttempt();
+    Shader::CompilationLogs compilationLogs(GLShader::NumVersions);
+
     for (int version = 0; version < GLShader::NumVersions; version++) {
         auto& programObject = programObjects[version];
 
         // Let's go through every shaders and make sure they are ready to go
         std::vector< GLuint > shaderGLObjects;
         for (auto subShader : program.getShaders()) {
-            auto object = GLShader::sync((*this), *subShader);
+            auto object = GLShader::sync((*this), *subShader, handler);
             if (object) {
                 shaderGLObjects.push_back(object->_shaderObjects[version].glshader);
             } else {
                 qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - One of the shaders of the program is not compiled?";
+                compilationLogs[version].compiled = false;
+                compilationLogs[version].message = std::string("Failed to compile, one of the shaders of the program is not compiled ?");
+                program.setCompilationLogs(compilationLogs);
                 return nullptr;
             }
         }
 
-        std::string error;
+        GLuint glprogram = ::gl::compileProgram(shaderGLObjects, compilationLogs[version].message, compilationLogs[version].binary);
-        GLuint glprogram = ::gl::compileProgram(shaderGLObjects, error);
         if (glprogram == 0) {
-            qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Program didn't link:\n" << error.c_str();
+            qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Program didn't link:\n" << compilationLogs[version].message.c_str(); 
+            program.setCompilationLogs(compilationLogs);
             return nullptr;
         }
-
+        compilationLogs[version].compiled = true;
         programObject.glprogram = glprogram;
 
         makeProgramBindings(programObject);
     }
+    // Compilation feedback
+    program.setCompilationLogs(compilationLogs);
 
     // So far so good, the program versions have all been created successfully
     GLShader* object = new GLShader(this->shared_from_this());

libraries/gpu-gl/src/gpu/gl/GLShader.cpp

@@ -30,7 +30,7 @@ GLShader::~GLShader() {
     }
 }
 
-GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) {
+GLShader* GLShader::sync(GLBackend& backend, const Shader& shader, const Shader::CompilationHandler& handler) {
     GLShader* object = Backend::getGPUObject<GLShader>(shader);
 
     // If GPU object already created then good
@@ -39,13 +39,13 @@ GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) {
     }
     // need to have a gpu object?
     if (shader.isProgram()) {
-        GLShader* tempObject = backend.compileBackendProgram(shader);
+        GLShader* tempObject = backend.compileBackendProgram(shader, handler);
         if (tempObject) {
             object = tempObject;
             Backend::setGPUObject(shader, object);
         }
     } else if (shader.isDomain()) {
-        GLShader* tempObject = backend.compileBackendShader(shader);
+        GLShader* tempObject = backend.compileBackendShader(shader, handler);
         if (tempObject) {
             object = tempObject;
             Backend::setGPUObject(shader, object);
@@ -56,10 +56,10 @@ GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) {
     return object;
 }
 
-bool GLShader::makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings) {
+bool GLShader::makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings, const Shader::CompilationHandler& handler) {
 
     // First make sure the Shader has been compiled
-    GLShader* object = sync(backend, shader);
+    GLShader* object = sync(backend, shader, handler);
     if (!object) {
         return false;
     }

libraries/gpu-gl/src/gpu/gl/GLShader.h

@@ -21,8 +21,8 @@ struct ShaderObject {
 
 class GLShader : public GPUObject {
 public:
-    static GLShader* sync(GLBackend& backend, const Shader& shader);
+    static GLShader* sync(GLBackend& backend, const Shader& shader, const Shader::CompilationHandler& handler = nullptr);
-    static bool makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings);
+    static bool makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings, const Shader::CompilationHandler& handler);
 
     enum Version {
         Mono = 0,

libraries/gpu-gles/src/gpu/gl/GLBackend.cpp

@@ -61,8 +61,8 @@ GLBackend& getBackend() {
     return *INSTANCE;
 }
 
-bool GLBackend::makeProgram(Shader& shader, const Shader::BindingSet& slotBindings) {
+bool GLBackend::makeProgram(Shader& shader, const Shader::BindingSet& slotBindings, const Shader::CompilationHandler& handler) {
-    return GLShader::makeProgram(getBackend(), shader, slotBindings);
+    return GLShader::makeProgram(getBackend(), shader, slotBindings, handler);
 }
 
 

libraries/gpu-gles/src/gpu/gl/GLBackend.h

@@ -61,7 +61,7 @@ protected:
     explicit GLBackend(bool syncCache);
     GLBackend();
 public:
-    static bool makeProgram(Shader& shader, const Shader::BindingSet& slotBindings = Shader::BindingSet());
+    static bool makeProgram(Shader& shader, const Shader::BindingSet& slotBindings = Shader::BindingSet(), const Shader::CompilationHandler& handler = nullptr);
 
     virtual ~GLBackend();
 
@@ -420,8 +420,8 @@ protected:
     } _pipeline;
 
     // Backend dependant compilation of the shader
-    virtual GLShader* compileBackendProgram(const Shader& program);
+    virtual GLShader* compileBackendProgram(const Shader& program, const Shader::CompilationHandler& handler);
-    virtual GLShader* compileBackendShader(const Shader& shader);
+    virtual GLShader* compileBackendShader(const Shader& shader, const Shader::CompilationHandler& handler);
     virtual std::string getBackendShaderHeader() const;
     virtual void makeProgramBindings(ShaderObject& shaderObject);
     class ElementResource {

libraries/gpu-gles/src/gpu/gl/GLBackendShader.cpp

@@ -56,11 +56,12 @@ static const std::array<std::string, GLShader::NumVersions> VERSION_DEFINES { {
     stereoVersion
 } };
 
-GLShader* GLBackend::compileBackendShader(const Shader& shader) {
+GLShader* GLBackend::compileBackendShader(const Shader& shader, const Shader::CompilationHandler& handler) {
     // Any GLSLprogram ? normally yes...
     const std::string& shaderSource = shader.getSource().getCode();
     GLenum shaderDomain = SHADER_DOMAINS[shader.getType()];
     GLShader::ShaderObjects shaderObjects;
+    Shader::CompilationLogs compilationLogs(GLShader::NumVersions);
 
     for (int version = 0; version < GLShader::NumVersions; version++) {
         auto& shaderObject = shaderObjects[version];
@@ -70,18 +71,35 @@ GLShader* GLBackend::compileBackendShader(const Shader& shader) {
             + "\nprecision lowp float; // check precision 2"
             + "\nprecision lowp samplerBuffer;"
             + "\nprecision lowp sampler2DShadow;";
-        std::string error;
+        if (handler) {
-
+            bool retest = true;
-#ifdef SEPARATE_PROGRAM
+            std::string currentSrc = shaderSource;
-        bool result = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, shaderObject.glprogram, error);
+            // When a Handler is specified, we can try multiple times to build the shader and let the handler change the source if the compilation fails.
-#else
+            // The retest bool is set to false as soon as the compilation succeed to wexit the while loop.
-        bool result = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, error);
+            // The handler tells us if we should retry or not while returning a modified version of the source.
-#endif
+            while (retest) {
+                bool result = ::gl::compileShader(shaderDomain, currentSrc, shaderDefines, shaderObject.glshader, compilationLogs[version].message);
+                compilationLogs[version].compiled = result;
                 if (!result) {
-            qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Shader didn't compile:\n" << error.c_str();
+                    std::string newSrc;
+                    retest = handler(shader, currentSrc, compilationLogs[version], newSrc);
+                    currentSrc = newSrc;
+                } else {
+                    retest = false;
+                }
+            }
+        } else {
+            compilationLogs[version].compiled = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, compilationLogs[version].message);
+        }
+
+         if (!compilationLogs[version].compiled) {
+            qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Shader didn't compile:\n" << compilationLogs[version].message.c_str();
+            shader.setCompilationLogs(compilationLogs);
             return nullptr;
         }
     }
+    // Compilation feedback
+    shader.setCompilationLogs(compilationLogs);
 
     // So far so good, the shader is created successfully
     GLShader* object = new GLShader(this->shared_from_this());
@@ -90,32 +108,35 @@ GLShader* GLBackend::compileBackendShader(const Shader& shader) {
     return object;
 }
 
-GLShader* GLBackend::compileBackendProgram(const Shader& program) {
+GLShader* GLBackend::compileBackendProgram(const Shader& program, const Shader::CompilationHandler& handler) {
     if (!program.isProgram()) {
         return nullptr;
     }
 
     GLShader::ShaderObjects programObjects;
 
+    Shader::CompilationLogs compilationLogs(GLShader::NumVersions);
+
     for (int version = 0; version < GLShader::NumVersions; version++) {
         auto& programObject = programObjects[version];
 
         // Let's go through every shaders and make sure they are ready to go
-        std::vector<GLuint> shaderGLObjects;
+        std::vector< GLuint > shaderGLObjects;
         for (auto subShader : program.getShaders()) {
-            auto object = GLShader::sync((*this), *subShader);
+            auto object = GLShader::sync((*this), *subShader, handler);
             if (object) {
                 shaderGLObjects.push_back(object->_shaderObjects[version].glshader);
             } else {
                 qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - One of the shaders of the program is not compiled?";
+                program.setCompilationLogs(compilationLogs);
                 return nullptr;
             }
         }
 
-        std::string error;
+        GLuint glprogram = ::gl::compileProgram(shaderGLObjects, compilationLogs[version].message, compilationLogs[version].binary);
-        GLuint glprogram = ::gl::compileProgram(shaderGLObjects, error);
         if (glprogram == 0) {
-            qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Program didn't link:\n" << error.c_str();
+            qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Program didn't link:\n" << compilationLogs[version].message.c_str(); 
+            program.setCompilationLogs(compilationLogs);
             return nullptr;
         }
 
@@ -123,6 +144,8 @@ GLShader* GLBackend::compileBackendProgram(const Shader& program) {
 
         makeProgramBindings(programObject);
     }
+    // Compilation feedback
+    program.setCompilationLogs(compilationLogs);
 
     // So far so good, the program versions have all been created successfully
     GLShader* object = new GLShader(this->shared_from_this());

libraries/gpu-gles/src/gpu/gl/GLShader.cpp

@@ -30,7 +30,7 @@ GLShader::~GLShader() {
     }
 }
 
-GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) {
+GLShader* GLShader::sync(GLBackend& backend, const Shader& shader, const Shader::CompilationHandler& handler) {
     GLShader* object = Backend::getGPUObject<GLShader>(shader);
 
     // If GPU object already created then good
@@ -39,13 +39,13 @@ GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) {
     }
     // need to have a gpu object?
     if (shader.isProgram()) {
-        GLShader* tempObject = backend.compileBackendProgram(shader);
+        GLShader* tempObject = backend.compileBackendProgram(shader, handler);
         if (tempObject) {
             object = tempObject;
             Backend::setGPUObject(shader, object);
         }
     } else if (shader.isDomain()) {
-        GLShader* tempObject = backend.compileBackendShader(shader);
+        GLShader* tempObject = backend.compileBackendShader(shader, handler);
         if (tempObject) {
             object = tempObject;
             Backend::setGPUObject(shader, object);
@@ -56,10 +56,10 @@ GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) {
     return object;
 }
 
-bool GLShader::makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings) {
+bool GLShader::makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings, const Shader::CompilationHandler& handler) {
 
     // First make sure the Shader has been compiled
-    GLShader* object = sync(backend, shader);
+    GLShader* object = sync(backend, shader, handler);
     if (!object) {
         return false;
     }

libraries/gpu-gles/src/gpu/gl/GLShader.h

@@ -21,8 +21,8 @@ struct ShaderObject {
 
 class GLShader : public GPUObject {
 public:
-    static GLShader* sync(GLBackend& backend, const Shader& shader);
+    static GLShader* sync(GLBackend& backend, const Shader& shader, const Shader::CompilationHandler& handler = nullptr);
-    static bool makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings);
+    static bool makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings, const Shader::CompilationHandler& handler = nullptr);
 
     enum Version {
         Mono = 0,

libraries/gpu/src/gpu/Context.cpp

@@ -127,7 +127,7 @@ void Context::executeFrame(const FramePointer& frame) const {
     _frameStats.evalDelta(beginStats, endStats);
 }
 
-bool Context::makeProgram(Shader& shader, const Shader::BindingSet& bindings) {
+bool Context::makeProgram(Shader& shader, const Shader::BindingSet& bindings, const Shader::CompilationHandler& handler) {
     // If we're running in another DLL context, we need to fetch the program callback out of the application
     // FIXME find a way to do this without reliance on Qt app properties
     if (!_makeProgramCallback) {
@@ -135,7 +135,7 @@ bool Context::makeProgram(Shader& shader, const Shader::BindingSet& bindings) {
         _makeProgramCallback = reinterpret_cast<Context::MakeProgram>(rawCallback);
     }
     if (shader.isProgram() && _makeProgramCallback) {
-        return _makeProgramCallback(shader, bindings);
+        return _makeProgramCallback(shader, bindings, handler);
     }
     return false;
 }

libraries/gpu/src/gpu/Context.h

@@ -143,7 +143,7 @@ class Context {
 public:
     using Size = Resource::Size;
     typedef BackendPointer (*CreateBackend)();
-    typedef bool (*MakeProgram)(Shader& shader, const Shader::BindingSet& bindings);
+    typedef bool (*MakeProgram)(Shader& shader, const Shader::BindingSet& bindings, const Shader::CompilationHandler& handler);
 
 
     // This one call must happen before any context is created or used (Shader::MakeProgram) in order to setup the Backend and any singleton data needed
@@ -262,7 +262,7 @@ protected:
     // makeProgramShader(...) make a program shader ready to be used in a Batch.
     // It compiles the sub shaders, link them and defines the Slots and their bindings.
     // If the shader passed is not a program, nothing happens. 
-    static bool makeProgram(Shader& shader, const Shader::BindingSet& bindings);
+    static bool makeProgram(Shader& shader, const Shader::BindingSet& bindings, const Shader::CompilationHandler& handler);
 
     static CreateBackend _createBackendCallback;
     static MakeProgram _makeProgramCallback;

libraries/gpu/src/gpu/Shader.cpp

@@ -17,59 +17,111 @@
 
 using namespace gpu;
 
-Shader::Shader(Type type, const Source& source):
+std::atomic<uint32_t> Shader::_nextShaderID( 1 );
+Shader::DomainShaderMaps Shader::_domainShaderMaps;
+Shader::ProgramMap Shader::_programMap;
+
+
+Shader::Shader(Type type, const Source& source) :
     _source(source),
-    _type(type)
+    _type(type),
+    _ID(_nextShaderID++)
 {
 }
 
-Shader::Shader(Type type, const Pointer& vertex, const Pointer& pixel):
+Shader::Shader(Type type, const Pointer& vertex, const Pointer& geometry, const Pointer& pixel):
-    _type(type)
+    _type(type),
+    _ID(_nextShaderID++)
 {
-    _shaders.resize(2);
+    if (geometry) {
-    _shaders[VERTEX] = vertex;
-    _shaders[PIXEL] = pixel;
-}
-
-Shader::Shader(Type type, const Pointer& vertex, const Pointer& geometry, const Pointer& pixel) :
-_type(type) {
         _shaders.resize(3);
         _shaders[VERTEX] = vertex;
         _shaders[GEOMETRY] = geometry;
         _shaders[PIXEL] = pixel;
+    } else {
+        _shaders.resize(2);
+        _shaders[VERTEX] = vertex;
+        _shaders[PIXEL] = pixel;
+    }
 }
 
 Shader::~Shader()
 {
 }
 
+Shader::Pointer Shader::createOrReuseDomainShader(Type type, const Source& source) {
+    auto found = _domainShaderMaps[type].find(source);
+    if (found != _domainShaderMaps[type].end()) {
+        auto sharedShader = (*found).second.lock();
+        if (sharedShader) {
+            return sharedShader;
+        }
+    }
+    auto shader = Pointer(new Shader(type, source));
+    _domainShaderMaps[type].emplace(source, std::weak_ptr<Shader>(shader));
+    return shader;
+}
+
 Shader::Pointer Shader::createVertex(const Source& source) {
-    return Pointer(new Shader(VERTEX, source));
+    return createOrReuseDomainShader(VERTEX, source);
 }
 
 Shader::Pointer Shader::createPixel(const Source& source) {
-    return Pointer(new Shader(PIXEL, source));
+    return createOrReuseDomainShader(PIXEL, source);
 }
 
 Shader::Pointer Shader::createGeometry(const Source& source) {
-    return Pointer(new Shader(GEOMETRY, source));
+    return createOrReuseDomainShader(GEOMETRY, source);
 }
 
-Shader::Pointer Shader::createProgram(const Pointer& vertexShader, const Pointer& pixelShader) {
+ShaderPointer Shader::createOrReuseProgramShader(Type type, const Pointer& vertexShader, const Pointer& geometryShader, const Pointer& pixelShader) {
-    if (vertexShader && vertexShader->getType() == VERTEX &&
+    ProgramMapKey key(0);
-        pixelShader && pixelShader->getType() == PIXEL) {
+
-        return Pointer(new Shader(PROGRAM, vertexShader, pixelShader));
+    if (vertexShader && vertexShader->getType() == VERTEX) {
-    }
+        key.x = vertexShader->getID();
+    } else {
+        // Shader is not valid, exit
         return Pointer();
+    }
+
+    if (pixelShader && pixelShader->getType() == PIXEL) {
+        key.y = pixelShader->getID();
+    } else {
+        // Shader is not valid, exit
+        return Pointer();
+    }
+
+    if (geometryShader) {
+        if (geometryShader->getType() == GEOMETRY) {
+            key.z = geometryShader->getID();
+        } else {
+            // Shader is not valid, exit
+            return Pointer();
+        }
+    }
+
+    // program key is defined, now try to reuse
+    auto found = _programMap.find(key);
+    if (found != _programMap.end()) {
+        auto sharedShader = (*found).second.lock();
+        if (sharedShader) {
+            return sharedShader;
+        }
+    }
+
+    // Program is a new one, let's create it
+    auto program = Pointer(new Shader(type, vertexShader, geometryShader, pixelShader));
+    _programMap.emplace(key, std::weak_ptr<Shader>(program));
+    return program;
+}
+
+
+Shader::Pointer Shader::createProgram(const Pointer& vertexShader, const Pointer& pixelShader) {
+    return createOrReuseProgramShader(PROGRAM, vertexShader, nullptr, pixelShader);
 }
 
 Shader::Pointer Shader::createProgram(const Pointer& vertexShader, const Pointer& geometryShader, const Pointer& pixelShader) {
-    if (vertexShader && vertexShader->getType() == VERTEX &&
+    return createOrReuseProgramShader(PROGRAM, vertexShader, geometryShader, pixelShader);
-        geometryShader && geometryShader->getType() == GEOMETRY &&
-        pixelShader && pixelShader->getType() == PIXEL) {
-        return Pointer(new Shader(PROGRAM, vertexShader, geometryShader, pixelShader));
-    }
-    return Pointer();
 }
 
 void Shader::defineSlots(const SlotSet& uniforms, const SlotSet& uniformBuffers, const SlotSet& resourceBuffers, const SlotSet& textures, const SlotSet& samplers, const SlotSet& inputs, const SlotSet& outputs) {
@@ -82,9 +134,21 @@ void Shader::defineSlots(const SlotSet& uniforms, const SlotSet& uniformBuffers,
     _outputs = outputs;
 }
 
-bool Shader::makeProgram(Shader& shader, const Shader::BindingSet& bindings) {
+bool Shader::makeProgram(Shader& shader, const Shader::BindingSet& bindings, const CompilationHandler& handler) {
     if (shader.isProgram()) {
-        return Context::makeProgram(shader, bindings);
+        return Context::makeProgram(shader, bindings, handler);
     }
     return false;
 }
+
+void Shader::setCompilationLogs(const CompilationLogs& logs) const {
+    _compilationLogs.clear();
+    for (const auto& log : logs) {
+        _compilationLogs.emplace_back(CompilationLog(log));
+    }
+}
+
+void Shader::incrementCompilationAttempt() const {
+    _numCompilationAttempts++;
+}
+

libraries/gpu/src/gpu/Shader.h

@@ -15,6 +15,7 @@
 #include <string>
 #include <memory>
 #include <set>
+#include <map>
 
 #include <QUrl>
  
@@ -22,6 +23,8 @@ namespace gpu {
 
 class Shader {
 public:
+    // unique identifier of a shader
+    using ID = uint32_t;
 
     typedef std::shared_ptr< Shader > Pointer;
     typedef std::vector< Pointer > Shaders;
@@ -39,11 +42,29 @@ public:
  
         virtual const std::string& getCode() const { return _code; }
 
+        class Less {
+        public:
+            bool operator() (const Source& x, const Source& y) const { if (x._lang == y._lang) { return x._code < y._code; } else { return (x._lang < y._lang); } }
+        };
+
     protected:
         std::string _code;
         Language _lang = GLSL;
     };
 
+    struct CompilationLog {
+        std::string message;
+        std::vector<char> binary;
+        bool compiled{ false };
+
+        CompilationLog() {}
+        CompilationLog(const CompilationLog& src) :
+            message(src.message),
+            binary(src.binary),
+            compiled(src.compiled) {}
+    };
+    using CompilationLogs = std::vector<CompilationLog>;
+
     static const int32 INVALID_LOCATION = -1;
 
     class Slot {
@@ -121,13 +142,12 @@ public:
 
     ~Shader();
 
+    ID getID() const { return _ID; }
+
     Type getType() const { return _type; }
     bool isProgram() const { return getType() > NUM_DOMAINS; }
     bool isDomain() const { return getType() < NUM_DOMAINS; }
 
-    void setCompilationHasFailed(bool compilationHasFailed) { _compilationHasFailed = compilationHasFailed; }
-    bool compilationHasFailed() const { return _compilationHasFailed; }
-
     const Source& getSource() const { return _source; }
 
     const Shaders& getShaders() const { return _shaders; }
@@ -155,6 +175,15 @@ public:
                      const SlotSet& inputs,
                      const SlotSet& outputs);
 
+    // Compilation Handler can be passed while compiling a shader (in the makeProgram call) to be able to give the hand to
+    // the caller thread if the comilation fails and to prvide a different version of the source for it
+    // @param0 the Shader object that just failed to compile
+    // @param1 the original source code as submited to the compiler
+    // @param2 the compilation log containing the error message
+    // @param3 a new string ready to be filled with the new version of the source that could be proposed from the handler functor
+    // @return boolean true if the backend should keep trying to compile the shader with the new source returned or false to stop and fail that shader compilation
+    using CompilationHandler = std::function<bool (const Shader&, const std::string&, CompilationLog&, std::string&)>; 
+
     // makeProgram(...) make a program shader ready to be used in a Batch.
     // It compiles the sub shaders, link them and defines the Slots and their bindings.
     // If the shader passed is not a program, nothing happens. 
@@ -168,18 +197,29 @@ public:
     // on a gl Context and the driver to compile the glsl shader. 
     // Hoppefully in a few years the shader compilation will be completely abstracted in a separate shader compiler library
     // independant of the graphics api in use underneath (looking at you opengl & vulkan).
-    static bool makeProgram(Shader& shader, const Shader::BindingSet& bindings = Shader::BindingSet());
+    static bool makeProgram(Shader& shader, const Shader::BindingSet& bindings = Shader::BindingSet(), const CompilationHandler& handler = nullptr);
+
+    // Check the compilation state
+    bool compilationHasFailed() const { return _compilationHasFailed; }
+    const CompilationLogs& getCompilationLogs() const { return _compilationLogs; }
+    uint32_t getNumCompilationAttempts() const { return _numCompilationAttempts; }
+
+    // Set COmpilation logs can only be called by the Backend layers
+    void setCompilationHasFailed(bool compilationHasFailed) { _compilationHasFailed = compilationHasFailed; }
+    void setCompilationLogs(const CompilationLogs& logs) const;
+    void incrementCompilationAttempt() const;
+
 
     const GPUObjectPointer gpuObject {};
 
 protected:
     Shader(Type type, const Source& source);
-    Shader(Type type, const Pointer& vertex, const Pointer& pixel);
     Shader(Type type, const Pointer& vertex, const Pointer& geometry, const Pointer& pixel);
 
     Shader(const Shader& shader); // deep copy of the sysmem shader
     Shader& operator=(const Shader& shader); // deep copy of the sysmem texture
 
+
     // Source contains the actual source code or nothing if the shader is a program
     Source _source;
 
@@ -198,8 +238,49 @@ protected:
     // The type of the shader, the master key
     Type _type;
 
+    // The unique identifier of a shader in the GPU lib
+    uint32_t _ID{ 0 };
+
+    // Number of attempts to compile the shader
+    mutable uint32_t _numCompilationAttempts{ 0 };
+    // Compilation logs (one for each versions generated)
+    mutable CompilationLogs _compilationLogs;
+
     // Whether or not the shader compilation failed
     bool _compilationHasFailed { false };
+
+
+    // Global maps of the shaders 
+    // Unique shader ID
+    static std::atomic<ID> _nextShaderID;
+
+    using ShaderMap =  std::map<Source, std::weak_ptr<Shader>, Source::Less>;
+    using DomainShaderMaps = std::array<ShaderMap, NUM_DOMAINS>;
+    static DomainShaderMaps _domainShaderMaps;
+
+    static ShaderPointer createOrReuseDomainShader(Type type, const Source& source);
+
+    using ProgramMapKey = glm::uvec3; // The IDs of the shaders in a program make its key
+    class ProgramKeyLess {
+    public:
+        bool operator() (const ProgramMapKey& l, const ProgramMapKey& r) const {
+            if (l.x == r.x) {
+                if (l.y == r.y) {
+                    return (l.z < r.z);
+                }
+                else {
+                    return (l.y < r.y);
+                }
+            }
+            else {
+                return (l.x < r.x);
+            }
+        }
+    };
+    using ProgramMap = std::map<ProgramMapKey, std::weak_ptr<Shader>, ProgramKeyLess>;
+    static ProgramMap _programMap;
+
+    static ShaderPointer createOrReuseProgramShader(Type type, const Pointer& vertexShader, const Pointer& geometryShader, const Pointer& pixelShader);
 };
 
 typedef Shader::Pointer ShaderPointer;

libraries/gpu/src/gpu/null/NullBackend.h

@@ -28,7 +28,7 @@ class Backend : public gpu::Backend {
     friend class gpu::Context;
     static void init() {}
     static gpu::Backend* createBackend() { return new Backend(); }
-    static bool makeProgram(Shader& shader, const Shader::BindingSet& slotBindings) { return true; }
+    static bool makeProgram(Shader& shader, const Shader::BindingSet& slotBindings, const Shader::CompilationHandler& handler) { return true; }
 
 protected:
     explicit Backend(bool syncCache) : Parent() { }

libraries/render-utils/src/LightAmbient.slh

@@ -46,7 +46,7 @@ vec3 evalAmbientSpecularIrradiance(LightAmbient ambient, SurfaceData surface) {
             float levels = getLightAmbientMapNumMips(ambient);
             float m = 12.0 / (1.0+11.0*surface.roughness);
             float lod = levels - m;
-            lod = max(lod, 0);
+            lod = max(lod, 0.0);
             specularLight = evalSkyboxLight(lightDir, lod).xyz;
         }
     <@endif@>

libraries/render/src/render/ShapePipeline.cpp

@@ -73,8 +73,9 @@ void ShapePlumber::addPipeline(const Filter& filter, const gpu::ShaderPointer& p
 
     ShapeKey key{ filter._flags };
 
+
     // don't call makeProgram on shaders that are already made.
-    if (program->getUniformBuffers().empty()) {
+    if (program->getNumCompilationAttempts() < 1) {
         gpu::Shader::BindingSet slotBindings;
         slotBindings.insert(gpu::Shader::Binding(std::string("lightingModelBuffer"), Slot::BUFFER::LIGHTING_MODEL));
         slotBindings.insert(gpu::Shader::Binding(std::string("skinClusterBuffer"), Slot::BUFFER::SKINNING));

plugins/openvr/src/OpenVrDisplayPlugin.cpp

@@ -200,9 +200,10 @@ public:
             std::string fsSource = HMD_REPROJECTION_FRAG;
             GLuint vertexShader { 0 }, fragmentShader { 0 };
             std::string error;
+            std::vector<char> binary;
             ::gl::compileShader(GL_VERTEX_SHADER, vsSource, "", vertexShader, error);
             ::gl::compileShader(GL_FRAGMENT_SHADER, fsSource, "", fragmentShader, error);
-            _program = ::gl::compileProgram({ { vertexShader, fragmentShader } }, error);
+            _program = ::gl::compileProgram({ { vertexShader, fragmentShader } }, error, binary);
             glDeleteShader(vertexShader);
             glDeleteShader(fragmentShader);
             qDebug() << "Rebuild proigram";

tests/shaders/src/main.cpp

@@ -137,12 +137,13 @@ const std::string PIXEL_SHADER_DEFINES{ R"GLSL(
 
 void testShaderBuild(const char* vs_src, const char * fs_src) {
     std::string error;
+    std::vector<char> binary;
     GLuint vs, fs;
     if (!gl::compileShader(GL_VERTEX_SHADER, vs_src, VERTEX_SHADER_DEFINES, vs, error) || 
         !gl::compileShader(GL_FRAGMENT_SHADER, fs_src, PIXEL_SHADER_DEFINES, fs, error)) {
         throw std::runtime_error("Failed to compile shader");
     }
-    auto pr = gl::compileProgram({ vs, fs }, error);
+    auto pr = gl::compileProgram({ vs, fs }, error, binary);
     if (!pr) {
         throw std::runtime_error("Failed to link shader");
     }