diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 4f4c528abf..1a54c94d53 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2296,16 +2296,16 @@ void Application::initializeGL() { #endif _renderEngine->addJob("RenderMainView", cullFunctor, !DISABLE_DEFERRED, render::ItemKey::TAG_BITS_0, render::ItemKey::TAG_BITS_0); -#ifdef Q_OS_OSX - DeadlockWatchdogThread::resume(); -#endif - _renderEngine->load(); _renderEngine->registerScene(_main3DScene); // Now that OpenGL is initialized, we are sure we have a valid context and can create the various pipeline shaders with success. DependencyManager::get()->initializeShapePipelines(); +#ifdef Q_OS_OSX + DeadlockWatchdogThread::resume(); +#endif + _offscreenContext = new OffscreenGLCanvas(); _offscreenContext->setObjectName("MainThreadContext"); _offscreenContext->create(_glWidget->qglContext()); diff --git a/libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp b/libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp index 90bb83a663..d35d5c5317 100644 --- a/libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp +++ b/libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp @@ -398,7 +398,7 @@ void HmdDisplayPlugin::HUDRenderer::updatePipeline() { auto vs = gpu::Shader::createVertex(std::string(hmd_ui_vert)); auto ps = gpu::Shader::createPixel(std::string(hmd_ui_frag)); auto program = gpu::Shader::createProgram(vs, ps); - gpu::gl::GLBackend::makeProgram(*program, gpu::Shader::BindingSet()); + gpu::Shader::makeProgram(*program, gpu::Shader::BindingSet()); uniformsLocation = program->getUniformBuffers().findLocation("hudBuffer"); gpu::StatePointer state = gpu::StatePointer(new gpu::State()); diff --git a/libraries/gl/src/gl/GLShaders.cpp b/libraries/gl/src/gl/GLShaders.cpp index 017c92b71c..ecd6fe3323 100644 --- a/libraries/gl/src/gl/GLShaders.cpp +++ b/libraries/gl/src/gl/GLShaders.cpp @@ -6,9 +6,9 @@ namespace gl { #ifdef SEPARATE_PROGRAM - bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, GLuint &programObject, std::string& error) { + bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, GLuint &programObject, std::string& message) { #else - bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, std::string& error) { + bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, std::string& message) { #endif if (shaderSource.empty()) { qCDebug(glLogging) << "GLShader::compileShader - no GLSL shader source code ? so failed to create"; @@ -34,52 +34,57 @@ namespace gl { GLint compiled = 0; glGetShaderiv(glshader, GL_COMPILE_STATUS, &compiled); - // if compilation fails - if (!compiled) { - - // save the source code to a temp file so we can debug easily - /* - std::ofstream filestream; - filestream.open("debugshader.glsl"); - if (filestream.is_open()) { - filestream << srcstr[0]; - filestream << srcstr[1]; - filestream.close(); - } - */ - - GLint infoLength = 0; - glGetShaderiv(glshader, GL_INFO_LOG_LENGTH, &infoLength); + GLint infoLength = 0; + glGetShaderiv(glshader, GL_INFO_LOG_LENGTH, &infoLength); + if ((infoLength > 0) || !compiled) { char* temp = new char[infoLength]; glGetShaderInfoLog(glshader, infoLength, NULL, temp); + message = std::string(temp); - /* - filestream.open("debugshader.glsl.info.txt"); - if (filestream.is_open()) { - filestream << std::string(temp); - filestream.close(); - } - */ - - qCCritical(glLogging) << "GLShader::compileShader - failed to compile the gl shader object:"; - int lineNumber = 0; - for (auto s : srcstr) { - QString str(s); - QStringList lines = str.split("\n"); - for (auto& line : lines) { - qCCritical(glLogging).noquote() << QString("%1: %2").arg(lineNumber++, 5, 10, QChar('0')).arg(line); + // if compilation fails + if (!compiled) { + // save the source code to a temp file so we can debug easily + /* + std::ofstream filestream; + filestream.open("debugshader.glsl"); + if (filestream.is_open()) { + filestream << srcstr[0]; + filestream << srcstr[1]; + filestream.close(); } + */ + + /* + filestream.open("debugshader.glsl.info.txt"); + if (filestream.is_open()) { + filestream << std::string(temp); + filestream.close(); + } + */ + + qCCritical(glLogging) << "GLShader::compileShader - failed to compile the gl shader object:"; + int lineNumber = 0; + for (auto s : srcstr) { + QString str(s); + QStringList lines = str.split("\n"); + for (auto& line : lines) { + qCCritical(glLogging).noquote() << QString("%1: %2").arg(lineNumber++, 5, 10, QChar('0')).arg(line); + } + } + qCCritical(glLogging) << "GLShader::compileShader - errors:"; + qCCritical(glLogging) << temp; + + delete[] temp; + glDeleteShader(glshader); + return false; } - qCCritical(glLogging) << "GLShader::compileShader - errors:"; - qCCritical(glLogging) << temp; - error = std::string(temp); + // Compilation success + qCWarning(glLogging) << "GLShader::compileShader - Success:"; + qCWarning(glLogging) << temp; delete[] temp; - - glDeleteShader(glshader); - return false; } #ifdef SEPARATE_PROGRAM @@ -137,7 +142,7 @@ namespace gl { return true; } -GLuint compileProgram(const std::vector& glshaders, std::string& error) { +GLuint compileProgram(const std::vector& glshaders, std::string& message, std::vector& binary) { // A brand new program: GLuint glprogram = glCreateProgram(); if (!glprogram) { @@ -157,39 +162,65 @@ GLuint compileProgram(const std::vector& glshaders, std::string& error) GLint linked = 0; glGetProgramiv(glprogram, GL_LINK_STATUS, &linked); - if (!linked) { - /* - // save the source code to a temp file so we can debug easily - std::ofstream filestream; - filestream.open("debugshader.glsl"); - if (filestream.is_open()) { - filestream << shaderSource->source; - filestream.close(); - } - */ - - GLint infoLength = 0; - glGetProgramiv(glprogram, GL_INFO_LOG_LENGTH, &infoLength); + GLint infoLength = 0; + glGetProgramiv(glprogram, GL_INFO_LOG_LENGTH, &infoLength); + if ((infoLength > 0) || !linked) { char* temp = new char[infoLength]; glGetProgramInfoLog(glprogram, infoLength, NULL, temp); - qCDebug(glLogging) << "GLShader::compileProgram - failed to LINK the gl program object :"; - qCDebug(glLogging) << temp; + message = std::string(temp); - error = std::string(temp); - delete[] temp; + if (!linked) { + /* + // save the source code to a temp file so we can debug easily + std::ofstream filestream; + filestream.open("debugshader.glsl"); + if (filestream.is_open()) { + filestream << shaderSource->source; + filestream.close(); + } + */ - /* - filestream.open("debugshader.glsl.info.txt"); - if (filestream.is_open()) { - filestream << std::string(temp); - filestream.close(); + qCDebug(glLogging) << "GLShader::compileProgram - failed to LINK the gl program object :"; + qCDebug(glLogging) << temp; + + delete[] temp; + + /* + filestream.open("debugshader.glsl.info.txt"); + if (filestream.is_open()) { + filestream << std::string(temp); + filestream.close(); + } + */ + + glDeleteProgram(glprogram); + return 0; + } else { + qCDebug(glLogging) << "GLShader::compileProgram - success:"; + qCDebug(glLogging) << temp; + delete[] temp; } - */ + } - glDeleteProgram(glprogram); - return 0; + // If linked get the binaries + if (linked) { + GLint binaryLength = 0; + glGetProgramiv(glprogram, GL_PROGRAM_BINARY_LENGTH, &binaryLength); + + if (binaryLength > 0) { + GLint numBinFormats = 0; + glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, &numBinFormats); + if (numBinFormats > 0) { + binary.resize(binaryLength); + std::vector binFormats(numBinFormats); + glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, binFormats.data()); + + GLenum programBinFormat; + glGetProgramBinary(glprogram, binaryLength, NULL, &programBinFormat, binary.data()); + } + } } return glprogram; diff --git a/libraries/gl/src/gl/GLShaders.h b/libraries/gl/src/gl/GLShaders.h index a6213fd280..fc070d7659 100644 --- a/libraries/gl/src/gl/GLShaders.h +++ b/libraries/gl/src/gl/GLShaders.h @@ -17,12 +17,12 @@ namespace gl { #ifdef SEPARATE_PROGRAM - bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, GLuint &programObject, std::string& error); + bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, GLuint &programObject, std::string& message); #else - bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, std::string& error); + bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, std::string& message); #endif - GLuint compileProgram(const std::vector& glshaders, std::string& error); + GLuint compileProgram(const std::vector& glshaders, std::string& message, std::vector& binary); } diff --git a/libraries/gpu-gl/src/gpu/gl/GLBackend.cpp b/libraries/gpu-gl/src/gpu/gl/GLBackend.cpp index eb6de5df13..08bd20be66 100644 --- a/libraries/gpu-gl/src/gpu/gl/GLBackend.cpp +++ b/libraries/gpu-gl/src/gpu/gl/GLBackend.cpp @@ -68,8 +68,8 @@ GLBackend& getBackend() { return *INSTANCE; } -bool GLBackend::makeProgram(Shader& shader, const Shader::BindingSet& slotBindings) { - return GLShader::makeProgram(getBackend(), shader, slotBindings); +bool GLBackend::makeProgram(Shader& shader, const Shader::BindingSet& slotBindings, const Shader::CompilationHandler& handler) { + return GLShader::makeProgram(getBackend(), shader, slotBindings, handler); } GLBackend::CommandCall GLBackend::_commandCalls[Batch::NUM_COMMANDS] = diff --git a/libraries/gpu-gl/src/gpu/gl/GLBackend.h b/libraries/gpu-gl/src/gpu/gl/GLBackend.h index 5558d3ada1..18916ac18c 100644 --- a/libraries/gpu-gl/src/gpu/gl/GLBackend.h +++ b/libraries/gpu-gl/src/gpu/gl/GLBackend.h @@ -64,7 +64,7 @@ protected: explicit GLBackend(bool syncCache); GLBackend(); public: - static bool makeProgram(Shader& shader, const Shader::BindingSet& slotBindings = Shader::BindingSet()); + static bool makeProgram(Shader& shader, const Shader::BindingSet& slotBindings, const Shader::CompilationHandler& handler); virtual ~GLBackend(); @@ -423,8 +423,8 @@ protected: } _pipeline; // Backend dependant compilation of the shader - virtual GLShader* compileBackendProgram(const Shader& program); - virtual GLShader* compileBackendShader(const Shader& shader); + virtual GLShader* compileBackendProgram(const Shader& program, const Shader::CompilationHandler& handler); + virtual GLShader* compileBackendShader(const Shader& shader, const Shader::CompilationHandler& handler); virtual std::string getBackendShaderHeader() const; virtual void makeProgramBindings(ShaderObject& shaderObject); class ElementResource { diff --git a/libraries/gpu-gl/src/gpu/gl/GLBackendShader.cpp b/libraries/gpu-gl/src/gpu/gl/GLBackendShader.cpp index 9adfd550ef..93c9b0d2ff 100644 --- a/libraries/gpu-gl/src/gpu/gl/GLBackendShader.cpp +++ b/libraries/gpu-gl/src/gpu/gl/GLBackendShader.cpp @@ -56,28 +56,47 @@ static const std::array VERSION_DEFINES { { stereoVersion } }; -GLShader* GLBackend::compileBackendShader(const Shader& shader) { +GLShader* GLBackend::compileBackendShader(const Shader& shader, const Shader::CompilationHandler& handler) { // Any GLSLprogram ? normally yes... const std::string& shaderSource = shader.getSource().getCode(); GLenum shaderDomain = SHADER_DOMAINS[shader.getType()]; GLShader::ShaderObjects shaderObjects; + Shader::CompilationLogs compilationLogs(GLShader::NumVersions); + shader.incrementCompilationAttempt(); for (int version = 0; version < GLShader::NumVersions; version++) { auto& shaderObject = shaderObjects[version]; std::string shaderDefines = getBackendShaderHeader() + "\n" + DOMAIN_DEFINES[shader.getType()] + "\n" + VERSION_DEFINES[version]; - std::string error; + if (handler) { + bool retest = true; + std::string currentSrc = shaderSource; + // When a Handler is specified, we can try multiple times to build the shader and let the handler change the source if the compilation fails. + // The retest bool is set to false as soon as the compilation succeed to wexit the while loop. + // The handler tells us if we should retry or not while returning a modified version of the source. + while (retest) { + bool result = ::gl::compileShader(shaderDomain, currentSrc, shaderDefines, shaderObject.glshader, compilationLogs[version].message); + compilationLogs[version].compiled = result; + if (!result) { + std::string newSrc; + retest = handler(shader, currentSrc, compilationLogs[version], newSrc); + currentSrc = newSrc; + } else { + retest = false; + } + } + } else { + compilationLogs[version].compiled = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, compilationLogs[version].message); + } -#ifdef SEPARATE_PROGRAM - bool result = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, shaderObject.glprogram, error); -#else - bool result = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, error); -#endif - if (!result) { - qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Shader didn't compile:\n" << error.c_str(); + if (!compilationLogs[version].compiled) { + qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Shader didn't compile:\n" << compilationLogs[version].message.c_str(); + shader.setCompilationLogs(compilationLogs); return nullptr; } } + // Compilation feedback + shader.setCompilationLogs(compilationLogs); // So far so good, the shader is created successfully GLShader* object = new GLShader(this->shared_from_this()); @@ -86,39 +105,47 @@ GLShader* GLBackend::compileBackendShader(const Shader& shader) { return object; } -GLShader* GLBackend::compileBackendProgram(const Shader& program) { +GLShader* GLBackend::compileBackendProgram(const Shader& program, const Shader::CompilationHandler& handler) { if (!program.isProgram()) { return nullptr; } GLShader::ShaderObjects programObjects; + program.incrementCompilationAttempt(); + Shader::CompilationLogs compilationLogs(GLShader::NumVersions); + for (int version = 0; version < GLShader::NumVersions; version++) { auto& programObject = programObjects[version]; // Let's go through every shaders and make sure they are ready to go std::vector< GLuint > shaderGLObjects; for (auto subShader : program.getShaders()) { - auto object = GLShader::sync((*this), *subShader); + auto object = GLShader::sync((*this), *subShader, handler); if (object) { shaderGLObjects.push_back(object->_shaderObjects[version].glshader); } else { qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - One of the shaders of the program is not compiled?"; + compilationLogs[version].compiled = false; + compilationLogs[version].message = std::string("Failed to compile, one of the shaders of the program is not compiled ?"); + program.setCompilationLogs(compilationLogs); return nullptr; } } - std::string error; - GLuint glprogram = ::gl::compileProgram(shaderGLObjects, error); + GLuint glprogram = ::gl::compileProgram(shaderGLObjects, compilationLogs[version].message, compilationLogs[version].binary); if (glprogram == 0) { - qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Program didn't link:\n" << error.c_str(); + qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Program didn't link:\n" << compilationLogs[version].message.c_str(); + program.setCompilationLogs(compilationLogs); return nullptr; } - + compilationLogs[version].compiled = true; programObject.glprogram = glprogram; makeProgramBindings(programObject); } + // Compilation feedback + program.setCompilationLogs(compilationLogs); // So far so good, the program versions have all been created successfully GLShader* object = new GLShader(this->shared_from_this()); diff --git a/libraries/gpu-gl/src/gpu/gl/GLShader.cpp b/libraries/gpu-gl/src/gpu/gl/GLShader.cpp index 7ed9121978..010a7c479c 100644 --- a/libraries/gpu-gl/src/gpu/gl/GLShader.cpp +++ b/libraries/gpu-gl/src/gpu/gl/GLShader.cpp @@ -30,7 +30,7 @@ GLShader::~GLShader() { } } -GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) { +GLShader* GLShader::sync(GLBackend& backend, const Shader& shader, const Shader::CompilationHandler& handler) { GLShader* object = Backend::getGPUObject(shader); // If GPU object already created then good @@ -39,13 +39,13 @@ GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) { } // need to have a gpu object? if (shader.isProgram()) { - GLShader* tempObject = backend.compileBackendProgram(shader); + GLShader* tempObject = backend.compileBackendProgram(shader, handler); if (tempObject) { object = tempObject; Backend::setGPUObject(shader, object); } } else if (shader.isDomain()) { - GLShader* tempObject = backend.compileBackendShader(shader); + GLShader* tempObject = backend.compileBackendShader(shader, handler); if (tempObject) { object = tempObject; Backend::setGPUObject(shader, object); @@ -56,10 +56,10 @@ GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) { return object; } -bool GLShader::makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings) { +bool GLShader::makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings, const Shader::CompilationHandler& handler) { // First make sure the Shader has been compiled - GLShader* object = sync(backend, shader); + GLShader* object = sync(backend, shader, handler); if (!object) { return false; } diff --git a/libraries/gpu-gl/src/gpu/gl/GLShader.h b/libraries/gpu-gl/src/gpu/gl/GLShader.h index dcf2dc330d..3259982e93 100644 --- a/libraries/gpu-gl/src/gpu/gl/GLShader.h +++ b/libraries/gpu-gl/src/gpu/gl/GLShader.h @@ -21,8 +21,8 @@ struct ShaderObject { class GLShader : public GPUObject { public: - static GLShader* sync(GLBackend& backend, const Shader& shader); - static bool makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings); + static GLShader* sync(GLBackend& backend, const Shader& shader, const Shader::CompilationHandler& handler = nullptr); + static bool makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings, const Shader::CompilationHandler& handler); enum Version { Mono = 0, diff --git a/libraries/gpu-gles/src/gpu/gl/GLBackend.cpp b/libraries/gpu-gles/src/gpu/gl/GLBackend.cpp index 6fd5df6f81..fc1bc39929 100644 --- a/libraries/gpu-gles/src/gpu/gl/GLBackend.cpp +++ b/libraries/gpu-gles/src/gpu/gl/GLBackend.cpp @@ -61,8 +61,8 @@ GLBackend& getBackend() { return *INSTANCE; } -bool GLBackend::makeProgram(Shader& shader, const Shader::BindingSet& slotBindings) { - return GLShader::makeProgram(getBackend(), shader, slotBindings); +bool GLBackend::makeProgram(Shader& shader, const Shader::BindingSet& slotBindings, const Shader::CompilationHandler& handler) { + return GLShader::makeProgram(getBackend(), shader, slotBindings, handler); } diff --git a/libraries/gpu-gles/src/gpu/gl/GLBackend.h b/libraries/gpu-gles/src/gpu/gl/GLBackend.h index ea06b6b672..3681fc0492 100644 --- a/libraries/gpu-gles/src/gpu/gl/GLBackend.h +++ b/libraries/gpu-gles/src/gpu/gl/GLBackend.h @@ -61,7 +61,7 @@ protected: explicit GLBackend(bool syncCache); GLBackend(); public: - static bool makeProgram(Shader& shader, const Shader::BindingSet& slotBindings = Shader::BindingSet()); + static bool makeProgram(Shader& shader, const Shader::BindingSet& slotBindings = Shader::BindingSet(), const Shader::CompilationHandler& handler = nullptr); virtual ~GLBackend(); @@ -420,8 +420,8 @@ protected: } _pipeline; // Backend dependant compilation of the shader - virtual GLShader* compileBackendProgram(const Shader& program); - virtual GLShader* compileBackendShader(const Shader& shader); + virtual GLShader* compileBackendProgram(const Shader& program, const Shader::CompilationHandler& handler); + virtual GLShader* compileBackendShader(const Shader& shader, const Shader::CompilationHandler& handler); virtual std::string getBackendShaderHeader() const; virtual void makeProgramBindings(ShaderObject& shaderObject); class ElementResource { diff --git a/libraries/gpu-gles/src/gpu/gl/GLBackendShader.cpp b/libraries/gpu-gles/src/gpu/gl/GLBackendShader.cpp index fd44ad462f..677bba97ca 100644 --- a/libraries/gpu-gles/src/gpu/gl/GLBackendShader.cpp +++ b/libraries/gpu-gles/src/gpu/gl/GLBackendShader.cpp @@ -56,32 +56,50 @@ static const std::array VERSION_DEFINES { { stereoVersion } }; -GLShader* GLBackend::compileBackendShader(const Shader& shader) { +GLShader* GLBackend::compileBackendShader(const Shader& shader, const Shader::CompilationHandler& handler) { // Any GLSLprogram ? normally yes... const std::string& shaderSource = shader.getSource().getCode(); GLenum shaderDomain = SHADER_DOMAINS[shader.getType()]; GLShader::ShaderObjects shaderObjects; + Shader::CompilationLogs compilationLogs(GLShader::NumVersions); for (int version = 0; version < GLShader::NumVersions; version++) { auto& shaderObject = shaderObjects[version]; std::string shaderDefines = getBackendShaderHeader() + "\n" + DOMAIN_DEFINES[shader.getType()] + "\n" + VERSION_DEFINES[version] - + "\n#extension GL_EXT_texture_buffer : enable" - + "\nprecision lowp float; // check precision 2" - + "\nprecision lowp samplerBuffer;" - + "\nprecision lowp sampler2DShadow;"; - std::string error; + + "\n#extension GL_EXT_texture_buffer : enable" + + "\nprecision lowp float; // check precision 2" + + "\nprecision lowp samplerBuffer;" + + "\nprecision lowp sampler2DShadow;"; + if (handler) { + bool retest = true; + std::string currentSrc = shaderSource; + // When a Handler is specified, we can try multiple times to build the shader and let the handler change the source if the compilation fails. + // The retest bool is set to false as soon as the compilation succeed to wexit the while loop. + // The handler tells us if we should retry or not while returning a modified version of the source. + while (retest) { + bool result = ::gl::compileShader(shaderDomain, currentSrc, shaderDefines, shaderObject.glshader, compilationLogs[version].message); + compilationLogs[version].compiled = result; + if (!result) { + std::string newSrc; + retest = handler(shader, currentSrc, compilationLogs[version], newSrc); + currentSrc = newSrc; + } else { + retest = false; + } + } + } else { + compilationLogs[version].compiled = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, compilationLogs[version].message); + } -#ifdef SEPARATE_PROGRAM - bool result = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, shaderObject.glprogram, error); -#else - bool result = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, error); -#endif - if (!result) { - qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Shader didn't compile:\n" << error.c_str(); + if (!compilationLogs[version].compiled) { + qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Shader didn't compile:\n" << compilationLogs[version].message.c_str(); + shader.setCompilationLogs(compilationLogs); return nullptr; } } + // Compilation feedback + shader.setCompilationLogs(compilationLogs); // So far so good, the shader is created successfully GLShader* object = new GLShader(this->shared_from_this()); @@ -90,32 +108,35 @@ GLShader* GLBackend::compileBackendShader(const Shader& shader) { return object; } -GLShader* GLBackend::compileBackendProgram(const Shader& program) { +GLShader* GLBackend::compileBackendProgram(const Shader& program, const Shader::CompilationHandler& handler) { if (!program.isProgram()) { return nullptr; } GLShader::ShaderObjects programObjects; + Shader::CompilationLogs compilationLogs(GLShader::NumVersions); + for (int version = 0; version < GLShader::NumVersions; version++) { auto& programObject = programObjects[version]; // Let's go through every shaders and make sure they are ready to go - std::vector shaderGLObjects; + std::vector< GLuint > shaderGLObjects; for (auto subShader : program.getShaders()) { - auto object = GLShader::sync((*this), *subShader); + auto object = GLShader::sync((*this), *subShader, handler); if (object) { shaderGLObjects.push_back(object->_shaderObjects[version].glshader); } else { qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - One of the shaders of the program is not compiled?"; + program.setCompilationLogs(compilationLogs); return nullptr; } } - std::string error; - GLuint glprogram = ::gl::compileProgram(shaderGLObjects, error); + GLuint glprogram = ::gl::compileProgram(shaderGLObjects, compilationLogs[version].message, compilationLogs[version].binary); if (glprogram == 0) { - qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Program didn't link:\n" << error.c_str(); + qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Program didn't link:\n" << compilationLogs[version].message.c_str(); + program.setCompilationLogs(compilationLogs); return nullptr; } @@ -123,6 +144,8 @@ GLShader* GLBackend::compileBackendProgram(const Shader& program) { makeProgramBindings(programObject); } + // Compilation feedback + program.setCompilationLogs(compilationLogs); // So far so good, the program versions have all been created successfully GLShader* object = new GLShader(this->shared_from_this()); diff --git a/libraries/gpu-gles/src/gpu/gl/GLShader.cpp b/libraries/gpu-gles/src/gpu/gl/GLShader.cpp index 7ed9121978..010a7c479c 100644 --- a/libraries/gpu-gles/src/gpu/gl/GLShader.cpp +++ b/libraries/gpu-gles/src/gpu/gl/GLShader.cpp @@ -30,7 +30,7 @@ GLShader::~GLShader() { } } -GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) { +GLShader* GLShader::sync(GLBackend& backend, const Shader& shader, const Shader::CompilationHandler& handler) { GLShader* object = Backend::getGPUObject(shader); // If GPU object already created then good @@ -39,13 +39,13 @@ GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) { } // need to have a gpu object? if (shader.isProgram()) { - GLShader* tempObject = backend.compileBackendProgram(shader); + GLShader* tempObject = backend.compileBackendProgram(shader, handler); if (tempObject) { object = tempObject; Backend::setGPUObject(shader, object); } } else if (shader.isDomain()) { - GLShader* tempObject = backend.compileBackendShader(shader); + GLShader* tempObject = backend.compileBackendShader(shader, handler); if (tempObject) { object = tempObject; Backend::setGPUObject(shader, object); @@ -56,10 +56,10 @@ GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) { return object; } -bool GLShader::makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings) { +bool GLShader::makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings, const Shader::CompilationHandler& handler) { // First make sure the Shader has been compiled - GLShader* object = sync(backend, shader); + GLShader* object = sync(backend, shader, handler); if (!object) { return false; } diff --git a/libraries/gpu-gles/src/gpu/gl/GLShader.h b/libraries/gpu-gles/src/gpu/gl/GLShader.h index dcf2dc330d..f2a144a81c 100644 --- a/libraries/gpu-gles/src/gpu/gl/GLShader.h +++ b/libraries/gpu-gles/src/gpu/gl/GLShader.h @@ -21,8 +21,8 @@ struct ShaderObject { class GLShader : public GPUObject { public: - static GLShader* sync(GLBackend& backend, const Shader& shader); - static bool makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings); + static GLShader* sync(GLBackend& backend, const Shader& shader, const Shader::CompilationHandler& handler = nullptr); + static bool makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings, const Shader::CompilationHandler& handler = nullptr); enum Version { Mono = 0, diff --git a/libraries/gpu/src/gpu/Context.cpp b/libraries/gpu/src/gpu/Context.cpp index 24128524da..d7d86c3ef7 100644 --- a/libraries/gpu/src/gpu/Context.cpp +++ b/libraries/gpu/src/gpu/Context.cpp @@ -127,7 +127,7 @@ void Context::executeFrame(const FramePointer& frame) const { _frameStats.evalDelta(beginStats, endStats); } -bool Context::makeProgram(Shader& shader, const Shader::BindingSet& bindings) { +bool Context::makeProgram(Shader& shader, const Shader::BindingSet& bindings, const Shader::CompilationHandler& handler) { // If we're running in another DLL context, we need to fetch the program callback out of the application // FIXME find a way to do this without reliance on Qt app properties if (!_makeProgramCallback) { @@ -135,7 +135,7 @@ bool Context::makeProgram(Shader& shader, const Shader::BindingSet& bindings) { _makeProgramCallback = reinterpret_cast(rawCallback); } if (shader.isProgram() && _makeProgramCallback) { - return _makeProgramCallback(shader, bindings); + return _makeProgramCallback(shader, bindings, handler); } return false; } diff --git a/libraries/gpu/src/gpu/Context.h b/libraries/gpu/src/gpu/Context.h index 7b7575e9ed..195565f438 100644 --- a/libraries/gpu/src/gpu/Context.h +++ b/libraries/gpu/src/gpu/Context.h @@ -143,7 +143,7 @@ class Context { public: using Size = Resource::Size; typedef BackendPointer (*CreateBackend)(); - typedef bool (*MakeProgram)(Shader& shader, const Shader::BindingSet& bindings); + typedef bool (*MakeProgram)(Shader& shader, const Shader::BindingSet& bindings, const Shader::CompilationHandler& handler); // This one call must happen before any context is created or used (Shader::MakeProgram) in order to setup the Backend and any singleton data needed @@ -262,7 +262,7 @@ protected: // makeProgramShader(...) make a program shader ready to be used in a Batch. // It compiles the sub shaders, link them and defines the Slots and their bindings. // If the shader passed is not a program, nothing happens. - static bool makeProgram(Shader& shader, const Shader::BindingSet& bindings); + static bool makeProgram(Shader& shader, const Shader::BindingSet& bindings, const Shader::CompilationHandler& handler); static CreateBackend _createBackendCallback; static MakeProgram _makeProgramCallback; diff --git a/libraries/gpu/src/gpu/Shader.cpp b/libraries/gpu/src/gpu/Shader.cpp index 398a269f3f..aa7898569b 100755 --- a/libraries/gpu/src/gpu/Shader.cpp +++ b/libraries/gpu/src/gpu/Shader.cpp @@ -17,59 +17,111 @@ using namespace gpu; -Shader::Shader(Type type, const Source& source): +std::atomic Shader::_nextShaderID( 1 ); +Shader::DomainShaderMaps Shader::_domainShaderMaps; +Shader::ProgramMap Shader::_programMap; + + +Shader::Shader(Type type, const Source& source) : _source(source), - _type(type) + _type(type), + _ID(_nextShaderID++) { } -Shader::Shader(Type type, const Pointer& vertex, const Pointer& pixel): - _type(type) +Shader::Shader(Type type, const Pointer& vertex, const Pointer& geometry, const Pointer& pixel): + _type(type), + _ID(_nextShaderID++) { - _shaders.resize(2); - _shaders[VERTEX] = vertex; - _shaders[PIXEL] = pixel; -} - -Shader::Shader(Type type, const Pointer& vertex, const Pointer& geometry, const Pointer& pixel) : -_type(type) { - _shaders.resize(3); - _shaders[VERTEX] = vertex; - _shaders[GEOMETRY] = geometry; - _shaders[PIXEL] = pixel; + if (geometry) { + _shaders.resize(3); + _shaders[VERTEX] = vertex; + _shaders[GEOMETRY] = geometry; + _shaders[PIXEL] = pixel; + } else { + _shaders.resize(2); + _shaders[VERTEX] = vertex; + _shaders[PIXEL] = pixel; + } } Shader::~Shader() { } +Shader::Pointer Shader::createOrReuseDomainShader(Type type, const Source& source) { + auto found = _domainShaderMaps[type].find(source); + if (found != _domainShaderMaps[type].end()) { + auto sharedShader = (*found).second.lock(); + if (sharedShader) { + return sharedShader; + } + } + auto shader = Pointer(new Shader(type, source)); + _domainShaderMaps[type].emplace(source, std::weak_ptr(shader)); + return shader; +} + Shader::Pointer Shader::createVertex(const Source& source) { - return Pointer(new Shader(VERTEX, source)); + return createOrReuseDomainShader(VERTEX, source); } Shader::Pointer Shader::createPixel(const Source& source) { - return Pointer(new Shader(PIXEL, source)); + return createOrReuseDomainShader(PIXEL, source); } Shader::Pointer Shader::createGeometry(const Source& source) { - return Pointer(new Shader(GEOMETRY, source)); + return createOrReuseDomainShader(GEOMETRY, source); } -Shader::Pointer Shader::createProgram(const Pointer& vertexShader, const Pointer& pixelShader) { - if (vertexShader && vertexShader->getType() == VERTEX && - pixelShader && pixelShader->getType() == PIXEL) { - return Pointer(new Shader(PROGRAM, vertexShader, pixelShader)); +ShaderPointer Shader::createOrReuseProgramShader(Type type, const Pointer& vertexShader, const Pointer& geometryShader, const Pointer& pixelShader) { + ProgramMapKey key(0); + + if (vertexShader && vertexShader->getType() == VERTEX) { + key.x = vertexShader->getID(); + } else { + // Shader is not valid, exit + return Pointer(); } - return Pointer(); + + if (pixelShader && pixelShader->getType() == PIXEL) { + key.y = pixelShader->getID(); + } else { + // Shader is not valid, exit + return Pointer(); + } + + if (geometryShader) { + if (geometryShader->getType() == GEOMETRY) { + key.z = geometryShader->getID(); + } else { + // Shader is not valid, exit + return Pointer(); + } + } + + // program key is defined, now try to reuse + auto found = _programMap.find(key); + if (found != _programMap.end()) { + auto sharedShader = (*found).second.lock(); + if (sharedShader) { + return sharedShader; + } + } + + // Program is a new one, let's create it + auto program = Pointer(new Shader(type, vertexShader, geometryShader, pixelShader)); + _programMap.emplace(key, std::weak_ptr(program)); + return program; +} + + +Shader::Pointer Shader::createProgram(const Pointer& vertexShader, const Pointer& pixelShader) { + return createOrReuseProgramShader(PROGRAM, vertexShader, nullptr, pixelShader); } Shader::Pointer Shader::createProgram(const Pointer& vertexShader, const Pointer& geometryShader, const Pointer& pixelShader) { - if (vertexShader && vertexShader->getType() == VERTEX && - geometryShader && geometryShader->getType() == GEOMETRY && - pixelShader && pixelShader->getType() == PIXEL) { - return Pointer(new Shader(PROGRAM, vertexShader, geometryShader, pixelShader)); - } - return Pointer(); + return createOrReuseProgramShader(PROGRAM, vertexShader, geometryShader, pixelShader); } void Shader::defineSlots(const SlotSet& uniforms, const SlotSet& uniformBuffers, const SlotSet& resourceBuffers, const SlotSet& textures, const SlotSet& samplers, const SlotSet& inputs, const SlotSet& outputs) { @@ -82,9 +134,21 @@ void Shader::defineSlots(const SlotSet& uniforms, const SlotSet& uniformBuffers, _outputs = outputs; } -bool Shader::makeProgram(Shader& shader, const Shader::BindingSet& bindings) { +bool Shader::makeProgram(Shader& shader, const Shader::BindingSet& bindings, const CompilationHandler& handler) { if (shader.isProgram()) { - return Context::makeProgram(shader, bindings); + return Context::makeProgram(shader, bindings, handler); } return false; } + +void Shader::setCompilationLogs(const CompilationLogs& logs) const { + _compilationLogs.clear(); + for (const auto& log : logs) { + _compilationLogs.emplace_back(CompilationLog(log)); + } +} + +void Shader::incrementCompilationAttempt() const { + _numCompilationAttempts++; +} + diff --git a/libraries/gpu/src/gpu/Shader.h b/libraries/gpu/src/gpu/Shader.h index 181c9b5e78..4504337789 100755 --- a/libraries/gpu/src/gpu/Shader.h +++ b/libraries/gpu/src/gpu/Shader.h @@ -15,6 +15,7 @@ #include #include #include +#include #include @@ -22,6 +23,8 @@ namespace gpu { class Shader { public: + // unique identifier of a shader + using ID = uint32_t; typedef std::shared_ptr< Shader > Pointer; typedef std::vector< Pointer > Shaders; @@ -39,11 +42,29 @@ public: virtual const std::string& getCode() const { return _code; } + class Less { + public: + bool operator() (const Source& x, const Source& y) const { if (x._lang == y._lang) { return x._code < y._code; } else { return (x._lang < y._lang); } } + }; + protected: std::string _code; Language _lang = GLSL; }; + struct CompilationLog { + std::string message; + std::vector binary; + bool compiled{ false }; + + CompilationLog() {} + CompilationLog(const CompilationLog& src) : + message(src.message), + binary(src.binary), + compiled(src.compiled) {} + }; + using CompilationLogs = std::vector; + static const int32 INVALID_LOCATION = -1; class Slot { @@ -121,13 +142,12 @@ public: ~Shader(); + ID getID() const { return _ID; } + Type getType() const { return _type; } bool isProgram() const { return getType() > NUM_DOMAINS; } bool isDomain() const { return getType() < NUM_DOMAINS; } - void setCompilationHasFailed(bool compilationHasFailed) { _compilationHasFailed = compilationHasFailed; } - bool compilationHasFailed() const { return _compilationHasFailed; } - const Source& getSource() const { return _source; } const Shaders& getShaders() const { return _shaders; } @@ -155,6 +175,15 @@ public: const SlotSet& inputs, const SlotSet& outputs); + // Compilation Handler can be passed while compiling a shader (in the makeProgram call) to be able to give the hand to + // the caller thread if the comilation fails and to prvide a different version of the source for it + // @param0 the Shader object that just failed to compile + // @param1 the original source code as submited to the compiler + // @param2 the compilation log containing the error message + // @param3 a new string ready to be filled with the new version of the source that could be proposed from the handler functor + // @return boolean true if the backend should keep trying to compile the shader with the new source returned or false to stop and fail that shader compilation + using CompilationHandler = std::function; + // makeProgram(...) make a program shader ready to be used in a Batch. // It compiles the sub shaders, link them and defines the Slots and their bindings. // If the shader passed is not a program, nothing happens. @@ -168,18 +197,29 @@ public: // on a gl Context and the driver to compile the glsl shader. // Hoppefully in a few years the shader compilation will be completely abstracted in a separate shader compiler library // independant of the graphics api in use underneath (looking at you opengl & vulkan). - static bool makeProgram(Shader& shader, const Shader::BindingSet& bindings = Shader::BindingSet()); + static bool makeProgram(Shader& shader, const Shader::BindingSet& bindings = Shader::BindingSet(), const CompilationHandler& handler = nullptr); + + // Check the compilation state + bool compilationHasFailed() const { return _compilationHasFailed; } + const CompilationLogs& getCompilationLogs() const { return _compilationLogs; } + uint32_t getNumCompilationAttempts() const { return _numCompilationAttempts; } + + // Set COmpilation logs can only be called by the Backend layers + void setCompilationHasFailed(bool compilationHasFailed) { _compilationHasFailed = compilationHasFailed; } + void setCompilationLogs(const CompilationLogs& logs) const; + void incrementCompilationAttempt() const; + const GPUObjectPointer gpuObject {}; - + protected: Shader(Type type, const Source& source); - Shader(Type type, const Pointer& vertex, const Pointer& pixel); Shader(Type type, const Pointer& vertex, const Pointer& geometry, const Pointer& pixel); Shader(const Shader& shader); // deep copy of the sysmem shader Shader& operator=(const Shader& shader); // deep copy of the sysmem texture + // Source contains the actual source code or nothing if the shader is a program Source _source; @@ -198,8 +238,49 @@ protected: // The type of the shader, the master key Type _type; + // The unique identifier of a shader in the GPU lib + uint32_t _ID{ 0 }; + + // Number of attempts to compile the shader + mutable uint32_t _numCompilationAttempts{ 0 }; + // Compilation logs (one for each versions generated) + mutable CompilationLogs _compilationLogs; + // Whether or not the shader compilation failed bool _compilationHasFailed { false }; + + + // Global maps of the shaders + // Unique shader ID + static std::atomic _nextShaderID; + + using ShaderMap = std::map, Source::Less>; + using DomainShaderMaps = std::array; + static DomainShaderMaps _domainShaderMaps; + + static ShaderPointer createOrReuseDomainShader(Type type, const Source& source); + + using ProgramMapKey = glm::uvec3; // The IDs of the shaders in a program make its key + class ProgramKeyLess { + public: + bool operator() (const ProgramMapKey& l, const ProgramMapKey& r) const { + if (l.x == r.x) { + if (l.y == r.y) { + return (l.z < r.z); + } + else { + return (l.y < r.y); + } + } + else { + return (l.x < r.x); + } + } + }; + using ProgramMap = std::map, ProgramKeyLess>; + static ProgramMap _programMap; + + static ShaderPointer createOrReuseProgramShader(Type type, const Pointer& vertexShader, const Pointer& geometryShader, const Pointer& pixelShader); }; typedef Shader::Pointer ShaderPointer; diff --git a/libraries/gpu/src/gpu/null/NullBackend.h b/libraries/gpu/src/gpu/null/NullBackend.h index c9d249aec7..57b8fbafbc 100644 --- a/libraries/gpu/src/gpu/null/NullBackend.h +++ b/libraries/gpu/src/gpu/null/NullBackend.h @@ -28,7 +28,7 @@ class Backend : public gpu::Backend { friend class gpu::Context; static void init() {} static gpu::Backend* createBackend() { return new Backend(); } - static bool makeProgram(Shader& shader, const Shader::BindingSet& slotBindings) { return true; } + static bool makeProgram(Shader& shader, const Shader::BindingSet& slotBindings, const Shader::CompilationHandler& handler) { return true; } protected: explicit Backend(bool syncCache) : Parent() { } diff --git a/libraries/render-utils/src/LightAmbient.slh b/libraries/render-utils/src/LightAmbient.slh index acd30d527d..89d3f4faee 100644 --- a/libraries/render-utils/src/LightAmbient.slh +++ b/libraries/render-utils/src/LightAmbient.slh @@ -46,7 +46,7 @@ vec3 evalAmbientSpecularIrradiance(LightAmbient ambient, SurfaceData surface) { float levels = getLightAmbientMapNumMips(ambient); float m = 12.0 / (1.0+11.0*surface.roughness); float lod = levels - m; - lod = max(lod, 0); + lod = max(lod, 0.0); specularLight = evalSkyboxLight(lightDir, lod).xyz; } <@endif@> diff --git a/libraries/render/src/render/ShapePipeline.cpp b/libraries/render/src/render/ShapePipeline.cpp index 6d9d8230e4..f6ed00b493 100644 --- a/libraries/render/src/render/ShapePipeline.cpp +++ b/libraries/render/src/render/ShapePipeline.cpp @@ -73,8 +73,9 @@ void ShapePlumber::addPipeline(const Filter& filter, const gpu::ShaderPointer& p ShapeKey key{ filter._flags }; + // don't call makeProgram on shaders that are already made. - if (program->getUniformBuffers().empty()) { + if (program->getNumCompilationAttempts() < 1) { gpu::Shader::BindingSet slotBindings; slotBindings.insert(gpu::Shader::Binding(std::string("lightingModelBuffer"), Slot::BUFFER::LIGHTING_MODEL)); slotBindings.insert(gpu::Shader::Binding(std::string("skinClusterBuffer"), Slot::BUFFER::SKINNING)); diff --git a/plugins/openvr/src/OpenVrDisplayPlugin.cpp b/plugins/openvr/src/OpenVrDisplayPlugin.cpp index 4403eaeff7..4b5f0e6517 100644 --- a/plugins/openvr/src/OpenVrDisplayPlugin.cpp +++ b/plugins/openvr/src/OpenVrDisplayPlugin.cpp @@ -200,9 +200,10 @@ public: std::string fsSource = HMD_REPROJECTION_FRAG; GLuint vertexShader { 0 }, fragmentShader { 0 }; std::string error; + std::vector binary; ::gl::compileShader(GL_VERTEX_SHADER, vsSource, "", vertexShader, error); ::gl::compileShader(GL_FRAGMENT_SHADER, fsSource, "", fragmentShader, error); - _program = ::gl::compileProgram({ { vertexShader, fragmentShader } }, error); + _program = ::gl::compileProgram({ { vertexShader, fragmentShader } }, error, binary); glDeleteShader(vertexShader); glDeleteShader(fragmentShader); qDebug() << "Rebuild proigram"; diff --git a/tests/shaders/src/main.cpp b/tests/shaders/src/main.cpp index cd307ba362..3f48e37a76 100644 --- a/tests/shaders/src/main.cpp +++ b/tests/shaders/src/main.cpp @@ -137,12 +137,13 @@ const std::string PIXEL_SHADER_DEFINES{ R"GLSL( void testShaderBuild(const char* vs_src, const char * fs_src) { std::string error; + std::vector binary; GLuint vs, fs; if (!gl::compileShader(GL_VERTEX_SHADER, vs_src, VERTEX_SHADER_DEFINES, vs, error) || !gl::compileShader(GL_FRAGMENT_SHADER, fs_src, PIXEL_SHADER_DEFINES, fs, error)) { throw std::runtime_error("Failed to compile shader"); } - auto pr = gl::compileProgram({ vs, fs }, error); + auto pr = gl::compileProgram({ vs, fs }, error, binary); if (!pr) { throw std::runtime_error("Failed to link shader"); }