From 2dcedf9f3901dac39067d6bf6aedad9b4ead28df Mon Sep 17 00:00:00 2001 From: samcake Date: Tue, 30 Jan 2018 12:52:05 -0800 Subject: [PATCH] cherry picking Tony's fix for shader compilation time not taking soo long and adding better feedback from shader compilation --- interface/src/Application.cpp | 23 ++- libraries/gl/src/gl/GLShaders.cpp | 161 +++++++++++------- libraries/gl/src/gl/GLShaders.h | 6 +- libraries/gpu-gl/src/gpu/gl/GLBackend.h | 4 +- .../gpu-gl/src/gpu/gl/GLBackendShader.cpp | 44 +++-- libraries/gpu-gl/src/gpu/gl/GLShader.cpp | 8 +- libraries/gpu-gl/src/gpu/gl/GLShader.h | 4 +- libraries/gpu-gles/src/gpu/gl/GLBackend.h | 4 +- .../gpu-gles/src/gpu/gl/GLBackendShader.cpp | 65 ++++++- libraries/gpu-gles/src/gpu/gl/GLShader.cpp | 8 +- libraries/gpu-gles/src/gpu/gl/GLShader.h | 4 +- libraries/gpu/src/gpu/Context.cpp | 4 +- libraries/gpu/src/gpu/Context.h | 4 +- libraries/gpu/src/gpu/Shader.cpp | 11 +- libraries/gpu/src/gpu/Shader.h | 29 +++- libraries/render/src/render/ShapePipeline.cpp | 52 +++--- 16 files changed, 297 insertions(+), 134 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 64e9feea02..f1d0378792 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -396,6 +396,7 @@ public: setObjectName("Deadlock Watchdog"); // Give the heartbeat an initial value _heartbeat = usecTimestampNow(); + _paused = false; connect(qApp, &QCoreApplication::aboutToQuit, [this] { _quit = true; }); @@ -413,11 +414,20 @@ public: *crashTrigger = 0xDEAD10CC; } + static void pause() { + _paused = true; + } + + static void resume() { + _paused = false; + updateHeartbeat(); + } + void run() override { while (!_quit) { QThread::sleep(HEARTBEAT_UPDATE_INTERVAL_SECS); // Don't do heartbeat detection under nsight - if (nsightActive()) { + if (nsightActive() || _paused) { continue; } uint64_t lastHeartbeat = _heartbeat; // sample atomic _heartbeat, because we could context switch away and have it updated on us @@ -473,6 +483,7 @@ public: } } + static std::atomic _paused; static std::atomic _heartbeat; static std::atomic _maxElapsed; static std::atomic _maxElapsedAverage; @@ -481,6 +492,7 @@ public: bool _quit { false }; }; +std::atomic DeadlockWatchdogThread::_paused; std::atomic DeadlockWatchdogThread::_heartbeat; std::atomic DeadlockWatchdogThread::_maxElapsed; std::atomic DeadlockWatchdogThread::_maxElapsedAverage; @@ -2269,6 +2281,11 @@ void Application::initializeGL() { initDisplay(); qCDebug(interfaceapp, "Initialized Display."); +#ifdef Q_OS_OSX + // FIXME: on mac os the shaders take up to 1 minute to compile, so we pause the deadlock watchdog thread. + +DeadlockWatchdogThread::pause(); +#endif + // Set up the render engine render::CullFunctor cullFunctor = LODManager::shouldRender; static const QString RENDER_FORWARD = "HIFI_RENDER_FORWARD"; @@ -2283,6 +2300,10 @@ void Application::initializeGL() { // Now that OpenGL is initialized, we are sure we have a valid context and can create the various pipeline shaders with success. DependencyManager::get()->initializeShapePipelines(); +#ifdef Q_OS_OSX + DeadlockWatchdogThread::resume(); +#endif + _offscreenContext = new OffscreenGLCanvas(); _offscreenContext->setObjectName("MainThreadContext"); _offscreenContext->create(_glWidget->qglContext()); diff --git a/libraries/gl/src/gl/GLShaders.cpp b/libraries/gl/src/gl/GLShaders.cpp index 017c92b71c..ecd6fe3323 100644 --- a/libraries/gl/src/gl/GLShaders.cpp +++ b/libraries/gl/src/gl/GLShaders.cpp @@ -6,9 +6,9 @@ namespace gl { #ifdef SEPARATE_PROGRAM - bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, GLuint &programObject, std::string& error) { + bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, GLuint &programObject, std::string& message) { #else - bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, std::string& error) { + bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, std::string& message) { #endif if (shaderSource.empty()) { qCDebug(glLogging) << "GLShader::compileShader - no GLSL shader source code ? so failed to create"; @@ -34,52 +34,57 @@ namespace gl { GLint compiled = 0; glGetShaderiv(glshader, GL_COMPILE_STATUS, &compiled); - // if compilation fails - if (!compiled) { - - // save the source code to a temp file so we can debug easily - /* - std::ofstream filestream; - filestream.open("debugshader.glsl"); - if (filestream.is_open()) { - filestream << srcstr[0]; - filestream << srcstr[1]; - filestream.close(); - } - */ - - GLint infoLength = 0; - glGetShaderiv(glshader, GL_INFO_LOG_LENGTH, &infoLength); + GLint infoLength = 0; + glGetShaderiv(glshader, GL_INFO_LOG_LENGTH, &infoLength); + if ((infoLength > 0) || !compiled) { char* temp = new char[infoLength]; glGetShaderInfoLog(glshader, infoLength, NULL, temp); + message = std::string(temp); - /* - filestream.open("debugshader.glsl.info.txt"); - if (filestream.is_open()) { - filestream << std::string(temp); - filestream.close(); - } - */ - - qCCritical(glLogging) << "GLShader::compileShader - failed to compile the gl shader object:"; - int lineNumber = 0; - for (auto s : srcstr) { - QString str(s); - QStringList lines = str.split("\n"); - for (auto& line : lines) { - qCCritical(glLogging).noquote() << QString("%1: %2").arg(lineNumber++, 5, 10, QChar('0')).arg(line); + // if compilation fails + if (!compiled) { + // save the source code to a temp file so we can debug easily + /* + std::ofstream filestream; + filestream.open("debugshader.glsl"); + if (filestream.is_open()) { + filestream << srcstr[0]; + filestream << srcstr[1]; + filestream.close(); } + */ + + /* + filestream.open("debugshader.glsl.info.txt"); + if (filestream.is_open()) { + filestream << std::string(temp); + filestream.close(); + } + */ + + qCCritical(glLogging) << "GLShader::compileShader - failed to compile the gl shader object:"; + int lineNumber = 0; + for (auto s : srcstr) { + QString str(s); + QStringList lines = str.split("\n"); + for (auto& line : lines) { + qCCritical(glLogging).noquote() << QString("%1: %2").arg(lineNumber++, 5, 10, QChar('0')).arg(line); + } + } + qCCritical(glLogging) << "GLShader::compileShader - errors:"; + qCCritical(glLogging) << temp; + + delete[] temp; + glDeleteShader(glshader); + return false; } - qCCritical(glLogging) << "GLShader::compileShader - errors:"; - qCCritical(glLogging) << temp; - error = std::string(temp); + // Compilation success + qCWarning(glLogging) << "GLShader::compileShader - Success:"; + qCWarning(glLogging) << temp; delete[] temp; - - glDeleteShader(glshader); - return false; } #ifdef SEPARATE_PROGRAM @@ -137,7 +142,7 @@ namespace gl { return true; } -GLuint compileProgram(const std::vector& glshaders, std::string& error) { +GLuint compileProgram(const std::vector& glshaders, std::string& message, std::vector& binary) { // A brand new program: GLuint glprogram = glCreateProgram(); if (!glprogram) { @@ -157,39 +162,65 @@ GLuint compileProgram(const std::vector& glshaders, std::string& error) GLint linked = 0; glGetProgramiv(glprogram, GL_LINK_STATUS, &linked); - if (!linked) { - /* - // save the source code to a temp file so we can debug easily - std::ofstream filestream; - filestream.open("debugshader.glsl"); - if (filestream.is_open()) { - filestream << shaderSource->source; - filestream.close(); - } - */ - - GLint infoLength = 0; - glGetProgramiv(glprogram, GL_INFO_LOG_LENGTH, &infoLength); + GLint infoLength = 0; + glGetProgramiv(glprogram, GL_INFO_LOG_LENGTH, &infoLength); + if ((infoLength > 0) || !linked) { char* temp = new char[infoLength]; glGetProgramInfoLog(glprogram, infoLength, NULL, temp); - qCDebug(glLogging) << "GLShader::compileProgram - failed to LINK the gl program object :"; - qCDebug(glLogging) << temp; + message = std::string(temp); - error = std::string(temp); - delete[] temp; + if (!linked) { + /* + // save the source code to a temp file so we can debug easily + std::ofstream filestream; + filestream.open("debugshader.glsl"); + if (filestream.is_open()) { + filestream << shaderSource->source; + filestream.close(); + } + */ - /* - filestream.open("debugshader.glsl.info.txt"); - if (filestream.is_open()) { - filestream << std::string(temp); - filestream.close(); + qCDebug(glLogging) << "GLShader::compileProgram - failed to LINK the gl program object :"; + qCDebug(glLogging) << temp; + + delete[] temp; + + /* + filestream.open("debugshader.glsl.info.txt"); + if (filestream.is_open()) { + filestream << std::string(temp); + filestream.close(); + } + */ + + glDeleteProgram(glprogram); + return 0; + } else { + qCDebug(glLogging) << "GLShader::compileProgram - success:"; + qCDebug(glLogging) << temp; + delete[] temp; } - */ + } - glDeleteProgram(glprogram); - return 0; + // If linked get the binaries + if (linked) { + GLint binaryLength = 0; + glGetProgramiv(glprogram, GL_PROGRAM_BINARY_LENGTH, &binaryLength); + + if (binaryLength > 0) { + GLint numBinFormats = 0; + glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, &numBinFormats); + if (numBinFormats > 0) { + binary.resize(binaryLength); + std::vector binFormats(numBinFormats); + glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, binFormats.data()); + + GLenum programBinFormat; + glGetProgramBinary(glprogram, binaryLength, NULL, &programBinFormat, binary.data()); + } + } } return glprogram; diff --git a/libraries/gl/src/gl/GLShaders.h b/libraries/gl/src/gl/GLShaders.h index a6213fd280..c5262b6b61 100644 --- a/libraries/gl/src/gl/GLShaders.h +++ b/libraries/gl/src/gl/GLShaders.h @@ -17,12 +17,12 @@ namespace gl { #ifdef SEPARATE_PROGRAM - bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, GLuint &programObject, std::string& error); + bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, GLuint &programObject, std::string& message); #else - bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, std::string& error); + bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, std::string& message); #endif - GLuint compileProgram(const std::vector& glshaders, std::string& error); + GLuint compileProgram(const std::vector& glshaders, std::string& message, , std::vector& binary); } diff --git a/libraries/gpu-gl/src/gpu/gl/GLBackend.h b/libraries/gpu-gl/src/gpu/gl/GLBackend.h index 5558d3ada1..004bfe1c85 100644 --- a/libraries/gpu-gl/src/gpu/gl/GLBackend.h +++ b/libraries/gpu-gl/src/gpu/gl/GLBackend.h @@ -64,7 +64,7 @@ protected: explicit GLBackend(bool syncCache); GLBackend(); public: - static bool makeProgram(Shader& shader, const Shader::BindingSet& slotBindings = Shader::BindingSet()); + static bool makeProgram(Shader& shader, const Shader::BindingSet& slotBindings = Shader::BindingSet(), Shader::CompilationHandler handler = nullptr); virtual ~GLBackend(); @@ -424,7 +424,7 @@ protected: // Backend dependant compilation of the shader virtual GLShader* compileBackendProgram(const Shader& program); - virtual GLShader* compileBackendShader(const Shader& shader); + virtual GLShader* compileBackendShader(const Shader& shader, Shader::CompilationHandler handler); virtual std::string getBackendShaderHeader() const; virtual void makeProgramBindings(ShaderObject& shaderObject); class ElementResource { diff --git a/libraries/gpu-gl/src/gpu/gl/GLBackendShader.cpp b/libraries/gpu-gl/src/gpu/gl/GLBackendShader.cpp index 9adfd550ef..84170dbffd 100644 --- a/libraries/gpu-gl/src/gpu/gl/GLBackendShader.cpp +++ b/libraries/gpu-gl/src/gpu/gl/GLBackendShader.cpp @@ -56,28 +56,43 @@ static const std::array VERSION_DEFINES { { stereoVersion } }; -GLShader* GLBackend::compileBackendShader(const Shader& shader) { +GLShader* GLBackend::compileBackendShader(const Shader& shader, Shader::CompilationHandler handler) { // Any GLSLprogram ? normally yes... const std::string& shaderSource = shader.getSource().getCode(); GLenum shaderDomain = SHADER_DOMAINS[shader.getType()]; GLShader::ShaderObjects shaderObjects; + Shader::CompilationLogs compilationLogs(GLShader::NumVersions); for (int version = 0; version < GLShader::NumVersions; version++) { auto& shaderObject = shaderObjects[version]; std::string shaderDefines = getBackendShaderHeader() + "\n" + DOMAIN_DEFINES[shader.getType()] + "\n" + VERSION_DEFINES[version]; - std::string error; + if (handler) { + bool retest = true; + std::string currentSrc = shaderSource; + while (retest) { + bool result = ::gl::compileShader(shaderDomain, currentSrc, shaderDefines, shaderObject.glshader, compilationLogs[version].message); + compilationLogs[version].compiled = result; + if (!result) { + std::string newSrc; + retest = handler(shader, currentSrc, compilationLogs[version], newSrc); + currentSrc = newSrc; + } else { + retest = false; + } + } + } else { + compilationLogs[version].compiled = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, compilationLogs[version].message); + } -#ifdef SEPARATE_PROGRAM - bool result = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, shaderObject.glprogram, error); -#else - bool result = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, error); -#endif - if (!result) { - qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Shader didn't compile:\n" << error.c_str(); + if (!compilationLogs[version].compiled) { + qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Shader didn't compile:\n" << compilationLogs[version].message.c_str(); + shader.setCompilationLogs(compilationLogs); return nullptr; } } + // Compilation feedback + shader.setCompilationLogs(compilationLogs); // So far so good, the shader is created successfully GLShader* object = new GLShader(this->shared_from_this()); @@ -93,6 +108,8 @@ GLShader* GLBackend::compileBackendProgram(const Shader& program) { GLShader::ShaderObjects programObjects; + Shader::CompilationLogs compilationLogs(GLShader::NumVersions); + for (int version = 0; version < GLShader::NumVersions; version++) { auto& programObject = programObjects[version]; @@ -104,14 +121,15 @@ GLShader* GLBackend::compileBackendProgram(const Shader& program) { shaderGLObjects.push_back(object->_shaderObjects[version].glshader); } else { qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - One of the shaders of the program is not compiled?"; + program.setCompilationLogs(compilationLogs); return nullptr; } } - std::string error; - GLuint glprogram = ::gl::compileProgram(shaderGLObjects, error); + GLuint glprogram = ::gl::compileProgram(shaderGLObjects, compilationLogs[version].message, compilationLogs[version].binary); if (glprogram == 0) { - qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Program didn't link:\n" << error.c_str(); + qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Program didn't link:\n" << compilationLogs[version].message.c_str(); + program.setCompilationLogs(compilationLogs); return nullptr; } @@ -119,6 +137,8 @@ GLShader* GLBackend::compileBackendProgram(const Shader& program) { makeProgramBindings(programObject); } + // Compilation feedback + program.setCompilationLogs(compilationLogs); // So far so good, the program versions have all been created successfully GLShader* object = new GLShader(this->shared_from_this()); diff --git a/libraries/gpu-gl/src/gpu/gl/GLShader.cpp b/libraries/gpu-gl/src/gpu/gl/GLShader.cpp index 7ed9121978..a626376e27 100644 --- a/libraries/gpu-gl/src/gpu/gl/GLShader.cpp +++ b/libraries/gpu-gl/src/gpu/gl/GLShader.cpp @@ -30,7 +30,7 @@ GLShader::~GLShader() { } } -GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) { +GLShader* GLShader::sync(GLBackend& backend, const Shader& shader, Shader::CompilationHandler handler) { GLShader* object = Backend::getGPUObject(shader); // If GPU object already created then good @@ -45,7 +45,7 @@ GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) { Backend::setGPUObject(shader, object); } } else if (shader.isDomain()) { - GLShader* tempObject = backend.compileBackendShader(shader); + GLShader* tempObject = backend.compileBackendShader(shader, handler); if (tempObject) { object = tempObject; Backend::setGPUObject(shader, object); @@ -56,10 +56,10 @@ GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) { return object; } -bool GLShader::makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings) { +bool GLShader::makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings, Shader::CompilationHandler handler) { // First make sure the Shader has been compiled - GLShader* object = sync(backend, shader); + GLShader* object = sync(backend, shader, handler); if (!object) { return false; } diff --git a/libraries/gpu-gl/src/gpu/gl/GLShader.h b/libraries/gpu-gl/src/gpu/gl/GLShader.h index dcf2dc330d..42d63f8dfb 100644 --- a/libraries/gpu-gl/src/gpu/gl/GLShader.h +++ b/libraries/gpu-gl/src/gpu/gl/GLShader.h @@ -21,8 +21,8 @@ struct ShaderObject { class GLShader : public GPUObject { public: - static GLShader* sync(GLBackend& backend, const Shader& shader); - static bool makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings); + static GLShader* sync(GLBackend& backend, const Shader& shader, Shader::CompilationHandler handler = nullptr); + static bool makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings, Shader::CompilationHandler handler = nullptr); enum Version { Mono = 0, diff --git a/libraries/gpu-gles/src/gpu/gl/GLBackend.h b/libraries/gpu-gles/src/gpu/gl/GLBackend.h index ea06b6b672..bb8e15bad3 100644 --- a/libraries/gpu-gles/src/gpu/gl/GLBackend.h +++ b/libraries/gpu-gles/src/gpu/gl/GLBackend.h @@ -61,7 +61,7 @@ protected: explicit GLBackend(bool syncCache); GLBackend(); public: - static bool makeProgram(Shader& shader, const Shader::BindingSet& slotBindings = Shader::BindingSet()); + static bool makeProgram(Shader& shader, const Shader::BindingSet& slotBindings = Shader::BindingSet(), Shader::CompilationHandler handler = nullptr); virtual ~GLBackend(); @@ -421,7 +421,7 @@ protected: // Backend dependant compilation of the shader virtual GLShader* compileBackendProgram(const Shader& program); - virtual GLShader* compileBackendShader(const Shader& shader); + virtual GLShader* compileBackendShader(const Shader& shader, Shader::CompilationHandler handler); virtual std::string getBackendShaderHeader() const; virtual void makeProgramBindings(ShaderObject& shaderObject); class ElementResource { diff --git a/libraries/gpu-gles/src/gpu/gl/GLBackendShader.cpp b/libraries/gpu-gles/src/gpu/gl/GLBackendShader.cpp index fd44ad462f..26dbb12cf7 100644 --- a/libraries/gpu-gles/src/gpu/gl/GLBackendShader.cpp +++ b/libraries/gpu-gles/src/gpu/gl/GLBackendShader.cpp @@ -56,11 +56,12 @@ static const std::array VERSION_DEFINES { { stereoVersion } }; -GLShader* GLBackend::compileBackendShader(const Shader& shader) { +GLShader* GLBackend::compileBackendShader(const Shader& shader, Shader::CompilationHandler handler) { // Any GLSLprogram ? normally yes... const std::string& shaderSource = shader.getSource().getCode(); GLenum shaderDomain = SHADER_DOMAINS[shader.getType()]; GLShader::ShaderObjects shaderObjects; + Shader::CompilationLogs compilationLogs(GLShader::NumVersions); for (int version = 0; version < GLShader::NumVersions; version++) { auto& shaderObject = shaderObjects[version]; @@ -90,6 +91,55 @@ GLShader* GLBackend::compileBackendShader(const Shader& shader) { return object; } +GLShader* GLBackend::compileBackendShader(const Shader& shader, Shader::CompilationHandler handler) { + // Any GLSLprogram ? normally yes... + const std::string& shaderSource = shader.getSource().getCode(); + GLenum shaderDomain = SHADER_DOMAINS[shader.getType()]; + GLShader::ShaderObjects shaderObjects; + Shader::CompilationLogs compilationLogs(GLShader::NumVersions); + + for (int version = 0; version < GLShader::NumVersions; version++) { + auto& shaderObject = shaderObjects[version]; + + std::string shaderDefines = getBackendShaderHeader() + "\n" + DOMAIN_DEFINES[shader.getType()] + "\n" + VERSION_DEFINES[version] + + "\n#extension GL_EXT_texture_buffer : enable" + + "\nprecision lowp float; // check precision 2" + + "\nprecision lowp samplerBuffer;" + + "\nprecision lowp sampler2DShadow;"; + if (handler) { + bool retest = true; + std::string currentSrc = shaderSource; + while (retest) { + bool result = ::gl::compileShader(shaderDomain, currentSrc, shaderDefines, shaderObject.glshader, compilationLogs[version].message); + compilationLogs[version].compiled = result; + if (!result) { + std::string newSrc; + retest = handler(shader, currentSrc, compilationLogs[version], newSrc); + currentSrc = newSrc; + } else { + retest = false; + } + } + } else { + compilationLogs[version].compiled = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, compilationLogs[version].message); + } + + if (!compilationLogs[version].compiled) { + qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Shader didn't compile:\n" << compilationLogs[version].message.c_str(); + shader.setCompilationLogs(compilationLogs); + return nullptr; + } + } + // Compilation feedback + shader.setCompilationLogs(compilationLogs); + + // So far so good, the shader is created successfully + GLShader* object = new GLShader(this->shared_from_this()); + object->_shaderObjects = shaderObjects; + + return object; +} + GLShader* GLBackend::compileBackendProgram(const Shader& program) { if (!program.isProgram()) { return nullptr; @@ -97,25 +147,28 @@ GLShader* GLBackend::compileBackendProgram(const Shader& program) { GLShader::ShaderObjects programObjects; + Shader::CompilationLogs compilationLogs(GLShader::NumVersions); + for (int version = 0; version < GLShader::NumVersions; version++) { auto& programObject = programObjects[version]; // Let's go through every shaders and make sure they are ready to go - std::vector shaderGLObjects; + std::vector< GLuint > shaderGLObjects; for (auto subShader : program.getShaders()) { auto object = GLShader::sync((*this), *subShader); if (object) { shaderGLObjects.push_back(object->_shaderObjects[version].glshader); } else { qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - One of the shaders of the program is not compiled?"; + program.setCompilationLogs(compilationLogs); return nullptr; } } - std::string error; - GLuint glprogram = ::gl::compileProgram(shaderGLObjects, error); + GLuint glprogram = ::gl::compileProgram(shaderGLObjects, compilationLogs[version].message, compilationLogs[version].binary); if (glprogram == 0) { - qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Program didn't link:\n" << error.c_str(); + qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Program didn't link:\n" << compilationLogs[version].message.c_str(); + program.setCompilationLogs(compilationLogs); return nullptr; } @@ -123,6 +176,8 @@ GLShader* GLBackend::compileBackendProgram(const Shader& program) { makeProgramBindings(programObject); } + // Compilation feedback + program.setCompilationLogs(compilationLogs); // So far so good, the program versions have all been created successfully GLShader* object = new GLShader(this->shared_from_this()); diff --git a/libraries/gpu-gles/src/gpu/gl/GLShader.cpp b/libraries/gpu-gles/src/gpu/gl/GLShader.cpp index 7ed9121978..a626376e27 100644 --- a/libraries/gpu-gles/src/gpu/gl/GLShader.cpp +++ b/libraries/gpu-gles/src/gpu/gl/GLShader.cpp @@ -30,7 +30,7 @@ GLShader::~GLShader() { } } -GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) { +GLShader* GLShader::sync(GLBackend& backend, const Shader& shader, Shader::CompilationHandler handler) { GLShader* object = Backend::getGPUObject(shader); // If GPU object already created then good @@ -45,7 +45,7 @@ GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) { Backend::setGPUObject(shader, object); } } else if (shader.isDomain()) { - GLShader* tempObject = backend.compileBackendShader(shader); + GLShader* tempObject = backend.compileBackendShader(shader, handler); if (tempObject) { object = tempObject; Backend::setGPUObject(shader, object); @@ -56,10 +56,10 @@ GLShader* GLShader::sync(GLBackend& backend, const Shader& shader) { return object; } -bool GLShader::makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings) { +bool GLShader::makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings, Shader::CompilationHandler handler) { // First make sure the Shader has been compiled - GLShader* object = sync(backend, shader); + GLShader* object = sync(backend, shader, handler); if (!object) { return false; } diff --git a/libraries/gpu-gles/src/gpu/gl/GLShader.h b/libraries/gpu-gles/src/gpu/gl/GLShader.h index dcf2dc330d..42d63f8dfb 100644 --- a/libraries/gpu-gles/src/gpu/gl/GLShader.h +++ b/libraries/gpu-gles/src/gpu/gl/GLShader.h @@ -21,8 +21,8 @@ struct ShaderObject { class GLShader : public GPUObject { public: - static GLShader* sync(GLBackend& backend, const Shader& shader); - static bool makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings); + static GLShader* sync(GLBackend& backend, const Shader& shader, Shader::CompilationHandler handler = nullptr); + static bool makeProgram(GLBackend& backend, Shader& shader, const Shader::BindingSet& slotBindings, Shader::CompilationHandler handler = nullptr); enum Version { Mono = 0, diff --git a/libraries/gpu/src/gpu/Context.cpp b/libraries/gpu/src/gpu/Context.cpp index 24128524da..2399d3ddc3 100644 --- a/libraries/gpu/src/gpu/Context.cpp +++ b/libraries/gpu/src/gpu/Context.cpp @@ -127,7 +127,7 @@ void Context::executeFrame(const FramePointer& frame) const { _frameStats.evalDelta(beginStats, endStats); } -bool Context::makeProgram(Shader& shader, const Shader::BindingSet& bindings) { +bool Context::makeProgram(Shader& shader, const Shader::BindingSet& bindings, Shader::CompilationHandler handler) { // If we're running in another DLL context, we need to fetch the program callback out of the application // FIXME find a way to do this without reliance on Qt app properties if (!_makeProgramCallback) { @@ -135,7 +135,7 @@ bool Context::makeProgram(Shader& shader, const Shader::BindingSet& bindings) { _makeProgramCallback = reinterpret_cast(rawCallback); } if (shader.isProgram() && _makeProgramCallback) { - return _makeProgramCallback(shader, bindings); + return _makeProgramCallback(shader, bindings, handler); } return false; } diff --git a/libraries/gpu/src/gpu/Context.h b/libraries/gpu/src/gpu/Context.h index 7b7575e9ed..7d04463609 100644 --- a/libraries/gpu/src/gpu/Context.h +++ b/libraries/gpu/src/gpu/Context.h @@ -143,7 +143,7 @@ class Context { public: using Size = Resource::Size; typedef BackendPointer (*CreateBackend)(); - typedef bool (*MakeProgram)(Shader& shader, const Shader::BindingSet& bindings); + typedef bool (*MakeProgram)(Shader& shader, const Shader::BindingSet& bindings, Shader::CompilationHandler handler); // This one call must happen before any context is created or used (Shader::MakeProgram) in order to setup the Backend and any singleton data needed @@ -262,7 +262,7 @@ protected: // makeProgramShader(...) make a program shader ready to be used in a Batch. // It compiles the sub shaders, link them and defines the Slots and their bindings. // If the shader passed is not a program, nothing happens. - static bool makeProgram(Shader& shader, const Shader::BindingSet& bindings); + static bool makeProgram(Shader& shader, const Shader::BindingSet& bindings, Shader::CompilationHandler handler = nullptr); static CreateBackend _createBackendCallback; static MakeProgram _makeProgramCallback; diff --git a/libraries/gpu/src/gpu/Shader.cpp b/libraries/gpu/src/gpu/Shader.cpp index 398a269f3f..35c9ad8ae2 100755 --- a/libraries/gpu/src/gpu/Shader.cpp +++ b/libraries/gpu/src/gpu/Shader.cpp @@ -82,9 +82,16 @@ void Shader::defineSlots(const SlotSet& uniforms, const SlotSet& uniformBuffers, _outputs = outputs; } -bool Shader::makeProgram(Shader& shader, const Shader::BindingSet& bindings) { +bool Shader::makeProgram(Shader& shader, const Shader::BindingSet& bindings, CompilationHandler handler) { if (shader.isProgram()) { - return Context::makeProgram(shader, bindings); + return Context::makeProgram(shader, bindings, handler); } return false; } + +void Shader::setCompilationLogs(const CompilationLogs& logs) const { + _compilationLogs.clear(); + for (const auto& log : logs) { + _compilationLogs.emplace_back(CompilationLog(log)); + } +} diff --git a/libraries/gpu/src/gpu/Shader.h b/libraries/gpu/src/gpu/Shader.h index 181c9b5e78..33449fe656 100755 --- a/libraries/gpu/src/gpu/Shader.h +++ b/libraries/gpu/src/gpu/Shader.h @@ -44,6 +44,19 @@ public: Language _lang = GLSL; }; + struct CompilationLog { + std::string message; + std::vector binary; + bool compiled{ false }; + + CompilationLog() {} + CompilationLog(const CompilationLog& src) : + message(src.message), + binary(src.binary), + compiled(src.compiled) {} + }; + using CompilationLogs = std::vector; + static const int32 INVALID_LOCATION = -1; class Slot { @@ -155,6 +168,8 @@ public: const SlotSet& inputs, const SlotSet& outputs); + typedef bool(*CompilationHandler)(const Shader& shader, const std::string& src, CompilationLog& log, std::string& newSrc); + // makeProgram(...) make a program shader ready to be used in a Batch. // It compiles the sub shaders, link them and defines the Slots and their bindings. // If the shader passed is not a program, nothing happens. @@ -168,7 +183,16 @@ public: // on a gl Context and the driver to compile the glsl shader. // Hoppefully in a few years the shader compilation will be completely abstracted in a separate shader compiler library // independant of the graphics api in use underneath (looking at you opengl & vulkan). - static bool makeProgram(Shader& shader, const Shader::BindingSet& bindings = Shader::BindingSet()); + static bool makeProgram(Shader& shader, const Shader::BindingSet& bindings = Shader::BindingSet(), CompilationHandler handler = nullptr); + + // Check the compilation state + bool compilationHasFailed() const { return _compilationHasFailed; } + const CompilationLogs& getCompilationLogs() const { return _compilationLogs; } + + // Set COmpilation logs can only be called by the Backend layers + void setCompilationHasFailed(bool compilationHasFailed) { _compilationHasFailed = compilationHasFailed; } + void setCompilationLogs(const CompilationLogs& logs) const; + const GPUObjectPointer gpuObject {}; @@ -198,6 +222,9 @@ protected: // The type of the shader, the master key Type _type; + // Compilation logs (one for each versions generated) + mutable CompilationLogs _compilationLogs; + // Whether or not the shader compilation failed bool _compilationHasFailed { false }; }; diff --git a/libraries/render/src/render/ShapePipeline.cpp b/libraries/render/src/render/ShapePipeline.cpp index fb58c21dde..597a6a0fab 100644 --- a/libraries/render/src/render/ShapePipeline.cpp +++ b/libraries/render/src/render/ShapePipeline.cpp @@ -72,34 +72,36 @@ void ShapePlumber::addPipeline(const Filter& filter, const gpu::ShaderPointer& p BatchSetter batchSetter, ItemSetter itemSetter) { ShapeKey key{ filter._flags }; - gpu::Shader::BindingSet slotBindings; - slotBindings.insert(gpu::Shader::Binding(std::string("lightingModelBuffer"), Slot::BUFFER::LIGHTING_MODEL)); - slotBindings.insert(gpu::Shader::Binding(std::string("skinClusterBuffer"), Slot::BUFFER::SKINNING)); - slotBindings.insert(gpu::Shader::Binding(std::string("materialBuffer"), Slot::BUFFER::MATERIAL)); - slotBindings.insert(gpu::Shader::Binding(std::string("texMapArrayBuffer"), Slot::BUFFER::TEXMAPARRAY)); - slotBindings.insert(gpu::Shader::Binding(std::string("albedoMap"), Slot::MAP::ALBEDO)); - slotBindings.insert(gpu::Shader::Binding(std::string("roughnessMap"), Slot::MAP::ROUGHNESS)); - slotBindings.insert(gpu::Shader::Binding(std::string("normalMap"), Slot::MAP::NORMAL)); - slotBindings.insert(gpu::Shader::Binding(std::string("metallicMap"), Slot::MAP::METALLIC)); - slotBindings.insert(gpu::Shader::Binding(std::string("emissiveMap"), Slot::MAP::EMISSIVE_LIGHTMAP)); - slotBindings.insert(gpu::Shader::Binding(std::string("occlusionMap"), Slot::MAP::OCCLUSION)); - slotBindings.insert(gpu::Shader::Binding(std::string("scatteringMap"), Slot::MAP::SCATTERING)); - slotBindings.insert(gpu::Shader::Binding(std::string("keyLightBuffer"), Slot::BUFFER::KEY_LIGHT)); - slotBindings.insert(gpu::Shader::Binding(std::string("lightBuffer"), Slot::BUFFER::LIGHT)); - slotBindings.insert(gpu::Shader::Binding(std::string("lightAmbientBuffer"), Slot::BUFFER::LIGHT_AMBIENT_BUFFER)); - slotBindings.insert(gpu::Shader::Binding(std::string("skyboxMap"), Slot::MAP::LIGHT_AMBIENT)); - slotBindings.insert(gpu::Shader::Binding(std::string("fadeMaskMap"), Slot::MAP::FADE_MASK)); - slotBindings.insert(gpu::Shader::Binding(std::string("fadeParametersBuffer"), Slot::BUFFER::FADE_PARAMETERS)); - slotBindings.insert(gpu::Shader::Binding(std::string("hazeBuffer"), Slot::BUFFER::HAZE_MODEL)); + if (program->getInputs().empty()) { + gpu::Shader::BindingSet slotBindings; + slotBindings.insert(gpu::Shader::Binding(std::string("lightingModelBuffer"), Slot::BUFFER::LIGHTING_MODEL)); + slotBindings.insert(gpu::Shader::Binding(std::string("skinClusterBuffer"), Slot::BUFFER::SKINNING)); + slotBindings.insert(gpu::Shader::Binding(std::string("materialBuffer"), Slot::BUFFER::MATERIAL)); + slotBindings.insert(gpu::Shader::Binding(std::string("texMapArrayBuffer"), Slot::BUFFER::TEXMAPARRAY)); + slotBindings.insert(gpu::Shader::Binding(std::string("albedoMap"), Slot::MAP::ALBEDO)); + slotBindings.insert(gpu::Shader::Binding(std::string("roughnessMap"), Slot::MAP::ROUGHNESS)); + slotBindings.insert(gpu::Shader::Binding(std::string("normalMap"), Slot::MAP::NORMAL)); + slotBindings.insert(gpu::Shader::Binding(std::string("metallicMap"), Slot::MAP::METALLIC)); + slotBindings.insert(gpu::Shader::Binding(std::string("emissiveMap"), Slot::MAP::EMISSIVE_LIGHTMAP)); + slotBindings.insert(gpu::Shader::Binding(std::string("occlusionMap"), Slot::MAP::OCCLUSION)); + slotBindings.insert(gpu::Shader::Binding(std::string("scatteringMap"), Slot::MAP::SCATTERING)); + slotBindings.insert(gpu::Shader::Binding(std::string("keyLightBuffer"), Slot::BUFFER::KEY_LIGHT)); + slotBindings.insert(gpu::Shader::Binding(std::string("lightBuffer"), Slot::BUFFER::LIGHT)); + slotBindings.insert(gpu::Shader::Binding(std::string("lightAmbientBuffer"), Slot::BUFFER::LIGHT_AMBIENT_BUFFER)); + slotBindings.insert(gpu::Shader::Binding(std::string("skyboxMap"), Slot::MAP::LIGHT_AMBIENT)); + slotBindings.insert(gpu::Shader::Binding(std::string("fadeMaskMap"), Slot::MAP::FADE_MASK)); + slotBindings.insert(gpu::Shader::Binding(std::string("fadeParametersBuffer"), Slot::BUFFER::FADE_PARAMETERS)); + slotBindings.insert(gpu::Shader::Binding(std::string("hazeBuffer"), Slot::BUFFER::HAZE_MODEL)); - if (key.isTranslucent()) { - slotBindings.insert(gpu::Shader::Binding(std::string("clusterGridBuffer"), Slot::BUFFER::LIGHT_CLUSTER_GRID_CLUSTER_GRID_SLOT)); - slotBindings.insert(gpu::Shader::Binding(std::string("clusterContentBuffer"), Slot::BUFFER::LIGHT_CLUSTER_GRID_CLUSTER_CONTENT_SLOT)); - slotBindings.insert(gpu::Shader::Binding(std::string("frustumGridBuffer"), Slot::BUFFER::LIGHT_CLUSTER_GRID_FRUSTUM_GRID_SLOT)); + if (key.isTranslucent()) { + slotBindings.insert(gpu::Shader::Binding(std::string("clusterGridBuffer"), Slot::BUFFER::LIGHT_CLUSTER_GRID_CLUSTER_GRID_SLOT)); + slotBindings.insert(gpu::Shader::Binding(std::string("clusterContentBuffer"), Slot::BUFFER::LIGHT_CLUSTER_GRID_CLUSTER_CONTENT_SLOT)); + slotBindings.insert(gpu::Shader::Binding(std::string("frustumGridBuffer"), Slot::BUFFER::LIGHT_CLUSTER_GRID_FRUSTUM_GRID_SLOT)); + } + + gpu::Shader::makeProgram(*program, slotBindings); } - gpu::Shader::makeProgram(*program, slotBindings); - auto locations = std::make_shared(); locations->albedoTextureUnit = program->getTextures().findLocation("albedoMap");