Unit testing cached and uncached shader loading

libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.cpp

@@ -46,6 +46,8 @@
 
 const char* SRGB_TO_LINEAR_FRAG = R"SCRIBE(
 
+// OpenGLDisplayPlugin_present.frag
+
 uniform sampler2D colorMap;
 
 in vec2 varTexCoord0;

libraries/gl/src/gl/GLShaders.cpp

@@ -275,7 +275,7 @@ GLuint gl::compileProgram(const std::vector<GLuint>& glshaders, std::string& mes
     return glprogram;
 }
 
-static const QString& getShaderCacheFile() {
+const QString& getShaderCacheFile() {
     static const QString SHADER_CACHE_FOLDER{ "shaders" };
     static const QString SHADER_CACHE_FILE_NAME{ "cache.json" };
     static const QString SHADER_CACHE_FILE = FileUtils::standardPath(SHADER_CACHE_FOLDER) + SHADER_CACHE_FILE_NAME;

libraries/gpu-gl-common/src/gpu/gl/GLBackendShader.cpp

@@ -154,6 +154,8 @@ GLShader* GLBackend::compileBackendShader(const Shader& shader, const Shader::Co
     return object;
 }
 
+std::atomic<size_t> gpuBinaryShadersLoaded;
+
 GLShader* GLBackend::compileBackendProgram(const Shader& program, const Shader::CompilationHandler& handler) {
     if (!program.isProgram()) {
         return nullptr;
@@ -182,6 +184,9 @@ GLShader* GLBackend::compileBackendProgram(const Shader& program, const Shader::
         // If we have a cached binary program, try to load it instead of compiling the individual shaders
         if (cachedBinary) {
             glprogram = ::gl::compileProgram({}, compilationLogs[version].message, cachedBinary);
+            if (0 != glprogram) {
+                ++gpuBinaryShadersLoaded;
+            }
         }
 
         // If we have no program, then either no cached binary, or the binary failed to load (perhaps a GPU driver update invalidated the cache)

tests/gpu/src/ShaderLoadTest.cpp

@@ -16,18 +16,19 @@
 #include <gl/Config.h>
 #include <gl/GLHelpers.h>
 #include <gl/GLShaders.h>
-
+#include <gpu/gl/GLShader.h>
 #include <gpu/gl/GLBackend.h>
 #include <shared/FileUtils.h>
 #include <SettingManager.h>
 
 #include <test-utils/QTestExtensions.h>
 
-//#pragma optimize("", off)
-
 QTEST_MAIN(ShaderLoadTest)
 
-#define FAIL_AFTER_SECONDS 30
+extern std::atomic<size_t> gpuBinaryShadersLoaded;
+
+extern const QString& getShaderCacheFile();
+
 
 QtMessageHandler originalHandler;
 
@@ -39,24 +40,188 @@ void messageHandler(QtMsgType type, const QMessageLogContext& context, const QSt
     originalHandler(type, context, message);
 }
 
-void ShaderLoadTest::loadProgramSources() {
-    _programSources.clear();
-    QString json = FileUtils::readFile(":cache.json");
+std::pair<int, std::vector<std::pair<QString, QString>>> parseCachedShaderString(const QString& cachedShaderString) {
+
+    std::pair<int, std::vector<std::pair<QString, QString>>> result;
+    {
+        static const QRegularExpression versionRegex("^// VERSION (\\d+)");
+        auto match = versionRegex.match(cachedShaderString);
+        result.first = match.captured(1).toInt();
+    }
+
+
+    int rangeStart = 0;
+    QString type;
+    static const QRegularExpression regex("//-------- (\\w+)");
+    auto match = regex.match(cachedShaderString, rangeStart);
+    while (match.hasMatch()) {
+        auto newType = match.captured(1);
+        auto start = match.capturedStart(0);
+        auto end = match.capturedEnd(0);
+        if (rangeStart != 0) {
+            QString subString = cachedShaderString.mid(rangeStart, start - rangeStart);
+            result.second.emplace_back(type, subString);
+        }
+        rangeStart = end;
+        type = newType;
+        match = regex.match(cachedShaderString, rangeStart);
+    }
+
+    if (rangeStart != 0) {
+        QString subString = cachedShaderString.mid(rangeStart);
+        result.second.emplace_back(type, subString);
+    }
+    return result;
+}
+
+std::string getShaderName(const QString& shader) {
+    static const QRegularExpression nameExp("//\\s+(\\w+\\.(?:vert|frag))");
+    auto match = nameExp.match(shader);
+    if (!match.hasMatch()) {
+        return (QCryptographicHash::hash(shader.toUtf8(), QCryptographicHash::Md5).toHex() + ".shader").toStdString();
+    }
+    return match.captured(1).trimmed().toStdString();
+}
+
+void ShaderLoadTest::randomizeShaderSources() {
+    for (auto& entry : _shaderSources) {
+        entry.second += ("\n//" + QUuid::createUuid().toString()).toStdString();
+    }
+}
+
+#if USE_LOCAL_SHADERS
+const QString SHADER_CACHE_FILENAME = "c:/Users/bdavi/AppData/Local/High Fidelity - dev/Interface/shaders/cache.json";
+static const QString SHADER_FOLDER = "D:/shaders/";
+void ShaderLoadTest::parseCacheDirectory() {
+    for (const auto& shaderFile : QDir(SHADER_FOLDER).entryList(QDir::Files)) {
+        QString shaderSource = FileUtils::readFile(SHADER_FOLDER + "/" + shaderFile);
+        _shaderSources[shaderFile.trimmed().toStdString()] = shaderSource.toStdString();
+    }
+
+    auto programsDoc = QJsonDocument::fromJson(FileUtils::readFile(SHADER_FOLDER + "programs.json").toUtf8());
+    for (const auto& programElement : programsDoc.array()) {
+        auto programObj = programElement.toObject();
+        QString vertexSource = programObj["vertex"].toString();
+        QString pixelSource = programObj["pixel"].toString();
+        _programs.insert({ vertexSource.toStdString(), pixelSource.toStdString() });
+    }
+}
+
+void ShaderLoadTest::persistCacheDirectory() {
+    for (const auto& shaderFile : QDir(SHADER_FOLDER).entryList(QDir::Files)) {
+        QFile(SHADER_FOLDER + "/" + shaderFile).remove();
+    }
+
+    // Write the shader source files
+    for (const auto& entry : _shaderSources) {
+        const QString name = entry.first.c_str();
+        const QString shader = entry.second.c_str();
+        QString fullFile = SHADER_FOLDER + name;
+        QVERIFY(!QFileInfo(fullFile).exists());
+        QFile shaderFile(fullFile);
+        shaderFile.open(QIODevice::WriteOnly);
+        shaderFile.write(shader.toUtf8());
+        shaderFile.close();
+    }
+
+    // Write the list of programs
+    {
+        QVariantList programsList;
+        for (const auto& program : _programs) {
+            QVariantMap programMap;
+            programMap["vertex"] = program.first.c_str();
+            programMap["pixel"] = program.second.c_str();
+            programsList.push_back(programMap);
+        }
+
+        QFile saveFile(SHADER_FOLDER + "programs.json");
+        saveFile.open(QFile::WriteOnly | QFile::Text | QFile::Truncate);
+        saveFile.write(QJsonDocument::fromVariant(programsList).toJson(QJsonDocument::Indented));
+        saveFile.close();
+    }
+}
+#else
+const QString SHADER_CACHE_FILENAME = ":cache.json";
+#endif
+
+void ShaderLoadTest::parseCacheFile() {
+    QString json = FileUtils::readFile(SHADER_CACHE_FILENAME);
     auto root = QJsonDocument::fromJson(json.toUtf8()).object();
-    _programSources.reserve(root.size());
-    QRegularExpression regex("//-------- \\d");
-    for (auto shaderKey : root.keys()) {
+    _programs.clear();
+    _programs.reserve(root.size());
+
+    const auto keys = root.keys();
+    Program program;
+    for (auto shaderKey : keys) {
         auto cacheEntry = root[shaderKey].toObject();
         auto source = cacheEntry["source"].toString();
-        auto split = source.split(regex, QString::SplitBehavior::SkipEmptyParts);
-        _programSources.emplace_back(split.at(0).trimmed().toStdString(), split.at(1).trimmed().toStdString());
+        auto shaders = parseCachedShaderString(source);
+        for (const auto& entry : shaders.second) {
+            const auto& type = entry.first;
+            const auto& source = entry.second;
+            const auto name = getShaderName(source);
+            if (name.empty()) {
+                continue;
+            }
+            if (0 == _shaderSources.count(name)) {
+                _shaderSources[name] = source.toStdString();
+            }
+            if (type == "vertex") {
+                program.first = name;
+            } else if (type == "pixel") {
+                program.second = name;
+            }
+        }
+        // FIXME support geometry / tesselation shaders eventually
+        if (program.first.empty() || program.second.empty()) {
+            qFatal("Bad Shader Setup");
+        }
+        _programs.insert(program);
     }
 }        
 
+bool ShaderLoadTest::buildProgram(const Program& programFiles) {
+    const auto& vertexName = programFiles.first;
+    const auto& vertexSource = _shaderSources[vertexName];
+    auto vertexShader = gpu::Shader::createVertex({ vertexSource });
+
+    const auto& pixelName = programFiles.second;
+    const auto& pixelSource = _shaderSources[pixelName];
+    auto pixelShader = gpu::Shader::createPixel({ pixelSource });
+
+    auto program = gpu::Shader::createProgram(vertexShader, pixelShader);
+    return gpu::gl::GLBackend::makeProgram(*program, {}, {});
+}
 
 void ShaderLoadTest::initTestCase() {
     originalHandler = qInstallMessageHandler(messageHandler);
-    loadProgramSources();
+    DependencyManager::set<Setting::Manager>();
+    {
+        const auto& shaderCacheFile = getShaderCacheFile();
+        if (QFileInfo(shaderCacheFile).exists()) {
+            QFile(shaderCacheFile).remove();
+        }
+    }
+
+    // For local debugging
+#if USE_LOCAL_SHADERS
+    parseCacheFile();
+    persistCacheDirectory();
+    parseCacheDirectory();
+#else
+    parseCacheFile();
+#endif
+
+    // We use this to defeat shader caching both by the GPU backend
+    // and the OpenGL driver
+    randomizeShaderSources();
+
+    QVERIFY(!_programs.empty());
+    for (const auto& program : _programs) {
+        QVERIFY(_shaderSources.count(program.first) == 1);
+        QVERIFY(_shaderSources.count(program.second) == 1);
+    }
+
     getDefaultOpenGLSurfaceFormat();
     _canvas.create();
     if (!_canvas.makeCurrent()) {
@@ -64,40 +229,59 @@ void ShaderLoadTest::initTestCase() {
     }
     gl::initModuleGl();
     gpu::Context::init<gpu::gl::GLBackend>();
-    _gpuContext = std::make_shared<gpu::Context>();
     _canvas.makeCurrent();
-    DependencyManager::set<Setting::Manager>();
 }
 
 void ShaderLoadTest::cleanupTestCase() {
-    _gpuContext->recycle();
-    _gpuContext->shutdown();
-    _gpuContext.reset();
     DependencyManager::destroy<Setting::Manager>();
 }
 
-std::string randomString() {
-    return "\n//" + QUuid::createUuid().toString().toStdString();
-}
-
-std::unordered_map<std::string, gpu::ShaderPointer> cachedShaders;
-
-gpu::ShaderPointer getShader(const std::string& shaderSource, bool pixel) {
-    if (0 != cachedShaders.count(shaderSource)) {
-        return cachedShaders[shaderSource];
-    }
-    auto shader = pixel ? gpu::Shader::createPixel({ shaderSource + randomString() })  : gpu::Shader::createVertex({ shaderSource + randomString() });
-    cachedShaders.insert({shaderSource, shader});
-    return shader;
-}
-
 void ShaderLoadTest::testShaderLoad() {
-    QBENCHMARK {
-        for (const auto& programSource : _programSources) {
-            auto vertexShader = getShader(programSource.first, false);
-            auto pixelShader = getShader(programSource.second, true);
-            auto program = gpu::Shader::createProgram(vertexShader, pixelShader);
-            QVERIFY(gpu::gl::GLBackend::makeProgram(*program, {}, {}));
+    auto gpuContext = std::make_shared<gpu::Context>();
+    QVERIFY(gpuBinaryShadersLoaded == 0);
+
+    QElapsedTimer timer;
+
+    // Initial load of all the shaders
+    // No caching
+    {
+        timer.start();
+        for (const auto& program : _programs) {
+            QVERIFY(buildProgram(program));
         }
+        qDebug() << "Uncached shader load took" << timer.elapsed() << "ms";
+        QVERIFY(gpuBinaryShadersLoaded == 0);
     }
+    gpuContext->recycle();
+    glFinish();
+
+    // Reload the shaders within the same GPU context lifetime.
+    // Shaders will use the cached binaries in memory
+    {
+        timer.start();
+        for (const auto& program : _programs) {
+            QVERIFY(buildProgram(program));
+        }
+        qDebug() << "Cached shader load took" << timer.elapsed() << "ms";
+        QVERIFY(gpuBinaryShadersLoaded == _programs.size() * gpu::gl::GLShader::NumVersions);
+    }
+
+    // Simulate reloading the shader cache from disk by destroying and recreating the gpu context
+    // Shaders will use the cached binaries from disk
+    {
+        gpuBinaryShadersLoaded = 0;
+        gpuContext->recycle();
+        gpuContext->shutdown();
+        gpuContext.reset();
+        gpuContext = std::make_shared<gpu::Context>();
+        _canvas.makeCurrent();
+        timer.start();
+        for (const auto& program : _programs) {
+            QVERIFY(buildProgram(program));
+        }
+        qDebug() << "Cached shader load took" << timer.elapsed() << "ms";
+        QVERIFY(gpuBinaryShadersLoaded == _programs.size() * gpu::gl::GLShader::NumVersions);
+    }
+
 }
+

tests/gpu/src/ShaderLoadTest.h

@@ -8,17 +8,44 @@
 
 #pragma once
 
+#include <unordered_map>
+#include <unordered_set>
+
 #include <QtTest/QtTest>
 #include <QtCore/QTemporaryDir>
 
 #include <gpu/Forward.h>
 #include <gl/OffscreenGLCanvas.h>
 
+#define USE_LOCAL_SHADERS 0
+
+namespace std {
+    template <>
+    struct hash<std::pair<std::string, std::string>> {
+        size_t operator()(const std::pair<std::string, std::string>& a) const {
+            std::hash<std::string> hasher;
+            return hasher(a.first) + hasher(a.second);
+        }
+    };
+
+}
+
+using ShadersByName = std::unordered_map<std::string, std::string>;
+using Program = std::pair<std::string, std::string>;
+using Programs = std::unordered_set<Program>;
+
 class ShaderLoadTest : public QObject {
     Q_OBJECT
 
 private:
-    void loadProgramSources();
+
+    void parseCacheFile();
+#if USE_LOCAL_SHADERS
+    void parseCacheDirectory();
+    void persistCacheDirectory();
+#endif
+    bool buildProgram(const Program& program);
+    void randomizeShaderSources();
 
 private slots:
     void initTestCase();
@@ -27,12 +54,10 @@ private slots:
 
 
 private:
-    using ProgramSource = std::pair<std::string, std::string>;
-    using ProgramSources = std::vector<ProgramSource>;
 
-    ProgramSources _programSources;
+    ShadersByName _shaderSources;
+    Programs _programs;
     QString _resourcesPath;
     OffscreenGLCanvas _canvas;
-    gpu::ContextPointer _gpuContext;
     const glm::uvec2 _size{ 640, 480 };
 };