Mirrors/overte-HifiExperiments
3
0
Fork 0
mirror of https://github.com/HifiExperiments/overte.git synced 2025-04-06 06:12:50 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #13082 from jherico/feature/shader_cache

Browse source 
Shader load time performance optimiziations
This commit is contained in:
John Conklin II 2018-05-30 13:55:34 -07:00 committed by GitHub
commit 6095a3b319
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
54 changed files with 1301 additions and 236 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
cmake/macros/SetupHifiTestCase.cmake
View file

@ -61,16 +61,21 @@ macro(SETUP_HIFI_TESTCASE)
endif()
endforeach()
# Find test classes to build into test executables.
# Warn about any .cpp files that are *not* test classes (*Test[s].cpp), since those files will not be used.
foreach (SRC_FILE ${TEST_PROJ_SRC_FILES})
string(REGEX MATCH ".+Tests?\\.cpp$" TEST_CPP_FILE ${SRC_FILE})
string(REGEX MATCH ".+\\.cpp$" NON_TEST_CPP_FILE ${SRC_FILE})
string(REGEX MATCH ".+\\.qrc$" QRC_FILE ${SRC_FILE})
if (TEST_CPP_FILE)
list(APPEND TEST_CASE_FILES ${TEST_CPP_FILE})
elseif (NON_TEST_CPP_FILE)
message(WARNING "ignoring .cpp file (not a test class -- this will not be linked or compiled!): " ${NON_TEST_CPP_FILE})
endif ()
if (QRC_FILE)
list(APPEND EXTRA_FILES ${QRC_FILE})
endif()
endforeach ()
if (TEST_CASE_FILES)
@ -88,7 +93,7 @@ macro(SETUP_HIFI_TESTCASE)
# grab the implemenation and header files
set(TARGET_SRCS ${TEST_FILE}) # only one source / .cpp file (the test class)
add_executable(${TARGET_NAME} ${TEST_FILE})
add_executable(${TARGET_NAME} ${TEST_FILE} ${EXTRA_FILES})
add_test(${TARGET_NAME}-test ${TARGET_NAME})
set_target_properties(${TARGET_NAME} PROPERTIES
EXCLUDE_FROM_DEFAULT_BUILD TRUE

1
interface/src/Application.cpp
View file

@ -2492,6 +2492,7 @@ void Application::cleanupBeforeQuit() {
}
_window->saveGeometry();
_gpuContext->shutdown();
// Destroy third party processes after scripts have finished using them.
#ifdef HAVE_DDE

2
libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.cpp
View file

@ -46,6 +46,8 @@
const char* SRGB_TO_LINEAR_FRAG = R"SCRIBE(
// OpenGLDisplayPlugin_present.frag
uniform sampler2D colorMap;
in vec2 varTexCoord0;

162
libraries/gl/src/gl/GLShaders.cpp
View file

@ -2,15 +2,64 @@
#include "GLLogging.h"
namespace gl {
#include <QtCore/QJsonDocument>
#include <QtCore/QJsonValue>
#include <QtCore/QJsonObject>
#include <QtCore/QFileInfo>
#include <QtCore/QCryptographicHash>
#include <shared/FileUtils.h>
using namespace gl;
void Uniform::load(GLuint glprogram, int index) {
const GLint NAME_LENGTH = 256;
GLchar glname[NAME_LENGTH];
GLint length = 0;
glGetActiveUniform(glprogram, index, NAME_LENGTH, &length, &size, &type, glname);
name = std::string(glname, length);
location = glGetUniformLocation(glprogram, glname);
}
Uniforms gl::loadUniforms(GLuint glprogram) {
GLint uniformsCount = 0;
glGetProgramiv(glprogram, GL_ACTIVE_UNIFORMS, &uniformsCount);
Uniforms result;
result.resize(uniformsCount);
for (int i = 0; i < uniformsCount; i++) {
result[i].load(glprogram, i);
}
return result;
}
#ifdef SEPARATE_PROGRAM
bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, GLuint &programObject, std::string& message) {
bool gl::compileShader(GLenum shaderDomain,
const std::string& shaderSource,
GLuint& shaderObject,
GLuint& programObject,
std::string& message) {
return compileShader(shaderDomain, std::vector<std::string>{ shaderSource }, shaderObject, programObject, message);
}
#else
bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, std::string& message) {
bool gl::compileShader(GLenum shaderDomain, const std::string& shaderSource, GLuint& shaderObject, std::string& message) {
return compileShader(shaderDomain, std::vector<std::string>{ shaderSource }, shaderObject, message);
}
#endif
if (shaderSource.empty()) {
#ifdef SEPARATE_PROGRAM
bool gl::compileShader(GLenum shaderDomain,
const std::string& shaderSource,
GLuint& shaderObject,
GLuint& programObject,
std::string& message) {
#else
bool gl::compileShader(GLenum shaderDomain,
const std::vector<std::string>& shaderSources,
GLuint& shaderObject,
std::string& message) {
#endif
if (shaderSources.empty()) {
qCDebug(glLogging) << "GLShader::compileShader - no GLSL shader source code ? so failed to create";
return false;
}
@ -23,9 +72,11 @@ namespace gl {
}
// Assign the source
const int NUM_SOURCE_STRINGS = 2;
const GLchar* srcstr[] = { defines.c_str(), shaderSource.c_str() };
glShaderSource(glshader, NUM_SOURCE_STRINGS, srcstr, NULL);
std::vector<const GLchar*> cstrs;
for (const auto& str : shaderSources) {
cstrs.push_back(str.c_str());
}
glShaderSource(glshader, static_cast<GLint>(cstrs.size()), cstrs.data(), NULL);
// Compile !
glCompileShader(glshader);
@ -66,7 +117,7 @@ namespace gl {
qCCritical(glLogging) << "GLShader::compileShader - failed to compile the gl shader object:";
int lineNumber = 0;
for (auto s : srcstr) {
for (const auto& s : cstrs) {
QString str(s);
QStringList lines = str.split("\n");
for (auto& line : lines) {
@ -142,7 +193,7 @@ namespace gl {
return true;
}
GLuint compileProgram(const std::vector<GLuint>& glshaders, std::string& message, std::vector<GLchar>& binary) {
GLuint gl::compileProgram(const std::vector<GLuint>& glshaders, std::string& message, CachedShader& cachedShader) {
// A brand new program:
GLuint glprogram = glCreateProgram();
if (!glprogram) {
@ -150,14 +201,21 @@ GLuint compileProgram(const std::vector<GLuint>& glshaders, std::string& message
return 0;
}
// glProgramParameteri(glprogram, GL_PROGRAM_, GL_TRUE);
// Create the program from the sub shaders
for (auto so : glshaders) {
glAttachShader(glprogram, so);
}
bool binaryLoaded = false;
// Link!
glLinkProgram(glprogram);
if (glshaders.empty() && cachedShader) {
glProgramBinary(glprogram, cachedShader.format, cachedShader.binary.data(), (GLsizei)cachedShader.binary.size());
binaryLoaded = true;
} else {
// glProgramParameteri(glprogram, GL_PROGRAM_, GL_TRUE);
// Create the program from the sub shaders
for (auto so : glshaders) {
glAttachShader(glprogram, so);
}
// Link!
glLinkProgram(glprogram);
}
GLint linked = 0;
glGetProgramiv(glprogram, GL_LINK_STATUS, &linked);
@ -205,25 +263,73 @@ GLuint compileProgram(const std::vector<GLuint>& glshaders, std::string& message
}
// If linked get the binaries
if (linked) {
if (linked && !binaryLoaded) {
GLint binaryLength = 0;
glGetProgramiv(glprogram, GL_PROGRAM_BINARY_LENGTH, &binaryLength);
if (binaryLength > 0) {
GLint numBinFormats = 0;
glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, &numBinFormats);
if (numBinFormats > 0) {
binary.resize(binaryLength);
std::vector<GLint> binFormats(numBinFormats);
glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, binFormats.data());
GLenum programBinFormat;
glGetProgramBinary(glprogram, binaryLength, NULL, &programBinFormat, binary.data());
}
cachedShader.binary.resize(binaryLength);
glGetProgramBinary(glprogram, binaryLength, NULL, &cachedShader.format, cachedShader.binary.data());
}
}
return glprogram;
}
const QString& getShaderCacheFile() {
static const QString SHADER_CACHE_FOLDER{ "shaders" };
static const QString SHADER_CACHE_FILE_NAME{ "cache.json" };
static const QString SHADER_CACHE_FILE = FileUtils::standardPath(SHADER_CACHE_FOLDER) + SHADER_CACHE_FILE_NAME;
return SHADER_CACHE_FILE;
}
static const char* SHADER_JSON_TYPE_KEY = "type";
static const char* SHADER_JSON_SOURCE_KEY = "source";
static const char* SHADER_JSON_DATA_KEY = "data";
void gl::loadShaderCache(ShaderCache& cache) {
QString shaderCacheFile = getShaderCacheFile();
if (QFileInfo(shaderCacheFile).exists()) {
QString json = FileUtils::readFile(shaderCacheFile);
auto root = QJsonDocument::fromJson(json.toUtf8()).object();
for (const auto& qhash : root.keys()) {
auto programObject = root[qhash].toObject();
QByteArray qbinary = QByteArray::fromBase64(programObject[SHADER_JSON_DATA_KEY].toString().toUtf8());
std::string hash = qhash.toStdString();
auto& cachedShader = cache[hash];
cachedShader.binary.resize(qbinary.size());
memcpy(cachedShader.binary.data(), qbinary.data(), qbinary.size());
cachedShader.format = (GLenum)programObject[SHADER_JSON_TYPE_KEY].toInt();
cachedShader.source = programObject[SHADER_JSON_SOURCE_KEY].toString().toStdString();
}
}
}
void gl::saveShaderCache(const ShaderCache& cache) {
QByteArray json;
{
QVariantMap variantMap;
for (const auto& entry : cache) {
const auto& key = entry.first;
const auto& type = entry.second.format;
const auto& binary = entry.second.binary;
QVariantMap qentry;
qentry[SHADER_JSON_TYPE_KEY] = QVariant(type);
qentry[SHADER_JSON_SOURCE_KEY] = QString(entry.second.source.c_str());
qentry[SHADER_JSON_DATA_KEY] = QByteArray{ binary.data(), (int)binary.size() }.toBase64();
variantMap[key.c_str()] = qentry;
}
json = QJsonDocument::fromVariant(variantMap).toJson(QJsonDocument::Indented);
}
if (!json.isEmpty()) {
QString shaderCacheFile = getShaderCacheFile();
QFile saveFile(shaderCacheFile);
saveFile.open(QFile::WriteOnly | QFile::Text | QFile::Truncate);
saveFile.write(json);
saveFile.close();
}
}
std::string gl::getShaderHash(const std::string& shaderSource) {
return QCryptographicHash::hash(QByteArray(shaderSource.c_str()), QCryptographicHash::Md5).toBase64().toStdString();
}

38
libraries/gl/src/gl/GLShaders.h
View file

@ -14,15 +14,47 @@
#include <vector>
#include <string>
#include <unordered_map>
namespace gl {
struct Uniform {
std::string name;
GLint size{ -1 };
GLenum type{ GL_FLOAT };
GLint location{ -1 };
void load(GLuint glprogram, int index);
};
using Uniforms = std::vector<Uniform>;
Uniforms loadUniforms(GLuint glprogram);
struct CachedShader {
GLenum format{ 0 };
std::string source;
std::vector<char> binary;
inline operator bool() const {
return format != 0 && !binary.empty();
}
};
using ShaderCache = std::unordered_map<std::string, CachedShader>;
std::string getShaderHash(const std::string& shaderSource);
void loadShaderCache(ShaderCache& cache);
void saveShaderCache(const ShaderCache& cache);
#ifdef SEPARATE_PROGRAM
bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, GLuint &programObject, std::string& message);
bool compileShader(GLenum shaderDomain, const std::string& shaderSource, GLuint &shaderObject, GLuint &programObject, std::string& message);
bool compileShader(GLenum shaderDomain, const std::vector<std::string>& shaderSources, GLuint &shaderObject, GLuint &programObject, std::string& message);
#else
bool compileShader(GLenum shaderDomain, const std::string& shaderSource, const std::string& defines, GLuint &shaderObject, std::string& message);
bool compileShader(GLenum shaderDomain, const std::string& shaderSource, GLuint &shaderObject, std::string& message);
bool compileShader(GLenum shaderDomain, const std::vector<std::string>& shaderSources, GLuint &shaderObject, std::string& message);
#endif
GLuint compileProgram(const std::vector<GLuint>& glshaders, std::string& message, std::vector<GLchar>& binary);
GLuint compileProgram(const std::vector<GLuint>& glshaders, std::string& message, CachedShader& binary);
}

5
libraries/gpu-gl-common/src/gpu/gl/GLBackend.cpp
View file

@ -124,13 +124,16 @@ void GLBackend::init() {
GLBackend::GLBackend() {
_pipeline._cameraCorrectionBuffer._buffer->flush();
glGetIntegerv(GL_UNIFORM_BUFFER_OFFSET_ALIGNMENT, &_uboAlignment);
initShaderBinaryCache();
}
GLBackend::~GLBackend() {}
GLBackend::~GLBackend() {
void GLBackend::shutdown() {
killInput();
killTransform();
killTextureManagementStage();
killShaderBinaryCache();
}
void GLBackend::renderPassTransfer(const Batch& batch) {

34
libraries/gpu-gl-common/src/gpu/gl/GLBackend.h
View file

@ -23,6 +23,7 @@
#include <QtCore/QLoggingCategory>
#include <gl/Config.h>
#include <gl/GLShaders.h>
#include <gpu/Forward.h>
#include <gpu/Context.h>
@ -71,6 +72,9 @@ public:
virtual ~GLBackend();
// Shutdown rendering and persist any required resources
void shutdown() override;
void setCameraCorrection(const Mat4& correction, const Mat4& prevRenderView, bool reset = false);
void render(const Batch& batch) final override;
@ -455,6 +459,13 @@ protected:
virtual GLShader* compileBackendProgram(const Shader& program, const Shader::CompilationHandler& handler);
virtual GLShader* compileBackendShader(const Shader& shader, const Shader::CompilationHandler& handler);
virtual std::string getBackendShaderHeader() const = 0;
// For a program, this will return a string containing all the source files (without any
// backend headers or defines). For a vertex, fragment or geometry shader, this will
// return the fully customized shader with all the version and backend specific
// preprocessor directives
// The program string returned can be used as a key for a cache of shader binaries
// The shader strings can be reliably sent to the low level `compileShader` functions
virtual std::string getShaderSource(const Shader& shader, int version) final;
virtual void makeProgramBindings(ShaderObject& shaderObject);
class ElementResource {
public:
@ -465,12 +476,12 @@ protected:
ElementResource getFormatFromGLUniform(GLenum gltype);
static const GLint UNUSED_SLOT {-1};
static bool isUnusedSlot(GLint binding) { return (binding == UNUSED_SLOT); }
virtual int makeUniformSlots(GLuint glprogram, const Shader::BindingSet& slotBindings,
virtual int makeUniformSlots(const ShaderObject& program, const Shader::BindingSet& slotBindings,
Shader::SlotSet& uniforms, Shader::SlotSet& textures, Shader::SlotSet& samplers);
virtual int makeUniformBlockSlots(GLuint glprogram, const Shader::BindingSet& slotBindings, Shader::SlotSet& buffers);
virtual int makeResourceBufferSlots(GLuint glprogram, const Shader::BindingSet& slotBindings, Shader::SlotSet& resourceBuffers) = 0;
virtual int makeInputSlots(GLuint glprogram, const Shader::BindingSet& slotBindings, Shader::SlotSet& inputs);
virtual int makeOutputSlots(GLuint glprogram, const Shader::BindingSet& slotBindings, Shader::SlotSet& outputs);
virtual int makeUniformBlockSlots(const ShaderObject& program, const Shader::BindingSet& slotBindings, Shader::SlotSet& buffers);
virtual int makeResourceBufferSlots(const ShaderObject& program, const Shader::BindingSet& slotBindings, Shader::SlotSet& resourceBuffers) = 0;
virtual int makeInputSlots(const ShaderObject& program, const Shader::BindingSet& slotBindings, Shader::SlotSet& inputs);
virtual int makeOutputSlots(const ShaderObject& program, const Shader::BindingSet& slotBindings, Shader::SlotSet& outputs);
// Synchronize the state cache of this Backend with the actual real state of the GL Context
@ -489,6 +500,19 @@ protected:
void resetStages();
// Stores cached binary versions of the shaders for quicker startup on subsequent runs
// Note that shaders in the cache can still fail to load due to hardware or driver
// changes that invalidate the cached binary, in which case we fall back on compiling
// the source again
struct ShaderBinaryCache {
std::mutex _mutex;
std::vector<GLint> _formats;
std::unordered_map<std::string, ::gl::CachedShader> _binaries;
} _shaderBinaryCache;
virtual void initShaderBinaryCache();
virtual void killShaderBinaryCache();
struct TextureManagementStageState {
bool _sparseCapable { false };
GLTextureTransferEnginePointer _transferEngine;

226
libraries/gpu-gl-common/src/gpu/gl/GLBackendShader.cpp
View file

@ -11,6 +11,8 @@
using namespace gpu;
using namespace gpu::gl;
using CachedShader = ::gl::CachedShader;
// Shader domain
static const size_t NUM_SHADER_DOMAINS = 3;
@ -68,9 +70,45 @@ static const std::array<std::string, GLShader::NumVersions> VERSION_DEFINES { {
stereoVersion
} };
static std::string getShaderTypeString(Shader::Type type) {
switch (type) {
case Shader::Type::VERTEX:
return "vertex";
case Shader::Type::PIXEL:
return "pixel";
case Shader::Type::GEOMETRY:
return "geometry";
case Shader::Type::PROGRAM:
return "program";
default:
qFatal("Unexpected shader type %d", type);
Q_UNREACHABLE();
}
}
std::string GLBackend::getShaderSource(const Shader& shader, int version) {
if (shader.isProgram()) {
std::string result;
result.append("// VERSION " + std::to_string(version));
for (const auto& subShader : shader.getShaders()) {
result.append("//-------- ");
result.append(getShaderTypeString(subShader->getType()));
result.append("\n");
result.append(subShader->getSource().getCode());
}
return result;
}
std::string shaderDefines = getBackendShaderHeader() + "\n"
+ (supportsBindless() ? textureTableVersion : "\n")
+ DOMAIN_DEFINES[shader.getType()] + "\n"
+ VERSION_DEFINES[version];
return shaderDefines + "\n" + shader.getSource().getCode();
}
GLShader* GLBackend::compileBackendShader(const Shader& shader, const Shader::CompilationHandler& handler) {
// Any GLSLprogram ? normally yes...
const std::string& shaderSource = shader.getSource().getCode();
GLenum shaderDomain = SHADER_DOMAINS[shader.getType()];
GLShader::ShaderObjects shaderObjects;
Shader::CompilationLogs compilationLogs(GLShader::NumVersions);
@ -78,11 +116,7 @@ GLShader* GLBackend::compileBackendShader(const Shader& shader, const Shader::Co
for (int version = 0; version < GLShader::NumVersions; version++) {
auto& shaderObject = shaderObjects[version];
std::string shaderDefines = getBackendShaderHeader() + "\n"
+ (supportsBindless() ? textureTableVersion : "\n")
+ DOMAIN_DEFINES[shader.getType()] + "\n"
+ VERSION_DEFINES[version];
auto shaderSource = getShaderSource(shader, version);
if (handler) {
bool retest = true;
std::string currentSrc = shaderSource;
@ -90,7 +124,7 @@ GLShader* GLBackend::compileBackendShader(const Shader& shader, const Shader::Co
// The retest bool is set to false as soon as the compilation succeed to wexit the while loop.
// The handler tells us if we should retry or not while returning a modified version of the source.
while (retest) {
bool result = ::gl::compileShader(shaderDomain, currentSrc, shaderDefines, shaderObject.glshader, compilationLogs[version].message);
bool result = ::gl::compileShader(shaderDomain, currentSrc, shaderObject.glshader, compilationLogs[version].message);
compilationLogs[version].compiled = result;
if (!result) {
std::string newSrc;
@ -101,7 +135,7 @@ GLShader* GLBackend::compileBackendShader(const Shader& shader, const Shader::Co
}
}
} else {
compilationLogs[version].compiled = ::gl::compileShader(shaderDomain, shaderSource, shaderDefines, shaderObject.glshader, compilationLogs[version].message);
compilationLogs[version].compiled = ::gl::compileShader(shaderDomain, shaderSource, shaderObject.glshader, compilationLogs[version].message);
}
if (!compilationLogs[version].compiled) {
@ -120,43 +154,80 @@ GLShader* GLBackend::compileBackendShader(const Shader& shader, const Shader::Co
return object;
}
std::atomic<size_t> gpuBinaryShadersLoaded;
GLShader* GLBackend::compileBackendProgram(const Shader& program, const Shader::CompilationHandler& handler) {
if (!program.isProgram()) {
return nullptr;
}
GLShader::ShaderObjects programObjects;
program.incrementCompilationAttempt();
Shader::CompilationLogs compilationLogs(GLShader::NumVersions);
for (int version = 0; version < GLShader::NumVersions; version++) {
auto& programObject = programObjects[version];
auto programSource = getShaderSource(program, version);
auto hash = ::gl::getShaderHash(programSource);
// Let's go through every shaders and make sure they are ready to go
std::vector< GLuint > shaderGLObjects;
for (auto subShader : program.getShaders()) {
auto object = GLShader::sync((*this), *subShader, handler);
if (object) {
shaderGLObjects.push_back(object->_shaderObjects[version].glshader);
} else {
qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - One of the shaders of the program is not compiled?";
compilationLogs[version].compiled = false;
compilationLogs[version].message = std::string("Failed to compile, one of the shaders of the program is not compiled ?");
program.setCompilationLogs(compilationLogs);
return nullptr;
CachedShader cachedBinary;
{
Lock shaderCacheLock{ _shaderBinaryCache._mutex };
if (_shaderBinaryCache._binaries.count(hash) != 0) {
cachedBinary = _shaderBinaryCache._binaries[hash];
}
}
GLuint glprogram = 0;
// If we have a cached binary program, try to load it instead of compiling the individual shaders
if (cachedBinary) {
glprogram = ::gl::compileProgram({}, compilationLogs[version].message, cachedBinary);
if (0 != glprogram) {
++gpuBinaryShadersLoaded;
}
}
// If we have no program, then either no cached binary, or the binary failed to load (perhaps a GPU driver update invalidated the cache)
if (0 == glprogram) {
cachedBinary = CachedShader();
{
std::unique_lock<std::mutex> shaderCacheLock{ _shaderBinaryCache._mutex };
_shaderBinaryCache._binaries.erase(hash);
}
// Let's go through every shaders and make sure they are ready to go
std::vector<GLuint> shaderGLObjects;
shaderGLObjects.reserve(program.getShaders().size());
for (auto subShader : program.getShaders()) {
auto object = GLShader::sync((*this), *subShader, handler);
if (object) {
shaderGLObjects.push_back(object->_shaderObjects[version].glshader);
} else {
qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - One of the shaders of the program is not compiled?";
compilationLogs[version].compiled = false;
compilationLogs[version].message = std::string("Failed to compile, one of the shaders of the program is not compiled ?");
program.setCompilationLogs(compilationLogs);
return nullptr;
}
}
glprogram = ::gl::compileProgram(shaderGLObjects, compilationLogs[version].message, cachedBinary);
if (cachedBinary) {
cachedBinary.source = programSource;
std::unique_lock<std::mutex> shaderCacheLock{ _shaderBinaryCache._mutex };
_shaderBinaryCache._binaries[hash] = cachedBinary;
}
}
GLuint glprogram = ::gl::compileProgram(shaderGLObjects, compilationLogs[version].message, compilationLogs[version].binary);
if (glprogram == 0) {
qCWarning(gpugllogging) << "GLBackend::compileBackendProgram - Program didn't link:\n" << compilationLogs[version].message.c_str();
program.setCompilationLogs(compilationLogs);
return nullptr;
}
compilationLogs[version].compiled = true;
programObject.glprogram = glprogram;
makeProgramBindings(programObject);
}
// Compilation feedback
@ -338,20 +409,15 @@ GLBackend::ElementResource GLBackend::getFormatFromGLUniform(GLenum gltype) {
};
int GLBackend::makeUniformSlots(GLuint glprogram, const Shader::BindingSet& slotBindings,
int GLBackend::makeUniformSlots(const ShaderObject& shaderProgram, const Shader::BindingSet& slotBindings,
Shader::SlotSet& uniforms, Shader::SlotSet& textures, Shader::SlotSet& samplers) {
GLint uniformsCount = 0;
auto& glprogram = shaderProgram.glprogram;
glGetProgramiv(glprogram, GL_ACTIVE_UNIFORMS, &uniformsCount);
for (int i = 0; i < uniformsCount; i++) {
const GLint NAME_LENGTH = 256;
GLchar name[NAME_LENGTH];
GLint length = 0;
GLint size = 0;
GLenum type = 0;
glGetActiveUniform(glprogram, i, NAME_LENGTH, &length, &size, &type, name);
GLint location = glGetUniformLocation(glprogram, name);
for (const auto& uniform : shaderProgram.uniforms) {
const auto& type = uniform.type;
const auto& location = uniform.location;
const auto& size = uniform.size;
const auto& name = uniform.name;
const GLint INVALID_UNIFORM_LOCATION = -1;
// Try to make sense of the gltype
@ -359,8 +425,8 @@ int GLBackend::makeUniformSlots(GLuint glprogram, const Shader::BindingSet& slot
// The uniform as a standard var type
if (location != INVALID_UNIFORM_LOCATION) {
auto sname = uniform.name;
// Let's make sure the name doesn't contains an array element
std::string sname(name);
auto foundBracket = sname.find_first_of('[');
if (foundBracket != std::string::npos) {
// std::string arrayname = sname.substr(0, foundBracket);
@ -397,10 +463,11 @@ int GLBackend::makeUniformSlots(GLuint glprogram, const Shader::BindingSet& slot
}
}
return uniformsCount;
return static_cast<uint32_t>(shaderProgram.uniforms.size());
}
int GLBackend::makeUniformBlockSlots(GLuint glprogram, const Shader::BindingSet& slotBindings, Shader::SlotSet& buffers) {
int GLBackend::makeUniformBlockSlots(const ShaderObject& shaderProgram, const Shader::BindingSet& slotBindings, Shader::SlotSet& buffers) {
const auto& glprogram = shaderProgram.glprogram;
GLint buffersCount = 0;
glGetProgramiv(glprogram, GL_ACTIVE_UNIFORM_BLOCKS, &buffersCount);
@ -479,7 +546,8 @@ int GLBackend::makeUniformBlockSlots(GLuint glprogram, const Shader::BindingSet&
return buffersCount;
}
int GLBackend::makeInputSlots(GLuint glprogram, const Shader::BindingSet& slotBindings, Shader::SlotSet& inputs) {
int GLBackend::makeInputSlots(const ShaderObject& shaderProgram, const Shader::BindingSet& slotBindings, Shader::SlotSet& inputs) {
const auto& glprogram = shaderProgram.glprogram;
GLint inputsCount = 0;
glGetProgramiv(glprogram, GL_ACTIVE_ATTRIBUTES, &inputsCount);
@ -501,7 +569,7 @@ int GLBackend::makeInputSlots(GLuint glprogram, const Shader::BindingSet& slotBi
return inputsCount;
}
int GLBackend::makeOutputSlots(GLuint glprogram, const Shader::BindingSet& slotBindings, Shader::SlotSet& outputs) {
int GLBackend::makeOutputSlots(const ShaderObject& shaderProgram, const Shader::BindingSet& slotBindings, Shader::SlotSet& outputs) {
/* GLint outputsCount = 0;
glGetProgramiv(glprogram, GL_ACTIVE_, &outputsCount);
@ -525,67 +593,19 @@ void GLBackend::makeProgramBindings(ShaderObject& shaderObject) {
if (!shaderObject.glprogram) {
return;
}
GLuint glprogram = shaderObject.glprogram;
GLint loc = -1;
//Check for gpu specific attribute slotBindings
loc = glGetAttribLocation(glprogram, "inPosition");
if (loc >= 0 && loc != gpu::Stream::POSITION) {
glBindAttribLocation(glprogram, gpu::Stream::POSITION, "inPosition");
}
loc = glGetAttribLocation(glprogram, "inNormal");
if (loc >= 0 && loc != gpu::Stream::NORMAL) {
glBindAttribLocation(glprogram, gpu::Stream::NORMAL, "inNormal");
}
loc = glGetAttribLocation(glprogram, "inColor");
if (loc >= 0 && loc != gpu::Stream::COLOR) {
glBindAttribLocation(glprogram, gpu::Stream::COLOR, "inColor");
}
loc = glGetAttribLocation(glprogram, "inTexCoord0");
if (loc >= 0 && loc != gpu::Stream::TEXCOORD) {
glBindAttribLocation(glprogram, gpu::Stream::TEXCOORD, "inTexCoord0");
}
loc = glGetAttribLocation(glprogram, "inTangent");
if (loc >= 0 && loc != gpu::Stream::TANGENT) {
glBindAttribLocation(glprogram, gpu::Stream::TANGENT, "inTangent");
}
char attribName[] = "inTexCoordn";
for (auto i = 0; i < 4; i++) {
auto streamId = gpu::Stream::TEXCOORD1 + i;
attribName[strlen(attribName) - 1] = '1' + i;
loc = glGetAttribLocation(glprogram, attribName);
if (loc >= 0 && loc != streamId) {
glBindAttribLocation(glprogram, streamId, attribName);
}
}
loc = glGetAttribLocation(glprogram, "inSkinClusterIndex");
if (loc >= 0 && loc != gpu::Stream::SKIN_CLUSTER_INDEX) {
glBindAttribLocation(glprogram, gpu::Stream::SKIN_CLUSTER_INDEX, "inSkinClusterIndex");
}
loc = glGetAttribLocation(glprogram, "inSkinClusterWeight");
if (loc >= 0 && loc != gpu::Stream::SKIN_CLUSTER_WEIGHT) {
glBindAttribLocation(glprogram, gpu::Stream::SKIN_CLUSTER_WEIGHT, "inSkinClusterWeight");
}
loc = glGetAttribLocation(glprogram, "_drawCallInfo");
if (loc >= 0 && loc != gpu::Stream::DRAW_CALL_INFO) {
glBindAttribLocation(glprogram, gpu::Stream::DRAW_CALL_INFO, "_drawCallInfo");
}
// Link again to take into account the assigned attrib location
glLinkProgram(glprogram);
GLint linked = 0;
glGetProgramiv(glprogram, GL_LINK_STATUS, &linked);
if (!linked) {
qCWarning(gpugllogging) << "GLShader::makeBindings - failed to link after assigning slotBindings?";
}
}
void GLBackend::initShaderBinaryCache() {
GLint numBinFormats = 0;
glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, &numBinFormats);
if (numBinFormats > 0) {
_shaderBinaryCache._formats.resize(numBinFormats);
glGetIntegerv(GL_PROGRAM_BINARY_FORMATS, _shaderBinaryCache._formats.data());
}
::gl::loadShaderCache(_shaderBinaryCache._binaries);
}
void GLBackend::killShaderBinaryCache() {
::gl::saveShaderCache(_shaderBinaryCache._binaries);
}

11
libraries/gpu-gl-common/src/gpu/gl/GLShader.cpp
View file

@ -68,22 +68,23 @@ bool GLShader::makeProgram(GLBackend& backend, Shader& shader, const Shader::Bin
for (int version = 0; version < GLShader::NumVersions; version++) {
auto& shaderObject = object->_shaderObjects[version];
if (shaderObject.glprogram) {
shaderObject.uniforms = ::gl::loadUniforms(shaderObject.glprogram);
Shader::SlotSet buffers;
backend.makeUniformBlockSlots(shaderObject.glprogram, slotBindings, buffers);
backend.makeUniformBlockSlots(shaderObject, slotBindings, buffers);
Shader::SlotSet uniforms;
Shader::SlotSet textures;
Shader::SlotSet samplers;
backend.makeUniformSlots(shaderObject.glprogram, slotBindings, uniforms, textures, samplers);
backend.makeUniformSlots(shaderObject, slotBindings, uniforms, textures, samplers);
Shader::SlotSet resourceBuffers;
backend.makeResourceBufferSlots(shaderObject.glprogram, slotBindings, resourceBuffers);
backend.makeResourceBufferSlots(shaderObject, slotBindings, resourceBuffers);
Shader::SlotSet inputs;
backend.makeInputSlots(shaderObject.glprogram, slotBindings, inputs);
backend.makeInputSlots(shaderObject, slotBindings, inputs);
Shader::SlotSet outputs;
backend.makeOutputSlots(shaderObject.glprogram, slotBindings, outputs);
backend.makeOutputSlots(shaderObject, slotBindings, outputs);
// Define the public slots only from the default version
if (version == 0) {

3
libraries/gpu-gl-common/src/gpu/gl/GLShader.h
View file

@ -9,14 +9,17 @@
#define hifi_gpu_gl_GLShader_h
#include "GLShared.h"
#include <gl/GLShaders.h>
namespace gpu { namespace gl {
struct ShaderObject {
using Uniforms = ::gl::Uniforms;
GLuint glshader { 0 };
GLuint glprogram { 0 };
GLint transformCameraSlot { -1 };
GLint transformObjectSlot { -1 };
Uniforms uniforms;
};
class GLShader : public GPUObject {

2
libraries/gpu-gl/src/gpu/gl41/GL41Backend.h
View file

@ -173,7 +173,7 @@ protected:
std::string getBackendShaderHeader() const override;
void makeProgramBindings(ShaderObject& shaderObject) override;
int makeResourceBufferSlots(GLuint glprogram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) override;
int makeResourceBufferSlots(const ShaderObject& shaderProgram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) override;
};

19
libraries/gpu-gl/src/gpu/gl41/GL41BackendShader.cpp
View file

@ -22,20 +22,13 @@ std::string GL41Backend::getBackendShaderHeader() const {
return header;
}
int GL41Backend::makeResourceBufferSlots(GLuint glprogram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) {
int GL41Backend::makeResourceBufferSlots(const ShaderObject& shaderProgram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) {
GLint ssboCount = 0;
GLint uniformsCount = 0;
glGetProgramiv(glprogram, GL_ACTIVE_UNIFORMS, &uniformsCount);
for (int i = 0; i < uniformsCount; i++) {
const GLint NAME_LENGTH = 256;
GLchar name[NAME_LENGTH];
GLint length = 0;
GLint size = 0;
GLenum type = 0;
glGetActiveUniform(glprogram, i, NAME_LENGTH, &length, &size, &type, name);
GLint location = glGetUniformLocation(glprogram, name);
const auto& glprogram = shaderProgram.glprogram;
for (const auto& uniform : shaderProgram.uniforms) {
const auto& name = uniform.name;
const auto& type = uniform.type;
const auto& location = uniform.location;
const GLint INVALID_UNIFORM_LOCATION = -1;
// Try to make sense of the gltype

2
libraries/gpu-gl/src/gpu/gl45/GL45Backend.h
View file

@ -274,7 +274,7 @@ protected:
// Shader Stage
std::string getBackendShaderHeader() const override;
void makeProgramBindings(ShaderObject& shaderObject) override;
int makeResourceBufferSlots(GLuint glprogram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) override;
int makeResourceBufferSlots(const ShaderObject& shaderProgram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) override;
// Texture Management Stage
void initTextureManagementStage() override;

3
libraries/gpu-gl/src/gpu/gl45/GL45BackendShader.cpp
View file

@ -27,7 +27,8 @@ std::string GL45Backend::getBackendShaderHeader() const {
return header;
}
int GL45Backend::makeResourceBufferSlots(GLuint glprogram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) {
int GL45Backend::makeResourceBufferSlots(const ShaderObject& shaderProgram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) {
const auto& glprogram = shaderProgram.glprogram;
GLint buffersCount = 0;
glGetProgramInterfaceiv(glprogram, GL_SHADER_STORAGE_BLOCK, GL_ACTIVE_RESOURCES, &buffersCount);

2
libraries/gpu-gles/src/gpu/gles/GLESBackend.h
View file

@ -164,7 +164,7 @@ protected:
std::string getBackendShaderHeader() const override;
void makeProgramBindings(ShaderObject& shaderObject) override;
int makeResourceBufferSlots(GLuint glprogram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) override;
int makeResourceBufferSlots(const ShaderObject& shaderObject, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) override;
};

19
libraries/gpu-gles/src/gpu/gles/GLESBackendShader.cpp
View file

@ -25,20 +25,15 @@ std::string GLESBackend::getBackendShaderHeader() const {
return header;
}
int GLESBackend::makeResourceBufferSlots(GLuint glprogram, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) {
int GLESBackend::makeResourceBufferSlots(const ShaderObject& shaderObject, const Shader::BindingSet& slotBindings,Shader::SlotSet& resourceBuffers) {
GLint ssboCount = 0;
GLint uniformsCount = 0;
GLint uniformsCount = 0;
const auto& glprogram = shaderObject.glprogram;
glGetProgramiv(glprogram, GL_ACTIVE_UNIFORMS, &uniformsCount);
for (int i = 0; i < uniformsCount; i++) {
const GLint NAME_LENGTH = 256;
GLchar name[NAME_LENGTH];
GLint length = 0;
GLint size = 0;
GLenum type = 0;
glGetActiveUniform(glprogram, i, NAME_LENGTH, &length, &size, &type, name);
GLint location = glGetUniformLocation(glprogram, name);
for (const auto& uniform : shaderObject.uniforms) {
const auto& type = uniform.type;
const auto& location = uniform.location;
const auto& name = uniform.name;
const GLint INVALID_UNIFORM_LOCATION = -1;
// Try to make sense of the gltype

7
libraries/gpu/src/gpu/Context.cpp
View file

@ -53,6 +53,13 @@ Context::~Context() {
_batchPool.clear();
}
void Context::shutdown() {
if (_backend) {
_backend->shutdown();
_backend.reset();
}
}
const std::string& Context::getBackendVersion() const {
return _backend->getVersion();
}

2
libraries/gpu/src/gpu/Context.h
View file

@ -54,6 +54,7 @@ class Backend {
public:
virtual ~Backend(){};
virtual void shutdown() {}
virtual const std::string& getVersion() const = 0;
void setStereoState(const StereoState& stereo) { _stereo = stereo; }
@ -154,6 +155,7 @@ public:
Context();
~Context();
void shutdown();
const std::string& getBackendVersion() const;
void beginFrame(const glm::mat4& renderView = glm::mat4(), const glm::mat4& renderPose = glm::mat4());

2
libraries/gpu/src/gpu/Shader.h
View file

@ -54,13 +54,11 @@ public:
struct CompilationLog {
std::string message;
std::vector<char> binary;
bool compiled{ false };
CompilationLog() {}
CompilationLog(const CompilationLog& src) :
message(src.message),
binary(src.binary),
compiled(src.compiled) {}
};
using CompilationLogs = std::vector<CompilationLog>;

4
libraries/networking/src/ResourceCache.cpp
View file

@ -218,8 +218,8 @@ ScriptableResource* ResourceCache::prefetch(const QUrl& url, void* extra) {
}
ResourceCache::ResourceCache(QObject* parent) : QObject(parent) {
auto nodeList = DependencyManager::get<NodeList>();
if (nodeList) {
if (DependencyManager::isSet<NodeList>()) {
auto nodeList = DependencyManager::get<NodeList>();
auto& domainHandler = nodeList->getDomainHandler();
connect(&domainHandler, &DomainHandler::disconnectedFromDomain,
this, &ResourceCache::clearATPAssets, Qt::DirectConnection);

22
libraries/shared/src/SharedUtil.h
View file

@ -25,6 +25,7 @@
#include <QtCore/QCoreApplication>
#include <QUuid>
#include "NumericalConstants.h"
// When writing out avatarEntities to a QByteArray, if the parentID is the ID of MyAvatar, use this ID instead. This allows
// the value to be reset when the sessionID changes.
const QUuid AVATAR_SELF_ID = QUuid("{00000000-0000-0000-0000-000000000001}");
@ -122,6 +123,27 @@ const QByteArray HIGH_FIDELITY_USER_AGENT = "Mozilla/5.0 (HighFidelityInterface)
quint64 usecTimestampNow(bool wantDebug = false);
void usecTimestampNowForceClockSkew(qint64 clockSkew);
inline bool afterUsecs(quint64& startUsecs, quint64 maxIntervalUecs) {
auto now = usecTimestampNow();
auto interval = now - startUsecs;
if (interval > maxIntervalUecs) {
startUsecs = now;
return true;
}
return false;
}
inline bool afterSecs(quint64& startUsecs, quint64 maxIntervalSecs) {
return afterUsecs(startUsecs, maxIntervalSecs * USECS_PER_SECOND);
}
template <typename F>
void doEvery(quint64& lastReportUsecs, quint64 secs, F lamdba) {
if (afterSecs(lastReportUsecs, secs)) {
lamdba();
}
}
// Number of seconds expressed since the first call to this function, expressed as a float
// Maximum accuracy in msecs
float secTimestampNow();

2
libraries/test-utils/CMakeLists.txt
View file

@ -1,3 +1,3 @@
set(TARGET_NAME test-utils)
setup_hifi_library(Network Gui)
link_hifi_libraries(shared)

0
tests/GLMTestUtils.h → libraries/test-utils/src/test-utils/GLMTestUtils.h
View file

21
tests/QTestExtensions.h → libraries/test-utils/src/test-utils/QTestExtensions.h
View file

@ -313,27 +313,6 @@ inline QString getTestResource(const QString& relativePath) {
return QDir::cleanPath(dir.absoluteFilePath(relativePath));
}
inline bool afterUsecs(quint64& startUsecs, quint64 maxIntervalUecs) {
auto now = usecTimestampNow();
auto interval = now - startUsecs;
if (interval > maxIntervalUecs) {
startUsecs = now;
return true;
}
return false;
}
inline bool afterSecs(quint64& startUsecs, quint64 maxIntervalSecs) {
return afterUsecs(startUsecs, maxIntervalSecs * USECS_PER_SECOND);
}
template <typename F>
void doEvery(quint64& lastReportUsecs, quint64 secs, F lamdba) {
if (afterSecs(lastReportUsecs, secs)) {
lamdba();
}
}
inline void failAfter(quint64 startUsecs, quint64 secs, const char* message) {
if (afterSecs(startUsecs, secs)) {
QFAIL(message);

6
plugins/openvr/src/OpenVrDisplayPlugin.cpp
View file

@ -198,9 +198,9 @@ public:
std::string fsSource = HMD_REPROJECTION_FRAG;
GLuint vertexShader { 0 }, fragmentShader { 0 };
std::string error;
std::vector<char> binary;
::gl::compileShader(GL_VERTEX_SHADER, vsSource, "", vertexShader, error);
::gl::compileShader(GL_FRAGMENT_SHADER, fsSource, "", fragmentShader, error);
::gl::CachedShader binary;
::gl::compileShader(GL_VERTEX_SHADER, vsSource, vertexShader, error);
::gl::compileShader(GL_FRAGMENT_SHADER, fsSource, fragmentShader, error);
_program = ::gl::compileProgram({ { vertexShader, fragmentShader } }, error, binary);
glDeleteShader(vertexShader);
glDeleteShader(fragmentShader);

6
tests-manual/shaders/src/main.cpp
View file

@ -134,12 +134,12 @@ const std::string PIXEL_SHADER_DEFINES{ R"GLSL(
void testShaderBuild(const std::string& vs_src, const std::string& fs_src) {
std::string error;
std::vector<char> binary;
GLuint vs, fs;
if (!gl::compileShader(GL_VERTEX_SHADER, vs_src, VERTEX_SHADER_DEFINES, vs, error) ||
!gl::compileShader(GL_FRAGMENT_SHADER, fs_src, PIXEL_SHADER_DEFINES, fs, error)) {
if (!gl::compileShader(GL_VERTEX_SHADER, VERTEX_SHADER_DEFINES + vs_src, vs, error) ||
!gl::compileShader(GL_FRAGMENT_SHADER, PIXEL_SHADER_DEFINES + fs_src, fs, error)) {
throw std::runtime_error("Failed to compile shader");
}
gl::CachedShader binary;
auto pr = gl::compileProgram({ vs, fs }, error, binary);
if (!pr) {
throw std::runtime_error("Failed to link shader");

2
tests/animation/CMakeLists.txt
View file

@ -1,7 +1,7 @@
# Declare dependencies
macro (setup_testcase_dependencies)
# link in the shared libraries
link_hifi_libraries(shared animation gpu fbx graphics networking)
link_hifi_libraries(shared animation gpu fbx graphics networking test-utils)
package_libraries_for_deployment()
endmacro ()

2
tests/animation/src/AnimInverseKinematicsTests.cpp
View file

@ -16,7 +16,7 @@
#include <AnimationLogging.h>
#include <NumericalConstants.h>
#include "../QTestExtensions.h"
#include <test-utils/QTestExtensions.h>
QTEST_MAIN(AnimInverseKinematicsTests)

2
tests/animation/src/AnimTests.cpp
View file

@ -20,7 +20,7 @@
#include <AccountManager.h>
#include <ResourceManager.h>
#include <StatTracker.h>
#include <../QTestExtensions.h>
#include <test-utils/QTestExtensions.h>
QTEST_MAIN(AnimTests)

2
tests/animation/src/RotationConstraintTests.cpp
View file

@ -17,7 +17,7 @@
#include <NumericalConstants.h>
#include <SwingTwistConstraint.h>
#include "../QTestExtensions.h"
#include <test-utils/QTestExtensions.h>
QTEST_MAIN(RotationConstraintTests)

287
tests/gpu/src/ShaderLoadTest.cpp Normal file
View file

@ -0,0 +1,287 @@
//
// Created by Bradley Austin Davis on 2018/01/11
// Copyright 2014 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "ShaderLoadTest.h"
#include <iostream>
#include <QtCore/QTemporaryFile>
#include <NumericalConstants.h>
#include <gpu/Forward.h>
#include <gl/Config.h>
#include <gl/GLHelpers.h>
#include <gl/GLShaders.h>
#include <gpu/gl/GLShader.h>
#include <gpu/gl/GLBackend.h>
#include <shared/FileUtils.h>
#include <SettingManager.h>
#include <test-utils/QTestExtensions.h>
QTEST_MAIN(ShaderLoadTest)
extern std::atomic<size_t> gpuBinaryShadersLoaded;
extern const QString& getShaderCacheFile();
QtMessageHandler originalHandler;
void messageHandler(QtMsgType type, const QMessageLogContext& context, const QString& message) {
#if defined(Q_OS_WIN)
OutputDebugStringA(message.toStdString().c_str());
OutputDebugStringA("\n");
#endif
originalHandler(type, context, message);
}
std::pair<int, std::vector<std::pair<QString, QString>>> parseCachedShaderString(const QString& cachedShaderString) {
std::pair<int, std::vector<std::pair<QString, QString>>> result;
{
static const QRegularExpression versionRegex("^// VERSION (\\d+)");
auto match = versionRegex.match(cachedShaderString);
result.first = match.captured(1).toInt();
}
int rangeStart = 0;
QString type;
static const QRegularExpression regex("//-------- (\\w+)");
auto match = regex.match(cachedShaderString, rangeStart);
while (match.hasMatch()) {
auto newType = match.captured(1);
auto start = match.capturedStart(0);
auto end = match.capturedEnd(0);
if (rangeStart != 0) {
QString subString = cachedShaderString.mid(rangeStart, start - rangeStart);
result.second.emplace_back(type, subString);
}
rangeStart = end;
type = newType;
match = regex.match(cachedShaderString, rangeStart);
}
if (rangeStart != 0) {
QString subString = cachedShaderString.mid(rangeStart);
result.second.emplace_back(type, subString);
}
return result;
}
std::string getShaderName(const QString& shader) {
static const QRegularExpression nameExp("//\\s+(\\w+\\.(?:vert|frag))");
auto match = nameExp.match(shader);
if (!match.hasMatch()) {
return (QCryptographicHash::hash(shader.toUtf8(), QCryptographicHash::Md5).toHex() + ".shader").toStdString();
}
return match.captured(1).trimmed().toStdString();
}
void ShaderLoadTest::randomizeShaderSources() {
for (auto& entry : _shaderSources) {
entry.second += ("\n//" + QUuid::createUuid().toString()).toStdString();
}
}
#if USE_LOCAL_SHADERS
const QString SHADER_CACHE_FILENAME = "c:/Users/bdavi/AppData/Local/High Fidelity - dev/Interface/shaders/cache.json";
static const QString SHADER_FOLDER = "D:/shaders/";
void ShaderLoadTest::parseCacheDirectory() {
for (const auto& shaderFile : QDir(SHADER_FOLDER).entryList(QDir::Files)) {
QString shaderSource = FileUtils::readFile(SHADER_FOLDER + "/" + shaderFile);
_shaderSources[shaderFile.trimmed().toStdString()] = shaderSource.toStdString();
}
auto programsDoc = QJsonDocument::fromJson(FileUtils::readFile(SHADER_FOLDER + "programs.json").toUtf8());
for (const auto& programElement : programsDoc.array()) {
auto programObj = programElement.toObject();
QString vertexSource = programObj["vertex"].toString();
QString pixelSource = programObj["pixel"].toString();
_programs.insert({ vertexSource.toStdString(), pixelSource.toStdString() });
}
}
void ShaderLoadTest::persistCacheDirectory() {
for (const auto& shaderFile : QDir(SHADER_FOLDER).entryList(QDir::Files)) {
QFile(SHADER_FOLDER + "/" + shaderFile).remove();
}
// Write the shader source files
for (const auto& entry : _shaderSources) {
const QString name = entry.first.c_str();
const QString shader = entry.second.c_str();
QString fullFile = SHADER_FOLDER + name;
QVERIFY(!QFileInfo(fullFile).exists());
QFile shaderFile(fullFile);
shaderFile.open(QIODevice::WriteOnly);
shaderFile.write(shader.toUtf8());
shaderFile.close();
}
// Write the list of programs
{
QVariantList programsList;
for (const auto& program : _programs) {
QVariantMap programMap;
programMap["vertex"] = program.first.c_str();
programMap["pixel"] = program.second.c_str();
programsList.push_back(programMap);
}
QFile saveFile(SHADER_FOLDER + "programs.json");
saveFile.open(QFile::WriteOnly | QFile::Text | QFile::Truncate);
saveFile.write(QJsonDocument::fromVariant(programsList).toJson(QJsonDocument::Indented));
saveFile.close();
}
}
#else
const QString SHADER_CACHE_FILENAME = ":cache.json";
#endif
void ShaderLoadTest::parseCacheFile() {
QString json = FileUtils::readFile(SHADER_CACHE_FILENAME);
auto root = QJsonDocument::fromJson(json.toUtf8()).object();
_programs.clear();
_programs.reserve(root.size());
const auto keys = root.keys();
Program program;
for (auto shaderKey : keys) {
auto cacheEntry = root[shaderKey].toObject();
auto source = cacheEntry["source"].toString();
auto shaders = parseCachedShaderString(source);
for (const auto& entry : shaders.second) {
const auto& type = entry.first;
const auto& source = entry.second;
const auto name = getShaderName(source);
if (name.empty()) {
continue;
}
if (0 == _shaderSources.count(name)) {
_shaderSources[name] = source.toStdString();
}
if (type == "vertex") {
program.first = name;
} else if (type == "pixel") {
program.second = name;
}
}
// FIXME support geometry / tesselation shaders eventually
if (program.first.empty() || program.second.empty()) {
qFatal("Bad Shader Setup");
}
_programs.insert(program);
}
}
bool ShaderLoadTest::buildProgram(const Program& programFiles) {
const auto& vertexName = programFiles.first;
const auto& vertexSource = _shaderSources[vertexName];
auto vertexShader = gpu::Shader::createVertex({ vertexSource });
const auto& pixelName = programFiles.second;
const auto& pixelSource = _shaderSources[pixelName];
auto pixelShader = gpu::Shader::createPixel({ pixelSource });
auto program = gpu::Shader::createProgram(vertexShader, pixelShader);
return gpu::gl::GLBackend::makeProgram(*program, {}, {});
}
void ShaderLoadTest::initTestCase() {
originalHandler = qInstallMessageHandler(messageHandler);
DependencyManager::set<Setting::Manager>();
{
const auto& shaderCacheFile = getShaderCacheFile();
if (QFileInfo(shaderCacheFile).exists()) {
QFile(shaderCacheFile).remove();
}
}
// For local debugging
#if USE_LOCAL_SHADERS
parseCacheFile();
persistCacheDirectory();
parseCacheDirectory();
#else
parseCacheFile();
#endif
// We use this to defeat shader caching both by the GPU backend
// and the OpenGL driver
randomizeShaderSources();
QVERIFY(!_programs.empty());
for (const auto& program : _programs) {
QVERIFY(_shaderSources.count(program.first) == 1);
QVERIFY(_shaderSources.count(program.second) == 1);
}
getDefaultOpenGLSurfaceFormat();
_canvas.create();
if (!_canvas.makeCurrent()) {
qFatal("Unable to make test GL context current");
}
gl::initModuleGl();
gpu::Context::init<gpu::gl::GLBackend>();
_canvas.makeCurrent();
}
void ShaderLoadTest::cleanupTestCase() {
DependencyManager::destroy<Setting::Manager>();
}
void ShaderLoadTest::testShaderLoad() {
auto gpuContext = std::make_shared<gpu::Context>();
QVERIFY(gpuBinaryShadersLoaded == 0);
QElapsedTimer timer;
// Initial load of all the shaders
// No caching
{
timer.start();
for (const auto& program : _programs) {
QVERIFY(buildProgram(program));
}
qDebug() << "Uncached shader load took" << timer.elapsed() << "ms";
QVERIFY(gpuBinaryShadersLoaded == 0);
}
gpuContext->recycle();
glFinish();
// Reload the shaders within the same GPU context lifetime.
// Shaders will use the cached binaries in memory
{
timer.start();
for (const auto& program : _programs) {
QVERIFY(buildProgram(program));
}
qDebug() << "Cached shader load took" << timer.elapsed() << "ms";
QVERIFY(gpuBinaryShadersLoaded == _programs.size() * gpu::gl::GLShader::NumVersions);
}
// Simulate reloading the shader cache from disk by destroying and recreating the gpu context
// Shaders will use the cached binaries from disk
{
gpuBinaryShadersLoaded = 0;
gpuContext->recycle();
gpuContext->shutdown();
gpuContext.reset();
gpuContext = std::make_shared<gpu::Context>();
_canvas.makeCurrent();
timer.start();
for (const auto& program : _programs) {
QVERIFY(buildProgram(program));
}
qDebug() << "Cached shader load took" << timer.elapsed() << "ms";
QVERIFY(gpuBinaryShadersLoaded == _programs.size() * gpu::gl::GLShader::NumVersions);
}
}

63
tests/gpu/src/ShaderLoadTest.h Normal file
View file

@ -0,0 +1,63 @@
//
// Created by Bradley Austin Davis on 2018/05/08
// Copyright 2013-2018 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#pragma once
#include <unordered_map>
#include <unordered_set>
#include <QtTest/QtTest>
#include <QtCore/QTemporaryDir>
#include <gpu/Forward.h>
#include <gl/OffscreenGLCanvas.h>
#define USE_LOCAL_SHADERS 0
namespace std {
template <>
struct hash<std::pair<std::string, std::string>> {
size_t operator()(const std::pair<std::string, std::string>& a) const {
std::hash<std::string> hasher;
return hasher(a.first) + hasher(a.second);
}
};
}
using ShadersByName = std::unordered_map<std::string, std::string>;
using Program = std::pair<std::string, std::string>;
using Programs = std::unordered_set<Program>;
class ShaderLoadTest : public QObject {
Q_OBJECT
private:
void parseCacheFile();
#if USE_LOCAL_SHADERS
void parseCacheDirectory();
void persistCacheDirectory();
#endif
bool buildProgram(const Program& program);
void randomizeShaderSources();
private slots:
void initTestCase();
void cleanupTestCase();
void testShaderLoad();
private:
ShadersByName _shaderSources;
Programs _programs;
QString _resourcesPath;
OffscreenGLCanvas _canvas;
const glm::uvec2 _size{ 640, 480 };
};

2
tests/gpu/src/TextureTest.cpp
View file

@ -21,7 +21,7 @@
#include <quazip5/quazip.h>
#include <quazip5/JlCompress.h>
#include "../../QTestExtensions.h"
#include <test-utils/QTestExtensions.h>
QTEST_MAIN(TextureTest)

512
tests/gpu/src/cache.json Normal file
View file

File diff suppressed because one or more lines are too long

6
tests/gpu/src/data.qrc Normal file
View file

@ -0,0 +1,6 @@
<!DOCTYPE RCC>
<RCC version="1.0">
<qresource>
<file>cache.json</file>
</qresource>
</RCC>

2
tests/networking/CMakeLists.txt
View file

@ -2,7 +2,7 @@
# Declare dependencies
macro (setup_testcase_dependencies)
# link in the shared libraries
link_hifi_libraries(shared networking)
link_hifi_libraries(shared test-utils networking)
package_libraries_for_deployment()
endmacro ()

2
tests/networking/src/PacketTests.cpp
View file

@ -10,7 +10,7 @@
//
#include "PacketTests.h"
#include "../QTestExtensions.h"
#include <test-utils/QTestExtensions.h>
#include <NLPacket.h>

2
tests/octree/CMakeLists.txt
View file

@ -2,7 +2,7 @@
# Declare dependencies
macro (setup_testcase_dependencies)
# link in the shared libraries
link_hifi_libraries(shared octree gpu graphics fbx networking entities avatars audio animation script-engine physics)
link_hifi_libraries(shared test-utils octree gpu graphics fbx networking entities avatars audio animation script-engine physics)
package_libraries_for_deployment()
endmacro ()

4
tests/octree/src/ViewFrustumTests.cpp
View file

@ -18,8 +18,8 @@
#include <ViewFrustum.h>
//#include <StreamUtils.h>
#include <../GLMTestUtils.h>
#include <../QTestExtensions.h>
#include <test-utils/GLMTestUtils.h>
#include <test-utils/QTestExtensions.h>
const float ACCEPTABLE_FLOAT_ERROR = 1.0e-6f;

2
tests/physics/CMakeLists.txt
View file

@ -2,7 +2,7 @@
# Declare dependencies
macro (SETUP_TESTCASE_DEPENDENCIES)
target_bullet()
link_hifi_libraries(shared physics gpu graphics)
link_hifi_libraries(shared test-utils physics gpu graphics)
package_libraries_for_deployment()
endmacro ()

2
tests/physics/src/BulletUtilTests.cpp
View file

@ -18,7 +18,7 @@
#include <GLMHelpers.h>
// Add additional qtest functionality (the include order is important!)
#include "../QTestExtensions.h"
#include <test-utils/QTestExtensions.h>
// Constants
const glm::vec3 origin(0.0f);

2
tests/physics/src/MeshMassPropertiesTests.cpp
View file

@ -16,7 +16,7 @@
// Add additional qtest functionality (the include order is important!)
#include "BulletTestUtils.h"
#include "../QTestExtensions.h"
#include <test-utils/QTestExtensions.h>
const btScalar acceptableRelativeError(1.0e-5f);
const btScalar acceptableAbsoluteError(1.0e-4f);

2
tests/shared/CMakeLists.txt
View file

@ -3,7 +3,7 @@
macro (setup_testcase_dependencies)
# link in the shared libraries
link_hifi_libraries(shared)
link_hifi_libraries(shared test-utils)
package_libraries_for_deployment()
endmacro ()

4
tests/shared/src/AABoxTests.cpp
View file

@ -17,8 +17,8 @@
#include <NumericalConstants.h>
#include <StreamUtils.h>
#include <../GLMTestUtils.h>
#include <../QTestExtensions.h>
#include <test-utils/GLMTestUtils.h>
#include <test-utils/QTestExtensions.h>
QTEST_MAIN(AABoxTests)

4
tests/shared/src/AACubeTests.cpp
View file

@ -17,8 +17,8 @@
#include <NumericalConstants.h>
#include <StreamUtils.h>
#include <../GLMTestUtils.h>
#include <../QTestExtensions.h>
#include <test-utils/GLMTestUtils.h>
#include <test-utils/QTestExtensions.h>
QTEST_MAIN(AACubeTests)

2
tests/shared/src/BitVectorHelperTests.cpp
View file

@ -12,7 +12,7 @@
#include <SharedLogging.h>
#include "../QTestExtensions.h"
#include <test-utils/QTestExtensions.h>
#include <QtCore/QDebug>
#include <StreamUtils.h>
#include <glm/glm.hpp>

2
tests/shared/src/CubicHermiteSplineTests.cpp
View file

@ -9,7 +9,7 @@
//
#include "CubicHermiteSplineTests.h"
#include "../QTestExtensions.h"
#include <test-utils/QTestExtensions.h>
#include <QtCore/QDebug>
#include "CubicHermiteSpline.h"

4
tests/shared/src/DualQuaternionTests.cpp
View file

@ -17,8 +17,8 @@
#include <NumericalConstants.h>
#include <StreamUtils.h>
#include <../GLMTestUtils.h>
#include <../QTestExtensions.h>
#include <test-utils/GLMTestUtils.h>
#include <test-utils/QTestExtensions.h>
QTEST_MAIN(DualQuaternionTests)

2
tests/shared/src/GLMHelpersTests.cpp
View file

@ -14,7 +14,7 @@
#include <NumericalConstants.h>
#include <StreamUtils.h>
#include <../QTestExtensions.h>
#include <test-utils/QTestExtensions.h>
#include <glm/gtc/matrix_transform.hpp>
#include <glm/simd/matrix.h>

3
tests/shared/src/GeometryUtilTests.cpp
View file

@ -18,7 +18,8 @@
#include <NumericalConstants.h>
#include <StreamUtils.h>
#include <../QTestExtensions.h>
#include <test-utils/GLMTestUtils.h>
#include <test-utils/QTestExtensions.h>
QTEST_MAIN(GeometryUtilTests)

3
tests/shared/src/MovingMinMaxAvgTests.cpp
View file

@ -17,7 +17,8 @@
#include <NumericalConstants.h>
#include <SharedUtil.h>
#include "../QTestExtensions.h"
#include <test-utils/GLMTestUtils.h>
#include <test-utils/QTestExtensions.h>
QTEST_MAIN(MovingMinMaxAvgTests)

5
tests/shared/src/MovingPercentileTests.cpp
View file

@ -15,8 +15,9 @@
#include "MovingPercentile.h"
#include <limits>
#include <qqueue.h>
#include <../QTestExtensions.h>
#include <QtCore/QQueue>
#include <test-utils/GLMTestUtils.h>
#include <test-utils/QTestExtensions.h>
QTEST_MAIN(MovingPercentileTests)

2
tests/shared/src/TraceTests.cpp
View file

@ -14,7 +14,7 @@
#include <Profile.h>
#include <NumericalConstants.h>
#include <../QTestExtensions.h>
#include <test-utils/QTestExtensions.h>
QTEST_MAIN(TraceTests)
Q_LOGGING_CATEGORY(trace_test, "trace.test")

2
tests/shared/src/TransformTests.cpp
View file

@ -14,7 +14,7 @@
#include <SharedLogging.h>
#include "../QTestExtensions.h"
#include <test-utils/QTestExtensions.h>
#include <QtCore/QDebug>
#include <Transform.h>
#include <StreamUtils.h>