From 68d2f9581ff6bab383fbbff1f37774e2464ac0e6 Mon Sep 17 00:00:00 2001 From: SamGondelman Date: Mon, 3 Jun 2019 19:16:42 -0700 Subject: [PATCH] add ability for slp files to define shader variants --- cmake/macros/AutoScribeShader.cmake | 160 +++++++++++++++++++++++++--- tools/shadergen.py | 46 +++++--- 2 files changed, 178 insertions(+), 28 deletions(-) diff --git a/cmake/macros/AutoScribeShader.cmake b/cmake/macros/AutoScribeShader.cmake index c5bb2b4054..9a18005424 100755 --- a/cmake/macros/AutoScribeShader.cmake +++ b/cmake/macros/AutoScribeShader.cmake @@ -66,6 +66,9 @@ macro(AUTOSCRIBE_PLATFORM_SHADER) list(APPEND SHADER_GEN_LINE ${TEMP_PATH}) file(RELATIVE_PATH TEMP_PATH ${CMAKE_SOURCE_DIR} ${AUTOSCRIBE_OUTPUT_FILE}) list(APPEND SHADER_GEN_LINE ${TEMP_PATH}) + if (NOT("${DEFINES}" STREQUAL "")) + list(APPEND SHADER_GEN_LINE "defines:${DEFINES}") + endif() list(APPEND SHADER_GEN_LINE ${AUTOSCRIBE_SHADER_SEEN_LIBS}) string(CONCAT AUTOSCRIBE_SHADERGEN_COMMANDS "${AUTOSCRIBE_SHADERGEN_COMMANDS}" "${SHADER_GEN_LINE}\n") endmacro() @@ -108,6 +111,10 @@ macro(AUTOSCRIBE_SHADER) set(SHADER_TYPE geom) endif() + if (NOT("${DEFINES}" STREQUAL "")) + string(CONCAT SHADER_NAME "${SHADER_NAME}" "_${DEFINES}") + endif() + set(SCRIBE_ARGS -D GLPROFILE ${GLPROFILE} -T ${SHADER_TYPE} ${SCRIBE_INCLUDES} ) # SHADER_SCRIBED -> the output of scribe @@ -135,11 +142,72 @@ macro(AUTOSCRIBE_SHADER) endif() endif() - string(CONCAT SHADER_ENUMS "${SHADER_ENUMS}" "${SHADER_NAME} = ${SHADER_COUNT},\n") + string(CONCAT SHADER_LIST "${SHADER_LIST}" "${SHADER_NAME} = ${SHADER_COUNT},\n") string(CONCAT SHADER_SHADERS_ARRAY "${SHADER_SHADERS_ARRAY}" "${SHADER_COUNT},\n") MATH(EXPR SHADER_COUNT "${SHADER_COUNT}+1") endmacro() +function(GENERATE_DEFINES_LIST_HELPER INPUT_LIST RETURN_LIST) + string(LENGTH "${INPUT_LIST}" STR_LENGTH) + set(OPEN_INDEX -1) + set(STR_INDEX 0) + set(NESTED_DEPTH 0) + while ("${STR_INDEX}" LESS "${STR_LENGTH}") + string(SUBSTRING "${INPUT_LIST}" ${STR_INDEX} 1 CURRENT_CHAR) + + if (("${CURRENT_CHAR}" STREQUAL "(") AND (OPEN_INDEX EQUAL -1)) + set(OPEN_INDEX ${STR_INDEX}) + MATH(EXPR STR_INDEX "${STR_INDEX}+1") + continue() + elseif (("${CURRENT_CHAR}" STREQUAL "(") AND NOT(OPEN_INDEX EQUAL -1)) + MATH(EXPR NESTED_DEPTH "${NESTED_DEPTH}+1") + MATH(EXPR STR_INDEX "${STR_INDEX}+1") + continue() + elseif (("${CURRENT_CHAR}" STREQUAL ")") AND NOT(OPEN_INDEX EQUAL -1) AND (NESTED_DEPTH GREATER 0)) + MATH(EXPR NESTED_DEPTH "${NESTED_DEPTH}-1") + MATH(EXPR STR_INDEX "${STR_INDEX}+1") + continue() + elseif (("${CURRENT_CHAR}" STREQUAL ")") AND NOT(OPEN_INDEX EQUAL -1) AND (NESTED_DEPTH EQUAL 0)) + MATH(EXPR OPEN_INDEX "${OPEN_INDEX}+1") + MATH(EXPR SUBSTR_LENGTH "${STR_INDEX}-${OPEN_INDEX}") + string(SUBSTRING "${INPUT_LIST}" ${OPEN_INDEX} ${SUBSTR_LENGTH} GROUP_STR) + GENERATE_DEFINES_LIST_HELPER("${GROUP_STR}" EXPANDED_GROUP_LIST) + string(REPLACE ";" "/" EXPANDED_GROUP_LIST "${EXPANDED_GROUP_LIST}") + string(REPLACE "(${GROUP_STR})" "${EXPANDED_GROUP_LIST}" INPUT_LIST "${INPUT_LIST}") + MATH(EXPR STR_INDEX "${OPEN_INDEX}-1") + set(OPEN_INDEX -1) + string(LENGTH "${INPUT_LIST}" STR_LENGTH) + continue() + endif() + + MATH(EXPR STR_INDEX "${STR_INDEX}+1") + endwhile() + + list(LENGTH INPUT_LIST NUM_DEFINES) + if (NUM_DEFINES EQUAL 1) + string(REPLACE "/" ";" INPUT_LIST "${INPUT_LIST}") + set(${RETURN_LIST} ${INPUT_LIST} PARENT_SCOPE) + elseif (NUM_DEFINES GREATER 1) + list(GET INPUT_LIST 0 CURRENT_DEFINES) + string(REPLACE "/" ";" CURRENT_DEFINES "${CURRENT_DEFINES}") + list(REMOVE_AT INPUT_LIST 0) + GENERATE_DEFINES_LIST_HELPER("${INPUT_LIST}" REMAINING_DEFINES_LIST) + set(TO_RETURN_LIST "${CURRENT_DEFINES}") + foreach(REMAINING_DEFINES ${REMAINING_DEFINES_LIST}) + list(APPEND TO_RETURN_LIST "${REMAINING_DEFINES}") + foreach(CURRENT_DEFINE ${CURRENT_DEFINES}) + list(APPEND TO_RETURN_LIST "${CURRENT_DEFINE}_${REMAINING_DEFINES}") + endforeach() + endforeach() + set(${RETURN_LIST} ${TO_RETURN_LIST} PARENT_SCOPE) + endif() +endfunction() + +macro(GENERATE_DEFINES_LIST) + set(DEFINES_LIST "") + GENERATE_DEFINES_LIST_HELPER("${ARGV0}" DEFINES_LIST) +endmacro() + macro(AUTOSCRIBE_SHADER_LIB) if (NOT ("${TARGET_NAME}" STREQUAL "shaders")) message(FATAL_ERROR "AUTOSCRIBE_SHADER_LIB can only be used by the shaders library") @@ -164,24 +232,24 @@ macro(AUTOSCRIBE_SHADER_LIB) if (SHADER_VERTEX_FILES) source_group("${SHADER_LIB}/Vertex" FILES ${SHADER_VERTEX_FILES}) list(APPEND ALL_SCRIBE_SHADERS ${SHADER_VERTEX_FILES}) - string(CONCAT SHADER_ENUMS "${SHADER_ENUMS}" "namespace vertex { enum {\n") - foreach(SHADER_FILE ${SHADER_VERTEX_FILES}) + set(SHADER_LIST "namespace vertex { enum {\n") + foreach(SHADER_FILE ${SHADER_VERTEX_FILES}) AUTOSCRIBE_SHADER(${ALL_SHADER_HEADERS}) endforeach() - string(CONCAT SHADER_ENUMS "${SHADER_ENUMS}" "}; } // vertex \n") + set(VERTEX_ENUMS "${SHADER_LIST}") endif() file(GLOB_RECURSE SHADER_FRAGMENT_FILES ${SRC_FOLDER}/*.slf) if (SHADER_FRAGMENT_FILES) source_group("${SHADER_LIB}/Fragment" FILES ${SHADER_FRAGMENT_FILES}) list(APPEND ALL_SCRIBE_SHADERS ${SHADER_FRAGMENT_FILES}) - string(CONCAT SHADER_ENUMS "${SHADER_ENUMS}" "namespace fragment { enum {\n") - foreach(SHADER_FILE ${SHADER_FRAGMENT_FILES}) + set(SHADER_LIST "namespace fragment { enum {\n") + foreach(SHADER_FILE ${SHADER_FRAGMENT_FILES}) AUTOSCRIBE_SHADER(${ALL_SHADER_HEADERS}) endforeach() - string(CONCAT SHADER_ENUMS "${SHADER_ENUMS}" "}; } // fragment \n") + set(FRAGMENT_ENUMS "${SHADER_LIST}") endif() - + # FIXME add support for geometry, compute and tesselation shaders #file(GLOB_RECURSE SHADER_GEOMETRY_FILES ${SRC_FOLDER}/*.slg) #file(GLOB_RECURSE SHADER_COMPUTE_FILES ${SRC_FOLDER}/*.slc) @@ -191,13 +259,13 @@ macro(AUTOSCRIBE_SHADER_LIB) if (SHADER_PROGRAM_FILES) source_group("${SHADER_LIB}/Program" FILES ${SHADER_PROGRAM_FILES}) list(APPEND ALL_SCRIBE_SHADERS ${SHADER_PROGRAM_FILES}) - string(CONCAT SHADER_ENUMS "${SHADER_ENUMS}" "namespace program { enum {\n") + set(PROGRAM_ENUMS "namespace program { enum {\n") foreach(PROGRAM_FILE ${SHADER_PROGRAM_FILES}) get_filename_component(PROGRAM_NAME ${PROGRAM_FILE} NAME_WE) - set(AUTOSCRIBE_PROGRAM_FRAGMENT ${PROGRAM_NAME}) file(READ ${PROGRAM_FILE} PROGRAM_CONFIG) set(AUTOSCRIBE_PROGRAM_VERTEX ${PROGRAM_NAME}) set(AUTOSCRIBE_PROGRAM_FRAGMENT ${PROGRAM_NAME}) + set(AUTOSCRIBE_PROGRAM_DEFINES "") if (NOT("${PROGRAM_CONFIG}" STREQUAL "")) string(REGEX MATCH ".*VERTEX +([_\\:A-Z0-9a-z]+)" MVERT ${PROGRAM_CONFIG}) @@ -208,6 +276,12 @@ macro(AUTOSCRIBE_SHADER_LIB) if (CMAKE_MATCH_1) set(AUTOSCRIBE_PROGRAM_FRAGMENT ${CMAKE_MATCH_1}) endif() + string(REGEX MATCH ".*DEFINES +([a-zA-Z\(\)/: ]+)" MDEF ${PROGRAM_CONFIG}) + if (CMAKE_MATCH_1) + set(AUTOSCRIBE_PROGRAM_DEFINES ${CMAKE_MATCH_1}) + string(TOLOWER AUTOSCRIBE_PROGRAM_DEFINES "${AUTOSCRIBE_PROGRAM_DEFINES}") + string(REGEX REPLACE " +" ";" AUTOSCRIBE_PROGRAM_DEFINES "${AUTOSCRIBE_PROGRAM_DEFINES}") + endif() endif() if (NOT (${AUTOSCRIBE_PROGRAM_VERTEX} MATCHES ".*::.*")) @@ -216,12 +290,72 @@ macro(AUTOSCRIBE_SHADER_LIB) if (NOT (${AUTOSCRIBE_PROGRAM_FRAGMENT} MATCHES ".*::.*")) set(AUTOSCRIBE_PROGRAM_FRAGMENT "fragment::${AUTOSCRIBE_PROGRAM_FRAGMENT}") endif() + string(REGEX REPLACE ".*::" "" VERTEX_NAME "${AUTOSCRIBE_PROGRAM_VERTEX}") + string(REGEX REPLACE ".*::" "" FRAGMENT_NAME "${AUTOSCRIBE_PROGRAM_FRAGMENT}") - set(PROGRAM_ENTRY "${PROGRAM_NAME} = (${AUTOSCRIBE_PROGRAM_VERTEX} << 16) | ${AUTOSCRIBE_PROGRAM_FRAGMENT},\n") - string(CONCAT SHADER_ENUMS "${SHADER_ENUMS}" "${PROGRAM_ENTRY}") + GENERATE_DEFINES_LIST("${AUTOSCRIBE_PROGRAM_DEFINES}") + + string(CONCAT PROGRAM_ENUMS "${PROGRAM_ENUMS}" "${PROGRAM_NAME} = (${AUTOSCRIBE_PROGRAM_VERTEX} << 16) | ${AUTOSCRIBE_PROGRAM_FRAGMENT},\n") string(CONCAT SHADER_PROGRAMS_ARRAY "${SHADER_PROGRAMS_ARRAY} ${SHADER_NAMESPACE}::program::${PROGRAM_NAME},\n") + + foreach(DEFINES ${DEFINES_LIST}) + set(ORIG_DEFINES "${DEFINES}") + + string(REPLACE ":v" "" VERTEX_DEFINES "${ORIG_DEFINES}") + string(FIND "${ORIG_DEFINES}" ":f" HAS_FRAGMENT) + if (HAS_FRAGMENT EQUAL -1) + set(DEFINES "${VERTEX_DEFINES}") + set(SHADER_LIST "") + set(SHADER_FILE "${SRC_FOLDER}/${VERTEX_NAME}.slv") + AUTOSCRIBE_SHADER(${ALL_SHADER_HEADERS}) + string(CONCAT VERTEX_ENUMS "${VERTEX_ENUMS}" "${SHADER_LIST}") + else() + string(REGEX REPLACE "_*[^_]*:f" "" VERTEX_DEFINES "${VERTEX_DEFINES}") + endif() + + if (NOT("${VERTEX_DEFINES}" STREQUAL "") AND NOT("${VERTEX_DEFINES}" MATCHES "^_.*")) + set(VERTEX_DEFINES "_${VERTEX_DEFINES}") + endif() + + string(REPLACE ":f" "" FRAGMENT_DEFINES "${ORIG_DEFINES}") + string(FIND "${ORIG_DEFINES}" ":v" HAS_VERTEX) + if (HAS_VERTEX EQUAL -1) + set(DEFINES "${FRAGMENT_DEFINES}") + set(SHADER_LIST "") + set(SHADER_FILE "${SRC_FOLDER}/${FRAGMENT_NAME}.slf") + AUTOSCRIBE_SHADER(${ALL_SHADER_HEADERS}) + string(CONCAT FRAGMENT_ENUMS "${FRAGMENT_ENUMS}" "${SHADER_LIST}") + else() + string(REGEX REPLACE "_*[^_]*:v" "" FRAGMENT_DEFINES "${FRAGMENT_DEFINES}") + endif() + + if (NOT("${FRAGMENT_DEFINES}" STREQUAL "") AND NOT("${FRAGMENT_DEFINES}" MATCHES "^_.*")) + set(FRAGMENT_DEFINES "_${FRAGMENT_DEFINES}") + endif() + + string(REGEX REPLACE ":(f|v)" "" PROGRAM_DEFINES "${ORIG_DEFINES}") + + if (NOT("${PROGRAM_DEFINES}" STREQUAL "")) + string(CONCAT PROGRAM_ENUMS "${PROGRAM_ENUMS}" "${PROGRAM_NAME}_${PROGRAM_DEFINES} = (${AUTOSCRIBE_PROGRAM_VERTEX}${VERTEX_DEFINES} << 16) | ${AUTOSCRIBE_PROGRAM_FRAGMENT}${FRAGMENT_DEFINES},\n") + string(CONCAT SHADER_PROGRAMS_ARRAY "${SHADER_PROGRAMS_ARRAY} ${SHADER_NAMESPACE}::program::${PROGRAM_NAME}_${PROGRAM_DEFINES},\n") + endif() + endforeach() endforeach() - string(CONCAT SHADER_ENUMS "${SHADER_ENUMS}" "}; } // program \n") + endif() + + if (SHADER_VERTEX_FILES) + string(CONCAT VERTEX_ENUMS "${VERTEX_ENUMS}" "}; } // vertex \n") + string(CONCAT SHADER_ENUMS "${SHADER_ENUMS}" "${VERTEX_ENUMS}") + endif() + + if (SHADER_FRAGMENT_FILES) + string(CONCAT FRAGMENT_ENUMS "${FRAGMENT_ENUMS}" "}; } // fragment \n") + string(CONCAT SHADER_ENUMS "${SHADER_ENUMS}" "${FRAGMENT_ENUMS}") + endif() + + if (SHADER_PROGRAM_FILES) + string(CONCAT PROGRAM_ENUMS "${PROGRAM_ENUMS}" "}; } // program \n") + string(CONCAT SHADER_ENUMS "${SHADER_ENUMS}" "${PROGRAM_ENUMS}") endif() # Finish the shader enums diff --git a/tools/shadergen.py b/tools/shadergen.py index f82b471f17..1f4acae915 100644 --- a/tools/shadergen.py +++ b/tools/shadergen.py @@ -66,7 +66,6 @@ def getExtensionsHeader(dialect, variant, extensions): extensionsHeaderMutex.release() return extensionHeader - def getDialectAndVariantHeaders(dialect, variant, extensions=None): result = [] headerPath = args.source_dir + '/libraries/shaders/headers/' @@ -80,6 +79,14 @@ def getDialectAndVariantHeaders(dialect, variant, extensions=None): result.append(variantHeader) return result +def getDefines(defines): + definesList = [] + if defines: + definesSplit = defines.split("_") + for define in definesSplit: + definesList.append('HIFI_USE_{} 1'.format(define.upper())) + return definesList + class ScribeDependenciesCache: cache = {} lock = Lock() @@ -99,9 +106,9 @@ class ScribeDependenciesCache: with open(self.filename, "w") as f: f.write(json.dumps(self.cache)) - def get(self, scribefile, dialect, variant): + def get(self, scribefile, dialect, variant, defines): self.lock.acquire() - key = self.key(scribefile, dialect, variant) + key = self.key(scribefile, dialect, variant, defines) try: if key in self.cache: return self.cache[key].copy() @@ -109,25 +116,26 @@ class ScribeDependenciesCache: self.lock.release() return None - def key(self, scribeFile, dialect, variant): - return ':'.join([scribeFile, dialect, variant]) + def key(self, scribeFile, dialect, variant, defines): + return ':'.join([scribeFile, dialect, variant, defines]) - def getOrGen(self, scribefile, includeLibs, dialect, variant): - result = self.get(scribefile, dialect, variant) - if (None == result): - result = self.gen(scribefile, includeLibs, dialect, variant) + def getOrGen(self, scribefile, includeLibs, dialect, variant, defines): + result = self.get(scribefile, dialect, variant, defines) + if result is None: + result = self.gen(scribefile, includeLibs, dialect, variant, defines) return result - def gen(self, scribefile, includeLibs, dialect, variant): + def gen(self, scribefile, includeLibs, dialect, variant, defines): scribeArgs = getCommonScribeArgs(scribefile, includeLibs) scribeArgs.extend(['-M']) processResult = subprocess.run(scribeArgs, stdout=subprocess.PIPE) if (0 != processResult.returncode): - raise RuntimeError("Unable to parse scribe dependencies") + raise RuntimeError("Unable to parse scribe dependencies for file {} with defines: {}".format(scribefile, defines)) result = processResult.stdout.decode("utf-8").splitlines(False) result.append(scribefile) result.extend(getDialectAndVariantHeaders(dialect, variant)) - key = self.key(scribefile, dialect, variant) + result.extend(getDefines(defines)) + key = self.key(scribefile, dialect, variant, defines) self.lock.acquire() self.cache[key] = result.copy() self.lock.release() @@ -166,6 +174,10 @@ def processCommand(line): variant = params.pop(0) scribeFile = args.source_dir + '/' + params.pop(0) unoptGlslFile = args.source_dir + '/' + params.pop(0) + defines = "" + if len(params) > 1 and params[0].startswith("defines:"): + defines = params.pop(0) + defines = defines[len("defines:"):] libs = params upoptSpirvFile = unoptGlslFile + '.spv' @@ -184,19 +196,23 @@ def processCommand(line): os.makedirs(scribeOutputDir) folderMutex.release() - scribeDeps = scribeDepCache.getOrGen(scribeFile, libs, dialect, variant) + scribeDeps = scribeDepCache.getOrGen(scribeFile, libs, dialect, variant, defines) # if the scribe sources (slv, slf, slh, etc), or the dialect/ variant headers are out of date # regenerate the scribe GLSL output if args.force or outOfDate(scribeDeps, outputFiles): - print('Processing file {} dialect {} variant {}'.format(scribeFile, dialect, variant)) + print('Processing file {} dialect {} variant {} defines {}'.format(scribeFile, dialect, variant, defines)) if args.dry_run: return True - scribeDepCache.gen(scribeFile, libs, dialect, variant) + scribeDepCache.gen(scribeFile, libs, dialect, variant, defines) scribeArgs = getCommonScribeArgs(scribeFile, libs) for header in getDialectAndVariantHeaders(dialect, variant, args.extensions): scribeArgs.extend(['-H', header]) + for define in getDefines(defines): + defineArgs = ['-D'] + defineArgs.extend(define.split(' ')) + scribeArgs.extend(defineArgs) scribeArgs.extend(['-o', unoptGlslFile]) executeSubprocess(scribeArgs)