From a39fe7452ce7f828c9925b5358fe4de875cc7756 Mon Sep 17 00:00:00 2001 From: Olivier Prat Date: Wed, 27 Mar 2019 16:31:08 +0100 Subject: [PATCH] Preparing for cubemap convolution --- libraries/image/src/image/CubeMap.cpp | 28 ++++ libraries/image/src/image/CubeMap.h | 43 ++++++ libraries/image/src/image/Image.cpp | 190 ++++++++++++++++++++------ libraries/image/src/image/Image.h | 16 ++- 4 files changed, 231 insertions(+), 46 deletions(-) create mode 100644 libraries/image/src/image/CubeMap.cpp create mode 100644 libraries/image/src/image/CubeMap.h diff --git a/libraries/image/src/image/CubeMap.cpp b/libraries/image/src/image/CubeMap.cpp new file mode 100644 index 0000000000..303cb98fe7 --- /dev/null +++ b/libraries/image/src/image/CubeMap.cpp @@ -0,0 +1,28 @@ +// +// CubeMap.h +// image/src/image +// +// Created by Olivier Prat on 03/27/2019. +// Copyright 2019 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#include "CubeMap.h" + +using namespace image; + +CubeMap::CubeMap(int width, int height, int mipCount) : + _width(width), _height(height) { + assert(mipCount >0 && _width > 0 && _height > 0); + _mips.resize(mipCount); + for (auto mipLevel = 0; mipLevel < mipCount; mipLevel++) { + auto mipWidth = std::max(1, width >> mipLevel); + auto mipHeight = std::max(1, height >> mipLevel); + auto mipPixelCount = mipWidth * mipHeight; + + for (auto& face : _mips[mipLevel]) { + face.resize(mipPixelCount); + } + } +} diff --git a/libraries/image/src/image/CubeMap.h b/libraries/image/src/image/CubeMap.h new file mode 100644 index 0000000000..05a571cafc --- /dev/null +++ b/libraries/image/src/image/CubeMap.h @@ -0,0 +1,43 @@ +// +// CubeMap.h +// image/src/image +// +// Created by Olivier Prat on 03/27/2019. +// Copyright 2019 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_image_CubeMap_h +#define hifi_image_CubeMap_h + +#include +#include +#include +#include + +namespace image { + + class CubeMap { + public: + + using Face = std::vector; + using Faces = std::array; + + CubeMap(int width, int height, int mipCount); + + gpu::uint16 getMipCount() const { return (gpu::uint16)_mips.size(); } + Faces& editMip(gpu::uint16 mipLevel) { return _mips[mipLevel]; } + const Faces& getMip(gpu::uint16 mipLevel) const { return _mips[mipLevel]; } + + private: + + int _width; + int _height; + std::vector _mips; + }; + +} + +#endif // hifi_image_CubeMap_h diff --git a/libraries/image/src/image/Image.cpp b/libraries/image/src/image/Image.cpp index 88ca440908..6e7e08ea89 100644 --- a/libraries/image/src/image/Image.cpp +++ b/libraries/image/src/image/Image.cpp @@ -30,6 +30,7 @@ #include "OpenEXRReader.h" #endif #include "ImageLogging.h" +#include "CubeMap.h" using namespace gpu; @@ -101,12 +102,12 @@ TextureUsage::TextureLoader TextureUsage::getTextureLoaderForType(Type type, con case LIGHTMAP_TEXTURE: return image::TextureUsage::createLightmapTextureFromImage; case SKY_TEXTURE: - return image::TextureUsage::createCubeTextureFromImageWithoutIrradiance; + return image::TextureUsage::createCubeTextureFromImage; case AMBIENT_TEXTURE: if (options.value("generateIrradiance", true).toBool()) { - return image::TextureUsage::createCubeTextureFromImage; + return image::TextureUsage::createCubeTextureAndIrradianceFromImage; } else { - return image::TextureUsage::createCubeTextureFromImageWithoutIrradiance; + return image::TextureUsage::createCubeTextureFromImage; } case BUMP_TEXTURE: return image::TextureUsage::createNormalTextureFromBumpImage; @@ -177,14 +178,24 @@ gpu::TexturePointer TextureUsage::createMetallicTextureFromImage(QImage&& srcIma return process2DTextureGrayscaleFromImage(std::move(srcImage), srcImageName, compress, target, false, abortProcessing); } -gpu::TexturePointer TextureUsage::createCubeTextureFromImage(QImage&& srcImage, const std::string& srcImageName, +gpu::TexturePointer TextureUsage::createCubeTextureAndIrradianceFromImage(QImage&& srcImage, const std::string& srcImageName, bool compress, BackendTarget target, const std::atomic& abortProcessing) { - return processCubeTextureColorFromImage(std::move(srcImage), srcImageName, compress, target, true, abortProcessing); + return processCubeTextureColorFromImage(std::move(srcImage), srcImageName, compress, target, CUBE_GENERATE_IRRADIANCE, abortProcessing); } -gpu::TexturePointer TextureUsage::createCubeTextureFromImageWithoutIrradiance(QImage&& srcImage, const std::string& srcImageName, +gpu::TexturePointer TextureUsage::createCubeTextureFromImage(QImage&& srcImage, const std::string& srcImageName, bool compress, BackendTarget target, const std::atomic& abortProcessing) { - return processCubeTextureColorFromImage(std::move(srcImage), srcImageName, compress, target, false, abortProcessing); + return processCubeTextureColorFromImage(std::move(srcImage), srcImageName, compress, target, CUBE_DEFAULT, abortProcessing); +} + +gpu::TexturePointer TextureUsage::createGGXConvolvedCubeTextureFromImage(QImage&& image, const std::string& srcImageName, + bool compress, gpu::BackendTarget target, const std::atomic& abortProcessing) { + return processCubeTextureColorFromImage(std::move(image), srcImageName, compress, target, CUBE_GGX_CONVOLVE, abortProcessing); +} + +gpu::TexturePointer TextureUsage::createGGXConvolvedCubeTextureAndIrradianceFromImage(QImage&& image, const std::string& srcImageName, + bool compress, gpu::BackendTarget target, const std::atomic& abortProcessing) { + return processCubeTextureColorFromImage(std::move(image), srcImageName, compress, target, CUBE_GENERATE_IRRADIANCE | CUBE_GGX_CONVOLVE, abortProcessing); } static float denormalize(float value, const float minValue) { @@ -206,11 +217,17 @@ static uint32 packR11G11B10F(const glm::vec3& color) { return glm::packF2x11_1x10(ucolor); } -static std::function getHDRPackingFunction(const gpu::Element& format) { +static uint32 packUnorm4x8(const glm::vec3& color) { + return glm::packUnorm4x8(glm::vec4(color, 1.0f)); +} + +static std::function getPackingFunction(const gpu::Element& format) { if (format == gpu::Element::COLOR_RGB9E5) { return glm::packF3x9_E1x5; } else if (format == gpu::Element::COLOR_R11G11B10) { return packR11G11B10F; + } else if (format == gpu::Element::COLOR_RGBA_32 || format == gpu::Element::COLOR_SRGBA_32 || format == gpu::Element::COLOR_BGRA_32 || format == gpu::Element::COLOR_SBGRA_32) { + return packUnorm4x8; } else { qCWarning(imagelogging) << "Unknown handler format"; Q_UNREACHABLE(); @@ -219,21 +236,27 @@ static std::function getHDRPackingFunction(const gpu:: } std::function getHDRPackingFunction() { - return getHDRPackingFunction(HDR_FORMAT); + return getPackingFunction(HDR_FORMAT); } -std::function getHDRUnpackingFunction() { - if (HDR_FORMAT == gpu::Element::COLOR_RGB9E5) { +std::function getUnpackingFunction(const gpu::Element& format) { + if (format == gpu::Element::COLOR_RGB9E5) { return glm::unpackF3x9_E1x5; - } else if (HDR_FORMAT == gpu::Element::COLOR_R11G11B10) { + } else if (format == gpu::Element::COLOR_R11G11B10) { return glm::unpackF2x11_1x10; + } else if (format == gpu::Element::COLOR_RGBA_32 || format == gpu::Element::COLOR_SRGBA_32 || format == gpu::Element::COLOR_BGRA_32 || format == gpu::Element::COLOR_SBGRA_32) { + return glm::unpackUnorm4x8; } else { - qCWarning(imagelogging) << "Unknown HDR encoding format in QImage"; + qCWarning(imagelogging) << "Unknown handler format"; Q_UNREACHABLE(); return nullptr; } } +std::function getHDRUnpackingFunction() { + return getUnpackingFunction(HDR_FORMAT); +} + QImage processRawImageData(QIODevice& content, const std::string& filename) { // Help the QImage loader by extracting the image file format from the url filename ext. // Some tga are not created properly without it. @@ -440,7 +463,7 @@ struct OutputHandler : public nvtt::OutputHandler { struct PackedFloatOutputHandler : public OutputHandler { PackedFloatOutputHandler(gpu::Texture* texture, int face, gpu::Element format) : OutputHandler(texture, face) { - _packFunc = getHDRPackingFunction(format); + _packFunc = getPackingFunction(format); } virtual void beginImage(int size, int width, int height, int depth, int face, int miplevel) override { @@ -498,6 +521,43 @@ public: } }; +void convertToFloat(const unsigned char* source, int width, int height, int lineStride, gpu::Element sourceFormat, std::vector& output) { + std::vector::iterator outputIt; + auto unpackFunc = getUnpackingFunction(sourceFormat); + + output.resize(width * height); + outputIt = output.begin(); + for (auto lineNb = 0; lineNb < height; lineNb++) { + const uint32* srcPixelIt = reinterpret_cast(source + lineNb * lineStride); + const uint32* srcPixelEnd = srcPixelIt + width; + + while (srcPixelIt < srcPixelEnd) { + *outputIt = glm::vec4(unpackFunc(*srcPixelIt), 1.0f); + ++srcPixelIt; + ++outputIt; + } + } + assert(outputIt == output.end()); +} + +void convertFromFloat(unsigned char* output, int width, int height, int lineStride, gpu::Element outputFormat, const std::vector& source) { + std::vector::const_iterator sourceIt; + auto packFunc = getPackingFunction(outputFormat); + + sourceIt = source.begin(); + for (auto lineNb = 0; lineNb < height; lineNb++) { + uint32* outPixelIt = reinterpret_cast(output + lineNb * lineStride); + uint32* outPixelEnd = outPixelIt + width; + + while (outPixelIt < outPixelEnd) { + *outPixelIt = packFunc(*sourceIt); + ++outPixelIt; + ++sourceIt; + } + } + assert(sourceIt == source.end()); +} + void generateHDRMips(gpu::Texture* texture, QImage&& image, BackendTarget target, const std::atomic& abortProcessing, int face) { // Take a local copy to force move construction // https://github.com/isocpp/CppCoreGuidelines/blob/master/CppCoreGuidelines.md#f18-for-consume-parameters-pass-by-x-and-stdmove-the-parameter @@ -509,7 +569,6 @@ void generateHDRMips(gpu::Texture* texture, QImage&& image, BackendTarget target std::vector data; std::vector::iterator dataIt; auto mipFormat = texture->getStoredMipFormat(); - std::function unpackFunc = getHDRUnpackingFunction(); nvtt::InputFormat inputFormat = nvtt::InputFormat_RGBA_32F; nvtt::WrapMode wrapMode = nvtt::WrapMode_Mirror; @@ -535,19 +594,7 @@ void generateHDRMips(gpu::Texture* texture, QImage&& image, BackendTarget target return; } - data.resize(width * height); - dataIt = data.begin(); - for (auto lineNb = 0; lineNb < height; lineNb++) { - const uint32* srcPixelIt = reinterpret_cast(localCopy.constScanLine(lineNb)); - const uint32* srcPixelEnd = srcPixelIt + width; - - while (srcPixelIt < srcPixelEnd) { - *dataIt = glm::vec4(unpackFunc(*srcPixelIt), 1.0f); - ++srcPixelIt; - ++dataIt; - } - } - assert(dataIt == data.end()); + convertToFloat(localCopy.bits(), width, height, localCopy.bytesPerLine(), HDR_FORMAT, data); // We're done with the localCopy, free up the memory to avoid bloating the heap localCopy = QImage(); // QImage doesn't have a clear function, so override it with an empty one. @@ -785,22 +832,74 @@ void generateLDRMips(gpu::Texture* texture, QImage&& image, BackendTarget target #endif -void generateMips(gpu::Texture* texture, QImage&& image, BackendTarget target, const std::atomic& abortProcessing = false, int face = -1) { -#if CPU_MIPMAPS - PROFILE_RANGE(resource_parse, "generateMips"); +void generateMips(gpu::Texture* texture, QImage&& image, BackendTarget target, const std::atomic& abortProcessing = false, int face = -1, bool forceCPUBuild = false) { + if (forceCPUBuild || CPU_MIPMAPS) { + PROFILE_RANGE(resource_parse, "generateMips"); - if (target == BackendTarget::GLES32) { - generateLDRMips(texture, std::move(image), target, abortProcessing, face); - } else { - if (image.format() == QIMAGE_HDRFORMAT) { - generateHDRMips(texture, std::move(image), target, abortProcessing, face); - } else { + if (target == BackendTarget::GLES32) { generateLDRMips(texture, std::move(image), target, abortProcessing, face); + } else { + if (image.format() == QIMAGE_HDRFORMAT) { + generateHDRMips(texture, std::move(image), target, abortProcessing, face); + } else { + generateLDRMips(texture, std::move(image), target, abortProcessing, face); + } + } + } else { + texture->setAutoGenerateMips(true); + } +} + +void convolveFaceWithGGX(const CubeMap& source, int face, const std::atomic& abortProcessing) { + +} + +void convolveWithGGX(gpu::Texture* texture, BackendTarget target, const std::atomic& abortProcessing = false) { + PROFILE_RANGE(resource_parse, "convolveWithGGX"); + CubeMap source(texture->getWidth(), texture->getHeight(), texture->getNumMips()); + gpu::uint16 mipLevel; + int face; + const auto textureFormat = texture->getTexelFormat(); + + // Convert all mip data to float as source + for (mipLevel = 0; mipLevel < source.getMipCount(); ++mipLevel) { + auto mipDims = texture->evalMipDimensions(mipLevel); + auto& mip = source.editMip(mipLevel); + + for (face = 0; face < 6; face++) { + auto sourcePixels = texture->accessStoredMipFace(mipLevel, face)->data(); + convertToFloat(sourcePixels, mipDims.x, mipDims.y, sizeof(uint32)*mipDims.x, textureFormat, mip[face]); + if (abortProcessing.load()) { + return; + } } } -#else - texture->setAutoGenerateMips(true); -#endif + + for (face = 0; face < 6; face++) { + convolveFaceWithGGX(source, face, abortProcessing); + } + + if (!abortProcessing) { + // Convert all mip data back from float + unsigned char* convertedPixels = new unsigned char[texture->getWidth() * texture->getHeight() * sizeof(uint32)]; + + for (mipLevel = 0; mipLevel < source.getMipCount(); ++mipLevel) { + auto mipDims = texture->evalMipDimensions(mipLevel); + auto mipSize = texture->evalMipFaceSize(mipLevel); + auto& mip = source.getMip(mipLevel); + + for (face = 0; face < 6; face++) { + convertFromFloat(convertedPixels, mipDims.x, mipDims.y, sizeof(uint32)*mipDims.x, textureFormat, mip[face]); + texture->assignStoredMipFace(mipLevel, face, mipSize, convertedPixels); + if (abortProcessing.load()) { + delete[] convertedPixels; + return; + } + } + } + + delete[] convertedPixels; + } } void processTextureAlpha(const QImage& srcImage, bool& validAlpha, bool& alphaAsMask) { @@ -1407,7 +1506,7 @@ QImage convertToHDRFormat(QImage&& srcImage, gpu::Element format) { } gpu::TexturePointer TextureUsage::processCubeTextureColorFromImage(QImage&& srcImage, const std::string& srcImageName, - bool compress, BackendTarget target, bool generateIrradiance, + bool compress, BackendTarget target, int options, const std::atomic& abortProcessing) { PROFILE_RANGE(resource_parse, "processCubeTextureColorFromImage"); @@ -1492,7 +1591,7 @@ gpu::TexturePointer TextureUsage::processCubeTextureColorFromImage(QImage&& srcI theTexture->setStoredMipFormat(formatMip); // Generate irradiance while we are at it - if (generateIrradiance) { + if (options & CUBE_GENERATE_IRRADIANCE) { PROFILE_RANGE(resource_parse, "generateIrradiance"); gpu::Element irradianceFormat; // TODO: we could locally compress the irradiance texture on Android, but we don't need to @@ -1516,7 +1615,12 @@ gpu::TexturePointer TextureUsage::processCubeTextureColorFromImage(QImage&& srcI } for (uint8 face = 0; face < faces.size(); ++face) { - generateMips(theTexture.get(), std::move(faces[face]), target, abortProcessing, face); + // Force building the mip maps right now on CPU if we are convolving for GGX later on + generateMips(theTexture.get(), std::move(faces[face]), target, abortProcessing, face, (options & CUBE_GGX_CONVOLVE) == CUBE_GGX_CONVOLVE); + } + + if (options & CUBE_GGX_CONVOLVE) { + convolveWithGGX(theTexture.get(), target, abortProcessing); } } diff --git a/libraries/image/src/image/Image.h b/libraries/image/src/image/Image.h index b816edac39..9c27b0cf3c 100644 --- a/libraries/image/src/image/Image.h +++ b/libraries/image/src/image/Image.h @@ -72,8 +72,12 @@ gpu::TexturePointer createMetallicTextureFromImage(QImage&& image, const std::st bool compress, gpu::BackendTarget target, const std::atomic& abortProcessing); gpu::TexturePointer createCubeTextureFromImage(QImage&& image, const std::string& srcImageName, bool compress, gpu::BackendTarget target, const std::atomic& abortProcessing); -gpu::TexturePointer createCubeTextureFromImageWithoutIrradiance(QImage&& image, const std::string& srcImageName, - bool compress, gpu::BackendTarget target, const std::atomic& abortProcessing); +gpu::TexturePointer createCubeTextureAndIrradianceFromImage(QImage&& image, const std::string& srcImageName, + bool compress, gpu::BackendTarget target, const std::atomic& abortProcessing); +gpu::TexturePointer createGGXConvolvedCubeTextureFromImage(QImage&& image, const std::string& srcImageName, + bool compress, gpu::BackendTarget target, const std::atomic& abortProcessing); +gpu::TexturePointer createGGXConvolvedCubeTextureAndIrradianceFromImage(QImage&& image, const std::string& srcImageName, + bool compress, gpu::BackendTarget target, const std::atomic& abortProcessing); gpu::TexturePointer createLightmapTextureFromImage(QImage&& image, const std::string& srcImageName, bool compress, gpu::BackendTarget target, const std::atomic& abortProcessing); gpu::TexturePointer process2DTextureColorFromImage(QImage&& srcImage, const std::string& srcImageName, bool compress, @@ -82,8 +86,14 @@ gpu::TexturePointer process2DTextureNormalMapFromImage(QImage&& srcImage, const gpu::BackendTarget target, bool isBumpMap, const std::atomic& abortProcessing); gpu::TexturePointer process2DTextureGrayscaleFromImage(QImage&& srcImage, const std::string& srcImageName, bool compress, gpu::BackendTarget target, bool isInvertedPixels, const std::atomic& abortProcessing); + +enum CubeTextureOptions { + CUBE_DEFAULT = 0x0, + CUBE_GENERATE_IRRADIANCE = 0x1, + CUBE_GGX_CONVOLVE = 0x2 +}; gpu::TexturePointer processCubeTextureColorFromImage(QImage&& srcImage, const std::string& srcImageName, bool compress, - gpu::BackendTarget target, bool generateIrradiance, const std::atomic& abortProcessing); + gpu::BackendTarget target, int option, const std::atomic& abortProcessing); } // namespace TextureUsage