From 24f022ec3b33a81ba11893fbed6c6075061ed1a0 Mon Sep 17 00:00:00 2001 From: Anna Date: Wed, 3 Jul 2019 18:06:05 -0700 Subject: [PATCH 01/41] init commit for this pr, still buggy --- .../render-utils/src/RenderDeferredTask.cpp | 11 +- .../render-utils/src/RenderForwardTask.cpp | 11 +- .../src/ToneMapAndResampleTask.cpp | 139 ++++++++++++++++++ ...ppingEffect.h => ToneMapAndResampleTask.h} | 100 +++++++------ .../render-utils/src/ToneMappingEffect.cpp | 96 ------------ .../src/render-utils/toneMappingMirroredX.slp | 1 + .../render-utils/src/toneMappingMirroredX.slf | 71 +++++++++ 7 files changed, 272 insertions(+), 157 deletions(-) create mode 100644 libraries/render-utils/src/ToneMapAndResampleTask.cpp rename libraries/render-utils/src/{ToneMappingEffect.h => ToneMapAndResampleTask.h} (65%) delete mode 100644 libraries/render-utils/src/ToneMappingEffect.cpp create mode 100644 libraries/render-utils/src/render-utils/toneMappingMirroredX.slp create mode 100644 libraries/render-utils/src/toneMappingMirroredX.slf diff --git a/libraries/render-utils/src/RenderDeferredTask.cpp b/libraries/render-utils/src/RenderDeferredTask.cpp index c26f3b613c..cb7ef696ad 100644 --- a/libraries/render-utils/src/RenderDeferredTask.cpp +++ b/libraries/render-utils/src/RenderDeferredTask.cpp @@ -51,7 +51,7 @@ #include "AmbientOcclusionEffect.h" #include "AntialiasingEffect.h" -#include "ToneMappingEffect.h" +#include "ToneMapAndResampleTask.h" #include "SubsurfaceScattering.h" #include "DrawHaze.h" #include "BloomEffect.h" @@ -239,8 +239,8 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren task.addJob("Bloom", bloomInputs); // Lighting Buffer ready for tone mapping - const auto toneMappingInputs = ToneMappingDeferred::Input(lightingFramebuffer, scaledPrimaryFramebuffer).asVarying(); - const auto toneMappedBuffer = task.addJob("ToneMapping", toneMappingInputs); + const auto toneMappingInputs = ToneMapAndResample::Input(lightingFramebuffer, scaledPrimaryFramebuffer).asVarying(); + const auto toneMappedBuffer = task.addJob("ToneMapAndResample", toneMappingInputs); // Debugging task is happening in the "over" layer after tone mapping and just before HUD { // Debug the bounds of the rendered items, still look at the zbuffer @@ -250,11 +250,8 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren task.addJob("DebugRenderDeferredTask", debugInputs); } - // Upscale to finale resolution - const auto primaryFramebuffer = task.addJob("PrimaryBufferUpscale", toneMappedBuffer); - // HUD Layer - const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(primaryFramebuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); + const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(toneMappedBuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); task.addJob("RenderHUDLayer", renderHUDLayerInputs); } diff --git a/libraries/render-utils/src/RenderForwardTask.cpp b/libraries/render-utils/src/RenderForwardTask.cpp index b6b17ee376..0c13d2cfb8 100755 --- a/libraries/render-utils/src/RenderForwardTask.cpp +++ b/libraries/render-utils/src/RenderForwardTask.cpp @@ -28,7 +28,7 @@ #include "StencilMaskPass.h" #include "ZoneRenderer.h" #include "FadeEffect.h" -#include "ToneMappingEffect.h" +#include "ToneMapAndResampleTask.h" #include "BackgroundStage.h" #include "FramebufferCache.h" #include "TextureCache.h" @@ -159,16 +159,13 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend // Lighting Buffer ready for tone mapping // Forward rendering on GLES doesn't support tonemapping to and from the same FBO, so we specify // the output FBO as null, which causes the tonemapping to target the blit framebuffer - const auto toneMappingInputs = ToneMappingDeferred::Input(resolvedFramebuffer, resolvedFramebuffer).asVarying(); - const auto toneMappedBuffer = task.addJob("ToneMapping", toneMappingInputs); + const auto toneMappingInputs = ToneMapAndResample::Input(resolvedFramebuffer, resolvedFramebuffer).asVarying(); + const auto toneMappedBuffer = task.addJob("ToneMapAndResample", toneMappingInputs); #endif - // Upscale to finale resolution - const auto primaryFramebuffer = task.addJob("PrimaryBufferUpscale", toneMappedBuffer); - // HUD Layer - const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(primaryFramebuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); + const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(toneMappedBuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); task.addJob("RenderHUDLayer", renderHUDLayerInputs); } diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.cpp b/libraries/render-utils/src/ToneMapAndResampleTask.cpp new file mode 100644 index 0000000000..521c23685d --- /dev/null +++ b/libraries/render-utils/src/ToneMapAndResampleTask.cpp @@ -0,0 +1,139 @@ +// +// ToneMapAndResampleTask.cpp +// libraries/render-utils/src +// +// Created by Sam Gateau on 12/7/2015. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "ToneMapAndResampleTask.h" + +#include +#include + +#include "render-utils/ShaderConstants.h" +#include "StencilMaskPass.h" +#include "FramebufferCache.h" + +using namespace shader::render_utils::program; + +gpu::PipelinePointer ToneMapAndResample::_pipeline; +gpu::PipelinePointer ToneMapAndResample::_mirrorPipeline; + +ToneMapAndResample::ToneMapAndResample() { + Parameters parameters; + _parametersBuffer = gpu::BufferView(std::make_shared(sizeof(Parameters), (const gpu::Byte*) ¶meters)); +} + +void ToneMapAndResample::init(RenderArgs* args) { + + // shared_ptr to gpu::State + gpu::StatePointer blitState = gpu::StatePointer(new gpu::State()); + + // TODO why was this in the upsample task + //blitState->setDepthTest(gpu::State::DepthTest(false, false)); + blitState->setColorWriteMask(true, true, true, true); + + _pipeline = gpu::PipelinePointer(gpu::Pipeline::create(gpu::Shader::createProgram(toneMapping), blitState)); + _mirrorPipeline = gpu::PipelinePointer(gpu::Pipeline::create(gpu::Shader::createProgram(toneMappingMirroredX), blitState)); +} + +void ToneMapAndResample::setExposure(float exposure) { + auto& params = _parametersBuffer.get(); + if (params._exposure != exposure) { + _parametersBuffer.edit()._exposure = exposure; + _parametersBuffer.edit()._twoPowExposure = pow(2.0, exposure); + } +} + +void ToneMapAndResample::setToneCurve(ToneCurve curve) { + auto& params = _parametersBuffer.get(); + if (params._toneCurve != (int)curve) { + _parametersBuffer.edit()._toneCurve = (int)curve; + } +} + +gpu::FramebufferPointer ToneMapAndResample::getResampledFrameBuffer(const gpu::FramebufferPointer& sourceFramebuffer) { + if (_factor == 1.0f) { + return sourceFramebuffer; + } + + auto resampledFramebufferSize = glm::uvec2(glm::vec2(sourceFramebuffer->getSize()) * _factor); + + if (!_destinationFrameBuffer || resampledFramebufferSize != _destinationFrameBuffer->getSize()) { + _destinationFrameBuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("ResampledOutput")); + + auto sampler = gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR); + auto target = gpu::Texture::createRenderBuffer(sourceFramebuffer->getRenderBuffer(0)->getTexelFormat(), resampledFramebufferSize.x, resampledFramebufferSize.y, gpu::Texture::SINGLE_MIP, sampler); + _destinationFrameBuffer->setRenderBuffer(0, target); + } + return _destinationFrameBuffer; +} + +// TODO why was destination const +void ToneMapAndResample::render(RenderArgs* args, const gpu::TexturePointer& lightingBuffer, gpu::FramebufferPointer& destinationFramebuffer) { + + if (!_pipeline) { + init(args); + } + + if (!lightingBuffer || !destinationFramebuffer) { + return; + } + + //auto framebufferSize = glm::ivec2(lightingBuffer->getDimensions()); + + auto framebufferSize = destinationFramebuffer->getSize(); + + gpu::doInBatch("ToneMapAndResample::render", args->_context, [&](gpu::Batch& batch) { + batch.enableStereo(false); + batch.setFramebuffer(destinationFramebuffer); + + // FIXME: Generate the Luminosity map + //batch.generateTextureMips(lightingBuffer); + + batch.setViewportTransform(args->_viewport); + batch.setProjectionTransform(glm::mat4()); + batch.resetViewTransform(); + batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(framebufferSize, args->_viewport)); + batch.setPipeline(args->_renderMode == RenderArgs::MIRROR_RENDER_MODE ? _mirrorPipeline : _pipeline); + + batch.setUniformBuffer(render_utils::slot::buffer::ToneMappingParams, _parametersBuffer); + batch.setResourceTexture(render_utils::slot::texture::ToneMappingColor, lightingBuffer); + batch.draw(gpu::TRIANGLE_STRIP, 4); + }); + + destinationFramebuffer = getResampledFrameBuffer(destinationFramebuffer); + + const auto bufferSize = destinationFramebuffer->getSize(); + glm::ivec4 viewport{ 0, 0, bufferSize.x, bufferSize.y }; + + //Set full final viewport + args->_viewport = viewport; +} + +void ToneMapAndResample::configure(const Config& config) { + setExposure(config.exposure); + setToneCurve((ToneCurve)config.curve); +} + +void ToneMapAndResample::run(const render::RenderContextPointer& renderContext, const Input& input, Output& output) { + + assert(renderContext->args); + assert(renderContext->args->hasViewFrustum()); + + RenderArgs* args = renderContext->args; + + auto lightingBuffer = input.get0()->getRenderBuffer(0); + + auto resampledFramebuffer = args->_blitFramebuffer; + + render(args, lightingBuffer, resampledFramebuffer); + + output = resampledFramebuffer; +} + + diff --git a/libraries/render-utils/src/ToneMappingEffect.h b/libraries/render-utils/src/ToneMapAndResampleTask.h similarity index 65% rename from libraries/render-utils/src/ToneMappingEffect.h rename to libraries/render-utils/src/ToneMapAndResampleTask.h index faf6e514e9..439146e05f 100644 --- a/libraries/render-utils/src/ToneMappingEffect.h +++ b/libraries/render-utils/src/ToneMapAndResampleTask.h @@ -1,5 +1,5 @@ // -// ToneMappingEffect.h +// ToneMapAndResample.h // libraries/render-utils/src // // Created by Sam Gateau on 12/7/2015. @@ -9,8 +9,8 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#ifndef hifi_ToneMappingEffect_h -#define hifi_ToneMappingEffect_h +#ifndef hifi_ToneMapAndResample_h +#define hifi_ToneMapAndResample_h #include #include @@ -20,27 +20,64 @@ #include #include +enum class ToneCurve { + // Different tone curve available + None, + Gamma22, + Reinhard, + Filmic, +}; -class ToneMappingEffect { +class ToneMappingConfig : public render::Job::Config { + Q_OBJECT + Q_PROPERTY(float exposure MEMBER exposure WRITE setExposure); + Q_PROPERTY(int curve MEMBER curve WRITE setCurve); public: - ToneMappingEffect(); - virtual ~ToneMappingEffect() {} + ToneMappingConfig() : render::Job::Config(true) {} - void render(RenderArgs* args, const gpu::TexturePointer& lightingBuffer, const gpu::FramebufferPointer& destinationBuffer); + void setExposure(float newExposure) { exposure = newExposure; emit dirty(); } + void setCurve(int newCurve) { curve = std::max((int)ToneCurve::None, std::min((int)ToneCurve::Filmic, newCurve)); emit dirty(); } + + + float exposure{ 0.0f }; + int curve{ (int)ToneCurve::Gamma22 }; +signals: + void dirty(); +}; + +class ToneMapAndResample { +public: + ToneMapAndResample(); + virtual ~ToneMapAndResample() {} + + void render(RenderArgs* args, const gpu::TexturePointer& lightingBuffer, gpu::FramebufferPointer& destinationBuffer); void setExposure(float exposure); float getExposure() const { return _parametersBuffer.get()._exposure; } - // Different tone curve available - enum ToneCurve { - None = 0, - Gamma22, - Reinhard, - Filmic, - }; void setToneCurve(ToneCurve curve); ToneCurve getToneCurve() const { return (ToneCurve)_parametersBuffer.get()._toneCurve; } + // Inputs: lightingFramebuffer, destinationFramebuffer + using Input = render::VaryingSet2; + using Output = gpu::FramebufferPointer; + using Config = ToneMappingConfig; + using JobModel = render::Job::ModelIO; + + void configure(const Config& config); + void run(const render::RenderContextPointer& renderContext, const Input& input, Output& output); + +protected: + + static gpu::PipelinePointer _pipeline; + static gpu::PipelinePointer _mirrorPipeline; + + gpu::FramebufferPointer _destinationFrameBuffer; + + float _factor{ 2.0f }; + + gpu::FramebufferPointer ToneMapAndResample::getResampledFrameBuffer(const gpu::FramebufferPointer& sourceFramebuffer); + private: gpu::PipelinePointer _blitLightBuffer; @@ -51,7 +88,7 @@ private: float _exposure = 0.0f; float _twoPowExposure = 1.0f; glm::vec2 spareA; - int _toneCurve = Gamma22; + int _toneCurve = (int)ToneCurve::Gamma22; glm::vec3 spareB; Parameters() {} @@ -62,35 +99,4 @@ private: void init(RenderArgs* args); }; -class ToneMappingConfig : public render::Job::Config { - Q_OBJECT - Q_PROPERTY(float exposure MEMBER exposure WRITE setExposure); - Q_PROPERTY(int curve MEMBER curve WRITE setCurve); -public: - ToneMappingConfig() : render::Job::Config(true) {} - - void setExposure(float newExposure) { exposure = newExposure; emit dirty(); } - void setCurve(int newCurve) { curve = std::max((int)ToneMappingEffect::None, std::min((int)ToneMappingEffect::Filmic, newCurve)); emit dirty(); } - - - float exposure{ 0.0f }; - int curve{ ToneMappingEffect::Gamma22 }; -signals: - void dirty(); -}; - -class ToneMappingDeferred { -public: - // Inputs: lightingFramebuffer, destinationFramebuffer - using Input = render::VaryingSet2; - using Output = gpu::FramebufferPointer; - using Config = ToneMappingConfig; - using JobModel = render::Job::ModelIO; - - void configure(const Config& config); - void run(const render::RenderContextPointer& renderContext, const Input& input, Output& output); - - ToneMappingEffect _toneMappingEffect; -}; - -#endif // hifi_ToneMappingEffect_h +#endif // hifi_ToneMapAndResample_h diff --git a/libraries/render-utils/src/ToneMappingEffect.cpp b/libraries/render-utils/src/ToneMappingEffect.cpp deleted file mode 100644 index b7cc5d3d80..0000000000 --- a/libraries/render-utils/src/ToneMappingEffect.cpp +++ /dev/null @@ -1,96 +0,0 @@ -// -// ToneMappingEffect.cpp -// libraries/render-utils/src -// -// Created by Sam Gateau on 12/7/2015. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "ToneMappingEffect.h" - -#include -#include - -#include "render-utils/ShaderConstants.h" -#include "StencilMaskPass.h" -#include "FramebufferCache.h" - - -ToneMappingEffect::ToneMappingEffect() { - Parameters parameters; - _parametersBuffer = gpu::BufferView(std::make_shared(sizeof(Parameters), (const gpu::Byte*) ¶meters)); -} - -void ToneMappingEffect::init(RenderArgs* args) { - auto blitProgram = gpu::Shader::createProgram(shader::render_utils::program::toneMapping); - - auto blitState = std::make_shared(); - blitState->setColorWriteMask(true, true, true, true); - _blitLightBuffer = gpu::PipelinePointer(gpu::Pipeline::create(blitProgram, blitState)); -} - -void ToneMappingEffect::setExposure(float exposure) { - auto& params = _parametersBuffer.get(); - if (params._exposure != exposure) { - _parametersBuffer.edit()._exposure = exposure; - _parametersBuffer.edit()._twoPowExposure = pow(2.0, exposure); - } -} - -void ToneMappingEffect::setToneCurve(ToneCurve curve) { - auto& params = _parametersBuffer.get(); - if (params._toneCurve != curve) { - _parametersBuffer.edit()._toneCurve = curve; - } -} - -void ToneMappingEffect::render(RenderArgs* args, const gpu::TexturePointer& lightingBuffer, const gpu::FramebufferPointer& destinationFramebuffer) { - if (!_blitLightBuffer) { - init(args); - } - - if (!lightingBuffer || !destinationFramebuffer) { - return; - } - - auto framebufferSize = glm::ivec2(lightingBuffer->getDimensions()); - gpu::doInBatch("ToneMappingEffect::render", args->_context, [&](gpu::Batch& batch) { - batch.enableStereo(false); - batch.setFramebuffer(destinationFramebuffer); - - // FIXME: Generate the Luminosity map - //batch.generateTextureMips(lightingBuffer); - - batch.setViewportTransform(args->_viewport); - batch.setProjectionTransform(glm::mat4()); - batch.resetViewTransform(); - batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(framebufferSize, args->_viewport)); - batch.setPipeline(_blitLightBuffer); - - batch.setUniformBuffer(render_utils::slot::buffer::ToneMappingParams, _parametersBuffer); - batch.setResourceTexture(render_utils::slot::texture::ToneMappingColor, lightingBuffer); - batch.draw(gpu::TRIANGLE_STRIP, 4); - }); -} - - -void ToneMappingDeferred::configure(const Config& config) { - _toneMappingEffect.setExposure(config.exposure); - _toneMappingEffect.setToneCurve((ToneMappingEffect::ToneCurve)config.curve); -} - -void ToneMappingDeferred::run(const render::RenderContextPointer& renderContext, const Input& input, Output& output) { - - auto lightingBuffer = input.get0()->getRenderBuffer(0); - auto destFbo = input.get1(); - - if (!destFbo) { - destFbo = renderContext->args->_blitFramebuffer; - } - - _toneMappingEffect.render(renderContext->args, lightingBuffer, destFbo); - output = destFbo; -} diff --git a/libraries/render-utils/src/render-utils/toneMappingMirroredX.slp b/libraries/render-utils/src/render-utils/toneMappingMirroredX.slp new file mode 100644 index 0000000000..d4d8ec4b01 --- /dev/null +++ b/libraries/render-utils/src/render-utils/toneMappingMirroredX.slp @@ -0,0 +1 @@ +VERTEX gpu::vertex::DrawViewportQuadTransformTexcoord diff --git a/libraries/render-utils/src/toneMappingMirroredX.slf b/libraries/render-utils/src/toneMappingMirroredX.slf new file mode 100644 index 0000000000..7c510cdc1b --- /dev/null +++ b/libraries/render-utils/src/toneMappingMirroredX.slf @@ -0,0 +1,71 @@ +<@include gpu/Config.slh@> +<$VERSION_HEADER$> +// Generated on Sat Oct 24 09:34:37 2015 +// +// toneMapping.frag +// +// Draw texture 0 fetched at texcoord.xy +// +// Created by Sam Gateau on 6/22/2015 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +<@include render-utils/ShaderConstants.h@> + +struct ToneMappingParams { + vec4 _exp_2powExp_s0_s1; + ivec4 _toneCurve_s0_s1_s2; +}; + +const float GAMMA_22 = 2.2; +const float INV_GAMMA_22 = 1.0 / 2.2; +const int ToneCurveNone = 0; +const int ToneCurveGamma22 = 1; +const int ToneCurveReinhard = 2; +const int ToneCurveFilmic = 3; + +LAYOUT(binding=RENDER_UTILS_BUFFER_TM_PARAMS) uniform toneMappingParamsBuffer { + ToneMappingParams params; +}; +float getTwoPowExposure() { + return params._exp_2powExp_s0_s1.y; +} +int getToneCurve() { + return params._toneCurve_s0_s1_s2.x; +} + +LAYOUT(binding=RENDER_UTILS_TEXTURE_TM_COLOR) uniform sampler2D colorMap; + +layout(location=0) in vec2 varTexCoord0; +layout(location=0) out vec4 outFragColor; + +void main(void) { + vec4 fragColorRaw = texture(colorMap, vec2(1 - varTexCoord0.x, varTexCoord0.y)); + vec3 fragColor = fragColorRaw.xyz; + + vec3 srcColor = fragColor * getTwoPowExposure(); + + int toneCurve = getToneCurve(); + vec3 tonedColor = srcColor; + if (toneCurve == ToneCurveFilmic) { + vec3 x = max(vec3(0.0), srcColor-0.004); + tonedColor = pow((x * (6.2 * x + 0.5)) / (x * (6.2 * x + 1.7) + 0.06), vec3(GAMMA_22)); + } else if (toneCurve == ToneCurveReinhard) { + tonedColor = srcColor/(1.0 + srcColor); + } else if (toneCurve == ToneCurveGamma22) { + // We use glEnable(GL_FRAMEBUFFER_SRGB), which automatically converts textures from RGB to SRGB + // when writing from an RGB framebuffer to an SRGB framebuffer (note that it doesn't do anything + // when writing from an SRGB framebuffer to an RGB framebuffer). + // Since the conversion happens automatically, we don't need to do anything in this shader + } else { + // toneCurve == ToneCurveNone + // For debugging purposes, we may want to see what the colors look like before the automatic OpenGL + // conversion mentioned above, so we undo it here + tonedColor = pow(srcColor, vec3(GAMMA_22)); + } + + outFragColor = vec4(tonedColor, 1.0); +} From 0b51e3c129e6057e8aaaedd69c7f894c711746bd Mon Sep 17 00:00:00 2001 From: Anna Date: Tue, 9 Jul 2019 09:37:07 -0700 Subject: [PATCH 02/41] progress --- interface/src/ui/Snapshot.cpp | 2 +- libraries/render-utils/src/BloomEffect.cpp | 2 +- .../render-utils/src/RenderDeferredTask.cpp | 4 +-- .../render-utils/src/RenderForwardTask.cpp | 3 +- .../src/ToneMapAndResampleTask.cpp | 36 ++++++++++--------- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/interface/src/ui/Snapshot.cpp b/interface/src/ui/Snapshot.cpp index 926588e4ca..363e7a5cf1 100644 --- a/interface/src/ui/Snapshot.cpp +++ b/interface/src/ui/Snapshot.cpp @@ -41,7 +41,7 @@ #include "MainWindow.h" #include "Snapshot.h" #include "SnapshotUploader.h" -#include "ToneMappingEffect.h" +#include "ToneMapAndResampleTask.h" // filename format: hifi-snap-by-%username%-on-%date%_%time%_@-%location%.jpg // %1 <= username, %2 <= date and time, %3 <= current location diff --git a/libraries/render-utils/src/BloomEffect.cpp b/libraries/render-utils/src/BloomEffect.cpp index 414a1c3f91..10df138fad 100644 --- a/libraries/render-utils/src/BloomEffect.cpp +++ b/libraries/render-utils/src/BloomEffect.cpp @@ -15,7 +15,7 @@ #include #include -#include +#include "ToneMapAndResampleTask.h" #include "render-utils/ShaderConstants.h" #define BLOOM_BLUR_LEVEL_COUNT 3 diff --git a/libraries/render-utils/src/RenderDeferredTask.cpp b/libraries/render-utils/src/RenderDeferredTask.cpp index cb7ef696ad..a4f1d6b6c9 100644 --- a/libraries/render-utils/src/RenderDeferredTask.cpp +++ b/libraries/render-utils/src/RenderDeferredTask.cpp @@ -29,7 +29,6 @@ #include #include #include -#include #include "RenderHifi.h" #include "render-utils/ShaderConstants.h" @@ -246,7 +245,7 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren { // Debug the bounds of the rendered items, still look at the zbuffer const auto extraDebugBuffers = RenderDeferredTaskDebug::ExtraBuffers(linearDepthTarget, surfaceGeometryFramebuffer, ambientOcclusionFramebuffer, ambientOcclusionUniforms, scatteringResource, velocityBuffer); const auto debugInputs = RenderDeferredTaskDebug::Input(fetchedItems, shadowTaskOutputs, lightingStageInputs, lightClusters, prepareDeferredOutputs, extraDebugBuffers, - deferredFrameTransform, jitter, lightingModel).asVarying(); + deferredFrameTransform, jitter, lightingModel).asVarying(); task.addJob("DebugRenderDeferredTask", debugInputs); } @@ -412,7 +411,6 @@ void RenderDeferredTaskDebug::build(JobModel& task, const render::Varying& input const auto debugZoneInputs = DebugZoneLighting::Inputs(deferredFrameTransform, lightFrame, backgroundFrame).asVarying(); task.addJob("DrawZoneStack", debugZoneInputs); - } diff --git a/libraries/render-utils/src/RenderForwardTask.cpp b/libraries/render-utils/src/RenderForwardTask.cpp index 0c13d2cfb8..c16af23218 100755 --- a/libraries/render-utils/src/RenderForwardTask.cpp +++ b/libraries/render-utils/src/RenderForwardTask.cpp @@ -19,7 +19,6 @@ #include #include #include -#include #include @@ -100,6 +99,8 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend // GPU jobs: Start preparing the main framebuffer const auto scaledPrimaryFramebuffer = task.addJob("PreparePrimaryBuffer"); + qDebug() << "anna forward " << "scaled " << renderContext->args->_viewport.z << " x " << renderContext->args->_viewport.w; + // Prepare deferred, generate the shared Deferred Frame Transform. Only valid with the scaled frame buffer const auto deferredFrameTransform = task.addJob("DeferredFrameTransform"); diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.cpp b/libraries/render-utils/src/ToneMapAndResampleTask.cpp index 521c23685d..474c56525a 100644 --- a/libraries/render-utils/src/ToneMapAndResampleTask.cpp +++ b/libraries/render-utils/src/ToneMapAndResampleTask.cpp @@ -34,7 +34,7 @@ void ToneMapAndResample::init(RenderArgs* args) { gpu::StatePointer blitState = gpu::StatePointer(new gpu::State()); // TODO why was this in the upsample task - //blitState->setDepthTest(gpu::State::DepthTest(false, false)); + blitState->setDepthTest(gpu::State::DepthTest(false, false)); blitState->setColorWriteMask(true, true, true, true); _pipeline = gpu::PipelinePointer(gpu::Pipeline::create(gpu::Shader::createProgram(toneMapping), blitState)); @@ -63,13 +63,16 @@ gpu::FramebufferPointer ToneMapAndResample::getResampledFrameBuffer(const gpu::F auto resampledFramebufferSize = glm::uvec2(glm::vec2(sourceFramebuffer->getSize()) * _factor); - if (!_destinationFrameBuffer || resampledFramebufferSize != _destinationFrameBuffer->getSize()) { - _destinationFrameBuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("ResampledOutput")); + _destinationFrameBuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("ResampledOutput")); + + auto sampler = gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR); + auto target = gpu::Texture::createRenderBuffer(sourceFramebuffer->getRenderBuffer(0)->getTexelFormat(), resampledFramebufferSize.x, resampledFramebufferSize.y, gpu::Texture::SINGLE_MIP, sampler); + _destinationFrameBuffer->setRenderBuffer(0, target); + + auto depthFormat = gpu::Element(gpu::SCALAR, gpu::UINT32, gpu::DEPTH_STENCIL); // Depth24_Stencil8 texel format + auto primaryDepthTexture = gpu::Texture::createRenderBuffer(depthFormat, resampledFramebufferSize.x, resampledFramebufferSize.y, gpu::Texture::SINGLE_MIP, sampler); + _destinationFrameBuffer->setDepthStencilBuffer(primaryDepthTexture, depthFormat); - auto sampler = gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR); - auto target = gpu::Texture::createRenderBuffer(sourceFramebuffer->getRenderBuffer(0)->getTexelFormat(), resampledFramebufferSize.x, resampledFramebufferSize.y, gpu::Texture::SINGLE_MIP, sampler); - _destinationFrameBuffer->setRenderBuffer(0, target); - } return _destinationFrameBuffer; } @@ -84,9 +87,11 @@ void ToneMapAndResample::render(RenderArgs* args, const gpu::TexturePointer& lig return; } - //auto framebufferSize = glm::ivec2(lightingBuffer->getDimensions()); + const auto destBufferSize = destinationFramebuffer->getSize(); - auto framebufferSize = destinationFramebuffer->getSize(); + auto srcBufferSize = glm::ivec2(lightingBuffer->getDimensions()); + + glm::ivec4 destViewport{ 0, 0, destBufferSize.x, destBufferSize.y }; gpu::doInBatch("ToneMapAndResample::render", args->_context, [&](gpu::Batch& batch) { batch.enableStereo(false); @@ -95,10 +100,10 @@ void ToneMapAndResample::render(RenderArgs* args, const gpu::TexturePointer& lig // FIXME: Generate the Luminosity map //batch.generateTextureMips(lightingBuffer); - batch.setViewportTransform(args->_viewport); + batch.setViewportTransform(destViewport); batch.setProjectionTransform(glm::mat4()); batch.resetViewTransform(); - batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(framebufferSize, args->_viewport)); + batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(srcBufferSize, args->_viewport)); batch.setPipeline(args->_renderMode == RenderArgs::MIRROR_RENDER_MODE ? _mirrorPipeline : _pipeline); batch.setUniformBuffer(render_utils::slot::buffer::ToneMappingParams, _parametersBuffer); @@ -106,13 +111,10 @@ void ToneMapAndResample::render(RenderArgs* args, const gpu::TexturePointer& lig batch.draw(gpu::TRIANGLE_STRIP, 4); }); - destinationFramebuffer = getResampledFrameBuffer(destinationFramebuffer); + //Set full final viewport + args->_viewport = destViewport; - const auto bufferSize = destinationFramebuffer->getSize(); - glm::ivec4 viewport{ 0, 0, bufferSize.x, bufferSize.y }; - - //Set full final viewport - args->_viewport = viewport; + // TODO access violation reading 0xFFFFFFFFFFFFFFFF } void ToneMapAndResample::configure(const Config& config) { From b59d967b6c2a8a50662e26cadb7f9cc16513dbf3 Mon Sep 17 00:00:00 2001 From: Anna Date: Thu, 11 Jul 2019 16:44:56 -0700 Subject: [PATCH 03/41] fixed luci, made tone mapping mirrored auto generate, changed color format on forward, bug fixes --- libraries/render-utils/src/BloomEffect.cpp | 3 +- .../src/DeferredLightingEffect.cpp | 8 ++ .../render-utils/src/RenderDeferredTask.cpp | 7 +- .../render-utils/src/RenderForwardTask.cpp | 17 ++- .../render-utils/src/RenderForwardTask.h | 2 + .../src/ToneMapAndResampleTask.cpp | 105 +++++++----------- .../render-utils/src/ToneMapAndResampleTask.h | 2 +- .../src/render-utils/toneMapping.slp | 1 + libraries/render-utils/src/toneMapping.slf | 4 + .../utilities/render/deferredLighting.qml | 2 +- 10 files changed, 69 insertions(+), 82 deletions(-) diff --git a/libraries/render-utils/src/BloomEffect.cpp b/libraries/render-utils/src/BloomEffect.cpp index 10df138fad..a069aa60fa 100644 --- a/libraries/render-utils/src/BloomEffect.cpp +++ b/libraries/render-utils/src/BloomEffect.cpp @@ -60,7 +60,8 @@ void BloomThreshold::run(const render::RenderContextPointer& renderContext, cons auto colorTexture = gpu::TexturePointer(gpu::Texture::createRenderBuffer(inputBuffer->getTexelFormat(), bufferSize.x, bufferSize.y, gpu::Texture::SINGLE_MIP, gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR_MIP_POINT, gpu::Sampler::WRAP_CLAMP))); - _outputBuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("BloomThreshold")); + //_outputBuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("BloomThreshold")); + _outputBuffer = args->_blitFramebuffer; _outputBuffer->setRenderBuffer(0, colorTexture); _parameters.edit()._deltaUV = { 1.0f / bufferSize.x, 1.0f / bufferSize.y }; diff --git a/libraries/render-utils/src/DeferredLightingEffect.cpp b/libraries/render-utils/src/DeferredLightingEffect.cpp index 3b7c66d42d..88b9f09161 100644 --- a/libraries/render-utils/src/DeferredLightingEffect.cpp +++ b/libraries/render-utils/src/DeferredLightingEffect.cpp @@ -29,6 +29,10 @@ #include "TextureCache.h" #include "FramebufferCache.h" +#include +#include +#include + namespace ru { using render_utils::slot::texture::Texture; using render_utils::slot::buffer::Buffer; @@ -284,6 +288,10 @@ void PrepareDeferred::run(const RenderContextPointer& renderContext, const Input } _deferredFramebuffer->updatePrimaryDepth(primaryFramebuffer->getDepthStencilBuffer()); + auto dummy = _deferredFramebuffer->getFrameSize(); + auto height = _deferredFramebuffer->getLightingFramebuffer()->getHeight(); + auto width = _deferredFramebuffer->getLightingFramebuffer()->getWidth(); + outputs.edit0() = _deferredFramebuffer; outputs.edit1() = _deferredFramebuffer->getLightingFramebuffer(); diff --git a/libraries/render-utils/src/RenderDeferredTask.cpp b/libraries/render-utils/src/RenderDeferredTask.cpp index a4f1d6b6c9..eafc2463f5 100644 --- a/libraries/render-utils/src/RenderDeferredTask.cpp +++ b/libraries/render-utils/src/RenderDeferredTask.cpp @@ -95,7 +95,7 @@ RenderDeferredTask::RenderDeferredTask() void RenderDeferredTask::configure(const Config& config) { // Propagate resolution scale to sub jobs who need it - auto preparePrimaryBufferConfig = config.getConfig("PreparePrimaryBuffer"); + auto preparePrimaryBufferConfig = config.getConfig("PreparePrimaryBufferDeferred"); assert(preparePrimaryBufferConfig); preparePrimaryBufferConfig->setResolutionScale(config.resolutionScale); } @@ -145,7 +145,7 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren const auto jitter = task.addJob("JitterCam"); // GPU jobs: Start preparing the primary, deferred and lighting buffer - const auto scaledPrimaryFramebuffer = task.addJob("PreparePrimaryBuffer"); + const auto scaledPrimaryFramebuffer = task.addJob("PreparePrimaryBufferDeferred"); // Prepare deferred, generate the shared Deferred Frame Transform. Only valid with the scaled frame buffer const auto deferredFrameTransform = task.addJob("DeferredFrameTransform", jitter); @@ -238,7 +238,7 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren task.addJob("Bloom", bloomInputs); // Lighting Buffer ready for tone mapping - const auto toneMappingInputs = ToneMapAndResample::Input(lightingFramebuffer, scaledPrimaryFramebuffer).asVarying(); + const auto toneMappingInputs = lightingFramebuffer; const auto toneMappedBuffer = task.addJob("ToneMapAndResample", toneMappingInputs); // Debugging task is happening in the "over" layer after tone mapping and just before HUD @@ -252,6 +252,7 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren // HUD Layer const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(toneMappedBuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); task.addJob("RenderHUDLayer", renderHUDLayerInputs); + } RenderDeferredTaskDebug::RenderDeferredTaskDebug() { diff --git a/libraries/render-utils/src/RenderForwardTask.cpp b/libraries/render-utils/src/RenderForwardTask.cpp index c16af23218..2c8a2b498d 100755 --- a/libraries/render-utils/src/RenderForwardTask.cpp +++ b/libraries/render-utils/src/RenderForwardTask.cpp @@ -34,6 +34,8 @@ #include "RenderCommonTask.h" #include "RenderHUDLayerTask.h" +#include + namespace ru { using render_utils::slot::texture::Texture; using render_utils::slot::buffer::Buffer; @@ -50,7 +52,7 @@ extern void initForwardPipelines(ShapePlumber& plumber); void RenderForwardTask::configure(const Config& config) { // Propagate resolution scale to sub jobs who need it - auto preparePrimaryBufferConfig = config.getConfig("PreparePrimaryBuffer"); + auto preparePrimaryBufferConfig = config.getConfig("PreparePrimaryBufferForward"); assert(preparePrimaryBufferConfig); preparePrimaryBufferConfig->setResolutionScale(config.resolutionScale); } @@ -98,9 +100,7 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend // GPU jobs: Start preparing the main framebuffer - const auto scaledPrimaryFramebuffer = task.addJob("PreparePrimaryBuffer"); - qDebug() << "anna forward " << "scaled " << renderContext->args->_viewport.z << " x " << renderContext->args->_viewport.w; - + const auto scaledPrimaryFramebuffer = task.addJob("PreparePrimaryBufferForward"); // Prepare deferred, generate the shared Deferred Frame Transform. Only valid with the scaled frame buffer const auto deferredFrameTransform = task.addJob("DeferredFrameTransform"); @@ -152,15 +152,11 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend #else const auto newResolvedFramebuffer = task.addJob("MakeResolvingFramebuffer"); - - // Just resolve the msaa const auto resolveInputs = ResolveFramebuffer::Inputs(scaledPrimaryFramebuffer, newResolvedFramebuffer).asVarying(); const auto resolvedFramebuffer = task.addJob("Resolve", resolveInputs); // Lighting Buffer ready for tone mapping - // Forward rendering on GLES doesn't support tonemapping to and from the same FBO, so we specify - // the output FBO as null, which causes the tonemapping to target the blit framebuffer - const auto toneMappingInputs = ToneMapAndResample::Input(resolvedFramebuffer, resolvedFramebuffer).asVarying(); + const auto toneMappingInputs = resolvedFramebuffer; const auto toneMappedBuffer = task.addJob("ToneMapAndResample", toneMappingInputs); #endif @@ -175,7 +171,8 @@ gpu::FramebufferPointer PreparePrimaryFramebufferMSAA::createFramebuffer(const c auto defaultSampler = gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR); - auto colorFormat = gpu::Element::COLOR_SRGBA_32; + //auto colorFormat = gpu::Element::COLOR_SRGBA_32; + auto colorFormat = gpu::Element(gpu::SCALAR, gpu::FLOAT, gpu::R11G11B10); auto colorTexture = gpu::Texture::createRenderBufferMultisample(colorFormat, frameSize.x, frameSize.y, numSamples, defaultSampler); framebuffer->setRenderBuffer(0, colorTexture); diff --git a/libraries/render-utils/src/RenderForwardTask.h b/libraries/render-utils/src/RenderForwardTask.h index 2abf248692..dd0abbc9ab 100755 --- a/libraries/render-utils/src/RenderForwardTask.h +++ b/libraries/render-utils/src/RenderForwardTask.h @@ -50,11 +50,13 @@ public: const float SCALE_RANGE_MIN = 0.1f; const float SCALE_RANGE_MAX = 2.0f; resolutionScale = std::max(SCALE_RANGE_MIN, std::min(SCALE_RANGE_MAX, scale)); + //emit dirty(); } int getNumSamples() const { return numSamples; } void setNumSamples(int num) { numSamples = std::max(1, std::min(32, num)); + emit dirty(); } signals: diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.cpp b/libraries/render-utils/src/ToneMapAndResampleTask.cpp index 474c56525a..5f3e643a69 100644 --- a/libraries/render-utils/src/ToneMapAndResampleTask.cpp +++ b/libraries/render-utils/src/ToneMapAndResampleTask.cpp @@ -38,7 +38,7 @@ void ToneMapAndResample::init(RenderArgs* args) { blitState->setColorWriteMask(true, true, true, true); _pipeline = gpu::PipelinePointer(gpu::Pipeline::create(gpu::Shader::createProgram(toneMapping), blitState)); - _mirrorPipeline = gpu::PipelinePointer(gpu::Pipeline::create(gpu::Shader::createProgram(toneMappingMirroredX), blitState)); + _mirrorPipeline = gpu::PipelinePointer(gpu::Pipeline::create(gpu::Shader::createProgram(toneMapping_mirrored), blitState)); } void ToneMapAndResample::setExposure(float exposure) { @@ -56,67 +56,6 @@ void ToneMapAndResample::setToneCurve(ToneCurve curve) { } } -gpu::FramebufferPointer ToneMapAndResample::getResampledFrameBuffer(const gpu::FramebufferPointer& sourceFramebuffer) { - if (_factor == 1.0f) { - return sourceFramebuffer; - } - - auto resampledFramebufferSize = glm::uvec2(glm::vec2(sourceFramebuffer->getSize()) * _factor); - - _destinationFrameBuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("ResampledOutput")); - - auto sampler = gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR); - auto target = gpu::Texture::createRenderBuffer(sourceFramebuffer->getRenderBuffer(0)->getTexelFormat(), resampledFramebufferSize.x, resampledFramebufferSize.y, gpu::Texture::SINGLE_MIP, sampler); - _destinationFrameBuffer->setRenderBuffer(0, target); - - auto depthFormat = gpu::Element(gpu::SCALAR, gpu::UINT32, gpu::DEPTH_STENCIL); // Depth24_Stencil8 texel format - auto primaryDepthTexture = gpu::Texture::createRenderBuffer(depthFormat, resampledFramebufferSize.x, resampledFramebufferSize.y, gpu::Texture::SINGLE_MIP, sampler); - _destinationFrameBuffer->setDepthStencilBuffer(primaryDepthTexture, depthFormat); - - return _destinationFrameBuffer; -} - -// TODO why was destination const -void ToneMapAndResample::render(RenderArgs* args, const gpu::TexturePointer& lightingBuffer, gpu::FramebufferPointer& destinationFramebuffer) { - - if (!_pipeline) { - init(args); - } - - if (!lightingBuffer || !destinationFramebuffer) { - return; - } - - const auto destBufferSize = destinationFramebuffer->getSize(); - - auto srcBufferSize = glm::ivec2(lightingBuffer->getDimensions()); - - glm::ivec4 destViewport{ 0, 0, destBufferSize.x, destBufferSize.y }; - - gpu::doInBatch("ToneMapAndResample::render", args->_context, [&](gpu::Batch& batch) { - batch.enableStereo(false); - batch.setFramebuffer(destinationFramebuffer); - - // FIXME: Generate the Luminosity map - //batch.generateTextureMips(lightingBuffer); - - batch.setViewportTransform(destViewport); - batch.setProjectionTransform(glm::mat4()); - batch.resetViewTransform(); - batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(srcBufferSize, args->_viewport)); - batch.setPipeline(args->_renderMode == RenderArgs::MIRROR_RENDER_MODE ? _mirrorPipeline : _pipeline); - - batch.setUniformBuffer(render_utils::slot::buffer::ToneMappingParams, _parametersBuffer); - batch.setResourceTexture(render_utils::slot::texture::ToneMappingColor, lightingBuffer); - batch.draw(gpu::TRIANGLE_STRIP, 4); - }); - - //Set full final viewport - args->_viewport = destViewport; - - // TODO access violation reading 0xFFFFFFFFFFFFFFFF -} - void ToneMapAndResample::configure(const Config& config) { setExposure(config.exposure); setToneCurve((ToneCurve)config.curve); @@ -129,13 +68,47 @@ void ToneMapAndResample::run(const render::RenderContextPointer& renderContext, RenderArgs* args = renderContext->args; - auto lightingBuffer = input.get0()->getRenderBuffer(0); + auto lightingBuffer = input->getRenderBuffer(0); - auto resampledFramebuffer = args->_blitFramebuffer; + auto blitFramebuffer = args->_blitFramebuffer; - render(args, lightingBuffer, resampledFramebuffer); + if (!_pipeline) { + init(args); + } - output = resampledFramebuffer; + if (!lightingBuffer || !blitFramebuffer) { + return; + } + + const auto blitBufferSize = blitFramebuffer->getSize(); + + auto srcBufferSize = glm::ivec2(lightingBuffer->getDimensions()); + + glm::ivec4 destViewport{ 0, 0, blitBufferSize.x, blitBufferSize.y }; + + gpu::doInBatch("ToneMapAndResample::render", args->_context, [&](gpu::Batch& batch) { + batch.enableStereo(false); + batch.setFramebuffer(blitFramebuffer); + + // FIXME: Generate the Luminosity map + //batch.generateTextureMips(lightingBuffer); + + batch.setViewportTransform(destViewport); + batch.setProjectionTransform(glm::mat4()); + batch.resetViewTransform(); + batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(srcBufferSize, args->_viewport)); + + batch.setPipeline(args->_renderMode == RenderArgs::MIRROR_RENDER_MODE ? _mirrorPipeline : _pipeline); + + batch.setUniformBuffer(render_utils::slot::buffer::ToneMappingParams, _parametersBuffer); + batch.setResourceTexture(render_utils::slot::texture::ToneMappingColor, lightingBuffer); + batch.draw(gpu::TRIANGLE_STRIP, 4); + }); + + //Set full final viewport + args->_viewport = destViewport; + + output = blitFramebuffer; } diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.h b/libraries/render-utils/src/ToneMapAndResampleTask.h index 439146e05f..e9e95534b7 100644 --- a/libraries/render-utils/src/ToneMapAndResampleTask.h +++ b/libraries/render-utils/src/ToneMapAndResampleTask.h @@ -59,7 +59,7 @@ public: ToneCurve getToneCurve() const { return (ToneCurve)_parametersBuffer.get()._toneCurve; } // Inputs: lightingFramebuffer, destinationFramebuffer - using Input = render::VaryingSet2; + using Input = gpu::FramebufferPointer; using Output = gpu::FramebufferPointer; using Config = ToneMappingConfig; using JobModel = render::Job::ModelIO; diff --git a/libraries/render-utils/src/render-utils/toneMapping.slp b/libraries/render-utils/src/render-utils/toneMapping.slp index d4d8ec4b01..614df45551 100644 --- a/libraries/render-utils/src/render-utils/toneMapping.slp +++ b/libraries/render-utils/src/render-utils/toneMapping.slp @@ -1 +1,2 @@ VERTEX gpu::vertex::DrawViewportQuadTransformTexcoord +DEFINES mirrored:f \ No newline at end of file diff --git a/libraries/render-utils/src/toneMapping.slf b/libraries/render-utils/src/toneMapping.slf index 4f7ed6374d..6de12b3a23 100644 --- a/libraries/render-utils/src/toneMapping.slf +++ b/libraries/render-utils/src/toneMapping.slf @@ -43,7 +43,11 @@ layout(location=0) in vec2 varTexCoord0; layout(location=0) out vec4 outFragColor; void main(void) { +<@if HIFI_USE_MIRROREDX@> + vec4 fragColorRaw = texture(colorMap, vec2(1.0 - varTexCoord0.x, varTexCoord0.y); +<@else@> vec4 fragColorRaw = texture(colorMap, varTexCoord0); +<@endif@> vec3 fragColor = fragColorRaw.xyz; vec3 srcColor = fragColor * getTwoPowExposure(); diff --git a/scripts/developer/utilities/render/deferredLighting.qml b/scripts/developer/utilities/render/deferredLighting.qml index 80ca8b09e1..f2891ddc55 100644 --- a/scripts/developer/utilities/render/deferredLighting.qml +++ b/scripts/developer/utilities/render/deferredLighting.qml @@ -45,7 +45,7 @@ Rectangle { anchors.right: parent.right spacing: 5 Repeater { - model: [ "MSAA:PrepareFramebuffer:numSamples:4:1" + model: [ "MSAA:PreparePrimaryBufferForward:numSamples:4:1" ] ConfigSlider { label: qsTr(modelData.split(":")[0]) From e129f887234026e06ab107f0f269eca52091e472 Mon Sep 17 00:00:00 2001 From: Anna Date: Thu, 11 Jul 2019 17:10:08 -0700 Subject: [PATCH 04/41] deleted tone mapping mirrored slf and slp since they auto generate --- .../src/ToneMapAndResampleTask.cpp | 1 - .../src/render-utils/toneMappingMirroredX.slp | 1 - .../render-utils/src/toneMappingMirroredX.slf | 71 ------------------- 3 files changed, 73 deletions(-) delete mode 100644 libraries/render-utils/src/render-utils/toneMappingMirroredX.slp delete mode 100644 libraries/render-utils/src/toneMappingMirroredX.slf diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.cpp b/libraries/render-utils/src/ToneMapAndResampleTask.cpp index 5f3e643a69..b6079ef0a7 100644 --- a/libraries/render-utils/src/ToneMapAndResampleTask.cpp +++ b/libraries/render-utils/src/ToneMapAndResampleTask.cpp @@ -33,7 +33,6 @@ void ToneMapAndResample::init(RenderArgs* args) { // shared_ptr to gpu::State gpu::StatePointer blitState = gpu::StatePointer(new gpu::State()); - // TODO why was this in the upsample task blitState->setDepthTest(gpu::State::DepthTest(false, false)); blitState->setColorWriteMask(true, true, true, true); diff --git a/libraries/render-utils/src/render-utils/toneMappingMirroredX.slp b/libraries/render-utils/src/render-utils/toneMappingMirroredX.slp deleted file mode 100644 index d4d8ec4b01..0000000000 --- a/libraries/render-utils/src/render-utils/toneMappingMirroredX.slp +++ /dev/null @@ -1 +0,0 @@ -VERTEX gpu::vertex::DrawViewportQuadTransformTexcoord diff --git a/libraries/render-utils/src/toneMappingMirroredX.slf b/libraries/render-utils/src/toneMappingMirroredX.slf deleted file mode 100644 index 7c510cdc1b..0000000000 --- a/libraries/render-utils/src/toneMappingMirroredX.slf +++ /dev/null @@ -1,71 +0,0 @@ -<@include gpu/Config.slh@> -<$VERSION_HEADER$> -// Generated on Sat Oct 24 09:34:37 2015 -// -// toneMapping.frag -// -// Draw texture 0 fetched at texcoord.xy -// -// Created by Sam Gateau on 6/22/2015 -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -<@include render-utils/ShaderConstants.h@> - -struct ToneMappingParams { - vec4 _exp_2powExp_s0_s1; - ivec4 _toneCurve_s0_s1_s2; -}; - -const float GAMMA_22 = 2.2; -const float INV_GAMMA_22 = 1.0 / 2.2; -const int ToneCurveNone = 0; -const int ToneCurveGamma22 = 1; -const int ToneCurveReinhard = 2; -const int ToneCurveFilmic = 3; - -LAYOUT(binding=RENDER_UTILS_BUFFER_TM_PARAMS) uniform toneMappingParamsBuffer { - ToneMappingParams params; -}; -float getTwoPowExposure() { - return params._exp_2powExp_s0_s1.y; -} -int getToneCurve() { - return params._toneCurve_s0_s1_s2.x; -} - -LAYOUT(binding=RENDER_UTILS_TEXTURE_TM_COLOR) uniform sampler2D colorMap; - -layout(location=0) in vec2 varTexCoord0; -layout(location=0) out vec4 outFragColor; - -void main(void) { - vec4 fragColorRaw = texture(colorMap, vec2(1 - varTexCoord0.x, varTexCoord0.y)); - vec3 fragColor = fragColorRaw.xyz; - - vec3 srcColor = fragColor * getTwoPowExposure(); - - int toneCurve = getToneCurve(); - vec3 tonedColor = srcColor; - if (toneCurve == ToneCurveFilmic) { - vec3 x = max(vec3(0.0), srcColor-0.004); - tonedColor = pow((x * (6.2 * x + 0.5)) / (x * (6.2 * x + 1.7) + 0.06), vec3(GAMMA_22)); - } else if (toneCurve == ToneCurveReinhard) { - tonedColor = srcColor/(1.0 + srcColor); - } else if (toneCurve == ToneCurveGamma22) { - // We use glEnable(GL_FRAMEBUFFER_SRGB), which automatically converts textures from RGB to SRGB - // when writing from an RGB framebuffer to an SRGB framebuffer (note that it doesn't do anything - // when writing from an SRGB framebuffer to an RGB framebuffer). - // Since the conversion happens automatically, we don't need to do anything in this shader - } else { - // toneCurve == ToneCurveNone - // For debugging purposes, we may want to see what the colors look like before the automatic OpenGL - // conversion mentioned above, so we undo it here - tonedColor = pow(srcColor, vec3(GAMMA_22)); - } - - outFragColor = vec4(tonedColor, 1.0); -} From 8d9760ae0c05e2634ed5e2b3ba7ddb9305798ad2 Mon Sep 17 00:00:00 2001 From: Anna Date: Fri, 12 Jul 2019 09:19:03 -0700 Subject: [PATCH 05/41] typo fix --- libraries/render-utils/src/ToneMapAndResampleTask.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.h b/libraries/render-utils/src/ToneMapAndResampleTask.h index e9e95534b7..ba7ca260d9 100644 --- a/libraries/render-utils/src/ToneMapAndResampleTask.h +++ b/libraries/render-utils/src/ToneMapAndResampleTask.h @@ -76,7 +76,7 @@ protected: float _factor{ 2.0f }; - gpu::FramebufferPointer ToneMapAndResample::getResampledFrameBuffer(const gpu::FramebufferPointer& sourceFramebuffer); + gpu::FramebufferPointer getResampledFrameBuffer(const gpu::FramebufferPointer& sourceFramebuffer); private: From 2cd2e357360ba6f994c77b3f3bba5f0dea59dc74 Mon Sep 17 00:00:00 2001 From: Anna Date: Fri, 12 Jul 2019 14:38:00 -0700 Subject: [PATCH 06/41] fixing jenkins build errors --- libraries/render-utils/src/DeferredLightingEffect.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/render-utils/src/DeferredLightingEffect.cpp b/libraries/render-utils/src/DeferredLightingEffect.cpp index 88b9f09161..c7cb90e7e2 100644 --- a/libraries/render-utils/src/DeferredLightingEffect.cpp +++ b/libraries/render-utils/src/DeferredLightingEffect.cpp @@ -29,7 +29,6 @@ #include "TextureCache.h" #include "FramebufferCache.h" -#include #include #include From 58a04c1ccfed2c3e06148fa680b4057e35b71d83 Mon Sep 17 00:00:00 2001 From: Anna Date: Tue, 16 Jul 2019 09:47:30 -0700 Subject: [PATCH 07/41] fix jenkins errors for real this time --- libraries/render-utils/src/RenderForwardTask.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/render-utils/src/RenderForwardTask.cpp b/libraries/render-utils/src/RenderForwardTask.cpp index 2c8a2b498d..cd57fae7bf 100755 --- a/libraries/render-utils/src/RenderForwardTask.cpp +++ b/libraries/render-utils/src/RenderForwardTask.cpp @@ -34,8 +34,6 @@ #include "RenderCommonTask.h" #include "RenderHUDLayerTask.h" -#include - namespace ru { using render_utils::slot::texture::Texture; using render_utils::slot::buffer::Buffer; From 45d1672edd30d9d544cef0b7fe6da3a80c90371b Mon Sep 17 00:00:00 2001 From: Anna Date: Tue, 16 Jul 2019 14:38:36 -0700 Subject: [PATCH 08/41] remove testing variables --- libraries/render-utils/src/DeferredLightingEffect.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libraries/render-utils/src/DeferredLightingEffect.cpp b/libraries/render-utils/src/DeferredLightingEffect.cpp index c7cb90e7e2..d042a8305b 100644 --- a/libraries/render-utils/src/DeferredLightingEffect.cpp +++ b/libraries/render-utils/src/DeferredLightingEffect.cpp @@ -287,10 +287,6 @@ void PrepareDeferred::run(const RenderContextPointer& renderContext, const Input } _deferredFramebuffer->updatePrimaryDepth(primaryFramebuffer->getDepthStencilBuffer()); - auto dummy = _deferredFramebuffer->getFrameSize(); - auto height = _deferredFramebuffer->getLightingFramebuffer()->getHeight(); - auto width = _deferredFramebuffer->getLightingFramebuffer()->getWidth(); - outputs.edit0() = _deferredFramebuffer; outputs.edit1() = _deferredFramebuffer->getLightingFramebuffer(); From ca24693f39f6871d45196521d20d0941ad066a04 Mon Sep 17 00:00:00 2001 From: Anna Date: Tue, 23 Jul 2019 16:03:20 -0700 Subject: [PATCH 09/41] applying android crash fix for testing purposes --- .../src/platform/backend/PlatformInstance.cpp | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/libraries/platform/src/platform/backend/PlatformInstance.cpp b/libraries/platform/src/platform/backend/PlatformInstance.cpp index 038521d398..2f68972c4a 100644 --- a/libraries/platform/src/platform/backend/PlatformInstance.cpp +++ b/libraries/platform/src/platform/backend/PlatformInstance.cpp @@ -104,34 +104,34 @@ void Instance::enumerateNics() { } } } - json Instance::getCPU(int index) { - assert(index <(int) _cpus.size()); - if (index >= (int)_cpus.size()) + assert(index < (int)_cpus.size()); + + if (index < 0 || (int)_cpus.size() <= index) return json(); return _cpus.at(index); } json Instance::getGPU(int index) { - assert(index <(int) _gpus.size()); + assert(index < (int)_gpus.size()); - if (index >=(int) _gpus.size()) + if (index < 0 || (int)_gpus.size() <= index) return json(); - + return _gpus.at(index); } - json Instance::getDisplay(int index) { - assert(index <(int) _displays.size()); - - if (index >=(int) _displays.size()) + assert(index < (int)_displays.size()); + + if (index < 0 || (int)_displays.size() <= index) return json(); return _displays.at(index); } + Instance::~Instance() { if (_cpus.size() > 0) { _cpus.clear(); From 27586727db5a70b461ae3bb4f603f7020f62e6a9 Mon Sep 17 00:00:00 2001 From: Anna Date: Wed, 24 Jul 2019 11:38:47 -0700 Subject: [PATCH 10/41] make tonemapandresampletask run on android --- libraries/render-utils/src/RenderForwardTask.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/render-utils/src/RenderForwardTask.cpp b/libraries/render-utils/src/RenderForwardTask.cpp index cd57fae7bf..367224f937 100755 --- a/libraries/render-utils/src/RenderForwardTask.cpp +++ b/libraries/render-utils/src/RenderForwardTask.cpp @@ -153,12 +153,12 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend const auto resolveInputs = ResolveFramebuffer::Inputs(scaledPrimaryFramebuffer, newResolvedFramebuffer).asVarying(); const auto resolvedFramebuffer = task.addJob("Resolve", resolveInputs); +#endif + // Lighting Buffer ready for tone mapping const auto toneMappingInputs = resolvedFramebuffer; const auto toneMappedBuffer = task.addJob("ToneMapAndResample", toneMappingInputs); -#endif - // HUD Layer const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(toneMappedBuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); task.addJob("RenderHUDLayer", renderHUDLayerInputs); From 1b0d8ee60366ab4b6fd7115d23ca8d9c512b61f1 Mon Sep 17 00:00:00 2001 From: Anna Date: Wed, 24 Jul 2019 13:37:33 -0700 Subject: [PATCH 11/41] removed typo --- libraries/render-utils/src/RenderForwardTask.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/render-utils/src/RenderForwardTask.cpp b/libraries/render-utils/src/RenderForwardTask.cpp index 367224f937..0d25763613 100755 --- a/libraries/render-utils/src/RenderForwardTask.cpp +++ b/libraries/render-utils/src/RenderForwardTask.cpp @@ -146,7 +146,7 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend const auto resolveInputs = ResolveFramebuffer::Inputs(scaledPrimaryFramebuffer, static_cast(nullptr)).asVarying(); const auto resolvedFramebuffer = task.addJob("Resolve", resolveInputs); - const auto toneMappedBuffer = resolvedFramebuffer; + //const auto toneMappedBuffer = resolvedFramebuffer; #else const auto newResolvedFramebuffer = task.addJob("MakeResolvingFramebuffer"); From 635ed89d456dae6238285672b4f72ad3235a1e2d Mon Sep 17 00:00:00 2001 From: Anna Date: Wed, 24 Jul 2019 17:05:59 -0700 Subject: [PATCH 12/41] fixed cmake --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5e4746418d..f8fd5b7637 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ endif() include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/macros/TargetPython.cmake") target_python() -if (WIN32) +if (WIN32 AND NOT HIFI_ANDROID) # Force x64 toolset set(CMAKE_GENERATOR_TOOLSET "host=x64" CACHE STRING "64-bit toolset" FORCE) endif() From 16595d9412da005364fa76a7e548703274e5109c Mon Sep 17 00:00:00 2001 From: Anna Date: Fri, 26 Jul 2019 11:56:25 -0700 Subject: [PATCH 13/41] created tonemapandresample --- libraries/render-utils/src/BloomEffect.cpp | 2 +- .../render-utils/src/RenderDeferredTask.cpp | 8 +- .../render-utils/src/RenderForwardTask.cpp | 7 +- .../src/render/ToneMapAndResampleTask.cpp | 180 ++++++++++++++++++ .../src/render/ToneMapAndResampleTask.h | 87 +++++++++ 5 files changed, 277 insertions(+), 7 deletions(-) create mode 100644 libraries/render/src/render/ToneMapAndResampleTask.cpp create mode 100644 libraries/render/src/render/ToneMapAndResampleTask.h diff --git a/libraries/render-utils/src/BloomEffect.cpp b/libraries/render-utils/src/BloomEffect.cpp index 414a1c3f91..523fbdbd2b 100644 --- a/libraries/render-utils/src/BloomEffect.cpp +++ b/libraries/render-utils/src/BloomEffect.cpp @@ -15,7 +15,7 @@ #include #include -#include +#include #include "render-utils/ShaderConstants.h" #define BLOOM_BLUR_LEVEL_COUNT 3 diff --git a/libraries/render-utils/src/RenderDeferredTask.cpp b/libraries/render-utils/src/RenderDeferredTask.cpp index c26f3b613c..8af3c81dfa 100644 --- a/libraries/render-utils/src/RenderDeferredTask.cpp +++ b/libraries/render-utils/src/RenderDeferredTask.cpp @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include "RenderHifi.h" #include "render-utils/ShaderConstants.h" @@ -251,11 +251,11 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren } // Upscale to finale resolution - const auto primaryFramebuffer = task.addJob("PrimaryBufferUpscale", toneMappedBuffer); + //const auto primaryFramebuffer = task.addJob("PrimaryBufferUpscale", toneMappedBuffer); // HUD Layer - const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(primaryFramebuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); - task.addJob("RenderHUDLayer", renderHUDLayerInputs); + //const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(primaryFramebuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); + //task.addJob("RenderHUDLayer", renderHUDLayerInputs); } RenderDeferredTaskDebug::RenderDeferredTaskDebug() { diff --git a/libraries/render-utils/src/RenderForwardTask.cpp b/libraries/render-utils/src/RenderForwardTask.cpp index b6b17ee376..0e3f036e6b 100755 --- a/libraries/render-utils/src/RenderForwardTask.cpp +++ b/libraries/render-utils/src/RenderForwardTask.cpp @@ -19,7 +19,8 @@ #include #include #include -#include +//#include +#include #include @@ -29,6 +30,7 @@ #include "ZoneRenderer.h" #include "FadeEffect.h" #include "ToneMappingEffect.h" +//#include "ToneMapAndResampleTask.h" #include "BackgroundStage.h" #include "FramebufferCache.h" #include "TextureCache.h" @@ -165,7 +167,8 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend #endif // Upscale to finale resolution - const auto primaryFramebuffer = task.addJob("PrimaryBufferUpscale", toneMappedBuffer); + //const auto primaryFramebuffer = task.addJob("ToneMapAndResample", toneMappedBuffer); + const auto primaryFramebuffer = task.addJob("ToneMapAndResample", toneMappedBuffer); // HUD Layer const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(primaryFramebuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); diff --git a/libraries/render/src/render/ToneMapAndResampleTask.cpp b/libraries/render/src/render/ToneMapAndResampleTask.cpp new file mode 100644 index 0000000000..c304f12f4a --- /dev/null +++ b/libraries/render/src/render/ToneMapAndResampleTask.cpp @@ -0,0 +1,180 @@ +// +// ResampleTask.cpp +// render/src/render +// +// Various to Resample or downsample textures into framebuffers. +// +// Created by Olivier Prat on 10/09/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#include "ToneMapAndResampleTask.h" + +#include +#include + +using namespace render; +using namespace shader::gpu::program; +/* +gpu::PipelinePointer HalfDownsample::_pipeline; + +HalfDownsample::HalfDownsample() { + +} + +void HalfDownsample::configure(const Config& config) { + +} + +gpu::FramebufferPointer HalfDownsample::getResampledFrameBuffer(const gpu::FramebufferPointer& sourceFramebuffer) { + auto resampledFramebufferSize = sourceFramebuffer->getSize(); + + resampledFramebufferSize.x /= 2U; + resampledFramebufferSize.y /= 2U; + + if (!_destinationFrameBuffer || resampledFramebufferSize != _destinationFrameBuffer->getSize()) { + _destinationFrameBuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("HalfOutput")); + + auto sampler = gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR_MIP_POINT); + auto target = gpu::Texture::createRenderBuffer(sourceFramebuffer->getRenderBuffer(0)->getTexelFormat(), resampledFramebufferSize.x, resampledFramebufferSize.y, gpu::Texture::SINGLE_MIP, sampler); + _destinationFrameBuffer->setRenderBuffer(0, target); + } + return _destinationFrameBuffer; +} + +void HalfDownsample::run(const RenderContextPointer& renderContext, const gpu::FramebufferPointer& sourceFramebuffer, gpu::FramebufferPointer& resampledFrameBuffer) { + assert(renderContext->args); + assert(renderContext->args->hasViewFrustum()); + RenderArgs* args = renderContext->args; + + resampledFrameBuffer = getResampledFrameBuffer(sourceFramebuffer); + + if (!_pipeline) { + gpu::ShaderPointer program = gpu::Shader::createProgram(shader::gpu::program::drawTransformUnitQuadTextureOpaque); + gpu::StatePointer state = gpu::StatePointer(new gpu::State()); + state->setDepthTest(gpu::State::DepthTest(false, false)); + _pipeline = gpu::Pipeline::create(program, state); + } + + const auto bufferSize = resampledFrameBuffer->getSize(); + glm::ivec4 viewport{ 0, 0, bufferSize.x, bufferSize.y }; + + gpu::doInBatch("HalfDownsample::run", args->_context, [&](gpu::Batch& batch) { + batch.enableStereo(false); + + batch.setFramebuffer(resampledFrameBuffer); + + batch.setViewportTransform(viewport); + batch.setProjectionTransform(glm::mat4()); + batch.resetViewTransform(); + batch.setPipeline(_pipeline); + + batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(bufferSize, viewport)); + batch.setResourceTexture(0, sourceFramebuffer->getRenderBuffer(0)); + batch.draw(gpu::TRIANGLE_STRIP, 4); + }); +} +*/ +gpu::PipelinePointer Resample::_pipeline; + +void Resample::configure(const Config& config) { + _factor = config.factor; +} + +gpu::FramebufferPointer Resample::getResampledFrameBuffer(const gpu::FramebufferPointer& sourceFramebuffer) { + if (_factor == 1.0f) { + return sourceFramebuffer; + } + + auto resampledFramebufferSize = glm::uvec2(glm::vec2(sourceFramebuffer->getSize()) * _factor); + + if (!_destinationFrameBuffer || resampledFramebufferSize != _destinationFrameBuffer->getSize()) { + _destinationFrameBuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("ResampledOutput")); + + auto sampler = gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR); + auto target = gpu::Texture::createRenderBuffer(sourceFramebuffer->getRenderBuffer(0)->getTexelFormat(), resampledFramebufferSize.x, resampledFramebufferSize.y, gpu::Texture::SINGLE_MIP, sampler); + _destinationFrameBuffer->setRenderBuffer(0, target); + } + return _destinationFrameBuffer; +} + +void Resample::run(const RenderContextPointer& renderContext, const gpu::FramebufferPointer& sourceFramebuffer, gpu::FramebufferPointer& resampledFrameBuffer) { + assert(renderContext->args); + assert(renderContext->args->hasViewFrustum()); + RenderArgs* args = renderContext->args; + + resampledFrameBuffer = getResampledFrameBuffer(sourceFramebuffer); + if (resampledFrameBuffer != sourceFramebuffer) { + if (!_pipeline) { + gpu::ShaderPointer program = gpu::Shader::createProgram(shader::gpu::program::drawTransformUnitQuadTextureOpaque); + gpu::StatePointer state = gpu::StatePointer(new gpu::State()); + state->setDepthTest(gpu::State::DepthTest(false, false)); + _pipeline = gpu::Pipeline::create(program, state); + } + + const auto bufferSize = resampledFrameBuffer->getSize(); + glm::ivec4 viewport{ 0, 0, bufferSize.x, bufferSize.y }; + + gpu::doInBatch("Resample::run", args->_context, [&](gpu::Batch& batch) { + batch.enableStereo(false); + + batch.setFramebuffer(resampledFrameBuffer); + + batch.setViewportTransform(viewport); + batch.setProjectionTransform(glm::mat4()); + batch.resetViewTransform(); + batch.setPipeline(_pipeline); + + batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(bufferSize, viewport)); + batch.setResourceTexture(0, sourceFramebuffer->getRenderBuffer(0)); + batch.draw(gpu::TRIANGLE_STRIP, 4); + }); + + // Set full final viewport + args->_viewport = viewport; + } +} + +gpu::PipelinePointer ResampleToBlitFramebuffer::_pipeline; +gpu::PipelinePointer ResampleToBlitFramebuffer::_mirrorPipeline; + +void ResampleToBlitFramebuffer::run(const RenderContextPointer& renderContext, const Input& input, gpu::FramebufferPointer& resampledFrameBuffer) { + assert(renderContext->args); + assert(renderContext->args->hasViewFrustum()); + RenderArgs* args = renderContext->args; + auto sourceFramebuffer = input; + + resampledFrameBuffer = args->_blitFramebuffer; + + if (resampledFrameBuffer != sourceFramebuffer) { + if (!_pipeline) { + gpu::StatePointer state = gpu::StatePointer(new gpu::State()); + state->setDepthTest(gpu::State::DepthTest(false, false)); + + _pipeline = gpu::Pipeline::create(gpu::Shader::createProgram(drawTransformUnitQuadTextureOpaque), state); + _mirrorPipeline = gpu::Pipeline::create(gpu::Shader::createProgram(DrawTextureMirroredX), state); + } + const auto bufferSize = resampledFrameBuffer->getSize(); + glm::ivec4 viewport{ 0, 0, bufferSize.x, bufferSize.y }; + + gpu::doInBatch("Resample::run", args->_context, [&](gpu::Batch& batch) { + batch.enableStereo(false); + + batch.setFramebuffer(resampledFrameBuffer); + + batch.setViewportTransform(viewport); + batch.setProjectionTransform(glm::mat4()); + batch.resetViewTransform(); + batch.setPipeline(args->_renderMode == RenderArgs::MIRROR_RENDER_MODE ? _mirrorPipeline : _pipeline); + + batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(bufferSize, viewport)); + batch.setResourceTexture(0, sourceFramebuffer->getRenderBuffer(0)); + batch.draw(gpu::TRIANGLE_STRIP, 4); + }); + + // Set full final viewport + args->_viewport = viewport; + } +} \ No newline at end of file diff --git a/libraries/render/src/render/ToneMapAndResampleTask.h b/libraries/render/src/render/ToneMapAndResampleTask.h new file mode 100644 index 0000000000..f900bf85fa --- /dev/null +++ b/libraries/render/src/render/ToneMapAndResampleTask.h @@ -0,0 +1,87 @@ +// +// ResampleTask.h +// render/src/render +// +// Various to upsample or downsample textures into framebuffers. +// +// Created by Olivier Prat on 10/09/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_render_ToneMapAndResampleTask_h +#define hifi_render_ToneMapAndResampleTask_h + +#include "Engine.h" + +namespace render { +/* + class HalfDownsample { + public: + using Config = JobConfig; + using JobModel = Job::ModelIO; + + HalfDownsample(); + + void configure(const Config& config); + void run(const RenderContextPointer& renderContext, const gpu::FramebufferPointer& sourceFramebuffer, gpu::FramebufferPointer& resampledFrameBuffer); + + protected: + + static gpu::PipelinePointer _pipeline; + + gpu::FramebufferPointer _destinationFrameBuffer; + + gpu::FramebufferPointer getResampledFrameBuffer(const gpu::FramebufferPointer& sourceFramebuffer); + }; +*/ + class ResampleConfig : public render::Job::Config { + Q_OBJECT + Q_PROPERTY(float factor MEMBER factor NOTIFY dirty) + public: + + float factor{ 1.0f }; + + signals: + void dirty(); + }; + + class Resample { + public: + using Config = ResampleConfig; + using JobModel = Job::ModelIO; + + Resample(float factor = 2.0f) : _factor{ factor } {} + + void configure(const Config& config); + void run(const RenderContextPointer& renderContext, const gpu::FramebufferPointer& sourceFramebuffer, gpu::FramebufferPointer& resampledFrameBuffer); + + protected: + + static gpu::PipelinePointer _pipeline; + + gpu::FramebufferPointer _destinationFrameBuffer; + float _factor{ 2.0f }; + + gpu::FramebufferPointer getResampledFrameBuffer(const gpu::FramebufferPointer& sourceFramebuffer); + }; + + class ResampleToBlitFramebuffer { + public: + using Input = gpu::FramebufferPointer; + using JobModel = Job::ModelIO; + + ResampleToBlitFramebuffer() {} + + void run(const RenderContextPointer& renderContext, const Input& input, gpu::FramebufferPointer& resampledFrameBuffer); + + protected: + + static gpu::PipelinePointer _pipeline; + static gpu::PipelinePointer _mirrorPipeline; + }; +} + +#endif // hifi_render_ResampleTask_h \ No newline at end of file From 0d2b2a2aefc60aa3547c09066b2bf11f723cc0c1 Mon Sep 17 00:00:00 2001 From: Anna Date: Fri, 26 Jul 2019 12:53:18 -0700 Subject: [PATCH 14/41] added tone mapping config --- .../render-utils/src/RenderForwardTask.cpp | 2 +- .../src/render/ToneMapAndResampleTask.cpp | 52 ++++++++++++- .../src/render/ToneMapAndResampleTask.h | 74 +++++++++++++++++-- 3 files changed, 118 insertions(+), 10 deletions(-) diff --git a/libraries/render-utils/src/RenderForwardTask.cpp b/libraries/render-utils/src/RenderForwardTask.cpp index 0e3f036e6b..fb2222f98c 100755 --- a/libraries/render-utils/src/RenderForwardTask.cpp +++ b/libraries/render-utils/src/RenderForwardTask.cpp @@ -168,7 +168,7 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend // Upscale to finale resolution //const auto primaryFramebuffer = task.addJob("ToneMapAndResample", toneMappedBuffer); - const auto primaryFramebuffer = task.addJob("ToneMapAndResample", toneMappedBuffer); + const auto primaryFramebuffer = task.addJob("ToneMapAndResample", toneMappedBuffer); // HUD Layer const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(primaryFramebuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); diff --git a/libraries/render/src/render/ToneMapAndResampleTask.cpp b/libraries/render/src/render/ToneMapAndResampleTask.cpp index c304f12f4a..b551aa14ce 100644 --- a/libraries/render/src/render/ToneMapAndResampleTask.cpp +++ b/libraries/render/src/render/ToneMapAndResampleTask.cpp @@ -77,6 +77,7 @@ void HalfDownsample::run(const RenderContextPointer& renderContext, const gpu::F }); } */ +/* gpu::PipelinePointer Resample::_pipeline; void Resample::configure(const Config& config) { @@ -136,32 +137,74 @@ void Resample::run(const RenderContextPointer& renderContext, const gpu::Framebu args->_viewport = viewport; } } +*/ +ToneMapAndResample::ToneMapAndResample() { + Parameters parameters; + _parametersBuffer = gpu::BufferView(std::make_shared(sizeof(Parameters), (const gpu::Byte*) ¶meters)); +} -gpu::PipelinePointer ResampleToBlitFramebuffer::_pipeline; -gpu::PipelinePointer ResampleToBlitFramebuffer::_mirrorPipeline; +void ToneMapAndResample::setExposure(float exposure) { + auto& params = _parametersBuffer.get(); + if (params._exposure != exposure) { + _parametersBuffer.edit()._exposure = exposure; + _parametersBuffer.edit()._twoPowExposure = pow(2.0, exposure); + } +} -void ResampleToBlitFramebuffer::run(const RenderContextPointer& renderContext, const Input& input, gpu::FramebufferPointer& resampledFrameBuffer) { +void ToneMapAndResample::setToneCurve(ToneCurve curve) { + auto& params = _parametersBuffer.get(); + if (params._toneCurve != (int)curve) { + _parametersBuffer.edit()._toneCurve = (int)curve; + } +} + +void ToneMapAndResample::configure(const Config& config) { + setExposure(config.exposure); + setToneCurve((ToneCurve)config.curve); +} + +gpu::PipelinePointer ToneMapAndResample::_pipeline; +gpu::PipelinePointer ToneMapAndResample::_mirrorPipeline; + +void ToneMapAndResample::run(const RenderContextPointer& renderContext, const Input& input, gpu::FramebufferPointer& resampledFrameBuffer) { assert(renderContext->args); assert(renderContext->args->hasViewFrustum()); RenderArgs* args = renderContext->args; auto sourceFramebuffer = input; + //auto lightingBuffer = input->getRenderBuffer(0); + resampledFrameBuffer = args->_blitFramebuffer; +/* + if (!_pipeline) { + init(args); + } + + if (!lightingBuffer || !blitFramebuffer) { + return; + } + */ if (resampledFrameBuffer != sourceFramebuffer) { + if (!_pipeline) { gpu::StatePointer state = gpu::StatePointer(new gpu::State()); + state->setDepthTest(gpu::State::DepthTest(false, false)); + //blitState->setColorWriteMask(true, true, true, true); _pipeline = gpu::Pipeline::create(gpu::Shader::createProgram(drawTransformUnitQuadTextureOpaque), state); _mirrorPipeline = gpu::Pipeline::create(gpu::Shader::createProgram(DrawTextureMirroredX), state); } + const auto bufferSize = resampledFrameBuffer->getSize(); + + //auto srcBufferSize = glm::ivec2(lightingBuffer->getDimensions()); + glm::ivec4 viewport{ 0, 0, bufferSize.x, bufferSize.y }; gpu::doInBatch("Resample::run", args->_context, [&](gpu::Batch& batch) { batch.enableStereo(false); - batch.setFramebuffer(resampledFrameBuffer); batch.setViewportTransform(viewport); @@ -169,6 +212,7 @@ void ResampleToBlitFramebuffer::run(const RenderContextPointer& renderContext, c batch.resetViewTransform(); batch.setPipeline(args->_renderMode == RenderArgs::MIRROR_RENDER_MODE ? _mirrorPipeline : _pipeline); + // viewport = args->_viewport ?? batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(bufferSize, viewport)); batch.setResourceTexture(0, sourceFramebuffer->getRenderBuffer(0)); batch.draw(gpu::TRIANGLE_STRIP, 4); diff --git a/libraries/render/src/render/ToneMapAndResampleTask.h b/libraries/render/src/render/ToneMapAndResampleTask.h index f900bf85fa..1365cabf71 100644 --- a/libraries/render/src/render/ToneMapAndResampleTask.h +++ b/libraries/render/src/render/ToneMapAndResampleTask.h @@ -17,6 +17,31 @@ #include "Engine.h" namespace render { + + enum class ToneCurve { + // Different tone curve available + None, + Gamma22, + Reinhard, + Filmic, + }; + + class ToneMappingConfig : public render::Job::Config { + Q_OBJECT + Q_PROPERTY(float exposure MEMBER exposure WRITE setExposure); + Q_PROPERTY(int curve MEMBER curve WRITE setCurve); + + public: + ToneMappingConfig() : render::Job::Config(true) {} + + void setExposure(float newExposure) { exposure = newExposure; emit dirty(); } + void setCurve(int newCurve) { curve = std::max((int)ToneCurve::None, std::min((int)ToneCurve::Filmic, newCurve)); emit dirty(); } + + float exposure{ 0.0f }; + int curve{ (int)ToneCurve::Gamma22 }; + signals: + void dirty(); + }; /* class HalfDownsample { public: @@ -37,6 +62,7 @@ namespace render { gpu::FramebufferPointer getResampledFrameBuffer(const gpu::FramebufferPointer& sourceFramebuffer); }; */ +/* class ResampleConfig : public render::Job::Config { Q_OBJECT Q_PROPERTY(float factor MEMBER factor NOTIFY dirty) @@ -67,20 +93,58 @@ namespace render { gpu::FramebufferPointer getResampledFrameBuffer(const gpu::FramebufferPointer& sourceFramebuffer); }; - - class ResampleToBlitFramebuffer { + */ + class ToneMapAndResample{ public: + ToneMapAndResample(); + virtual ~ToneMapAndResample() {} + + void setExposure(float exposure); + float getExposure() const { return _parametersBuffer.get()._exposure; } + + void setToneCurve(ToneCurve curve); + ToneCurve getToneCurve() const { return (ToneCurve)_parametersBuffer.get()._toneCurve; } + + // Inputs: lightingFramebuffer, destinationFramebuffer using Input = gpu::FramebufferPointer; - using JobModel = Job::ModelIO; - - ResampleToBlitFramebuffer() {} + using Output = gpu::FramebufferPointer; + using Config = ToneMappingConfig; + //using JobModel = render::Job::ModelIO; + using JobModel = Job::ModelIO; + void configure(const Config& config); void run(const RenderContextPointer& renderContext, const Input& input, gpu::FramebufferPointer& resampledFrameBuffer); protected: static gpu::PipelinePointer _pipeline; static gpu::PipelinePointer _mirrorPipeline; + + gpu::FramebufferPointer _destinationFrameBuffer; + + float _factor{ 2.0f }; + + gpu::FramebufferPointer getResampledFrameBuffer(const gpu::FramebufferPointer& sourceFramebuffer); + + private: + + gpu::PipelinePointer _blitLightBuffer; + + // Class describing the uniform buffer with all the parameters common to the tone mapping shaders + class Parameters { + public: + float _exposure = 0.0f; + float _twoPowExposure = 1.0f; + glm::vec2 spareA; + int _toneCurve = (int)ToneCurve::Gamma22; + glm::vec3 spareB; + + Parameters() {} + }; + typedef gpu::BufferView UniformBufferView; + gpu::BufferView _parametersBuffer; + + void init(RenderArgs* args); }; } From 6a6b2f6d27c3f07125f9432ed6de78858e181355 Mon Sep 17 00:00:00 2001 From: Anna Date: Fri, 26 Jul 2019 18:55:08 -0700 Subject: [PATCH 15/41] added tone mapping back --- interface/src/ui/Snapshot.cpp | 2 +- libraries/render-utils/src/BloomEffect.cpp | 2 +- .../render-utils/src/RenderDeferredTask.cpp | 8 +- .../render-utils/src/RenderForwardTask.cpp | 9 +- .../src/ToneMapAndResampleTask.cpp | 109 +++++++++ .../src}/ToneMapAndResampleTask.h | 17 +- .../render-utils/src/ToneMappingEffect.cpp | 96 -------- .../render-utils/src/ToneMappingEffect.h | 96 -------- .../src/render-utils/toneMapping.slp | 1 + libraries/render-utils/src/toneMapping.slf | 6 +- .../src/render/ToneMapAndResampleTask.cpp | 224 ------------------ 11 files changed, 136 insertions(+), 434 deletions(-) create mode 100644 libraries/render-utils/src/ToneMapAndResampleTask.cpp rename libraries/{render/src/render => render-utils/src}/ToneMapAndResampleTask.h (91%) delete mode 100644 libraries/render-utils/src/ToneMappingEffect.cpp delete mode 100644 libraries/render-utils/src/ToneMappingEffect.h delete mode 100644 libraries/render/src/render/ToneMapAndResampleTask.cpp diff --git a/interface/src/ui/Snapshot.cpp b/interface/src/ui/Snapshot.cpp index 926588e4ca..363e7a5cf1 100644 --- a/interface/src/ui/Snapshot.cpp +++ b/interface/src/ui/Snapshot.cpp @@ -41,7 +41,7 @@ #include "MainWindow.h" #include "Snapshot.h" #include "SnapshotUploader.h" -#include "ToneMappingEffect.h" +#include "ToneMapAndResampleTask.h" // filename format: hifi-snap-by-%username%-on-%date%_%time%_@-%location%.jpg // %1 <= username, %2 <= date and time, %3 <= current location diff --git a/libraries/render-utils/src/BloomEffect.cpp b/libraries/render-utils/src/BloomEffect.cpp index 523fbdbd2b..10df138fad 100644 --- a/libraries/render-utils/src/BloomEffect.cpp +++ b/libraries/render-utils/src/BloomEffect.cpp @@ -15,7 +15,7 @@ #include #include -#include +#include "ToneMapAndResampleTask.h" #include "render-utils/ShaderConstants.h" #define BLOOM_BLUR_LEVEL_COUNT 3 diff --git a/libraries/render-utils/src/RenderDeferredTask.cpp b/libraries/render-utils/src/RenderDeferredTask.cpp index 8af3c81dfa..301288da38 100644 --- a/libraries/render-utils/src/RenderDeferredTask.cpp +++ b/libraries/render-utils/src/RenderDeferredTask.cpp @@ -29,7 +29,6 @@ #include #include #include -#include #include "RenderHifi.h" #include "render-utils/ShaderConstants.h" @@ -51,7 +50,7 @@ #include "AmbientOcclusionEffect.h" #include "AntialiasingEffect.h" -#include "ToneMappingEffect.h" +#include "ToneMapAndResampleTask.h" #include "SubsurfaceScattering.h" #include "DrawHaze.h" #include "BloomEffect.h" @@ -239,8 +238,9 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren task.addJob("Bloom", bloomInputs); // Lighting Buffer ready for tone mapping - const auto toneMappingInputs = ToneMappingDeferred::Input(lightingFramebuffer, scaledPrimaryFramebuffer).asVarying(); - const auto toneMappedBuffer = task.addJob("ToneMapping", toneMappingInputs); + //TODO re-enable last 3 steps + //const auto toneMappingInputs = ToneMappingDeferred::Input(lightingFramebuffer, scaledPrimaryFramebuffer).asVarying(); + //const auto toneMappedBuffer = task.addJob("ToneMapping", toneMappingInputs); // Debugging task is happening in the "over" layer after tone mapping and just before HUD { // Debug the bounds of the rendered items, still look at the zbuffer diff --git a/libraries/render-utils/src/RenderForwardTask.cpp b/libraries/render-utils/src/RenderForwardTask.cpp index fb2222f98c..f7c62c9c8e 100755 --- a/libraries/render-utils/src/RenderForwardTask.cpp +++ b/libraries/render-utils/src/RenderForwardTask.cpp @@ -20,7 +20,7 @@ #include #include //#include -#include +//#include #include @@ -29,8 +29,8 @@ #include "StencilMaskPass.h" #include "ZoneRenderer.h" #include "FadeEffect.h" -#include "ToneMappingEffect.h" -//#include "ToneMapAndResampleTask.h" +//#include "ToneMappingEffect.h" +#include "ToneMapAndResampleTask.h" #include "BackgroundStage.h" #include "FramebufferCache.h" #include "TextureCache.h" @@ -167,8 +167,7 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend #endif // Upscale to finale resolution - //const auto primaryFramebuffer = task.addJob("ToneMapAndResample", toneMappedBuffer); - const auto primaryFramebuffer = task.addJob("ToneMapAndResample", toneMappedBuffer); + const auto primaryFramebuffer = task.addJob("ToneMapAndResample", toneMappedBuffer); // HUD Layer const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(primaryFramebuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.cpp b/libraries/render-utils/src/ToneMapAndResampleTask.cpp new file mode 100644 index 0000000000..5fc331fab9 --- /dev/null +++ b/libraries/render-utils/src/ToneMapAndResampleTask.cpp @@ -0,0 +1,109 @@ +// +// ResampleTask.cpp +// render/src/render +// +// Various to Resample or downsample textures into framebuffers. +// +// Created by Olivier Prat on 10/09/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#include "ToneMapAndResampleTask.h" + +#include +#include + +#include "render-utils/ShaderConstants.h" +#include "StencilMaskPass.h" +#include "FramebufferCache.h" + +using namespace render; +using namespace shader::gpu::program; +using namespace shader::render_utils::program; + +ToneMapAndResample::ToneMapAndResample() { + Parameters parameters; + _parametersBuffer = gpu::BufferView(std::make_shared(sizeof(Parameters), (const gpu::Byte*) ¶meters)); +} + +void ToneMapAndResample::setExposure(float exposure) { + auto& params = _parametersBuffer.get(); + if (params._exposure != exposure) { + _parametersBuffer.edit()._exposure = exposure; + _parametersBuffer.edit()._twoPowExposure = pow(2.0, exposure); + } +} + +void ToneMapAndResample::setToneCurve(ToneCurve curve) { + auto& params = _parametersBuffer.get(); + if (params._toneCurve != (int)curve) { + _parametersBuffer.edit()._toneCurve = (int)curve; + } +} + +void ToneMapAndResample::configure(const Config& config) { + setExposure(config.exposure); + setToneCurve((ToneCurve)config.curve); +} + +gpu::PipelinePointer ToneMapAndResample::_pipeline; +gpu::PipelinePointer ToneMapAndResample::_mirrorPipeline; + +void ToneMapAndResample::run(const RenderContextPointer& renderContext, const Input& input, gpu::FramebufferPointer& resampledFrameBuffer) { + assert(renderContext->args); + assert(renderContext->args->hasViewFrustum()); + RenderArgs* args = renderContext->args; + auto sourceFramebuffer = input; + + //auto lightingBuffer = input->getRenderBuffer(0); + + resampledFrameBuffer = args->_blitFramebuffer; +/* + if (!_pipeline) { + init(args); + } + + if (!lightingBuffer || !blitFramebuffer) { + return; + } + */ + + if (resampledFrameBuffer != sourceFramebuffer) { + + if (!_pipeline) { + gpu::StatePointer state = gpu::StatePointer(new gpu::State()); + + state->setDepthTest(gpu::State::DepthTest(false, false)); + //blitState->setColorWriteMask(true, true, true, true); + + _pipeline = gpu::Pipeline::create(gpu::Shader::createProgram(toneMapping), state); + _mirrorPipeline = gpu::Pipeline::create(gpu::Shader::createProgram(toneMapping_mirrored), state); + } + + const auto bufferSize = resampledFrameBuffer->getSize(); + + //auto srcBufferSize = glm::ivec2(lightingBuffer->getDimensions()); + + glm::ivec4 viewport{ 0, 0, bufferSize.x, bufferSize.y }; + + gpu::doInBatch("Resample::run", args->_context, [&](gpu::Batch& batch) { + batch.enableStereo(false); + batch.setFramebuffer(resampledFrameBuffer); + + batch.setViewportTransform(viewport); + batch.setProjectionTransform(glm::mat4()); + batch.resetViewTransform(); + batch.setPipeline(args->_renderMode == RenderArgs::MIRROR_RENDER_MODE ? _mirrorPipeline : _pipeline); + + // viewport = args->_viewport ?? + batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(bufferSize, viewport)); + batch.setResourceTexture(0, sourceFramebuffer->getRenderBuffer(0)); + batch.draw(gpu::TRIANGLE_STRIP, 4); + }); + + // Set full final viewport + args->_viewport = viewport; + } +} \ No newline at end of file diff --git a/libraries/render/src/render/ToneMapAndResampleTask.h b/libraries/render-utils/src/ToneMapAndResampleTask.h similarity index 91% rename from libraries/render/src/render/ToneMapAndResampleTask.h rename to libraries/render-utils/src/ToneMapAndResampleTask.h index 1365cabf71..6bf6fb3c85 100644 --- a/libraries/render/src/render/ToneMapAndResampleTask.h +++ b/libraries/render-utils/src/ToneMapAndResampleTask.h @@ -14,13 +14,18 @@ #ifndef hifi_render_ToneMapAndResampleTask_h #define hifi_render_ToneMapAndResampleTask_h -#include "Engine.h" +#include "render/Engine.h" -namespace render { +#include +#include +#include +#include + +//namespace render { enum class ToneCurve { // Different tone curve available - None, + None, Gamma22, Reinhard, Filmic, @@ -110,10 +115,10 @@ namespace render { using Output = gpu::FramebufferPointer; using Config = ToneMappingConfig; //using JobModel = render::Job::ModelIO; - using JobModel = Job::ModelIO; + using JobModel = render::Job::ModelIO; void configure(const Config& config); - void run(const RenderContextPointer& renderContext, const Input& input, gpu::FramebufferPointer& resampledFrameBuffer); + void run(const render::RenderContextPointer& renderContext, const Input& input, gpu::FramebufferPointer& resampledFrameBuffer); protected: @@ -146,6 +151,6 @@ namespace render { void init(RenderArgs* args); }; -} +//} #endif // hifi_render_ResampleTask_h \ No newline at end of file diff --git a/libraries/render-utils/src/ToneMappingEffect.cpp b/libraries/render-utils/src/ToneMappingEffect.cpp deleted file mode 100644 index b7cc5d3d80..0000000000 --- a/libraries/render-utils/src/ToneMappingEffect.cpp +++ /dev/null @@ -1,96 +0,0 @@ -// -// ToneMappingEffect.cpp -// libraries/render-utils/src -// -// Created by Sam Gateau on 12/7/2015. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "ToneMappingEffect.h" - -#include -#include - -#include "render-utils/ShaderConstants.h" -#include "StencilMaskPass.h" -#include "FramebufferCache.h" - - -ToneMappingEffect::ToneMappingEffect() { - Parameters parameters; - _parametersBuffer = gpu::BufferView(std::make_shared(sizeof(Parameters), (const gpu::Byte*) ¶meters)); -} - -void ToneMappingEffect::init(RenderArgs* args) { - auto blitProgram = gpu::Shader::createProgram(shader::render_utils::program::toneMapping); - - auto blitState = std::make_shared(); - blitState->setColorWriteMask(true, true, true, true); - _blitLightBuffer = gpu::PipelinePointer(gpu::Pipeline::create(blitProgram, blitState)); -} - -void ToneMappingEffect::setExposure(float exposure) { - auto& params = _parametersBuffer.get(); - if (params._exposure != exposure) { - _parametersBuffer.edit()._exposure = exposure; - _parametersBuffer.edit()._twoPowExposure = pow(2.0, exposure); - } -} - -void ToneMappingEffect::setToneCurve(ToneCurve curve) { - auto& params = _parametersBuffer.get(); - if (params._toneCurve != curve) { - _parametersBuffer.edit()._toneCurve = curve; - } -} - -void ToneMappingEffect::render(RenderArgs* args, const gpu::TexturePointer& lightingBuffer, const gpu::FramebufferPointer& destinationFramebuffer) { - if (!_blitLightBuffer) { - init(args); - } - - if (!lightingBuffer || !destinationFramebuffer) { - return; - } - - auto framebufferSize = glm::ivec2(lightingBuffer->getDimensions()); - gpu::doInBatch("ToneMappingEffect::render", args->_context, [&](gpu::Batch& batch) { - batch.enableStereo(false); - batch.setFramebuffer(destinationFramebuffer); - - // FIXME: Generate the Luminosity map - //batch.generateTextureMips(lightingBuffer); - - batch.setViewportTransform(args->_viewport); - batch.setProjectionTransform(glm::mat4()); - batch.resetViewTransform(); - batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(framebufferSize, args->_viewport)); - batch.setPipeline(_blitLightBuffer); - - batch.setUniformBuffer(render_utils::slot::buffer::ToneMappingParams, _parametersBuffer); - batch.setResourceTexture(render_utils::slot::texture::ToneMappingColor, lightingBuffer); - batch.draw(gpu::TRIANGLE_STRIP, 4); - }); -} - - -void ToneMappingDeferred::configure(const Config& config) { - _toneMappingEffect.setExposure(config.exposure); - _toneMappingEffect.setToneCurve((ToneMappingEffect::ToneCurve)config.curve); -} - -void ToneMappingDeferred::run(const render::RenderContextPointer& renderContext, const Input& input, Output& output) { - - auto lightingBuffer = input.get0()->getRenderBuffer(0); - auto destFbo = input.get1(); - - if (!destFbo) { - destFbo = renderContext->args->_blitFramebuffer; - } - - _toneMappingEffect.render(renderContext->args, lightingBuffer, destFbo); - output = destFbo; -} diff --git a/libraries/render-utils/src/ToneMappingEffect.h b/libraries/render-utils/src/ToneMappingEffect.h deleted file mode 100644 index faf6e514e9..0000000000 --- a/libraries/render-utils/src/ToneMappingEffect.h +++ /dev/null @@ -1,96 +0,0 @@ -// -// ToneMappingEffect.h -// libraries/render-utils/src -// -// Created by Sam Gateau on 12/7/2015. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_ToneMappingEffect_h -#define hifi_ToneMappingEffect_h - -#include -#include - -#include -#include -#include -#include - - -class ToneMappingEffect { -public: - ToneMappingEffect(); - virtual ~ToneMappingEffect() {} - - void render(RenderArgs* args, const gpu::TexturePointer& lightingBuffer, const gpu::FramebufferPointer& destinationBuffer); - - void setExposure(float exposure); - float getExposure() const { return _parametersBuffer.get()._exposure; } - - // Different tone curve available - enum ToneCurve { - None = 0, - Gamma22, - Reinhard, - Filmic, - }; - void setToneCurve(ToneCurve curve); - ToneCurve getToneCurve() const { return (ToneCurve)_parametersBuffer.get()._toneCurve; } - -private: - - gpu::PipelinePointer _blitLightBuffer; - - // Class describing the uniform buffer with all the parameters common to the tone mapping shaders - class Parameters { - public: - float _exposure = 0.0f; - float _twoPowExposure = 1.0f; - glm::vec2 spareA; - int _toneCurve = Gamma22; - glm::vec3 spareB; - - Parameters() {} - }; - typedef gpu::BufferView UniformBufferView; - gpu::BufferView _parametersBuffer; - - void init(RenderArgs* args); -}; - -class ToneMappingConfig : public render::Job::Config { - Q_OBJECT - Q_PROPERTY(float exposure MEMBER exposure WRITE setExposure); - Q_PROPERTY(int curve MEMBER curve WRITE setCurve); -public: - ToneMappingConfig() : render::Job::Config(true) {} - - void setExposure(float newExposure) { exposure = newExposure; emit dirty(); } - void setCurve(int newCurve) { curve = std::max((int)ToneMappingEffect::None, std::min((int)ToneMappingEffect::Filmic, newCurve)); emit dirty(); } - - - float exposure{ 0.0f }; - int curve{ ToneMappingEffect::Gamma22 }; -signals: - void dirty(); -}; - -class ToneMappingDeferred { -public: - // Inputs: lightingFramebuffer, destinationFramebuffer - using Input = render::VaryingSet2; - using Output = gpu::FramebufferPointer; - using Config = ToneMappingConfig; - using JobModel = render::Job::ModelIO; - - void configure(const Config& config); - void run(const render::RenderContextPointer& renderContext, const Input& input, Output& output); - - ToneMappingEffect _toneMappingEffect; -}; - -#endif // hifi_ToneMappingEffect_h diff --git a/libraries/render-utils/src/render-utils/toneMapping.slp b/libraries/render-utils/src/render-utils/toneMapping.slp index d4d8ec4b01..2bcb4497c4 100644 --- a/libraries/render-utils/src/render-utils/toneMapping.slp +++ b/libraries/render-utils/src/render-utils/toneMapping.slp @@ -1 +1,2 @@ VERTEX gpu::vertex::DrawViewportQuadTransformTexcoord +DEFINES mirrored:f diff --git a/libraries/render-utils/src/toneMapping.slf b/libraries/render-utils/src/toneMapping.slf index 4f7ed6374d..e0d1d6f452 100644 --- a/libraries/render-utils/src/toneMapping.slf +++ b/libraries/render-utils/src/toneMapping.slf @@ -43,7 +43,11 @@ layout(location=0) in vec2 varTexCoord0; layout(location=0) out vec4 outFragColor; void main(void) { - vec4 fragColorRaw = texture(colorMap, varTexCoord0); + <@if HIFI_USE_MIRRORED@> + vec4 fragColorRaw = texture(colorMap, vec2(1.0 - varTexCoord0.x, varTexCoord0.y)); + <@else@> + vec4 fragColorRaw = texture(colorMap, varTexCoord0); + <@endif@> vec3 fragColor = fragColorRaw.xyz; vec3 srcColor = fragColor * getTwoPowExposure(); diff --git a/libraries/render/src/render/ToneMapAndResampleTask.cpp b/libraries/render/src/render/ToneMapAndResampleTask.cpp deleted file mode 100644 index b551aa14ce..0000000000 --- a/libraries/render/src/render/ToneMapAndResampleTask.cpp +++ /dev/null @@ -1,224 +0,0 @@ -// -// ResampleTask.cpp -// render/src/render -// -// Various to Resample or downsample textures into framebuffers. -// -// Created by Olivier Prat on 10/09/17. -// Copyright 2017 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// -#include "ToneMapAndResampleTask.h" - -#include -#include - -using namespace render; -using namespace shader::gpu::program; -/* -gpu::PipelinePointer HalfDownsample::_pipeline; - -HalfDownsample::HalfDownsample() { - -} - -void HalfDownsample::configure(const Config& config) { - -} - -gpu::FramebufferPointer HalfDownsample::getResampledFrameBuffer(const gpu::FramebufferPointer& sourceFramebuffer) { - auto resampledFramebufferSize = sourceFramebuffer->getSize(); - - resampledFramebufferSize.x /= 2U; - resampledFramebufferSize.y /= 2U; - - if (!_destinationFrameBuffer || resampledFramebufferSize != _destinationFrameBuffer->getSize()) { - _destinationFrameBuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("HalfOutput")); - - auto sampler = gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR_MIP_POINT); - auto target = gpu::Texture::createRenderBuffer(sourceFramebuffer->getRenderBuffer(0)->getTexelFormat(), resampledFramebufferSize.x, resampledFramebufferSize.y, gpu::Texture::SINGLE_MIP, sampler); - _destinationFrameBuffer->setRenderBuffer(0, target); - } - return _destinationFrameBuffer; -} - -void HalfDownsample::run(const RenderContextPointer& renderContext, const gpu::FramebufferPointer& sourceFramebuffer, gpu::FramebufferPointer& resampledFrameBuffer) { - assert(renderContext->args); - assert(renderContext->args->hasViewFrustum()); - RenderArgs* args = renderContext->args; - - resampledFrameBuffer = getResampledFrameBuffer(sourceFramebuffer); - - if (!_pipeline) { - gpu::ShaderPointer program = gpu::Shader::createProgram(shader::gpu::program::drawTransformUnitQuadTextureOpaque); - gpu::StatePointer state = gpu::StatePointer(new gpu::State()); - state->setDepthTest(gpu::State::DepthTest(false, false)); - _pipeline = gpu::Pipeline::create(program, state); - } - - const auto bufferSize = resampledFrameBuffer->getSize(); - glm::ivec4 viewport{ 0, 0, bufferSize.x, bufferSize.y }; - - gpu::doInBatch("HalfDownsample::run", args->_context, [&](gpu::Batch& batch) { - batch.enableStereo(false); - - batch.setFramebuffer(resampledFrameBuffer); - - batch.setViewportTransform(viewport); - batch.setProjectionTransform(glm::mat4()); - batch.resetViewTransform(); - batch.setPipeline(_pipeline); - - batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(bufferSize, viewport)); - batch.setResourceTexture(0, sourceFramebuffer->getRenderBuffer(0)); - batch.draw(gpu::TRIANGLE_STRIP, 4); - }); -} -*/ -/* -gpu::PipelinePointer Resample::_pipeline; - -void Resample::configure(const Config& config) { - _factor = config.factor; -} - -gpu::FramebufferPointer Resample::getResampledFrameBuffer(const gpu::FramebufferPointer& sourceFramebuffer) { - if (_factor == 1.0f) { - return sourceFramebuffer; - } - - auto resampledFramebufferSize = glm::uvec2(glm::vec2(sourceFramebuffer->getSize()) * _factor); - - if (!_destinationFrameBuffer || resampledFramebufferSize != _destinationFrameBuffer->getSize()) { - _destinationFrameBuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("ResampledOutput")); - - auto sampler = gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR); - auto target = gpu::Texture::createRenderBuffer(sourceFramebuffer->getRenderBuffer(0)->getTexelFormat(), resampledFramebufferSize.x, resampledFramebufferSize.y, gpu::Texture::SINGLE_MIP, sampler); - _destinationFrameBuffer->setRenderBuffer(0, target); - } - return _destinationFrameBuffer; -} - -void Resample::run(const RenderContextPointer& renderContext, const gpu::FramebufferPointer& sourceFramebuffer, gpu::FramebufferPointer& resampledFrameBuffer) { - assert(renderContext->args); - assert(renderContext->args->hasViewFrustum()); - RenderArgs* args = renderContext->args; - - resampledFrameBuffer = getResampledFrameBuffer(sourceFramebuffer); - if (resampledFrameBuffer != sourceFramebuffer) { - if (!_pipeline) { - gpu::ShaderPointer program = gpu::Shader::createProgram(shader::gpu::program::drawTransformUnitQuadTextureOpaque); - gpu::StatePointer state = gpu::StatePointer(new gpu::State()); - state->setDepthTest(gpu::State::DepthTest(false, false)); - _pipeline = gpu::Pipeline::create(program, state); - } - - const auto bufferSize = resampledFrameBuffer->getSize(); - glm::ivec4 viewport{ 0, 0, bufferSize.x, bufferSize.y }; - - gpu::doInBatch("Resample::run", args->_context, [&](gpu::Batch& batch) { - batch.enableStereo(false); - - batch.setFramebuffer(resampledFrameBuffer); - - batch.setViewportTransform(viewport); - batch.setProjectionTransform(glm::mat4()); - batch.resetViewTransform(); - batch.setPipeline(_pipeline); - - batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(bufferSize, viewport)); - batch.setResourceTexture(0, sourceFramebuffer->getRenderBuffer(0)); - batch.draw(gpu::TRIANGLE_STRIP, 4); - }); - - // Set full final viewport - args->_viewport = viewport; - } -} -*/ -ToneMapAndResample::ToneMapAndResample() { - Parameters parameters; - _parametersBuffer = gpu::BufferView(std::make_shared(sizeof(Parameters), (const gpu::Byte*) ¶meters)); -} - -void ToneMapAndResample::setExposure(float exposure) { - auto& params = _parametersBuffer.get(); - if (params._exposure != exposure) { - _parametersBuffer.edit()._exposure = exposure; - _parametersBuffer.edit()._twoPowExposure = pow(2.0, exposure); - } -} - -void ToneMapAndResample::setToneCurve(ToneCurve curve) { - auto& params = _parametersBuffer.get(); - if (params._toneCurve != (int)curve) { - _parametersBuffer.edit()._toneCurve = (int)curve; - } -} - -void ToneMapAndResample::configure(const Config& config) { - setExposure(config.exposure); - setToneCurve((ToneCurve)config.curve); -} - -gpu::PipelinePointer ToneMapAndResample::_pipeline; -gpu::PipelinePointer ToneMapAndResample::_mirrorPipeline; - -void ToneMapAndResample::run(const RenderContextPointer& renderContext, const Input& input, gpu::FramebufferPointer& resampledFrameBuffer) { - assert(renderContext->args); - assert(renderContext->args->hasViewFrustum()); - RenderArgs* args = renderContext->args; - auto sourceFramebuffer = input; - - //auto lightingBuffer = input->getRenderBuffer(0); - - resampledFrameBuffer = args->_blitFramebuffer; -/* - if (!_pipeline) { - init(args); - } - - if (!lightingBuffer || !blitFramebuffer) { - return; - } - */ - - if (resampledFrameBuffer != sourceFramebuffer) { - - if (!_pipeline) { - gpu::StatePointer state = gpu::StatePointer(new gpu::State()); - - state->setDepthTest(gpu::State::DepthTest(false, false)); - //blitState->setColorWriteMask(true, true, true, true); - - _pipeline = gpu::Pipeline::create(gpu::Shader::createProgram(drawTransformUnitQuadTextureOpaque), state); - _mirrorPipeline = gpu::Pipeline::create(gpu::Shader::createProgram(DrawTextureMirroredX), state); - } - - const auto bufferSize = resampledFrameBuffer->getSize(); - - //auto srcBufferSize = glm::ivec2(lightingBuffer->getDimensions()); - - glm::ivec4 viewport{ 0, 0, bufferSize.x, bufferSize.y }; - - gpu::doInBatch("Resample::run", args->_context, [&](gpu::Batch& batch) { - batch.enableStereo(false); - batch.setFramebuffer(resampledFrameBuffer); - - batch.setViewportTransform(viewport); - batch.setProjectionTransform(glm::mat4()); - batch.resetViewTransform(); - batch.setPipeline(args->_renderMode == RenderArgs::MIRROR_RENDER_MODE ? _mirrorPipeline : _pipeline); - - // viewport = args->_viewport ?? - batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(bufferSize, viewport)); - batch.setResourceTexture(0, sourceFramebuffer->getRenderBuffer(0)); - batch.draw(gpu::TRIANGLE_STRIP, 4); - }); - - // Set full final viewport - args->_viewport = viewport; - } -} \ No newline at end of file From 8aef9db4b9710463966091425056a5d8586a2128 Mon Sep 17 00:00:00 2001 From: Anna Date: Fri, 26 Jul 2019 20:09:18 -0700 Subject: [PATCH 16/41] works on android! --- .../render-utils/src/RenderCommonTask.cpp | 24 ++++++++- libraries/render-utils/src/RenderCommonTask.h | 11 ++++ .../render-utils/src/RenderForwardTask.cpp | 16 +++--- .../src/ToneMapAndResampleTask.cpp | 20 ++++---- .../render-utils/src/ToneMapAndResampleTask.h | 51 ------------------- 5 files changed, 50 insertions(+), 72 deletions(-) diff --git a/libraries/render-utils/src/RenderCommonTask.cpp b/libraries/render-utils/src/RenderCommonTask.cpp index 9ea4ac9f3c..8868d2fe7c 100644 --- a/libraries/render-utils/src/RenderCommonTask.cpp +++ b/libraries/render-utils/src/RenderCommonTask.cpp @@ -148,6 +148,28 @@ void Blit::run(const RenderContextPointer& renderContext, const gpu::Framebuffer }); } +void NewFramebuffer::run(const render::RenderContextPointer& renderContext, const Input& input, Output& output) { + RenderArgs* args = renderContext->args; + // auto frameSize = input; + glm::uvec2 frameSize(args->_viewport.z, args->_viewport.w); + output.reset(); + + // Else use the lodal Framebuffer + if (_outputFramebuffer && _outputFramebuffer->getSize() != frameSize) { + _outputFramebuffer.reset(); + } + + if (!_outputFramebuffer) { + _outputFramebuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("newFramebuffer.out")); + auto colorFormat = gpu::Element(gpu::SCALAR, gpu::FLOAT, gpu::R11G11B10); + auto defaultSampler = gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR); + auto colorTexture = gpu::Texture::createRenderBuffer(colorFormat, frameSize.x, frameSize.y, gpu::Texture::SINGLE_MIP, defaultSampler); + _outputFramebuffer->setRenderBuffer(0, colorTexture); + } + + output = _outputFramebuffer; +} + void NewOrDefaultFramebuffer::run(const render::RenderContextPointer& renderContext, const Input& input, Output& output) { RenderArgs* args = renderContext->args; // auto frameSize = input; @@ -168,7 +190,7 @@ void NewOrDefaultFramebuffer::run(const render::RenderContextPointer& renderCont if (!_outputFramebuffer) { _outputFramebuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("newFramebuffer.out")); - auto colorFormat = gpu::Element::COLOR_SRGBA_32; + auto colorFormat = gpu::Element(gpu::SCALAR, gpu::FLOAT, gpu::R11G11B10); auto defaultSampler = gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR); auto colorTexture = gpu::Texture::createRenderBuffer(colorFormat, frameSize.x, frameSize.y, gpu::Texture::SINGLE_MIP, defaultSampler); _outputFramebuffer->setRenderBuffer(0, colorTexture); diff --git a/libraries/render-utils/src/RenderCommonTask.h b/libraries/render-utils/src/RenderCommonTask.h index ec50fbf2cc..a9bea8b672 100644 --- a/libraries/render-utils/src/RenderCommonTask.h +++ b/libraries/render-utils/src/RenderCommonTask.h @@ -83,6 +83,17 @@ public: void run(const render::RenderContextPointer& renderContext, const gpu::FramebufferPointer& srcFramebuffer); }; +class NewFramebuffer { +public: + using Input = glm::uvec2; + using Output = gpu::FramebufferPointer; + using JobModel = render::Job::ModelIO; + + void run(const render::RenderContextPointer& renderContext, const Input& input, Output& output); +private: + gpu::FramebufferPointer _outputFramebuffer; +}; + class NewOrDefaultFramebuffer { public: using Input = glm::uvec2; diff --git a/libraries/render-utils/src/RenderForwardTask.cpp b/libraries/render-utils/src/RenderForwardTask.cpp index f7c62c9c8e..6a39a937a0 100755 --- a/libraries/render-utils/src/RenderForwardTask.cpp +++ b/libraries/render-utils/src/RenderForwardTask.cpp @@ -143,7 +143,7 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend task.addJob("DrawZoneStack", debugZoneInputs); } -#if defined(Q_OS_ANDROID) +#if !defined(Q_OS_ANDROID) // Just resolve the msaa const auto resolveInputs = ResolveFramebuffer::Inputs(scaledPrimaryFramebuffer, static_cast(nullptr)).asVarying(); @@ -151,8 +151,7 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend const auto toneMappedBuffer = resolvedFramebuffer; #else - const auto newResolvedFramebuffer = task.addJob("MakeResolvingFramebuffer"); - + const auto newResolvedFramebuffer = task.addJob("MakeResolvingFramebuffer"); // Just resolve the msaa const auto resolveInputs = ResolveFramebuffer::Inputs(scaledPrimaryFramebuffer, newResolvedFramebuffer).asVarying(); @@ -161,13 +160,12 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend // Lighting Buffer ready for tone mapping // Forward rendering on GLES doesn't support tonemapping to and from the same FBO, so we specify // the output FBO as null, which causes the tonemapping to target the blit framebuffer - const auto toneMappingInputs = ToneMappingDeferred::Input(resolvedFramebuffer, resolvedFramebuffer).asVarying(); - const auto toneMappedBuffer = task.addJob("ToneMapping", toneMappingInputs); + //const auto toneMappingInputs = ToneMappingDeferred::Input(resolvedFramebuffer, resolvedFramebuffer).asVarying(); + //const auto toneMappedBuffer = task.addJob("ToneMapping", toneMappingInputs); #endif - // Upscale to finale resolution - const auto primaryFramebuffer = task.addJob("ToneMapAndResample", toneMappedBuffer); + const auto primaryFramebuffer = task.addJob("ToneMapAndResample", resolvedFramebuffer); // HUD Layer const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(primaryFramebuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); @@ -178,8 +176,8 @@ gpu::FramebufferPointer PreparePrimaryFramebufferMSAA::createFramebuffer(const c gpu::FramebufferPointer framebuffer = gpu::FramebufferPointer(gpu::Framebuffer::create(name)); auto defaultSampler = gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR); - - auto colorFormat = gpu::Element::COLOR_SRGBA_32; + + auto colorFormat = gpu::Element(gpu::SCALAR, gpu::FLOAT, gpu::R11G11B10); auto colorTexture = gpu::Texture::createRenderBufferMultisample(colorFormat, frameSize.x, frameSize.y, numSamples, defaultSampler); framebuffer->setRenderBuffer(0, colorTexture); diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.cpp b/libraries/render-utils/src/ToneMapAndResampleTask.cpp index 5fc331fab9..b67b70b76c 100644 --- a/libraries/render-utils/src/ToneMapAndResampleTask.cpp +++ b/libraries/render-utils/src/ToneMapAndResampleTask.cpp @@ -57,18 +57,14 @@ void ToneMapAndResample::run(const RenderContextPointer& renderContext, const In RenderArgs* args = renderContext->args; auto sourceFramebuffer = input; - //auto lightingBuffer = input->getRenderBuffer(0); + auto lightingBuffer = input->getRenderBuffer(0); resampledFrameBuffer = args->_blitFramebuffer; -/* - if (!_pipeline) { - init(args); - } +/* if (!lightingBuffer || !blitFramebuffer) { return; - } - */ + }*/ if (resampledFrameBuffer != sourceFramebuffer) { @@ -76,7 +72,7 @@ void ToneMapAndResample::run(const RenderContextPointer& renderContext, const In gpu::StatePointer state = gpu::StatePointer(new gpu::State()); state->setDepthTest(gpu::State::DepthTest(false, false)); - //blitState->setColorWriteMask(true, true, true, true); + state->setColorWriteMask(true, true, true, true); _pipeline = gpu::Pipeline::create(gpu::Shader::createProgram(toneMapping), state); _mirrorPipeline = gpu::Pipeline::create(gpu::Shader::createProgram(toneMapping_mirrored), state); @@ -84,7 +80,7 @@ void ToneMapAndResample::run(const RenderContextPointer& renderContext, const In const auto bufferSize = resampledFrameBuffer->getSize(); - //auto srcBufferSize = glm::ivec2(lightingBuffer->getDimensions()); + auto srcBufferSize = glm::ivec2(lightingBuffer->getDimensions()); glm::ivec4 viewport{ 0, 0, bufferSize.x, bufferSize.y }; @@ -98,8 +94,10 @@ void ToneMapAndResample::run(const RenderContextPointer& renderContext, const In batch.setPipeline(args->_renderMode == RenderArgs::MIRROR_RENDER_MODE ? _mirrorPipeline : _pipeline); // viewport = args->_viewport ?? - batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(bufferSize, viewport)); - batch.setResourceTexture(0, sourceFramebuffer->getRenderBuffer(0)); + batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(srcBufferSize, args->_viewport)); + batch.setUniformBuffer(render_utils::slot::buffer::ToneMappingParams, _parametersBuffer); + batch.setResourceTexture(render_utils::slot::texture::ToneMappingColor, lightingBuffer); + //batch.setResourceTexture(0, sourceFramebuffer->getRenderBuffer(0)); batch.draw(gpu::TRIANGLE_STRIP, 4); }); diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.h b/libraries/render-utils/src/ToneMapAndResampleTask.h index 6bf6fb3c85..1a4459bda3 100644 --- a/libraries/render-utils/src/ToneMapAndResampleTask.h +++ b/libraries/render-utils/src/ToneMapAndResampleTask.h @@ -47,58 +47,7 @@ signals: void dirty(); }; -/* - class HalfDownsample { - public: - using Config = JobConfig; - using JobModel = Job::ModelIO; - HalfDownsample(); - - void configure(const Config& config); - void run(const RenderContextPointer& renderContext, const gpu::FramebufferPointer& sourceFramebuffer, gpu::FramebufferPointer& resampledFrameBuffer); - - protected: - - static gpu::PipelinePointer _pipeline; - - gpu::FramebufferPointer _destinationFrameBuffer; - - gpu::FramebufferPointer getResampledFrameBuffer(const gpu::FramebufferPointer& sourceFramebuffer); - }; -*/ -/* - class ResampleConfig : public render::Job::Config { - Q_OBJECT - Q_PROPERTY(float factor MEMBER factor NOTIFY dirty) - public: - - float factor{ 1.0f }; - - signals: - void dirty(); - }; - - class Resample { - public: - using Config = ResampleConfig; - using JobModel = Job::ModelIO; - - Resample(float factor = 2.0f) : _factor{ factor } {} - - void configure(const Config& config); - void run(const RenderContextPointer& renderContext, const gpu::FramebufferPointer& sourceFramebuffer, gpu::FramebufferPointer& resampledFrameBuffer); - - protected: - - static gpu::PipelinePointer _pipeline; - - gpu::FramebufferPointer _destinationFrameBuffer; - float _factor{ 2.0f }; - - gpu::FramebufferPointer getResampledFrameBuffer(const gpu::FramebufferPointer& sourceFramebuffer); - }; - */ class ToneMapAndResample{ public: ToneMapAndResample(); From db49a0b9d65cb66054fbbbb181d65d74b55a7acb Mon Sep 17 00:00:00 2001 From: Anna Date: Mon, 29 Jul 2019 11:55:12 -0700 Subject: [PATCH 17/41] fixed some typos from merge --- .../render-utils/src/RenderForwardTask.cpp | 25 ++++++------------- .../src/ToneMapAndResampleTask.cpp | 6 ++--- 2 files changed, 9 insertions(+), 22 deletions(-) diff --git a/libraries/render-utils/src/RenderForwardTask.cpp b/libraries/render-utils/src/RenderForwardTask.cpp index 866c1db2a1..714291d103 100755 --- a/libraries/render-utils/src/RenderForwardTask.cpp +++ b/libraries/render-utils/src/RenderForwardTask.cpp @@ -140,29 +140,18 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend task.addJob("DrawZoneStack", debugZoneInputs); } -#if !defined(Q_OS_ANDROID) - - // Just resolve the msaa - const auto resolveInputs = ResolveFramebuffer::Inputs(scaledPrimaryFramebuffer, static_cast(nullptr)).asVarying(); - const auto resolvedFramebuffer = task.addJob("Resolve", resolveInputs); - - //const auto toneMappedBuffer = resolvedFramebuffer; -#else const auto newResolvedFramebuffer = task.addJob("MakeResolvingFramebuffer"); - // Just resolve the msaa + #if 1 + const auto resolveInputs = ResolveFramebuffer::Inputs(scaledPrimaryFramebuffer, static_cast(nullptr)).asVarying(); + #else const auto resolveInputs = ResolveFramebuffer::Inputs(scaledPrimaryFramebuffer, newResolvedFramebuffer).asVarying(); + #endif + const auto resolvedFramebuffer = task.addJob("Resolve", resolveInputs); - // Lighting Buffer ready for tone mapping - // Forward rendering on GLES doesn't support tonemapping to and from the same FBO, so we specify - // the output FBO as null, which causes the tonemapping to target the blit framebuffer - //const auto toneMappingInputs = ToneMappingDeferred::Input(resolvedFramebuffer, resolvedFramebuffer).asVarying(); - //const auto toneMappedBuffer = task.addJob("ToneMapping", toneMappingInputs); - -#endif - - const auto primaryFramebuffer = task.addJob("ToneMapAndResample", resolvedFramebuffer); + //const auto toneMappingInputs = ToneMappingDeferred::Input(resolvedFramebuffer, args->blitFramebuffer).asVarying(); + const auto toneMappedBuffer = task.addJob("ToneMapAndResample", resolvedFramebuffer); // HUD Layer const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(toneMappedBuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.cpp b/libraries/render-utils/src/ToneMapAndResampleTask.cpp index 9608309ee2..adafdf0531 100644 --- a/libraries/render-utils/src/ToneMapAndResampleTask.cpp +++ b/libraries/render-utils/src/ToneMapAndResampleTask.cpp @@ -73,7 +73,7 @@ void ToneMapAndResample::run(const RenderContextPointer& renderContext, const In resampledFrameBuffer = args->_blitFramebuffer; - if (!lightingBuffer || !blitFramebuffer) { + if (!lightingBuffer || !resampledFramebuffer) { return; } @@ -105,8 +105,6 @@ void ToneMapAndResample::run(const RenderContextPointer& renderContext, const In }); // Set full final viewport - args->_viewport = viewport; - - output = blitFramebuffer; + args->_viewport = destViewport; } } \ No newline at end of file From 2fca77867ba42a3b7d5a137868b325c79efc5797 Mon Sep 17 00:00:00 2001 From: Anna Date: Mon, 29 Jul 2019 13:45:08 -0700 Subject: [PATCH 18/41] broken on android again somehow --- libraries/render-utils/src/BloomEffect.cpp | 3 +-- .../render-utils/src/RenderCommonTask.cpp | 8 ++++++- .../render-utils/src/RenderForwardTask.cpp | 7 +++--- .../src/ToneMapAndResampleTask.cpp | 22 ++++++++++++------- libraries/render-utils/src/toneMapping.slf | 2 +- 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/libraries/render-utils/src/BloomEffect.cpp b/libraries/render-utils/src/BloomEffect.cpp index a069aa60fa..10df138fad 100644 --- a/libraries/render-utils/src/BloomEffect.cpp +++ b/libraries/render-utils/src/BloomEffect.cpp @@ -60,8 +60,7 @@ void BloomThreshold::run(const render::RenderContextPointer& renderContext, cons auto colorTexture = gpu::TexturePointer(gpu::Texture::createRenderBuffer(inputBuffer->getTexelFormat(), bufferSize.x, bufferSize.y, gpu::Texture::SINGLE_MIP, gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR_MIP_POINT, gpu::Sampler::WRAP_CLAMP))); - //_outputBuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("BloomThreshold")); - _outputBuffer = args->_blitFramebuffer; + _outputBuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("BloomThreshold")); _outputBuffer->setRenderBuffer(0, colorTexture); _parameters.edit()._deltaUV = { 1.0f / bufferSize.x, 1.0f / bufferSize.y }; diff --git a/libraries/render-utils/src/RenderCommonTask.cpp b/libraries/render-utils/src/RenderCommonTask.cpp index 8868d2fe7c..cddf1a6410 100644 --- a/libraries/render-utils/src/RenderCommonTask.cpp +++ b/libraries/render-utils/src/RenderCommonTask.cpp @@ -154,13 +154,19 @@ void NewFramebuffer::run(const render::RenderContextPointer& renderContext, cons glm::uvec2 frameSize(args->_viewport.z, args->_viewport.w); output.reset(); + auto destBlitFbo = args->_blitFramebuffer; + if (destBlitFbo && destBlitFbo->getSize() == frameSize) { + output = destBlitFbo; + return; + } + // Else use the lodal Framebuffer if (_outputFramebuffer && _outputFramebuffer->getSize() != frameSize) { _outputFramebuffer.reset(); } if (!_outputFramebuffer) { - _outputFramebuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("newFramebuffer.out")); + _outputFramebuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("newFramebuffer2.out")); auto colorFormat = gpu::Element(gpu::SCALAR, gpu::FLOAT, gpu::R11G11B10); auto defaultSampler = gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR); auto colorTexture = gpu::Texture::createRenderBuffer(colorFormat, frameSize.x, frameSize.y, gpu::Texture::SINGLE_MIP, defaultSampler); diff --git a/libraries/render-utils/src/RenderForwardTask.cpp b/libraries/render-utils/src/RenderForwardTask.cpp index 714291d103..97d0bbbc2e 100755 --- a/libraries/render-utils/src/RenderForwardTask.cpp +++ b/libraries/render-utils/src/RenderForwardTask.cpp @@ -140,7 +140,7 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend task.addJob("DrawZoneStack", debugZoneInputs); } - const auto newResolvedFramebuffer = task.addJob("MakeResolvingFramebuffer"); + //const auto newResolvedFramebuffer = task.addJob("MakeResolvingFramebuffer"); #if 1 const auto resolveInputs = ResolveFramebuffer::Inputs(scaledPrimaryFramebuffer, static_cast(nullptr)).asVarying(); @@ -149,12 +149,13 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend #endif const auto resolvedFramebuffer = task.addJob("Resolve", resolveInputs); + const auto toneMappedBuffer = resolvedFramebuffer; //const auto toneMappingInputs = ToneMappingDeferred::Input(resolvedFramebuffer, args->blitFramebuffer).asVarying(); - const auto toneMappedBuffer = task.addJob("ToneMapAndResample", resolvedFramebuffer); + const auto primaryFramebuffer = task.addJob("ToneMapAndResample", toneMappedBuffer); // HUD Layer - const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(toneMappedBuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); + const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(primaryFramebuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); task.addJob("RenderHUDLayer", renderHUDLayerInputs); } diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.cpp b/libraries/render-utils/src/ToneMapAndResampleTask.cpp index adafdf0531..dbd735f098 100644 --- a/libraries/render-utils/src/ToneMapAndResampleTask.cpp +++ b/libraries/render-utils/src/ToneMapAndResampleTask.cpp @@ -29,9 +29,8 @@ ToneMapAndResample::ToneMapAndResample() { Parameters parameters; _parametersBuffer = gpu::BufferView(std::make_shared(sizeof(Parameters), (const gpu::Byte*) ¶meters)); } - -void ToneMapAndResample::init(RenderArgs* args) { - +/* +void ToneMapAndResample::init() { // shared_ptr to gpu::State gpu::StatePointer blitState = gpu::StatePointer(new gpu::State()); @@ -41,6 +40,7 @@ void ToneMapAndResample::init(RenderArgs* args) { _pipeline = gpu::PipelinePointer(gpu::Pipeline::create(gpu::Shader::createProgram(toneMapping), blitState)); _mirrorPipeline = gpu::PipelinePointer(gpu::Pipeline::create(gpu::Shader::createProgram(toneMapping_mirrored), blitState)); } + */ void ToneMapAndResample::setExposure(float exposure) { auto& params = _parametersBuffer.get(); @@ -72,20 +72,26 @@ void ToneMapAndResample::run(const RenderContextPointer& renderContext, const In auto lightingBuffer = input->getRenderBuffer(0); resampledFrameBuffer = args->_blitFramebuffer; - - if (!lightingBuffer || !resampledFramebuffer) { +/* + if (!lightingBuffer || !resampledFrameBuffer) { return; - } + }*/ if (resampledFrameBuffer != sourceFramebuffer) { if (!_pipeline) { - init(args); + gpu::StatePointer blitState = gpu::StatePointer(new gpu::State()); + + blitState->setDepthTest(gpu::State::DepthTest(false, false)); + blitState->setColorWriteMask(true, true, true, true); + + _pipeline = gpu::PipelinePointer(gpu::Pipeline::create(gpu::Shader::createProgram(toneMapping), blitState)); + _mirrorPipeline = gpu::PipelinePointer(gpu::Pipeline::create(gpu::Shader::createProgram(toneMapping_mirrored), blitState)); } const auto bufferSize = resampledFrameBuffer->getSize(); - const auto srcBufferSize = glm::ivec2(lightingBuffer->getDimensions()); + auto srcBufferSize = glm::ivec2(lightingBuffer->getDimensions()); glm::ivec4 destViewport{ 0, 0, bufferSize.x, bufferSize.y }; diff --git a/libraries/render-utils/src/toneMapping.slf b/libraries/render-utils/src/toneMapping.slf index d90d1b3c8a..32aa2b0788 100644 --- a/libraries/render-utils/src/toneMapping.slf +++ b/libraries/render-utils/src/toneMapping.slf @@ -44,7 +44,7 @@ layout(location=0) out vec4 outFragColor; void main(void) { <@if HIFI_USE_MIRRORED@> - vec4 fragColorRaw = texture(colorMap, vec2(1.0 - varTexCoord0.x, varTexCoord0.y); + vec4 fragColorRaw = texture(colorMap, vec2(1.0 - varTexCoord0.x, varTexCoord0.y)); <@else@> vec4 fragColorRaw = texture(colorMap, varTexCoord0); <@endif@> From 3a437cf19013f1b147a8acfd24687debf825a175 Mon Sep 17 00:00:00 2001 From: Anna Date: Mon, 29 Jul 2019 14:59:15 -0700 Subject: [PATCH 19/41] fix on android, merge android and windows forward pipelines --- libraries/render-utils/src/RenderCommonTask.cpp | 10 ++-------- libraries/render-utils/src/RenderForwardTask.cpp | 13 +++---------- .../render-utils/src/ToneMapAndResampleTask.cpp | 15 ++++----------- .../render-utils/src/ToneMapAndResampleTask.h | 2 +- 4 files changed, 10 insertions(+), 30 deletions(-) diff --git a/libraries/render-utils/src/RenderCommonTask.cpp b/libraries/render-utils/src/RenderCommonTask.cpp index cddf1a6410..9338cefa82 100644 --- a/libraries/render-utils/src/RenderCommonTask.cpp +++ b/libraries/render-utils/src/RenderCommonTask.cpp @@ -154,19 +154,13 @@ void NewFramebuffer::run(const render::RenderContextPointer& renderContext, cons glm::uvec2 frameSize(args->_viewport.z, args->_viewport.w); output.reset(); - auto destBlitFbo = args->_blitFramebuffer; - if (destBlitFbo && destBlitFbo->getSize() == frameSize) { - output = destBlitFbo; - return; - } - // Else use the lodal Framebuffer if (_outputFramebuffer && _outputFramebuffer->getSize() != frameSize) { _outputFramebuffer.reset(); } if (!_outputFramebuffer) { - _outputFramebuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("newFramebuffer2.out")); + _outputFramebuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("newFramebuffer.out")); auto colorFormat = gpu::Element(gpu::SCALAR, gpu::FLOAT, gpu::R11G11B10); auto defaultSampler = gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR); auto colorTexture = gpu::Texture::createRenderBuffer(colorFormat, frameSize.x, frameSize.y, gpu::Texture::SINGLE_MIP, defaultSampler); @@ -195,7 +189,7 @@ void NewOrDefaultFramebuffer::run(const render::RenderContextPointer& renderCont } if (!_outputFramebuffer) { - _outputFramebuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("newFramebuffer.out")); + _outputFramebuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("newOrDefaultFramebuffer.out")); auto colorFormat = gpu::Element(gpu::SCALAR, gpu::FLOAT, gpu::R11G11B10); auto defaultSampler = gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR); auto colorTexture = gpu::Texture::createRenderBuffer(colorFormat, frameSize.x, frameSize.y, gpu::Texture::SINGLE_MIP, defaultSampler); diff --git a/libraries/render-utils/src/RenderForwardTask.cpp b/libraries/render-utils/src/RenderForwardTask.cpp index 97d0bbbc2e..814c5a9080 100755 --- a/libraries/render-utils/src/RenderForwardTask.cpp +++ b/libraries/render-utils/src/RenderForwardTask.cpp @@ -140,22 +140,15 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend task.addJob("DrawZoneStack", debugZoneInputs); } - //const auto newResolvedFramebuffer = task.addJob("MakeResolvingFramebuffer"); + const auto newResolvedFramebuffer = task.addJob("MakeResolvingFramebuffer"); - #if 1 - const auto resolveInputs = ResolveFramebuffer::Inputs(scaledPrimaryFramebuffer, static_cast(nullptr)).asVarying(); - #else const auto resolveInputs = ResolveFramebuffer::Inputs(scaledPrimaryFramebuffer, newResolvedFramebuffer).asVarying(); - #endif - const auto resolvedFramebuffer = task.addJob("Resolve", resolveInputs); - const auto toneMappedBuffer = resolvedFramebuffer; - //const auto toneMappingInputs = ToneMappingDeferred::Input(resolvedFramebuffer, args->blitFramebuffer).asVarying(); - const auto primaryFramebuffer = task.addJob("ToneMapAndResample", toneMappedBuffer); + const auto toneMappedBuffer = task.addJob("ToneMapAndResample", resolvedFramebuffer); // HUD Layer - const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(primaryFramebuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); + const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(toneMappedBuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); task.addJob("RenderHUDLayer", renderHUDLayerInputs); } diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.cpp b/libraries/render-utils/src/ToneMapAndResampleTask.cpp index dbd735f098..45370d1dee 100644 --- a/libraries/render-utils/src/ToneMapAndResampleTask.cpp +++ b/libraries/render-utils/src/ToneMapAndResampleTask.cpp @@ -29,7 +29,7 @@ ToneMapAndResample::ToneMapAndResample() { Parameters parameters; _parametersBuffer = gpu::BufferView(std::make_shared(sizeof(Parameters), (const gpu::Byte*) ¶meters)); } -/* + void ToneMapAndResample::init() { // shared_ptr to gpu::State gpu::StatePointer blitState = gpu::StatePointer(new gpu::State()); @@ -40,7 +40,6 @@ void ToneMapAndResample::init() { _pipeline = gpu::PipelinePointer(gpu::Pipeline::create(gpu::Shader::createProgram(toneMapping), blitState)); _mirrorPipeline = gpu::PipelinePointer(gpu::Pipeline::create(gpu::Shader::createProgram(toneMapping_mirrored), blitState)); } - */ void ToneMapAndResample::setExposure(float exposure) { auto& params = _parametersBuffer.get(); @@ -72,21 +71,15 @@ void ToneMapAndResample::run(const RenderContextPointer& renderContext, const In auto lightingBuffer = input->getRenderBuffer(0); resampledFrameBuffer = args->_blitFramebuffer; -/* + if (!lightingBuffer || !resampledFrameBuffer) { return; - }*/ + } if (resampledFrameBuffer != sourceFramebuffer) { if (!_pipeline) { - gpu::StatePointer blitState = gpu::StatePointer(new gpu::State()); - - blitState->setDepthTest(gpu::State::DepthTest(false, false)); - blitState->setColorWriteMask(true, true, true, true); - - _pipeline = gpu::PipelinePointer(gpu::Pipeline::create(gpu::Shader::createProgram(toneMapping), blitState)); - _mirrorPipeline = gpu::PipelinePointer(gpu::Pipeline::create(gpu::Shader::createProgram(toneMapping_mirrored), blitState)); + init(); } const auto bufferSize = resampledFrameBuffer->getSize(); diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.h b/libraries/render-utils/src/ToneMapAndResampleTask.h index 659d06c433..6d7def5f85 100644 --- a/libraries/render-utils/src/ToneMapAndResampleTask.h +++ b/libraries/render-utils/src/ToneMapAndResampleTask.h @@ -96,7 +96,7 @@ private: typedef gpu::BufferView UniformBufferView; gpu::BufferView _parametersBuffer; - void init(RenderArgs* args); + void init(); }; #endif // hifi_ToneMapAndResample_h From b4fee1140cf1470b8e6d45f2f050824ee74c751f Mon Sep 17 00:00:00 2001 From: Roxanne Skelly Date: Mon, 29 Jul 2019 16:25:51 -0700 Subject: [PATCH 20/41] BUGZ-319 - Add 'Installed By' username to installed content section on settings. --- .../resources/web/content/js/content.js | 6 +-- domain-server/src/AssetsBackupHandler.cpp | 2 +- domain-server/src/AssetsBackupHandler.h | 2 +- domain-server/src/BackupHandler.h | 2 +- .../src/ContentSettingsBackupHandler.cpp | 4 +- .../src/ContentSettingsBackupHandler.h | 2 +- .../src/DomainContentBackupManager.cpp | 24 ++++----- .../src/DomainContentBackupManager.h | 8 +-- domain-server/src/DomainServer.cpp | 54 ++++++++++++------- domain-server/src/DomainServer.h | 4 +- domain-server/src/EntitiesBackupHandler.cpp | 2 +- domain-server/src/EntitiesBackupHandler.h | 2 +- 12 files changed, 64 insertions(+), 48 deletions(-) diff --git a/domain-server/resources/web/content/js/content.js b/domain-server/resources/web/content/js/content.js index 9b5c807245..bf4cf07b26 100644 --- a/domain-server/resources/web/content/js/content.js +++ b/domain-server/resources/web/content/js/content.js @@ -130,12 +130,12 @@ $(document).ready(function(){ html += "File Name"; html += "Created"; html += "Installed"; - //html += "Installed By"; + html += "Installed By"; html += ""; html += ""; html += ""; html += ""; - //html += ""; + html += ""; html += ""; $('#' + Settings.INSTALLED_CONTENT + ' .panel-body').html(html); } @@ -379,7 +379,7 @@ $(document).ready(function(){ $('#' + INSTALLED_CONTENT_FILENAME_ID).text(data.installed_content.filename); $('#' + INSTALLED_CONTENT_CREATED_ID).text(data.installed_content.creation_time ? moment(data.installed_content.creation_time).format('lll') : ""); $('#' + INSTALLED_CONTENT_INSTALLED_ID).text(data.installed_content.install_time ? moment(data.installed_content.install_time).format('lll') : ""); - //$('#' + INSTALLED_CONTENT_INSTALLED_BY_ID).text(data.installed_content.installed_by); + $('#' + INSTALLED_CONTENT_INSTALLED_BY_ID).text(data.installed_content.installed_by); // update the progress bars for current restore status if (data.status.isRecovering) { diff --git a/domain-server/src/AssetsBackupHandler.cpp b/domain-server/src/AssetsBackupHandler.cpp index d978e4ea56..6ccfb2ad57 100644 --- a/domain-server/src/AssetsBackupHandler.cpp +++ b/domain-server/src/AssetsBackupHandler.cpp @@ -278,7 +278,7 @@ void AssetsBackupHandler::createBackup(const QString& backupName, QuaZip& zip) { _backups.emplace_back(backupName, mappings, false); } -std::pair AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip, const QString& sourceFilename) { +std::pair AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip, const QString& username, const QString& sourceFilename) { Q_ASSERT(QThread::currentThread() == thread()); if (operationInProgress()) { diff --git a/domain-server/src/AssetsBackupHandler.h b/domain-server/src/AssetsBackupHandler.h index c8f20ab965..703b844afc 100644 --- a/domain-server/src/AssetsBackupHandler.h +++ b/domain-server/src/AssetsBackupHandler.h @@ -38,7 +38,7 @@ public: void loadBackup(const QString& backupName, QuaZip& zip) override; void loadingComplete() override; void createBackup(const QString& backupName, QuaZip& zip) override; - std::pair recoverBackup(const QString& backupName, QuaZip& zip, const QString& sourceFilename) override; + std::pair recoverBackup(const QString& backupName, QuaZip& zip, const QString& username, const QString& sourceFilename) override; void deleteBackup(const QString& backupName) override; void consolidateBackup(const QString& backupName, QuaZip& zip) override; bool isCorruptedBackup(const QString& backupName) override; diff --git a/domain-server/src/BackupHandler.h b/domain-server/src/BackupHandler.h index 278d43ade3..8ef11b432a 100644 --- a/domain-server/src/BackupHandler.h +++ b/domain-server/src/BackupHandler.h @@ -30,7 +30,7 @@ public: virtual void loadBackup(const QString& backupName, QuaZip& zip) = 0; virtual void loadingComplete() = 0; virtual void createBackup(const QString& backupName, QuaZip& zip) = 0; - virtual std::pair recoverBackup(const QString& backupName, QuaZip& zip, const QString& sourceFilename) = 0; + virtual std::pair recoverBackup(const QString& backupName, QuaZip& zip, const QString& username, const QString& sourceFilename) = 0; virtual void deleteBackup(const QString& backupName) = 0; virtual void consolidateBackup(const QString& backupName, QuaZip& zip) = 0; virtual bool isCorruptedBackup(const QString& backupName) = 0; diff --git a/domain-server/src/ContentSettingsBackupHandler.cpp b/domain-server/src/ContentSettingsBackupHandler.cpp index b3748a66a3..97a10e81c3 100644 --- a/domain-server/src/ContentSettingsBackupHandler.cpp +++ b/domain-server/src/ContentSettingsBackupHandler.cpp @@ -84,7 +84,7 @@ void ContentSettingsBackupHandler::createBackup(const QString& backupName, QuaZi } } -std::pair ContentSettingsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip, const QString& sourceFilename) { +std::pair ContentSettingsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip, const QString& username, const QString& sourceFilename) { if (!zip.setCurrentFile(CONTENT_SETTINGS_BACKUP_FILENAME)) { QString errorStr("Failed to find " + CONTENT_SETTINGS_BACKUP_FILENAME + " while recovering backup"); qWarning() << errorStr; @@ -117,7 +117,7 @@ std::pair ContentSettingsBackupHandler::recoverBackup(const QStri { INSTALLED_CONTENT_NAME, archiveJson[INSTALLED_CONTENT_NAME].toString()}, { INSTALLED_CONTENT_CREATION_TIME, archiveJson[INSTALLED_CONTENT_CREATION_TIME].toVariant().toLongLong() }, { INSTALLED_CONTENT_INSTALL_TIME, QDateTime::currentDateTime().currentMSecsSinceEpoch() }, - { INSTALLED_CONTENT_INSTALLED_BY, "" } + { INSTALLED_CONTENT_INSTALLED_BY, username } }; jsonObject.insert(INSTALLED_CONTENT, installed_content); diff --git a/domain-server/src/ContentSettingsBackupHandler.h b/domain-server/src/ContentSettingsBackupHandler.h index 0e44a18424..0872bce59a 100644 --- a/domain-server/src/ContentSettingsBackupHandler.h +++ b/domain-server/src/ContentSettingsBackupHandler.h @@ -28,7 +28,7 @@ public: void createBackup(const QString& backupName, QuaZip& zip) override; - std::pair recoverBackup(const QString& backupName, QuaZip& zip, const QString& sourceFilename) override; + std::pair recoverBackup(const QString& backupName, QuaZip& zip, const QString& username, const QString& sourceFilename) override; void deleteBackup(const QString& backupName) override {} diff --git a/domain-server/src/DomainContentBackupManager.cpp b/domain-server/src/DomainContentBackupManager.cpp index 11930f0b49..b3f85884da 100644 --- a/domain-server/src/DomainContentBackupManager.cpp +++ b/domain-server/src/DomainContentBackupManager.cpp @@ -279,7 +279,7 @@ void DomainContentBackupManager::deleteBackup(MiniPromise::Promise promise, cons }); } -bool DomainContentBackupManager::recoverFromBackupZip(const QString& backupName, QuaZip& zip, const QString& sourceFilename, bool rollingBack) { +bool DomainContentBackupManager::recoverFromBackupZip(const QString& backupName, QuaZip& zip, const QString& username, const QString& sourceFilename, bool rollingBack) { if (!zip.open(QuaZip::Mode::mdUnzip)) { qWarning() << "Failed to unzip file: " << backupName; return false; @@ -290,7 +290,7 @@ bool DomainContentBackupManager::recoverFromBackupZip(const QString& backupName, for (auto& handler : _backupHandlers) { bool success; QString errorStr; - std::tie(success, errorStr) = handler->recoverBackup(backupName, zip, sourceFilename); + std::tie(success, errorStr) = handler->recoverBackup(backupName, zip, username, sourceFilename); if (!success) { if (!rollingBack) { _recoveryError = errorStr; @@ -304,7 +304,7 @@ bool DomainContentBackupManager::recoverFromBackupZip(const QString& backupName, } } -void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise, const QString& backupName) { +void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise, const QString& backupName, const QString& username) { if (_isRecovering) { promise->resolve({ { "success", false } @@ -314,7 +314,7 @@ void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise, if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "recoverFromBackup", Q_ARG(MiniPromise::Promise, promise), - Q_ARG(const QString&, backupName)); + Q_ARG(const QString&, backupName), Q_ARG(const QString&, username)); return; } @@ -327,7 +327,7 @@ void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise, if (backupFile.open(QIODevice::ReadOnly)) { QuaZip zip { &backupFile }; - success = recoverFromBackupZip(backupName, zip, backupName); + success = recoverFromBackupZip(backupName, zip, username, backupName); backupFile.close(); } else { @@ -340,11 +340,11 @@ void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise, }); } -void DomainContentBackupManager::recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup) { +void DomainContentBackupManager::recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup, QString username) { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "recoverFromUploadedBackup", Q_ARG(MiniPromise::Promise, promise), - Q_ARG(QByteArray, uploadedBackup)); + Q_ARG(QByteArray, uploadedBackup), Q_ARG(QString, username)); return; } @@ -355,17 +355,17 @@ void DomainContentBackupManager::recoverFromUploadedBackup(MiniPromise::Promise QuaZip uploadedZip { &uploadedBackupBuffer }; QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip"; - bool success = recoverFromBackupZip(backupName, uploadedZip, QString()); + bool success = recoverFromBackupZip(backupName, uploadedZip, username, QString()); promise->resolve({ { "success", success } }); } -void DomainContentBackupManager::recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename, QString sourceFilename) { +void DomainContentBackupManager::recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename, const QString username, QString sourceFilename) { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "recoverFromUploadedFile", Q_ARG(MiniPromise::Promise, promise), - Q_ARG(QString, uploadedFilename), Q_ARG(QString, sourceFilename)); + Q_ARG(QString, uploadedFilename), Q_ARG(QString, username), Q_ARG(QString, sourceFilename)); return; } @@ -382,7 +382,7 @@ void DomainContentBackupManager::recoverFromUploadedFile(MiniPromise::Promise pr QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip"; - bool success = recoverFromBackupZip(backupName, uploadedZip, sourceFilename); + bool success = recoverFromBackupZip(backupName, uploadedZip, username, sourceFilename); if (!success) { @@ -394,7 +394,7 @@ void DomainContentBackupManager::recoverFromUploadedFile(MiniPromise::Promise pr QuaZip uploadedZip { &uploadedFile }; QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip"; - recoverFromBackupZip(backupName, uploadedZip, sourceFilename, true); + recoverFromBackupZip(backupName, uploadedZip, sourceFilename, username, true); } } diff --git a/domain-server/src/DomainContentBackupManager.h b/domain-server/src/DomainContentBackupManager.h index f5957d74f5..f2ee71d498 100644 --- a/domain-server/src/DomainContentBackupManager.h +++ b/domain-server/src/DomainContentBackupManager.h @@ -95,9 +95,9 @@ public: public slots: void getAllBackupsAndStatus(MiniPromise::Promise promise); void createManualBackup(MiniPromise::Promise promise, const QString& name); - void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName); - void recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup); - void recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename, QString sourceFilename); + void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName, const QString& username); + void recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup, QString username); + void recoverFromUploadedFile(MiniPromise::Promise promise, QString uploadedFilename, QString username, QString sourceFilename); void deleteBackup(MiniPromise::Promise promise, const QString& backupName); signals: @@ -119,7 +119,7 @@ protected: std::pair createBackup(const QString& prefix, const QString& name); - bool recoverFromBackupZip(const QString& backupName, QuaZip& backupZip, const QString& sourceFilename, bool rollingBack = false); + bool recoverFromBackupZip(const QString& backupName, QuaZip& backupZip, const QString& username, const QString& sourceFilename, bool rollingBack = false); private slots: void removeOldConsolidatedBackups(); diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index fa4bf89ad6..09438b31bc 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -1957,6 +1957,7 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url QPointer connectionPtr { connection }; auto nodeList = DependencyManager::get(); + QString username; auto getSetting = [this](QString keyPath, QVariant& value) -> bool { @@ -2024,7 +2025,9 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url } // all requests below require a cookie to prove authentication so check that first - if (!isAuthenticatedRequest(connection, url)) { + bool isAuthenticated { false }; + std::tie(isAuthenticated, username) = isAuthenticatedRequest(connection); + if (!isAuthenticated) { // this is not an authenticated request // return true from the handler since it was handled with a 401 or re-direct to auth return true; @@ -2361,7 +2364,7 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url connectionPtr->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(), JSON_MIME_TYPE.toUtf8()); }); - _contentManager->recoverFromBackup(deferred, id); + _contentManager->recoverFromBackup(deferred, id, username); return true; } } else if (connection->requestOperation() == QNetworkAccessManager::PutOperation) { @@ -2557,6 +2560,9 @@ bool DomainServer::processPendingContent(HTTPConnection* connection, QString ite int sessionId = sessionIdBytes.toInt(); bool newUpload = itemName == "restore-file" || itemName == "restore-file-chunk-initial" || itemName == "restore-file-chunk-only"; + bool isAuthenticated; + QString username; + std::tie(isAuthenticated, username) = isAuthenticatedRequest(connection); if (filename.endsWith(".zip", Qt::CaseInsensitive)) { static const QString TEMPORARY_CONTENT_FILEPATH { QDir::tempPath() + "/hifiUploadContent_XXXXXX.zip" }; @@ -2591,7 +2597,7 @@ bool DomainServer::processPendingContent(HTTPConnection* connection, QString ite _pendingContentFiles.erase(sessionId); }); - _contentManager->recoverFromUploadedFile(deferred, _pendingFileContent.fileName(), filename); + _contentManager->recoverFromUploadedFile(deferred, _pendingFileContent.fileName(), username, filename); } } else if (filename.endsWith(".json", Qt::CaseInsensitive) || filename.endsWith(".json.gz", Qt::CaseInsensitive)) { @@ -2604,7 +2610,7 @@ bool DomainServer::processPendingContent(HTTPConnection* connection, QString ite if (itemName == "restore-file" || itemName == "restore-file-chunk-final" || itemName == "restore-file-chunk-only") { // invoke our method to hand the new octree file off to the octree server - if (!handleOctreeFileReplacement(_pendingUploadedContent, filename, QString())) { + if (!handleOctreeFileReplacement(_pendingUploadedContent, filename, QString(), username)) { connection->respond(HTTPConnection::StatusCode400); return false; } @@ -2680,7 +2686,7 @@ void DomainServer::profileRequestFinished() { } } -bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl& url) { +std::pair DomainServer::isAuthenticatedRequest(HTTPConnection* connection) { static const QByteArray HTTP_COOKIE_HEADER_KEY = "Cookie"; static const QString ADMIN_USERS_CONFIG_KEY = "admin-users"; @@ -2717,7 +2723,7 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl if (_settingsManager.valueForKeyPath(ADMIN_USERS_CONFIG_KEY).toStringList().contains(profileUsername)) { // this is an authenticated user - return true; + return { true, profileUsername }; } // loop the roles of this user and see if they are in the admin-roles array @@ -2727,7 +2733,7 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl foreach(const QString& userRole, sessionData.getRoles()) { if (adminRolesArray.contains(userRole)) { // this user has a role that allows them to administer the domain-server - return true; + return { true, profileUsername }; } } } @@ -2735,7 +2741,7 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl connection->respond(HTTPConnection::StatusCode401, UNAUTHENTICATED_BODY); // the user does not have allowed username or role, return 401 - return false; + return { false, QString() }; } else { static const QByteArray REQUESTED_WITH_HEADER = "X-Requested-With"; static const QString XML_REQUESTED_WITH = "XMLHttpRequest"; @@ -2764,7 +2770,7 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl } // we don't know about this user yet, so they are not yet authenticated - return false; + return { false, QString() }; } } else if (_settingsManager.valueForKeyPath(BASIC_AUTH_USERNAME_KEY_PATH).isValid()) { // config file contains username and password combinations for basic auth @@ -2793,7 +2799,7 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl "" : QCryptographicHash::hash(headerPassword.toUtf8(), QCryptographicHash::Sha256).toHex(); if (settingsUsername == headerUsername && hexHeaderPassword == settingsPassword) { - return true; + return { true, headerUsername }; } } } @@ -2815,11 +2821,11 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl HTTPConnection::DefaultContentType, basicAuthHeader); // not authenticated, bubble up false - return false; + return { false, QString() }; } else { // we don't have an OAuth URL + admin roles/usernames, so all users are authenticated - return true; + return { true, QString() }; } } @@ -3493,7 +3499,7 @@ void DomainServer::maybeHandleReplacementEntityFile() { } } -bool DomainServer::handleOctreeFileReplacement(QByteArray octreeFile, QString sourceFilename, QString name) { +bool DomainServer::handleOctreeFileReplacement(QByteArray octreeFile, QString sourceFilename, QString name, QString username) { OctreeUtils::RawEntityData data; if (data.readOctreeDataInfoFromData(octreeFile)) { data.resetIdAndVersion(); @@ -3514,7 +3520,7 @@ bool DomainServer::handleOctreeFileReplacement(QByteArray octreeFile, QString so { INSTALLED_CONTENT_NAME, name }, { INSTALLED_CONTENT_CREATION_TIME, 0 }, { INSTALLED_CONTENT_INSTALL_TIME, QDateTime::currentDateTime().currentMSecsSinceEpoch() }, - { INSTALLED_CONTENT_INSTALLED_BY, "" } + { INSTALLED_CONTENT_INSTALLED_BY, username } }; QJsonObject jsonObject { { INSTALLED_CONTENT, installed_content } }; @@ -3539,6 +3545,11 @@ void DomainServer::handleDomainContentReplacementFromURLRequest(QSharedPointer()->findNodeWithAddr(message->getSenderSockAddr()); if (node && node->getCanReplaceContent()) { + DomainServerNodeData* nodeData = static_cast(node->getLinkedData()); + QString username; + if (nodeData) { + username = nodeData->getUsername(); + } // Convert message data into our URL QString url(message->getMessage()); QUrl modelsURL = QUrl(url, QUrl::StrictMode); @@ -3548,17 +3559,17 @@ void DomainServer::handleDomainContentReplacementFromURLRequest(QSharedPointererror(); if (networkError == QNetworkReply::NoError) { if (modelsURL.fileName().endsWith(".json.gz")) { QUrlQuery urlQuery(modelsURL.query(QUrl::FullyEncoded)); QString itemName = urlQuery.queryItemValue(CONTENT_SET_NAME_QUERY_PARAM); - handleOctreeFileReplacement(reply->readAll(), modelsURL.fileName(), itemName); + handleOctreeFileReplacement(reply->readAll(), modelsURL.fileName(), itemName, username); } else if (modelsURL.fileName().endsWith(".zip")) { auto deferred = makePromise("recoverFromUploadedBackup"); - _contentManager->recoverFromUploadedBackup(deferred, reply->readAll()); + _contentManager->recoverFromUploadedBackup(deferred, reply->readAll(), username); } } else { qDebug() << "Error downloading JSON from specified file: " << modelsURL; @@ -3569,7 +3580,12 @@ void DomainServer::handleDomainContentReplacementFromURLRequest(QSharedPointer message) { auto node = DependencyManager::get()->nodeWithLocalID(message->getSourceID()); - if (node->getCanReplaceContent()) { - handleOctreeFileReplacement(message->readAll(), QString(), QString()); + if (node && node->getCanReplaceContent()) { + QString username; + DomainServerNodeData* nodeData = static_cast(node->getLinkedData()); + if (nodeData) { + username = nodeData->getUsername(); + } + handleOctreeFileReplacement(message->readAll(), QString(), QString(), username); } } diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index aef59a4e4a..02362abd7b 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -99,7 +99,7 @@ private slots: void handleDomainContentReplacementFromURLRequest(QSharedPointer message); void handleOctreeFileReplacementRequest(QSharedPointer message); - bool handleOctreeFileReplacement(QByteArray octreeFile, QString sourceFilename, QString name); + bool handleOctreeFileReplacement(QByteArray octreeFile, QString sourceFilename, QString name, QString username); void processOctreeDataRequestMessage(QSharedPointer message); void processOctreeDataPersistMessage(QSharedPointer message); @@ -194,7 +194,7 @@ private: QUrl oauthRedirectURL(); QUrl oauthAuthorizationURL(const QUuid& stateUUID = QUuid::createUuid()); - bool isAuthenticatedRequest(HTTPConnection* connection, const QUrl& url); + std::pair isAuthenticatedRequest(HTTPConnection* connection); QNetworkReply* profileRequestGivenTokenReply(QNetworkReply* tokenReply); Headers setupCookieHeadersFromProfileReply(QNetworkReply* profileReply); diff --git a/domain-server/src/EntitiesBackupHandler.cpp b/domain-server/src/EntitiesBackupHandler.cpp index 03baec9164..e7e8b5a90d 100644 --- a/domain-server/src/EntitiesBackupHandler.cpp +++ b/domain-server/src/EntitiesBackupHandler.cpp @@ -57,7 +57,7 @@ void EntitiesBackupHandler::createBackup(const QString& backupName, QuaZip& zip) } } -std::pair EntitiesBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip, const QString& sourceFilename) { +std::pair EntitiesBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip, const QString& username, const QString& sourceFilename) { if (!zip.setCurrentFile(ENTITIES_BACKUP_FILENAME)) { QString errorStr("Failed to find " + ENTITIES_BACKUP_FILENAME + " while recovering backup"); qWarning() << errorStr; diff --git a/domain-server/src/EntitiesBackupHandler.h b/domain-server/src/EntitiesBackupHandler.h index f8b6cba8a0..1cdfdd89ed 100644 --- a/domain-server/src/EntitiesBackupHandler.h +++ b/domain-server/src/EntitiesBackupHandler.h @@ -29,7 +29,7 @@ public: void createBackup(const QString& backupName, QuaZip& zip) override; // Recover from a full backup - std::pair recoverBackup(const QString& backupName, QuaZip& zip, const QString& sourceFilename) override; + std::pair recoverBackup(const QString& backupName, QuaZip& zip, const QString& username, const QString& sourceFilename) override; // Delete a skeleton backup void deleteBackup(const QString& backupName) override {} From d3107efb8280a418ee17248608ae2e94e68daad9 Mon Sep 17 00:00:00 2001 From: Anna Date: Mon, 29 Jul 2019 16:27:54 -0700 Subject: [PATCH 21/41] added optional output buffer to ToneMapAndResampleTask --- libraries/render-utils/CMakeLists.txt | 4 +- .../render-utils/src/RenderForwardTask.cpp | 8 ++- .../src/ToneMapAndResampleTask.cpp | 71 ++++++++++--------- .../render-utils/src/ToneMapAndResampleTask.h | 4 +- 4 files changed, 48 insertions(+), 39 deletions(-) diff --git a/libraries/render-utils/CMakeLists.txt b/libraries/render-utils/CMakeLists.txt index 833d78bb74..4fbd076d05 100644 --- a/libraries/render-utils/CMakeLists.txt +++ b/libraries/render-utils/CMakeLists.txt @@ -12,7 +12,9 @@ include_hifi_library_headers(hfm) # tell CMake to exclude qrc_fonts.cpp for policy CMP0071 set_property(SOURCE qrc_fonts.cpp PROPERTY SKIP_AUTOMOC ON) -if (NOT ANDROID) +if (ANDROID) + target_link_libraries(${TARGET_NAME} android log) +else() target_nsight() endif () diff --git a/libraries/render-utils/src/RenderForwardTask.cpp b/libraries/render-utils/src/RenderForwardTask.cpp index 814c5a9080..33af6aaa71 100755 --- a/libraries/render-utils/src/RenderForwardTask.cpp +++ b/libraries/render-utils/src/RenderForwardTask.cpp @@ -145,8 +145,14 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend const auto resolveInputs = ResolveFramebuffer::Inputs(scaledPrimaryFramebuffer, newResolvedFramebuffer).asVarying(); const auto resolvedFramebuffer = task.addJob("Resolve", resolveInputs); - const auto toneMappedBuffer = task.addJob("ToneMapAndResample", resolvedFramebuffer); +#if defined(Q_OS_ANDROID) + const auto destFramebuffer = static_cast(nullptr); +#else + const auto destFramebuffer = resolvedFramebuffer; +#endif + const auto toneMappingInputs = ToneMapAndResample::Input(resolvedFramebuffer, destFramebuffer).asVarying(); + const auto toneMappedBuffer = task.addJob("ToneMapAndResample", toneMappingInputs); // HUD Layer const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(toneMappedBuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); task.addJob("RenderHUDLayer", renderHUDLayerInputs); diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.cpp b/libraries/render-utils/src/ToneMapAndResampleTask.cpp index 45370d1dee..d9439f7a92 100644 --- a/libraries/render-utils/src/ToneMapAndResampleTask.cpp +++ b/libraries/render-utils/src/ToneMapAndResampleTask.cpp @@ -3,7 +3,7 @@ // libraries/render-utils/src // // Created by Anna Brewer on 7/3/19. -// Copyright 2015 High Fidelity, Inc. +// Copyright 2019 High Fidelity, Inc. // // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html @@ -61,49 +61,50 @@ void ToneMapAndResample::configure(const Config& config) { setToneCurve((ToneCurve)config.curve); } -void ToneMapAndResample::run(const RenderContextPointer& renderContext, const Input& input, gpu::FramebufferPointer& resampledFrameBuffer) { +void ToneMapAndResample::run(const RenderContextPointer& renderContext, const Input& input, Output& output) { assert(renderContext->args); assert(renderContext->args->hasViewFrustum()); RenderArgs* args = renderContext->args; - auto sourceFramebuffer = input; - auto lightingBuffer = input->getRenderBuffer(0); + auto lightingBuffer = input.get0()->getRenderBuffer(0); + auto resampledFramebuffer = input.get1(); - resampledFrameBuffer = args->_blitFramebuffer; + if (!resampledFramebuffer) { + resampledFramebuffer = args->_blitFramebuffer; + } - if (!lightingBuffer || !resampledFrameBuffer) { + if (!lightingBuffer || !resampledFramebuffer) { return; } - if (resampledFrameBuffer != sourceFramebuffer) { - - if (!_pipeline) { - init(); - } - - const auto bufferSize = resampledFrameBuffer->getSize(); - - auto srcBufferSize = glm::ivec2(lightingBuffer->getDimensions()); - - glm::ivec4 destViewport{ 0, 0, bufferSize.x, bufferSize.y }; - - gpu::doInBatch("Resample::run", args->_context, [&](gpu::Batch& batch) { - batch.enableStereo(false); - batch.setFramebuffer(resampledFrameBuffer); - - batch.setViewportTransform(destViewport); - batch.setProjectionTransform(glm::mat4()); - batch.resetViewTransform(); - batch.setPipeline(args->_renderMode == RenderArgs::MIRROR_RENDER_MODE ? _mirrorPipeline : _pipeline); - - batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(srcBufferSize, args->_viewport)); - batch.setUniformBuffer(render_utils::slot::buffer::ToneMappingParams, _parametersBuffer); - batch.setResourceTexture(render_utils::slot::texture::ToneMappingColor, lightingBuffer); - batch.draw(gpu::TRIANGLE_STRIP, 4); - }); - - // Set full final viewport - args->_viewport = destViewport; + if (!_pipeline) { + init(); } + + const auto bufferSize = resampledFramebuffer->getSize(); + + auto srcBufferSize = glm::ivec2(lightingBuffer->getDimensions()); + + glm::ivec4 destViewport{ 0, 0, bufferSize.x, bufferSize.y }; + + gpu::doInBatch("Resample::run", args->_context, [&](gpu::Batch& batch) { + batch.enableStereo(false); + batch.setFramebuffer(resampledFramebuffer); + + batch.setViewportTransform(destViewport); + batch.setProjectionTransform(glm::mat4()); + batch.resetViewTransform(); + batch.setPipeline(args->_renderMode == RenderArgs::MIRROR_RENDER_MODE ? _mirrorPipeline : _pipeline); + + batch.setModelTransform(gpu::Framebuffer::evalSubregionTexcoordTransform(srcBufferSize, args->_viewport)); + batch.setUniformBuffer(render_utils::slot::buffer::ToneMappingParams, _parametersBuffer); + batch.setResourceTexture(render_utils::slot::texture::ToneMappingColor, lightingBuffer); + batch.draw(gpu::TRIANGLE_STRIP, 4); + }); + + // Set full final viewport + args->_viewport = destViewport; + + output = resampledFramebuffer; } \ No newline at end of file diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.h b/libraries/render-utils/src/ToneMapAndResampleTask.h index 6d7def5f85..d02cb4afda 100644 --- a/libraries/render-utils/src/ToneMapAndResampleTask.h +++ b/libraries/render-utils/src/ToneMapAndResampleTask.h @@ -3,7 +3,7 @@ // libraries/render-utils/src // // Created by Anna Brewer on 7/3/19. -// Copyright 2015 High Fidelity, Inc. +// Copyright 2019 High Fidelity, Inc. // // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html @@ -59,7 +59,7 @@ public: ToneCurve getToneCurve() const { return (ToneCurve)_parametersBuffer.get()._toneCurve; } // Inputs: lightingFramebuffer, destinationFramebuffer - using Input = gpu::FramebufferPointer; + using Input = render::VaryingSet2; using Output = gpu::FramebufferPointer; using Config = ToneMappingConfig; using JobModel = render::Job::ModelIO; From 379a262b072b84909a24dab25f83941a3da01af4 Mon Sep 17 00:00:00 2001 From: Anna Date: Mon, 29 Jul 2019 18:15:54 -0700 Subject: [PATCH 22/41] bug fix in deferred, finish merging android & windows forward --- libraries/render-utils/src/RenderDeferredTask.cpp | 4 +++- libraries/render-utils/src/RenderForwardTask.cpp | 4 ---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/libraries/render-utils/src/RenderDeferredTask.cpp b/libraries/render-utils/src/RenderDeferredTask.cpp index cffbad0e44..7f91c6eaf2 100644 --- a/libraries/render-utils/src/RenderDeferredTask.cpp +++ b/libraries/render-utils/src/RenderDeferredTask.cpp @@ -237,8 +237,10 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren const auto bloomInputs = BloomEffect::Inputs(deferredFrameTransform, lightingFramebuffer, bloomFrame).asVarying(); task.addJob("Bloom", bloomInputs); + const auto destFramebuffer = static_cast(nullptr); + // Lighting Buffer ready for tone mapping - const auto toneMappingInputs = lightingFramebuffer; + const auto toneMappingInputs = ToneMapAndResample::Input(lightingFramebuffer, destFramebuffer).asVarying(); const auto toneMappedBuffer = task.addJob("ToneMapAndResample", toneMappingInputs); // Debugging task is happening in the "over" layer after tone mapping and just before HUD diff --git a/libraries/render-utils/src/RenderForwardTask.cpp b/libraries/render-utils/src/RenderForwardTask.cpp index 33af6aaa71..fdcb6c6741 100755 --- a/libraries/render-utils/src/RenderForwardTask.cpp +++ b/libraries/render-utils/src/RenderForwardTask.cpp @@ -145,11 +145,7 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend const auto resolveInputs = ResolveFramebuffer::Inputs(scaledPrimaryFramebuffer, newResolvedFramebuffer).asVarying(); const auto resolvedFramebuffer = task.addJob("Resolve", resolveInputs); -#if defined(Q_OS_ANDROID) const auto destFramebuffer = static_cast(nullptr); -#else - const auto destFramebuffer = resolvedFramebuffer; -#endif const auto toneMappingInputs = ToneMapAndResample::Input(resolvedFramebuffer, destFramebuffer).asVarying(); const auto toneMappedBuffer = task.addJob("ToneMapAndResample", toneMappingInputs); From 458b3f18df4e64453d25b627019379d7ac0675f3 Mon Sep 17 00:00:00 2001 From: Anna Date: Mon, 29 Jul 2019 18:25:36 -0700 Subject: [PATCH 23/41] changes from cr --- libraries/render-utils/src/DeferredLightingEffect.cpp | 3 --- libraries/render-utils/src/ToneMapAndResampleTask.h | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/libraries/render-utils/src/DeferredLightingEffect.cpp b/libraries/render-utils/src/DeferredLightingEffect.cpp index d8ccef0064..8d7fc345ac 100644 --- a/libraries/render-utils/src/DeferredLightingEffect.cpp +++ b/libraries/render-utils/src/DeferredLightingEffect.cpp @@ -29,9 +29,6 @@ #include "TextureCache.h" #include "FramebufferCache.h" -#include -#include - namespace ru { using render_utils::slot::texture::Texture; using render_utils::slot::buffer::Buffer; diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.h b/libraries/render-utils/src/ToneMapAndResampleTask.h index d02cb4afda..d2b166392e 100644 --- a/libraries/render-utils/src/ToneMapAndResampleTask.h +++ b/libraries/render-utils/src/ToneMapAndResampleTask.h @@ -30,7 +30,7 @@ enum class ToneCurve { class ToneMappingConfig : public render::Job::Config { Q_OBJECT - Q_PROPERTY(float exposure MEMBER exposure WRITE setExposure); + Q_PROPERTY(float exposure MEMBER exposure WRITE setExposure); Q_PROPERTY(int curve MEMBER curve WRITE setCurve); public: ToneMappingConfig() : render::Job::Config(true) {} From 0cd532a47b6fe8c187f4e76f09cbbb1ad5b9acaf Mon Sep 17 00:00:00 2001 From: Anna Date: Tue, 30 Jul 2019 10:08:38 -0700 Subject: [PATCH 24/41] cleaning up --- libraries/platform/src/platform/backend/PlatformInstance.cpp | 2 +- libraries/render-utils/CMakeLists.txt | 4 +--- libraries/render-utils/src/RenderCommonTask.cpp | 1 - libraries/render-utils/src/ToneMapAndResampleTask.cpp | 2 +- libraries/render-utils/src/ToneMapAndResampleTask.h | 5 +++-- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/libraries/platform/src/platform/backend/PlatformInstance.cpp b/libraries/platform/src/platform/backend/PlatformInstance.cpp index 2f68972c4a..e02eaf2837 100644 --- a/libraries/platform/src/platform/backend/PlatformInstance.cpp +++ b/libraries/platform/src/platform/backend/PlatformInstance.cpp @@ -104,6 +104,7 @@ void Instance::enumerateNics() { } } } + json Instance::getCPU(int index) { assert(index < (int)_cpus.size()); @@ -146,7 +147,6 @@ Instance::~Instance() { } } - json Instance::listAllKeys() { json allKeys; allKeys.array({{ diff --git a/libraries/render-utils/CMakeLists.txt b/libraries/render-utils/CMakeLists.txt index 4fbd076d05..833d78bb74 100644 --- a/libraries/render-utils/CMakeLists.txt +++ b/libraries/render-utils/CMakeLists.txt @@ -12,9 +12,7 @@ include_hifi_library_headers(hfm) # tell CMake to exclude qrc_fonts.cpp for policy CMP0071 set_property(SOURCE qrc_fonts.cpp PROPERTY SKIP_AUTOMOC ON) -if (ANDROID) - target_link_libraries(${TARGET_NAME} android log) -else() +if (NOT ANDROID) target_nsight() endif () diff --git a/libraries/render-utils/src/RenderCommonTask.cpp b/libraries/render-utils/src/RenderCommonTask.cpp index 9338cefa82..bdcba3e931 100644 --- a/libraries/render-utils/src/RenderCommonTask.cpp +++ b/libraries/render-utils/src/RenderCommonTask.cpp @@ -154,7 +154,6 @@ void NewFramebuffer::run(const render::RenderContextPointer& renderContext, cons glm::uvec2 frameSize(args->_viewport.z, args->_viewport.w); output.reset(); - // Else use the lodal Framebuffer if (_outputFramebuffer && _outputFramebuffer->getSize() != frameSize) { _outputFramebuffer.reset(); } diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.cpp b/libraries/render-utils/src/ToneMapAndResampleTask.cpp index d9439f7a92..54821b0380 100644 --- a/libraries/render-utils/src/ToneMapAndResampleTask.cpp +++ b/libraries/render-utils/src/ToneMapAndResampleTask.cpp @@ -107,4 +107,4 @@ void ToneMapAndResample::run(const RenderContextPointer& renderContext, const In args->_viewport = destViewport; output = resampledFramebuffer; -} \ No newline at end of file +} diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.h b/libraries/render-utils/src/ToneMapAndResampleTask.h index d2b166392e..1c7ef2cf48 100644 --- a/libraries/render-utils/src/ToneMapAndResampleTask.h +++ b/libraries/render-utils/src/ToneMapAndResampleTask.h @@ -32,6 +32,7 @@ class ToneMappingConfig : public render::Job::Config { Q_OBJECT Q_PROPERTY(float exposure MEMBER exposure WRITE setExposure); Q_PROPERTY(int curve MEMBER curve WRITE setCurve); + public: ToneMappingConfig() : render::Job::Config(true) {} @@ -41,6 +42,7 @@ public: float exposure{ 0.0f }; int curve{ (int)ToneCurve::Gamma22 }; + signals: void dirty(); }; @@ -68,7 +70,6 @@ public: void run(const render::RenderContextPointer& renderContext, const Input& input, Output& output); protected: - static gpu::PipelinePointer _pipeline; static gpu::PipelinePointer _mirrorPipeline; @@ -79,7 +80,6 @@ protected: gpu::FramebufferPointer getResampledFrameBuffer(const gpu::FramebufferPointer& sourceFramebuffer); private: - gpu::PipelinePointer _blitLightBuffer; // Class describing the uniform buffer with all the parameters common to the tone mapping shaders @@ -93,6 +93,7 @@ private: Parameters() {} }; + typedef gpu::BufferView UniformBufferView; gpu::BufferView _parametersBuffer; From de1aa68cffd906dc1c17c966db7149fe17c1d747 Mon Sep 17 00:00:00 2001 From: Roxanne Skelly Date: Tue, 30 Jul 2019 11:30:23 -0700 Subject: [PATCH 25/41] oopse --- domain-server/src/DomainContentBackupManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domain-server/src/DomainContentBackupManager.cpp b/domain-server/src/DomainContentBackupManager.cpp index b3f85884da..3a7897ec61 100644 --- a/domain-server/src/DomainContentBackupManager.cpp +++ b/domain-server/src/DomainContentBackupManager.cpp @@ -394,7 +394,7 @@ void DomainContentBackupManager::recoverFromUploadedFile(MiniPromise::Promise pr QuaZip uploadedZip { &uploadedFile }; QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip"; - recoverFromBackupZip(backupName, uploadedZip, sourceFilename, username, true); + recoverFromBackupZip(backupName, uploadedZip, username, sourceFilename, true); } } From f58400950a0d11035cfa2740e292475c2e55eb59 Mon Sep 17 00:00:00 2001 From: Anna Date: Tue, 30 Jul 2019 11:57:01 -0700 Subject: [PATCH 26/41] made color format for resolve FB a parameter, reverted name of tonemapping task to preserve back compatibility --- interface/src/ui/Snapshot.cpp | 1 - libraries/render-utils/src/BloomEffect.cpp | 1 - libraries/render-utils/src/RenderCommonTask.cpp | 11 +++++++---- libraries/render-utils/src/RenderCommonTask.h | 9 ++++++--- libraries/render-utils/src/RenderDeferredTask.cpp | 2 +- libraries/render-utils/src/RenderForwardTask.cpp | 4 ++-- .../render-utils/src/ToneMapAndResampleTask.cpp | 14 +++++++------- 7 files changed, 23 insertions(+), 19 deletions(-) diff --git a/interface/src/ui/Snapshot.cpp b/interface/src/ui/Snapshot.cpp index 363e7a5cf1..bb9971e582 100644 --- a/interface/src/ui/Snapshot.cpp +++ b/interface/src/ui/Snapshot.cpp @@ -41,7 +41,6 @@ #include "MainWindow.h" #include "Snapshot.h" #include "SnapshotUploader.h" -#include "ToneMapAndResampleTask.h" // filename format: hifi-snap-by-%username%-on-%date%_%time%_@-%location%.jpg // %1 <= username, %2 <= date and time, %3 <= current location diff --git a/libraries/render-utils/src/BloomEffect.cpp b/libraries/render-utils/src/BloomEffect.cpp index 10df138fad..e58d07ac33 100644 --- a/libraries/render-utils/src/BloomEffect.cpp +++ b/libraries/render-utils/src/BloomEffect.cpp @@ -15,7 +15,6 @@ #include #include -#include "ToneMapAndResampleTask.h" #include "render-utils/ShaderConstants.h" #define BLOOM_BLUR_LEVEL_COUNT 3 diff --git a/libraries/render-utils/src/RenderCommonTask.cpp b/libraries/render-utils/src/RenderCommonTask.cpp index bdcba3e931..43983b76f7 100644 --- a/libraries/render-utils/src/RenderCommonTask.cpp +++ b/libraries/render-utils/src/RenderCommonTask.cpp @@ -148,9 +148,12 @@ void Blit::run(const RenderContextPointer& renderContext, const gpu::Framebuffer }); } -void NewFramebuffer::run(const render::RenderContextPointer& renderContext, const Input& input, Output& output) { +NewFramebuffer::NewFramebuffer(gpu::Element pixelFormat = gpu::Element::COLOR_SRGBA_32) { + _pixelFormat = pixelFormat; +} + +void NewFramebuffer::run(const render::RenderContextPointer& renderContext, Output& output) { RenderArgs* args = renderContext->args; - // auto frameSize = input; glm::uvec2 frameSize(args->_viewport.z, args->_viewport.w); output.reset(); @@ -160,7 +163,7 @@ void NewFramebuffer::run(const render::RenderContextPointer& renderContext, cons if (!_outputFramebuffer) { _outputFramebuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("newFramebuffer.out")); - auto colorFormat = gpu::Element(gpu::SCALAR, gpu::FLOAT, gpu::R11G11B10); + auto colorFormat = _pixelFormat; auto defaultSampler = gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR); auto colorTexture = gpu::Texture::createRenderBuffer(colorFormat, frameSize.x, frameSize.y, gpu::Texture::SINGLE_MIP, defaultSampler); _outputFramebuffer->setRenderBuffer(0, colorTexture); @@ -189,7 +192,7 @@ void NewOrDefaultFramebuffer::run(const render::RenderContextPointer& renderCont if (!_outputFramebuffer) { _outputFramebuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("newOrDefaultFramebuffer.out")); - auto colorFormat = gpu::Element(gpu::SCALAR, gpu::FLOAT, gpu::R11G11B10); + auto colorFormat = gpu::Element::COLOR_SRGBA_32; auto defaultSampler = gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_LINEAR); auto colorTexture = gpu::Texture::createRenderBuffer(colorFormat, frameSize.x, frameSize.y, gpu::Texture::SINGLE_MIP, defaultSampler); _outputFramebuffer->setRenderBuffer(0, colorTexture); diff --git a/libraries/render-utils/src/RenderCommonTask.h b/libraries/render-utils/src/RenderCommonTask.h index a9bea8b672..446d6a351c 100644 --- a/libraries/render-utils/src/RenderCommonTask.h +++ b/libraries/render-utils/src/RenderCommonTask.h @@ -85,11 +85,14 @@ public: class NewFramebuffer { public: - using Input = glm::uvec2; using Output = gpu::FramebufferPointer; - using JobModel = render::Job::ModelIO; + using JobModel = render::Job::ModelO; - void run(const render::RenderContextPointer& renderContext, const Input& input, Output& output); + NewFramebuffer(gpu::Element pixelFormat); + + void run(const render::RenderContextPointer& renderContext, Output& output); +protected: + gpu::Element _pixelFormat; private: gpu::FramebufferPointer _outputFramebuffer; }; diff --git a/libraries/render-utils/src/RenderDeferredTask.cpp b/libraries/render-utils/src/RenderDeferredTask.cpp index 7f91c6eaf2..4561cf903d 100644 --- a/libraries/render-utils/src/RenderDeferredTask.cpp +++ b/libraries/render-utils/src/RenderDeferredTask.cpp @@ -241,7 +241,7 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren // Lighting Buffer ready for tone mapping const auto toneMappingInputs = ToneMapAndResample::Input(lightingFramebuffer, destFramebuffer).asVarying(); - const auto toneMappedBuffer = task.addJob("ToneMapAndResample", toneMappingInputs); + const auto toneMappedBuffer = task.addJob("ToneMapping", toneMappingInputs); // Debugging task is happening in the "over" layer after tone mapping and just before HUD { // Debug the bounds of the rendered items, still look at the zbuffer diff --git a/libraries/render-utils/src/RenderForwardTask.cpp b/libraries/render-utils/src/RenderForwardTask.cpp index fdcb6c6741..14f2e51697 100755 --- a/libraries/render-utils/src/RenderForwardTask.cpp +++ b/libraries/render-utils/src/RenderForwardTask.cpp @@ -140,7 +140,7 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend task.addJob("DrawZoneStack", debugZoneInputs); } - const auto newResolvedFramebuffer = task.addJob("MakeResolvingFramebuffer"); + const auto newResolvedFramebuffer = task.addJob("MakeResolvingFramebuffer", gpu::Element(gpu::SCALAR, gpu::FLOAT, gpu::R11G11B10)); const auto resolveInputs = ResolveFramebuffer::Inputs(scaledPrimaryFramebuffer, newResolvedFramebuffer).asVarying(); const auto resolvedFramebuffer = task.addJob("Resolve", resolveInputs); @@ -148,7 +148,7 @@ void RenderForwardTask::build(JobModel& task, const render::Varying& input, rend const auto destFramebuffer = static_cast(nullptr); const auto toneMappingInputs = ToneMapAndResample::Input(resolvedFramebuffer, destFramebuffer).asVarying(); - const auto toneMappedBuffer = task.addJob("ToneMapAndResample", toneMappingInputs); + const auto toneMappedBuffer = task.addJob("ToneMapping", toneMappingInputs); // HUD Layer const auto renderHUDLayerInputs = RenderHUDLayerTask::Input(toneMappedBuffer, lightingModel, hudOpaque, hudTransparent, hazeFrame).asVarying(); task.addJob("RenderHUDLayer", renderHUDLayerInputs); diff --git a/libraries/render-utils/src/ToneMapAndResampleTask.cpp b/libraries/render-utils/src/ToneMapAndResampleTask.cpp index 54821b0380..8d4a3c485d 100644 --- a/libraries/render-utils/src/ToneMapAndResampleTask.cpp +++ b/libraries/render-utils/src/ToneMapAndResampleTask.cpp @@ -68,13 +68,13 @@ void ToneMapAndResample::run(const RenderContextPointer& renderContext, const In RenderArgs* args = renderContext->args; auto lightingBuffer = input.get0()->getRenderBuffer(0); - auto resampledFramebuffer = input.get1(); + auto destinationFramebuffer = input.get1(); - if (!resampledFramebuffer) { - resampledFramebuffer = args->_blitFramebuffer; + if (!destinationFramebuffer) { + destinationFramebuffer = args->_blitFramebuffer; } - if (!lightingBuffer || !resampledFramebuffer) { + if (!lightingBuffer || !destinationFramebuffer) { return; } @@ -82,7 +82,7 @@ void ToneMapAndResample::run(const RenderContextPointer& renderContext, const In init(); } - const auto bufferSize = resampledFramebuffer->getSize(); + const auto bufferSize = destinationFramebuffer->getSize(); auto srcBufferSize = glm::ivec2(lightingBuffer->getDimensions()); @@ -90,7 +90,7 @@ void ToneMapAndResample::run(const RenderContextPointer& renderContext, const In gpu::doInBatch("Resample::run", args->_context, [&](gpu::Batch& batch) { batch.enableStereo(false); - batch.setFramebuffer(resampledFramebuffer); + batch.setFramebuffer(destinationFramebuffer); batch.setViewportTransform(destViewport); batch.setProjectionTransform(glm::mat4()); @@ -106,5 +106,5 @@ void ToneMapAndResample::run(const RenderContextPointer& renderContext, const In // Set full final viewport args->_viewport = destViewport; - output = resampledFramebuffer; + output = destinationFramebuffer; } From 4e4f8866cd077e0125bc5da02392fb25f7236c84 Mon Sep 17 00:00:00 2001 From: Anna Date: Tue, 30 Jul 2019 13:15:26 -0700 Subject: [PATCH 27/41] fix android/mac build error --- libraries/render-utils/src/RenderCommonTask.cpp | 2 +- libraries/render-utils/src/RenderCommonTask.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/render-utils/src/RenderCommonTask.cpp b/libraries/render-utils/src/RenderCommonTask.cpp index 43983b76f7..7cf7f1129f 100644 --- a/libraries/render-utils/src/RenderCommonTask.cpp +++ b/libraries/render-utils/src/RenderCommonTask.cpp @@ -148,7 +148,7 @@ void Blit::run(const RenderContextPointer& renderContext, const gpu::Framebuffer }); } -NewFramebuffer::NewFramebuffer(gpu::Element pixelFormat = gpu::Element::COLOR_SRGBA_32) { +NewFramebuffer::NewFramebuffer(gpu::Element pixelFormat) { _pixelFormat = pixelFormat; } diff --git a/libraries/render-utils/src/RenderCommonTask.h b/libraries/render-utils/src/RenderCommonTask.h index 446d6a351c..15d6ff9895 100644 --- a/libraries/render-utils/src/RenderCommonTask.h +++ b/libraries/render-utils/src/RenderCommonTask.h @@ -88,7 +88,7 @@ public: using Output = gpu::FramebufferPointer; using JobModel = render::Job::ModelO; - NewFramebuffer(gpu::Element pixelFormat); + NewFramebuffer(gpu::Element pixelFormat = gpu::Element::COLOR_SRGBA_32); void run(const render::RenderContextPointer& renderContext, Output& output); protected: From 32d5ab7b1f0ce18cf4def40bf61c8b946bd48a25 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Thu, 27 Jun 2019 08:43:59 -0700 Subject: [PATCH 28/41] Make WebRTC library available --- cmake/macros/TargetWebRTC.cmake | 16 ++++++++++ cmake/ports/hifi-deps/CONTROL | 2 +- cmake/ports/webrtc/CONTROL | 3 ++ cmake/ports/webrtc/portfile.cmake | 35 ++++++++++++++++++++++ libraries/audio-client/CMakeLists.txt | 2 ++ libraries/audio-client/src/AudioClient.cpp | 9 +++++- libraries/audio-client/src/AudioClient.h | 5 ++++ libraries/shared/src/shared/WebRTC.h | 34 +++++++++++++++++++++ 8 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 cmake/macros/TargetWebRTC.cmake create mode 100644 cmake/ports/webrtc/CONTROL create mode 100644 cmake/ports/webrtc/portfile.cmake create mode 100644 libraries/shared/src/shared/WebRTC.h diff --git a/cmake/macros/TargetWebRTC.cmake b/cmake/macros/TargetWebRTC.cmake new file mode 100644 index 0000000000..e117011101 --- /dev/null +++ b/cmake/macros/TargetWebRTC.cmake @@ -0,0 +1,16 @@ +# +# Copyright 2019 High Fidelity, Inc. +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +# +macro(TARGET_WEBRTC) + +if (ANDROID) +else() + target_include_directories(${TARGET_NAME} SYSTEM PUBLIC "${VCPKG_INSTALL_ROOT}/include/webrtc") + find_library(WEBRTC_LIBRARY NAMES webrtc PATHS ${VCPKG_INSTALL_ROOT}/lib/ NO_DEFAULT_PATH) + target_link_libraries(${TARGET_NAME} ${WEBRTC_LIBRARY}) +endif() + +endmacro() diff --git a/cmake/ports/hifi-deps/CONTROL b/cmake/ports/hifi-deps/CONTROL index 5f860a1620..2441de9002 100644 --- a/cmake/ports/hifi-deps/CONTROL +++ b/cmake/ports/hifi-deps/CONTROL @@ -1,4 +1,4 @@ Source: hifi-deps Version: 0.1 Description: Collected dependencies for High Fidelity applications -Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib +Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib, webrtc (!android) diff --git a/cmake/ports/webrtc/CONTROL b/cmake/ports/webrtc/CONTROL new file mode 100644 index 0000000000..12a76920b9 --- /dev/null +++ b/cmake/ports/webrtc/CONTROL @@ -0,0 +1,3 @@ +Source: webrtc +Version: 20190626 +Description: WebRTC diff --git a/cmake/ports/webrtc/portfile.cmake b/cmake/ports/webrtc/portfile.cmake new file mode 100644 index 0000000000..85479d4201 --- /dev/null +++ b/cmake/ports/webrtc/portfile.cmake @@ -0,0 +1,35 @@ +include(vcpkg_common_functions) +set(WEBRTC_VERSION 20190626) +set(MASTER_COPY_SOURCE_PATH ${CURRENT_BUILDTREES_DIR}/src) + +if (WIN32) + vcpkg_download_distfile( + WEBRTC_SOURCE_ARCHIVE + URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-windows.zip + SHA512 f4444a95f87800b446d95a4c381b69a8ddc36d89a08ce57bf2b03d684ce528b592a3260ef2f9dee314e38b58dd236a3495677e1c7773ce3d4057f59e80b48f32 + FILENAME webrtc-20190626-windows.zip + ) +elseif (APPLE) + vcpkg_download_distfile( + WEBRTC_SOURCE_ARCHIVE + URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-osx.tar.gz + SHA512 fc70cec1b5ee87395137b7090f424e2fc2300fc17d744d5ffa1cf7aa0e0f1a069a9d72ba1ad2fb4a640ebeb6c218bda24351ba0083e1ff96c4a4b5032648a9d2 + FILENAME webrtc-20190626-osx.tar.gz + ) +elseif (ANDROID) +# then not desktop Linux +else () + vcpkg_download_distfile( + WEBRTC_SOURCE_ARCHIVE + URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-linux.tar.gz + SHA512 7e41c3350e4dd3bbbedff9070ef0887db4e8d1607270ac2057cee3c8c2bf17aa2bdccb6c1bfa14526338a51b57d7896b7e1230c6cedb41f3afe5c49a5f1a7319 + FILENAME webrtc-20190626-linux.tar.gz + ) +endif () + +vcpkg_extract_source_archive(${WEBRTC_SOURCE_ARCHIVE}) + +file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/include DESTINATION ${CURRENT_PACKAGES_DIR}) +file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/lib DESTINATION ${CURRENT_PACKAGES_DIR}) +file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/share DESTINATION ${CURRENT_PACKAGES_DIR}) +file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/debug DESTINATION ${CURRENT_PACKAGES_DIR}) diff --git a/libraries/audio-client/CMakeLists.txt b/libraries/audio-client/CMakeLists.txt index 6ca7962c39..48d3d74cd6 100644 --- a/libraries/audio-client/CMakeLists.txt +++ b/libraries/audio-client/CMakeLists.txt @@ -7,6 +7,8 @@ link_hifi_libraries(audio plugins) include_hifi_library_headers(shared) include_hifi_library_headers(networking) +target_webrtc() + # append audio includes to our list of includes to bubble target_include_directories(${TARGET_NAME} PUBLIC "${HIFI_LIBRARY_DIR}/audio/src") diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 04ab0f7973..3394543ddd 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -24,7 +24,7 @@ #endif #ifdef WIN32 -#define WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN 1 #include #include #include @@ -302,6 +302,13 @@ AudioClient::AudioClient() : _isHeadsetPluggedIn(false), #endif _orientationGetter(DEFAULT_ORIENTATION_GETTER) { + +#if defined(WEBRTC_ENABLED) + qDebug() << "QQQQ calling AudioProcessingBuilder"; + _apm = webrtc::AudioProcessingBuilder().Create(); + qDebug() << "QQQQ done calling AudioProcessingBuilder"; +#endif + // avoid putting a lock in the device callback assert(_localSamplesAvailable.is_lock_free()); diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index decf0f7751..67a064929c 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -475,6 +476,10 @@ private: QTimer* _checkPeakValuesTimer { nullptr }; bool _isRecording { false }; + +#if WEBRTC_ENABLED + webrtc::AudioProcessing* _apm; +#endif }; diff --git a/libraries/shared/src/shared/WebRTC.h b/libraries/shared/src/shared/WebRTC.h new file mode 100644 index 0000000000..3085f46c0a --- /dev/null +++ b/libraries/shared/src/shared/WebRTC.h @@ -0,0 +1,34 @@ +// +// WebRTC.h +// libraries/shared/src/shared/ +// +// Copyright 2019 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_WebRTC_h +#define hifi_WebRTC_h + +#if defined(Q_OS_MAC) +# define WEBRTC_ENABLED 1 +# define WEBRTC_POSIX 1 +#elif defined(WIN32) +# define WEBRTC_ENABLED 1 +# define WEBRTC_WIN 1 +# define NOMINMAX 1 +# define WIN32_LEAN_AND_MEAN 1 +#elif defined(Q_OS_ANDROID) +// no webrtc for android -- this is here so the LINUX clause doesn't get used, below +#elif defined(Q_OS_LINUX) +# define WEBRTC_ENABLED 1 +# define WEBRTC_POSIX 1 +#endif + +#if defined(WEBRTC_ENABLED) +# include +# include "modules/audio_processing/audio_processing_impl.h" +#endif + +#endif // hifi_WebRTC_h From 80aeeaa3e0cac81b85af73f686adfaf532844e5a Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Thu, 18 Jul 2019 11:43:46 -0700 Subject: [PATCH 29/41] Refactor the audio output callback, to expose the final output (far-end) needed for AEC --- libraries/audio-client/src/AudioClient.cpp | 75 +++++++++++----------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 3394543ddd..946b47fec5 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -2114,15 +2114,16 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { return maxSize; } - // samples requested from OUTPUT_CHANNEL_COUNT + // max samples requested from OUTPUT_CHANNEL_COUNT int deviceChannelCount = _audio->_outputFormat.channelCount(); - int samplesRequested = (int)(maxSize / AudioConstants::SAMPLE_SIZE) * OUTPUT_CHANNEL_COUNT / deviceChannelCount; + int maxSamplesRequested = (int)(maxSize / AudioConstants::SAMPLE_SIZE) * OUTPUT_CHANNEL_COUNT / deviceChannelCount; // restrict samplesRequested to the size of our mix/scratch buffers - samplesRequested = std::min(samplesRequested, _audio->_outputPeriod); + maxSamplesRequested = std::min(maxSamplesRequested, _audio->_outputPeriod); int16_t* scratchBuffer = _audio->_outputScratchBuffer; float* mixBuffer = _audio->_outputMixBuffer; + int samplesRequested = maxSamplesRequested; int networkSamplesPopped; if ((networkSamplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) { qCDebug(audiostream, "Read %d samples from buffer (%d available, %d requested)", networkSamplesPopped, _receivedAudioStream.getSamplesAvailable(), samplesRequested); @@ -2167,40 +2168,42 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { }); int samplesPopped = std::max(networkSamplesPopped, injectorSamplesPopped); - int framesPopped = samplesPopped / AudioConstants::STEREO; - int bytesWritten; - if (samplesPopped > 0) { - - // apply output gain - float newGain = _audio->_outputGain.load(std::memory_order_acquire); - float oldGain = _audio->_lastOutputGain; - _audio->_lastOutputGain = newGain; - - applyGainSmoothing(mixBuffer, framesPopped, oldGain, newGain); - - if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) { - // limit the audio - _audio->_audioLimiter.render(mixBuffer, (int16_t*)data, framesPopped); - } else { - _audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped); - - // upmix or downmix to deviceChannelCount - if (deviceChannelCount > OUTPUT_CHANNEL_COUNT) { - int extraChannels = deviceChannelCount - OUTPUT_CHANNEL_COUNT; - channelUpmix(scratchBuffer, (int16_t*)data, samplesPopped, extraChannels); - } else { - channelDownmix(scratchBuffer, (int16_t*)data, samplesPopped); - } - } - - bytesWritten = framesPopped * AudioConstants::SAMPLE_SIZE * deviceChannelCount; - assert(bytesWritten <= maxSize); - - } else { - // nothing on network, don't grab anything from injectors, and just return 0s - memset(data, 0, maxSize); - bytesWritten = maxSize; + if (samplesPopped == 0) { + // nothing on network, don't grab anything from injectors, and fill with silence + samplesPopped = maxSamplesRequested; + memset(mixBuffer, 0, samplesPopped * sizeof(float)); } + int framesPopped = samplesPopped / OUTPUT_CHANNEL_COUNT; + + // apply output gain + float newGain = _audio->_outputGain.load(std::memory_order_acquire); + float oldGain = _audio->_lastOutputGain; + _audio->_lastOutputGain = newGain; + + applyGainSmoothing(mixBuffer, framesPopped, oldGain, newGain); + + // limit the audio + _audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped); + + // TODO: + // At this point, scratchBuffer contains the final (mixed, limited) output audio. + // format = interleaved int16_t + // samples = samplesPopped + // channels = OUTPUT_CHANNEL_COUNT + // sampleRate = _outputFormat.sampleRate() + // This can be used as the far-end signal for AEC. + + // if required, upmix or downmix to deviceChannelCount + if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) { + memcpy(data, scratchBuffer, samplesPopped * AudioConstants::SAMPLE_SIZE); + } else if (deviceChannelCount > OUTPUT_CHANNEL_COUNT) { + int extraChannels = deviceChannelCount - OUTPUT_CHANNEL_COUNT; + channelUpmix(scratchBuffer, (int16_t*)data, samplesPopped, extraChannels); + } else { + channelDownmix(scratchBuffer, (int16_t*)data, samplesPopped); + } + int bytesWritten = framesPopped * AudioConstants::SAMPLE_SIZE * deviceChannelCount; + assert(bytesWritten <= maxSize); // send output buffer for recording if (_audio->_isRecording) { From c3b9e4806a14f889fab0918d43f493da8031c713 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Thu, 18 Jul 2019 11:54:01 -0700 Subject: [PATCH 30/41] Fix longstanding bug in Audio Scope feature that records output to WAV --- libraries/audio-client/src/AudioClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 946b47fec5..9bf7a24c56 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -2208,7 +2208,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { // send output buffer for recording if (_audio->_isRecording) { Lock lock(_recordMutex); - _audio->_audioFileWav.addRawAudioChunk(reinterpret_cast(scratchBuffer), bytesWritten); + _audio->_audioFileWav.addRawAudioChunk(data, bytesWritten); } int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree(); From 569c76c8ba51ff25a90ebd7b1ec85206f4b0184d Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Sat, 20 Jul 2019 13:05:11 -0700 Subject: [PATCH 31/41] Baseline AEC using WebRTC, always enabled. Audio streams are hooked as close as possible to device input/output, re-buffering as needed. --- libraries/audio-client/src/AudioClient.cpp | 156 +++++++++++++++++++-- libraries/audio-client/src/AudioClient.h | 12 +- 2 files changed, 150 insertions(+), 18 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 9bf7a24c56..b95dd96d06 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -303,12 +303,6 @@ AudioClient::AudioClient() : #endif _orientationGetter(DEFAULT_ORIENTATION_GETTER) { -#if defined(WEBRTC_ENABLED) - qDebug() << "QQQQ calling AudioProcessingBuilder"; - _apm = webrtc::AudioProcessingBuilder().Create(); - qDebug() << "QQQQ done calling AudioProcessingBuilder"; -#endif - // avoid putting a lock in the device callback assert(_localSamplesAvailable.is_lock_free()); @@ -360,6 +354,10 @@ AudioClient::AudioClient() : configureReverb(); +#if defined(WEBRTC_ENABLED) + configureWebrtc(); +#endif + auto nodeList = DependencyManager::get(); auto& packetReceiver = nodeList->getPacketReceiver(); packetReceiver.registerListener(PacketType::AudioStreamStats, &_stats, "processStreamStatsPacket"); @@ -1091,6 +1089,137 @@ void AudioClient::setReverbOptions(const AudioEffectOptions* options) { } } +#if defined(WEBRTC_ENABLED) + +static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * webrtc::AudioProcessing::kMaxNativeSampleRateHz / 1000; +static const int WEBRTC_CHANNELS_MAX = 2; + +static void deinterleaveToFloat(const int16_t* src, float* const* dst, int numFrames, int numChannels) { + for (int i = 0; i < numFrames; i++) { + for (int ch = 0; ch < numChannels; ch++) { + float f = *src++; + f *= (1/32768.0f); // scale + dst[ch][i] = f; // deinterleave + } + } +} + +static void interleaveToInt16(const float* const* src, int16_t* dst, int numFrames, int numChannels) { + for (int i = 0; i < numFrames; i++) { + for (int ch = 0; ch < numChannels; ch++) { + float f = src[ch][i]; + f *= 32768.0f; // scale + f += (f < 0.0f) ? -0.5f : 0.5f; // round + f = std::max(std::min(f, 32767.0f), -32768.0f); // saturate + *dst++ = (int16_t)f; // interleave + } + } +} + +void AudioClient::configureWebrtc() { + _apm = webrtc::AudioProcessingBuilder().Create(); + + webrtc::AudioProcessing::Config config; + + config.pre_amplifier.enabled = false; + config.high_pass_filter.enabled = false; + config.echo_canceller.enabled = true; + config.echo_canceller.mobile_mode = false; + config.echo_canceller.use_legacy_aec = false; + config.noise_suppression.enabled = false; + config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kModerate; + config.voice_detection.enabled = false; + config.gain_controller1.enabled = false; + config.gain_controller2.enabled = false; + config.gain_controller2.fixed_digital.gain_db = 0.0f; + config.gain_controller2.adaptive_digital.enabled = false; + config.residual_echo_detector.enabled = true; + config.level_estimation.enabled = false; + + _apm->ApplyConfig(config); + + qCDebug(audioclient) << "WebRTC enabled for acoustic echo cancellation."; +} + +// rebuffer into 10ms chunks +void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate) { + + // TODO: move to AudioClient.h + static int16_t _fifo[WEBRTC_CHANNELS_MAX * WEBRTC_FRAMES_MAX]; + static int _numFifo = 0; // numFrames saved in fifo + + const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels); + const int numChunk = (int)streamConfig.num_frames(); + + if (sampleRate > webrtc::AudioProcessing::kMaxNativeSampleRateHz) { + qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "output sample rate."; + return; + } + if (numChannels > WEBRTC_CHANNELS_MAX) { + qCWarning(audioclient) << "WebRTC does not support" << numChannels << "output channels."; + return; + } + + while (numFrames > 0) { + + // number of frames to fill + int numFill = std::min(numFrames, numChunk - _numFifo); + + // refill fifo + memcpy(&_fifo[_numFifo], samples, numFill * numChannels * sizeof(int16_t)); + samples += numFill * numChannels; + numFrames -= numFill; + _numFifo += numFill; + + if (_numFifo == numChunk) { + + // convert audio format + float buffer[WEBRTC_CHANNELS_MAX][WEBRTC_FRAMES_MAX]; + float* const buffers[WEBRTC_CHANNELS_MAX] = { buffer[0], buffer[1] }; + deinterleaveToFloat(_fifo, buffers, numChunk, numChannels); + + // process one chunk + if (_apm->kNoError != _apm->ProcessReverseStream(buffers, streamConfig, streamConfig, buffers)) { + qCWarning(audioclient) << "WebRTC ProcessReverseStream() returned an ERROR."; + } + _numFifo = 0; + } + } +} + +void AudioClient::processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate) { + + const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels); + const int numChunk = (int)streamConfig.num_frames(); + + if (sampleRate > webrtc::AudioProcessing::kMaxNativeSampleRateHz) { + qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "input sample rate."; + return; + } + if (numChannels > WEBRTC_CHANNELS_MAX) { + qCWarning(audioclient) << "WebRTC does not support" << numChannels << "input channels."; + return; + } + if (numFrames != numChunk) { + qCWarning(audioclient) << "WebRTC requires exactly 10ms of input."; + return; + } + + // convert audio format + float buffer[WEBRTC_CHANNELS_MAX][WEBRTC_FRAMES_MAX]; + float* const buffers[WEBRTC_CHANNELS_MAX] = { buffer[0], buffer[1] }; + deinterleaveToFloat(samples, buffers, numFrames, numChannels); + + // process one chunk + if (_apm->kNoError != _apm->ProcessStream(buffers, streamConfig, streamConfig, buffers)) { + qCWarning(audioclient) << "WebRTC ProcessStream() returned an ERROR."; + } + // modify samples in-place + interleaveToInt16(buffers, samples, numFrames, numChannels); +} + +#endif // WEBRTC_ENABLED + void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { // If there is server echo, reverb will be applied to the recieved audio stream so no need to have it here. bool hasReverb = _reverb || _receivedAudioStream.hasReverb(); @@ -1269,6 +1398,11 @@ void AudioClient::handleMicAudioInput() { _inputRingBuffer.readSamples(inputAudioSamples.get(), inputSamplesRequired); +#if defined(WEBRTC_ENABLED) + processWebrtcNearEnd(inputAudioSamples.get(), inputSamplesRequired / _inputFormat.channelCount(), + _inputFormat.channelCount(), _inputFormat.sampleRate()); +#endif + // detect loudness and clipping on the raw input bool isClipping = false; float loudness = computeLoudness(inputAudioSamples.get(), inputSamplesRequired, _inputFormat.channelCount(), isClipping); @@ -2185,13 +2319,9 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { // limit the audio _audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped); - // TODO: - // At this point, scratchBuffer contains the final (mixed, limited) output audio. - // format = interleaved int16_t - // samples = samplesPopped - // channels = OUTPUT_CHANNEL_COUNT - // sampleRate = _outputFormat.sampleRate() - // This can be used as the far-end signal for AEC. +#if defined(WEBRTC_ENABLED) + _audio->processWebrtcFarEnd(scratchBuffer, framesPopped, OUTPUT_CHANNEL_COUNT, _audio->_outputFormat.sampleRate()); +#endif // if required, upmix or downmix to deviceChannelCount if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) { diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 67a064929c..8cf54c1a0d 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -415,9 +415,15 @@ private: // Adds Reverb void configureReverb(); void updateReverbOptions(); - void handleLocalEchoAndReverb(QByteArray& inputByteArray); +#if defined(WEBRTC_ENABLED) + webrtc::AudioProcessing* _apm { nullptr }; + void configureWebrtc(); + void processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate); + void processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate); +#endif + bool switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest = false); bool switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest = false); @@ -476,10 +482,6 @@ private: QTimer* _checkPeakValuesTimer { nullptr }; bool _isRecording { false }; - -#if WEBRTC_ENABLED - webrtc::AudioProcessing* _apm; -#endif }; From c1cadf670e89d8324f6ac85fcb775dbc884b8f44 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Mon, 22 Jul 2019 10:43:43 -0700 Subject: [PATCH 32/41] update webrtc library, enable for android --- cmake/macros/TargetWebRTC.cmake | 17 ++++++++++++----- cmake/ports/hifi-deps/CONTROL | 2 +- cmake/ports/webrtc/portfile.cmake | 11 ++++++----- hifi_android.py | 5 +++++ libraries/shared/src/shared/WebRTC.h | 5 +++-- 5 files changed, 27 insertions(+), 13 deletions(-) diff --git a/cmake/macros/TargetWebRTC.cmake b/cmake/macros/TargetWebRTC.cmake index e117011101..fd5db5fcbe 100644 --- a/cmake/macros/TargetWebRTC.cmake +++ b/cmake/macros/TargetWebRTC.cmake @@ -5,12 +5,19 @@ # See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html # macro(TARGET_WEBRTC) + if (ANDROID) + include(SelectLibraryConfigurations) + set(INSTALL_DIR ${HIFI_ANDROID_PRECOMPILED}/webrtc/webrtc) + set(WEBRTC_INCLUDE_DIRS "${INSTALL_DIR}/include/webrtc") + set(WEBRTC_LIBRARY_DEBUG ${INSTALL_DIR}/debug/lib/libwebrtc.a) + set(WEBRTC_LIBRARY_RELEASE ${INSTALL_DIR}/lib/libwebrtc.a) + select_library_configurations(WEBRTC) + else() + set(WEBRTC_INCLUDE_DIRS "${VCPKG_INSTALL_ROOT}/include/webrtc") + find_library(WEBRTC_LIBRARY NAMES webrtc PATHS ${VCPKG_INSTALL_ROOT}/lib/ NO_DEFAULT_PATH) + endif() -if (ANDROID) -else() - target_include_directories(${TARGET_NAME} SYSTEM PUBLIC "${VCPKG_INSTALL_ROOT}/include/webrtc") - find_library(WEBRTC_LIBRARY NAMES webrtc PATHS ${VCPKG_INSTALL_ROOT}/lib/ NO_DEFAULT_PATH) + target_include_directories(${TARGET_NAME} SYSTEM PUBLIC ${WEBRTC_INCLUDE_DIRS}) target_link_libraries(${TARGET_NAME} ${WEBRTC_LIBRARY}) -endif() endmacro() diff --git a/cmake/ports/hifi-deps/CONTROL b/cmake/ports/hifi-deps/CONTROL index 2441de9002..0a3d194ea1 100644 --- a/cmake/ports/hifi-deps/CONTROL +++ b/cmake/ports/hifi-deps/CONTROL @@ -1,4 +1,4 @@ Source: hifi-deps Version: 0.1 Description: Collected dependencies for High Fidelity applications -Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib, webrtc (!android) +Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib, webrtc diff --git a/cmake/ports/webrtc/portfile.cmake b/cmake/ports/webrtc/portfile.cmake index 85479d4201..3f2fb7a6ab 100644 --- a/cmake/ports/webrtc/portfile.cmake +++ b/cmake/ports/webrtc/portfile.cmake @@ -2,11 +2,13 @@ include(vcpkg_common_functions) set(WEBRTC_VERSION 20190626) set(MASTER_COPY_SOURCE_PATH ${CURRENT_BUILDTREES_DIR}/src) -if (WIN32) +if (ANDROID) + # this is handled by hifi_android.py +elseif (WIN32) vcpkg_download_distfile( WEBRTC_SOURCE_ARCHIVE URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-windows.zip - SHA512 f4444a95f87800b446d95a4c381b69a8ddc36d89a08ce57bf2b03d684ce528b592a3260ef2f9dee314e38b58dd236a3495677e1c7773ce3d4057f59e80b48f32 + SHA512 c0848eddb1579b3bb0496b8785e24f30470f3c477145035fd729264a326a467b9467ae9f426aa5d72d168ad9e9bf2c279150744832736bdf39064d24b04de1a3 FILENAME webrtc-20190626-windows.zip ) elseif (APPLE) @@ -16,13 +18,12 @@ elseif (APPLE) SHA512 fc70cec1b5ee87395137b7090f424e2fc2300fc17d744d5ffa1cf7aa0e0f1a069a9d72ba1ad2fb4a640ebeb6c218bda24351ba0083e1ff96c4a4b5032648a9d2 FILENAME webrtc-20190626-osx.tar.gz ) -elseif (ANDROID) -# then not desktop Linux else () + # else Linux desktop vcpkg_download_distfile( WEBRTC_SOURCE_ARCHIVE URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-linux.tar.gz - SHA512 7e41c3350e4dd3bbbedff9070ef0887db4e8d1607270ac2057cee3c8c2bf17aa2bdccb6c1bfa14526338a51b57d7896b7e1230c6cedb41f3afe5c49a5f1a7319 + SHA512 07d7776551aa78cb09a3ef088a8dee7762735c168c243053b262083d90a1d258cec66dc386f6903da5c4461921a3c2db157a1ee106a2b47e7756cb424b66cc43 FILENAME webrtc-20190626-linux.tar.gz ) endif () diff --git a/hifi_android.py b/hifi_android.py index 42b472e960..926a671a6c 100644 --- a/hifi_android.py +++ b/hifi_android.py @@ -94,6 +94,11 @@ ANDROID_PACKAGES = { 'checksum': 'ddcb23df336b08017042ba4786db1d9e', 'sharedLibFolder': 'lib', 'includeLibs': {'libbreakpad_client.a'} + }, + 'webrtc': { + 'file': 'webrtc-20190626-android.tar.gz', + 'checksum': '8be046cf27e1fabff8a1029eabbe52c8', + 'versionId': '8TU1LgZswAgZpBiNYvB5s3ZLwSqNJHuj', } } diff --git a/libraries/shared/src/shared/WebRTC.h b/libraries/shared/src/shared/WebRTC.h index 3085f46c0a..6f05d38a3d 100644 --- a/libraries/shared/src/shared/WebRTC.h +++ b/libraries/shared/src/shared/WebRTC.h @@ -14,13 +14,14 @@ #if defined(Q_OS_MAC) # define WEBRTC_ENABLED 1 # define WEBRTC_POSIX 1 -#elif defined(WIN32) +#elif defined(Q_OS_WIN) # define WEBRTC_ENABLED 1 # define WEBRTC_WIN 1 # define NOMINMAX 1 # define WIN32_LEAN_AND_MEAN 1 #elif defined(Q_OS_ANDROID) -// no webrtc for android -- this is here so the LINUX clause doesn't get used, below +# define WEBRTC_ENABLED 1 +# define WEBRTC_POSIX 1 #elif defined(Q_OS_LINUX) # define WEBRTC_ENABLED 1 # define WEBRTC_POSIX 1 From 8ec4505b953ffe6c1f7e968211ab0dd9eaa73c1b Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Wed, 24 Jul 2019 18:51:02 -0700 Subject: [PATCH 33/41] add an on/off control for audio echo cancelation to Audio Settings page. It's not yet hooked to anything. --- interface/resources/qml/hifi/audio/Audio.qml | 24 +++++++++++++++---- interface/src/scripting/Audio.cpp | 25 ++++++++++++++++++++ interface/src/scripting/Audio.h | 16 +++++++++++++ libraries/audio-client/src/AudioClient.cpp | 10 ++++++++ libraries/audio-client/src/AudioClient.h | 5 ++++ 5 files changed, 76 insertions(+), 4 deletions(-) diff --git a/interface/resources/qml/hifi/audio/Audio.qml b/interface/resources/qml/hifi/audio/Audio.qml index af517be55d..9db19e72e2 100644 --- a/interface/resources/qml/hifi/audio/Audio.qml +++ b/interface/resources/qml/hifi/audio/Audio.qml @@ -166,16 +166,16 @@ Rectangle { x: 2 * margins.paddings; width: parent.width; // switch heights + 2 * top margins - height: (root.switchHeight) * 3 + 48; + height: (root.switchHeight) * 6 + 48; anchors.top: firstSeparator.bottom; anchors.topMargin: 10; - // mute is in its own row Item { id: switchContainer; x: margins.paddings; width: parent.width / 2; height: parent.height; + anchors.top: parent.top anchors.left: parent.left; HifiControlsUit.Switch { id: muteMic; @@ -222,12 +222,29 @@ Rectangle { } HifiControlsUit.Switch { - id: pttSwitch + id: acousticEchoCancellationSwitch; height: root.switchHeight; switchWidth: root.switchWidth; anchors.top: noiseReductionSwitch.bottom anchors.topMargin: 24 anchors.left: parent.left + labelTextOn: "Echo Cancellation"; + labelTextSize: 16; + backgroundOnColor: "#E3E3E3"; + checked: AudioScriptingInterface.acousticEchoCancellation; + onCheckedChanged: { + AudioScriptingInterface.acousticEchoCancellation = checked; + checked = Qt.binding(function() { return AudioScriptingInterface.acousticEchoCancellation; }); + } + } + + HifiControlsUit.Switch { + id: pttSwitch + height: root.switchHeight; + switchWidth: root.switchWidth; + anchors.top: acousticEchoCancellationSwitch.bottom; + anchors.topMargin: 24 + anchors.left: parent.left labelTextOn: (bar.currentIndex === 0) ? qsTr("Push To Talk (T)") : qsTr("Push To Talk"); labelTextSize: 16; backgroundOnColor: "#E3E3E3"; @@ -298,7 +315,6 @@ Rectangle { checked = Qt.binding(function() { return AudioScriptingInterface.isStereoInput; }); // restore binding } } - } } diff --git a/interface/src/scripting/Audio.cpp b/interface/src/scripting/Audio.cpp index a0bea256ad..f674b533a7 100644 --- a/interface/src/scripting/Audio.cpp +++ b/interface/src/scripting/Audio.cpp @@ -26,6 +26,7 @@ QString Audio::HMD { "VR" }; Setting::Handle enableNoiseReductionSetting { QStringList { Audio::AUDIO, "NoiseReduction" }, true }; Setting::Handle enableWarnWhenMutedSetting { QStringList { Audio::AUDIO, "WarnWhenMuted" }, true }; +Setting::Handle enableAcousticEchoCancellationSetting { QStringList { Audio::AUDIO, "AcousticEchoCancellation" }, true }; float Audio::loudnessToLevel(float loudness) { @@ -40,12 +41,14 @@ Audio::Audio() : _devices(_contextIsHMD) { connect(client, &AudioClient::muteToggled, this, &Audio::setMuted); connect(client, &AudioClient::noiseReductionChanged, this, &Audio::enableNoiseReduction); connect(client, &AudioClient::warnWhenMutedChanged, this, &Audio::enableWarnWhenMuted); + connect(client, &AudioClient::acousticEchoCancellationChanged, this, &Audio::enableAcousticEchoCancellation); connect(client, &AudioClient::inputLoudnessChanged, this, &Audio::onInputLoudnessChanged); connect(client, &AudioClient::inputVolumeChanged, this, &Audio::setInputVolume); connect(this, &Audio::contextChanged, &_devices, &AudioDevices::onContextChanged); connect(this, &Audio::pushingToTalkChanged, this, &Audio::handlePushedToTalk); enableNoiseReduction(enableNoiseReductionSetting.get()); enableWarnWhenMuted(enableWarnWhenMutedSetting.get()); + enableAcousticEchoCancellation(enableAcousticEchoCancellationSetting.get()); onContextChanged(); } @@ -277,6 +280,28 @@ void Audio::enableWarnWhenMuted(bool enable) { } } +bool Audio::acousticEchoCancellationEnabled() const { + return resultWithReadLock([&] { + return _enableAcousticEchoCancellation; + }); +} + +void Audio::enableAcousticEchoCancellation(bool enable) { + bool changed = false; + withWriteLock([&] { + if (_enableAcousticEchoCancellation != enable) { + _enableAcousticEchoCancellation = enable; + auto client = DependencyManager::get().data(); + QMetaObject::invokeMethod(client, "setAcousticEchoCancellation", Q_ARG(bool, enable), Q_ARG(bool, false)); + enableAcousticEchoCancellationSetting.set(enable); + changed = true; + } + }); + if (changed) { + emit acousticEchoCancellationChanged(enable); + } +} + float Audio::getInputVolume() const { return resultWithReadLock([&] { return _inputVolume; diff --git a/interface/src/scripting/Audio.h b/interface/src/scripting/Audio.h index aab1ade95b..d2d1ee36c9 100644 --- a/interface/src/scripting/Audio.h +++ b/interface/src/scripting/Audio.h @@ -72,6 +72,9 @@ class Audio : public AudioScriptingInterface, protected ReadWriteLockable { * @property {number} systemInjectorGain - The gain (relative volume) that system sounds are played at. * @property {number} pushingToTalkOutputGainDesktop - The gain (relative volume) that all sounds are played at when the user is holding * the push-to-talk key in Desktop mode. + * @property {boolean} acousticEchoCancellation - true if audio-echo-cancellation is enabled, otherwise + * false. When enabled, sound from the audio output will be suppressed when it echos back to the + * input audio signal. * * @comment The following properties are from AudioScriptingInterface.h. * @property {boolean} isStereoInput - true if the input audio is being used in stereo, otherwise @@ -85,6 +88,8 @@ class Audio : public AudioScriptingInterface, protected ReadWriteLockable { Q_PROPERTY(bool muted READ isMuted WRITE setMuted NOTIFY mutedChanged) Q_PROPERTY(bool noiseReduction READ noiseReductionEnabled WRITE enableNoiseReduction NOTIFY noiseReductionChanged) Q_PROPERTY(bool warnWhenMuted READ warnWhenMutedEnabled WRITE enableWarnWhenMuted NOTIFY warnWhenMutedChanged) + Q_PROPERTY(bool acousticEchoCancellation + READ acousticEchoCancellationEnabled WRITE enableAcousticEchoCancellation NOTIFY acousticEchoCancellationChanged) Q_PROPERTY(float inputVolume READ getInputVolume WRITE setInputVolume NOTIFY inputVolumeChanged) Q_PROPERTY(float inputLevel READ getInputLevel NOTIFY inputLevelChanged) Q_PROPERTY(bool clipping READ isClipping NOTIFY clippingChanged) @@ -115,6 +120,7 @@ public: bool isMuted() const; bool noiseReductionEnabled() const; bool warnWhenMutedEnabled() const; + bool acousticEchoCancellationEnabled() const; float getInputVolume() const; float getInputLevel() const; bool isClipping() const; @@ -396,6 +402,14 @@ signals: */ void warnWhenMutedChanged(bool isEnabled); + /**jsdoc + * Triggered when acoustic echo cancellation is enabled or disabled. + * @function Audio.acousticEchoCancellationChanged + * @param {boolean} isEnabled - true if acoustic echo cancellation is enabled, otherwise false. + * @returns {Signal} + */ + void acousticEchoCancellationChanged(bool isEnabled); + /**jsdoc * Triggered when the input audio volume changes. * @function Audio.inputVolumeChanged @@ -494,6 +508,7 @@ private slots: void setMuted(bool muted); void enableNoiseReduction(bool enable); void enableWarnWhenMuted(bool enable); + void enableAcousticEchoCancellation(bool enable); void setInputVolume(float volume); void onInputLoudnessChanged(float loudness, bool isClipping); @@ -512,6 +527,7 @@ private: bool _isClipping { false }; bool _enableNoiseReduction { true }; // Match default value of AudioClient::_isNoiseGateEnabled. bool _enableWarnWhenMuted { true }; + bool _enableAcousticEchoCancellation { true }; // AudioClient::_isAECEnabled bool _contextIsHMD { false }; AudioDevices* getDevices() { return &_devices; } AudioDevices _devices; diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index b95dd96d06..6f5fe0e050 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -286,6 +286,7 @@ AudioClient::AudioClient() : _shouldEchoLocally(false), _shouldEchoToServer(false), _isNoiseGateEnabled(true), + _isAECEnabled(true), _reverb(false), _reverbOptions(&_scriptReverbOptions), _inputToNetworkResampler(NULL), @@ -1715,6 +1716,15 @@ void AudioClient::setWarnWhenMuted(bool enable, bool emitSignal) { } } +void AudioClient::setAcousticEchoCancellation(bool enable, bool emitSignal) { + if (_isAECEnabled != enable) { + _isAECEnabled = enable; + if (emitSignal) { + emit acousticEchoCancellationChanged(_isAECEnabled); + } + } +} + bool AudioClient::setIsStereoInput(bool isStereoInput) { bool stereoInputChanged = false; if (isStereoInput != _isStereoInput && _inputDeviceInfo.supportedChannelCounts().contains(2)) { diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 8cf54c1a0d..638c1b61e9 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -216,6 +216,9 @@ public slots: void setWarnWhenMuted(bool isNoiseGateEnabled, bool emitSignal = true); bool isWarnWhenMutedEnabled() const { return _warnWhenMuted; } + void setAcousticEchoCancellation(bool isAECEnabled, bool emitSignal = true); + bool isAcousticEchoCancellationEnabled() const { return _isAECEnabled; } + virtual bool getLocalEcho() override { return _shouldEchoLocally; } virtual void setLocalEcho(bool localEcho) override { _shouldEchoLocally = localEcho; } virtual void toggleLocalEcho() override { _shouldEchoLocally = !_shouldEchoLocally; } @@ -257,6 +260,7 @@ signals: void muteToggled(bool muted); void noiseReductionChanged(bool noiseReductionEnabled); void warnWhenMutedChanged(bool warnWhenMutedEnabled); + void acousticEchoCancellationChanged(bool acousticEchoCancellationEnabled); void mutedByMixer(); void inputReceived(const QByteArray& inputSamples); void inputLoudnessChanged(float loudness, bool isClipping); @@ -378,6 +382,7 @@ private: bool _shouldEchoToServer; bool _isNoiseGateEnabled; bool _warnWhenMuted; + bool _isAECEnabled; bool _reverb; AudioEffectOptions _scriptReverbOptions; From 5c86c22076c9196af0c052f36e181aa3e71e6ce0 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Fri, 26 Jul 2019 12:04:00 -0700 Subject: [PATCH 34/41] Move WebRTC far-end FIFO into AudioClient.h --- libraries/audio-client/src/AudioClient.cpp | 19 ++++++------------- libraries/audio-client/src/AudioClient.h | 7 +++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 6f5fe0e050..f4a01869e2 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1092,9 +1092,6 @@ void AudioClient::setReverbOptions(const AudioEffectOptions* options) { #if defined(WEBRTC_ENABLED) -static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * webrtc::AudioProcessing::kMaxNativeSampleRateHz / 1000; -static const int WEBRTC_CHANNELS_MAX = 2; - static void deinterleaveToFloat(const int16_t* src, float* const* dst, int numFrames, int numChannels) { for (int i = 0; i < numFrames; i++) { for (int ch = 0; ch < numChannels; ch++) { @@ -1145,10 +1142,6 @@ void AudioClient::configureWebrtc() { // rebuffer into 10ms chunks void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate) { - // TODO: move to AudioClient.h - static int16_t _fifo[WEBRTC_CHANNELS_MAX * WEBRTC_FRAMES_MAX]; - static int _numFifo = 0; // numFrames saved in fifo - const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels); const int numChunk = (int)streamConfig.num_frames(); @@ -1164,26 +1157,26 @@ void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int while (numFrames > 0) { // number of frames to fill - int numFill = std::min(numFrames, numChunk - _numFifo); + int numFill = std::min(numFrames, numChunk - _numFifoFarEnd); // refill fifo - memcpy(&_fifo[_numFifo], samples, numFill * numChannels * sizeof(int16_t)); + memcpy(&_fifoFarEnd[_numFifoFarEnd], samples, numFill * numChannels * sizeof(int16_t)); samples += numFill * numChannels; numFrames -= numFill; - _numFifo += numFill; + _numFifoFarEnd += numFill; - if (_numFifo == numChunk) { + if (_numFifoFarEnd == numChunk) { // convert audio format float buffer[WEBRTC_CHANNELS_MAX][WEBRTC_FRAMES_MAX]; float* const buffers[WEBRTC_CHANNELS_MAX] = { buffer[0], buffer[1] }; - deinterleaveToFloat(_fifo, buffers, numChunk, numChannels); + deinterleaveToFloat(_fifoFarEnd, buffers, numChunk, numChannels); // process one chunk if (_apm->kNoError != _apm->ProcessReverseStream(buffers, streamConfig, streamConfig, buffers)) { qCWarning(audioclient) << "WebRTC ProcessReverseStream() returned an ERROR."; } - _numFifo = 0; + _numFifoFarEnd = 0; } } } diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 638c1b61e9..e07a730d11 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -423,7 +423,14 @@ private: void handleLocalEchoAndReverb(QByteArray& inputByteArray); #if defined(WEBRTC_ENABLED) + static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * webrtc::AudioProcessing::kMaxNativeSampleRateHz / 1000; + static const int WEBRTC_CHANNELS_MAX = 2; + webrtc::AudioProcessing* _apm { nullptr }; + + int16_t _fifoFarEnd[WEBRTC_CHANNELS_MAX * WEBRTC_FRAMES_MAX] {}; + int _numFifoFarEnd = 0; // numFrames saved in fifo + void configureWebrtc(); void processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate); void processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate); From bd0849fd4fe54e0aba3407372348eaa9acd632e0 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Fri, 26 Jul 2019 12:18:30 -0700 Subject: [PATCH 35/41] Allow AEC with device sample rate up to 96KHz --- libraries/audio-client/src/AudioClient.cpp | 9 ++------- libraries/audio-client/src/AudioClient.h | 3 ++- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index f4a01869e2..f9451d196e 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1145,12 +1145,10 @@ void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels); const int numChunk = (int)streamConfig.num_frames(); - if (sampleRate > webrtc::AudioProcessing::kMaxNativeSampleRateHz) { - qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "output sample rate."; + if (sampleRate > WEBRTC_SAMPLE_RATE_MAX) { return; } if (numChannels > WEBRTC_CHANNELS_MAX) { - qCWarning(audioclient) << "WebRTC does not support" << numChannels << "output channels."; return; } @@ -1186,16 +1184,13 @@ void AudioClient::processWebrtcNearEnd(int16_t* samples, int numFrames, int numC const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels); const int numChunk = (int)streamConfig.num_frames(); - if (sampleRate > webrtc::AudioProcessing::kMaxNativeSampleRateHz) { - qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "input sample rate."; + if (sampleRate > WEBRTC_SAMPLE_RATE_MAX) { return; } if (numChannels > WEBRTC_CHANNELS_MAX) { - qCWarning(audioclient) << "WebRTC does not support" << numChannels << "input channels."; return; } if (numFrames != numChunk) { - qCWarning(audioclient) << "WebRTC requires exactly 10ms of input."; return; } diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index e07a730d11..ab12393ebf 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -423,8 +423,9 @@ private: void handleLocalEchoAndReverb(QByteArray& inputByteArray); #if defined(WEBRTC_ENABLED) - static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * webrtc::AudioProcessing::kMaxNativeSampleRateHz / 1000; + static const int WEBRTC_SAMPLE_RATE_MAX = 96000; static const int WEBRTC_CHANNELS_MAX = 2; + static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * WEBRTC_SAMPLE_RATE_MAX / 1000; webrtc::AudioProcessing* _apm { nullptr }; From e6e8491098a89cda9d2f9d08fffe341a1445231c Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Fri, 26 Jul 2019 12:32:38 -0700 Subject: [PATCH 36/41] Improved WebRTC error reporting --- libraries/audio-client/src/AudioClient.cpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index f9451d196e..2d21958f92 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1146,9 +1146,11 @@ void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int const int numChunk = (int)streamConfig.num_frames(); if (sampleRate > WEBRTC_SAMPLE_RATE_MAX) { + qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "output sample rate."; return; } if (numChannels > WEBRTC_CHANNELS_MAX) { + qCWarning(audioclient) << "WebRTC does not support" << numChannels << "output channels."; return; } @@ -1171,8 +1173,9 @@ void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int deinterleaveToFloat(_fifoFarEnd, buffers, numChunk, numChannels); // process one chunk - if (_apm->kNoError != _apm->ProcessReverseStream(buffers, streamConfig, streamConfig, buffers)) { - qCWarning(audioclient) << "WebRTC ProcessReverseStream() returned an ERROR."; + int error = _apm->ProcessReverseStream(buffers, streamConfig, streamConfig, buffers); + if (error != _apm->kNoError) { + qCWarning(audioclient) << "WebRTC ProcessReverseStream() returned ERROR:" << error; } _numFifoFarEnd = 0; } @@ -1185,12 +1188,15 @@ void AudioClient::processWebrtcNearEnd(int16_t* samples, int numFrames, int numC const int numChunk = (int)streamConfig.num_frames(); if (sampleRate > WEBRTC_SAMPLE_RATE_MAX) { + qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "input sample rate."; return; } if (numChannels > WEBRTC_CHANNELS_MAX) { + qCWarning(audioclient) << "WebRTC does not support" << numChannels << "input channels."; return; } if (numFrames != numChunk) { + qCWarning(audioclient) << "WebRTC requires exactly 10ms of input."; return; } @@ -1200,11 +1206,13 @@ void AudioClient::processWebrtcNearEnd(int16_t* samples, int numFrames, int numC deinterleaveToFloat(samples, buffers, numFrames, numChannels); // process one chunk - if (_apm->kNoError != _apm->ProcessStream(buffers, streamConfig, streamConfig, buffers)) { - qCWarning(audioclient) << "WebRTC ProcessStream() returned an ERROR."; + int error = _apm->ProcessStream(buffers, streamConfig, streamConfig, buffers); + if (error =! _apm->kNoError) { + qCWarning(audioclient) << "WebRTC ProcessStream() returned ERROR:" << error; + } else { + // modify samples in-place + interleaveToInt16(buffers, samples, numFrames, numChannels); } - // modify samples in-place - interleaveToInt16(buffers, samples, numFrames, numChannels); } #endif // WEBRTC_ENABLED From 8c6cee4e4dbe094b75dbf89754787a4f098434e8 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Fri, 26 Jul 2019 12:45:21 -0700 Subject: [PATCH 37/41] Bypass WebRTC processing when not enabled. Note: WebRTC processing with all features disabled still holds locks and performs resampling in the callback. --- libraries/audio-client/src/AudioClient.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 2d21958f92..c16e297c28 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1135,8 +1135,6 @@ void AudioClient::configureWebrtc() { config.level_estimation.enabled = false; _apm->ApplyConfig(config); - - qCDebug(audioclient) << "WebRTC enabled for acoustic echo cancellation."; } // rebuffer into 10ms chunks @@ -1207,7 +1205,7 @@ void AudioClient::processWebrtcNearEnd(int16_t* samples, int numFrames, int numC // process one chunk int error = _apm->ProcessStream(buffers, streamConfig, streamConfig, buffers); - if (error =! _apm->kNoError) { + if (error != _apm->kNoError) { qCWarning(audioclient) << "WebRTC ProcessStream() returned ERROR:" << error; } else { // modify samples in-place @@ -1396,8 +1394,10 @@ void AudioClient::handleMicAudioInput() { _inputRingBuffer.readSamples(inputAudioSamples.get(), inputSamplesRequired); #if defined(WEBRTC_ENABLED) - processWebrtcNearEnd(inputAudioSamples.get(), inputSamplesRequired / _inputFormat.channelCount(), - _inputFormat.channelCount(), _inputFormat.sampleRate()); + if (_isAECEnabled) { + processWebrtcNearEnd(inputAudioSamples.get(), inputSamplesRequired / _inputFormat.channelCount(), + _inputFormat.channelCount(), _inputFormat.sampleRate()); + } #endif // detect loudness and clipping on the raw input @@ -2326,7 +2326,9 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { _audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped); #if defined(WEBRTC_ENABLED) - _audio->processWebrtcFarEnd(scratchBuffer, framesPopped, OUTPUT_CHANNEL_COUNT, _audio->_outputFormat.sampleRate()); + if (_audio->_isAECEnabled) { + _audio->processWebrtcFarEnd(scratchBuffer, framesPopped, OUTPUT_CHANNEL_COUNT, _audio->_outputFormat.sampleRate()); + } #endif // if required, upmix or downmix to deviceChannelCount From e5ad87925e9135ee85abba14bb47b72789dabaa2 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Sat, 27 Jul 2019 09:26:03 -0700 Subject: [PATCH 38/41] update webrtc library for android --- hifi_android.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hifi_android.py b/hifi_android.py index 926a671a6c..0c2ea07cc7 100644 --- a/hifi_android.py +++ b/hifi_android.py @@ -97,8 +97,7 @@ ANDROID_PACKAGES = { }, 'webrtc': { 'file': 'webrtc-20190626-android.tar.gz', - 'checksum': '8be046cf27e1fabff8a1029eabbe52c8', - 'versionId': '8TU1LgZswAgZpBiNYvB5s3ZLwSqNJHuj', + 'checksum': 'e2dccd3d8efdcba6d428c87ba7fb2a53' } } From d8b940cc9624872deffeaa04b11ba51d89baf97d Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Mon, 29 Jul 2019 10:35:44 -0700 Subject: [PATCH 39/41] add control to enable/disable acoustic echo cancellation to simplified UI --- .../qml/hifi/simplifiedUI/settingsApp/audio/Audio.qml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/interface/resources/qml/hifi/simplifiedUI/settingsApp/audio/Audio.qml b/interface/resources/qml/hifi/simplifiedUI/settingsApp/audio/Audio.qml index 8827bb3834..bfc0bc5200 100644 --- a/interface/resources/qml/hifi/simplifiedUI/settingsApp/audio/Audio.qml +++ b/interface/resources/qml/hifi/simplifiedUI/settingsApp/audio/Audio.qml @@ -222,6 +222,17 @@ Flickable { } } } + + SimplifiedControls.Switch { + id: acousticEchoCancellationSwitch + Layout.preferredHeight: 18 + Layout.preferredWidth: parent.width + labelTextOn: "Acoustic Echo Cancellation" + checked: AudioScriptingInterface.acousticEchoCancellation + onClicked: { + AudioScriptingInterface.acousticEchoCancellation = !AudioScriptingInterface.acousticEchoCancellation; + } + } } } From 27b574cf14cb9b2a53026b61dde3c87596358c41 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Tue, 30 Jul 2019 14:09:04 -0700 Subject: [PATCH 40/41] disable webrtc for android --- cmake/macros/TargetWebRTC.cmake | 17 +++++++++-------- cmake/ports/hifi-deps/CONTROL | 2 +- libraries/audio-client/CMakeLists.txt | 5 ++++- libraries/shared/src/shared/WebRTC.h | 5 +++-- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/cmake/macros/TargetWebRTC.cmake b/cmake/macros/TargetWebRTC.cmake index fd5db5fcbe..d2821528df 100644 --- a/cmake/macros/TargetWebRTC.cmake +++ b/cmake/macros/TargetWebRTC.cmake @@ -6,18 +6,19 @@ # macro(TARGET_WEBRTC) if (ANDROID) - include(SelectLibraryConfigurations) - set(INSTALL_DIR ${HIFI_ANDROID_PRECOMPILED}/webrtc/webrtc) - set(WEBRTC_INCLUDE_DIRS "${INSTALL_DIR}/include/webrtc") - set(WEBRTC_LIBRARY_DEBUG ${INSTALL_DIR}/debug/lib/libwebrtc.a) - set(WEBRTC_LIBRARY_RELEASE ${INSTALL_DIR}/lib/libwebrtc.a) - select_library_configurations(WEBRTC) + # I don't yet have working libwebrtc for android + # include(SelectLibraryConfigurations) + # set(INSTALL_DIR ${HIFI_ANDROID_PRECOMPILED}/webrtc/webrtc) + # set(WEBRTC_INCLUDE_DIRS "${INSTALL_DIR}/include/webrtc") + # set(WEBRTC_LIBRARY_DEBUG ${INSTALL_DIR}/debug/lib/libwebrtc.a) + # set(WEBRTC_LIBRARY_RELEASE ${INSTALL_DIR}/lib/libwebrtc.a) + # select_library_configurations(WEBRTC) else() set(WEBRTC_INCLUDE_DIRS "${VCPKG_INSTALL_ROOT}/include/webrtc") find_library(WEBRTC_LIBRARY NAMES webrtc PATHS ${VCPKG_INSTALL_ROOT}/lib/ NO_DEFAULT_PATH) + target_include_directories(${TARGET_NAME} SYSTEM PUBLIC ${WEBRTC_INCLUDE_DIRS}) + target_link_libraries(${TARGET_NAME} ${WEBRTC_LIBRARY}) endif() - target_include_directories(${TARGET_NAME} SYSTEM PUBLIC ${WEBRTC_INCLUDE_DIRS}) - target_link_libraries(${TARGET_NAME} ${WEBRTC_LIBRARY}) endmacro() diff --git a/cmake/ports/hifi-deps/CONTROL b/cmake/ports/hifi-deps/CONTROL index 0a3d194ea1..2441de9002 100644 --- a/cmake/ports/hifi-deps/CONTROL +++ b/cmake/ports/hifi-deps/CONTROL @@ -1,4 +1,4 @@ Source: hifi-deps Version: 0.1 Description: Collected dependencies for High Fidelity applications -Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib, webrtc +Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib, webrtc (!android) diff --git a/libraries/audio-client/CMakeLists.txt b/libraries/audio-client/CMakeLists.txt index 48d3d74cd6..6b88292dd4 100644 --- a/libraries/audio-client/CMakeLists.txt +++ b/libraries/audio-client/CMakeLists.txt @@ -7,7 +7,10 @@ link_hifi_libraries(audio plugins) include_hifi_library_headers(shared) include_hifi_library_headers(networking) -target_webrtc() +if (ANDROID) +else () + target_webrtc() +endif () # append audio includes to our list of includes to bubble target_include_directories(${TARGET_NAME} PUBLIC "${HIFI_LIBRARY_DIR}/audio/src") diff --git a/libraries/shared/src/shared/WebRTC.h b/libraries/shared/src/shared/WebRTC.h index 6f05d38a3d..2f0e444bff 100644 --- a/libraries/shared/src/shared/WebRTC.h +++ b/libraries/shared/src/shared/WebRTC.h @@ -20,8 +20,9 @@ # define NOMINMAX 1 # define WIN32_LEAN_AND_MEAN 1 #elif defined(Q_OS_ANDROID) -# define WEBRTC_ENABLED 1 -# define WEBRTC_POSIX 1 +// I don't yet have a working libwebrtc for android +// # define WEBRTC_ENABLED 1 +// # define WEBRTC_POSIX 1 #elif defined(Q_OS_LINUX) # define WEBRTC_ENABLED 1 # define WEBRTC_POSIX 1 From f07352ab2c4d7e0bd60fbda9aba71f7b58bb3221 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Wed, 31 Jul 2019 13:39:52 -0700 Subject: [PATCH 41/41] add MaterialCache to oven's DependencyManager --- tools/oven/src/Oven.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/oven/src/Oven.cpp b/tools/oven/src/Oven.cpp index a680dd1f89..d7e0cec67b 100644 --- a/tools/oven/src/Oven.cpp +++ b/tools/oven/src/Oven.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -42,6 +43,7 @@ Oven::Oven() { DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); + DependencyManager::set(); MaterialBaker::setNextOvenWorkerThreadOperator([] { return Oven::instance().getNextWorkerThread();