From bd28498833d72686b92e31c8dda5dacbd3cc9192 Mon Sep 17 00:00:00 2001
From: samcake <samuel.gateau@gmail.com>
Date: Tue, 21 Jun 2016 18:25:02 -0700
Subject: [PATCH] trying to fix the curvature artefact

---
 .../render-utils/src/DeferredTransform.slh    |  7 ++++
 .../src/surfaceGeometry_makeCurvature.slf     | 37 +++++++++++++------
 .../utilities/render/framebuffer.qml          |  3 +-
 3 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/libraries/render-utils/src/DeferredTransform.slh b/libraries/render-utils/src/DeferredTransform.slh
index 20c1cfdd52..520ac77fde 100644
--- a/libraries/render-utils/src/DeferredTransform.slh
+++ b/libraries/render-utils/src/DeferredTransform.slh
@@ -50,6 +50,13 @@ mat4 getProjection(int side) {
     return frameTransform._projection[side];
 }
 
+// positive near distance of the projection
+float getProjectionNear() {
+    float planeC = frameTransform._projection[0][2][3] + frameTransform._projection[0][2][2];
+    float planeD = frameTransform._projection[0][3][2];
+    return planeD / planeC;
+}
+
 mat4 getViewInverse() {
     return frameTransform._viewInverse;
 }
diff --git a/libraries/render-utils/src/surfaceGeometry_makeCurvature.slf b/libraries/render-utils/src/surfaceGeometry_makeCurvature.slf
index dc222dab4c..5316ef4211 100644
--- a/libraries/render-utils/src/surfaceGeometry_makeCurvature.slf
+++ b/libraries/render-utils/src/surfaceGeometry_makeCurvature.slf
@@ -118,6 +118,8 @@ void main(void) {
     // Fetch the z under the pixel (stereo or not)
     float Zeye = getZEye(framePixelPos);
 
+    float nearPlaneScale = min(-Zeye / getCurvatureBasisScale(), 1.0);
+
     vec3 worldNormal = getWorldNormal(frameTexcoordPos);
 
     // The position of the pixel fragment in Eye space then in world space
@@ -125,26 +127,40 @@ void main(void) {
     vec3 worldPos = (frameTransform._viewInverse * vec4(eyePos, 1.0)).xyz;
 
     // Calculate the perspective scale.
-    float perspectiveScale = (-getProjScaleEye() / Zeye);
-
-    vec2 viewportScale = perspectiveScale * getInvWidthHeight();
+    // Clamp to 0.5
+    float perspectiveScale = max(0.5, (-getProjScaleEye() / Zeye));
+   // float perspectiveScale = max(0.5, (-getProjectionNear() / Zeye));
 
     // Calculate dF/du and dF/dv
-    float threshold = getCurvatureDepthThreshold();
-    vec2 du = vec2( 1.0f, 0.0f ) * viewportScale.x;
-    vec2 dv = vec2( 0.0f, 1.0f ) * viewportScale.y;
+    vec2 viewportScale = perspectiveScale * getInvWidthHeight();
+    vec2 du = vec2( viewportScale.x, 0.0f );
+    vec2 dv = vec2( 0.0f, viewportScale.y );
 
     vec4 dFdu = vec4(getWorldNormalDiff(frameTexcoordPos, du), getEyeDepthDiff(frameTexcoordPos, du));
     vec4 dFdv = vec4(getWorldNormalDiff(frameTexcoordPos, dv), getEyeDepthDiff(frameTexcoordPos, dv));
+
+    float threshold = getCurvatureDepthThreshold();
     dFdu *= step(abs(dFdu.w), threshold);
     dFdv *= step(abs(dFdv.w), threshold);
 
-   // outFragColor = vec4(dFdu.xyz, 1.0);
-
+    //outFragColor = vec4(du.x, du.y, 0.0, 1.0);
+    // outFragColor = vec4(viewportScale, 0.0, 1.0);
+   /* if (perspectiveScale < getCurvatureBasisScale()) {
+        //outFragColor = vec4(0.0, 0.0, 4 * perspectiveScale, 1.0);
+    } else if (perspectiveScale < 0.5) {
+        outFragColor = vec4(0.0, 0.0, 2 * perspectiveScale, 1.0);
+    return;
+    } else if (perspectiveScale > 1.0) {
+        outFragColor = vec4(perspectiveScale, 0.0, 0.0, 1.0);s
+    return;
+    } else {
+        outFragColor = vec4(0.0, 0.5 * perspectiveScale, 0.0, 1.0);
+    return;
+    }*/
      // Calculate ( du/dx, du/dy, du/dz ) and ( dv/dx, dv/dy, dv/dz )
 
     // Eval px, py, pz world positions of the basis centered on the world pos of the fragment
-    float dist = getCurvatureBasisScale();
+    float dist = getCurvatureBasisScale() * nearPlaneScale;
     vec4 px = vec4(worldPos, 1.0) + vec4(dist, 0.0f, 0.0f, 0.0f);
     vec4 py = vec4(worldPos, 1.0) + vec4(0.0f, dist, 0.0f, 0.0f);
     vec4 pz = vec4(worldPos, 1.0) + vec4(0.0f, 0.0f, dist, 0.0f);
@@ -183,8 +199,7 @@ void main(void) {
     return;
 */
 
-    float pixPerspectiveScaleInv = 1.0 / (perspectiveScale);
-    //vec2 pixPerspectiveScaleInv = 1.0 / viewportScale;
+    float pixPerspectiveScaleInv = 1.0 / (perspectiveScale * nearPlaneScale);
     px.xy = (px.xy - nclipPos) * pixPerspectiveScaleInv;
     py.xy = (py.xy - nclipPos) * pixPerspectiveScaleInv;
     pz.xy = (pz.xy - nclipPos) * pixPerspectiveScaleInv;
diff --git a/scripts/developer/utilities/render/framebuffer.qml b/scripts/developer/utilities/render/framebuffer.qml
index d74ca03076..48f1550409 100644
--- a/scripts/developer/utilities/render/framebuffer.qml
+++ b/scripts/developer/utilities/render/framebuffer.qml
@@ -49,6 +49,7 @@ Column {
                 "Unlit",
                 "Occlusion",
                 "Lightmap",
+                "Scattering",
                 "Lighting",
                 "Shadow",
                 "Pyramid Depth",
@@ -56,7 +57,7 @@ Column {
                 "NormalCurvature",
                 "DiffusedCurvature",
                 "DiffusedNormalCurvature",
-                "Scattering",
+                "Debug Scattering",
                 "Ambient Occlusion",
                 "Ambient Occlusion Blurred",
                 "Custom Shader"