From ef7c002237059adb9c13908b8f7d7b5f0f2505b8 Mon Sep 17 00:00:00 2001
From: Sam Gateau <sam@highfidelity.io>
Date: Fri, 2 Jan 2015 10:31:34 -0800
Subject: [PATCH] Improving the shadow effect with lightmaps

---
 interface/src/Application.cpp                 |  2 +-
 .../src/directional_light_shadow_map.slf      | 30 +++++++++++++++++--
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp
index 4465899262..2c510c4980 100644
--- a/interface/src/Application.cpp
+++ b/interface/src/Application.cpp
@@ -2907,7 +2907,7 @@ void Application::updateShadowMap() {
 
         {
             PerformanceTimer perfTimer("entities");
-            _entities.render(RenderArgs::SHADOW_RENDER_MODE);
+         //   _entities.render(RenderArgs::SHADOW_RENDER_MODE);
         }
 
         // render JS/scriptable overlays
diff --git a/libraries/render-utils/src/directional_light_shadow_map.slf b/libraries/render-utils/src/directional_light_shadow_map.slf
index 2f6660234f..738fa8ee1f 100644
--- a/libraries/render-utils/src/directional_light_shadow_map.slf
+++ b/libraries/render-utils/src/directional_light_shadow_map.slf
@@ -51,15 +51,39 @@ void main(void) {
     // compute the view space position using the depth
     float z = near / (depthVal * depthScale - 1.0);
     vec4 position = vec4((depthTexCoordOffset + gl_TexCoord[0].st * depthTexCoordScale) * z, z, 1.0);
-    
+
+    // compute the corresponding texture coordinates
+    vec3 shadowTexCoord = vec3(dot(gl_EyePlaneS[0], position), dot(gl_EyePlaneT[0], position), dot(gl_EyePlaneR[0], position));
+
     // get the normal from the map
     vec4 normal = normalVal;
     if ((normalVal.a >= 0.45) && (normalVal.a <= 0.55)) {
         normal.a = 1.0;
         normalVal.a = 0.0;
-        gl_FragColor = vec4(diffuseVal.rgb * specularVal.rgb, 1.0);
+
+        // compute the corresponding texture coordinates
+        vec3 shadowTexCoord = vec3(dot(gl_EyePlaneS[0], position), dot(gl_EyePlaneT[0], position), dot(gl_EyePlaneR[0], position));
+        
+        // get the normal from the map
+        vec3 normalizedNormal = normalize(normal.xyz * 2.0 - vec3(1.0));
+
+        // how much this fragment faces the light direction
+        float faceLight = step(-0.005, dot(normalizedNormal, gl_LightSource[0].position.xyz));
+        
+        // evaluate the shadow test but only relevant for light facing fragments
+        float attenuation = (1 - faceLight) + faceLight * (0.25 *
+            (shadow2D(shadowMap, shadowTexCoord + vec3(-shadowScale, -shadowScale, 0.0)).r +
+            shadow2D(shadowMap, shadowTexCoord + vec3(-shadowScale, shadowScale, 0.0)).r +
+            shadow2D(shadowMap, shadowTexCoord + vec3(shadowScale, -shadowScale, 0.0)).r +
+            shadow2D(shadowMap, shadowTexCoord + vec3(shadowScale, shadowScale, 0.0)).r));
+            
+
+        gl_FragColor = vec4(diffuseVal.rgb * (
+            (gl_FrontLightModelProduct.sceneColor.rgb + gl_FrontLightProduct[0].ambient.rgb) * (1.0 - attenuation) +
+            specularVal.rgb * attenuation), 1.0);
+
+    //    gl_FragColor = vec4(diffuseVal.rgb * specularVal.rgb, 1.0);
     } else {
-     
         // compute the corresponding texture coordinates
         vec3 shadowTexCoord = vec3(dot(gl_EyePlaneS[0], position), dot(gl_EyePlaneT[0], position), dot(gl_EyePlaneR[0], position));