refining the shadowing

libraries/render-utils/src/directional_light_shadow_map.slf

@@ -55,48 +55,46 @@ void main(void) {
     // compute the corresponding texture coordinates
     vec3 shadowTexCoord = vec3(dot(gl_EyePlaneS[0], position), dot(gl_EyePlaneT[0], position), dot(gl_EyePlaneR[0], position));
 
+    // evaluate the shadow test but only relevant for light facing fragments
+    float shadowAttenuation = (0.25 *
+        (shadow2D(shadowMap, shadowTexCoord + vec3(-shadowScale, -shadowScale, 0.0)).r +
+        shadow2D(shadowMap, shadowTexCoord + vec3(-shadowScale, shadowScale, 0.0)).r +
+        shadow2D(shadowMap, shadowTexCoord + vec3(shadowScale, -shadowScale, 0.0)).r +
+        shadow2D(shadowMap, shadowTexCoord + vec3(shadowScale, shadowScale, 0.0)).r));
+
     // get the normal from the map
     vec4 normal = normalVal;
     if ((normalVal.a >= 0.45) && (normalVal.a <= 0.55)) {
         normal.a = 1.0;
         normalVal.a = 0.0;
 
-        // compute the corresponding texture coordinates
-        vec3 shadowTexCoord = vec3(dot(gl_EyePlaneS[0], position), dot(gl_EyePlaneT[0], position), dot(gl_EyePlaneR[0], position));
-        
         // get the normal from the map
         vec3 normalizedNormal = normalize(normal.xyz * 2.0 - vec3(1.0));
 
         // how much this fragment faces the light direction
-        float faceLight = step(-0.005, dot(normalizedNormal, gl_LightSource[0].position.xyz));
+        float diffuse = dot(normalizedNormal, gl_LightSource[0].position.xyz);
+
+        // need to catch normals perpendicular to the projection plane hence the magic number for the threshold
+        // it should be just 0, be we have innacurracy so we need to overshoot
+        const float PERPENDICULAR_THREASHOLD = -0.005;
+        float facingLight = step(PERPENDICULAR_THREASHOLD, diffuse); 
         
         // evaluate the shadow test but only relevant for light facing fragments
-        float attenuation = (1 - faceLight) + faceLight * (0.25 *
+        float lightAttenuation = (1 - facingLight) + facingLight * shadowAttenuation;
-            (shadow2D(shadowMap, shadowTexCoord + vec3(-shadowScale, -shadowScale, 0.0)).r +
-            shadow2D(shadowMap, shadowTexCoord + vec3(-shadowScale, shadowScale, 0.0)).r +
-            shadow2D(shadowMap, shadowTexCoord + vec3(shadowScale, -shadowScale, 0.0)).r +
-            shadow2D(shadowMap, shadowTexCoord + vec3(shadowScale, shadowScale, 0.0)).r));
             
+        // diffuse light is the lightmap dimmed by shadow
+        vec3 diffuseLight = lightAttenuation * specularVal.rgb;
+        // ambient is a tiny percentage of the lightmap and only when in the shadow
+        vec3 ambientLight = (1 - lightAttenuation) * 0.5 * specularVal.rgb; 
 
-        gl_FragColor = vec4(diffuseVal.rgb * (
+        gl_FragColor = vec4(diffuseVal.rgb * (ambientLight +  diffuseLight), 1.0);
-            (gl_FrontLightModelProduct.sceneColor.rgb + gl_FrontLightProduct[0].ambient.rgb) * (1.0 - attenuation) +
+    } else {    
-            specularVal.rgb * attenuation), 1.0);
-
-    //    gl_FragColor = vec4(diffuseVal.rgb * specularVal.rgb, 1.0);
-    } else {
-        // compute the corresponding texture coordinates
-        vec3 shadowTexCoord = vec3(dot(gl_EyePlaneS[0], position), dot(gl_EyePlaneT[0], position), dot(gl_EyePlaneR[0], position));
-        
         // get the normal from the map
         vec3 normalizedNormal = normalize(normal.xyz * 2.0 - vec3(1.0));
 
         // average values from the shadow map
         float diffuse = dot(normalizedNormal, gl_LightSource[0].position.xyz);
-        float facingLight = step(0.0, diffuse) * 0.25 *
+        float facingLight = step(0.0, diffuse) * shadowAttenuation;
-            (shadow2D(shadowMap, shadowTexCoord + vec3(-shadowScale, -shadowScale, 0.0)).r +
-            shadow2D(shadowMap, shadowTexCoord + vec3(-shadowScale, shadowScale, 0.0)).r +
-            shadow2D(shadowMap, shadowTexCoord + vec3(shadowScale, -shadowScale, 0.0)).r +
-            shadow2D(shadowMap, shadowTexCoord + vec3(shadowScale, shadowScale, 0.0)).r);
             
         // compute the base color based on OpenGL lighting model
         vec3 baseColor = diffuseVal.rgb * (gl_FrontLightModelProduct.sceneColor.rgb +