From c18cac642b23e0daeac65bb9b7b493a7749b16d7 Mon Sep 17 00:00:00 2001
From: Aaron Fenyes <aaron.fenyes@fareycircles.ooo>
Date: Sun, 25 Aug 2024 00:47:36 -0700
Subject: [PATCH] Ray-caster: generalize depth sorting

Switch from a hard-coded sorting network for four fragments to an
insertion sort, which should work for any number of fragments.
---
 .../inversive-display/src/inversive.frag      | 54 +++++++++++--------
 1 file changed, 31 insertions(+), 23 deletions(-)

diff --git a/app-proto/inversive-display/src/inversive.frag b/app-proto/inversive-display/src/inversive.frag
index 12b4129..93bdde4 100644
--- a/app-proto/inversive-display/src/inversive.frag
+++ b/app-proto/inversive-display/src/inversive.frag
@@ -137,29 +137,37 @@ void main() {
     // cast rays through the spheres
     vec2 depth_pairs [sphere_cnt];
     taggedFrag frags [2*sphere_cnt];
-    for (int i = 0; i < sphere_cnt; i++) {
+    int frag_cnt = 0;
+    for (int i = 0; i < sphere_cnt; ++i) {
         vec2 hit_depths = sphere_cast(sphere_list[i], dir);
-        frags[2*i] = sphere_shading(sphere_list[i], hit_depths[0] * dir, color_list[i], i);
-        frags[2*i+1] = sphere_shading(sphere_list[i], hit_depths[1] * dir, color_list[i], i);
+        if (!isnan(hit_depths[0])) {
+            frags[frag_cnt] = sphere_shading(sphere_list[i], hit_depths[0] * dir, color_list[i], i);
+            ++frag_cnt;
+        }
+        if (!isnan(hit_depths[1])) {
+            frags[frag_cnt] = sphere_shading(sphere_list[i], hit_depths[1] * dir, color_list[i], i);
+            ++frag_cnt;
+        }
     }
     
-    // shade and depth-sort the impact points
-    taggedFrag front_hits[2] = sort(frags[0], frags[2]);
-    taggedFrag back_hits[2] = sort(frags[1], frags[3]);
-    taggedFrag middle_frags[2] = sort(front_hits[1], back_hits[0]);
-    
-    // finish depth sorting
-    taggedFrag frags_by_depth[4];
-    frags_by_depth[0] = front_hits[0];
-    frags_by_depth[1] = middle_frags[0];
-    frags_by_depth[2] = middle_frags[1];
-    frags_by_depth[3] = back_hits[1];
+    // sort the fragments by depth, using an insertion sort
+    for (int take = 1; take < frag_cnt; ++take) {
+        taggedFrag pulled = frags[take];
+        for (int put = take; put >= 0; --put) {
+            if (put < 1 || frags[put-1].pt.z >= pulled.pt.z) {
+                frags[put] = pulled;
+                break;
+            } else {
+                frags[put] = frags[put-1];
+            }
+        }
+    }
     
     // highlight intersections and cusps
-    for (int i = 3; i >= 1; --i) {
+    for (int i = frag_cnt-1; i >= 1; --i) {
         // intersections
-        taggedFrag frag0 = frags_by_depth[i];
-        taggedFrag frag1 = frags_by_depth[i-1];
+        taggedFrag frag0 = frags[i];
+        taggedFrag frag1 = frags[i-1];
         float ixn_sin = length(cross(frag0.normal, frag1.normal));
         vec3 disp = frag0.pt - frag1.pt;
         float ixn_dist = max(
@@ -167,21 +175,21 @@ void main() {
             abs(dot(frag0.normal, disp))
         ) / ixn_sin;
         float ixn_highlight = 0.5 * highlight * (1. - smoothstep(2./3.*ixn_threshold, 1.5*ixn_threshold, ixn_dist));
-        frags_by_depth[i].color = mix(frags_by_depth[i].color, vec4(1.), ixn_highlight);
-        frags_by_depth[i-1].color = mix(frags_by_depth[i-1].color, vec4(1.), ixn_highlight);
+        frags[i].color = mix(frags[i].color, vec4(1.), ixn_highlight);
+        frags[i-1].color = mix(frags[i-1].color, vec4(1.), ixn_highlight);
         
         // cusps
         float cusp_cos = abs(dot(dir, frag0.normal));
         float cusp_threshold = 2.*sqrt(ixn_threshold * sphere_list[frag0.id].lt.s);
         float cusp_highlight = highlight * (1. - smoothstep(2./3.*cusp_threshold, 1.5*cusp_threshold, cusp_cos));
-        frags_by_depth[i].color = mix(frags_by_depth[i].color, vec4(1.), cusp_highlight);
+        frags[i].color = mix(frags[i].color, vec4(1.), cusp_highlight);
     }
     
     // composite the sphere fragments
     vec3 color = vec3(0.);
-    for (int i = 3; i >= layer_threshold; --i) {
-        if (frags_by_depth[i].pt.z < 0.) {
-            vec4 frag_color = frags_by_depth[i].color;
+    for (int i = frag_cnt-1; i >= layer_threshold; --i) {
+        if (frags[i].pt.z < 0.) {
+            vec4 frag_color = frags[i].color;
             color = mix(color, frag_color.rgb, frag_color.a);
         }
     }