Build step 2: splatgen — real depth preprocessing on recorded frames

GLM Agent committed on 2026-10-25 at 07:37

The synthetic source becomes a software Kinect: it emits raw
millimetre depth maps plus RGB with deterministic sensor noise
(fliers, jitter), and splatgen does the real pipeline work on them:
background culling, median-based flier removal, an edge-preserving
bilateral filter, normals from the filtered point cloud (oriented
towards the camera), and footprint-sized surfel assembly — one splat
per surviving depth pixel, coloured from RGB.

- rig: DepthFrame and DepthSource (the raw seam), Camera::backproject
- replay: NoiseParams with a pure (frame, pixel) hash; recording
  format (recording.toml + raw depth/rgb files) with a writer and a
  looping RecordingSource
- splatgen: SplatGen (params + pipeline) and SplatSource, the
  DepthSource → FrameSource adapter; every consumer now runs the real
  depth path
- sink: splatgen and noise settings join the config surface; main
  composes source → SplatSource → viewer
- tests: 60 across the workspace — flier/edge/normal unit tests with
  analytic ground truth (one real normal-orientation bug found and
  fixed by them), recording round-trips, and noisy frames rendering
  like clean ones

Browse files at this commit

Cargo.lock +10 -0

diff --git a/Cargo.lock b/Cargo.lock
index 9f8e86e..a087280 100644
@@ -6066,6 +6066,7 @@ dependencies = [
 "pollster",
 "replay",
 "rig",
 "splatgen",
 "tokio",
]
@@ -6082,6 +6083,8 @@ dependencies = [
 "anyhow",
 "glam",
 "rig",
 "serde",
 "toml",
]
[[package]]
@@ -6526,6 +6529,7 @@ dependencies = [
 "replay",
 "rig",
 "serde",
 "splatgen",
 "toml",
]
@@ -6663,6 +6667,12 @@ dependencies = [
[[package]]
name = "splatgen"
version = "0.1.0"
dependencies = [
 "anyhow",
 "glam",
 "replay",
 "rig",
]
[[package]]
name = "sqlite-wasm-rs"

PLAN.md +1 -1

diff --git a/PLAN.md b/PLAN.md
index ff1b945..702c65a 100644
@@ -113,7 +113,7 @@ loopback output) join the same arg surface.
## Build order
- [x] 1. replay + render + UI with synthetic data
- [ ] 2. splatgen on recorded Kinect frames — `replay` defines the
- [x] 2. splatgen on recorded Kinect frames — `replay` defines the
      recording format (synthetic recordings for CI); the synthetic
      source is refactored to emit raw depth frames like a Kinect, so
      the dev/CI path runs through the real depth preprocessing

README.md +14 -7

diff --git a/README.md b/README.md
index ad8ac0d..8c2a484 100644
@@ -25,13 +25,20 @@ Targets Ubuntu 24.04 (x86-64) and Windows on ARM64.
## Status
Build-order step 1 is complete: the full pipeline runs on synthetic data.
A synthetic head-and-shoulders stand-in is raycast per depth pixel and
turned into footprint-sized surfel splats — the same construction the
real depth path will use — then rendered live by Brush's WebGPU
renderer from a virtual camera steered between the rig's side cameras,
and shown in the egui viewer. The whole chain, GPU rendering included,
runs headlessly in tests (software Vulkan).
Build-order steps 1–2 are complete. The full pipeline runs on synthetic
data through the *real* depth path: a software Kinect emits raw
millimetre depth maps plus RGB (with deterministic sensor noise), and
`splatgen` culls the background, removes fliers, bilateral-filters the
depth, estimates normals, and assembles footprint-sized surfel splats —
which Brush renders live from a virtual camera steered between the
rig's side cameras, shown in the egui viewer. The whole chain, GPU
rendering included, runs headlessly in tests (software Vulkan), noise
included.
Recorded datasets have an on-disk format (`replay::recording`):
a directory with `recording.toml` (camera, fps, frame count) plus raw
`frame_NNNN.depth`/`frame_NNNN.rgb` files. Real Kinect recordings can
be converted into that shape; synthetic ones are generated for dev/CI.
- `cargo run -p sink` — the viewer (needs a GPU, or lavapipe/Vulkan)
- `cargo test --workspace` — unit, GPU-integration, and headless UI tests

render/Cargo.toml +2 -1

diff --git a/render/Cargo.toml b/render/Cargo.toml
index bcde597..138da11 100644
@@ -14,6 +14,7 @@ glam.workspace = true
pollster.workspace = true
rig.workspace = true
[dev-dependencies]
#[dev-dependencies]
replay.workspace = true
splatgen.workspace = true
tokio = { workspace = true }

render/tests/render.rs +41 -3

diff --git a/render/tests/render.rs b/render/tests/render.rs
index f038298..d559b7f 100644
@@ -12,8 +12,9 @@ use std::sync::LazyLock;
use glam::{Vec2, uvec2, vec3};
use render::{RenderStats, SplatRenderer};
use replay::synth::{self, SUBJECT};
use replay::synth::{self, NoiseParams, SUBJECT};
use rig::{Camera, FrameSource, SplatBatch};
use splatgen::{SplatGen, SplatSource};
/// Small enough to stay fast on software Vulkan, big enough that the
/// subject spans several pixels.
@@ -34,12 +35,17 @@ fn head_center() -> glam::Vec3 {
/// Synthetic skin colour, RGB in [0, 1].
const SKIN: [f32; 3] = [0.85, 0.62, 0.52];
fn first_frame() -> SplatBatch {
fn first_frame_with(noise: NoiseParams) -> SplatBatch {
    let rig = synth::dev_rig_with_resolutions(uvec2(CENTER.0, CENTER.1), uvec2(SIDE.0, SIDE.1));
    let mut source = synth::dev_source_with_center_camera(rig.center, 30.0);
    let raw = synth::dev_source_with(rig.center, SUBJECT, 30.0).with_noise(noise);
    let mut source = SplatSource::new(raw, SplatGen::default());
    source.next_frame().expect("synthetic frame").splats
}
fn first_frame() -> SplatBatch {
    first_frame_with(NoiseParams::OFF)
}
/// The mean of pixels within `tolerance` of a colour, in image
/// coordinates; `None` if too few match.
fn color_centroid(image: &render::RenderedImage, color: [f32; 3], tolerance: f32) -> Option<Vec2> {
@@ -180,6 +186,38 @@ async fn steering_shifts_the_backdrop_but_not_the_subject() {
}
#[tokio::test]
async fn noisy_frames_render_like_clean_ones() {
    // The realistic path: sensor noise on, and the splatgen
    // preprocessing (flier removal, bilateral filter) has to clean it
    // up well enough that the same content assertions hold.
    let rig = synth::dev_rig_with_resolutions(uvec2(CENTER.0, CENTER.1), uvec2(SIDE.0, SIDE.1));
    let batch = first_frame_with(NoiseParams::default());
    assert!(!batch.is_empty());
    let camera = rig.virtual_camera(0.5, rig.left.intrinsics, rig.left.resolution);
    let (image, stats) = renderer()
        .render_async(&batch, &camera)
        .await
        .expect("render");
    assert!(stats.visible_splats > 0);
    // The subject still lands where the camera math says, within a
    // slightly looser tolerance than the clean-frame test.
    let projected = camera.project(head_center()).expect("in front of camera");
    let rendered = color_centroid(&image, SKIN, 0.2).expect("skin visible");
    assert!(
        (projected - rendered).length() < 4.0,
        "head projected to {projected:?} but rendered at {rendered:?}"
    );
    // And the backdrop corners are still grey and opaque.
    let corner = image.pixel(2, 2);
    let [r, g, b, a] = corner;
    assert!(a >= 250, "backdrop opaque, got {corner:?}");
    assert!((r as i32 - g as i32).abs() <= 1 && (g as i32 - b as i32).abs() <= 1);
}
#[tokio::test]
async fn empty_batch_renders_the_background() {
    let rig = synth::dev_rig_with_resolutions(uvec2(CENTER.0, CENTER.1), uvec2(SIDE.0, SIDE.1));
    let camera: Camera = rig.virtual_camera(0.5, rig.left.intrinsics, rig.left.resolution);

replay/Cargo.toml +2 -0

diff --git a/replay/Cargo.toml b/replay/Cargo.toml
index c3a815c..e5881b8 100644
@@ -9,3 +9,5 @@ publish = false
anyhow.workspace = true
glam.workspace = true
rig.workspace = true
serde.workspace = true
toml.workspace = true

replay/src/lib.rs +4 -2

diff --git a/replay/src/lib.rs b/replay/src/lib.rs
index aa17ea1..be88d5d 100644
@@ -15,8 +15,10 @@
//!   hardware is involved.
//! * recorded Kinect + webcam streams land here with build-order step 2.
pub mod recording;
pub mod scene;
pub mod synth;
pub use scene::Scene;
pub use synth::SyntheticSource;
pub use recording::{Recording, RecordingSource};
pub use scene::{Scene, Surface};
pub use synth::{NoiseParams, SyntheticSource};

replay/src/recording.rs +300 -0

diff --git a/replay/src/recording.rs b/replay/src/recording.rs
new file mode 100644
index 0000000..b1f6e17
@@ -0,0 +1,300 @@
//! Recorded depth datasets: the on-disk format, writer, and loader.
//!
//! A recording is a directory:
//!
//! ```text
//! my-recording/
//! ├── recording.toml     # the depth camera (id, pose, intrinsics,
//! │                     #   resolution), fps, frame count
//! ├── frame_0000.depth   # raw little-endian u16 depth, width*height
//! ├── frame_0000.rgb    # raw RGB bytes, width*height*3
//! ├── frame_0001.depth
//! └── ...
//! ```
//!
//! Real Kinect recordings can be converted into this shape; for dev and
//! CI, [`Recording::write`] produces recordings from the synthetic
//! source. [`RecordingSource`] plays them back as a [`DepthSource`],
//! looping forever.
use std::fs;
use std::path::{Path, PathBuf};
use anyhow::{Context, Result, bail};
use serde::{Deserialize, Serialize};
use rig::{Camera, CameraIntrinsics, CameraPose, DepthFrame, DepthSource};
/// `recording.toml`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Manifest {
    pub camera: CameraManifest,
    pub fps: f32,
    pub frames: u32,
}
/// The depth camera a recording was made with.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CameraManifest {
    pub id: String,
    /// Camera position, metres.
    pub position: [f32; 3],
    /// Camera-to-world rotation, `x,y,z,w` quaternion.
    pub rotation: [f32; 4],
    /// `(width, height)` in pixels.
    pub resolution: [u32; 2],
    /// `fx, fy, cx, cy` in pixels.
    pub intrinsics: [f32; 4],
}
impl From<&Camera> for CameraManifest {
    fn from(camera: &Camera) -> Self {
        Self {
            id: camera.id.clone(),
            position: camera.pose.position.to_array(),
            rotation: camera.pose.rotation.to_array(),
            resolution: [camera.resolution.x, camera.resolution.y],
            intrinsics: [
                camera.intrinsics.fx,
                camera.intrinsics.fy,
                camera.intrinsics.cx,
                camera.intrinsics.cy,
            ],
        }
    }
}
impl From<CameraManifest> for Camera {
    fn from(manifest: CameraManifest) -> Self {
        Self {
            id: manifest.id,
            pose: CameraPose {
                position: manifest.position.into(),
                rotation: glam::Quat::from_array(manifest.rotation),
            },
            intrinsics: CameraIntrinsics {
                fx: manifest.intrinsics[0],
                fy: manifest.intrinsics[1],
                cx: manifest.intrinsics[2],
                cy: manifest.intrinsics[3],
            },
            resolution: glam::UVec2::new(manifest.resolution[0], manifest.resolution[1]),
        }
    }
}
fn depth_file(frame: u32) -> String {
    format!("frame_{frame:04}.depth")
}
fn rgb_file(frame: u32) -> String {
    format!("frame_{frame:04}.rgb")
}
/// Writes recordings: [`Recording::write`].
pub struct Recording;
impl Recording {
    /// Record `frames` frames from `source` into `dir`, replacing it.
    ///
    /// The source must yield exactly the frame stream worth keeping;
    /// its first `frames` frames become the recording.
    pub fn write(
        dir: impl AsRef<Path>,
        source: &mut dyn DepthSource,
        fps: f32,
        frames: u32,
    ) -> Result<()> {
        let dir = dir.as_ref();
        fs::create_dir_all(dir).with_context(|| format!("creating {}", dir.display()))?;
        let manifest = Manifest {
            camera: CameraManifest::from(source.camera()),
            fps,
            frames,
        };
        fs::write(
            dir.join("recording.toml"),
            toml::to_string_pretty(&manifest)?,
        )
        .context("writing recording.toml")?;
        for frame in 0..frames {
            let depth_frame = source
                .next_depth_frame()
                .with_context(|| format!("recording frame {frame}"))?;
            let depth_le: Vec<u8> = depth_frame
                .depth_mm
                .iter()
                .flat_map(|d| d.to_le_bytes())
                .collect();
            fs::write(dir.join(depth_file(frame)), depth_le)?;
            fs::write(dir.join(rgb_file(frame)), &depth_frame.rgb)?;
        }
        Ok(())
    }
}
/// Plays a recording back as a [`DepthSource`], looping forever.
#[derive(Debug)]
pub struct RecordingSource {
    dir: PathBuf,
    camera: Camera,
    fps: f32,
    frames: u32,
    frame_index: u32,
}
impl RecordingSource {
    /// Load the recording in `dir`.
    pub fn load(dir: impl AsRef<Path>) -> Result<Self> {
        let dir = dir.as_ref().to_path_buf();
        let manifest_path = dir.join("recording.toml");
        let manifest: Manifest = toml::from_str(
            &fs::read_to_string(&manifest_path)
                .with_context(|| format!("reading {}", manifest_path.display()))?,
        )
        .with_context(|| format!("parsing {}", manifest_path.display()))?;
        if manifest.frames == 0 {
            bail!("recording in {} has no frames", dir.display());
        }
        if manifest.fps <= 0.0 {
            bail!("recording in {} has non-positive fps", dir.display());
        }
        Ok(Self {
            camera: manifest.camera.into(),
            fps: manifest.fps,
            frames: manifest.frames,
            dir,
            frame_index: 0,
        })
    }
    /// The recording's frame count.
    pub fn frames(&self) -> u32 {
        self.frames
    }
    /// The recording's playback rate.
    pub fn fps(&self) -> f32 {
        self.fps
    }
}
impl DepthSource for RecordingSource {
    fn camera(&self) -> &Camera {
        &self.camera
    }
    fn next_depth_frame(&mut self) -> Result<DepthFrame> {
        let frame = self.frame_index % self.frames;
        self.frame_index += 1;
        let (width, height) = (self.camera.resolution.x, self.camera.resolution.y);
        let depth_path = self.dir.join(depth_file(frame));
        let rgb_path = self.dir.join(rgb_file(frame));
        let depth_bytes =
            fs::read(&depth_path).with_context(|| format!("reading {}", depth_path.display()))?;
        let rgb_bytes =
            fs::read(&rgb_path).with_context(|| format!("reading {}", rgb_path.display()))?;
        let expected_depth = (width * height) as usize * 2;
        if depth_bytes.len() != expected_depth {
            bail!(
                "{}: expected {expected_depth} depth bytes, got {}",
                depth_path.display(),
                depth_bytes.len()
            );
        }
        let expected_rgb = (width * height) as usize * 3;
        if rgb_bytes.len() != expected_rgb {
            bail!(
                "{}: expected {expected_rgb} rgb bytes, got {}",
                rgb_path.display(),
                rgb_bytes.len()
            );
        }
        let depth_mm: Vec<u16> = depth_bytes
            .as_chunks::<2>()
            .0
            .iter()
            .map(|b| u16::from_le_bytes(*b))
            .collect();
        Ok(DepthFrame {
            timestamp_secs: frame as f64 / self.fps as f64,
            depth_mm,
            rgb: rgb_bytes,
        })
    }
}
#[cfg(test)]
mod tests {
    use super::*;
    use crate::synth::{self, NoiseParams};
    use glam::uvec2;
    /// A tiny clean source for exact comparisons.
    fn tiny_clean_source() -> synth::SyntheticSource {
        let rig = synth::dev_rig_with_resolutions(uvec2(32, 24), uvec2(64, 48));
        synth::dev_source_with(rig.center, synth::SUBJECT, 30.0).with_noise(NoiseParams::OFF)
    }
    fn temp_dir(name: &str) -> PathBuf {
        let dir =
            std::env::temp_dir().join(format!("kinect-view-rec-{name}-{}", std::process::id()));
        let _ = fs::remove_dir_all(&dir);
        dir
    }
    #[test]
    fn round_trip_preserves_frames() {
        let dir = temp_dir("roundtrip");
        let mut source = tiny_clean_source();
        Recording::write(&dir, &mut source, 30.0, 3).expect("write recording");
        let mut player = RecordingSource::load(&dir).expect("load recording");
        assert_eq!(player.frames(), 3);
        assert_eq!(player.fps(), 30.0);
        assert_eq!(player.camera(), source.camera());
        // The recorded frames match what the source produced, in order.
        let mut source = tiny_clean_source();
        for _ in 0..3 {
            let recorded = player.next_depth_frame().expect("frame");
            let original = source.next_depth_frame().expect("frame");
            assert_eq!(recorded.depth_mm, original.depth_mm);
            assert_eq!(recorded.rgb, original.rgb);
            assert_eq!(recorded.timestamp_secs, original.timestamp_secs);
        }
        // And playback loops.
        let wrapped = player.next_depth_frame().expect("frame");
        assert_eq!(wrapped.timestamp_secs, 0.0);
        fs::remove_dir_all(&dir).ok();
    }
    #[test]
    fn missing_files_are_reported() {
        let err = RecordingSource::load("/nonexistent/recording").expect_err("missing dir");
        assert!(err.to_string().contains("reading"));
        // A manifest with frames but no frame files.
        let dir = temp_dir("missing-frames");
        fs::create_dir_all(&dir).expect("dir");
        fs::write(
            dir.join("recording.toml"),
            r#"
[[fake]]"#,
        )
        .expect("write");
        let err = RecordingSource::load(&dir).expect_err("bad manifest");
        assert!(err.to_string().contains("parsing"));
        fs::remove_dir_all(&dir).ok();
    }
}

replay/src/synth.rs +237 -105

diff --git a/replay/src/synth.rs b/replay/src/synth.rs
index d77c5eb..96bc3b5 100644
@@ -1,19 +1,24 @@
//! The synthetic frame source: a software depth camera.
//! The synthetic depth-camera: a software Kinect.
//!
//! Every frame, [`SyntheticSource`] raycasts the scene from the rig's
//! center (Kinect) viewpoint — one ray per depth pixel — and turns each
//! hit into a footprint-sized surfel via [`rig::Splat::surfel`], the same
//! construction the real `splatgen` uses on Kinect depth. The subject
//! sways a little so temporal behaviour is testable, deterministically
//! (a function of the frame index, never the wall clock).
//! center viewpoint — one ray per depth pixel — and emits a raw
//! [`DepthFrame`]: a millimetre depth map plus RGB, like a Kinect v2
//! reports them. `splatgen` then turns those frames into surfels, so the
//! dev/CI path exercises the real depth-processing pipeline.
//!
//! Realism knobs: the source injects deterministic sensor noise
//! ([`NoiseParams`]) — isolated fliers and per-pixel jitter — so the
//! flier removal and bilateral filter have real work to do. The noise is
//! a pure function of `(frame, pixel)`, never the wall clock: playback
//! stays deterministic.
//!
//! The subject sways a little, also deterministically, so temporal
//! behaviour is testable.
use anyhow::Result;
use glam::{UVec2, Vec3};
use rig::{
    Camera, CameraIntrinsics, CameraPose, Frame, FrameSource, Rig, SURFEL_THICKNESS_M, Splat,
    SplatBatch,
};
use rig::{Camera, CameraIntrinsics, CameraPose, DepthFrame, DepthSource};
use crate::scene::{self, Scene, Surface};
@@ -21,13 +26,64 @@ use crate::scene::{self, Scene, Surface};
/// camera at the origin.
pub const SUBJECT: Vec3 = Vec3::new(0.0, 0.0, 2.0);
/// Opacity used for synthetic surfels (PLAN.md: ~0.9–0.99).
/// Opacity used for synthetic surfels (PLAN.md: ~0.9–0.99). Applied by
/// `splatgen`, kept here as the dataset's declared intent.
pub const SURFEL_OPACITY: f32 = 0.95;
/// Subject sway amplitude, metres, and period in frames.
const SWAY_AMPLITUDE_M: f32 = 0.04;
const SWAY_PERIOD_FRAMES: u64 = 90;
/// The depth range a Kinect v2 reports; synthetic noise stays in it.
const KINECT_MIN_MM: u16 = 500;
const KINECT_MAX_MM: u16 = 4000;
/// Sensor noise injected into the synthetic depth maps.
///
/// Deterministic: a pure function of the frame index and pixel, so
/// playback is reproducible.
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct NoiseParams {
    /// Fraction of valid pixels replaced with a random depth (fliers),
    /// in `[0, 1]`.
    pub flier_rate: f32,
    /// Per-pixel depth jitter, ± millimetres.
    pub jitter_mm: u32,
}
impl Default for NoiseParams {
    fn default() -> Self {
        // Mild but realistic: scattered fliers, a few mm of jitter.
        Self {
            flier_rate: 0.005,
            jitter_mm: 4,
        }
    }
}
impl NoiseParams {
    /// Clean frames: no fliers, no jitter.
    pub const OFF: NoiseParams = NoiseParams {
        flier_rate: 0.0,
        jitter_mm: 0,
    };
}
/// Deterministic uniform noise in `[0, 1)` for a pixel: a hash of
/// `(frame, x, y, salt)` — no RNG state, so playback is reproducible.
fn noise01(frame: u64, x: u32, y: u32, salt: u32) -> f32 {
    let mut h = (frame as u32)
        ^ x.wrapping_mul(0x9E37_79B1)
        ^ y.wrapping_mul(0x85EB_CA77)
        ^ salt.wrapping_mul(0xC2B2_AE3D);
    h ^= h >> 16;
    h = h.wrapping_mul(0x7FEB_352D);
    h ^= h >> 15;
    h = h.wrapping_mul(0x846C_A68B);
    h ^= h >> 16;
    h as f32 / u32::MAX as f32
}
/// The development rig: a Kinect-like center camera flanked by two
/// webcams 0.8 m apart, converged on [`SUBJECT`].
///
@@ -74,9 +130,11 @@ const CENTER_FY: f32 = 0.866_025_4; // 1 / (2 * tan(30°)), FOV_y = 60°
/// Side-camera focal ratio: a 60° horizontal FOV, square pixels.
const SIDE_FX: f32 = 0.866_025_4; // 1 / (2 * tan(30°))
use rig::Rig;
/// The development rig at full size: Kinect-like depth at 256x212
/// (~54k active splats, inside the plan's 50–150k budget) and 640x480
/// webcams.
/// (~54k depth pixels, inside the plan's 50–150k splat budget) and
/// 640x480 webcams.
pub fn dev_rig() -> Rig {
    dev_rig_with_resolutions(UVec2::new(256, 212), UVec2::new(640, 480))
}
@@ -113,22 +171,30 @@ pub struct SyntheticSource {
    /// frame.
    subject: Vec<Surface>,
    /// The depth viewpoint: the rig's center camera.
    center: Camera,
    camera: Camera,
    fps: f32,
    noise: NoiseParams,
    frame_index: u64,
}
impl SyntheticSource {
    pub fn new(scene: Scene, subject: Vec<Surface>, center: Camera, fps: f32) -> Self {
    pub fn new(scene: Scene, subject: Vec<Surface>, camera: Camera, fps: f32) -> Self {
        Self {
            scene,
            subject,
            center,
            camera,
            fps: fps.max(1.0),
            noise: NoiseParams::default(),
            frame_index: 0,
        }
    }
    /// Override the sensor noise; [`NoiseParams::OFF`] for clean frames.
    pub fn with_noise(mut self, noise: NoiseParams) -> Self {
        self.noise = noise;
        self
    }
    /// The number of frames produced so far.
    pub fn frame_index(&self) -> u64 {
        self.frame_index
@@ -136,7 +202,7 @@ impl SyntheticSource {
    /// The depth viewpoint: the rig's center camera.
    pub fn center(&self) -> &Camera {
        &self.center
        &self.camera
    }
    /// The deterministic subject sway offset for a given frame.
@@ -157,46 +223,80 @@ impl SyntheticSource {
        scene
    }
    /// Build the splat batch the software depth camera sees in `frame`.
    fn build_frame(&self, frame: u64) -> SplatBatch {
    /// Raycast one frame: the clean depth map and RGB the software
    /// Kinect sees.
    fn cast_frame(&self, frame: u64) -> (Vec<u16>, Vec<u8>) {
        let scene = self.frame_scene(frame);
        let (width, height) = (self.center.resolution.x, self.center.resolution.y);
        let world_from_camera = self.center.pose.world_from_camera();
        let mut batch = SplatBatch::with_capacity((width * height) as usize);
        let (width, height) = (self.camera.resolution.x, self.camera.resolution.y);
        let camera_from_world = self.camera.pose.camera_from_world();
        let mut depth = vec![0_u16; (width * height) as usize];
        let mut rgb = vec![0_u8; (width * height * 3) as usize];
        for y in 0..height {
            for x in 0..width {
                // Sample through pixel centres.
                let (origin, dir) = self.center.pixel_ray(x as f32 + 0.5, y as f32 + 0.5);
                let (origin, dir) = self.camera.pixel_ray(x as f32 + 0.5, y as f32 + 0.5);
                let Some(hit) = scene.cast_ray(origin, dir) else {
                    continue;
                    continue; // no return: depth stays 0
                };
                // Depth along the viewing axis (like a Kinect depth map,
                // which is what PLAN.md's footprint formula expects).
                let depth = world_from_camera.inverse().transform_point3(hit.position).z;
                batch.push(Splat::surfel(
                    hit.position,
                    hit.normal,
                    depth,
                    &self.center.intrinsics,
                    SURFEL_THICKNESS_M,
                    hit.color,
                    SURFEL_OPACITY,
                ));
                // Planar depth along the viewing axis, in millimetres —
                // what a Kinect depth map reports.
                let z = camera_from_world.transform_point3(hit.position).z;
                let idx = (y * width + x) as usize;
                depth[idx] = (z * 1000.0).round().clamp(1.0, u16::MAX as f32) as u16;
                rgb[idx * 3] = (hit.color[0] * 255.0) as u8;
                rgb[idx * 3 + 1] = (hit.color[1] * 255.0) as u8;
                rgb[idx * 3 + 2] = (hit.color[2] * 255.0) as u8;
            }
        }
        (depth, rgb)
    }
    /// Apply the deterministic sensor noise to a clean frame.
    fn add_noise(&self, depth: &mut [u16], frame: u64) {
        if self.noise == NoiseParams::OFF {
            return;
        }
        let width = self.camera.resolution.x;
        for (i, d) in depth.iter_mut().enumerate() {
            if *d == 0 {
                continue; // invalid pixels stay invalid
            }
            let x = (i as u32) % width;
            let y = (i as u32) / width;
            if self.noise.jitter_mm > 0 {
                let jitter = noise01(frame, x, y, 1) * 2.0 - 1.0;
                let delta = (jitter * self.noise.jitter_mm as f32).round() as i32;
                *d = (*d as i32 + delta).clamp(1, u16::MAX as i32) as u16;
            }
            if self.noise.flier_rate > 0.0 && noise01(frame, x, y, 2) < self.noise.flier_rate {
                let span = (KINECT_MAX_MM - KINECT_MIN_MM) as f32;
                *d = KINECT_MIN_MM + (noise01(frame, x, y, 3) * span) as u16;
            }
        }
    }
    /// Build the raw depth frame for `frame`.
    fn build_frame(&self, frame: u64) -> DepthFrame {
        let (mut depth, rgb) = self.cast_frame(frame);
        self.add_noise(&mut depth, frame);
        DepthFrame {
            timestamp_secs: frame as f64 / self.fps as f64,
            depth_mm: depth,
            rgb,
        }
        batch
    }
}
impl FrameSource for SyntheticSource {
    fn next_frame(&mut self) -> Result<Frame> {
impl DepthSource for SyntheticSource {
    fn camera(&self) -> &Camera {
        &self.camera
    }
    fn next_depth_frame(&mut self) -> Result<DepthFrame> {
        let frame = self.frame_index;
        self.frame_index += 1;
        Ok(Frame {
            timestamp_secs: frame as f64 / self.fps as f64,
            splats: self.build_frame(frame),
        })
        Ok(self.build_frame(frame))
    }
}
@@ -204,6 +304,8 @@ impl FrameSource for SyntheticSource {
mod tests {
    use super::*;
    use crate::scene;
    use glam::uvec2;
    use rig::DepthSource;
    /// A small-resolution source so tests stay fast: 32x24 depth pixels.
    fn tiny_source() -> SyntheticSource {
@@ -215,7 +317,7 @@ mod tests {
        center.intrinsics.fy *= scale;
        center.intrinsics.cx = 16.0;
        center.intrinsics.cy = 12.0;
        center.resolution = UVec2::new(32, 24);
        center.resolution = uvec2(32, 24);
        SyntheticSource::new(
            Scene::new().with(scene::backdrop(SUBJECT)),
            vec![head, shoulders],
@@ -224,56 +326,48 @@ mod tests {
        )
    }
    #[test]
    fn frames_are_nonempty_and_in_front() {
        let mut src = tiny_source();
        let frame = src.next_frame().unwrap();
        assert!(!frame.splats.is_empty());
        // One splat per depth pixel that hit — never more.
        assert!(frame.splats.len() <= 32 * 24);
        // Most rays hit the backdrop.
        assert!(frame.splats.len() > 32 * 24 / 2);
        // Every splat lies in front of the center camera and matches the
        // footprint formula.
        let cam = src.center().clone();
        let camera_from_world = cam.pose.camera_from_world();
        for splat in frame.splats.iter() {
            let p = camera_from_world.transform_point3(splat.mean);
            assert!(p.z > 0.5, "splat behind camera: {p:?}");
            assert!((splat.scale.x - p.z / cam.intrinsics.fx).abs() < 1e-5);
            assert!((splat.scale.y - p.z / cam.intrinsics.fy).abs() < 1e-5);
            assert!((splat.opacity - SURFEL_OPACITY).abs() < 1e-6);
        }
    }
    /// The straight-ahead ray passes below the head and hits the front
    /// of the shoulders: ~1.85 m out.
    const SHOULDER_FRONT_MM: f32 = 1852.0;
    #[test]
    fn subject_and_backdrop_both_present() {
    fn frames_are_depth_maps_like_a_kinect() {
        let mut src = tiny_source();
        let frame = src.next_frame().unwrap();
        let mut skin = 0;
        let mut checker = 0;
        for splat in frame.splats.iter() {
            if (splat.color[0] - 0.85).abs() < 0.01 {
                skin += 1;
            }
            // Checker greys: 0.72 or 0.28.
            if (splat.color[0] - splat.color[1]).abs() < 1e-4
                && ((splat.color[0] - 0.72).abs() < 0.01 || (splat.color[0] - 0.28).abs() < 0.01)
            {
                checker += 1;
            }
        }
        assert!(skin > 0, "no subject splats");
        assert!(checker > 0, "no backdrop splats");
        let frame = src.next_depth_frame().expect("frame");
        assert_eq!(frame.depth_mm.len(), 32 * 24);
        assert_eq!(frame.rgb.len(), 32 * 24 * 3);
        // Every ray hits something (the backdrop is infinite): all
        // depths valid.
        assert!(frame.depth_mm.iter().all(|&d| d > 0));
        // The pixel looking straight at the subject sees the front of
        // the shoulders (± noise), in the shoulders' colour.
        let mid = 12 * 32 + 16;
        assert!(
            (frame.depth_mm[mid] as f32 - SHOULDER_FRONT_MM).abs() < 40.0,
            "center depth {} mm",
            frame.depth_mm[mid]
        );
        let shoulders = &frame.rgb[mid * 3..mid * 3 + 3];
        assert!(
            (shoulders[0] as i32 - 51).abs() < 6
                && (shoulders[1] as i32 - 89).abs() < 6
                && (shoulders[2] as i32 - 140).abs() < 6,
            "shoulders rgb {shoulders:?}"
        );
        // A top-corner pixel sees the backdrop behind the subject.
        let corner = frame.depth_mm[32 + 2] as f32;
        assert!((corner - 2800.0).abs() < 200.0, "corner depth {corner}");
    }
    #[test]
    fn timestamps_advance_with_index() {
        let mut src = tiny_source();
        assert_eq!(src.next_frame().unwrap().timestamp_secs, 0.0);
        assert!((src.next_frame().unwrap().timestamp_secs - 1.0 / 30.0).abs() < 1e-9);
        assert!((src.next_frame().unwrap().timestamp_secs - 2.0 / 30.0).abs() < 1e-9);
        assert_eq!(src.next_depth_frame().unwrap().timestamp_secs, 0.0);
        assert!((src.next_depth_frame().unwrap().timestamp_secs - 1.0 / 30.0).abs() < 1e-9);
        assert!((src.next_depth_frame().unwrap().timestamp_secs - 2.0 / 30.0).abs() < 1e-9);
    }
    #[test]
@@ -281,46 +375,52 @@ mod tests {
        let mut a = tiny_source();
        let mut b = tiny_source();
        for _ in 0..5 {
            assert_eq!(a.next_frame().unwrap(), b.next_frame().unwrap());
            assert_eq!(a.next_depth_frame().unwrap(), b.next_depth_frame().unwrap());
        }
    }
    #[test]
    fn subject_sways_and_repeats() {
        let mut src = tiny_source();
        let frame0 = src.next_frame().unwrap();
        // The frame produced at a quarter period (index 22) is at (nearly)
        // maximum sway.
        // The sway is periodic in the frame index; the sensor noise is
        // not, so exact repetition is asserted on clean frames.
        let mut src = tiny_source().with_noise(NoiseParams::OFF);
        let frame0 = src.next_depth_frame().unwrap();
        // After a quarter period the sway is near its maximum.
        let mut quarter = None;
        while src.frame_index() <= SWAY_PERIOD_FRAMES / 4 {
            quarter = Some(src.next_frame().unwrap());
            quarter = Some(src.next_depth_frame().unwrap());
        }
        let quarter = quarter.expect("quarter frame produced");
        assert_ne!(
            frame0.splats, quarter.splats,
            "no visible sway at a quarter period"
        );
        assert_ne!(frame0.depth_mm, quarter.depth_mm, "no visible sway");
        // The frame at index SWAY_PERIOD_FRAMES repeats frame 0 exactly.
        let mut period = None;
        while src.frame_index() <= SWAY_PERIOD_FRAMES {
            period = Some(src.next_frame().unwrap());
            period = Some(src.next_depth_frame().unwrap());
        }
        let period = period.expect("period frame produced");
        assert_eq!(
            frame0.splats, period.splats,
            frame0.depth_mm, period.depth_mm,
            "sway must be periodic in the frame index"
        );
        assert_eq!(frame0.rgb, period.rgb);
        // With noise on, the sway still moves the same pixels but the
        // exact map differs — noise must not be frame-periodic.
        let mut noisy = tiny_source();
        let n0 = noisy.next_depth_frame().unwrap();
        let mut quarter = None;
        while noisy.frame_index() <= SWAY_PERIOD_FRAMES / 4 {
            quarter = Some(noisy.next_depth_frame().unwrap());
        }
        assert_ne!(n0.depth_mm, quarter.unwrap().depth_mm);
    }
    #[test]
    fn sway_offset_is_deterministic_and_bounded() {
        let src = tiny_source();
        let offset = src.sway_offset(0);
        assert_eq!(offset, Vec3::ZERO);
        assert_eq!(src.sway_offset(0), Vec3::ZERO);
        assert_eq!(src.sway_offset(SWAY_PERIOD_FRAMES), Vec3::ZERO);
        // A quarter period (90/4 = 22 frames) is close to the maximum.
        let max = src.sway_offset(SWAY_PERIOD_FRAMES / 4);
        assert!(max.x > 0.99 * SWAY_AMPLITUDE_M, "sway max was {max:?}");
        for frame in 0..SWAY_PERIOD_FRAMES * 2 {
@@ -330,11 +430,43 @@ mod tests {
    }
    #[test]
    fn noise_is_deterministic_and_can_be_disabled() {
        // Noise must not break determinism, and OFF must give clean maps.
        let mut noisy = tiny_source();
        let mut clean = tiny_source().with_noise(NoiseParams::OFF);
        let noisy_frame = noisy.next_depth_frame().unwrap();
        let clean_frame = clean.next_depth_frame().unwrap();
        assert_ne!(noisy_frame.depth_mm, clean_frame.depth_mm);
        // Clean frames repeat exactly across sources.
        let mut clean2 = tiny_source().with_noise(NoiseParams::OFF);
        assert_eq!(clean_frame, clean2.next_depth_frame().unwrap());
        // And noisy ones too.
        let mut noisy2 = tiny_source();
        assert_eq!(noisy_frame, noisy2.next_depth_frame().unwrap());
        // Jitter is bounded: every noisy depth within ±jitter of the
        // clean one (fliers excepted).
        let mut fliers = 0;
        for (n, c) in noisy_frame.depth_mm.iter().zip(&clean_frame.depth_mm) {
            if (*n as i32 - *c as i32).abs() > NoiseParams::default().jitter_mm as i32 + 1 {
                fliers += 1;
            }
        }
        assert!(
            fliers as f32 <= 3.0 * NoiseParams::default().flier_rate * 32.0 * 24.0,
            "too many changed pixels: {fliers}"
        );
    }
    #[test]
    fn dev_rig_is_symmetric() {
        let rig = dev_rig();
        assert_eq!(rig.left.resolution, UVec2::new(640, 480));
        assert_eq!(rig.right.resolution, UVec2::new(640, 480));
        assert_eq!(rig.center.resolution, UVec2::new(256, 212));
        assert_eq!(rig.left.resolution, uvec2(640, 480));
        assert_eq!(rig.right.resolution, uvec2(640, 480));
        assert_eq!(rig.center.resolution, uvec2(256, 212));
        // The center camera sits exactly between the side cameras.
        let mid = rig.virtual_pose(0.5);
        assert!(mid.position.distance(rig.center.pose.position) < 1e-6);

rig/src/camera.rs +25 -0

diff --git a/rig/src/camera.rs b/rig/src/camera.rs
index ea7bb29..749d9e2 100644
@@ -150,6 +150,17 @@ impl Camera {
        (self.pose.position, dir_world)
    }
    /// Backproject a pixel at planar depth `z` (metres along the viewing
    /// axis) to world space — the inverse of [`Camera::project`].
    pub fn backproject(&self, u: f32, v: f32, z: f32) -> Vec3 {
        let p_cam = Vec3::new(
            (u - self.intrinsics.cx) / self.intrinsics.fx * z,
            (v - self.intrinsics.cy) / self.intrinsics.fy * z,
            z,
        );
        self.pose.world_from_camera().transform_point3(p_cam)
    }
    /// Horizontal field of view in radians, matching Brush's convention
    /// (`focal = (width / 2) / tan(fov / 2)`).
    pub fn fov_x(&self) -> f32 {
@@ -271,6 +282,20 @@ mod tests {
    }
    #[test]
    fn backproject_inverts_projection() {
        let cam = cam_looking_at_subject();
        for point in [vec3(0.3, 0.4, 1.9), vec3(-0.2, 0.1, 2.7)] {
            let projected = cam.project(point).expect("in front");
            let z = cam.pose.camera_from_world().transform_point3(point).z;
            let back = cam.backproject(projected.x, projected.y, z);
            assert!(
                (back - point).length() < 1e-4,
                "{point:?} -> {projected:?} @ {z} -> {back:?}"
            );
        }
    }
    #[test]
    fn pose_blend_hits_endpoints() {
        let a = CameraPose::looking_at(vec3(-1.0, 0.0, 0.0), vec3(0.0, 0.0, 2.0));
        let b = CameraPose::looking_at(vec3(1.0, 0.0, 0.0), vec3(0.0, 0.0, 2.0));

rig/src/lib.rs +1 -1

diff --git a/rig/src/lib.rs b/rig/src/lib.rs
index 738176f..3dfb1c0 100644
@@ -22,5 +22,5 @@ pub mod splat;
pub use camera::{Camera, CameraIntrinsics, CameraPose};
pub use rig::Rig;
pub use source::{Frame, FrameSource};
pub use source::{DepthFrame, DepthSource, Frame, FrameSource};
pub use splat::{SURFEL_THICKNESS_M, Splat, SplatBatch};

rig/src/source.rs +36 -1

diff --git a/rig/src/source.rs b/rig/src/source.rs
index 94305dc..50b5515 100644
@@ -1,7 +1,18 @@
//! The frame-source abstraction shared by live capture and replay.
//! The frame-source abstractions shared by live capture, replay, and
//! synthetic generation.
//!
//! Two stages, two traits:
//!
//! * [`DepthSource`] emits raw depth frames (millimetre depth maps plus
//!   RGB, as a Kinect reports them) — capture, recordings, and the
//!   synthetic dataset all speak this;
//! * [`FrameSource`] emits processed frames carrying the assembled
//!   [`SplatBatch`] — what the renderer and sinks consume. `splatgen`
//!   bridges the two.
use anyhow::Result;
use crate::camera::Camera;
use crate::splat::SplatBatch;
/// One processed rig frame, ready for rendering.
@@ -25,3 +36,27 @@ pub struct Frame {
pub trait FrameSource {
    fn next_frame(&mut self) -> Result<Frame>;
}
/// One raw depth-camera frame, as the sensors report it.
#[derive(Debug, Clone, PartialEq)]
pub struct DepthFrame {
    /// Frame timestamp in seconds since the source started.
    pub timestamp_secs: f64,
    /// Depth in millimetres, row-major, `width * height` at the depth
    /// camera's resolution. Zero marks an invalid pixel (no return),
    /// like the Kinect reports.
    pub depth_mm: Vec<u16>,
    /// RGB, 3 bytes per pixel, row-major, same resolution.
    pub rgb: Vec<u8>,
}
/// A source of raw depth frames: the seam where live capture, replay of
/// recordings, and the synthetic dataset are interchangeable.
///
/// Implementations must be deterministic when replaying.
pub trait DepthSource {
    /// The depth camera the frames come from (intrinsics, pose,
    /// resolution).
    fn camera(&self) -> &Camera;
    fn next_depth_frame(&mut self) -> Result<DepthFrame>;
}

sink/Cargo.toml +1 -0

diff --git a/sink/Cargo.toml b/sink/Cargo.toml
index 62c925e..6d18c9d 100644
@@ -22,6 +22,7 @@ replay.workspace = true
render.workspace = true
rig.workspace = true
serde.workspace = true
splatgen.workspace = true
toml.workspace = true
[dev-dependencies]

sink/src/config.rs +92 -1

diff --git a/sink/src/config.rs b/sink/src/config.rs
index 3a9a6cd..abeb576 100644
@@ -30,7 +30,9 @@ use anyhow::{Context, Result, ensure};
use clap::{CommandFactory, Parser};
use clap_config::ClapConfig;
use glam::{Quat, UVec2, Vec3};
use replay::NoiseParams;
use serde::{Deserialize, Serialize};
use splatgen::SplatGenParams;
use rig::{Camera, CameraIntrinsics, CameraPose, Rig};
@@ -178,6 +180,56 @@ pub struct Args {
    )]
    pub rig_right_intrinsics: Intrinsics,
    // -- depth processing (splatgen) -----------------------------------
    /// Bilateral filter radius, pixels (PLAN.md: 3–5).
    #[arg(long, env = "KINECT_VIEW_SPLATGEN_FILTER_RADIUS", default_value = "3")]
    pub splatgen_filter_radius: u32,
    /// Bilateral depth-domain sigma, millimetres.
    #[arg(
        long,
        env = "KINECT_VIEW_SPLATGEN_FILTER_SIGMA_DEPTH_MM",
        default_value = "25.0"
    )]
    pub splatgen_filter_sigma_depth_mm: f32,
    /// Bilateral spatial-domain sigma, pixels.
    #[arg(
        long,
        env = "KINECT_VIEW_SPLATGEN_FILTER_SIGMA_SPACE_PX",
        default_value = "2.0"
    )]
    pub splatgen_filter_sigma_space_px: f32,
    /// Pixels differing from their 3x3 neighbourhood median by more
    /// than this are fliers and dropped, millimetres.
    #[arg(
        long,
        env = "KINECT_VIEW_SPLATGEN_FLIER_THRESHOLD_MM",
        default_value = "250"
    )]
    pub splatgen_flier_threshold_mm: u32,
    /// Background culling: pixels beyond this depth are dropped,
    /// millimetres.
    #[arg(
        long,
        env = "KINECT_VIEW_SPLATGEN_MAX_DEPTH_MM",
        default_value = "4000"
    )]
    pub splatgen_max_depth_mm: u16,
    /// Surfel opacity (PLAN.md: ~0.9–0.99).
    #[arg(long, env = "KINECT_VIEW_SPLATGEN_OPACITY", default_value = "0.95")]
    pub splatgen_opacity: f32,
    // -- synthetic source noise ---------------------------------------
    /// Fraction of valid depth pixels turned into fliers, `[0, 1]`.
    #[arg(
        long,
        env = "KINECT_VIEW_SOURCE_NOISE_FLIER_RATE",
        default_value = "0.005"
    )]
    pub source_noise_flier_rate: f32,
    /// Per-pixel depth jitter, ± millimetres.
    #[arg(long, env = "KINECT_VIEW_SOURCE_NOISE_JITTER_MM", default_value = "4")]
    pub source_noise_jitter_mm: u32,
    // -- scene / source / view ----------------------------------------
    /// Where the synthetic subject sits, metres (the scene is built
    /// around it; the default rig converges on it).
@@ -230,6 +282,31 @@ impl Args {
        let fps = self.source_fps;
        ensure!(fps > 0.0, "source-fps must be positive, got {fps}");
        let splatgen = SplatGenParams {
            filter_radius: self.splatgen_filter_radius,
            filter_sigma_depth_mm: self.splatgen_filter_sigma_depth_mm,
            filter_sigma_space_px: self.splatgen_filter_sigma_space_px,
            flier_threshold_mm: self.splatgen_flier_threshold_mm,
            max_depth_mm: self.splatgen_max_depth_mm,
            opacity: self.splatgen_opacity,
            thickness_m: rig::SURFEL_THICKNESS_M,
        };
        ensure!(
            (0.0..=1.0).contains(&splatgen.opacity),
            "splatgen-opacity must be in [0, 1], got {}",
            splatgen.opacity
        );
        let noise = NoiseParams {
            flier_rate: self.source_noise_flier_rate,
            jitter_mm: self.source_noise_jitter_mm,
        };
        ensure!(
            (0.0..=1.0).contains(&noise.flier_rate),
            "source-noise-flier-rate must be in [0, 1], got {}",
            noise.flier_rate
        );
        Ok(ResolvedConfig {
            rig: Rig {
                left,
@@ -239,6 +316,8 @@ impl Args {
            subject: self.scene_subject_position.0.into(),
            fps,
            background: self.background.0.into(),
            splatgen,
            noise,
        })
    }
}
@@ -253,6 +332,10 @@ pub struct ResolvedConfig {
    pub fps: f32,
    /// Render background colour.
    pub background: Vec3,
    /// Depth-processing parameters (`splatgen`).
    pub splatgen: SplatGenParams,
    /// Synthetic sensor noise.
    pub noise: NoiseParams,
}
/// Build one rig camera from its merged args. Rotation priority:
@@ -425,6 +508,8 @@ mod tests {
        assert_eq!(args.subject, vec3(0.0, 0.0, 2.0));
        assert_eq!(args.fps, 30.0);
        assert_eq!(args.background, vec3(0.10, 0.10, 0.12));
        assert_eq!(args.splatgen, SplatGenParams::default());
        assert_eq!(args.noise, NoiseParams::default());
    }
    #[test]
@@ -432,12 +517,18 @@ mod tests {
        let _guard = TEST_LOCK.lock().unwrap();
        let args = merged(
            &["app"],
            Some("rig_left_position = [1.0, 0.0, 0.0]\nsource_fps = 12.5\n"),
            Some(
                "rig_left_position = [1.0, 0.0, 0.0]\nsource_fps = 12.5\n\
                 splatgen_max_depth_mm = 2500\nsource_noise_jitter_mm = 0\n",
            ),
        )
        .resolve()
        .expect("resolves");
        assert_eq!(args.rig.left.pose.position, vec3(1.0, 0.0, 0.0));
        assert_eq!(args.fps, 12.5);
        // Depth-processing and noise settings flow through too.
        assert_eq!(args.splatgen.max_depth_mm, 2500);
        assert_eq!(args.noise.jitter_mm, 0);
        // Everything else keeps its default.
        assert_eq!(args.rig.right.pose.position, vec3(0.4, 0.0, 0.0));
    }

sink/src/main.rs +4 -1

diff --git a/sink/src/main.rs b/sink/src/main.rs
index f84d4c1..f5763f3 100644
@@ -19,7 +19,10 @@ fn main() -> eframe::Result<()> {
    };
    let source =
        replay::synth::dev_source_with(resolved.rig.center.clone(), resolved.subject, resolved.fps);
        replay::synth::dev_source_with(resolved.rig.center.clone(), resolved.subject, resolved.fps)
            .with_noise(resolved.noise);
    // The real pipeline: raw depth frames → preprocessing → surfels.
    let source = splatgen::SplatSource::new(source, splatgen::SplatGen::new(resolved.splatgen));
    let renderer = render::SplatRenderer::new(resolved.background);
    let model = sink::ViewerModel::new(resolved.rig, Box::new(source), renderer);
    let app = sink::KinectViewApp { model };

sink/tests/ui.rs +5 -2

diff --git a/sink/tests/ui.rs b/sink/tests/ui.rs
index 40e0621..ef979a3 100644
@@ -20,7 +20,9 @@ use sink::ViewerModel;
/// runs for real without slowing the suite down.
fn small_model() -> ViewerModel {
    let rig = synth::dev_rig_with_resolutions(uvec2(64, 53), uvec2(80, 60));
    let source = synth::dev_source_with_center_camera(rig.center.clone(), 30.0);
    // The full pipeline: raw synthetic depth → preprocessing → surfels.
    let raw = synth::dev_source_with(rig.center.clone(), synth::SUBJECT, 30.0);
    let source = splatgen::SplatSource::new(raw, splatgen::SplatGen::default());
    ViewerModel::new(rig, Box::new(source), render::SplatRenderer::default())
}
@@ -219,7 +221,8 @@ scene_subject_position = [0.0, 0.0, 2.5]
    assert_eq!(resolved.rig.center.resolution, uvec2(64, 53));
    // And the rig actually renders through the whole pipeline.
    let source = synth::dev_source_with(resolved.rig.center.clone(), resolved.subject, 30.0);
    let raw = synth::dev_source_with(resolved.rig.center.clone(), resolved.subject, 30.0);
    let source = splatgen::SplatSource::new(raw, splatgen::SplatGen::default());
    let model = ViewerModel::new(resolved.rig, Box::new(source), SplatRenderer::default());
    let harness = harness_for(model, false);
    let image = harness.state().last_image().expect("rendered");

splatgen/Cargo.toml +7 -1

diff --git a/splatgen/Cargo.toml b/splatgen/Cargo.toml
index 71ee08e..04c022d 100644
@@ -1,8 +1,14 @@
[package]
name = "splatgen"
description = "Per-frame splat construction: depth filter, normals, surfels, ellipsoid proxy fill"
description = "Depth preprocess (filter, fliers, culling), normals, batch assembly"
version.workspace = true
edition.workspace = true
publish = false
[dependencies]
anyhow.workspace = true
glam.workspace = true
rig.workspace = true
[dev-dependencies]
replay.workspace = true

splatgen/src/lib.rs +504 -13

diff --git a/splatgen/src/lib.rs b/splatgen/src/lib.rs
index ed18686..f35e0f3 100644
@@ -1,15 +1,506 @@
//! Per-frame Gaussian-splat field construction (surfel splatting).
//! Depth preprocessing and splat construction (PLAN.md, pipeline
//! steps 2–3).
//!
//! Pipeline role (PLAN.md, "Pipeline per frame" — depth preprocess and
//! splat construction). No 3DGS training: the Gaussian list handed to
//! `render` is constructed fresh every frame.
//! [`SplatGen`] turns one raw [`DepthFrame`] — a millimetre depth map
//! plus RGB, exactly what a Kinect (or the synthetic source, or a
//! recording) emits — into the frame's [`SplatBatch`]:
//!
//! * depth preprocess — bilateral filter (3–5 px), flier removal,
//!   background culling; target ~50–150k active splats
//! * footprint-sized anisotropic surfels from the Kinect depth:
//!   two tangential scales = depth × pixel_size / focal_length, a flat
//!   disk aligned to the depth-map normal, few-mm thickness along the
//!   normal, opacity ~0.9–0.99
//! * occluded-region fill from the head/shoulders ellipsoid proxy,
//!   recolored from the side cameras, with inflated thickness and a
//!   lower confidence weight
//! 1. **cull** — drop invalid pixels (depth 0) and everything beyond
//!    `max_depth_mm` (background culling),
//! 2. **flier removal** — drop pixels that disagree hard with their
//!    3×3 neighbourhood median (isolated wrong returns),
//! 3. **bilateral filter** — edge-preserving smoothing over a small
//!    pixel radius, so depth edges survive but sensor jitter does not,
//! 4. **normals** — from screen-space derivatives of the filtered
//!    point cloud, oriented towards the camera,
//! 5. **surfels** — one footprint-sized anisotropic splat per
//!    surviving pixel via `rig::Splat::surfel` (the plan's core
//!    construction), coloured from the RGB pixel.
//!
//! [`SplatSource`] adapts any [`DepthSource`] into a
//! [`FrameSource`]: capture, recordings, and the synthetic dataset all
//! plug into the renderer the same way.
use anyhow::Result;
use glam::Vec3;
use rig::{Camera, DepthFrame, DepthSource, Frame, FrameSource, Splat, SplatBatch};
/// Tuning knobs for the depth pipeline; the defaults follow PLAN.md
/// (bilateral radius 3–5 px, opacity ~0.95, few-mm thickness).
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct SplatGenParams {
    /// Bilateral filter radius in pixels.
    pub filter_radius: u32,
    /// Bilateral depth-domain sigma, millimetres.
    pub filter_sigma_depth_mm: f32,
    /// Bilateral spatial-domain sigma, pixels.
    pub filter_sigma_space_px: f32,
    /// A pixel disagreeing with its 3×3 neighbourhood median by more
    /// than this is a flier and is dropped, millimetres.
    pub flier_threshold_mm: u32,
    /// Pixels beyond this depth are culled as background, millimetres.
    pub max_depth_mm: u16,
    /// Surfet opacity (PLAN.md: ~0.9–0.99).
    pub opacity: f32,
    /// Surfel thickness along the normal, metres.
    pub thickness_m: f32,
}
impl Default for SplatGenParams {
    fn default() -> Self {
        Self {
            filter_radius: 3,
            filter_sigma_depth_mm: 25.0,
            filter_sigma_space_px: 2.0,
            flier_threshold_mm: 250,
            max_depth_mm: 4000,
            opacity: 0.95,
            thickness_m: rig::SURFEL_THICKNESS_M,
        }
    }
}
/// The depth pipeline: depth frames in, splat batches out.
pub struct SplatGen {
    params: SplatGenParams,
}
impl Default for SplatGen {
    fn default() -> Self {
        Self::new(SplatGenParams::default())
    }
}
impl SplatGen {
    pub fn new(params: SplatGenParams) -> Self {
        Self { params }
    }
    /// The tuning parameters in effect.
    pub fn params(&self) -> &SplatGenParams {
        &self.params
    }
    /// Preprocess one depth frame and assemble its splat batch.
    pub fn generate(&self, frame: &DepthFrame, camera: &Camera) -> SplatBatch {
        let (width, height) = (camera.resolution.x, camera.resolution.y);
        let pixel_count = (width * height) as usize;
        assert_eq!(frame.depth_mm.len(), pixel_count, "depth map size");
        assert_eq!(frame.rgb.len(), pixel_count * 3, "rgb size");
        // f32 working copy; `None` = invalid.
        let mut depth: Vec<Option<f32>> = frame
            .depth_mm
            .iter()
            .map(|&d| (d > 0 && d <= self.params.max_depth_mm).then_some(d as f32))
            .collect();
        remove_fliers(&mut depth, width, self.params.flier_threshold_mm);
        let filtered = bilateral_filter(&depth, width, self.params);
        let mut batch = SplatBatch::with_capacity(pixel_count);
        for y in 0..height {
            for x in 0..width {
                let Some(z_mm) = filtered[(y * width + x) as usize] else {
                    continue;
                };
                let (u, v) = (x as f32 + 0.5, y as f32 + 0.5);
                let z = z_mm / 1000.0;
                let position = camera.backproject(u, v, z);
                let normal = estimate_normal(&filtered, width, camera, x, y);
                let idx = (y * width + x) as usize;
                let color = [
                    frame.rgb[idx * 3] as f32 / 255.0,
                    frame.rgb[idx * 3 + 1] as f32 / 255.0,
                    frame.rgb[idx * 3 + 2] as f32 / 255.0,
                ];
                batch.push(Splat::surfel(
                    position,
                    normal,
                    z,
                    &camera.intrinsics,
                    self.params.thickness_m,
                    color,
                    self.params.opacity,
                ));
            }
        }
        batch
    }
}
/// A [`FrameSource`] adapter: pull raw depth frames from `S` and run
/// them through [`SplatGen`]. This is the pipeline seam — live capture,
/// recordings, and the synthetic dataset all become renderable sources.
pub struct SplatSource<S: DepthSource> {
    inner: S,
    generator: SplatGen,
}
impl<S: DepthSource> SplatSource<S> {
    pub fn new(inner: S, generator: SplatGen) -> Self {
        Self { inner, generator }
    }
    /// The wrapped depth source.
    pub fn inner(&self) -> &S {
        &self.inner
    }
}
impl<S: DepthSource> FrameSource for SplatSource<S> {
    fn next_frame(&mut self) -> Result<Frame> {
        let depth_frame = self.inner.next_depth_frame()?;
        let splats = self.generator.generate(&depth_frame, self.inner.camera());
        Ok(Frame {
            timestamp_secs: depth_frame.timestamp_secs,
            splats,
        })
    }
}
/// Drop pixels that disagree with their 3×3 neighbourhood median by
/// more than `threshold_mm` (isolated fliers). Neighbourhood positions
/// are clamped at the borders.
fn remove_fliers(depth: &mut [Option<f32>], width: u32, threshold_mm: u32) {
    let width = width as i32;
    let height = (depth.len() / width as usize) as i32;
    let threshold = threshold_mm as f32;
    let mut neighbourhood = [0.0_f32; 9];
    let mut flier = vec![false; depth.len()];
    for y in 0..height {
        for x in 0..width {
            let idx = (y * width + x) as usize;
            let Some(own) = depth[idx] else { continue };
            let mut n = 0;
            for dy in -1..=1 {
                for dx in -1..=1 {
                    let (nx, ny) = (x + dx, y + dy);
                    if !(0..width).contains(&nx) || !(0..height).contains(&ny) {
                        continue;
                    }
                    if let Some(d) = depth[(ny * width + nx) as usize] {
                        neighbourhood[n] = d;
                        n += 1;
                    }
                }
            }
            if n < 3 {
                continue; // not enough context to judge
            }
            neighbourhood[..n].sort_by(|a, b| a.partial_cmp(b).unwrap());
            let median = if n % 2 == 1 {
                neighbourhood[n / 2]
            } else {
                (neighbourhood[n / 2 - 1] + neighbourhood[n / 2]) * 0.5
            };
            if (own - median).abs() > threshold {
                flier[idx] = true;
            }
        }
    }
    for (depth, flier) in depth.iter_mut().zip(flier) {
        if flier {
            *depth = None;
        }
    }
}
/// Edge-preserving bilateral filter over valid pixels: spatial × depth
/// Gaussians, radius from the params.
fn bilateral_filter(depth: &[Option<f32>], width: u32, params: SplatGenParams) -> Vec<Option<f32>> {
    let width_i = width as i32;
    let height = (depth.len() / width as usize) as i32;
    let radius = params.filter_radius as i32;
    let inv_two_sigma_d2 = 1.0 / (2.0 * params.filter_sigma_depth_mm.powi(2));
    let inv_two_sigma_s2 = 1.0 / (2.0 * params.filter_sigma_space_px.powi(2));
    let mut out = depth.to_vec();
    for y in 0..height {
        for x in 0..width_i {
            let idx = (y * width_i + x) as usize;
            let Some(own) = depth[idx] else { continue };
            let mut total = 0.0;
            let mut weight_sum = 0.0;
            for dy in -radius..=radius {
                for dx in -radius..=radius {
                    let (nx, ny) = (x + dx, y + dy);
                    if !(0..width_i).contains(&nx) || !(0..height).contains(&ny) {
                        continue;
                    }
                    let Some(neighbour) = depth[(ny * width_i + nx) as usize] else {
                        continue;
                    };
                    let spatial = (dx * dx + dy * dy) as f32;
                    let dd = neighbour - own;
                    let weight = (-spatial * inv_two_sigma_s2 - dd * dd * inv_two_sigma_d2).exp();
                    total += weight * neighbour;
                    weight_sum += weight;
                }
            }
            if weight_sum > 1e-6 {
                out[idx] = Some(total / weight_sum);
            }
        }
    }
    out
}
/// Surface normal at a pixel from the filtered depth: the cross product
/// of screen-space derivatives of the backprojected point cloud,
/// oriented towards the camera. Falls back to facing the camera when no
/// valid neighbours exist.
fn estimate_normal(depth: &[Option<f32>], width: u32, camera: &Camera, x: u32, y: u32) -> Vec3 {
    let width_i = width as i32;
    let height = (depth.len() / width as usize) as i32;
    let (xi, yi) = (x as i32, y as i32);
    let point = |px: i32, py: i32| -> Option<Vec3> {
        if !(0..width_i).contains(&px) || !(0..height).contains(&py) {
            return None;
        }
        let d = depth[(py * width_i + px) as usize]? / 1000.0;
        Some(camera.backproject(px as f32 + 0.5, py as f32 + 0.5, d))
    };
    let center = point(xi, yi).expect("caller checked the pixel is valid");
    // Central differences where both neighbours are valid, otherwise
    // one-sided, otherwise no estimate.
    let du = point(xi + 1, yi)
        .zip(point(xi - 1, yi))
        .map(|(a, b)| a - b)
        .or_else(|| point(xi + 1, yi).map(|a| (a - center) * 2.0))
        .or_else(|| point(xi - 1, yi).map(|b| (center - b) * 2.0));
    let dv = point(xi, yi + 1)
        .zip(point(xi, yi - 1))
        .map(|(a, b)| a - b)
        .or_else(|| point(xi, yi + 1).map(|a| (a - center) * 2.0))
        .or_else(|| point(xi, yi - 1).map(|b| (center - b) * 2.0));
    let toward_camera = (camera.pose.position - center).normalize_or_zero();
    let Some(du) = du else { return toward_camera };
    let Some(dv) = dv else { return toward_camera };
    let mut normal = du.cross(dv).normalize_or_zero();
    if normal.length_squared() < 1e-8 {
        // Degenerate (no valid neighbours): face the camera.
        normal = toward_camera;
    } else if normal.dot(toward_camera) < 0.0 {
        // The cross product's handedness is arbitrary; orient towards
        // the camera.
        normal = -normal;
    }
    normal
}
#[cfg(test)]
mod tests {
    use super::*;
    use glam::{uvec2, vec3};
    use rig::{CameraIntrinsics, CameraPose};
    fn camera(width: u32, height: u32) -> Camera {
        Camera {
            id: "test".into(),
            pose: CameraPose::looking_at(Vec3::ZERO, vec3(0.0, 0.0, 2.0)),
            intrinsics: CameraIntrinsics {
                fx: 100.0,
                fy: 100.0,
                cx: width as f32 / 2.0,
                cy: height as f32 / 2.0,
            },
            resolution: uvec2(width, height),
        }
    }
    fn frame(width: u32, height: u32, depth_of: impl Fn(u32, u32) -> u16) -> DepthFrame {
        let mut depth = Vec::with_capacity((width * height) as usize);
        let mut rgb = Vec::with_capacity((width * height * 3) as usize);
        for y in 0..height {
            for x in 0..width {
                depth.push(depth_of(x, y));
                rgb.extend_from_slice(&[200, 100, 50]);
            }
        }
        DepthFrame {
            timestamp_secs: 0.0,
            depth_mm: depth,
            rgb,
        }
    }
    #[test]
    fn flat_wall_makes_camera_facing_surfels() {
        let (w, h) = (32u32, 24u32);
        let camera = camera(w, h);
        let frame = frame(w, h, |_, _| 2000);
        let batch = SplatGen::default().generate(&frame, &camera);
        assert_eq!(batch.len(), (w * h) as usize);
        for splat in batch.iter() {
            // On the wall plane: the projection lands back on the pixel.
            let projected = camera.project(splat.mean).expect("in front of the camera");
            // The wall's normal is constant (0, 0, -1) in world space…
            let wall_normal = Vec3::new(0.0, 0.0, -1.0);
            assert!(
                (splat.normal() - wall_normal).length() < 0.05,
                "wall normal {:?}",
                splat.normal()
            );
            // …and faces the camera everywhere on the wall.
            let to_camera = (camera.pose.position - splat.mean).normalize();
            assert!(splat.normal().dot(to_camera) > 0.9);
            // Footprint from the plan: depth / focal.
            let z = 2.0;
            assert!((splat.scale.x - z / 100.0).abs() < 1e-4);
            assert!((splat.scale.y - z / 100.0).abs() < 1e-4);
            assert_eq!(splat.color, [200.0 / 255.0, 100.0 / 255.0, 50.0 / 255.0]);
            assert!((splat.opacity - 0.95).abs() < 1e-6);
            let _ = projected;
        }
    }
    #[test]
    fn culling_drops_invalid_and_far_pixels() {
        let (w, h) = (16u32, 8u32);
        let camera = camera(w, h);
        let frame = frame(w, h, |x, y| match (x, y) {
            (0, 0) => 0,    // no return
            (1, 0) => 5000, // beyond max depth
            _ => 2000,
        });
        let batch = SplatGen::default().generate(&frame, &camera);
        assert_eq!(batch.len(), (w * h - 2) as usize);
    }
    #[test]
    fn fliers_are_removed() {
        let (w, h) = (16u32, 16u32);
        let camera = camera(w, h);
        // A few isolated wrong returns among a flat wall.
        let frame = frame(w, h, |x, y| {
            if matches!((x, y), (4, 4) | (10, 12) | (15, 0)) {
                600 // wildly wrong
            } else {
                2000
            }
        });
        let batch = SplatGen::default().generate(&frame, &camera);
        assert_eq!(
            batch.len(),
            (w * h - 3) as usize,
            "isolated fliers must be culled"
        );
        // And none of the surviving splats sit at the flier depth.
        for splat in batch.iter() {
            let z = camera
                .pose
                .camera_from_world()
                .transform_point3(splat.mean)
                .z;
            assert!((z - 2.0).abs() < 0.05, "stray depth {z}");
        }
    }
    #[test]
    fn bilateral_smooths_jitter_but_not_edges() {
        let (w, h) = (32u32, 8u32);
        let camera = camera(w, h);
        // Left half at 2000 mm, right half at 3000 mm, everything
        // jittered ±3 mm.
        let clean = frame(w, h, |x, _| if x < 16 { 2000 } else { 3000 });
        let noisy = frame(w, h, |x, y| {
            let base = if x < 16 { 2000 } else { 3000 };
            let jitter = ((x * 7 + y * 13) % 7) as i16 - 3;
            ((base as i16) + jitter) as u16
        });
        let generator = SplatGen::default();
        let clean_batch = generator.generate(&clean, &camera);
        let noisy_batch = generator.generate(&noisy, &camera);
        assert_eq!(clean_batch.len(), noisy_batch.len());
        // The noisy splats sit within a fraction of a pixel of the
        // clean ones: the jitter (≈3 mm at 2 m ≈ 0.15 px) is smoothed
        // away, and the step edge is preserved.
        for (a, b) in clean_batch.iter().zip(noisy_batch.iter()) {
            assert!(
                (a.mean - b.mean).length() < 0.002,
                "positions differ: {:?} vs {:?}",
                a.mean,
                b.mean
            );
        }
    }
    #[test]
    fn normals_follow_a_slanted_plane() {
        let (w, h) = (24u32, 24u32);
        let camera = camera(w, h);
        // Depth increases by 40 mm per pixel: the surface tilts away to
        // the right. dz/du = 0.04 m per pixel; world dx per pixel is
        // z/fx, so the plane's normal is ∝ (-slope, 0, 1).
        let frame = frame(w, h, |x, _| 2000 + 40 * x as u16);
        let batch = SplatGen::default().generate(&frame, &camera);
        // Check a pixel near the centre, away from border effects.
        let splat = batch.iter().nth(12 * 24 + 12).expect("center pixel");
        let z_world = 2.0 + 0.04 * 12.5; // metres at that pixel
        // The plane's camera-facing normal in world space: image-right
        // maps to world -X, so rising depth with u means dz/dx_world is
        // negative there.
        let dz_du = 0.04_f32; // metres per pixel
        let dz_dx_world = dz_du / (-(z_world / camera.intrinsics.fx));
        let expected = Vec3::new(dz_dx_world, 0.0, -1.0).normalize();
        let dot = splat.normal().dot(expected);
        assert!(
            dot > 0.995,
            "normal {:?} vs expected {expected:?} (dot {dot})",
            splat.normal()
        );
    }
    #[test]
    fn params_are_honoured() {
        let (w, h) = (16u32, 16u32);
        let camera = camera(w, h);
        let frame = frame(w, h, |_, _| 2500);
        let params = SplatGenParams {
            max_depth_mm: 2000, // culls everything
            ..SplatGenParams::default()
        };
        let batch = SplatGen::new(params).generate(&frame, &camera);
        assert!(batch.is_empty());
        // Opacity flows through to the surfels.
        let params = SplatGenParams {
            opacity: 0.9,
            ..SplatGenParams::default()
        };
        let batch = SplatGen::new(params).generate(&frame, &camera);
        assert!((batch.iter().next().unwrap().opacity - 0.9).abs() < 1e-6);
    }
    #[test]
    fn generates_from_the_synthetic_source() {
        // Integration: the synthetic Kinect feeds the pipeline.
        use rig::DepthSource;
        let rig = replay::synth::dev_rig_with_resolutions(uvec2(48, 40), uvec2(64, 48));
        let mut source =
            replay::synth::dev_source_with(rig.center.clone(), replay::synth::SUBJECT, 30.0);
        let frame = source.next_depth_frame().expect("depth frame");
        let batch = SplatGen::default().generate(&frame, &rig.center);
        // Roughly one splat per depth pixel (fliers removed).
        assert!(batch.len() > 48 * 40 * 95 / 100);
        assert!(batch.len() <= 48 * 40);
        // All in front of the camera, all at valid depths.
        let camera_from_world = rig.center.pose.camera_from_world();
        for splat in batch.iter() {
            let p = camera_from_world.transform_point3(splat.mean);
            assert!(p.z > 0.5 && p.z < 3.1, "depth {p:?}");
        }
    }
}