Commit: 6d86186
Parent: 2b3bdb8

Build step 3: live Kinect capture + ellipsoid proxy from side cameras

GLM Agent committed on 2026-10-25 at 08:06
The occlusion proxy (PLAN.md core design): splatgen fits a
head/shoulders ellipsoid to each frame's depth cloud (the nearest
cluster; the backdrop is further out), then casts every side-camera
pixel ray onto it and keeps the hits the depth map has no data for —
behind the observed subject surface, or over background. Those become
splat with the ellipsoid's outward normal, the side camera's pixel
colour, inflated thickness, and lower opacity (the plan's lower
confidence weight). The synthetic head carries side tints only its
flanks show, so the recolouring is provable: proxy splats carry the
side cameras' colours, and a GPU test shows the tinted flanks appear
in the rendered view only with the proxy on.

Live capture: the capture crate opens the Kinect v2 (kinect-one: CPU
depth processing, pure-Rust JPEG, factory registration) plus the two
webcams (nokhwa, matched by id or name), runs each on its own thread,
and composes frames timestamp-aligned with ~1-frame skew tolerance.
The source's rig carries the Kinect's factory depth intrinsics.

- rig: shared Ellipsoid geometry; DepthFrame carries side RGB;
  DepthSource exposes the source's full rig
- replay: synthetic scene's head flanks tinted (proxy ground truth);
  source emits side RGB; recording format carries the rig + side files
- splatgen: SubjectFit (ellipsoid + cluster depth range), the
  occlusion rule, and the proxy fill; SplatSource owns the rig
- capture: KinectRigSource (code-complete; hardware verification
  pending — no USB in this container; opening fails fast and clearly
  without a Kinect, which is tested)
- sink: --source synthetic|recording|kinect, --recording-path, and
  proxy knobs join the config surface; main dispatches all sources
- tests: 75 across the workspace, including the recording path
  end-to-end and the proxy's side-colour assertions
Cargo.lock +670 -75
diff --git a/Cargo.lock b/Cargo.lock
index a087280..09a7156 100644
@@ -147,7 +147,7 @@ version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"cipher",
"cpufeatures 0.2.17",
]
@@ -158,7 +158,7 @@ version = "0.8.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"getrandom 0.3.4",
"once_cell",
"version_check",
@@ -409,7 +409,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc"
dependencies = [
"autocfg",
"cfg-if",
"cfg-if 1.0.5",
"concurrent-queue",
"futures-io",
"futures-lite",
@@ -443,7 +443,7 @@ dependencies = [
"async-signal",
"async-task",
"blocking",
"cfg-if",
"cfg-if 1.0.5",
"event-listener",
"futures-lite",
"rustix 1.1.5",
@@ -469,7 +469,7 @@ dependencies = [
"async-io",
"async-lock",
"atomic-waker",
"cfg-if",
"cfg-if 1.0.5",
"futures-core",
"futures-io",
"rustix 1.1.5",
@@ -579,7 +579,7 @@ dependencies = [
"anyhow",
"arrayvec",
"log",
"nom",
"nom 8.0.0",
"num-rational",
"v_frame",
]
@@ -674,7 +674,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6"
dependencies = [
"addr2line",
"cfg-if",
"cfg-if 1.0.5",
"libc",
"miniz_oxide 0.8.9",
"object",
@@ -701,6 +701,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
[[package]]
name = "bindgen"
version = "0.65.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5"
dependencies = [
"bitflags 1.3.2",
"cexpr",
"clang-sys",
"lazy_static",
"lazycell",
"log",
"peeking_take_while",
"prettyplease 0.2.37",
"proc-macro2",
"quote",
"regex",
"rustc-hash 1.1.0",
"shlex 1.3.0",
"syn 2.0.119",
"which",
]
[[package]]
name = "bit-set"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -746,6 +769,12 @@ dependencies = [
]
[[package]]
name = "block"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a"
[[package]]
name = "block-buffer"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -802,7 +831,7 @@ checksum = "93833e97ac7c46ba97045afcf247238c27057538fcf2f06344662ddd5f567be6"
dependencies = [
"darling 0.24.1",
"ident_case",
"prettyplease",
"prettyplease 0.3.0",
"proc-macro2",
"quote",
"syn 3.0.9",
@@ -875,7 +904,7 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b8e278b5bdc31756d01d65c17662e23692e0dbaa19e65adae9b58fc4683bc09"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"libc",
"log",
"winapi",
@@ -921,7 +950,7 @@ dependencies = [
"num-traits",
"parking_lot",
"portable-atomic",
"spin",
"spin 0.12.3",
]
[[package]]
@@ -939,7 +968,7 @@ dependencies = [
"rand 0.10.3",
"rand_distr",
"serde",
"spin",
"spin 0.12.3",
]
[[package]]
@@ -979,7 +1008,7 @@ dependencies = [
"regex",
"serde",
"serde_json",
"spin",
"spin 0.12.3",
]
[[package]]
@@ -1027,7 +1056,7 @@ dependencies = [
"derive-new",
"hashbrown 0.16.1",
"serde",
"spin",
"spin 0.12.3",
]
[[package]]
@@ -1104,7 +1133,7 @@ dependencies = [
"hashbrown 0.16.1",
"log",
"serde",
"spin",
"spin 0.12.3",
"tracing",
]
@@ -1231,7 +1260,7 @@ dependencies = [
"hashbrown 0.16.1",
"portable-atomic",
"serde",
"spin",
"spin 0.12.3",
]
[[package]]
@@ -1266,7 +1295,7 @@ dependencies = [
"rand_distr",
"serde",
"smallvec",
"spin",
"spin 0.12.3",
"thiserror 2.0.21",
"uuid",
]
@@ -1463,6 +1492,15 @@ dependencies = [
[[package]]
name = "capture"
version = "0.1.0"
dependencies = [
"anyhow",
"glam",
"kinect-one",
"log",
"nokhwa",
"rig",
"tokio",
]
[[package]]
name = "cc"
@@ -1473,11 +1511,26 @@ dependencies = [
"find-msvc-tools",
"jobserver",
"libc",
"shlex",
"shlex 2.0.1",
]
[[package]]
name = "cexpr"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
dependencies = [
"nom 7.1.3",
]
[[package]]
name = "cfg-if"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
[[package]]
name = "cfg-if"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
@@ -1503,7 +1556,7 @@ version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"cpufeatures 0.3.1",
"rand_core 0.10.1",
]
@@ -1546,6 +1599,17 @@ dependencies = [
]
[[package]]
name = "clang-sys"
version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a"
dependencies = [
"glob",
"libc",
"libloading 0.8.9",
]
[[package]]
name = "clap"
version = "4.6.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1610,6 +1674,34 @@ dependencies = [
]
[[package]]
name = "cocoa"
version = "0.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c49e86fc36d5704151f5996b7b3795385f50ce09e3be0f47a0cfde869681cf8"
dependencies = [
"bitflags 1.3.2",
"block",
"core-foundation 0.7.0",
"core-graphics 0.19.2",
"foreign-types 0.3.2",
"libc",
"objc",
]
[[package]]
name = "cocoa-foundation"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81411967c50ee9a1fc11365f8c585f863a22a9697c89239c452292c40ba79b0d"
dependencies = [
"bitflags 2.13.2",
"block",
"core-foundation 0.10.1",
"core-graphics-types 0.2.0",
"objc",
]
[[package]]
name = "codespan-reporting"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1676,7 +1768,7 @@ version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"wasm-bindgen",
]
@@ -1738,30 +1830,68 @@ dependencies = [
[[package]]
name = "core-foundation"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171"
dependencies = [
"core-foundation-sys 0.7.0",
"libc",
]
[[package]]
name = "core-foundation"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
dependencies = [
"core-foundation-sys",
"core-foundation-sys 0.8.7",
"libc",
]
[[package]]
name = "core-foundation"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
dependencies = [
"core-foundation-sys 0.8.7",
"libc",
]
[[package]]
name = "core-foundation-sys"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac"
[[package]]
name = "core-foundation-sys"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "core-graphics"
version = "0.19.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3889374e6ea6ab25dba90bb5d96202f61108058361f6dc72e8b03e6f8bbe923"
dependencies = [
"bitflags 1.3.2",
"core-foundation 0.7.0",
"foreign-types 0.3.2",
"libc",
]
[[package]]
name = "core-graphics"
version = "0.23.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081"
dependencies = [
"bitflags 1.3.2",
"core-foundation",
"core-graphics-types",
"foreign-types",
"core-foundation 0.9.4",
"core-graphics-types 0.1.3",
"foreign-types 0.5.0",
"libc",
]
@@ -1772,11 +1902,47 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf"
dependencies = [
"bitflags 1.3.2",
"core-foundation",
"core-foundation 0.9.4",
"libc",
]
[[package]]
name = "core-graphics-types"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb"
dependencies = [
"bitflags 2.13.2",
"core-foundation 0.10.1",
"libc",
]
[[package]]
name = "core-media-sys"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "273bf3fc5bf51fd06a7766a84788c1540b6527130a0bce39e00567d6ab9f31f1"
dependencies = [
"cfg-if 0.1.10",
"core-foundation-sys 0.7.0",
"libc",
]
[[package]]
name = "core-video-sys"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34ecad23610ad9757664d644e369246edde1803fcb43ed72876565098a5d3828"
dependencies = [
"cfg-if 0.1.10",
"core-foundation-sys 0.7.0",
"core-graphics 0.19.2",
"libc",
"metal",
"objc",
]
[[package]]
name = "cpufeatures"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1800,7 +1966,7 @@ version = "1.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
]
[[package]]
@@ -1887,7 +2053,7 @@ dependencies = [
"portable-atomic",
"portable-atomic-util",
"serde",
"spin",
"spin 0.12.3",
"tynm",
"xxhash-rust",
]
@@ -1961,7 +2127,7 @@ dependencies = [
"libc",
"log",
"pliron",
"spin",
"spin 0.12.3",
"sysinfo",
"winapi",
]
@@ -1994,7 +2160,7 @@ dependencies = [
"backtrace",
"bytemuck",
"bytes",
"cfg-if",
"cfg-if 1.0.5",
"cfg_aliases",
"ciborium",
"embassy-futures",
@@ -2013,7 +2179,7 @@ dependencies = [
"serde",
"serde_bytes",
"serde_json",
"spin",
"spin 0.12.3",
"toml",
"wasm-bindgen-futures",
"web-time",
@@ -2076,7 +2242,7 @@ dependencies = [
"rustc_apfloat",
"serde",
"smallvec",
"spin",
"spin 0.12.3",
"thiserror 2.0.21",
"variadics_please",
]
@@ -2111,7 +2277,7 @@ dependencies = [
"ident_case",
"inflections",
"itertools 0.15.0",
"prettyplease",
"prettyplease 0.3.0",
"proc-macro2",
"quote",
"syn 3.0.9",
@@ -2180,7 +2346,7 @@ source = "git+https://github.com/tracel-ai/cubecl?rev=467522442c7b68f02c7ff0c84b
dependencies = [
"buildid",
"bytemuck",
"cfg-if",
"cfg-if 1.0.5",
"cfg_aliases",
"cubecl-common",
"cubecl-environment",
@@ -2209,7 +2375,7 @@ source = "git+https://github.com/tracel-ai/cubecl?rev=467522442c7b68f02c7ff0c84b
dependencies = [
"buildid",
"bytemuck",
"cfg-if",
"cfg-if 1.0.5",
"cfg_aliases",
"cubecl-common",
"cubecl-environment",
@@ -2243,7 +2409,7 @@ dependencies = [
"num-traits",
"paste",
"serde",
"spin",
"spin 0.12.3",
"variadics_please",
]
@@ -2253,7 +2419,7 @@ version = "0.11.0-pre.4"
source = "git+https://github.com/tracel-ai/cubecl?rev=467522442c7b68f02c7ff0c84b2e249e36d8da91#467522442c7b68f02c7ff0c84b2e249e36d8da91"
dependencies = [
"bytemuck",
"cfg-if",
"cfg-if 1.0.5",
"cfg_aliases",
"cubecl-common",
"cubecl-core",
@@ -2731,6 +2897,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76"
[[package]]
name = "dunce"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
[[package]]
name = "dyn-clone"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2938,7 +3110,7 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "592b0c143ec626e821d4d90da51a2bd91d559d6c442b7c74a47d368c9e23d97a"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"critical-section",
"document-features",
"embassy-time-driver",
@@ -3146,7 +3318,7 @@ version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"windows-sys 0.61.2",
]
@@ -3243,7 +3415,7 @@ version = "0.2.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"libc",
]
@@ -3292,6 +3464,18 @@ dependencies = [
]
[[package]]
name = "flume"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095"
dependencies = [
"futures-core",
"futures-sink",
"nanorand",
"spin 0.9.9",
]
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3342,12 +3526,21 @@ dependencies = [
[[package]]
name = "foreign-types"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
dependencies = [
"foreign-types-shared 0.1.1",
]
[[package]]
name = "foreign-types"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965"
dependencies = [
"foreign-types-macros",
"foreign-types-shared",
"foreign-types-shared 0.3.1",
]
[[package]]
@@ -3363,6 +3556,12 @@ dependencies = [
[[package]]
name = "foreign-types-shared"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]]
name = "foreign-types-shared"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b"
@@ -3422,6 +3621,12 @@ dependencies = [
]
[[package]]
name = "futures-sink"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d"
[[package]]
name = "futures-task"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3583,9 +3788,11 @@ version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"js-sys",
"libc",
"wasi",
"wasm-bindgen",
]
[[package]]
@@ -3594,7 +3801,7 @@ version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"libc",
"r-efi 5.3.0",
"wasip2",
@@ -3606,7 +3813,7 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"libc",
"r-efi 6.0.0",
"rand_core 0.10.1",
@@ -3666,6 +3873,12 @@ dependencies = [
]
[[package]]
name = "glob"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
[[package]]
name = "gloo-timers"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3785,7 +3998,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
dependencies = [
"bytemuck",
"cfg-if",
"cfg-if 1.0.5",
"crunchy",
"num-traits",
"serde",
@@ -3896,6 +4109,15 @@ dependencies = [
]
[[package]]
name = "home"
version = "0.5.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d"
dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "http"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4117,6 +4339,16 @@ dependencies = [
]
[[package]]
name = "io-kit-sys"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "617ee6cf8e3f66f3b4ea67a4058564628cde41901316e19f559e14c7c72c5e7b"
dependencies = [
"core-foundation-sys 0.8.7",
"mach2",
]
[[package]]
name = "is_terminal_polyfill"
version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4189,7 +4421,7 @@ version = "0.22.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"combine",
"jni-macros",
"jni-sys 0.4.1",
@@ -4257,7 +4489,7 @@ version = "0.3.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"futures-util",
"wasm-bindgen",
]
@@ -4280,6 +4512,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc"
[[package]]
name = "kinect-one"
version = "0.1.0"
source = "git+https://github.com/Aytixel/kinect-one#24c6dc0f18485c934f218b853c26f91909172537"
dependencies = [
"nusb",
"rayon",
"thiserror 2.0.21",
"tokio",
"zune-jpeg",
]
[[package]]
name = "kittest"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4308,6 +4552,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb"
[[package]]
name = "lazycell"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "lebe"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4335,7 +4585,7 @@ version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"windows-link",
]
@@ -4345,7 +4595,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"windows-link",
]
@@ -4432,6 +4682,12 @@ checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
[[package]]
name = "linux-raw-sys"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
@@ -4515,6 +4771,24 @@ dependencies = [
]
[[package]]
name = "mach2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44"
dependencies = [
"libc",
]
[[package]]
name = "malloc_buf"
version = "0.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb"
dependencies = [
"libc",
]
[[package]]
name = "matrixmultiply"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4530,7 +4804,7 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"rayon",
]
@@ -4559,6 +4833,27 @@ dependencies = [
]
[[package]]
name = "metal"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e198a0ee42bdbe9ef2c09d0b9426f3b2b47d90d93a4a9b0395c4cea605e92dc0"
dependencies = [
"bitflags 1.3.2",
"block",
"cocoa",
"core-graphics 0.19.2",
"foreign-types 0.3.2",
"log",
"objc",
]
[[package]]
name = "minimal-lexical"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4606,6 +4901,31 @@ dependencies = [
]
[[package]]
name = "mozjpeg"
version = "0.10.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7891b80aaa86097d38d276eb98b3805d6280708c4e0a1e6f6aed9380c51fec9"
dependencies = [
"arrayvec",
"bytemuck",
"libc",
"mozjpeg-sys",
"rgb",
]
[[package]]
name = "mozjpeg-sys"
version = "2.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f0dc668bf9bf888c88e2fb1ab16a406d2c380f1d082b20d51dd540ab2aa70c1"
dependencies = [
"cc",
"dunce",
"libc",
"nasm-rs",
]
[[package]]
name = "mutants"
version = "0.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4620,7 +4940,7 @@ dependencies = [
"arrayvec",
"bit-set",
"bitflags 2.13.2",
"cfg-if",
"cfg-if 1.0.5",
"cfg_aliases",
"codespan-reporting",
"half",
@@ -4650,6 +4970,25 @@ dependencies = [
]
[[package]]
name = "nanorand"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3"
dependencies = [
"getrandom 0.2.17",
]
[[package]]
name = "nasm-rs"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "706bf8a5e8c8ddb99128c3291d31bd21f4bcde17f0f4c20ec678d85c74faa149"
dependencies = [
"jobserver",
"log",
]
[[package]]
name = "nb"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4746,6 +5085,84 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451"
[[package]]
name = "nokhwa"
version = "0.10.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d63f10b450319a0ace7aa8e0e25477d1fdb345313a97e220e886175539a1dbb"
dependencies = [
"flume",
"image",
"nokhwa-bindings-linux",
"nokhwa-bindings-macos",
"nokhwa-bindings-windows",
"nokhwa-core",
"parking_lot",
"paste",
"thiserror 2.0.21",
]
[[package]]
name = "nokhwa-bindings-linux"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb67e22201a53322291740ca064b20eaaade7222ef0349f312d9b37b004e1984"
dependencies = [
"libc",
"nokhwa-core",
"v4l",
]
[[package]]
name = "nokhwa-bindings-macos"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f70d3908ea68324e44a6b3a0f885aa59e433fb1f6678839d09e0df7d226fb42d"
dependencies = [
"block",
"cocoa-foundation",
"core-foundation 0.10.1",
"core-media-sys",
"core-video-sys",
"flume",
"nokhwa-core",
"objc",
"once_cell",
]
[[package]]
name = "nokhwa-bindings-windows"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5be28886bad8abcec3655c1f24b965b4cb596a72b23164c910c54439ce55d2a4"
dependencies = [
"nokhwa-core",
"once_cell",
"windows",
]
[[package]]
name = "nokhwa-core"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1cba20bebd3bd9ae22f9273ade5bbe49da3e047c8512b53fbaf8b4b9c80d496"
dependencies = [
"bytes",
"image",
"mozjpeg",
"thiserror 2.0.21",
]
[[package]]
name = "nom"
version = "7.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
dependencies = [
"memchr",
"minimal-lexical",
]
[[package]]
name = "nom"
version = "8.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4859,6 +5276,34 @@ dependencies = [
]
[[package]]
name = "nusb"
version = "0.2.1"
source = "git+https://github.com/FreddyFunk/nusb?branch=feature%2Fimplement-isochronous-transfers-for-linux#fef4a8337f7949a3a64303bd875f9714807e3617"
dependencies = [
"core-foundation 0.10.1",
"core-foundation-sys 0.8.7",
"futures-core",
"io-kit-sys",
"linux-raw-sys 0.9.4",
"log",
"once_cell",
"rustix 1.1.5",
"slab",
"tokio",
"windows-sys 0.60.2",
]
[[package]]
name = "objc"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1"
dependencies = [
"malloc_buf",
"objc_exception",
]
[[package]]
name = "objc-sys"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -5204,6 +5649,15 @@ dependencies = [
]
[[package]]
name = "objc_exception"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad970fb455818ad6cba4c122ad012fae53ae8b4795f86378bce65e4f6bab2ca4"
dependencies = [
"cc",
]
[[package]]
name = "object"
version = "0.37.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -5316,7 +5770,7 @@ version = "0.9.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"libc",
"redox_syscall 0.5.18",
"smallvec",
@@ -5365,6 +5819,12 @@ dependencies = [
]
[[package]]
name = "peeking_take_while"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
[[package]]
name = "peniko"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -5509,7 +5969,7 @@ dependencies = [
"rustc_apfloat",
"slotmap",
"smallvec",
"spin",
"spin 0.12.3",
"thiserror 2.0.21",
"utf8-chars",
]
@@ -5571,7 +6031,7 @@ version = "3.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"concurrent-queue",
"hermit-abi",
"pin-project-lite",
@@ -5645,6 +6105,16 @@ checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa"
[[package]]
name = "prettyplease"
version = "0.2.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
"proc-macro2",
"syn 2.0.119",
]
[[package]]
name = "prettyplease"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0"
@@ -5697,7 +6167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "046aa45b989642ec2e4717c8e72d677b13edd831a4d3b6cf37d9a3e54912496a"
dependencies = [
"bytemuck",
"cfg-if",
"cfg-if 1.0.5",
"libm",
"num-complex",
"paste",
@@ -5870,7 +6340,7 @@ dependencies = [
"av1-grain",
"bitstream-io",
"built",
"cfg-if",
"cfg-if 1.0.5",
"interpolate_name",
"itertools 0.14.0",
"libc",
@@ -6092,6 +6562,9 @@ name = "rgb"
version = "0.8.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4"
dependencies = [
"bytemuck",
]
[[package]]
name = "rig"
@@ -6108,7 +6581,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
dependencies = [
"cc",
"cfg-if",
"cfg-if 1.0.5",
"getrandom 0.2.17",
"libc",
"untrusted",
@@ -6448,7 +6921,7 @@ version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"cpufeatures 0.2.17",
"digest",
]
@@ -6459,13 +6932,19 @@ version = "0.10.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"cpufeatures 0.2.17",
"digest",
]
[[package]]
name = "shlex"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "shlex"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
@@ -6516,6 +6995,7 @@ name = "sink"
version = "0.1.0"
dependencies = [
"anyhow",
"capture",
"clap",
"clap_config",
"eframe",
@@ -6647,6 +7127,15 @@ dependencies = [
[[package]]
name = "spin"
version = "0.9.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e"
dependencies = [
"lock_api",
]
[[package]]
name = "spin"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0134f9043ed38b087ac4f7d4af44c79e2c9e5094421fe3164f435ce585953b10"
@@ -6955,7 +7444,7 @@ dependencies = [
"arrayref",
"arrayvec",
"bytemuck",
"cfg-if",
"cfg-if 1.0.5",
"log",
"tiny-skia-path",
]
@@ -7131,7 +7620,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8765b90061cba6c22b5831f675da109ae5561588290f9fa2317adab2714d5a6"
dependencies = [
"memchr",
"nom",
"nom 8.0.0",
"petgraph",
]
@@ -7156,7 +7645,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a21cdb0fc8f85c98b1ec812bc4cd69faf6c0fa2fc17d44ea3c2cdd38dc08e999"
dependencies = [
"nom",
"nom 8.0.0",
]
[[package]]
@@ -7341,6 +7830,26 @@ dependencies = [
]
[[package]]
name = "v4l"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8fbfea44a46799d62c55323f3c55d06df722fbe577851d848d328a1041c3403"
dependencies = [
"bitflags 1.3.2",
"libc",
"v4l2-sys-mit",
]
[[package]]
name = "v4l2-sys-mit"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6779878362b9bacadc7893eac76abe69612e8837ef746573c4a5239daf11990b"
dependencies = [
"bindgen",
]
[[package]]
name = "v_frame"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -7439,7 +7948,7 @@ version = "0.2.129"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409"
dependencies = [
"cfg-if",
"cfg-if 1.0.5",
"once_cell",
"rustversion",
"wasm-bindgen-macro",
@@ -7704,7 +8213,7 @@ dependencies = [
"arrayvec",
"bitflags 2.13.2",
"bytemuck",
"cfg-if",
"cfg-if 1.0.5",
"cfg_aliases",
"document-features",
"hashbrown 0.17.1",
@@ -7809,7 +8318,7 @@ dependencies = [
"bitflags 2.13.2",
"block2 0.6.2",
"bytemuck",
"cfg-if",
"cfg-if 1.0.5",
"cfg_aliases",
"glow",
"glutin_wgl_sys",
@@ -7879,6 +8388,18 @@ dependencies = [
]
[[package]]
name = "which"
version = "4.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
dependencies = [
"either",
"home",
"once_cell",
"rustix 0.38.44",
]
[[package]]
name = "wide"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -8026,7 +8547,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets",
"windows-targets 0.52.6",
]
[[package]]
@@ -8035,7 +8556,16 @@ version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
dependencies = [
"windows-targets 0.53.5",
]
[[package]]
@@ -8053,14 +8583,31 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
"windows_i686_gnullvm 0.52.6",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
]
[[package]]
name = "windows-targets"
version = "0.53.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
dependencies = [
"windows-link",
"windows_aarch64_gnullvm 0.53.1",
"windows_aarch64_msvc 0.53.1",
"windows_i686_gnu 0.53.1",
"windows_i686_gnullvm 0.53.1",
"windows_i686_msvc 0.53.1",
"windows_x86_64_gnu 0.53.1",
"windows_x86_64_gnullvm 0.53.1",
"windows_x86_64_msvc 0.53.1",
]
[[package]]
@@ -8079,48 +8626,96 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_aarch64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_i686_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "windows_x86_64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "winit"
version = "0.30.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -8135,8 +8730,8 @@ dependencies = [
"calloop 0.13.0",
"cfg_aliases",
"concurrent-queue",
"core-foundation",
"core-graphics",
"core-foundation 0.9.4",
"core-graphics 0.23.2",
"cursor-icon",
"dpi",
"js-sys",
Cargo.toml +9 -2
diff --git a/Cargo.toml b/Cargo.toml
index f6acac5..727c933 100644
@@ -33,8 +33,15 @@ glam = { version = "0.30", features = ["serde"] }
bytemuck = "1.20"
# Capture.
kinect-one = { git = "https://github.com/Aytixel/kinect-one" }
nokhwa = "0.10"
kinect-one = { git = "https://github.com/Aytixel/kinect-one", default-features = false, features = [
"parallel",
"cpu_depth",
"zune_color",
] }
nokhwa = { version = "0.10", features = [
"input-native",
"output-threaded",
] }
# Rendering: Brush's renderer as a library (not published to crates.io; it
# lives under crates/brush-render in the Brush repo). The burn/burn-wgpu
PLAN.md +3 -1
diff --git a/PLAN.md b/PLAN.md
index 702c65a..ee0e36e 100644
@@ -117,7 +117,9 @@ loopback output) join the same arg surface.
recording format (synthetic recordings for CI); the synthetic
source is refactored to emit raw depth frames like a Kinect, so
the dev/CI path runs through the real depth preprocessing
- [ ] 3. Live Kinect capture + ellipsoid proxy recoloring from side cameras
- [x] 3. Live Kinect capture + ellipsoid proxy recoloring from side
cameras — the capture crate is code-complete but **awaits hardware
verification** (the container has no USB; see Testing)
- [ ] 4. v4l2loopback sink
- [ ] 5. Polish: cross-camera color calibration, WB-lock webcams, seam tuning
README.md +25 -11
diff --git a/README.md b/README.md
index 8c2a484..852cba6 100644
@@ -25,20 +25,30 @@ Targets Ubuntu 24.04 (x86-64) and Windows on ARM64.
## Status
Build-order steps 1–2 are complete. The full pipeline runs on synthetic
Build-order steps 1–3 are complete. The full pipeline runs on synthetic
data through the *real* depth path: a software Kinect emits raw
millimetre depth maps plus RGB (with deterministic sensor noise), and
`splatgen` culls the background, removes fliers, bilateral-filters the
depth, estimates normals, and assembles footprint-sized surfel splats —
which Brush renders live from a virtual camera steered between the
rig's side cameras, shown in the egui viewer. The whole chain, GPU
rendering included, runs headlessly in tests (software Vulkan), noise
included.
millimetre depth maps plus RGB (with deterministic sensor noise) and
side-camera images, and `splatgen` culls the background, removes
fliers, bilateral-filters the depth, estimates normals, and assembles
footprint-sized surfel splats. Occluded regions — the subject's flanks
the Kinect never measured — are filled from a head/shoulders ellipsoid
fitted to each frame's depth cloud, recolored from the side webcams
(the plan's occlusion proxy). Brush renders it live from a virtual
camera steered between the rig's side cameras, shown in the egui
viewer. The whole chain, GPU rendering included, runs headlessly in
tests (software Vulkan), noise included.
Sources are selectable: the synthetic dataset (default), a recorded
dataset, or the live rig (`--source synthetic|recording|kinect`). The
live rig capture (Kinect v2 via `kinect-one` plus two webcams via
`nokhwa`) is implemented but **not yet hardware-verified** — this
container has no USB.
Recorded datasets have an on-disk format (`replay::recording`):
a directory with `recording.toml` (camera, fps, frame count) plus raw
`frame_NNNN.depth`/`frame_NNNN.rgb` files. Real Kinect recordings can
be converted into that shape; synthetic ones are generated for dev/CI.
a directory with `recording.toml` (the rig's cameras, fps, frame count)
plus raw `frame_NNNN.depth`/`.rgb`/`.left`/`.right` files. Real rig
recordings can be converted into that shape; synthetic ones are
generated for dev/CI.
- `cargo run -p sink` — the viewer (needs a GPU, or lavapipe/Vulkan)
- `cargo test --workspace` — unit, GPU-integration, and headless UI tests
@@ -68,6 +78,10 @@ rig_left_intrinsics = [554.256, 415.692, 320.0, 240.0]
scene_subject_position = [0.0, 0.0, 2.0]
source_fps = 30.0
background = [0.10, 0.10, 0.12]
source = "recording" # or "synthetic" / "kinect"
recording_path = "my-recording" # for --source recording
proxy_enabled = true # occlusion proxy from the side cameras
```
`kinect-view --help` lists every setting with its environment variable.
capture/Cargo.toml +7 -0
diff --git a/capture/Cargo.toml b/capture/Cargo.toml
index 2bb2bd0..a85a08c 100644
@@ -6,3 +6,10 @@ edition.workspace = true
publish = false
[dependencies]
anyhow.workspace = true
glam.workspace = true
kinect-one.workspace = true
log.workspace = true
nokhwa.workspace = true
rig.workspace = true
tokio.workspace = true
capture/src/lib.rs +490 -13
diff --git a/capture/src/lib.rs b/capture/src/lib.rs
index 2ddd10f..ceccbde 100644
@@ -1,15 +1,492 @@
//! Live capture from the fixed three-camera rig.
//! Live capture from the physical rig: a Kinect v2 (depth + registered
//! RGB, via [`kinect_one`]) flanked by two webcams ([`nokhwa`]), composed
//! into raw [`rig::DepthFrame`]s.
//!
//! Pipeline role (PLAN.md, "Pipeline per frame" — capture):
//! Each sensor runs on its own thread; frames are timestamp-aligned by
//! pairing each Kinect frame with the *latest* webcam frames (the plan
//! tolerates ~1 frame of skew). The Kinect's factory calibration
//! provides the depth camera's intrinsics (512x424, the registered
//! pixels' own convention); the config's center-camera pose — the
//! offline calibration's extrinsics — still applies.
//!
//! * Kinect v2 RGB + depth via `kinect-one`
//! * two side webcams via `nokhwa`
//! * timestamp alignment and frame sync across the three asynchronous
//! sensors (tolerate ~1 frame of skew)
//!
//! Camera extrinsics are known exactly from the existing offline
//! calibration; the camera overlap regions are checked at runtime as a
//! sanity check.
//!
//! Live capture must present the same rig-frame interface as `replay`, so
//! that live and recorded/synthetic input are interchangeable downstream.
//! **Hardware note:** none of this can run in the container — there is
//! no USB. The composition, device-matching, and conversion logic is
//! unit-tested with synthetic buffers; the device paths are exercised
//! only on real hardware. [`KinectRigSource::open`] fails fast with a
//! clear error when no Kinect is present.
use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
mpsc::{Receiver, SyncSender, sync_channel},
};
use std::thread::JoinHandle;
use std::time::Instant;
use anyhow::{Context, Result, bail};
use glam::UVec2;
use rig::{Camera, CameraIntrinsics, DepthFrame, DepthSource, Rig, SideRgb};
/// A composed frame from the capture threads.
struct PairedFrame {
depth_mm: Vec<u16>,
rgb: Vec<u8>,
left: Option<Vec<u8>>,
right: Option<Vec<u8>>,
timestamp_secs: f64,
}
/// The live rig source: Kinect v2 + two webcams.
pub struct KinectRigSource {
rig: Rig,
receiver: Receiver<PairedFrame>,
/// Set on drop; the worker threads check it and exit.
shutdown: Arc<AtomicBool>,
kinect_worker: Option<JoinHandle<()>>,
/// Kept alive so their capture threads keep streaming; stopping
/// on drop.
webcams: Vec<Webcam>,
}
impl KinectRigSource {
/// Open the rig: the Kinect plus the two webcams named by the rig
/// config's camera ids (a numeric id is a device index; otherwise
/// the id must match the camera's human-readable name).
///
/// The rig's center camera is *replaced* by the Kinect's factory
/// depth calibration (512x424 at the depth pixels' own intrinsics);
/// its pose comes from the config. The returned
/// [`DepthSource::rig`] is the effective rig the pipeline must
/// use.
pub fn open(rig: &Rig) -> Result<Self> {
let shutdown = Arc::new(AtomicBool::new(false));
// Kinect first, so a missing device fails before any webcam is
// touched. The worker reports the effective depth camera back;
// the webcam receivers are handed to it once opened.
let (ready_tx, ready_rx) = std::sync::mpsc::channel::<Result<Camera>>();
let (frames_tx, frames_rx) = sync_channel::<PairedFrame>(2);
let (side_tx, side_rx) =
std::sync::mpsc::channel::<(Receiver<Vec<u8>>, Receiver<Vec<u8>>)>();
let kinect_worker = {
let shutdown = shutdown.clone();
let center_config = rig.center.clone();
std::thread::spawn(move || {
kinect_loop(center_config, ready_tx, side_rx, frames_tx, &shutdown);
})
};
// If the Kinect fails before the ready signal, this surfaces it.
let center = ready_rx
.recv()
.map_err(|_| anyhow::anyhow!("the Kinect worker stopped before starting"))??;
// The webcams: each streams decoded RGB into its channel from
// nokhwa's own capture thread.
let mut webcams = Vec::new();
let mut side_channels = Vec::new();
for config in [&rig.left, &rig.right] {
let (tx, rx) = sync_channel(2);
let webcam = Webcam::open(config, tx)
.with_context(|| format!("opening webcam {:?}", config.id))?;
webcams.push(webcam);
side_channels.push(rx);
}
let mut it = side_channels.into_iter();
let (left_rx, right_rx) = (
it.next().expect("two side cameras"),
it.next().expect("two side cameras"),
);
side_tx
.send((left_rx, right_rx))
.map_err(|_| anyhow::anyhow!("the Kinect worker stopped"))?;
let rig = Rig {
center,
left: rig.left.clone(),
right: rig.right.clone(),
};
Ok(Self {
rig,
receiver: frames_rx,
shutdown,
kinect_worker: Some(kinect_worker),
webcams,
})
}
}
impl Drop for KinectRigSource {
fn drop(&mut self) {
self.shutdown.store(true, Ordering::Relaxed);
if let Some(worker) = self.kinect_worker.take() {
let _ = worker.join();
}
// The webcams stop their own capture threads on drop.
self.webcams.clear();
}
}
impl DepthSource for KinectRigSource {
fn camera(&self) -> &Camera {
&self.rig.center
}
fn rig(&self) -> Rig {
self.rig.clone()
}
fn next_depth_frame(&mut self) -> Result<DepthFrame> {
let paired = self
.receiver
.recv()
.map_err(|_| anyhow::anyhow!("the Kinect stopped producing frames"))?;
Ok(DepthFrame {
timestamp_secs: paired.timestamp_secs,
depth_mm: paired.depth_mm,
rgb: paired.rgb,
side_rgb: Some(SideRgb {
left: paired.left.unwrap_or_default(),
left_resolution: self.rig.left.resolution,
right: paired.right.unwrap_or_default(),
right_resolution: self.rig.right.resolution,
}),
})
}
}
/// The webcam receivers, one per side camera.
type SideChannels = (Receiver<Vec<u8>>, Receiver<Vec<u8>>);
/// The Kinect v2 worker: opens the device, then polls packets, decodes,
/// registers, and composes frames with the latest webcam images.
///
/// `ready` receives the effective depth camera (factory intrinsics,
/// config pose) once the device is up — or the open error.
fn kinect_loop(
center_config: Camera,
ready: std::sync::mpsc::Sender<Result<Camera>>,
side_handoff: Receiver<SideChannels>,
frames: SyncSender<PairedFrame>,
shutdown: &Arc<AtomicBool>,
) {
let result = (|| -> Result<()> {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.context("building the Kinect runtime")?;
runtime.block_on(async {
let mut device = kinect_one::DeviceEnumerator::open_default(true)
.await
.map_err(|e| {
anyhow::anyhow!("opening the Kinect failed — is it plugged in? ({e})")
})?;
device
.start()
.await
.map_err(|e| anyhow::anyhow!("starting the Kinect: {e}"))?;
// The depth camera: factory intrinsics at 512x424, pose
// from the offline calibration.
let ir = device.get_ir_params();
let camera = Camera {
id: center_config.id.clone(),
pose: center_config.pose,
intrinsics: CameraIntrinsics {
fx: ir.fx,
fy: ir.fy,
cx: ir.cx,
cy: ir.cy,
},
resolution: UVec2::new(
kinect_one::DEPTH_WIDTH as u32,
kinect_one::DEPTH_HEIGHT as u32,
),
};
ready
.send(Ok(camera))
.map_err(|_| anyhow::anyhow!("the app went away"))?;
let mut registration = kinect_one::processor::Registration::new();
registration.set_ir_params(device.get_ir_params());
registration.set_color_params(device.get_color_params());
let color_processor = kinect_one::processor::color::ZuneColorProcessor::new(
kinect_one::processor::color::ColorSpace::RGB,
)
.map_err(|e| anyhow::anyhow!("color processor: {e}"))?;
use kinect_one::processor::depth::DepthProcessorTrait;
let mut depth_processor = kinect_one::processor::depth::CpuDepthProcessor::new()
.map_err(|e| anyhow::anyhow!("depth processor: {e}"))?;
depth_processor
.set_p0_tables(device.get_p0_tables())
.map_err(|e| anyhow::anyhow!("depth p0 tables: {e}"))?;
depth_processor
.set_ir_params(device.get_ir_params())
.map_err(|e| anyhow::anyhow!("depth ir params: {e}"))?;
// The webcam receivers, handed over once the app has
// opened the side cameras.
let (left_rx, right_rx) = side_handoff
.recv()
.map_err(|_| anyhow::anyhow!("the app went away"))?;
let mut packet_sync = kinect_one::PacketSync::new();
let started = Instant::now();
while !shutdown.load(Ordering::Relaxed) {
if let Ok(Some(packet)) = device.poll_color_packet().await {
packet_sync.push_color_packet(packet);
}
if let Ok(Some(packet)) = device.poll_depth_packet().await {
packet_sync.push_depth_packet(packet);
}
let Some((color_packet, depth_packet)) = packet_sync.poll_packets() else {
continue;
};
use kinect_one::processor::ProcessTrait;
let color_frame = color_packet
.process(&color_processor)
.await
.map_err(|e| anyhow::anyhow!("color decode: {e}"))?;
let depth_frame = depth_packet
.process(&depth_processor)
.await
.map_err(|e| anyhow::anyhow!("depth decode: {e}"))?
.1;
let (registered, undistorted) =
registration.undistort_depth_and_color(&color_frame, &depth_frame, true);
let depth_mm = convert_depth(&undistorted.buffer);
if frames
.send(PairedFrame {
depth_mm,
rgb: registered.buffer,
left: latest(&left_rx),
right: latest(&right_rx),
timestamp_secs: started.elapsed().as_secs_f64(),
})
.is_err()
{
break; // receiver dropped: shut down
}
}
Ok(())
})
})();
if let Err(e) = result {
log::error!("kinect worker: {e:#}");
let _ = ready.send(Err(e));
}
}
/// libfreenect2-style depth (f32 metres, invalid as 0/NaN) → u16
/// millimetres with 0 for invalid.
fn convert_depth(buffer: &[f32]) -> Vec<u16> {
buffer
.iter()
.map(|&d| {
if d.is_finite() && (0.5..=6.0).contains(&d) {
(d * 1000.0).round() as u16
} else {
0
}
})
.collect()
}
/// The most recent frame from a webcam channel, if one arrived.
fn latest(channel: &Receiver<Vec<u8>>) -> Option<Vec<u8>> {
let mut newest = None;
while let Ok(frame) = channel.try_recv() {
newest = Some(frame);
}
newest
}
/// A webcam: a nokhwa [`CallbackCamera`] streaming decoded RGB into a
/// channel from its own capture thread (dropping it stops the stream).
struct Webcam {
/// Held for its side effect: dropping it stops the capture thread.
#[allow(dead_code)]
camera: nokhwa::CallbackCamera,
}
impl Webcam {
fn open(config: &Camera, frames: SyncSender<Vec<u8>>) -> Result<Self> {
use nokhwa::pixel_format::RgbFormat;
use nokhwa::utils::{
CameraFormat, FrameFormat, RequestedFormat, RequestedFormatType, Resolution,
};
// Match the device: numeric id → index, otherwise the
// human-readable name.
let devices = nokhwa::query(nokhwa::utils::ApiBackend::Auto)
.map_err(|e| anyhow::anyhow!("querying cameras: {e}"))?;
let index = device_index(&devices, &config.id)
.with_context(|| format!("finding webcam {:?}", config.id))?;
let format = CameraFormat::new(
Resolution::new(config.resolution.x, config.resolution.y),
FrameFormat::MJPEG,
30,
);
let mut camera = nokhwa::CallbackCamera::new(
index,
RequestedFormat::new::<RgbFormat>(RequestedFormatType::Closest(format)),
move |buffer| {
// Decode on the capture thread and hand the RGB over;
// a full channel just drops the frame (we always want
// the latest).
if let Ok(image) = buffer.decode_image::<RgbFormat>() {
let _ = frames.try_send(image.into_raw());
}
},
)
.map_err(|e| anyhow::anyhow!("opening webcam {:?}: {e}", config.id))?;
// The camera must deliver the configured resolution — the rig
// intrinsics are calibrated for it.
let actual = camera
.camera_format()
.map_err(|e| anyhow::anyhow!("webcam {:?} format: {e}", config.id))?;
if actual.resolution().width_x != config.resolution.x
|| actual.resolution().height_y != config.resolution.y
{
bail!(
"webcam {:?} delivers {}x{}, but the rig is calibrated for {}x{}",
config.id,
actual.resolution().width_x,
actual.resolution().height_y,
config.resolution.x,
config.resolution.y
);
}
camera
.open_stream()
.map_err(|e| anyhow::anyhow!("starting webcam {:?}: {e}", config.id))?;
Ok(Self { camera })
}
}
/// Resolve a configured camera id to a nokhwa device index: a numeric
/// id is the index itself; otherwise it must (uniquely) match a
/// device's human-readable name.
fn device_index(
devices: &[nokhwa::utils::CameraInfo],
id: &str,
) -> Result<nokhwa::utils::CameraIndex> {
// Numeric id: use it directly.
if let Ok(index) = id.parse::<u32>() {
return Ok(nokhwa::utils::CameraIndex::Index(index));
}
let matches: Vec<_> = devices
.iter()
.filter(|d| d.human_name().to_lowercase().contains(&id.to_lowercase()))
.collect();
match matches.as_slice() {
[device] => Ok(device.index().clone()),
[] => bail!(
"no camera name matches {id:?} (have: {:?})",
devices.iter().map(|d| d.human_name()).collect::<Vec<_>>()
),
_ => bail!(
"camera id {id:?} is ambiguous: {:?}",
matches.iter().map(|d| d.human_name()).collect::<Vec<_>>()
),
}
}
#[cfg(test)]
mod tests {
use super::*;
use glam::Vec3;
use glam::uvec2;
use nokhwa::utils::CameraIndex;
use rig::CameraPose;
#[test]
fn depth_conversion_drops_invalid_and_quantizes() {
// libfreenect2 depth: f32 metres, invalid as 0/NaN; out-of-range
// values are dropped too.
let out = convert_depth(&[1.9, 0.0, f32::NAN, 0.25, 6.5, 2.845]);
assert_eq!(out, vec![1900, 0, 0, 0, 0, 2845]);
}
#[test]
fn latest_takes_only_the_newest_frame() {
let (tx, rx) = sync_channel(4);
tx.send(vec![1]).unwrap();
tx.send(vec![2]).unwrap();
tx.send(vec![3]).unwrap();
assert_eq!(latest(&rx), Some(vec![3]));
assert_eq!(latest(&rx), None);
}
#[test]
fn device_matching_by_index_and_name() {
let camera = |index: u32, name: &str| {
nokhwa::utils::CameraInfo::new(name, "", "", CameraIndex::Index(index))
};
let devices = vec![camera(0, "Logitech C920"), camera(1, "Some Other Cam")];
// Numeric id → index.
assert_eq!(device_index(&devices, "1").unwrap(), CameraIndex::Index(1));
// Name match (case-insensitive substring).
assert_eq!(
device_index(&devices, "c920").unwrap(),
CameraIndex::Index(0)
);
// No match.
assert!(device_index(&devices, "nonexistent").is_err());
}
#[test]
fn opening_without_a_kinect_fails_clearly() {
// In the container there is no USB, so opening must fail with
// the "is it plugged in?" error rather than hang or panic.
let rig = rig::Rig {
center: Camera {
id: "kinect".into(),
pose: CameraPose::looking_at(Vec3::ZERO, Vec3::new(0.0, 0.0, 2.0)),
intrinsics: CameraIntrinsics {
fx: 365.0,
fy: 365.0,
cx: 256.0,
cy: 212.0,
},
resolution: uvec2(512, 424),
},
left: Camera {
id: "left".into(),
pose: CameraPose::looking_at(Vec3::new(-0.4, 0.0, 0.0), Vec3::new(0.0, 0.0, 2.0)),
intrinsics: CameraIntrinsics {
fx: 554.0,
fy: 554.0,
cx: 320.0,
cy: 240.0,
},
resolution: uvec2(640, 480),
},
right: Camera {
id: "right".into(),
pose: CameraPose::looking_at(Vec3::new(0.4, 0.0, 0.0), Vec3::new(0.0, 0.0, 2.0)),
intrinsics: CameraIntrinsics {
fx: 554.0,
fy: 554.0,
cx: 320.0,
cy: 240.0,
},
resolution: uvec2(640, 480),
},
};
let err = match KinectRigSource::open(&rig) {
Ok(_) => panic!("opened a rig with no Kinect in the container?"),
Err(e) => e,
};
assert!(
err.to_string().to_lowercase().contains("kinect"),
"unexpected error: {err:#}"
);
}
}
render/tests/render.rs +56 -4
diff --git a/render/tests/render.rs b/render/tests/render.rs
index d559b7f..16e19a2 100644
@@ -14,7 +14,7 @@ use glam::{Vec2, uvec2, vec3};
use render::{RenderStats, SplatRenderer};
use replay::synth::{self, NoiseParams, SUBJECT};
use rig::{Camera, FrameSource, SplatBatch};
use splatgen::{SplatGen, SplatSource};
use splatgen::{ProxyParams, SplatGen, SplatGenParams, SplatSource};
/// Small enough to stay fast on software Vulkan, big enough that the
/// subject spans several pixels.
@@ -36,9 +36,9 @@ fn head_center() -> glam::Vec3 {
const SKIN: [f32; 3] = [0.85, 0.62, 0.52];
fn first_frame_with(noise: NoiseParams) -> SplatBatch {
let rig = synth::dev_rig_with_resolutions(uvec2(CENTER.0, CENTER.1), uvec2(SIDE.0, SIDE.1));
let raw = synth::dev_source_with(rig.center, SUBJECT, 30.0).with_noise(noise);
let mut source = SplatSource::new(raw, SplatGen::default());
let rig = dev_rig();
let raw = synth::dev_source_with(&rig, SUBJECT, 30.0).with_noise(noise);
let mut source = SplatSource::new(rig, raw, SplatGen::default());
source.next_frame().expect("synthetic frame").splats
}
@@ -46,6 +46,11 @@ fn first_frame() -> SplatBatch {
first_frame_with(NoiseParams::OFF)
}
/// The dev rig at the test resolutions.
fn dev_rig() -> rig::Rig {
synth::dev_rig_with_resolutions(uvec2(CENTER.0, CENTER.1), uvec2(SIDE.0, SIDE.1))
}
/// The mean of pixels within `tolerance` of a colour, in image
/// coordinates; `None` if too few match.
fn color_centroid(image: &render::RenderedImage, color: [f32; 3], tolerance: f32) -> Option<Vec2> {
@@ -218,6 +223,53 @@ async fn noisy_frames_render_like_clean_ones() {
}
#[tokio::test]
async fn proxy_flanks_show_side_camera_colours() {
// The plan's occlusion proxy, end to end: with the proxy on, the
// virtual camera at the left webcam sees the subject's left flank
// in the *left camera's* tint — geometry the Kinect never measured.
// With the proxy off, those tint pixels barely exist (the depth
// surfels near the silhouette are edge-on disks).
let rig = synth::dev_rig_with_resolutions(uvec2(128, 106), uvec2(320, 240));
let make_batch = |proxy: bool| {
let raw = synth::dev_source_with(&rig, SUBJECT, 30.0).with_noise(NoiseParams::OFF);
let params = SplatGenParams {
proxy: ProxyParams {
enabled: proxy,
..Default::default()
},
..Default::default()
};
let mut source = SplatSource::new(rig.clone(), raw, SplatGen::new(params));
source.next_frame().expect("frame").splats
};
let camera = rig.virtual_camera(0.0, rig.left.intrinsics, rig.left.resolution);
async fn count_tinted(batch: &SplatBatch, camera: &rig::Camera) -> usize {
let (image, _) = renderer()
.render_async(batch, camera)
.await
.expect("render");
image
.rgba
.as_chunks::<4>()
.0
.iter()
.filter(|px| {
let (r, g, b) = (px[0] as i32, px[1] as i32, px[2] as i32);
r > 140 && r > 2 * g && r > 2 * b
})
.count()
}
let without_proxy = count_tinted(&make_batch(false), &camera).await;
let with_proxy = count_tinted(&make_batch(true), &camera).await;
assert!(
with_proxy > without_proxy * 2 + 10,
"proxy must reveal side-camera colours: {with_proxy} vs {without_proxy} tinted pixels"
);
}
#[tokio::test]
async fn empty_batch_renders_the_background() {
let rig = synth::dev_rig_with_resolutions(uvec2(CENTER.0, CENTER.1), uvec2(SIDE.0, SIDE.1));
let camera: Camera = rig.virtual_camera(0.5, rig.left.intrinsics, rig.left.resolution);
replay/src/recording.rs +121 -29
diff --git a/replay/src/recording.rs b/replay/src/recording.rs
index b1f6e17..294849a 100644
@@ -4,18 +4,21 @@
//!
//! ```text
//! my-recording/
//! ├── recording.toml # the depth camera (id, pose, intrinsics,
//! │ # resolution), fps, frame count
//! ├── recording.toml # the rig's cameras (center + optional left /
//! │ # right), fps, frame count
//! ├── frame_0000.depth # raw little-endian u16 depth, width*height
//! ├── frame_0000.rgb # raw RGB bytes, width*height*3
//! ├── frame_0001.depth
//! ├── frame_0000.rgb # raw RGB bytes, width*height*3 (registered)
//! ├── frame_0000.left # optional: left webcam RGB, w*h*3
//! ├── frame_0000.right # optional: right webcam RGB, w*h*3
//! └── ...
//! ```
//!
//! Real Kinect recordings can be converted into this shape; for dev and
//! Real rig recordings can be converted into this shape; for dev and
//! CI, [`Recording::write`] produces recordings from the synthetic
//! source. [`RecordingSource`] plays them back as a [`DepthSource`],
//! looping forever.
//! looping forever. The full recorded rig (center + side cameras) is
//! available via [`RecordingSource::rig`] — the pipeline needs the side
//! cameras for the occlusion proxy.
use std::fs;
use std::path::{Path, PathBuf};
@@ -23,17 +26,22 @@ use std::path::{Path, PathBuf};
use anyhow::{Context, Result, bail};
use serde::{Deserialize, Serialize};
use rig::{Camera, CameraIntrinsics, CameraPose, DepthFrame, DepthSource};
use rig::{Camera, DepthFrame, DepthSource, Rig, SideRgb};
/// `recording.toml`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Manifest {
pub camera: CameraManifest,
/// The depth (center) camera.
pub center: CameraManifest,
/// The left webcam, if the recording has side RGB.
pub left: Option<CameraManifest>,
/// The right webcam, if the recording has side RGB.
pub right: Option<CameraManifest>,
pub fps: f32,
pub frames: u32,
}
/// The depth camera a recording was made with.
/// A camera a recording was made with.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CameraManifest {
pub id: String,
@@ -68,11 +76,11 @@ impl From<CameraManifest> for Camera {
fn from(manifest: CameraManifest) -> Self {
Self {
id: manifest.id,
pose: CameraPose {
pose: rig::CameraPose {
position: manifest.position.into(),
rotation: glam::Quat::from_array(manifest.rotation),
},
intrinsics: CameraIntrinsics {
intrinsics: rig::CameraIntrinsics {
fx: manifest.intrinsics[0],
fy: manifest.intrinsics[1],
cx: manifest.intrinsics[2],
@@ -91,14 +99,18 @@ fn rgb_file(frame: u32) -> String {
format!("frame_{frame:04}.rgb")
}
fn side_file(frame: u32, side: &str) -> String {
format!("frame_{frame:04}.{side}")
}
/// Writes recordings: [`Recording::write`].
pub struct Recording;
impl Recording {
/// Record `frames` frames from `source` into `dir`, replacing it.
///
/// The source must yield exactly the frame stream worth keeping;
/// its first `frames` frames become the recording.
/// The rig (center + side cameras) comes from the source; side RGB
/// is stored when the source provides it.
pub fn write(
dir: impl AsRef<Path>,
source: &mut dyn DepthSource,
@@ -107,8 +119,12 @@ impl Recording {
) -> Result<()> {
let dir = dir.as_ref();
fs::create_dir_all(dir).with_context(|| format!("creating {}", dir.display()))?;
let rig = source.rig();
let manifest = Manifest {
camera: CameraManifest::from(source.camera()),
center: CameraManifest::from(&rig.center),
left: Some(CameraManifest::from(&rig.left)),
right: Some(CameraManifest::from(&rig.right)),
fps,
frames,
};
@@ -129,6 +145,10 @@ impl Recording {
.collect();
fs::write(dir.join(depth_file(frame)), depth_le)?;
fs::write(dir.join(rgb_file(frame)), &depth_frame.rgb)?;
if let Some(sides) = &depth_frame.side_rgb {
fs::write(dir.join(side_file(frame, "left")), &sides.left)?;
fs::write(dir.join(side_file(frame, "right")), &sides.right)?;
}
}
Ok(())
}
@@ -138,9 +158,10 @@ impl Recording {
#[derive(Debug)]
pub struct RecordingSource {
dir: PathBuf,
camera: Camera,
rig: Rig,
fps: f32,
frames: u32,
has_sides: bool,
frame_index: u32,
}
@@ -162,10 +183,45 @@ impl RecordingSource {
bail!("recording in {} has non-positive fps", dir.display());
}
let has_left = manifest.left.is_some();
let has_right = manifest.right.is_some();
let rig = Rig {
center: manifest.center.into(),
left: manifest.left.map(Into::into).unwrap_or_else(|| Camera {
id: "left".into(),
pose: rig::CameraPose::identity(),
intrinsics: rig::CameraIntrinsics {
fx: 1.0,
fy: 1.0,
cx: 0.0,
cy: 0.0,
},
resolution: glam::UVec2::ONE,
}),
right: manifest.right.map(Into::into).unwrap_or_else(|| Camera {
id: "right".into(),
pose: rig::CameraPose::identity(),
intrinsics: rig::CameraIntrinsics {
fx: 1.0,
fy: 1.0,
cx: 0.0,
cy: 0.0,
},
resolution: glam::UVec2::ONE,
}),
};
// Side RGB is present only when both cameras and both files for
// frame 0 exist.
let has_sides = has_left
&& has_right
&& dir.join(side_file(0, "left")).exists()
&& dir.join(side_file(0, "right")).exists();
Ok(Self {
camera: manifest.camera.into(),
rig,
fps: manifest.fps,
frames: manifest.frames,
has_sides,
dir,
frame_index: 0,
})
@@ -180,18 +236,27 @@ impl RecordingSource {
pub fn fps(&self) -> f32 {
self.fps
}
/// The recorded rig: center + side cameras.
pub fn rig(&self) -> &Rig {
&self.rig
}
}
impl DepthSource for RecordingSource {
fn camera(&self) -> &Camera {
&self.camera
&self.rig.center
}
fn rig(&self) -> Rig {
self.rig.clone()
}
fn next_depth_frame(&mut self) -> Result<DepthFrame> {
let frame = self.frame_index % self.frames;
self.frame_index += 1;
let (width, height) = (self.camera.resolution.x, self.camera.resolution.y);
let (width, height) = (self.rig.center.resolution.x, self.rig.center.resolution.y);
let depth_path = self.dir.join(depth_file(frame));
let rgb_path = self.dir.join(rgb_file(frame));
@@ -224,10 +289,36 @@ impl DepthSource for RecordingSource {
.map(|b| u16::from_le_bytes(*b))
.collect();
let side_rgb = if self.has_sides {
let read_side = |side: &str, camera: &Camera| -> Result<Vec<u8>> {
let path = self.dir.join(side_file(frame, side));
let bytes =
fs::read(&path).with_context(|| format!("reading {}", path.display()))?;
let expected = (camera.resolution.x * camera.resolution.y) as usize * 3;
if bytes.len() != expected {
bail!(
"{}: expected {expected} rgb bytes, got {}",
path.display(),
bytes.len()
);
}
Ok(bytes)
};
Some(SideRgb {
left: read_side("left", &self.rig.left)?,
left_resolution: self.rig.left.resolution,
right: read_side("right", &self.rig.right)?,
right_resolution: self.rig.right.resolution,
})
} else {
None
};
Ok(DepthFrame {
timestamp_secs: frame as f64 / self.fps as f64,
depth_mm,
rgb: rgb_bytes,
side_rgb,
})
}
}
@@ -241,7 +332,7 @@ mod tests {
/// A tiny clean source for exact comparisons.
fn tiny_clean_source() -> synth::SyntheticSource {
let rig = synth::dev_rig_with_resolutions(uvec2(32, 24), uvec2(64, 48));
synth::dev_source_with(rig.center, synth::SUBJECT, 30.0).with_noise(NoiseParams::OFF)
synth::dev_source_with(&rig, synth::SUBJECT, 30.0).with_noise(NoiseParams::OFF)
}
fn temp_dir(name: &str) -> PathBuf {
@@ -252,7 +343,7 @@ mod tests {
}
#[test]
fn round_trip_preserves_frames() {
fn round_trip_preserves_frames_and_rig() {
let dir = temp_dir("roundtrip");
let mut source = tiny_clean_source();
Recording::write(&dir, &mut source, 30.0, 3).expect("write recording");
@@ -260,15 +351,21 @@ mod tests {
let mut player = RecordingSource::load(&dir).expect("load recording");
assert_eq!(player.frames(), 3);
assert_eq!(player.fps(), 30.0);
assert_eq!(player.camera(), source.camera());
// The recorded frames match what the source produced, in order.
// The recorded rig round-trips: center + side cameras.
let recorded = player.rig();
assert_eq!(recorded.center.pose, source.rig().center.pose);
assert_eq!(recorded.left.pose, source.rig().left.pose);
assert_eq!(recorded.right.intrinsics, source.rig().right.intrinsics);
// The recorded frames match what the source produced, in order,
// including the side RGB.
let mut source = tiny_clean_source();
for _ in 0..3 {
let recorded = player.next_depth_frame().expect("frame");
let original = source.next_depth_frame().expect("frame");
assert_eq!(recorded.depth_mm, original.depth_mm);
assert_eq!(recorded.rgb, original.rgb);
assert_eq!(recorded.side_rgb, original.side_rgb);
assert_eq!(recorded.timestamp_secs, original.timestamp_secs);
}
@@ -287,12 +384,7 @@ mod tests {
// A manifest with frames but no frame files.
let dir = temp_dir("missing-frames");
fs::create_dir_all(&dir).expect("dir");
fs::write(
dir.join("recording.toml"),
r#"
[[fake]]"#,
)
.expect("write");
fs::write(dir.join("recording.toml"), "[[fake]]").expect("write");
let err = RecordingSource::load(&dir).expect_err("bad manifest");
assert!(err.to_string().contains("parsing"));
fs::remove_dir_all(&dir).ok();
replay/src/scene.rs +155 -92
diff --git a/replay/src/scene.rs b/replay/src/scene.rs
index 5b29e72..e84e2e9 100644
@@ -2,11 +2,13 @@
//!
//! Surfaces are simple enough to intersect exactly (ellipsoids, planes)
//! yet shaped like the real subject — a head, shoulders, and a backdrop —
//! so surfel construction, the virtual camera, and the UI can be
//! developed and tested against known ground truth.
//! so surfel construction, the virtual camera, the occlusion proxy, and
//! the UI can be developed and tested against known ground truth.
use glam::Vec3;
use rig::Ellipsoid;
/// A ray-surface intersection.
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Hit {
@@ -23,12 +25,28 @@ pub struct Hit {
/// A scene surface.
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum Surface {
/// An axis-aligned ellipsoid (sphere with per-axis radii).
/// An axis-aligned ellipsoid (sphere with per-axis radii) in a
/// single solid colour.
Ellipsoid {
center: Vec3,
radii: Vec3,
color: [f32; 3],
},
/// An ellipsoid whose left and right flanks carry a distinct tint:
/// beyond `tint_from` fractions of the x radius from the centre,
/// the colour switches to the side tint. Those flanks are what the
/// side webcams see but the center camera barely does — the ground
/// truth for the ellipsoid-proxy recolouring.
TintedEllipsoid {
center: Vec3,
radii: Vec3,
base: [f32; 3],
left_tint: [f32; 3],
right_tint: [f32; 3],
/// Flank tinting starts beyond this fraction of the x radius,
/// in `[0, 1]`.
tint_from: f32,
},
/// A backdrop plane with a checkerboard pattern, `cell` metres per
/// cell (0 disables the pattern: flat colour).
CheckerPlane {
@@ -53,32 +71,35 @@ impl Surface {
radii,
color,
} => {
// Scale space so the ellipsoid becomes the unit sphere.
let o = (origin - center) / radii;
let d = dir / radii;
// Solve |o + t d|² = 1.
let a = d.dot(d);
let b = 2.0 * o.dot(d);
let c = o.dot(o) - 1.0;
let disc = b * b - 4.0 * a * c;
if disc < 0.0 || a <= f32::EPSILON {
return None;
}
let sqrt_disc = disc.sqrt();
// Near root; for a ray starting inside the ellipsoid it is
// negative, so fall back to the far (exit) root.
let mut t = (-b - sqrt_disc) / (2.0 * a);
if t <= f32::EPSILON {
t = (-b + sqrt_disc) / (2.0 * a);
}
if t <= f32::EPSILON {
return None;
}
let ellipsoid = Ellipsoid::new(center, radii);
let (t, normal) = ellipsoid.intersect_ray(origin, dir)?;
Some(Hit {
t,
position: origin + dir * t,
normal,
color,
})
}
Surface::TintedEllipsoid {
center,
radii,
base,
left_tint,
right_tint,
tint_from,
} => {
let ellipsoid = Ellipsoid::new(center, radii);
let (t, normal) = ellipsoid.intersect_ray(origin, dir)?;
let position = origin + dir * t;
// Normal: gradient of the implicit function at the hit,
// mapped back to world space.
let grad = (position - center) / (radii * radii);
let normal = grad.normalize_or_zero();
// Tint by the hit's offset along the local x axis.
let dx = (position.x - center.x) / radii.x;
let color = if dx < -tint_from {
left_tint
} else if dx > tint_from {
right_tint
} else {
base
};
Some(Hit {
t,
position,
@@ -124,56 +145,7 @@ impl Surface {
}
}
}
}
/// In-plane coordinates of a point on a plane with the given normal,
/// for the checkerboard pattern.
fn plane_coords(p: Vec3, normal: Vec3) -> (f32, f32) {
// Build any orthonormal tangent basis.
let reference = if normal.abs().dot(Vec3::Y) > 0.999 {
Vec3::X
} else {
Vec3::Y
};
let t1 = normal.cross(reference).normalize_or_zero();
let t2 = normal.cross(t1).normalize_or_zero();
(p.dot(t1), p.dot(t2))
}
/// The default development scene's subject: a head-and-shoulders stand-in
/// at `subject`, facing the rig's center camera at the origin.
///
/// Returns `(head, shoulders)`; the backdrop is [`backdrop`].
pub fn head_and_shoulders(subject: Vec3) -> (Surface, Surface) {
(
// Head: slightly egg-shaped ellipsoid, ~22 cm tall.
Surface::Ellipsoid {
center: subject + Vec3::new(0.0, 0.18, 0.0),
radii: Vec3::new(0.09, 0.11, 0.10),
color: [0.85, 0.62, 0.52],
},
// Shoulders.
Surface::Ellipsoid {
center: subject + Vec3::new(0.0, -0.05, 0.0),
radii: Vec3::new(0.24, 0.13, 0.16),
color: [0.20, 0.35, 0.55],
},
)
}
/// The default development scene's backdrop: a checkerboard plane
/// `0.8 m` behind the subject, facing the rig.
pub fn backdrop(subject: Vec3) -> Surface {
Surface::CheckerPlane {
point: subject + Vec3::new(0.0, 0.0, 0.8),
normal: Vec3::new(0.0, 0.0, -1.0),
cell: 0.25,
color_a: [0.72, 0.72, 0.72],
color_b: [0.28, 0.28, 0.28],
}
}
impl Surface {
/// The surface translated by `offset`.
pub fn translated(&self, offset: Vec3) -> Self {
match *self {
@@ -186,6 +158,21 @@ impl Surface {
radii,
color,
},
Surface::TintedEllipsoid {
center,
radii,
base,
left_tint,
right_tint,
tint_from,
} => Surface::TintedEllipsoid {
center: center + offset,
radii,
base,
left_tint,
right_tint,
tint_from,
},
Surface::CheckerPlane {
point,
normal,
@@ -203,12 +190,35 @@ impl Surface {
}
}
/// In-plane coordinates of a point on a plane with the given normal,
/// for the checkerboard pattern.
fn plane_coords(p: Vec3, normal: Vec3) -> (f32, f32) {
// Build any orthonormal tangent basis.
let reference = if normal.abs().dot(Vec3::Y) > 0.999 {
Vec3::X
} else {
Vec3::Y
};
let t1 = normal.cross(reference).normalize_or_zero();
let t2 = normal.cross(t1).normalize_or_zero();
(p.dot(t1), p.dot(t2))
}
/// A synthetic scene: a list of surfaces, cast against per pixel.
#[derive(Debug, Clone, Default, PartialEq)]
pub struct Scene {
pub surfaces: Vec<Surface>,
}
/// The flanks of the synthetic head (the region tinted for the side
/// cameras) start beyond this fraction of the head's x radius.
pub const HEAD_TINT_FROM: f32 = 0.6;
/// The synthetic subject's head colour and its side tints.
pub const HEAD_BASE: [f32; 3] = [0.85, 0.62, 0.52];
pub const HEAD_LEFT_TINT: [f32; 3] = [0.85, 0.05, 0.05];
pub const HEAD_RIGHT_TINT: [f32; 3] = [0.05, 0.05, 0.85];
impl Scene {
pub fn new() -> Self {
Self::default()
@@ -220,19 +230,54 @@ impl Scene {
self
}
/// The default development scene: a head-and-shoulders stand-in in
/// front of a checkerboard backdrop, centred at `subject` (roughly
/// where the real subject sits, PLAN.md pipeline step 1).
/// The default development scene's subject: a head-and-shoulders
/// stand-in at `subject`, facing the rig's center camera at the
/// origin. The head's flanks carry the side tints (see
/// [`Surface::TintedEllipsoid`]).
///
/// The subject faces the rig's center camera at the origin (i.e. its
/// face points towards -Z). For the animated version used by
/// [`crate::synth`], see [`head_and_shoulders`] and [`backdrop`].
/// Returns `(head, shoulders)`; the backdrop is [`backdrop`].
pub fn head_and_shoulders(subject: Vec3) -> (Surface, Surface) {
(
// Head: slightly egg-shaped ellipsoid, ~22 cm tall, with
// tinted flanks for the proxy ground truth.
Surface::TintedEllipsoid {
center: subject + Vec3::new(0.0, 0.18, 0.0),
radii: Vec3::new(0.09, 0.11, 0.10),
base: HEAD_BASE,
left_tint: HEAD_LEFT_TINT,
right_tint: HEAD_RIGHT_TINT,
tint_from: HEAD_TINT_FROM,
},
// Shoulders.
Surface::Ellipsoid {
center: subject + Vec3::new(0.0, -0.05, 0.0),
radii: Vec3::new(0.24, 0.13, 0.16),
color: [0.20, 0.35, 0.55],
},
)
}
/// The default development scene's backdrop: a checkerboard plane
/// `0.8 m` behind the subject, facing the rig.
pub fn backdrop(subject: Vec3) -> Surface {
Surface::CheckerPlane {
point: subject + Vec3::new(0.0, 0.0, 0.8),
normal: Vec3::new(0.0, 0.0, -1.0),
cell: 0.25,
color_a: [0.72, 0.72, 0.72],
color_b: [0.28, 0.28, 0.28],
}
}
/// The default development scene: head, shoulders, backdrop, all
/// centred at `subject`. See [`head_and_shoulders`] for the
/// animated variant used by the sources.
pub fn head_stand_in(subject: Vec3) -> Self {
let (head, shoulders) = head_and_shoulders(subject);
let (head, shoulders) = Self::head_and_shoulders(subject);
Self::new()
.with(head)
.with(shoulders)
.with(backdrop(subject))
.with(Self::backdrop(subject))
}
/// Cast a ray against the scene; the nearest surface hit.
@@ -262,13 +307,31 @@ mod tests {
.expect("head in the way");
// The hit is on the ellipsoid's near side, on its surface.
assert!(hit.position.distance(head_center) < 0.11);
assert!(
hit.position.z < head_center.z - 0.05,
"must hit the near side, hit z = {}",
hit.position.z
);
// Normal faces back at the camera (front face of the ellipsoid).
assert!(hit.normal.dot(hit.position) < 0.0);
assert!(hit.position.z < head_center.z, "must hit the near side");
// Normal faces the camera (front face of the ellipsoid).
assert!(hit.normal.dot(hit.position - Vec3::ZERO) < 0.0);
}
#[test]
fn head_flanks_are_tinted() {
// Rays aimed straight at the head's left and right flanks (beyond
// HEAD_TINT_FROM of the x radius) hit tinted colour; the front
// is the base colour.
let head_center = vec3(0.0, 0.18, 2.0);
let left = scene()
.cast_ray(vec3(-3.0, 0.18, 2.0), vec3(1.0, 0.0, 0.0))
.expect("left flank hit");
assert_eq!(left.color, HEAD_LEFT_TINT);
assert!(left.position.x < head_center.x - 0.09 * HEAD_TINT_FROM);
let right = scene()
.cast_ray(vec3(3.0, 0.18, 2.0), vec3(-1.0, 0.0, 0.0))
.expect("right flank hit");
assert_eq!(right.color, HEAD_RIGHT_TINT);
assert!(right.position.x > head_center.x + 0.09 * HEAD_TINT_FROM);
let front = scene()
.cast_ray(Vec3::ZERO, head_center)
.expect("front hit");
assert_eq!(front.color, HEAD_BASE);
}
#[test]
replay/src/synth.rs +105 -67
diff --git a/replay/src/synth.rs b/replay/src/synth.rs
index 96bc3b5..77880a2 100644
@@ -1,9 +1,10 @@
//! The synthetic depth-camera: a software Kinect.
//!
//! Every frame, [`SyntheticSource`] raycasts the scene from the rig's
//! center viewpoint — one ray per depth pixel — and emits a raw
//! [`DepthFrame`]: a millimetre depth map plus RGB, like a Kinect v2
//! reports them. `splatgen` then turns those frames into surfels, so the
//! Every frame, [`SyntheticSource`] raycasts the scene — one ray per
//! depth pixel — and emits a raw [`DepthFrame`]: a millimetre depth map
//! plus RGB from the center (Kinect) viewpoint, *and* RGB from the two
//! side webcams, exactly what the live rig produces. `splatgen` then
//! turns those frames into surfels and occlusion-proxy splats, so the
//! dev/CI path exercises the real depth-processing pipeline.
//!
//! Realism knobs: the source injects deterministic sensor noise
@@ -13,23 +14,21 @@
//! stays deterministic.
//!
//! The subject sways a little, also deterministically, so temporal
//! behaviour is testable.
//! behaviour is testable. The head's flanks carry distinct tints (see
//! [`crate::scene`]), visible mainly to the side cameras — ground truth
//! for the ellipsoid-proxy recolouring.
use anyhow::Result;
use glam::{UVec2, Vec3};
use rig::{Camera, CameraIntrinsics, CameraPose, DepthFrame, DepthSource};
use rig::{Camera, DepthFrame, DepthSource, Rig, SideRgb};
use crate::scene::{self, Scene, Surface};
use crate::scene::{Scene, Surface};
/// Where the synthetic subject sits: 2 m in front of the rig's center
/// camera at the origin.
pub const SUBJECT: Vec3 = Vec3::new(0.0, 0.0, 2.0);
/// Opacity used for synthetic surfels (PLAN.md: ~0.9–0.99). Applied by
/// `splatgen`, kept here as the dataset's declared intent.
pub const SURFEL_OPACITY: f32 = 0.95;
/// Subject sway amplitude, metres, and period in frames.
const SWAY_AMPLITUDE_M: f32 = 0.04;
const SWAY_PERIOD_FRAMES: u64 = 90;
@@ -84,6 +83,13 @@ fn noise01(frame: u64, x: u32, y: u32, salt: u32) -> f32 {
h as f32 / u32::MAX as f32
}
/// Center-camera focal ratios (per resolution pixel): a ~70° horizontal
/// and 60° vertical field of view, like the Kinect v2 depth sensor.
const CENTER_FX: f32 = 0.714_074; // 1 / (2 * tan(35°)), FOV_x = 70°
const CENTER_FY: f32 = 0.866_025_4; // 1 / (2 * tan(30°)), FOV_y = 60°
/// Side-camera focal ratio: a 60° horizontal FOV, square pixels.
const SIDE_FX: f32 = 0.866_025_4; // 1 / (2 * tan(30°))
/// The development rig: a Kinect-like center camera flanked by two
/// webcams 0.8 m apart, converged on [`SUBJECT`].
///
@@ -95,8 +101,8 @@ pub fn dev_rig_with_resolutions(center: UVec2, side: UVec2) -> Rig {
// Kinect v2 depth: ~70° x 60° field of view.
let center = Camera {
id: "kinect".into(),
pose: CameraPose::looking_at(Vec3::ZERO, SUBJECT),
intrinsics: CameraIntrinsics {
pose: rig::CameraPose::looking_at(Vec3::ZERO, SUBJECT),
intrinsics: rig::CameraIntrinsics {
fx: CENTER_FX * center.x as f32,
fy: CENTER_FY * center.y as f32,
cx: center.x as f32 / 2.0,
@@ -107,8 +113,8 @@ pub fn dev_rig_with_resolutions(center: UVec2, side: UVec2) -> Rig {
// Typical webcams with a 60 degree horizontal FOV.
let make_side = |x: f32| Camera {
id: if x < 0.0 { "left" } else { "right" }.to_owned(),
pose: CameraPose::looking_at(Vec3::new(x, 0.0, 0.0), SUBJECT),
intrinsics: CameraIntrinsics {
pose: rig::CameraPose::looking_at(Vec3::new(x, 0.0, 0.0), SUBJECT),
intrinsics: rig::CameraIntrinsics {
fx: SIDE_FX * side.x as f32,
fy: SIDE_FX * side.y as f32,
cx: side.x as f32 / 2.0,
@@ -123,15 +129,6 @@ pub fn dev_rig_with_resolutions(center: UVec2, side: UVec2) -> Rig {
}
}
/// Center-camera focal ratios (per resolution pixel): a ~70° horizontal
/// and 60° vertical field of view, like the Kinect v2 depth sensor.
const CENTER_FX: f32 = 0.714_074; // 1 / (2 * tan(35°)), FOV_x = 70°
const CENTER_FY: f32 = 0.866_025_4; // 1 / (2 * tan(30°)), FOV_y = 60°
/// Side-camera focal ratio: a 60° horizontal FOV, square pixels.
const SIDE_FX: f32 = 0.866_025_4; // 1 / (2 * tan(30°))
use rig::Rig;
/// The development rig at full size: Kinect-like depth at 256x212
/// (~54k depth pixels, inside the plan's 50–150k splat budget) and
/// 640x480 webcams.
@@ -142,27 +139,23 @@ pub fn dev_rig() -> Rig {
/// A synthetic source over the default development scene (see
/// [`dev_rig`]), at `fps` frames per second.
pub fn dev_source(fps: f32) -> SyntheticSource {
dev_source_with_center_camera(dev_rig().center, fps)
dev_source_with(&dev_rig(), SUBJECT, fps)
}
/// Like [`dev_source`], but with a custom center (depth) camera and
/// subject position — what the configurable app uses.
pub fn dev_source_with(center: Camera, subject: Vec3, fps: f32) -> SyntheticSource {
let (head, shoulders) = scene::head_and_shoulders(subject);
/// A synthetic source over the default development scene with a custom
/// rig and subject position — what the configurable app uses. The rig's
/// center camera is the depth viewpoint; its side cameras provide the
/// side RGB.
pub fn dev_source_with(rig: &Rig, subject: Vec3, fps: f32) -> SyntheticSource {
let (head, shoulders) = Scene::head_and_shoulders(subject);
SyntheticSource::new(
Scene::new().with(scene::backdrop(subject)),
Scene::new().with(Scene::backdrop(subject)),
vec![head, shoulders],
center,
rig.clone(),
fps,
)
}
/// Like [`dev_source`], but with a custom center (depth) camera — e.g.
/// `dev_rig_with_resolutions(..).center` at a small resolution for tests.
pub fn dev_source_with_center_camera(center: Camera, fps: f32) -> SyntheticSource {
dev_source_with(center, SUBJECT, fps)
}
/// A synthetic depth-camera frame source.
pub struct SyntheticSource {
/// Static surroundings (backdrop).
@@ -170,19 +163,20 @@ pub struct SyntheticSource {
/// The animated subject surfaces, translated by the sway offset each
/// frame.
subject: Vec<Surface>,
/// The depth viewpoint: the rig's center camera.
camera: Camera,
/// The whole rig: the center camera is the depth viewpoint, the
/// side cameras provide side RGB.
rig: Rig,
fps: f32,
noise: NoiseParams,
frame_index: u64,
}
impl SyntheticSource {
pub fn new(scene: Scene, subject: Vec<Surface>, camera: Camera, fps: f32) -> Self {
pub fn new(scene: Scene, subject: Vec<Surface>, rig: Rig, fps: f32) -> Self {
Self {
scene,
subject,
camera,
rig,
fps: fps.max(1.0),
noise: NoiseParams::default(),
frame_index: 0,
@@ -202,7 +196,7 @@ impl SyntheticSource {
/// The depth viewpoint: the rig's center camera.
pub fn center(&self) -> &Camera {
&self.camera
&self.rig.center
}
/// The deterministic subject sway offset for a given frame.
@@ -223,19 +217,39 @@ impl SyntheticSource {
scene
}
/// The colour image one camera sees: RGB, 3 bytes per pixel.
fn cast_rgb(&self, scene: &Scene, camera: &Camera) -> Vec<u8> {
let (width, height) = (camera.resolution.x, camera.resolution.y);
let mut rgb = vec![0_u8; (width * height * 3) as usize];
for y in 0..height {
for x in 0..width {
let (origin, dir) = camera.pixel_ray(x as f32 + 0.5, y as f32 + 0.5);
let Some(hit) = scene.cast_ray(origin, dir) else {
continue;
};
let idx = (y * width + x) as usize;
rgb[idx * 3] = (hit.color[0] * 255.0) as u8;
rgb[idx * 3 + 1] = (hit.color[1] * 255.0) as u8;
rgb[idx * 3 + 2] = (hit.color[2] * 255.0) as u8;
}
}
rgb
}
/// Raycast one frame: the clean depth map and RGB the software
/// Kinect sees.
fn cast_frame(&self, frame: u64) -> (Vec<u16>, Vec<u8>) {
let scene = self.frame_scene(frame);
let (width, height) = (self.camera.resolution.x, self.camera.resolution.y);
let camera_from_world = self.camera.pose.camera_from_world();
let camera = &self.rig.center;
let (width, height) = (camera.resolution.x, camera.resolution.y);
let camera_from_world = camera.pose.camera_from_world();
let mut depth = vec![0_u16; (width * height) as usize];
let mut rgb = vec![0_u8; (width * height * 3) as usize];
for y in 0..height {
for x in 0..width {
// Sample through pixel centres.
let (origin, dir) = self.camera.pixel_ray(x as f32 + 0.5, y as f32 + 0.5);
let (origin, dir) = camera.pixel_ray(x as f32 + 0.5, y as f32 + 0.5);
let Some(hit) = scene.cast_ray(origin, dir) else {
continue; // no return: depth stays 0
};
@@ -257,7 +271,7 @@ impl SyntheticSource {
if self.noise == NoiseParams::OFF {
return;
}
let width = self.camera.resolution.x;
let width = self.rig.center.resolution.x;
for (i, d) in depth.iter_mut().enumerate() {
if *d == 0 {
continue; // invalid pixels stay invalid
@@ -276,21 +290,33 @@ impl SyntheticSource {
}
}
/// Build the raw depth frame for `frame`.
/// Build the raw depth frame for `frame`, including the side RGB.
fn build_frame(&self, frame: u64) -> DepthFrame {
let (mut depth, rgb) = self.cast_frame(frame);
self.add_noise(&mut depth, frame);
let scene = self.frame_scene(frame);
let side_rgb = SideRgb {
left: self.cast_rgb(&scene, &self.rig.left),
left_resolution: self.rig.left.resolution,
right: self.cast_rgb(&scene, &self.rig.right),
right_resolution: self.rig.right.resolution,
};
DepthFrame {
timestamp_secs: frame as f64 / self.fps as f64,
depth_mm: depth,
rgb,
side_rgb: Some(side_rgb),
}
}
}
impl DepthSource for SyntheticSource {
fn camera(&self) -> &Camera {
&self.camera
&self.rig.center
}
fn rig(&self) -> Rig {
self.rig.clone()
}
fn next_depth_frame(&mut self) -> Result<DepthFrame> {
@@ -309,21 +335,8 @@ mod tests {
/// A small-resolution source so tests stay fast: 32x24 depth pixels.
fn tiny_source() -> SyntheticSource {
let (head, shoulders) = scene::head_and_shoulders(SUBJECT);
let mut center = dev_rig().center;
// Keep the center camera's FOV, shrink the resolution.
let scale = 32.0 / center.resolution.x as f32;
center.intrinsics.fx *= scale;
center.intrinsics.fy *= scale;
center.intrinsics.cx = 16.0;
center.intrinsics.cy = 12.0;
center.resolution = uvec2(32, 24);
SyntheticSource::new(
Scene::new().with(scene::backdrop(SUBJECT)),
vec![head, shoulders],
center,
30.0,
)
let rig = dev_rig_with_resolutions(uvec2(32, 24), uvec2(64, 48));
dev_source_with(&rig, SUBJECT, 30.0)
}
/// The straight-ahead ray passes below the head and hits the front
@@ -363,6 +376,33 @@ mod tests {
}
#[test]
fn side_rgb_covers_the_webcams() {
// Higher-resolution webcams so the thin tint flanks are sampled.
let rig = dev_rig_with_resolutions(uvec2(32, 24), uvec2(160, 120));
let mut src = dev_source_with(&rig, SUBJECT, 30.0);
let frame = src.next_depth_frame().expect("frame");
let sides = frame.side_rgb.as_ref().expect("side rgb");
assert_eq!(sides.left.len(), 160 * 120 * 3);
assert_eq!(sides.right.len(), 160 * 120 * 3);
assert_eq!(sides.left_resolution, uvec2(160, 120));
// The head's flanks carry the side tints: the left camera's
// image must contain the left tint, the right camera's the
// right tint.
let has_tint = |rgb: &[u8], tint: [f32; 3]| {
rgb.as_chunks::<3>().0.iter().any(|px| {
(px[0] as i32 - (tint[0] * 255.0) as i32).abs() < 8
&& (px[2] as i32 - (tint[2] * 255.0) as i32).abs() < 8
})
};
assert!(
has_tint(&sides.left, scene::HEAD_LEFT_TINT),
"left tint in left image"
);
assert!(has_tint(&sides.right, scene::HEAD_RIGHT_TINT));
}
#[test]
fn timestamps_advance_with_index() {
let mut src = tiny_source();
assert_eq!(src.next_depth_frame().unwrap().timestamp_secs, 0.0);
@@ -399,11 +439,9 @@ mod tests {
period = Some(src.next_depth_frame().unwrap());
}
let period = period.expect("period frame produced");
assert_eq!(
frame0.depth_mm, period.depth_mm,
"sway must be periodic in the frame index"
);
assert_eq!(frame0.depth_mm, period.depth_mm, "sway must be periodic");
assert_eq!(frame0.rgb, period.rgb);
assert_eq!(frame0.side_rgb, period.side_rgb);
// With noise on, the sway still moves the same pixels but the
// exact map differs — noise must not be frame-periodic.
rig/src/ellipsoid.rs +146 -0
diff --git a/rig/src/ellipsoid.rs b/rig/src/ellipsoid.rs
new file mode 100644
index 0000000..a66e596
@@ -0,0 +1,146 @@
//! Ellipsoid geometry, shared by the synthetic scene (replay), the
//! occlusion proxy (splatgen), and tests.
use glam::Vec3;
/// An axis-aligned ellipsoid: all points `p` with
/// `((p - center) / radii)·length() = 1`.
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Ellipsoid {
pub center: Vec3,
/// Per-axis radii, metres. All positive.
pub radii: Vec3,
}
impl Ellipsoid {
pub fn new(center: Vec3, radii: Vec3) -> Self {
Self { center, radii }
}
/// Does the ellipsoid contain the point (strictly inside)?
pub fn contains(&self, p: Vec3) -> bool {
let s = (p - self.center) / self.radii;
s.dot(s) < 1.0
}
/// The outward unit normal at a surface point.
pub fn normal_at(&self, p: Vec3) -> Vec3 {
let grad = (p - self.center) / (self.radii * self.radii);
grad.normalize_or_zero()
}
/// The first intersection of the ray `origin + t * dir` with the
/// ellipsoid, at `t > 0`, with the surface normal at the hit.
///
/// `dir` need not be normalized; the returned `t` is in the ray's
/// own parameter space. Rays starting inside exit through the far
/// side.
pub fn intersect_ray(&self, origin: Vec3, dir: Vec3) -> Option<(f32, Vec3)> {
// Scale space so the ellipsoid is the unit sphere.
let o = (origin - self.center) / self.radii;
let d = dir / self.radii;
let a = d.dot(d);
if a <= f32::EPSILON {
return None;
}
let b = 2.0 * o.dot(d);
let c = o.dot(o) - 1.0;
let disc = b * b - 4.0 * a * c;
if disc < 0.0 {
return None;
}
let sqrt_disc = disc.sqrt();
let mut t = (-b - sqrt_disc) / (2.0 * a);
if t <= f32::EPSILON {
t = (-b + sqrt_disc) / (2.0 * a);
}
if t <= f32::EPSILON {
return None;
}
let position = origin + dir * t;
Some((t, self.normal_at(position)))
}
/// An approximate bounding check: does the ray ever come within
/// `margin` of the ellipsoid's surface? Used to skip work cheaply.
#[allow(dead_code)]
pub fn near_ray(&self, origin: Vec3, dir: Vec3, margin: f32) -> bool {
let o = (origin - self.center) / self.radii;
let d = dir / self.radii;
let a = d.dot(d);
if a <= f32::EPSILON {
return o.dot(o) < (1.0 + margin).powi(2);
}
let t = -o.dot(d) / a;
let closest = if t > 0.0 { o + d * t } else { o };
closest.dot(closest) < (1.0 + margin).powi(2)
}
}
#[cfg(test)]
mod tests {
use super::*;
use glam::vec3;
fn unit() -> Ellipsoid {
Ellipsoid::new(vec3(0.0, 0.0, 2.0), vec3(1.0, 1.0, 1.0))
}
#[test]
fn membership() {
let e = unit();
assert!(e.contains(vec3(0.0, 0.0, 2.0)));
assert!(e.contains(vec3(0.5, 0.0, 2.5)));
assert!(!e.contains(vec3(1.0, 0.0, 2.0)));
assert!(!e.contains(vec3(0.0, 0.0, 0.0)));
}
#[test]
fn ray_hits_sphere() {
let e = unit();
// Straight on: hits the front pole, normal faces the camera.
let (t, n) = e
.intersect_ray(Vec3::ZERO, vec3(0.0, 0.0, 1.0))
.expect("hit");
assert!((t - 1.0).abs() < 1e-5);
assert!((n - vec3(0.0, 0.0, -1.0)).length() < 1e-5);
// Miss.
assert!(
e.intersect_ray(vec3(5.0, 0.0, 0.0), vec3(0.0, 0.0, 1.0))
.is_none()
);
// Behind the ray.
assert!(
e.intersect_ray(vec3(0.0, 0.0, 5.0), vec3(0.0, 0.0, 1.0))
.is_none()
);
// From inside: exits at the far side.
let (t, _) = e
.intersect_ray(vec3(0.0, 0.0, 2.0), vec3(0.0, 0.0, 1.0))
.expect("exits");
assert!((t - 1.0).abs() < 1e-5);
}
#[test]
fn ray_hits_scaled_ellipsoid() {
let e = Ellipsoid::new(vec3(0.0, 0.18, 2.0), vec3(0.09, 0.11, 0.10));
// Straight at the front: t is in unnormalized ray units.
let (t, n) = e
.intersect_ray(Vec3::ZERO, vec3(0.0, 0.18, 2.0))
.expect("hit");
let p = Vec3::ZERO + vec3(0.0, 0.18, 2.0) * t;
// The hit is on the surface and the normal points back at us.
let s = (p - e.center) / e.radii;
assert!((s.length() - 1.0).abs() < 1e-4);
assert!(n.z < 0.0);
// Side rays hit with tilted normals.
let (t, n) = e
.intersect_ray(vec3(-1.0, 0.18, 2.0), vec3(1.0, 0.0, 0.0))
.expect("side hit");
let p = vec3(-1.0, 0.18, 2.0) + vec3(1.0, 0.0, 0.0) * t;
assert!((p.x - (-0.09)).abs() < 1e-4, "x = {}", p.x);
assert!(n.x < 0.0 && n.z.abs() < 0.01);
}
}
rig/src/lib.rs +3 -1
diff --git a/rig/src/lib.rs b/rig/src/lib.rs
index 3dfb1c0..fa469c9 100644
@@ -16,11 +16,13 @@
//! * intrinsics are in pixels at the camera's native resolution.
pub mod camera;
pub mod ellipsoid;
pub mod rig;
pub mod source;
pub mod splat;
pub use camera::{Camera, CameraIntrinsics, CameraPose};
pub use ellipsoid::Ellipsoid;
pub use rig::Rig;
pub use source::{DepthFrame, DepthSource, Frame, FrameSource};
pub use source::{DepthFrame, DepthSource, Frame, FrameSource, SideRgb};
pub use splat::{SURFEL_THICKNESS_M, Splat, SplatBatch};
rig/src/source.rs +28 -1
diff --git a/rig/src/source.rs b/rig/src/source.rs
index 50b5515..023fa38 100644
@@ -11,8 +11,10 @@
//! bridges the two.
use anyhow::Result;
use glam::UVec2;
use crate::camera::Camera;
use crate::rig::Rig;
use crate::splat::SplatBatch;
/// One processed rig frame, ready for rendering.
@@ -37,6 +39,23 @@ pub trait FrameSource {
fn next_frame(&mut self) -> Result<Frame>;
}
/// RGB images from the two side webcams, at their own resolutions.
///
/// Present when the rig source provides them (live capture, the
/// synthetic dataset, recordings with side files); the ellipsoid-proxy
/// fill needs them for recoloring.
#[derive(Debug, Clone, PartialEq)]
pub struct SideRgb {
/// Left camera, RGB, 3 bytes per pixel, row-major.
pub left: Vec<u8>,
/// Left camera resolution: `(width, height)`.
pub left_resolution: UVec2,
/// Right camera, RGB, 3 bytes per pixel, row-major.
pub right: Vec<u8>,
/// Right camera resolution: `(width, height)`.
pub right_resolution: UVec2,
}
/// One raw depth-camera frame, as the sensors report it.
#[derive(Debug, Clone, PartialEq)]
pub struct DepthFrame {
@@ -46,8 +65,11 @@ pub struct DepthFrame {
/// camera's resolution. Zero marks an invalid pixel (no return),
/// like the Kinect reports.
pub depth_mm: Vec<u16>,
/// RGB, 3 bytes per pixel, row-major, same resolution.
/// RGB, 3 bytes per pixel, row-major, same resolution as the depth
/// map (registered to depth, for a Kinect).
pub rgb: Vec<u8>,
/// RGB from the side webcams, when the source has them.
pub side_rgb: Option<SideRgb>,
}
/// A source of raw depth frames: the seam where live capture, replay of
@@ -58,5 +80,10 @@ pub trait DepthSource {
/// The depth camera the frames come from (intrinsics, pose,
/// resolution).
fn camera(&self) -> &Camera;
/// The full rig the source represents: the depth camera plus the
/// side cameras. The pipeline needs the side cameras (intrinsics,
/// poses, resolutions) for the occlusion-proxy recolouring, and
/// the recorded rig is the authority for recordings.
fn rig(&self) -> Rig;
fn next_depth_frame(&mut self) -> Result<DepthFrame>;
}
sink/Cargo.toml +1 -0
diff --git a/sink/Cargo.toml b/sink/Cargo.toml
index 6d18c9d..095ac62 100644
@@ -24,6 +24,7 @@ rig.workspace = true
serde.workspace = true
splatgen.workspace = true
toml.workspace = true
capture.workspace = true
[dev-dependencies]
egui_kittest.workspace = true
sink/src/config.rs +138 -1
diff --git a/sink/src/config.rs b/sink/src/config.rs
index abeb576..cf8f801 100644
@@ -32,10 +32,30 @@ use clap_config::ClapConfig;
use glam::{Quat, UVec2, Vec3};
use replay::NoiseParams;
use serde::{Deserialize, Serialize};
use splatgen::SplatGenParams;
use splatgen::{ProxyParams, SplatGenParams};
use rig::{Camera, CameraIntrinsics, CameraPose, Rig};
/// Which source feeds the pipeline. Selected with `--source`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum SourceArg {
/// The synthetic dataset (dev; the default).
Synthetic,
/// A recorded dataset directory (`--recording-path`).
Recording,
/// The live rig: Kinect + webcams.
Kinect,
}
/// The resolved source selection.
#[derive(Debug, Clone, PartialEq)]
pub enum SourceSelection {
Synthetic,
Recording(PathBuf),
Kinect,
}
/// Load the configuration from argv (CLI + environment), merged with
/// the config file named by `--config-file` / `KINECT_VIEW_CONFIG_FILE`,
/// if any.
@@ -180,6 +200,49 @@ pub struct Args {
)]
pub rig_right_intrinsics: Intrinsics,
// -- source selection ----------------------------------------------
/// Which source feeds the pipeline: synthetic, a recording, or the
/// live Kinect rig.
#[arg(
value_enum,
long,
env = "KINECT_VIEW_SOURCE",
default_value = "synthetic"
)]
pub source: SourceArg,
/// The recorded dataset directory (`--source recording`).
#[arg(long, env = "KINECT_VIEW_RECORDING_PATH")]
pub recording_path: Option<PathBuf>,
// -- occlusion proxy -----------------------------------------------
/// Fill occluded regions from the fitted subject ellipsoid,
/// recolored from the side cameras.
#[arg(long, env = "KINECT_VIEW_PROXY_ENABLED", default_value = "true")]
pub proxy_enabled: bool,
/// Fitted-ellipsoid radii as a multiple of the subject cluster's
/// per-axis standard deviation.
#[arg(long, env = "KINECT_VIEW_PROXY_SIGMA", default_value = "2.0")]
pub proxy_sigma: f32,
/// Proxy splats' thickness as a multiple of the regular surfels'.
#[arg(
long,
env = "KINECT_VIEW_PROXY_THICKNESS_MULTIPLE",
default_value = "3.0"
)]
pub proxy_thickness_multiple: f32,
/// Proxy splat opacity — lower than the surfels' (a lower
/// confidence weight).
#[arg(long, env = "KINECT_VIEW_PROXY_OPACITY", default_value = "0.85")]
pub proxy_opacity: f32,
/// Subject cluster depth range, metres (the nearest cluster in the
/// depth map is the subject; the background is further out).
#[arg(
long,
env = "KINECT_VIEW_PROXY_CLUSTER_EXTENT_M",
default_value = "0.6"
)]
pub proxy_cluster_extent_m: f32,
// -- depth processing (splatgen) -----------------------------------
/// Bilateral filter radius, pixels (PLAN.md: 3–5).
#[arg(long, env = "KINECT_VIEW_SPLATGEN_FILTER_RADIUS", default_value = "3")]
@@ -282,6 +345,20 @@ impl Args {
let fps = self.source_fps;
ensure!(fps > 0.0, "source-fps must be positive, got {fps}");
let proxy = ProxyParams {
enabled: self.proxy_enabled,
sigma: self.proxy_sigma,
thickness_multiple: self.proxy_thickness_multiple,
opacity: self.proxy_opacity,
cluster_extent_m: self.proxy_cluster_extent_m,
};
ensure!(
(0.0..=1.0).contains(&proxy.opacity),
"proxy-opacity must be in [0, 1], got {}",
proxy.opacity
);
ensure!(proxy.sigma > 0.0, "proxy-sigma must be positive");
let splatgen = SplatGenParams {
filter_radius: self.splatgen_filter_radius,
filter_sigma_depth_mm: self.splatgen_filter_sigma_depth_mm,
@@ -290,6 +367,7 @@ impl Args {
max_depth_mm: self.splatgen_max_depth_mm,
opacity: self.splatgen_opacity,
thickness_m: rig::SURFEL_THICKNESS_M,
proxy,
};
ensure!(
(0.0..=1.0).contains(&splatgen.opacity),
@@ -307,6 +385,19 @@ impl Args {
noise.flier_rate
);
// Which source feeds the pipeline.
let source = match self.source {
SourceArg::Synthetic => SourceSelection::Synthetic,
SourceArg::Recording => {
let path = self.recording_path.clone().context(
"--source recording needs --recording-path (or \
KINECT_VIEW_RECORDING_PATH)",
)?;
SourceSelection::Recording(path)
}
SourceArg::Kinect => SourceSelection::Kinect,
};
Ok(ResolvedConfig {
rig: Rig {
left,
@@ -316,6 +407,7 @@ impl Args {
subject: self.scene_subject_position.0.into(),
fps,
background: self.background.0.into(),
source,
splatgen,
noise,
})
@@ -332,6 +424,8 @@ pub struct ResolvedConfig {
pub fps: f32,
/// Render background colour.
pub background: Vec3,
/// Which source feeds the pipeline.
pub source: SourceSelection,
/// Depth-processing parameters (`splatgen`).
pub splatgen: SplatGenParams,
/// Synthetic sensor noise.
@@ -510,6 +604,7 @@ mod tests {
assert_eq!(args.background, vec3(0.10, 0.10, 0.12));
assert_eq!(args.splatgen, SplatGenParams::default());
assert_eq!(args.noise, NoiseParams::default());
assert_eq!(args.source, SourceSelection::Synthetic);
}
#[test]
@@ -649,6 +744,48 @@ mod tests {
}
#[test]
fn source_selection_is_configurable() {
let _guard = TEST_LOCK.lock().unwrap();
// Default: synthetic.
let args = merged(&["app"], None).resolve().expect("resolves");
assert_eq!(args.source, SourceSelection::Synthetic);
// From a file.
let args = merged(
&["app"],
Some("source = \"recording\"\nrecording_path = \"/tmp/some-rec\"\n"),
)
.resolve()
.expect("resolves");
assert_eq!(
args.source,
SourceSelection::Recording(PathBuf::from("/tmp/some-rec"))
);
// From the CLI.
let args = merged(&["app", "--source", "kinect"], None)
.resolve()
.expect("resolves");
assert_eq!(args.source, SourceSelection::Kinect);
// Recording without a path is a configuration error.
let err = merged(&["app", "--source", "recording"], None)
.resolve()
.expect_err("missing recording path");
assert!(err.to_string().contains("recording-path"));
// Proxy settings flow through.
let args = merged(
&["app"],
Some("proxy_opacity = 0.7\nproxy_cluster_extent_m = 0.9\n"),
)
.resolve()
.expect("resolves");
assert_eq!(args.splatgen.proxy.opacity, 0.7);
assert_eq!(args.splatgen.proxy.cluster_extent_m, 0.9);
}
#[test]
fn value_types_round_trip_as_strings_and_toml() {
let _guard = TEST_LOCK.lock().unwrap();
let pos: Position3 = "-0.4,0,1".parse().expect("parses");
sink/src/main.rs +55 -9
diff --git a/sink/src/main.rs b/sink/src/main.rs
index f5763f3..9edf7b4 100644
@@ -1,8 +1,8 @@
//! `kinect-view` application entry point.
//!
//! Runs the pipeline on the synthetic development dataset (build-order
//! step 1); live Kinect capture replaces the source in step 3. The
//! rig, scene, and view are configurable: CLI arguments, environment
//! Runs the pipeline on the configured source — synthetic (the
//! default), a recording, or the live Kinect rig — and shows the egui
//! viewer. Everything is configurable: CLI arguments, environment
//! variables, and a TOML config file — see `sink::config`.
use eframe::egui;
@@ -18,13 +18,59 @@ fn main() -> eframe::Result<()> {
}
};
let source =
replay::synth::dev_source_with(resolved.rig.center.clone(), resolved.subject, resolved.fps)
.with_noise(resolved.noise);
// The real pipeline: raw depth frames → preprocessing → surfels.
let source = splatgen::SplatSource::new(source, splatgen::SplatGen::new(resolved.splatgen));
// The frame source: raw depth frames → preprocessing → surfels.
// The source's own rig is the authority (a recording carries its
// cameras; the live Kinect contributes factory intrinsics).
let source: Box<dyn rig::FrameSource>;
let rig = match &resolved.source {
sink::config::SourceSelection::Synthetic => {
let rig = resolved.rig.clone();
let raw = replay::synth::dev_source_with(&rig, resolved.subject, resolved.fps)
.with_noise(resolved.noise);
source = Box::new(splatgen::SplatSource::new(
rig.clone(),
raw,
splatgen::SplatGen::new(resolved.splatgen),
));
rig
}
sink::config::SourceSelection::Recording(path) => {
let player = match replay::RecordingSource::load(path) {
Ok(player) => player,
Err(error) => {
eprintln!("loading recording {}: {error:#}", path.display());
std::process::exit(2);
}
};
let rig = player.rig().clone();
source = Box::new(splatgen::SplatSource::new(
player.rig().clone(),
player,
splatgen::SplatGen::new(resolved.splatgen),
));
rig
}
sink::config::SourceSelection::Kinect => {
let live = match capture::KinectRigSource::open(&resolved.rig) {
Ok(live) => live,
Err(error) => {
eprintln!("opening the live rig: {error:#}");
std::process::exit(2);
}
};
use rig::DepthSource;
let rig = live.rig();
source = Box::new(splatgen::SplatSource::new(
live.rig(),
live,
splatgen::SplatGen::new(resolved.splatgen),
));
rig
}
};
let renderer = render::SplatRenderer::new(resolved.background);
let model = sink::ViewerModel::new(resolved.rig, Box::new(source), renderer);
let model = sink::ViewerModel::new(rig, source, renderer);
let app = sink::KinectViewApp { model };
let options = eframe::NativeOptions {
sink/tests/ui.rs +46 -4
diff --git a/sink/tests/ui.rs b/sink/tests/ui.rs
index ef979a3..f7b2abe 100644
@@ -21,8 +21,8 @@ use sink::ViewerModel;
fn small_model() -> ViewerModel {
let rig = synth::dev_rig_with_resolutions(uvec2(64, 53), uvec2(80, 60));
// The full pipeline: raw synthetic depth → preprocessing → surfels.
let raw = synth::dev_source_with(rig.center.clone(), synth::SUBJECT, 30.0);
let source = splatgen::SplatSource::new(raw, splatgen::SplatGen::default());
let raw = synth::dev_source_with(&rig, synth::SUBJECT, 30.0);
let source = splatgen::SplatSource::new(rig.clone(), raw, splatgen::SplatGen::default());
ViewerModel::new(rig, Box::new(source), render::SplatRenderer::default())
}
@@ -221,8 +221,9 @@ scene_subject_position = [0.0, 0.0, 2.5]
assert_eq!(resolved.rig.center.resolution, uvec2(64, 53));
// And the rig actually renders through the whole pipeline.
let raw = synth::dev_source_with(resolved.rig.center.clone(), resolved.subject, 30.0);
let source = splatgen::SplatSource::new(raw, splatgen::SplatGen::default());
let raw = synth::dev_source_with(&resolved.rig, resolved.subject, 30.0);
let source =
splatgen::SplatSource::new(resolved.rig.clone(), raw, splatgen::SplatGen::default());
let model = ViewerModel::new(resolved.rig, Box::new(source), SplatRenderer::default());
let harness = harness_for(model, false);
let image = harness.state().last_image().expect("rendered");
@@ -237,6 +238,47 @@ scene_subject_position = [0.0, 0.0, 2.5]
std::fs::remove_dir(&config_dir).ok();
}
#[test]
fn recording_source_renders() {
// The `--source recording` path end to end: record a few synthetic
// frames to disk, load them back through the real config pipeline,
// and render through the viewer.
let dir = std::env::temp_dir().join(format!("kinect-view-ui-rec-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
let rig = synth::dev_rig_with_resolutions(uvec2(32, 24), uvec2(80, 60));
let mut raw =
synth::dev_source_with(&rig, synth::SUBJECT, 30.0).with_noise(synth::NoiseParams::OFF);
replay::recording::Recording::write(&dir, &mut raw, 30.0, 3).expect("write recording");
// Load through the real config path.
let resolved = sink::config::load_from([
"kinect-view",
"--source",
"recording",
"--recording-path",
dir.to_str().expect("utf8 path"),
])
.and_then(|args| args.resolve())
.expect("config resolves");
assert_eq!(
resolved.source,
sink::config::SourceSelection::Recording(dir.clone())
);
let player = replay::recording::RecordingSource::load(&dir).expect("load");
let rig = player.rig().clone();
let source = splatgen::SplatSource::new(rig.clone(), player, splatgen::SplatGen::default());
let model = ViewerModel::new(rig, Box::new(source), SplatRenderer::default());
let harness = harness_for(model, false);
let model = harness.state();
assert!(model.stats().splats > 0, "recorded frames produced splats");
assert!(model.last_image().is_some());
let _ = std::fs::remove_dir_all(&dir);
}
/// The mean x of pixels close to the synthetic skin colour, or `None` if
/// the subject is not visible.
fn skin_centroid_x(image: &RenderedImage) -> Option<f32> {
splatgen/src/lib.rs +508 -24
diff --git a/splatgen/src/lib.rs b/splatgen/src/lib.rs
index f35e0f3..434623b 100644
@@ -24,7 +24,42 @@
use anyhow::Result;
use glam::Vec3;
use rig::{Camera, DepthFrame, DepthSource, Frame, FrameSource, Splat, SplatBatch};
use rig::{
Camera, DepthFrame, DepthSource, Ellipsoid, Frame, FrameSource, Rig, SideRgb, Splat, SplatBatch,
};
/// Tuning knobs for the occlusion proxy (PLAN.md, "Core design": the
/// MVP fills the occluded far side of the face with a head/shoulders
/// ellipsoid fitted per frame, recolored from the side cameras).
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct ProxyParams {
/// Generate proxy splats at all.
pub enabled: bool,
/// Fitted-ellipsoid radii as a multiple of the cluster's per-axis
/// standard deviation.
pub sigma: f32,
/// Proxy splats are this many times thicker than regular surfels,
/// so the flat disks bridge the curvature between them.
pub thickness_multiple: f32,
/// Proxy splat opacity — lower than the surfels' (a lower
/// confidence weight, so real depth data dominates on overlap).
pub opacity: f32,
/// Points within this depth range of the nearest valid depth
/// belong to the subject cluster, metres.
pub cluster_extent_m: f32,
}
impl Default for ProxyParams {
fn default() -> Self {
Self {
enabled: true,
sigma: 2.0,
thickness_multiple: 3.0,
opacity: 0.85,
cluster_extent_m: 0.6,
}
}
}
/// Tuning knobs for the depth pipeline; the defaults follow PLAN.md
/// (bilateral radius 3–5 px, opacity ~0.95, few-mm thickness).
@@ -45,6 +80,8 @@ pub struct SplatGenParams {
pub opacity: f32,
/// Surfel thickness along the normal, metres.
pub thickness_m: f32,
/// Occlusion-proxy parameters.
pub proxy: ProxyParams,
}
impl Default for SplatGenParams {
@@ -57,6 +94,7 @@ impl Default for SplatGenParams {
max_depth_mm: 4000,
opacity: 0.95,
thickness_m: rig::SURFEL_THICKNESS_M,
proxy: ProxyParams::default(),
}
}
}
@@ -82,8 +120,11 @@ impl SplatGen {
&self.params
}
/// Preprocess one depth frame and assemble its splat batch.
pub fn generate(&self, frame: &DepthFrame, camera: &Camera) -> SplatBatch {
/// Preprocess one depth frame and assemble its splat batch:
/// surfels from the depth map, plus occlusion-proxy splats from the
/// fitted ellipsoid, recolored from the side cameras.
pub fn generate(&self, frame: &DepthFrame, rig: &Rig) -> SplatBatch {
let camera = &rig.center;
let (width, height) = (camera.resolution.x, camera.resolution.y);
let pixel_count = (width * height) as usize;
assert_eq!(frame.depth_mm.len(), pixel_count, "depth map size");
@@ -126,33 +167,110 @@ impl SplatGen {
));
}
}
// Occlusion proxy: splats on the fitted subject ellipsoid,
// recolored from the side cameras.
if self.params.proxy.enabled
&& let Some(sides) = &frame.side_rgb
&& let Some(fit) = fit_subject_ellipsoid(
&filtered,
camera,
self.params.proxy.cluster_extent_m,
self.params.proxy.sigma,
)
{
self.add_proxy_splats(&mut batch, &filtered, camera, &fit, rig, sides);
}
batch
}
/// Generate proxy splats from one side camera: every pixel whose
/// ray hits the fitted ellipsoid in a region the depth map does
/// cover gets a splat on the ellipsoid, recolored from that pixel.
fn add_proxy_splats(
&self,
batch: &mut SplatBatch,
filtered: &[Option<f32>],
center: &Camera,
fit: &SubjectFit,
rig: &Rig,
sides: &SideRgb,
) {
let side_data = [
(&rig.left, &sides.left, sides.left_resolution),
(&rig.right, &sides.right, sides.right_resolution),
];
for (camera, rgb, resolution) in side_data {
let (width, height) = (resolution.x, resolution.y);
assert_eq!(rgb.len(), (width * height) as usize * 3, "side rgb size");
let camera_from_world = camera.pose.camera_from_world();
for y in 0..height {
for x in 0..width {
let (u, v) = (x as f32 + 0.5, y as f32 + 0.5);
let (origin, dir) = camera.pixel_ray(u, v);
let Some((t, normal)) = fit.ellipsoid.intersect_ray(origin, dir) else {
continue;
};
let position = origin + dir * t;
// Only where the depth map has no data of its own.
if occluded_from_depth_map(filtered, center, position, fit) {
let depth = camera_from_world.transform_point3(position).z;
let idx = (y * width + x) as usize;
let color = [
rgb[idx * 3] as f32 / 255.0,
rgb[idx * 3 + 1] as f32 / 255.0,
rgb[idx * 3 + 2] as f32 / 255.0,
];
batch.push(Splat::surfel(
position,
normal,
depth,
&camera.intrinsics,
self.params.thickness_m * self.params.proxy.thickness_multiple,
color,
self.params.proxy.opacity,
));
}
}
}
}
}
}
/// A [`FrameSource`] adapter: pull raw depth frames from `S` and run
/// them through [`SplatGen`]. This is the pipeline seam — live capture,
/// recordings, and the synthetic dataset all become renderable sources.
/// The rig is the source's own (its side cameras drive the proxy).
pub struct SplatSource<S: DepthSource> {
rig: Rig,
inner: S,
generator: SplatGen,
}
impl<S: DepthSource> SplatSource<S> {
pub fn new(inner: S, generator: SplatGen) -> Self {
Self { inner, generator }
pub fn new(rig: Rig, inner: S, generator: SplatGen) -> Self {
Self {
rig,
inner,
generator,
}
}
/// The wrapped depth source.
pub fn inner(&self) -> &S {
&self.inner
}
/// The rig the pipeline renders with.
pub fn rig(&self) -> &Rig {
&self.rig
}
}
impl<S: DepthSource> FrameSource for SplatSource<S> {
fn next_frame(&mut self) -> Result<Frame> {
let depth_frame = self.inner.next_depth_frame()?;
let splats = self.generator.generate(&depth_frame, self.inner.camera());
let splats = self.generator.generate(&depth_frame, &self.rig);
Ok(Frame {
timestamp_secs: depth_frame.timestamp_secs,
splats,
@@ -160,6 +278,119 @@ impl<S: DepthSource> FrameSource for SplatSource<S> {
}
}
/// The subject cluster fit: the ellipsoid plus the depth range the
/// cluster spans (what separates subject pixels from background in the
/// depth map).
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct SubjectFit {
pub ellipsoid: Ellipsoid,
/// Cluster depth range in millimetres.
pub min_mm: f32,
pub max_mm: f32,
}
/// Is `position` a region the depth map does not cover — i.e. may the
/// proxy fill it?
///
/// * outside the depth image, at an invalid pixel, or behind the depth
/// camera → yes, no data there;
/// * over a background pixel (observed depth beyond the subject
/// cluster) → yes, the proxy adds subject geometry the depth map
/// never saw;
/// * over a subject pixel → only when the point sits *behind* the
/// observed surface by more than the margin — the occluded side. At
/// or in front of the observed surface the depth map's own surfels
/// already cover it (and a floating proxy point would z-fight).
fn occluded_from_depth_map(
filtered: &[Option<f32>],
center: &Camera,
position: Vec3,
fit: &SubjectFit,
) -> bool {
const PROXY_OCCLUSION_MARGIN_MM: f32 = 30.0;
let p = center.pose.camera_from_world().transform_point3(position);
if p.z <= 0.0 {
return true; // behind the depth camera entirely
}
let Some(pixel) = center.intrinsics.project_camera_point(p) else {
return true; // outside the depth image
};
let (width, height) = (center.resolution.x, center.resolution.y);
if pixel.x < 0.0 || pixel.y < 0.0 || pixel.x >= width as f32 || pixel.y >= height as f32 {
return true;
}
let idx = (pixel.y as u32 * width + pixel.x as u32) as usize;
let Some(observed) = filtered.get(idx).copied().flatten() else {
return true; // no data here
};
if observed > fit.max_mm {
return true; // background pixel: the proxy is new subject geometry
}
p.z * 1000.0 > observed + PROXY_OCCLUSION_MARGIN_MM
}
/// Fit the subject's ellipsoid to the depth cloud: the points within
/// `cluster_extent_m` of the nearest observed depth (the subject sits
/// closest to the rig; the backdrop is further out), mean centre,
/// `sigma`-scaled per-axis standard deviation as radii.
///
/// Returns `None` when there is no substantial cluster.
pub fn fit_subject_ellipsoid(
filtered: &[Option<f32>],
camera: &Camera,
cluster_extent_m: f32,
sigma: f32,
) -> Option<SubjectFit> {
let (width, height) = (camera.resolution.x, camera.resolution.y);
// Nearest valid depth (millimetres).
let min_mm = filtered
.iter()
.flatten()
.copied()
.fold(f32::INFINITY, f32::min);
if !min_mm.is_finite() {
return None;
}
let max_mm = min_mm + cluster_extent_m * 1000.0;
// The subject cluster, in world space.
let mut count = 0_usize;
let mut sum = Vec3::ZERO;
let mut sum_sq = Vec3::ZERO;
for y in 0..height {
for x in 0..width {
let Some(d) = filtered[(y * width + x) as usize] else {
continue;
};
if d < min_mm || d > max_mm {
continue;
}
let p = camera.backproject(x as f32 + 0.5, y as f32 + 0.5, d / 1000.0);
count += 1;
sum += p;
sum_sq += p * p;
}
}
// Too little subject to speak of an ellipsoid.
if count < 100 {
return None;
}
let center = sum / count as f32;
let variance = (sum_sq / count as f32 - center * center).max(Vec3::ZERO);
let std_dev = Vec3::new(variance.x.sqrt(), variance.y.sqrt(), variance.z.sqrt());
// Clamp so the ellipsoid never degenerates into a disk or point.
let min_radius = 0.02_f32;
let radii = (std_dev * sigma).max(Vec3::splat(min_radius));
Some(SubjectFit {
ellipsoid: Ellipsoid::new(center, radii),
min_mm,
max_mm,
})
}
/// Drop pixels that disagree with their 3×3 neighbourhood median by
/// more than `threshold_mm` (isolated fliers). Neighbourhood positions
/// are clamped at the borders.
@@ -300,6 +531,7 @@ mod tests {
use glam::{uvec2, vec3};
use rig::{CameraIntrinsics, CameraPose};
/// A test center camera.
fn camera(width: u32, height: u32) -> Camera {
Camera {
id: "test".into(),
@@ -314,6 +546,27 @@ mod tests {
}
}
/// A test rig: a converged center camera plus matching side cameras.
fn rig(width: u32, height: u32) -> rig::Rig {
let center = camera(width, height);
let make_side = |x: f32| Camera {
id: if x < 0.0 { "left" } else { "right" }.to_owned(),
pose: rig::CameraPose::looking_at(glam::vec3(x, 0.0, 0.0), glam::vec3(0.0, 0.0, 2.0)),
intrinsics: CameraIntrinsics {
fx: 100.0,
fy: 100.0,
cx: width as f32 / 2.0,
cy: height as f32 / 2.0,
},
resolution: glam::uvec2(width, height),
};
rig::Rig {
left: make_side(-0.4),
center,
right: make_side(0.4),
}
}
fn frame(width: u32, height: u32, depth_of: impl Fn(u32, u32) -> u16) -> DepthFrame {
let mut depth = Vec::with_capacity((width * height) as usize);
let mut rgb = Vec::with_capacity((width * height * 3) as usize);
@@ -327,15 +580,17 @@ mod tests {
timestamp_secs: 0.0,
depth_mm: depth,
rgb,
side_rgb: None,
}
}
#[test]
fn flat_wall_makes_camera_facing_surfels() {
let (w, h) = (32u32, 24u32);
let camera = camera(w, h);
let rig = rig(w, h);
let camera = &rig.center;
let frame = frame(w, h, |_, _| 2000);
let batch = SplatGen::default().generate(&frame, &camera);
let batch = SplatGen::default().generate(&frame, &rig);
assert_eq!(batch.len(), (w * h) as usize);
for splat in batch.iter() {
@@ -364,20 +619,22 @@ mod tests {
#[test]
fn culling_drops_invalid_and_far_pixels() {
let (w, h) = (16u32, 8u32);
let camera = camera(w, h);
let rig = rig(w, h);
let _camera = &rig.center;
let frame = frame(w, h, |x, y| match (x, y) {
(0, 0) => 0, // no return
(1, 0) => 5000, // beyond max depth
_ => 2000,
});
let batch = SplatGen::default().generate(&frame, &camera);
let batch = SplatGen::default().generate(&frame, &rig);
assert_eq!(batch.len(), (w * h - 2) as usize);
}
#[test]
fn fliers_are_removed() {
let (w, h) = (16u32, 16u32);
let camera = camera(w, h);
let rig = rig(w, h);
let camera = &rig.center;
// A few isolated wrong returns among a flat wall.
let frame = frame(w, h, |x, y| {
if matches!((x, y), (4, 4) | (10, 12) | (15, 0)) {
@@ -386,7 +643,7 @@ mod tests {
2000
}
});
let batch = SplatGen::default().generate(&frame, &camera);
let batch = SplatGen::default().generate(&frame, &rig);
assert_eq!(
batch.len(),
(w * h - 3) as usize,
@@ -406,7 +663,8 @@ mod tests {
#[test]
fn bilateral_smooths_jitter_but_not_edges() {
let (w, h) = (32u32, 8u32);
let camera = camera(w, h);
let rig = rig(w, h);
let _camera = &rig.center;
// Left half at 2000 mm, right half at 3000 mm, everything
// jittered ±3 mm.
let clean = frame(w, h, |x, _| if x < 16 { 2000 } else { 3000 });
@@ -417,8 +675,8 @@ mod tests {
});
let generator = SplatGen::default();
let clean_batch = generator.generate(&clean, &camera);
let noisy_batch = generator.generate(&noisy, &camera);
let clean_batch = generator.generate(&clean, &rig);
let noisy_batch = generator.generate(&noisy, &rig);
assert_eq!(clean_batch.len(), noisy_batch.len());
// The noisy splats sit within a fraction of a pixel of the
@@ -437,13 +695,14 @@ mod tests {
#[test]
fn normals_follow_a_slanted_plane() {
let (w, h) = (24u32, 24u32);
let camera = camera(w, h);
let rig = rig(w, h);
let camera = &rig.center;
// Depth increases by 40 mm per pixel: the surface tilts away to
// the right. dz/du = 0.04 m per pixel; world dx per pixel is
// z/fx, so the plane's normal is ∝ (-slope, 0, 1).
let frame = frame(w, h, |x, _| 2000 + 40 * x as u16);
let batch = SplatGen::default().generate(&frame, &camera);
let batch = SplatGen::default().generate(&frame, &rig);
// Check a pixel near the centre, away from border effects.
let splat = batch.iter().nth(12 * 24 + 12).expect("center pixel");
@@ -465,13 +724,14 @@ mod tests {
#[test]
fn params_are_honoured() {
let (w, h) = (16u32, 16u32);
let camera = camera(w, h);
let rig = rig(w, h);
let _camera = &rig.center;
let frame = frame(w, h, |_, _| 2500);
let params = SplatGenParams {
max_depth_mm: 2000, // culls everything
..SplatGenParams::default()
};
let batch = SplatGen::new(params).generate(&frame, &camera);
let batch = SplatGen::new(params).generate(&frame, &rig);
assert!(batch.is_empty());
// Opacity flows through to the surfels.
@@ -479,7 +739,7 @@ mod tests {
opacity: 0.9,
..SplatGenParams::default()
};
let batch = SplatGen::new(params).generate(&frame, &camera);
let batch = SplatGen::new(params).generate(&frame, &rig);
assert!((batch.iter().next().unwrap().opacity - 0.9).abs() < 1e-6);
}
@@ -487,11 +747,12 @@ mod tests {
fn generates_from_the_synthetic_source() {
// Integration: the synthetic Kinect feeds the pipeline.
use rig::DepthSource;
let rig = replay::synth::dev_rig_with_resolutions(uvec2(48, 40), uvec2(64, 48));
let mut source =
replay::synth::dev_source_with(rig.center.clone(), replay::synth::SUBJECT, 30.0);
let rig = replay::synth::dev_rig_with_resolutions(uvec2(48, 40), uvec2(160, 120));
let mut source = replay::synth::dev_source_with(&rig, replay::synth::SUBJECT, 30.0);
let frame = source.next_depth_frame().expect("depth frame");
let batch = SplatGen::default().generate(&frame, &rig.center);
let mut params = SplatGenParams::default();
params.proxy.enabled = false;
let batch = SplatGen::new(params).generate(&frame, &rig);
// Roughly one splat per depth pixel (fliers removed).
assert!(batch.len() > 48 * 40 * 95 / 100);
@@ -503,4 +764,227 @@ mod tests {
assert!(p.z > 0.5 && p.z < 3.1, "depth {p:?}");
}
}
/// The synthetic rig + a clean first frame, for proxy tests. The
/// depth resolution is high enough that the subject spans several
/// hundred pixels.
fn synthetic_proxy_setup() -> (rig::Rig, DepthFrame) {
use rig::DepthSource;
let rig = replay::synth::dev_rig_with_resolutions(uvec2(128, 106), uvec2(160, 120));
let mut source = replay::synth::dev_source_with(&rig, replay::synth::SUBJECT, 30.0)
.with_noise(replay::synth::NoiseParams::OFF);
let frame = source.next_depth_frame().expect("depth frame");
(rig, frame)
}
#[test]
fn ellipsoid_fit_matches_the_subject() {
let (rig, frame) = synthetic_proxy_setup();
let filtered = preprocess_for_fit(&frame, &rig);
let fit = fit_subject_ellipsoid(
&filtered,
&rig.center,
ProxyParams::default().cluster_extent_m,
ProxyParams::default().sigma,
)
.expect("subject cluster found");
let ellipsoid = fit.ellipsoid;
// The subject (head + shoulders, pixel-weighted to the wide
// shoulders) sits near the origin, slightly below head height,
// ~1.9 m out — the cluster is the visible front.
assert!(
ellipsoid.center.x.abs() < 0.05,
"fit centre {:?}",
ellipsoid.center
);
assert!(
(-0.1..0.15).contains(&ellipsoid.center.y),
"fit centre {:?}",
ellipsoid.center
);
assert!(
(1.8..2.1).contains(&ellipsoid.center.z),
"fit centre {:?}",
ellipsoid.center
);
// Head + shoulders: ~0.24 m half-width, ~0.47 m tall.
assert!(
ellipsoid.radii.x > 0.18 && ellipsoid.radii.x < 0.40,
"radii {:?}",
ellipsoid.radii
);
assert!(
ellipsoid.radii.y > 0.15 && ellipsoid.radii.y < 0.40,
"radii {:?}",
ellipsoid.radii
);
assert!(
ellipsoid.radii.z > 0.04 && ellipsoid.radii.z < 0.30,
"radii {:?}",
ellipsoid.radii
);
// The cluster range excludes the backdrop (2.8 m out).
assert!(
fit.min_mm > 1500.0 && fit.min_mm < 2100.0,
"min {}",
fit.min_mm
);
assert!(fit.max_mm < 2700.0, "max {}", fit.max_mm);
// And the fitted blob contains the subject's front — the head
// front pole and the shoulders' front.
assert!(ellipsoid.contains(replay::synth::SUBJECT + glam::vec3(0.0, 0.18, -0.10)));
assert!(ellipsoid.contains(replay::synth::SUBJECT + glam::vec3(0.0, -0.05, -0.16)));
}
/// The filtered depth the ellipsoid fit sees, for tests.
fn preprocess_for_fit(frame: &DepthFrame, rig: &rig::Rig) -> Vec<Option<f32>> {
let camera = &rig.center;
let mut depth: Vec<Option<f32>> = frame
.depth_mm
.iter()
.map(|&d| (d > 0 && d <= 4000).then_some(d as f32))
.collect();
remove_fliers(&mut depth, camera.resolution.x, 250);
bilateral_filter(&depth, camera.resolution.x, SplatGenParams::default())
}
#[test]
fn proxy_fills_occluded_flanks_with_side_colours() {
let (rig, frame) = synthetic_proxy_setup();
let plain_params = SplatGenParams {
proxy: ProxyParams {
enabled: false,
..ProxyParams::default()
},
..SplatGenParams::default()
};
let plain = SplatGen::new(plain_params).generate(&frame, &rig);
let with_proxy = SplatGen::default().generate(&frame, &rig);
assert!(
with_proxy.len() > plain.len(),
"proxy must add splats ({} vs {})",
with_proxy.len(),
plain.len()
);
// Recover the fitted ellipsoid and inspect the proxy splats:
// every added splat lies on it, with its outward normal, and
// carries lower confidence (opacity) and inflated thickness.
let filtered = preprocess_for_fit(&frame, &rig);
let fit = fit_subject_ellipsoid(
&filtered,
&rig.center,
ProxyParams::default().cluster_extent_m,
ProxyParams::default().sigma,
)
.expect("fit");
let ellipsoid = fit.ellipsoid;
let mut proxy_count = 0;
let mut left_tinted = 0;
let mut right_tinted = 0;
for splat in with_proxy.iter().skip(plain.len()) {
proxy_count += 1;
// On the fitted surface.
let s = (splat.mean - ellipsoid.center) / ellipsoid.radii;
assert!((s.length() - 1.0).abs() < 1e-3, "off-ellipsoid: {s:?}");
// Outward normal matches the ellipsoid's.
let n = ellipsoid.normal_at(splat.mean);
assert!(
splat.normal().dot(n) > 0.999,
"normal {:?} vs {n:?}",
splat.normal()
);
// Lower confidence and inflated thickness.
assert!((splat.opacity - 0.85).abs() < 1e-6);
assert!((splat.scale.z - 3.0 * rig::SURFEL_THICKNESS_M).abs() < 1e-6);
// Recolored from the side cameras: the tints show up.
let c = splat.color;
if c[0] > 0.7 && c[2] < 0.2 {
left_tinted += 1;
}
if c[2] > 0.7 && c[0] < 0.2 {
right_tinted += 1;
}
}
assert!(proxy_count > 20, "proxy produced {proxy_count} splats");
assert!(left_tinted > 0, "no left-tinted proxy splats");
assert!(right_tinted > 0, "no right-tinted proxy splats");
}
#[test]
fn occlusion_rule_drops_covered_geometry() {
// A flat wall 2 m out, treated as the subject (its depth is the
// cluster range): only points *behind* the observed surface —
// or where there is no data at all — may be filled by the
// proxy; background pixels always may.
let (w, h) = (16u32, 16u32);
let rig = rig(w, h);
let camera = &rig.center;
let filtered = vec![Some(2000.0_f32); (w * h) as usize];
let fit = SubjectFit {
ellipsoid: Ellipsoid::new(glam::vec3(0.0, 0.0, 2.0), glam::vec3(0.1, 0.1, 0.1)),
min_mm: 2000.0,
max_mm: 2600.0,
};
let at = |x: f32, y: f32, z: f32| camera.backproject(x, y, z);
// In front of the observed surface → covered → no proxy.
assert!(!occluded_from_depth_map(
&filtered,
camera,
at(8.5, 8.5, 1.95),
&fit
));
// Right at the surface (within the margin) → no proxy.
assert!(!occluded_from_depth_map(
&filtered,
camera,
at(8.5, 8.5, 2.02),
&fit
));
// Behind by more than the margin → occluded → proxy.
assert!(occluded_from_depth_map(
&filtered,
camera,
at(8.5, 8.5, 2.1),
&fit
));
// Over a background pixel (observed beyond the cluster) → the
// proxy is new subject geometry → always allowed.
let mut background = filtered.clone();
background[0] = Some(3500.0);
assert!(occluded_from_depth_map(
&background,
camera,
at(0.5, 0.5, 2.1),
&fit
));
// Outside the depth image → no data → proxy.
assert!(occluded_from_depth_map(
&filtered,
camera,
at(-3.0, 8.5, 2.1),
&fit
));
// Invalid depth pixel → no data → proxy.
let mut invalid = vec![Some(2000.0_f32); (w * h) as usize];
invalid[0] = None;
assert!(occluded_from_depth_map(
&invalid,
camera,
at(0.5, 0.5, 2.1),
&fit
));
// Behind the depth camera entirely → proxy.
assert!(occluded_from_depth_map(
&filtered,
camera,
at(8.5, 8.5, -1.0),
&fit
));
}
}