Commit:
a93bba1Parent:
9aa5c3cBuild step 4: the v4l2loopback virtual webcam sink
Rendered virtual-camera frames are converted to YUYV (BT.601 full range, chroma averaged per pixel pair) and written to a v4l2loopback device via v4l2r: capability check (with a clear error when the node isn't the output side of an exclusive_caps device), format negotiation, one write per frame. - sink::loopback: RGBA→YUYV conversion, the LoopbackWriter seam, a threaded sink with latest-frame handoff (a slow device never stalls the UI), inline mode for tests, and automatic disable-on- failure with a logged error - the viewer pushes every rendered frame to the sink and reports 'loopback on' in its stats line - --loopback-path joins the config surface (CLI/env/TOML); off Linux the setting is rejected up front - README documents the module setup (exclusive_caps is what makes Chrome-based consumers see the device) - tests: known-colour conversions, collector-writer integration through the real viewer pipeline, failure-disable behaviour, and real ioctl error paths (nonexistent path, /dev/null) — 81 total. Attaching a real /dev/video device needs the kernel module on a real machine (noted in PLAN.md's testing section)
Cargo.lock
+96
-15
diff --git a/Cargo.lock b/Cargo.lock
index 09a7156..1b8b21f 100644
@@ -563,7 +563,7 @@ dependencies = [
"log",
"num-rational",
"num-traits",
"pastey",
"pastey 0.1.1",
"rayon",
"thiserror 2.0.21",
"v_frame",
@@ -724,6 +724,26 @@ dependencies = [
]
[[package]]
name = "bindgen"
version = "0.72.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
dependencies = [
"bitflags 2.13.2",
"cexpr",
"clang-sys",
"itertools 0.13.0",
"log",
"prettyplease 0.2.37",
"proc-macro2",
"quote",
"regex",
"rustc-hash 2.1.3",
"shlex 1.3.0",
"syn 2.0.119",
]
[[package]]
name = "bit-set"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1537,6 +1557,12 @@ checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
[[package]]
name = "cfg_aliases"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
[[package]]
name = "cfg_aliases"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
@@ -2040,7 +2066,7 @@ version = "0.11.0-pre.4"
source = "git+https://github.com/tracel-ai/cubecl?rev=467522442c7b68f02c7ff0c84b2e249e36d8da91#467522442c7b68f02c7ff0c84b2e249e36d8da91"
dependencies = [
"bytemuck",
"cfg_aliases",
"cfg_aliases 0.2.2",
"cubecl-environment",
"derive-new",
"derive_more",
@@ -2161,7 +2187,7 @@ dependencies = [
"bytemuck",
"bytes",
"cfg-if 1.0.5",
"cfg_aliases",
"cfg_aliases 0.2.2",
"ciborium",
"embassy-futures",
"embassy-time",
@@ -2347,7 +2373,7 @@ dependencies = [
"buildid",
"bytemuck",
"cfg-if 1.0.5",
"cfg_aliases",
"cfg_aliases 0.2.2",
"cubecl-common",
"cubecl-environment",
"cubecl-ir",
@@ -2376,7 +2402,7 @@ dependencies = [
"buildid",
"bytemuck",
"cfg-if 1.0.5",
"cfg_aliases",
"cfg_aliases 0.2.2",
"cubecl-common",
"cubecl-environment",
"cubecl-ir",
@@ -2420,7 +2446,7 @@ source = "git+https://github.com/tracel-ai/cubecl?rev=467522442c7b68f02c7ff0c84b
dependencies = [
"bytemuck",
"cfg-if 1.0.5",
"cfg_aliases",
"cfg_aliases 0.2.2",
"cubecl-common",
"cubecl-core",
"cubecl-cpp",
@@ -3194,6 +3220,17 @@ dependencies = [
]
[[package]]
name = "enumn"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f9ed6b3789237c8a0c1c505af1c7eb2c560df6186f01b098c3a1064ea532f38"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "enumset"
version = "1.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3909,7 +3946,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12124de845cacfebedff80e877bb37b5b75c34c5a4c89e47e1cdd67fb6041325"
dependencies = [
"bitflags 2.13.2",
"cfg_aliases",
"cfg_aliases 0.2.2",
"cgl",
"dispatch2",
"glutin_egl_sys",
@@ -3933,7 +3970,7 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85edca7075f8fc728f28cb8fbb111a96c3b89e930574369e3e9c27eb75d3788f"
dependencies = [
"cfg_aliases",
"cfg_aliases 0.2.2",
"glutin",
"raw-window-handle",
"winit",
@@ -4356,6 +4393,15 @@ checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
[[package]]
name = "itertools"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
dependencies = [
"either",
]
[[package]]
name = "itertools"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
@@ -4749,7 +4795,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ddeaadb76e307b1d2b4836adee77910d405b9f6c1ca031bc81a7a398e2b22f2"
dependencies = [
"bytemuck",
"cfg_aliases",
"cfg_aliases 0.2.2",
"half",
"macerator-macros",
"moddef",
@@ -4941,7 +4987,7 @@ dependencies = [
"bit-set",
"bitflags 2.13.2",
"cfg-if 1.0.5",
"cfg_aliases",
"cfg_aliases 0.2.2",
"codespan-reporting",
"half",
"hashbrown 0.17.1",
@@ -5070,6 +5116,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
[[package]]
name = "nix"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4"
dependencies = [
"bitflags 2.13.2",
"cfg-if 1.0.5",
"cfg_aliases 0.1.1",
"libc",
]
[[package]]
name = "no_std_io2"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -5807,6 +5865,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec"
[[package]]
name = "pastey"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
[[package]]
name = "pbkdf2"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -7011,6 +7075,7 @@ dependencies = [
"serde",
"splatgen",
"toml",
"v4l2r",
]
[[package]]
@@ -7846,7 +7911,23 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6779878362b9bacadc7893eac76abe69612e8837ef746573c4a5239daf11990b"
dependencies = [
"bindgen",
"bindgen 0.65.1",
]
[[package]]
name = "v4l2r"
version = "0.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67d06043afd259b7e3f0b70bb8cc590b35c99006abc595399956392b4ebbe717"
dependencies = [
"anyhow",
"bindgen 0.72.1",
"bitflags 2.13.2",
"enumn",
"log",
"nix",
"pastey 0.2.3",
"thiserror 1.0.69",
]
[[package]]
@@ -8214,7 +8295,7 @@ dependencies = [
"bitflags 2.13.2",
"bytemuck",
"cfg-if 1.0.5",
"cfg_aliases",
"cfg_aliases 0.2.2",
"document-features",
"hashbrown 0.17.1",
"js-sys",
@@ -8245,7 +8326,7 @@ dependencies = [
"bit-vec",
"bitflags 2.13.2",
"bytemuck",
"cfg_aliases",
"cfg_aliases 0.2.2",
"document-features",
"hashbrown 0.17.1",
"indexmap",
@@ -8319,7 +8400,7 @@ dependencies = [
"block2 0.6.2",
"bytemuck",
"cfg-if 1.0.5",
"cfg_aliases",
"cfg_aliases 0.2.2",
"glow",
"glutin_wgl_sys",
"gpu-allocator",
@@ -8728,7 +8809,7 @@ dependencies = [
"block2 0.5.1",
"bytemuck",
"calloop 0.13.0",
"cfg_aliases",
"cfg_aliases 0.2.2",
"concurrent-queue",
"core-foundation 0.9.4",
"core-graphics 0.23.2",
Cargo.toml
+3
-2
diff --git a/Cargo.toml b/Cargo.toml
index 727c933..be59ea4 100644
@@ -75,8 +75,9 @@ egui = "0.37"
# Headless egui test harness (wgpu snapshots + interaction).
egui_kittest = { version = "0.37", features = ["wgpu"] }
pollster = "1"
# v4l2loopback output (Linux only). Fallback candidate if this proves
# awkward for output queues: edgefirst-v4l2.
# v4l2loopback output (Linux only). The frame-format writer logic is
# cross-platform and unit-tested; only the device writer itself is
# Linux-gated.
v4l2r = "0.0.8"
# Serialization / data for replay datasets.
PLAN.md
+9
-2
diff --git a/PLAN.md b/PLAN.md
index aeeda76..6e76188 100644
@@ -99,7 +99,11 @@ and no Kinect:
offscreen paint pass.
* The **Kinect v2 integration cannot be tested in the container**; replay
sources stand in for it until hardware testing happens on a real
machine.
machine. The same goes for the **v4l2loopback device**: creating
`/dev/video*` needs the kernel module, so the loopback sink's device
writer is exercised on a real machine (the frame-format writer,
negotiation, and error handling are unit-tested here, against real
ioctl failures where the container can produce them).
## Configuration
@@ -120,7 +124,10 @@ loopback output) join the same arg surface.
- [x] 3. Live Kinect capture + ellipsoid proxy recoloring from side
cameras — the capture crate is code-complete but **awaits hardware
verification** (the container has no USB; see Testing)
- [ ] 4. v4l2loopback sink
- [x] 4. v4l2loopback sink — the frame-format writer (RGBA→YUYV,
negotiation, error handling, latest-frame handoff) is fully
tested; attaching a real `/dev/video*` device needs the kernel
module on a real machine (see Testing)
- [ ] 5. Polish: cross-camera color calibration, WB-lock webcams, seam tuning
## Risks (ranked)
README.md
+19
-1
diff --git a/README.md b/README.md
index 5d796cd..49a835b 100644
@@ -25,7 +25,7 @@ Targets Ubuntu 24.04 (x86-64) and Windows on ARM64.
## Status
Build-order steps 1–3 are complete. The full pipeline runs on synthetic
Build-order steps 1–4 are complete. The full pipeline runs on synthetic
data through the *real* depth path: a software Kinect emits raw
millimetre depth maps plus RGB (with deterministic sensor noise) and
side-camera images, and `splatgen` culls the background, removes
@@ -50,6 +50,24 @@ plus raw `frame_NNNN.depth`/`.rgb`/`.left`/`.right` files. Real rig
recordings can be converted into that shape; synthetic ones are
generated for dev/CI.
## Virtual webcam (Linux)
The rendered virtual-camera frames can be published as a webcam for
Teams/Slack/Chrome via
[v4l2loopback](https://github.com/v4l2loopback/v4l2loopback):
```shell
sudo apt install v4l2loopback-dkms
sudo modprobe v4l2loopback video_nr=10 card_label="kinect-view" exclusive_caps=1
kinect-view --loopback-path /dev/video10
```
`exclusive_caps=1` is what makes Chrome-based consumers see the device:
it announces output capability until kinect-view attaches, then
capture capability only. A failing or removed device disables the sink
(with a logged error) rather than stopping the viewer. Off Linux the
setting is rejected up front.
- `cargo run -p sink` — the viewer (needs a GPU, or lavapipe/Vulkan)
- `cargo run -p render --release --example bench` — pipeline/render
frame times (`vulkaninfo --summary` shows which adapter is in use)
sink/Cargo.toml
+3
-0
diff --git a/sink/Cargo.toml b/sink/Cargo.toml
index 095ac62..b6bb980 100644
@@ -26,6 +26,9 @@ splatgen.workspace = true
toml.workspace = true
capture.workspace = true
[target.'cfg(target_os = "linux")'.dependencies]
v4l2r.workspace = true
[dev-dependencies]
egui_kittest.workspace = true
image.workspace = true
sink/src/config.rs
+26
-0
diff --git a/sink/src/config.rs b/sink/src/config.rs
index cf8f801..711084c 100644
@@ -200,6 +200,13 @@ pub struct Args {
)]
pub rig_right_intrinsics: Intrinsics,
// -- output ----------------------------------------------------------
/// Path to a v4l2loopback output device (e.g. /dev/video10): the
/// rendered virtual-camera frames become a webcam Teams/Slack can
/// use. Linux only.
#[arg(long, env = "KINECT_VIEW_LOOPBACK_PATH")]
pub loopback_path: Option<PathBuf>,
// -- source selection ----------------------------------------------
/// Which source feeds the pipeline: synthetic, a recording, or the
/// live Kinect rig.
@@ -398,6 +405,14 @@ impl Args {
SourceArg::Kinect => SourceSelection::Kinect,
};
// The loopback sink is Linux-only; reject the setting rather
// than silently ignoring it.
let loopback = self.loopback_path.clone();
#[cfg(not(target_os = "linux"))]
if loopback.is_some() {
anyhow::bail!("the v4l2loopback sink is Linux-only; unset loopback-path");
}
Ok(ResolvedConfig {
rig: Rig {
left,
@@ -408,6 +423,7 @@ impl Args {
fps,
background: self.background.0.into(),
source,
loopback,
splatgen,
noise,
})
@@ -426,6 +442,8 @@ pub struct ResolvedConfig {
pub background: Vec3,
/// Which source feeds the pipeline.
pub source: SourceSelection,
/// The v4l2loopback output device, when enabled (Linux only).
pub loopback: Option<PathBuf>,
/// Depth-processing parameters (`splatgen`).
pub splatgen: SplatGenParams,
/// Synthetic sensor noise.
@@ -774,6 +792,14 @@ mod tests {
.expect_err("missing recording path");
assert!(err.to_string().contains("recording-path"));
// The loopback path flows through (and defaults to off).
let args = merged(&["app"], None).resolve().expect("resolves");
assert_eq!(args.loopback, None);
let args = merged(&["app"], Some("loopback_path = \"/dev/video10\"\n"))
.resolve()
.expect("resolves");
assert_eq!(args.loopback, Some(PathBuf::from("/dev/video10")));
// Proxy settings flow through.
let args = merged(
&["app"],
sink/src/lib.rs
+2
-0
diff --git a/sink/src/lib.rs b/sink/src/lib.rs
index 81ede11..ce7b628 100644
@@ -18,8 +18,10 @@
pub mod app;
pub mod config;
pub mod loopback;
pub mod model;
pub use app::KinectViewApp;
pub use config::{Args, ResolvedConfig};
pub use loopback::LoopbackSink;
pub use model::{FrameStats, ViewerModel, ViewerState};
sink/src/loopback.rs
+470
-0
diff --git a/sink/src/loopback.rs b/sink/src/loopback.rs
new file mode 100644
index 0000000..36cb963
@@ -0,0 +1,470 @@
//! The v4l2loopback virtual webcam (PLAN.md, pipeline step 5 — the
//! Linux sink).
//!
//! Rendered virtual-camera frames are converted to YUYV and written to
//! a [v4l2loopback](https://github.com/v4l2loopback/v4l2loopback)
//! device, which conferencing tools (Teams, Slack, Chrome) then pick
//! up as an ordinary webcam.
//!
//! The device half is Linux-only; everything else — the RGBA→YUYV
//! conversion, the latest-frame handoff, the error handling — is
//! cross-platform and unit-tested. Creating an actual `/dev/video*`
//! device needs the v4l2loopback kernel module, which this container
//! cannot load; the device paths are exercised on a real machine.
//!
//! Typical setup (the `exclusive_caps` mode is what makes
//! Chrome-based consumers see the device):
//!
//! ```text
//! sudo apt install v4l2loopback-dkms
//! sudo modprobe v4l2loopback video_nr=10 card_label="kinect-view" exclusive_caps=1
//! kinect-view --loopback-path /dev/video10
//! ```
use std::path::Path;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread::JoinHandle;
use anyhow::{Context, Result, ensure};
use render::RenderedImage;
/// One pending frame handed to the writer thread.
struct PendingFrame {
rgba: Arc<[u8]>,
width: u32,
height: u32,
}
/// Where converted frames go: the v4l2loopback device, or a test
/// collector.
pub trait LoopbackWriter: Send + 'static {
/// Write one YUYV frame — exactly `width * height * 2` bytes.
fn write_frame(&mut self, yuyv: &[u8], width: u32, height: u32) -> Result<()>;
}
/// The loopback sink: receives rendered frames, converts them, and
/// writes them out — on the caller's thread ([`LoopbackSink::with_writer`])
/// or on a dedicated writer thread with latest-frame semantics
/// ([`LoopbackSink::open`], so a slow device never stalls the UI).
pub struct LoopbackSink {
mode: Mode,
}
enum Mode {
/// Writes on the calling thread (tests, tools).
Inline(Box<dyn LoopbackWriter>),
/// The caller stores the latest frame; a writer thread converts and
/// writes it.
Threaded {
latest: Arc<Mutex<Option<PendingFrame>>>,
error: Arc<Mutex<Option<String>>>,
shutdown: Arc<AtomicBool>,
worker: Option<JoinHandle<()>>,
},
}
impl LoopbackSink {
/// Open the v4l2loopback output device at `path` and negotiate a
/// YUYV format at `width x height`.
///
/// The device must report video-output capability (with
/// `exclusive_caps=1`, that is the *unattached* state — once this
/// sink produces, the device announces capture capability for the
/// conferencing tool). Fails clearly on Linux when the device is
/// absent, not a V4L2 output device, or refuses the format — and
/// on every non-Linux platform.
pub fn open(path: impl AsRef<Path>, width: u32, height: u32) -> Result<Self> {
let writer = open_device_writer(path.as_ref(), width, height)?;
Ok(Self::threaded(writer))
}
/// A sink writing to an arbitrary [`LoopbackWriter`] on the calling
/// thread — primarily for tests.
pub fn with_writer(writer: Box<dyn LoopbackWriter>) -> Self {
Self {
mode: Mode::Inline(writer),
}
}
/// Is the sink still active? (A writer failure disables it.)
pub fn is_active(&self) -> bool {
match &self.mode {
Mode::Inline(_) => true,
Mode::Threaded { error, .. } => error.lock().unwrap().is_none(),
}
}
/// Push one rendered frame. Cheap: with a real device this only
/// stores the latest frame; conversion and writing happen on the
/// writer thread.
pub fn push(&mut self, image: &RenderedImage) -> Result<()> {
match &mut self.mode {
Mode::Inline(writer) => {
let yuyv = rgba_to_yuyv(&image.rgba, image.width, image.height)?;
writer.write_frame(&yuyv, image.width, image.height)
}
Mode::Threaded { latest, error, .. } => {
if let Some(error) = error.lock().unwrap().take() {
anyhow::bail!("loopback writer failed: {error}");
}
let frame = PendingFrame {
rgba: Arc::from(image.rgba.as_slice()),
width: image.width,
height: image.height,
};
*latest.lock().unwrap() = Some(frame);
Ok(())
}
}
}
/// Build the threaded sink and start its writer loop.
fn threaded(writer: Box<dyn LoopbackWriter>) -> Self {
let latest = Arc::new(Mutex::new(None::<PendingFrame>));
let error = Arc::new(Mutex::new(None::<String>));
let shutdown = Arc::new(AtomicBool::new(false));
let worker = {
let latest = latest.clone();
let error = error.clone();
let shutdown = shutdown.clone();
std::thread::spawn(move || {
let mut writer = writer;
loop {
if shutdown.load(Ordering::Relaxed) {
break;
}
let frame = latest.lock().unwrap().take();
let Some(frame) = frame else {
std::thread::sleep(std::time::Duration::from_millis(5));
continue;
};
let result = (|| -> Result<()> {
let yuyv = rgba_to_yuyv(&frame.rgba, frame.width, frame.height)?;
writer.write_frame(&yuyv, frame.width, frame.height)
})();
if let Err(e) = result {
*error.lock().unwrap() = Some(format!("{e:#}"));
break;
}
}
})
};
Self {
mode: Mode::Threaded {
latest,
error,
shutdown,
worker: Some(worker),
},
}
}
}
impl Drop for LoopbackSink {
fn drop(&mut self) {
if let Mode::Threaded {
shutdown, worker, ..
} = &mut self.mode
{
shutdown.store(true, Ordering::Relaxed);
if let Some(worker) = worker.take() {
let _ = worker.join();
}
}
}
}
/// Convert an RGBA8 frame to YUYV (packed YUV 4:2:2): full-range
/// BT.601, the convention virtual-webcam consumers expect. Alpha is
/// ignored — rendered frames are already composited over the
/// background.
///
/// Layout: `[Y0, U, Y1, V]` per horizontal pixel pair, the chroma
/// averaged over the pair.
pub fn rgba_to_yuyv(rgba: &[u8], width: u32, height: u32) -> Result<Vec<u8>> {
ensure!(
width.is_multiple_of(2),
"YUYV needs an even width, got {width}"
);
let pixel_count = width as usize * height as usize;
ensure!(
rgba.len() == pixel_count * 4,
"expected {pixel_count} RGBA pixels, got {}",
rgba.len() / 4
);
let mut yuyv = vec![0_u8; pixel_count * 2];
for pair in 0..(width as usize * height as usize) / 2 {
// The pixel pair this YUYV quartet covers.
let px = pair * 2;
let i = px * 4;
let (r0, g0, b0) = (rgba[i] as f32, rgba[i + 1] as f32, rgba[i + 2] as f32);
let (r1, g1, b1) = (rgba[i + 4] as f32, rgba[i + 5] as f32, rgba[i + 6] as f32);
// BT.601 full range. Chroma is linear in RGB, so averaging the
// pair's RGB is exactly averaging the pair's chroma.
let y0 = clamp_u8(0.299 * r0 + 0.587 * g0 + 0.114 * b0 + 0.5);
let y1 = clamp_u8(0.299 * r1 + 0.587 * g1 + 0.114 * b1 + 0.5);
let (ra, ga, ba) = ((r0 + r1) * 0.5, (g0 + g1) * 0.5, (b0 + b1) * 0.5);
let u = clamp_u8(-0.168_736 * ra - 0.331_264 * ga + 0.5 * ba + 128.5);
let v = clamp_u8(0.5 * ra - 0.418_688 * ga - 0.081_312 * ba + 128.5);
let o = pair * 4;
yuyv[o] = y0;
yuyv[o + 1] = u;
yuyv[o + 2] = y1;
yuyv[o + 3] = v;
}
Ok(yuyv)
}
fn clamp_u8(x: f32) -> u8 {
x.clamp(0.0, 255.0) as u8
}
/// Open the v4l2loopback device writer. Linux only — elsewhere the
/// sink does not exist and the configuration is rejected up front.
#[cfg(target_os = "linux")]
fn open_device_writer(path: &Path, width: u32, height: u32) -> Result<Box<dyn LoopbackWriter>> {
use std::fs::OpenOptions;
use v4l2r::ioctl::{Capabilities, s_fmt};
use v4l2r::{Format, PixelFormat, PlaneLayout, QueueType};
let mut file = OpenOptions::new()
.write(true)
.open(path)
.with_context(|| format!("opening {}", path.display()))?;
// It must be an output device.
let capability = query_capabilities(&mut file, path)?;
if !capability
.device_caps()
.contains(Capabilities::VIDEO_OUTPUT)
{
anyhow::bail!(
"{} does not accept video output. With exclusive_caps=1, pass the \
v4l2loopback device *before* a producer attaches (it announces \
output capability only until then)",
path.display()
);
}
// Negotiate YUYV at the render size.
let frame_bytes = width as usize * height as usize * 2;
let format = Format {
width,
height,
pixelformat: PixelFormat::from_fourcc(b"YUYV"),
plane_fmt: vec![PlaneLayout {
bytesperline: width * 2,
sizeimage: (frame_bytes) as u32,
}],
};
let negotiated: Format = s_fmt(&mut file, (QueueType::VideoOutput, &format))
.with_context(|| format!("negotiating the output format on {}", path.display()))?;
if negotiated.width != width
|| negotiated.height != height
|| negotiated.pixelformat != format.pixelformat
{
anyhow::bail!(
"{} refused the format: asked for YUYV {width}x{height}, got {:?} {}x{} \
(the device was probably created with fixed dimensions — reload \
the module or set them accordingly)",
path.display(),
negotiated.pixelformat.to_fourcc(),
negotiated.width,
negotiated.height
);
}
Ok(Box::new(DeviceWriter {
file,
width,
height,
frame_bytes,
}))
}
#[cfg(target_os = "linux")]
fn query_capabilities(file: &mut std::fs::File, path: &Path) -> Result<v4l2r::ioctl::Capability> {
use v4l2r::ioctl::{QueryCapError, querycap};
match querycap(file) {
Ok(capability) => Ok(capability),
Err(QueryCapError::IoctlError(e)) => {
anyhow::bail!("{} is not a V4L2 device ({e})", path.display())
}
}
}
#[cfg(not(target_os = "linux"))]
fn open_device_writer(_path: &Path, _width: u32, _height: u32) -> Result<Box<dyn LoopbackWriter>> {
anyhow::bail!("the v4l2loopback sink is Linux-only")
}
/// The Linux device writer: frames go out with plain `write()`, one
/// YUYV frame per call.
#[cfg(target_os = "linux")]
struct DeviceWriter {
file: std::fs::File,
width: u32,
height: u32,
frame_bytes: usize,
}
#[cfg(target_os = "linux")]
impl LoopbackWriter for DeviceWriter {
fn write_frame(&mut self, yuyv: &[u8], width: u32, height: u32) -> Result<()> {
use std::io::Write;
ensure!(
width == self.width && height == self.height,
"frame size changed: wrote {}x{}, now {width}x{height}",
self.width,
self.height
);
ensure!(
yuyv.len() == self.frame_bytes,
"expected a {}-byte frame, got {}",
self.frame_bytes,
yuyv.len()
);
self.file
.write_all(yuyv)
.context("writing the frame (was the device removed?)")
}
}
#[cfg(test)]
mod tests {
use super::*;
/// A writer that records everything it receives.
struct Collector {
frames: CollectedFrames,
}
/// What the collector writer received: `(yuyv, width, height)` per
/// frame.
type CollectedFrames = Arc<Mutex<Vec<(Vec<u8>, u32, u32)>>>;
impl LoopbackWriter for Collector {
fn write_frame(&mut self, yuyv: &[u8], width: u32, height: u32) -> Result<()> {
self.frames
.lock()
.unwrap()
.push((yuyv.to_vec(), width, height));
Ok(())
}
}
/// A writer that always fails.
struct Failing;
impl LoopbackWriter for Failing {
fn write_frame(&mut self, _: &[u8], _: u32, _: u32) -> Result<()> {
anyhow::bail!("writer exploded")
}
}
fn image(width: u32, height: u32, rgba: &[u8]) -> RenderedImage {
RenderedImage {
width,
height,
rgba: rgba.to_vec(),
}
}
#[test]
fn yuyv_known_colours() {
// BT.601 full range: Y' = 0.299R + 0.587G + 0.114B. Chroma
// belongs to a pixel *pair*: computed from their average RGB.
// A 2x1 image: red | white.
let rgba = [255, 0, 0, 255, 255, 255, 255, 255];
let out = rgba_to_yuyv(&rgba, 2, 1).expect("converts");
assert_eq!(out.len(), 4);
// Luma per pixel: red ≈ 76, white = 255.
assert_eq!(out[0], 76);
assert_eq!(out[2], 255);
// Chroma of the red|white pair: averaged RGB (255, 127.5, 127.5)
// → U ≈ 107, V ≈ 192.
assert_eq!(out[1], 106);
assert_eq!(out[3], 192);
// Black is neutral: Y 0, U/V 128.
let out = rgba_to_yuyv(&[0, 0, 0, 255, 0, 0, 0, 255], 2, 1).expect("converts");
assert_eq!(out, vec![0, 128, 0, 128]);
// Odd width is rejected.
assert!(rgba_to_yuyv(&[0, 0, 0, 255], 1, 1).is_err());
// Wrong buffer length is rejected.
assert!(rgba_to_yuyv(&[0; 7], 2, 1).is_err());
}
#[test]
fn inline_sink_pushes_converted_frames() {
let frames = Arc::new(Mutex::new(Vec::new()));
let mut sink = LoopbackSink::with_writer(Box::new(Collector {
frames: frames.clone(),
}));
// A 2x2 red frame.
let rgba = [255, 0, 0, 255].repeat(4);
sink.push(&image(2, 2, &rgba)).expect("push");
sink.push(&image(2, 2, &rgba)).expect("push");
let frames = frames.lock().unwrap();
assert_eq!(frames.len(), 2, "every pushed frame is written");
for (yuyv, w, h) in frames.iter() {
assert_eq!((*w, *h), (2, 2));
assert_eq!(yuyv.len(), 2 * 2 * 2);
// All red: each pixel pair is one YUYV quartet
// [Y0, U, Y1, V] = [76, 85, 76, 255]; two rows.
assert_eq!(yuyv, &[76, 85, 76, 255, 76, 85, 76, 255]);
}
}
#[test]
fn writer_errors_disable_the_sink() {
let mut sink = LoopbackSink::with_writer(Box::new(Failing));
let rgba = [0, 0, 0, 255].repeat(4);
assert!(sink.push(&image(2, 2, &rgba)).is_err());
}
#[cfg(target_os = "linux")]
#[test]
fn device_open_failures_are_clear() {
// A nonexistent path.
let err = match LoopbackSink::open("/nonexistent/video", 640, 480) {
Ok(_) => panic!("opened a nonexistent device?"),
Err(e) => e,
};
assert!(err.to_string().contains("opening"), "got: {err:#}");
// A file that is not a V4L2 device.
let err = match LoopbackSink::open("/dev/null", 640, 480) {
Ok(_) => panic!("/dev/null is not a video device"),
Err(e) => e,
};
assert!(
err.to_string().contains("not a V4L2 device"),
"got: {err:#}"
);
}
#[cfg(not(target_os = "linux"))]
#[test]
fn device_open_fails_off_linux() {
let err = match LoopbackSink::open("/dev/video0", 640, 480) {
Ok(_) => panic!("opened a loopback device off Linux?"),
Err(e) => e,
};
assert!(err.to_string().contains("Linux-only"));
}
}
sink/src/main.rs
+25
-0
diff --git a/sink/src/main.rs b/sink/src/main.rs
index 9edf7b4..0098444 100644
@@ -70,7 +70,32 @@ fn main() -> eframe::Result<()> {
};
let renderer = render::SplatRenderer::new(resolved.background);
// The loopback sink (Linux): the rendered frames become a webcam.
let loopback = match &resolved.loopback {
Some(path) => {
let sink = match sink::LoopbackSink::open(
path,
rig.left.resolution.x,
rig.left.resolution.y,
) {
Ok(sink) => sink,
Err(error) => {
eprintln!("opening the loopback device {}: {error:#}", path.display());
std::process::exit(2);
}
};
log::info!("loopback sink active on {}", path.display());
Some(sink)
}
None => None,
};
let model = sink::ViewerModel::new(rig, source, renderer);
let model = match loopback {
Some(sink) => model.with_loopback(sink),
None => model,
};
let app = sink::KinectViewApp { model };
let options = eframe::NativeOptions {
sink/src/model.rs
+33
-1
diff --git a/sink/src/model.rs b/sink/src/model.rs
index 9e90469..307a0c4 100644
@@ -14,6 +14,8 @@ use egui::{ColorImage, TextureHandle, TextureOptions};
use render::{RenderedImage, SplatRenderer};
use rig::{Camera, FrameSource, Rig, SplatBatch};
use crate::loopback::LoopbackSink;
/// The viewer's interactive state, independent of egui internals so it
/// can be asserted on directly in tests.
#[derive(Debug, Clone, Copy, PartialEq)]
@@ -51,6 +53,9 @@ pub struct ViewerModel {
last_image: Option<RenderedImage>,
texture: Option<TextureHandle>,
stats: FrameStats,
/// The v4l2loopback virtual webcam, when enabled. Disabled
/// automatically if the device starts failing.
loopback: Option<LoopbackSink>,
}
impl ViewerModel {
@@ -67,9 +72,22 @@ impl ViewerModel {
last_image: None,
texture: None,
stats: FrameStats::default(),
loopback: None,
}
}
/// Send the rendered virtual-camera frames to the v4l2loopback
/// virtual webcam (see [`crate::loopback`]).
pub fn with_loopback(mut self, sink: LoopbackSink) -> Self {
self.loopback = Some(sink);
self
}
/// Is the loopback sink attached and healthy?
pub fn loopback_active(&self) -> bool {
self.loopback.as_ref().is_some_and(LoopbackSink::is_active)
}
/// The interactive state.
pub fn state(&self) -> ViewerState {
self.state
@@ -125,6 +143,15 @@ impl ViewerModel {
self.stats.splats = render_stats.total_splats;
self.stats.visible_splats = render_stats.visible_splats;
self.last_image = Some(image);
// Feed the loopback sink, if any: a failing device disables it
// rather than taking the viewer down.
if let Some(sink) = &mut self.loopback
&& let Err(error) = sink.push(self.last_image.as_ref().expect("just stored"))
{
log::error!("loopback sink disabled: {error:#}");
self.loopback = None;
}
Ok(())
}
@@ -148,8 +175,13 @@ impl ViewerModel {
);
ui.label(format!("{:.0}%", self.state.t * 100.0));
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
let loopback = if self.loopback_active() {
" · loopback on"
} else {
""
};
ui.label(format!(
"frame {} · {} splats · {} visible · {:.1} ms",
"frame {} · {} splats · {} visible · {:.1} ms{loopback}",
self.stats.frame_index,
self.stats.splats,
self.stats.visible_splats,
sink/tests/ui.rs
+56
-0
diff --git a/sink/tests/ui.rs b/sink/tests/ui.rs
index f7b2abe..5b9254f 100644
@@ -239,6 +239,62 @@ scene_subject_position = [0.0, 0.0, 2.5]
}
#[test]
fn loopback_sink_receives_rendered_frames() {
// End to end: the viewer pushes every rendered frame to the
// loopback sink (a collector writer stands in for the device).
type CollectedFrames = std::sync::Arc<std::sync::Mutex<Vec<(Vec<u8>, u32, u32)>>>;
pub struct Collector {
frames: CollectedFrames,
}
impl sink::loopback::LoopbackWriter for Collector {
fn write_frame(&mut self, yuyv: &[u8], width: u32, height: u32) -> anyhow::Result<()> {
self.frames
.lock()
.unwrap()
.push((yuyv.to_vec(), width, height));
Ok(())
}
}
let frames: CollectedFrames = Default::default();
let model = small_model().with_loopback(sink::LoopbackSink::with_writer(Box::new(Collector {
frames: frames.clone(),
})));
let mut harness = harness_for(model, false);
harness.run();
let frames = frames.lock().unwrap();
assert!(!frames.is_empty(), "the sink received no frames");
for (yuyv, w, h) in frames.iter() {
// The virtual camera renders at webcam resolution, YUYV is
// two bytes per pixel.
assert_eq!((*w, *h), (80, 60));
assert_eq!(yuyv.len(), 80 * 60 * 2);
}
// And the model reports the sink as active.
assert!(harness.state().loopback_active());
}
#[test]
fn loopback_sink_failure_disables_it() {
pub struct Failing;
impl sink::loopback::LoopbackWriter for Failing {
fn write_frame(&mut self, _: &[u8], _: u32, _: u32) -> anyhow::Result<()> {
anyhow::bail!("device went away")
}
}
let model = small_model().with_loopback(sink::LoopbackSink::with_writer(Box::new(Failing)));
let mut harness = harness_for(model, false);
harness.run();
// The failing writer is dropped and the model reports the sink
// inactive — the viewer itself keeps running.
assert!(!harness.state().loopback_active());
}
#[test]
fn recording_source_renders() {
// The `--source recording` path end to end: record a few synthetic
// frames to disk, load them back through the real config pipeline,