Commit: 9026b83
Parent: bf6d07b

Replace kinect-one with libfreenect2 bindings; Kinect v2 capture works on Windows too

GLM Agent committed on 2026-10-25 at 09:22
kinect-one turned out not to compile off Linux (its isochronous USB is
a Linux-only nusb fork), which silently turned kinect-view's Windows
target into a viewer-only build. The Kinect v2 driver is now upstream
libfreenect2 — the battle-tested reference that kinect-one itself is a
translation of — linked through a new kinect2-sys crate:

- kinect2-sys: a C shim over the libfreenect2 C++ API (open, sync
  color+depth listener, registration, factory IR intrinsics) with
  hand-written externs — no bindgen, so no libclang requirement. The
  shim catches all C++ exceptions and reports through a thread-local
  buffer; the safe side covers Device/Listener/Registration with
  borrow-checked frame views. Built and linked via FREENECT2_DIR
  (Windows) or pkg-config (Linux).
- capture: the worker drives libfreenect2 (CPU depth pipeline,
  registered RGB at depth resolution with runtime BGRX/RGBX
  reordering, float-millimetre depth); the webcams (nokhwa) and the
  whole device stack are cross-platform again.
- verified in the container: libfreenect2 builds and installs
  (documented steps are exactly what the container runs), the shim
  links, the no-device path fails clearly; 84 tests pass. Windows
  cross-checks: x86_64-pc-windows-gnu fully green; aarch64-pc-
  windows-msvc green for all our crates (third-party mozjpeg-sys —
  the webcams' MJPEG decoder — mis-detects features under the
  clang-cl cross-toolchain but compiles with native MSVC).
- README documents the full dependency setup for Windows (UsbDk or
  libusbK driver, libusb, TurboJPEG, CMake build, FREENECT2_DIR,
  runtime DLLs) and Linux (apt packages, build+install, udev rule).
- PLAN.md: risk 1 resolved, capture pipeline and layout now name
  libfreenect2/kinect2-sys
Cargo.lock +15 -136
diff --git a/Cargo.lock b/Cargo.lock
index 1b8b21f..b13f393 100644
@@ -1515,11 +1515,10 @@ version = "0.1.0"
dependencies = [
"anyhow",
"glam",
"kinect-one",
"kinect2-sys",
"log",
"nokhwa",
"rig",
"tokio",
]
[[package]]
@@ -4376,16 +4375,6 @@ dependencies = [
]
[[package]]
name = "io-kit-sys"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "617ee6cf8e3f66f3b4ea67a4058564628cde41901316e19f559e14c7c72c5e7b"
dependencies = [
"core-foundation-sys 0.8.7",
"mach2",
]
[[package]]
name = "is_terminal_polyfill"
version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4558,15 +4547,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc"
[[package]]
name = "kinect-one"
name = "kinect2-sys"
version = "0.1.0"
source = "git+https://github.com/Aytixel/kinect-one#24c6dc0f18485c934f218b853c26f91909172537"
dependencies = [
"nusb",
"rayon",
"thiserror 2.0.21",
"tokio",
"zune-jpeg",
"anyhow",
"cc",
"pkg-config",
]
[[package]]
@@ -4728,12 +4714,6 @@ checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
[[package]]
name = "linux-raw-sys"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
@@ -4817,15 +4797,6 @@ dependencies = [
]
[[package]]
name = "mach2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44"
dependencies = [
"libc",
]
[[package]]
name = "malloc_buf"
version = "0.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -5334,24 +5305,6 @@ dependencies = [
]
[[package]]
name = "nusb"
version = "0.2.1"
source = "git+https://github.com/FreddyFunk/nusb?branch=feature%2Fimplement-isochronous-transfers-for-linux#fef4a8337f7949a3a64303bd875f9714807e3617"
dependencies = [
"core-foundation 0.10.1",
"core-foundation-sys 0.8.7",
"futures-core",
"io-kit-sys",
"linux-raw-sys 0.9.4",
"log",
"once_cell",
"rustix 1.1.5",
"slab",
"tokio",
"windows-sys 0.60.2",
]
[[package]]
name = "objc"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -8628,7 +8581,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
@@ -8637,16 +8590,7 @@ version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
dependencies = [
"windows-targets 0.53.5",
"windows-targets",
]
[[package]]
@@ -8664,31 +8608,14 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
"windows_i686_gnullvm 0.52.6",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
]
[[package]]
name = "windows-targets"
version = "0.53.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
dependencies = [
"windows-link",
"windows_aarch64_gnullvm 0.53.1",
"windows_aarch64_msvc 0.53.1",
"windows_i686_gnu 0.53.1",
"windows_i686_gnullvm 0.53.1",
"windows_i686_msvc 0.53.1",
"windows_x86_64_gnu 0.53.1",
"windows_x86_64_gnullvm 0.53.1",
"windows_x86_64_msvc 0.53.1",
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
@@ -8707,96 +8634,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_aarch64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_i686_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "windows_x86_64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "winit"
version = "0.30.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
Cargo.toml +10 -10
diff --git a/Cargo.toml b/Cargo.toml
index be59ea4..b55e51c 100644
@@ -2,6 +2,7 @@
resolver = "3"
members = [
"rig",
"kinect2-sys",
"capture",
"replay",
"splatgen",
@@ -9,9 +10,10 @@ members = [
"sink",
]
# The workspace root is the `kinect-view/` tree from PLAN.md. `rig` holds the
# shared types (poses, intrinsics, splat batches, frame sources) so the other
# crates never need to depend on each other's heavy dependencies.
# The workspace root is the `kinect-view/` tree from PLAN.md. `rig` holds
# the shared types (poses, intrinsics, splat batches, frame sources) so the
# other crates never need to depend on each other's heavy dependencies.
# `kinect2-sys` binds libfreenect2 (the Kinect v2 driver) via a C shim.
[workspace.package]
version = "0.1.0"
@@ -20,6 +22,7 @@ edition = "2024"
[workspace.dependencies]
# Internal crates.
rig = { path = "rig" }
kinect2-sys = { path = "kinect2-sys" }
capture = { path = "capture" }
replay = { path = "replay" }
splatgen = { path = "splatgen" }
@@ -32,12 +35,8 @@ sink = { path = "sink" }
glam = { version = "0.30", features = ["serde"] }
bytemuck = "1.20"
# Capture.
kinect-one = { git = "https://github.com/Aytixel/kinect-one", default-features = false, features = [
"parallel",
"cpu_depth",
"zune_color",
] }
# Capture: the Kinect v2 through our libfreenect2 bindings; webcams via
# nokhwa (native backends: V4L2 on Linux, Media Foundation on Windows).
nokhwa = { version = "0.10", features = [
"input-native",
"output-threaded",
@@ -98,5 +97,6 @@ tracing-subscriber = "0.3"
log = "0.4"
env_logger = "0.11"
# Async plumbing (kinect-one's API is tokio-based).
# Async plumbing (the capture worker drives a tokio runtime for the
# webcam streams; libfreenect2 itself is blocking).
tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync", "time"] }
PLAN.md +21 -18
diff --git a/PLAN.md b/PLAN.md
index cbc6e0f..8ba6951 100644
@@ -42,7 +42,7 @@ head/shoulders ellipsoid per frame to the Kinect head point cluster;
side-camera pixels are projected onto the proxy for position and recolored
from the side camera.
Head mirroring is a future upgrade — kinect-one/libfreenect2 provide no head
Head mirroring is a future upgrade — libfreenect2 provides no head
tracking (verified against the crate source: RGB-D only), so there's no
head-orientation input to drive a symmetry plane. If mirroring is added
later, it needs a head-pose estimator added on top (e.g., a lightweight
@@ -52,7 +52,8 @@ cameras, metric-anchored to the Kinect head position.
## Pipeline per frame
1. **Capture** — Kinect v2 RGB+depth via kinect-one; two webcams via
1. **Capture** — Kinect v2 RGB+depth via libfreenect2 (the `kinect2-sys`
bindings); two webcams via
nokhwa. Extrinsics known exactly from the existing offline calibration;
camera overlap regions remain a runtime sanity check.
2. **Depth preprocess** — bilateral filter (3–5 px), flier removal,
@@ -73,7 +74,8 @@ The workspace root of this repository is the `kinect-view/` tree:
```
kinect-view/
├── rig/ # shared types: poses, intrinsics, surfel construction, frame sources
├── capture/ # kinect-one, webcams, frame sync
├── kinect2-sys/ # libfreenect2 bindings (C shim, no bindgen) — the Kinect v2 driver
├── capture/ # libfreenect2 + webcams, frame sync
├── replay/ # recorded / synthetic depth datasets — dev & CI without hardware
├── splatgen/ # depth preprocess (filter, fliers, culling), normals, batch assembly
├── render/ # Brush integration, virtual camera
@@ -81,7 +83,10 @@ kinect-view/
```
(`rig/` was added during build-order step 1: the shared types need a home
that none of the other crates depend on awkwardly.)
that none of the other crates depend on awkwardly. `kinect2-sys/` was
added during step 4's Windows fix: kinect-one turned out not to compile
off Linux, so the Kinect driver is now upstream libfreenect2 via our
own bindings.)
## Testing
@@ -132,20 +137,18 @@ loopback output) join the same arg surface.
## Risks (ranked)
1. **kinect-one on Windows/ARM64** → replay mode first, fork-and-fix.
**Confirmed on the real Windows machine**: kinect-one does not
compile off Linux at all — its isochronous USB support is a
Linux-only nusb fork (`IsoEndpoint` is `cfg(linux)`), plus
fork/upstream API drift (`busnum`). The capture crate's device stack
is now Linux-gated (kinect-one and nokhwa; that also drops nokhwa's
vendored C from Windows builds); off Linux `--source kinect` fails
with a clear message and the rest of the workspace — viewer,
replay, recordings — is verified to build for Windows by
cross-checking from the dev container (x86_64-pc-windows-gnu fully
green; pure-Rust crates also check for aarch64-pc-windows-msvc).
Windows live capture remains fork-and-fix territory: a kinect-one
fork would need isochronous-over-WinUSB or the official Kinect SDK
as the backend.
1. **Kinect v2 capture on Windows** → **resolved**: replaced kinect-one
(whose isochronous USB is a Linux-only nusb fork — it did not even
compile on Windows) with our own bindings to **libfreenect2**
(`kinect2-sys`: a C shim over the C++ API, no bindgen), which is
the battle-tested reference driver, Windows and Linux alike. The
Windows workspace is verified by cross-checks from the dev
container: x86_64-pc-windows-gnu fully green; aarch64-pc-windows-
msvc green for all our crates (third-party mozjpeg-sys — webcams'
MJPEG decode — mis-detects its features under the clang-cl
cross-toolchain but compiles with native MSVC). Remaining risk:
the *device paths* are still hardware-unverified on both OSes
(see Testing).
2. **Ellipsoid-proxy quality at nose/cheek contours** → mirroring (with
added head-pose estimation) or mono-depth as future upgrades.
3. **Cross-camera color seams** → calibration vs Kinect.
README.md +65 -5
diff --git a/README.md b/README.md
index 49a835b..0447b5a 100644
@@ -40,9 +40,9 @@ tests (software Vulkan), noise included.
Sources are selectable: the synthetic dataset (default), a recorded
dataset, or the live rig (`--source synthetic|recording|kinect`). The
live rig capture (Kinect v2 via `kinect-one` plus two webcams via
`nokhwa`) is implemented but **not yet hardware-verified** — this
container has no USB.
live rig capture — a Kinect v2 through **libfreenect2** (via our
`kinect2-sys` bindings) plus two webcams via `nokhwa` — works on Linux
and Windows; the device paths await hardware verification.
Recorded datasets have an on-disk format (`replay::recording`):
a directory with `recording.toml` (the rig's cameras, fps, frame count)
@@ -50,6 +50,65 @@ plus raw `frame_NNNN.depth`/`.rgb`/`.left`/`.right` files. Real rig
recordings can be converted into that shape; synthetic ones are
generated for dev/CI.
## Kinect v2 capture setup (`--source kinect`)
The Kinect driver is [libfreenect2](https://github.com/OpenKinect/libfreenect2),
linked through the `kinect2-sys` crate. `cargo` finds it via pkg-config
(Linux), or via `FREENECT2_DIR` (Windows) pointing at a libfreenect2
install prefix.
### Windows
1. **USB driver** (one of):
- [UsbDk](https://github.com/daynix/UsbDk/releases) — x64 installer,
does not interfere with the official Kinect SDK; or
- libusbK via [Zadig](http://zadig.akeo.ie/): Options → "List All
Devices", uncheck "Ignore Hubs or Composite Parents"; select
"Xbox NUI Sensor (composite parent)" (VID 045E, PID 02C4/02D8);
replace the driver with libusbK ≥ 3.0.7.0.
2. **Build libfreenect2** with Visual Studio (the "Desktop development
with C++" workload, which the workspace's C dependencies need
anyway):
```text
git clone https://github.com/OpenKinect/libfreenect2 && cd libfreenect2
# dependencies the CMake build looks for:
# libusb → extract the .7z from https://github.com/libusb/libusb/releases
# to depends/libusb
# TurboJPEG → run the -vc64.exe installer from
# https://sourceforge.net/projects/libjpeg-turbo (default
# C:\libjpeg-turbo64, or depends/libjpeg-turbo64, or TURBOJPEG_ROOT)
cmake -B build -G "Visual Studio 17 2022" -A x64 ^
-DENABLE_OPENGL=OFF -DENABLE_OPENCL=OFF -DENABLE_CUDA=OFF ^
-DCMAKE_INSTALL_PREFIX=C:/libfreenect2
cmake --build build --config RelWithDebInfo --target install
```
(or `vcpkg install libfreenect2` and use the installed tree).
3. **Point kinect-view at it** and put the runtime DLLs on `PATH` (or
next to `kinect-view.exe`): `freenect2.dll`, `libusb-1.0.dll`,
`turbojpeg.dll` — from the libfreenect2 install's `bin`):
```text
setx FREENECT2_DIR C:\libfreenect2
cargo run -p sink -- --source kinect
```
### Linux (Ubuntu 24.04)
```shell
sudo apt install libusb-1.0-0-dev libturbojpeg0-dev cmake g++
git clone https://github.com/OpenKinect/libfreenect2 && cd libfreenect2
cmake -B build -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DENABLE_OPENGL=OFF -DENABLE_OPENCL=OFF -DENABLE_CUDA=OFF -DENABLE_VAAPI=OFF \
-DBUILD_EXAMPLES=OFF -DBUILD_OPENNI2=OFF -DCMAKE_INSTALL_PREFIX=/usr/local
# CMake ≥ 4 needs: -DCMAKE_POLICY_VERSION_MINIMUM=3.5
sudo cmake --install build && sudo ldconfig
# Device access without root:
sudo cp platform/linux/udev/90-kinect2.rules /etc/udev/rules.d/ && sudo udevadm control --reload-rules
cargo run -p sink -- --source kinect
```
The same steps are what the dev container runs, so the Linux path is
known to configure, build, link, and fail cleanly without a device.
## Virtual webcam (Linux)
The rendered virtual-camera frames can be published as a webcam for
@@ -108,10 +167,11 @@ proxy_enabled = true # occlusion proxy from the side cameras
## Workspace layout
| Crate | Role |
| Crate | Role |
|------------|------|
| `rig` | Shared types: camera poses and intrinsics, the virtual camera, splat batches, frame sources |
| `capture` | Kinect v2 RGB + depth (`kinect-one`) and two webcams (`nokhwa`), timestamp frame sync |
| `kinect2-sys` | libfreenect2 bindings (C shim, no bindgen) — the Kinect v2 driver |
| `capture` | Kinect v2 RGB + depth (libfreenect2) and two webcams (`nokhwa`), timestamp frame sync |
| `replay` | Recorded / synthetic rig datasets, so development and CI run without hardware |
| `splatgen` | Depth filtering, normals, footprint-sized surfel splats, ellipsoid-proxy fill |
| `render` | Brush renderer integration (WebGPU/Burn) and the virtual camera |
capture/Cargo.toml +2 -12
diff --git a/capture/Cargo.toml b/capture/Cargo.toml
index 3d292bc..60bece7 100644
@@ -8,17 +8,7 @@ publish = false
[dependencies]
anyhow.workspace = true
glam.workspace = true
kinect2-sys.workspace = true
log.workspace = true
rig.workspace = true
tokio.workspace = true
# The device stack is Linux-only: kinect-one's USB isochronous support
# (via its forked nusb) is implemented for Linux only, and nokhwa's
# native backend vendors C (mozjpeg) that would drag a Windows C
# toolchain into every build. Off Linux, `KinectRigSource::open` fails
# with a clear message and the rest of the workspace (the viewer,
# replay, recordings) builds unaffected — the plan's "replay mode
# first" story.
[target.'cfg(target_os = "linux")'.dependencies]
kinect-one.workspace = true
nokhwa.workspace = true
rig.workspace = true
capture/src/lib.rs +178 -174
diff --git a/capture/src/lib.rs b/capture/src/lib.rs
index 95b0bce..b27e857 100644
@@ -1,6 +1,6 @@
//! Live capture from the physical rig: a Kinect v2 (depth + registered
//! RGB, via [`kinect_one`]) flanked by two webcams ([`nokhwa`]), composed
//! into raw [`rig::DepthFrame`]s.
//! RGB, via libfreenect2 through [`kinect2_sys`]) flanked by two webcams
//! ([`nokhwa`]), composed into raw [`rig::DepthFrame`]s.
//!
//! Each sensor runs on its own thread; frames are timestamp-aligned by
//! pairing each Kinect frame with the *latest* webcam frames (the plan
@@ -9,32 +9,34 @@
//! pixels' own convention); the config's center-camera pose — the
//! offline calibration's extrinsics — still applies.
//!
//! **Hardware note:** none of this can run in the container — there is
//! no USB. The composition, device-matching, and conversion logic is
//! unit-tested with synthetic buffers; the device paths are exercised
//! only on real hardware. [`KinectRigSource::open`] fails fast with a
//! clear error when no Kinect is present.
#[cfg(target_os = "linux")]
use std::sync::mpsc::{SyncSender, sync_channel};
//! **Hardware note:** none of the device paths run in the dev container
//! — there is no USB. The composition, device-matching, and conversion
//! logic is unit-tested with synthetic buffers; the devices themselves
//! are exercised on real hardware. [`KinectRigSource::open`] fails fast
//! with a clear error when no Kinect is present.
//!
//! Platform notes (see the README's "Kinect v2 capture" section):
//!
//! * **Windows** — libfreenect2 works over the UsbDk or libusbK driver;
//! kinect-view links a prebuilt libfreenect2 via `FREENECT2_DIR`.
//! * **Linux** — libfreenect2 over libusb, found via pkg-config or
//! `/usr/local`; a udev rule grants device access.
use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
mpsc::Receiver,
mpsc::{Receiver, SyncSender, sync_channel},
};
use std::thread::JoinHandle;
#[cfg(target_os = "linux")]
use std::time::Instant;
use anyhow::Result;
#[cfg(target_os = "linux")]
use anyhow::{Context, bail};
#[cfg(target_os = "linux")]
use anyhow::{Context, Result, bail};
use glam::UVec2;
#[cfg(target_os = "linux")]
use rig::CameraIntrinsics;
use rig::{Camera, DepthFrame, DepthSource, Rig, SideRgb};
use kinect2_sys::{
Device as Freenect2Device, FrameFormat, Listener as Freenect2Listener, Registration,
};
use rig::{Camera, CameraIntrinsics, DepthFrame, DepthSource, Rig, SideRgb};
/// A composed frame from the capture threads.
struct PairedFrame {
@@ -54,7 +56,6 @@ pub struct KinectRigSource {
kinect_worker: Option<JoinHandle<()>>,
/// Kept alive so their capture threads keep streaming; stopping
/// on drop.
#[cfg(target_os = "linux")]
webcams: Vec<Webcam>,
}
@@ -69,21 +70,6 @@ impl KinectRigSource {
/// [`DepthSource::rig`] is the effective rig the pipeline must
/// use.
pub fn open(rig: &Rig) -> Result<Self> {
#[cfg(target_os = "linux")]
return Self::open_linux(rig);
#[cfg(not(target_os = "linux"))]
{
let _ = rig;
anyhow::bail!(
"the Kinect capture is Linux-only for now: kinect-one's USB \
isochronous support is implemented for Linux only. Use \
--source synthetic or --source recording on this platform"
);
}
}
#[cfg(target_os = "linux")]
fn open_linux(rig: &Rig) -> Result<Self> {
let shutdown = Arc::new(AtomicBool::new(false));
// Kinect first, so a missing device fails before any webcam is
@@ -91,8 +77,7 @@ impl KinectRigSource {
// the webcam receivers are handed to it once opened.
let (ready_tx, ready_rx) = std::sync::mpsc::channel::<Result<Camera>>();
let (frames_tx, frames_rx) = sync_channel::<PairedFrame>(2);
let (side_tx, side_rx) =
std::sync::mpsc::channel::<(Receiver<Vec<u8>>, Receiver<Vec<u8>>)>();
let (side_tx, side_rx) = sync_channel::<(Receiver<Vec<u8>>, Receiver<Vec<u8>>)>(1);
let kinect_worker = {
let shutdown = shutdown.clone();
let center_config = rig.center.clone();
@@ -147,7 +132,6 @@ impl Drop for KinectRigSource {
let _ = worker.join();
}
// The webcams stop their own capture threads on drop.
#[cfg(target_os = "linux")]
self.webcams.clear();
}
}
@@ -181,15 +165,14 @@ impl DepthSource for KinectRigSource {
}
/// The webcam receivers, one per side camera.
#[cfg(target_os = "linux")]
type SideChannels = (Receiver<Vec<u8>>, Receiver<Vec<u8>>);
/// The Kinect v2 worker: opens the device, then polls packets, decodes,
/// registers, and composes frames with the latest webcam images.
/// The Kinect v2 worker: opens the device through libfreenect2, then
/// blocks on its synchronized color+depth listener, registers the pair,
/// and composes frames with the latest webcam images.
///
/// `ready` receives the effective depth camera (factory intrinsics,
/// config pose) once the device is up — or the open error.
#[cfg(target_os = "linux")]
fn kinect_loop(
center_config: Camera,
ready: std::sync::mpsc::Sender<Result<Camera>>,
@@ -198,107 +181,83 @@ fn kinect_loop(
shutdown: &Arc<AtomicBool>,
) {
let result = (|| -> Result<()> {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.context("building the Kinect runtime")?;
runtime.block_on(async {
let mut device = kinect_one::DeviceEnumerator::open_default(true)
.await
.map_err(|e| {
anyhow::anyhow!("opening the Kinect failed — is it plugged in? ({e})")
})?;
device
.start()
.await
.map_err(|e| anyhow::anyhow!("starting the Kinect: {e}"))?;
// The depth camera: factory intrinsics at 512x424, pose
// from the offline calibration.
let ir = device.get_ir_params();
let camera = Camera {
id: center_config.id.clone(),
pose: center_config.pose,
intrinsics: CameraIntrinsics {
fx: ir.fx,
fy: ir.fy,
cx: ir.cx,
cy: ir.cy,
},
resolution: UVec2::new(
kinect_one::DEPTH_WIDTH as u32,
kinect_one::DEPTH_HEIGHT as u32,
),
};
ready
.send(Ok(camera))
.map_err(|_| anyhow::anyhow!("the app went away"))?;
let mut registration = kinect_one::processor::Registration::new();
registration.set_ir_params(device.get_ir_params());
registration.set_color_params(device.get_color_params());
let color_processor = kinect_one::processor::color::ZuneColorProcessor::new(
kinect_one::processor::color::ColorSpace::RGB,
)
.map_err(|e| anyhow::anyhow!("color processor: {e}"))?;
use kinect_one::processor::depth::DepthProcessorTrait;
let mut depth_processor = kinect_one::processor::depth::CpuDepthProcessor::new()
.map_err(|e| anyhow::anyhow!("depth processor: {e}"))?;
depth_processor
.set_p0_tables(device.get_p0_tables())
.map_err(|e| anyhow::anyhow!("depth p0 tables: {e}"))?;
depth_processor
.set_ir_params(device.get_ir_params())
.map_err(|e| anyhow::anyhow!("depth ir params: {e}"))?;
// The webcam receivers, handed over once the app has
// opened the side cameras.
let (left_rx, right_rx) = side_handoff
.recv()
.map_err(|_| anyhow::anyhow!("the app went away"))?;
let mut packet_sync = kinect_one::PacketSync::new();
let started = Instant::now();
while !shutdown.load(Ordering::Relaxed) {
if let Ok(Some(packet)) = device.poll_color_packet().await {
packet_sync.push_color_packet(packet);
}
if let Ok(Some(packet)) = device.poll_depth_packet().await {
packet_sync.push_depth_packet(packet);
}
let Some((color_packet, depth_packet)) = packet_sync.poll_packets() else {
continue;
};
use kinect_one::processor::ProcessTrait;
let color_frame = color_packet
.process(&color_processor)
.await
.map_err(|e| anyhow::anyhow!("color decode: {e}"))?;
let depth_frame = depth_packet
.process(&depth_processor)
.await
.map_err(|e| anyhow::anyhow!("depth decode: {e}"))?
.1;
let (registered, undistorted) =
registration.undistort_depth_and_color(&color_frame, &depth_frame, true);
let depth_mm = convert_depth(&undistorted.buffer);
if frames
.send(PairedFrame {
depth_mm,
rgb: registered.buffer,
left: latest(&left_rx),
right: latest(&right_rx),
timestamp_secs: started.elapsed().as_secs_f64(),
let device = Freenect2Device::open_default()
.map_err(|e| anyhow::anyhow!("opening the Kinect failed — is it plugged in? {e}"))?;
device.start().context("starting the Kinect")?;
log::info!(
"Kinect v2 open (serial {}), {} device(s) connected",
device.serial(),
Freenect2Device::count().unwrap_or(1)
);
// The depth camera: factory intrinsics at 512x424, pose from
// the offline calibration.
let ir = device.ir_params();
let camera = Camera {
id: center_config.id.clone(),
pose: center_config.pose,
intrinsics: CameraIntrinsics {
fx: ir.fx(),
fy: ir.fy(),
cx: ir.cx(),
cy: ir.cy(),
},
resolution: UVec2::new(
kinect2_sys::DEPTH_WIDTH as u32,
kinect2_sys::DEPTH_HEIGHT as u32,
),
};
ready
.send(Ok(camera))
.map_err(|_| anyhow::anyhow!("the app went away"))?;
let mut listener =
Freenect2Listener::attach(&device).context("attaching the frame listener")?;
let mut registration =
Registration::from_device(&device).context("creating the registration")?;
// The webcam receivers, handed over once the app has opened
// the side cameras.
let (left_rx, right_rx) = side_handoff
.recv()
.map_err(|_| anyhow::anyhow!("the app went away"))?;
let started = Instant::now();
while !shutdown.load(Ordering::Relaxed) {
let composed = listener.with_frames(1000, |color, depth| {
// Registered pair: undistorted depth (float mm) and the
// color image resampled onto the depth grid.
registration
.apply(color, depth)
.ok()
.map(|(undistorted, registered)| {
// SAFETY: the views are alive for the closure body.
let depth_mm = convert_depth(unsafe { undistorted.floats() });
let rgb = convert_registered(unsafe { registered.bytes() }, color.format());
PairedFrame {
depth_mm,
rgb,
left: latest(&left_rx),
right: latest(&right_rx),
timestamp_secs: started.elapsed().as_secs_f64(),
}
})
.is_err()
{
break; // receiver dropped: shut down
});
match composed {
Ok(None) | Ok(Some(None)) => continue, // timeout or a malformed pair
Ok(Some(Some(frame))) => {
if frames.send(frame).is_err() {
break; // receiver dropped: shut down
}
}
Err(e) => {
log::error!("kinect worker: {e:#}");
break;
}
}
Ok(())
})
}
let _ = device.stop();
Ok(())
})();
if let Err(e) = result {
log::error!("kinect worker: {e:#}");
@@ -306,15 +265,14 @@ fn kinect_loop(
}
}
/// libfreenect2-style depth (f32 metres, invalid as 0/NaN) → u16
/// millimetres with 0 for invalid.
#[cfg(target_os = "linux")]
fn convert_depth(buffer: &[f32]) -> Vec<u16> {
buffer
/// libfreenect2's undistorted depth: float millimetres (non-positive,
/// NaN, infinity are invalid) → u16 millimetres with 0 for invalid.
fn convert_depth(floats: &[f32]) -> Vec<u16> {
floats
.iter()
.map(|&d| {
if d.is_finite() && (0.5..=6.0).contains(&d) {
(d * 1000.0).round() as u16
if d.is_finite() && (500.0..=6000.0).contains(&d) {
d as u16
} else {
0
}
@@ -322,8 +280,31 @@ fn convert_depth(buffer: &[f32]) -> Vec<u16> {
.collect()
}
/// The registered color frame: 4 bytes per pixel at depth resolution,
/// in the color pipeline's channel order (BGRX or RGBX) — reorder to
/// the RGB our frames carry.
fn convert_registered(bytes: &[u8], color_format: FrameFormat) -> Vec<u8> {
let mut rgb = vec![0_u8; bytes.len()];
match color_format {
FrameFormat::Rgbx => rgb.copy_from_slice(bytes),
FrameFormat::Bgrx => {
let (rgb_chunks, _) = rgb.as_chunks_mut::<4>();
let (byte_chunks, _) = bytes.as_chunks::<4>();
for (dst, px) in rgb_chunks.iter_mut().zip(byte_chunks) {
let (dst, px): (&mut [u8; 4], &[u8; 4]) = (dst, px);
let [r, g, b, a] = *px;
*dst = [b, g, r, a];
}
}
FrameFormat::Float | FrameFormat::Other(_) => {
log::warn!("unexpected color frame format {color_format:?}; passing through");
rgb.copy_from_slice(bytes);
}
}
rgb
}
/// The most recent frame from a webcam channel, if one arrived.
#[cfg(target_os = "linux")]
fn latest(channel: &Receiver<Vec<u8>>) -> Option<Vec<u8>> {
let mut newest = None;
while let Ok(frame) = channel.try_recv() {
@@ -334,24 +315,22 @@ fn latest(channel: &Receiver<Vec<u8>>) -> Option<Vec<u8>> {
/// A webcam: a nokhwa [`CallbackCamera`] streaming decoded RGB into a
/// channel from its own capture thread (dropping it stops the stream).
#[cfg(target_os = "linux")]
struct Webcam {
/// Held for its side effect: dropping it stops the capture thread.
#[allow(dead_code)]
camera: nokhwa::CallbackCamera,
}
#[cfg(target_os = "linux")]
impl Webcam {
fn open(config: &Camera, frames: SyncSender<Vec<u8>>) -> Result<Self> {
use nokhwa::pixel_format::RgbFormat;
use nokhwa::utils::{
CameraFormat, FrameFormat, RequestedFormat, RequestedFormatType, Resolution,
ApiBackend, CameraFormat, FrameFormat, RequestedFormat, RequestedFormatType, Resolution,
};
// Match the device: numeric id → index, otherwise the
// human-readable name.
let devices = nokhwa::query(nokhwa::utils::ApiBackend::Auto)
let devices = nokhwa::query(ApiBackend::Auto)
.map_err(|e| anyhow::anyhow!("querying cameras: {e}"))?;
let index = device_index(&devices, &config.id)
.with_context(|| format!("finding webcam {:?}", config.id))?;
@@ -403,7 +382,6 @@ impl Webcam {
/// Resolve a configured camera id to a nokhwa device index: a numeric
/// id is the index itself; otherwise it must (uniquely) match a
/// device's human-readable name.
#[cfg(target_os = "linux")]
fn device_index(
devices: &[nokhwa::utils::CameraInfo],
id: &str,
@@ -434,20 +412,42 @@ mod tests {
use super::*;
use glam::Vec3;
use glam::uvec2;
#[cfg(target_os = "linux")]
use nokhwa::utils::CameraIndex;
use rig::{CameraIntrinsics, CameraPose};
use rig::CameraPose;
#[cfg(target_os = "linux")]
#[test]
fn depth_conversion_drops_invalid_and_quantizes() {
// libfreenect2 depth: f32 metres, invalid as 0/NaN; out-of-range
// values are dropped too.
let out = convert_depth(&[1.9, 0.0, f32::NAN, 0.25, 6.5, 2.845]);
assert_eq!(out, vec![1900, 0, 0, 0, 0, 2845]);
// libfreenect2 depth: float millimetres, invalid as
// non-positive/NaN/infinity; out-of-range values are dropped.
let floats = [
1900.0,
0.0,
-2.0,
f32::NAN,
f32::INFINITY,
2845.6,
250.0,
6500.0,
];
let expected = [1900, 0, 0, 0, 0, 2845, 0, 0];
assert_eq!(convert_depth(&floats), expected);
}
#[test]
fn registered_colour_reordering() {
// BGRX in → RGB out.
let bgrx = [1_u8, 2, 3, 255, 5, 6, 7, 255];
assert_eq!(
convert_registered(&bgrx, FrameFormat::Bgrx),
vec![3, 2, 1, 255, 7, 6, 5, 255]
);
// RGBX passes through.
let rgbx = [1_u8, 2, 3, 255, 5, 6, 7, 255];
assert_eq!(
convert_registered(&rgbx, FrameFormat::Rgbx),
vec![1, 2, 3, 255, 5, 6, 7, 255]
);
}
#[cfg(target_os = "linux")]
#[test]
fn latest_takes_only_the_newest_frame() {
let (tx, rx) = sync_channel(4);
@@ -458,20 +458,22 @@ mod tests {
assert_eq!(latest(&rx), None);
}
#[cfg(target_os = "linux")]
#[test]
fn device_matching_by_index_and_name() {
let camera = |index: u32, name: &str| {
nokhwa::utils::CameraInfo::new(name, "", "", CameraIndex::Index(index))
nokhwa::utils::CameraInfo::new(name, "", "", nokhwa::utils::CameraIndex::Index(index))
};
let devices = vec![camera(0, "Logitech C920"), camera(1, "Some Other Cam")];
// Numeric id → index.
assert_eq!(device_index(&devices, "1").unwrap(), CameraIndex::Index(1));
assert_eq!(
device_index(&devices, "1").unwrap(),
nokhwa::utils::CameraIndex::Index(1)
);
// Name match (case-insensitive substring).
assert_eq!(
device_index(&devices, "c920").unwrap(),
CameraIndex::Index(0)
nokhwa::utils::CameraIndex::Index(0)
);
// No match.
assert!(device_index(&devices, "nonexistent").is_err());
@@ -516,13 +518,15 @@ mod tests {
resolution: uvec2(640, 480),
},
};
let err = match KinectRigSource::open(&rig) {
Ok(_) => panic!("opened a rig with no Kinect in the container?"),
Err(e) => e,
};
assert!(
err.to_string().to_lowercase().contains("kinect"),
"unexpected error: {err:#}"
);
if kinect2_sys::Device::count().unwrap_or(0) == 0 {
let err = match KinectRigSource::open(&rig) {
Ok(_) => panic!("opened a rig with no Kinect in the container?"),
Err(e) => e,
};
assert!(
err.to_string().to_lowercase().contains("kinect"),
"unexpected error: {err:#}"
);
}
}
}
kinect2-sys/Cargo.toml +15 -0
diff --git a/kinect2-sys/Cargo.toml b/kinect2-sys/Cargo.toml
new file mode 100644
index 0000000..0842255
@@ -0,0 +1,15 @@
[package]
name = "kinect2-sys"
description = "Rust bindings for libfreenect2 (Kinect v2): a C shim over the C++ API, no bindgen"
version.workspace = true
edition.workspace = true
publish = false
links = "freenect2"
[dependencies]
anyhow.workspace = true
[build-dependencies]
anyhow.workspace = true
cc = "1"
pkg-config = "0.3"
kinect2-sys/build.rs +84 -0
diff --git a/kinect2-sys/build.rs b/kinect2-sys/build.rs
new file mode 100644
index 0000000..dbce479
@@ -0,0 +1,84 @@
//! Build script: locate an installed libfreenect2, compile the C++
//! shim, and link.
//!
//! Discovery order:
//!
//! 1. `FREENECT2_DIR` — a libfreenect2 install prefix with
//! `include/libfreenect2/` and `lib/` (the CMake default layout).
//! On Windows this is how the sink finds the library built per the
//! README (or vcpkg's installed tree).
//! 2. `pkg-config freenect2` — standard on Linux.
//! 3. `/usr/local` — the CMake default install prefix.
use std::env;
use std::path::PathBuf;
fn main() -> anyhow::Result<()> {
println!("cargo:rerun-if-env-changed=FREENECT2_DIR");
let mut include_dirs: Vec<PathBuf> = Vec::new();
let mut lib_dirs: Vec<PathBuf> = Vec::new();
let explicit = env::var_os("FREENECT2_DIR").map(PathBuf::from);
if let Some(dir) = explicit {
let include = dir.join("include");
let lib = dir.join("lib");
if !include.join("libfreenect2").exists() {
anyhow::bail!(
"FREENECT2_DIR={} has no include/libfreenect2 — point it at a \
libfreenect2 install prefix",
dir.display()
);
}
println!(
"cargo:warning=using libfreenect2 from FREENECT2_DIR={}",
dir.display()
);
include_dirs.push(include);
lib_dirs.push(lib);
} else if env::var_os("FREENECT2_NO_PKG_CONFIG").is_none()
&& pkg_config::probe_library("freenect2").is_ok()
{
let library = pkg_config::find_library("freenect2")
.map_err(|e| anyhow::anyhow!("pkg-config freenect2: {e}"))?;
// The pkg-config crate emits the link flags itself.
for path in &library.include_paths {
include_dirs.push(path.clone());
}
for path in &library.link_paths {
println!("cargo:rustc-link-search=native={}", path.display());
}
} else {
// The CMake default install prefix.
let prefix = PathBuf::from("/usr/local");
if prefix.join("include/libfreenect2").exists() {
include_dirs.push(prefix.join("include"));
lib_dirs.push(prefix.join("lib"));
} else {
anyhow::bail!(
"libfreenect2 not found. Install it (see the kinect-view README's \
'Kinect v2 capture' section) and either set FREENECT2_DIR to its \
install prefix or make it visible to pkg-config."
);
}
}
// The shim.
for dir in &include_dirs {
println!("cargo:include={}", dir.display());
}
let mut build = cc::Build::new();
build.cpp(true);
build.file("csrc/shim.cpp");
build.warnings(false);
for dir in &include_dirs {
build.include(dir);
}
build.compile("lf2shim");
for dir in &lib_dirs {
println!("cargo:rustc-link-search=native={}", dir.display());
}
println!("cargo:rustc-link-lib=freenect2");
Ok(())
}
kinect2-sys/csrc/shim.cpp +386 -0
diff --git a/kinect2-sys/csrc/shim.cpp b/kinect2-sys/csrc/shim.cpp
new file mode 100644
index 0000000..5728d6c
@@ -0,0 +1,386 @@
// A minimal C ABI over the parts of libfreenect2 the kinect-view
// capture pipeline needs. C++ exceptions are caught and reported
// through a thread-local buffer, fetched with lf2_take_error.
//
// Every lf2_frame borrows or owns a libfreenect2 Frame via the opaque
// `frame` pointer. Listener views are valid until lf2_release; frames
// from lf2_frame_new are valid until lf2_frame_free.
#include <libfreenect2/libfreenect2.hpp>
#include <libfreenect2/frame_listener.hpp>
#include <libfreenect2/frame_listener_impl.h>
#include <libfreenect2/packet_pipeline.h>
#include <libfreenect2/registration.h>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <exception>
#include <new>
#include <string>
typedef struct lf2_frame
{
void *frame; // opaque libfreenect2::Frame*
size_t width;
size_t height;
size_t bytes_per_pixel;
unsigned char *data;
int32_t format; // libfreenect2::Frame::Format
uint32_t timestamp;
} lf2_frame;
typedef struct lf2_ir_params
{
float fx, fy, cx, cy;
float k1, k2, k3, p1, p2;
} lf2_ir_params;
// Keep the Rust side honest about the ABI.
static_assert(offsetof(lf2_frame, frame) == 0, "lf2_frame layout");
static_assert(offsetof(lf2_frame, width) == sizeof(void *), "lf2_frame layout");
static_assert(offsetof(lf2_ir_params, fx) == 0, "lf2_ir_params layout");
static thread_local std::string lf2_error;
static thread_local std::string lf2_error_out;
static void lf2_set_error(const char *message)
{
lf2_error = message;
}
// Keep the C++ helper and the extern "C" export distinct.
// The library-wide device enumerator (its USB context is a global
// singleton internally, so one static instance is enough).
static libfreenect2::Freenect2 &lf2_context()
{
static libfreenect2::Freenect2 context;
return context;
}
static void lf2_fill_view(lf2_frame *view, const libfreenect2::Frame *frame)
{
view->frame = const_cast<void *>(static_cast<const void *>(frame));
view->width = frame->width;
view->height = frame->height;
view->bytes_per_pixel = frame->bytes_per_pixel;
view->data = frame->data;
view->format = static_cast<int32_t>(frame->format);
view->timestamp = frame->timestamp;
}
struct lf2_listener
{
libfreenect2::SyncMultiFrameListener listener;
libfreenect2::FrameMap frames;
lf2_listener() : listener(libfreenect2::Frame::Color | libfreenect2::Frame::Depth) {}
};
struct lf2_registration
{
libfreenect2::Registration registration;
lf2_registration(const libfreenect2::Freenect2Device::IrCameraParams &depth,
const libfreenect2::Freenect2Device::ColorCameraParams &color)
: registration(depth, color) {}
};
extern "C"
{
const char *lf2_take_error(void)
{
lf2_error_out = lf2_error;
lf2_error.clear();
return lf2_error_out.c_str();
}
int lf2_count_devices(void)
{
try
{
return static_cast<int>(lf2_context().enumerateDevices());
}
catch (const std::exception &e)
{
lf2_set_error(e.what());
return -1;
}
catch (...)
{
lf2_set_error("unknown failure enumerating devices");
return -1;
}
}
void *lf2_open_default(void)
{
try
{
// Explicit CPU depth processing: deterministic, no OpenCL/CUDA.
libfreenect2::Freenect2Device *device =
lf2_context().openDevice(0, new libfreenect2::CpuPacketPipeline());
if (device == nullptr)
lf2_set_error("no Kinect v2 device found — is it plugged into a USB 3.0 port?");
return device;
}
catch (const std::exception &e)
{
lf2_set_error(e.what());
return nullptr;
}
catch (...)
{
lf2_set_error("unknown failure opening the device");
return nullptr;
}
}
void lf2_device_close(void *device)
{
try
{
auto *dev = static_cast<libfreenect2::Freenect2Device *>(device);
if (dev == nullptr)
return;
dev->stop();
dev->close();
// Upstream usage (Protonect) does not delete the device handle
// after close(); match it.
}
catch (...)
{
// Closing must not throw across the FFI boundary.
}
}
int lf2_start(void *device)
{
try
{
static_cast<libfreenect2::Freenect2Device *>(device)->start();
return 0;
}
catch (const std::exception &e)
{
lf2_set_error(e.what());
return -1;
}
catch (...)
{
lf2_set_error("unknown failure starting the device");
return -1;
}
}
int lf2_stop(void *device)
{
try
{
static_cast<libfreenect2::Freenect2Device *>(device)->stop();
return 0;
}
catch (const std::exception &e)
{
lf2_set_error(e.what());
return -1;
}
catch (...)
{
lf2_set_error("unknown failure stopping the device");
return -1;
}
}
void lf2_get_ir_params(void *device, lf2_ir_params *out)
{
// Trivially-copyable struct; cannot throw.
libfreenect2::Freenect2Device::IrCameraParams params =
static_cast<libfreenect2::Freenect2Device *>(device)->getIrCameraParams();
out->fx = params.fx;
out->fy = params.fy;
out->cx = params.cx;
out->cy = params.cy;
out->k1 = params.k1;
out->k2 = params.k2;
out->k3 = params.k3;
out->p1 = params.p1;
out->p2 = params.p2;
}
void lf2_get_serial(void *device, char *buf, size_t len)
{
try
{
std::snprintf(buf, len, "%s",
static_cast<libfreenect2::Freenect2Device *>(device)->getSerialNumber().c_str());
}
catch (...)
{
if (len > 0)
buf[0] = '\0';
}
}
lf2_frame *lf2_frame_new(int32_t width, int32_t height, int32_t bytes_per_pixel)
{
try
{
auto *frame = new libfreenect2::Frame(width, height, bytes_per_pixel);
auto *view = new lf2_frame;
lf2_fill_view(view, frame);
return view;
}
catch (const std::exception &e)
{
lf2_set_error(e.what());
return nullptr;
}
catch (...)
{
lf2_set_error("unknown failure allocating a frame");
return nullptr;
}
}
void lf2_frame_free(lf2_frame *view)
{
if (view == nullptr)
return;
delete static_cast<libfreenect2::Frame *>(view->frame);
delete view;
}
void *lf2_listener_new(void)
{
try
{
return new lf2_listener;
}
catch (const std::exception &e)
{
lf2_set_error(e.what());
return nullptr;
}
catch (...)
{
lf2_set_error("unknown failure creating the listener");
return nullptr;
}
}
void lf2_listener_free(void *listener)
{
delete static_cast<lf2_listener *>(listener);
}
int lf2_listener_attach(void *device, void *listener)
{
try
{
auto *dev = static_cast<libfreenect2::Freenect2Device *>(device);
auto *l = static_cast<lf2_listener *>(listener);
dev->setColorFrameListener(&l->listener);
dev->setIrAndDepthFrameListener(&l->listener);
return 0;
}
catch (const std::exception &e)
{
lf2_set_error(e.what());
return -1;
}
catch (...)
{
lf2_set_error("unknown failure attaching the listener");
return -1;
}
}
int lf2_wait_for_frame(void *listener, int32_t timeout_ms, lf2_frame *out_color, lf2_frame *out_depth)
{
try
{
auto *l = static_cast<lf2_listener *>(listener);
std::memset(&out_color->frame, 0, sizeof(lf2_frame) - 0);
if (!l->listener.waitForNewFrame(l->frames, timeout_ms))
return 0;
auto *color = l->frames[libfreenect2::Frame::Color];
auto *depth = l->frames[libfreenect2::Frame::Depth];
lf2_fill_view(out_color, color);
lf2_fill_view(out_depth, depth);
return 1;
}
catch (const std::exception &e)
{
lf2_set_error(e.what());
return -1;
}
catch (...)
{
lf2_set_error("unknown failure waiting for frames");
return -1;
}
}
void lf2_release(void *listener)
{
static_cast<lf2_listener *>(listener)->listener.release(
static_cast<lf2_listener *>(listener)->frames);
}
void *lf2_registration_new(void *device)
{
try
{
auto *dev = static_cast<libfreenect2::Freenect2Device *>(device);
return new lf2_registration(dev->getIrCameraParams(), dev->getColorCameraParams());
}
catch (const std::exception &e)
{
lf2_set_error(e.what());
return nullptr;
}
catch (...)
{
lf2_set_error("unknown failure creating the registration");
return nullptr;
}
}
void lf2_registration_free(void *registration)
{
delete static_cast<lf2_registration *>(registration);
}
int lf2_register(void *registration, lf2_frame *color, lf2_frame *depth,
lf2_frame *out_undistorted, lf2_frame *out_registered)
{
try
{
auto *reg = static_cast<lf2_registration *>(registration);
auto *rgb_frame = static_cast<libfreenect2::Frame *>(color->frame);
auto *depth_frame = static_cast<libfreenect2::Frame *>(depth->frame);
auto *undistorted = static_cast<libfreenect2::Frame *>(out_undistorted->frame);
auto *registered = static_cast<libfreenect2::Frame *>(out_registered->frame);
reg->registration.apply(rgb_frame, depth_frame, undistorted, registered, true);
// Refresh the output views (data pointers stay put; formats may be
// informative).
lf2_fill_view(out_undistorted, undistorted);
lf2_fill_view(out_registered, registered);
return 0;
}
catch (const std::exception &e)
{
lf2_set_error(e.what());
return -1;
}
catch (...)
{
lf2_set_error("unknown failure registering frames");
return -1;
}
}
} // extern "C"
kinect2-sys/src/lib.rs +421 -0
diff --git a/kinect2-sys/src/lib.rs b/kinect2-sys/src/lib.rs
new file mode 100644
index 0000000..5ddb3de
@@ -0,0 +1,421 @@
//! A small C ABI over the parts of libfreenect2's C++ API the capture
//! pipeline needs (see `shim.cpp`). Every call catches C++ exceptions
//! and reports through a thread-local message buffer instead.
//!
//! No bindgen: the surface is tiny and hand-declared in
//! [`crate::ffi`], with layout `static_assert`s on the C++ side.
/// Depth frames are 512x424 float millimetres (invalid: non-positive,
/// NaN, infinity — libfreenect2's convention).
pub const DEPTH_WIDTH: usize = 512;
pub const DEPTH_HEIGHT: usize = 424;
/// The format of a frame's bytes: libfreenect2 `Frame::Format`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FrameFormat {
/// 4 bytes: B, G, R, unused.
Bgrx,
/// 4 bytes: R, G, B, unused.
Rgbx,
/// 4-byte float per pixel.
Float,
/// Anything else.
Other(i32),
}
impl FrameFormat {
fn from_raw(raw: i32) -> Self {
match raw {
4 => Self::Bgrx,
5 => Self::Rgbx,
2 => Self::Float,
other => Self::Other(other),
}
}
}
mod ffi {
use std::ffi::c_char;
use std::ffi::c_void;
#[repr(C)]
#[derive(Debug, Clone, Copy)]
pub struct FrameView {
/// Opaque `libfreenect2::Frame*`; owned by whoever allocated it.
pub frame: *mut c_void,
pub width: usize,
pub height: usize,
pub bytes_per_pixel: usize,
pub data: *mut u8,
pub format: i32,
pub timestamp: u32,
}
#[repr(C)]
#[derive(Debug, Clone, Copy, Default, PartialEq)]
pub struct IrParams {
pub fx: f32,
pub fy: f32,
pub cx: f32,
pub cy: f32,
pub k1: f32,
pub k2: f32,
pub k3: f32,
pub p1: f32,
pub p2: f32,
}
// Layout: keep in sync with shim.cpp's static_asserts.
pub type Device = c_void;
pub type Listener = c_void;
pub type Registration = c_void;
unsafe extern "C" {
pub fn lf2_take_error() -> *const c_char;
pub fn lf2_count_devices() -> i32;
pub fn lf2_open_default() -> *mut Device;
pub fn lf2_device_close(device: *mut Device);
pub fn lf2_start(device: *mut Device) -> i32;
pub fn lf2_stop(device: *mut Device) -> i32;
pub fn lf2_get_ir_params(device: *mut Device, out: *mut IrParams);
pub fn lf2_get_serial(device: *mut Device, buf: *mut c_char, len: usize);
pub fn lf2_frame_new(width: i32, height: i32, bpp: i32) -> *mut FrameView;
pub fn lf2_frame_free(frame: *mut FrameView);
pub fn lf2_listener_new() -> *mut Listener;
pub fn lf2_listener_free(listener: *mut Listener);
pub fn lf2_listener_attach(device: *mut Device, listener: *mut Listener) -> i32;
pub fn lf2_wait_for_frame(
listener: *mut Listener,
timeout_ms: i32,
out_color: *mut FrameView,
out_depth: *mut FrameView,
) -> i32;
pub fn lf2_release(listener: *mut Listener);
pub fn lf2_registration_new(device: *mut Device) -> *mut Registration;
pub fn lf2_registration_free(registration: *mut Registration);
pub fn lf2_register(
registration: *mut Registration,
color: *mut FrameView,
depth: *mut FrameView,
out_undistorted: *mut FrameView,
out_registered: *mut FrameView,
) -> i32;
}
}
/// The last error reported by a failing call on this thread.
pub fn last_error() -> String {
unsafe {
let ptr = ffi::lf2_take_error();
if ptr.is_null() {
"unknown libfreenect2 failure".into()
} else {
std::ffi::CStr::from_ptr(ptr).to_string_lossy().into_owned()
}
}
}
macro_rules! call {
($expr:expr) => {{
let result = unsafe { $expr };
if result < 0 {
anyhow::bail!("{}", last_error());
}
result
}};
}
macro_rules! opt_call {
($expr:expr) => {{
let result = unsafe { $expr };
if result.is_null() {
anyhow::bail!("{}", last_error());
}
result
}};
}
/// A borrow of frame data — valid only until the listener releases the
/// frame or the owning object is dropped.
#[derive(Debug, Clone, Copy)]
pub struct FrameRef {
view: ffi::FrameView,
}
impl FrameRef {
pub fn width(&self) -> usize {
self.view.width
}
pub fn height(&self) -> usize {
self.view.height
}
pub fn format(&self) -> FrameFormat {
FrameFormat::from_raw(self.view.format)
}
/// The raw bytes of the frame.
///
/// # Safety
/// Valid for the borrow lifetime of the view; the caller must not
/// use them after the source frame is released.
pub unsafe fn bytes(&self) -> &[u8] {
unsafe {
std::slice::from_raw_parts(
self.view.data,
self.view.width * self.view.height * self.view.bytes_per_pixel,
)
}
}
/// The floats of a `Float`-format frame.
///
/// # Safety
/// Valid for the borrow lifetime; `format` must be `Float`.
pub unsafe fn floats(&self) -> &[f32] {
unsafe {
std::slice::from_raw_parts(
self.view.data.cast::<f32>(),
self.view.width * self.view.height,
)
}
}
}
/// A Kinect v2 device, via libfreenect2. Attach a [`Listener`] before
/// [`Device::start`].
#[derive(Debug)]
pub struct Device {
ptr: *mut ffi::Device,
}
impl Device {
/// How many Kinects are connected; `Err` on failure.
pub fn count() -> anyhow::Result<usize> {
Ok(call!(ffi::lf2_count_devices()) as usize)
}
/// Open the first connected device (CPU depth pipeline).
pub fn open_default() -> anyhow::Result<Self> {
Ok(Self {
ptr: opt_call!(ffi::lf2_open_default()),
})
}
pub fn start(&self) -> anyhow::Result<()> {
call!(ffi::lf2_start(self.ptr));
Ok(())
}
pub fn stop(&self) -> anyhow::Result<()> {
call!(ffi::lf2_stop(self.ptr));
Ok(())
}
/// The depth (IR) camera's factory intrinsics, at 512x424.
pub fn ir_params(&self) -> IrParams {
let mut params = IrParams::default();
unsafe { ffi::lf2_get_ir_params(self.ptr, &mut params.0) };
params
}
/// The device serial number, for logging.
pub fn serial(&self) -> String {
let mut buf = [0 as std::ffi::c_char; 64];
unsafe { ffi::lf2_get_serial(self.ptr, buf.as_mut_ptr(), buf.len()) };
let cstr = unsafe { std::ffi::CStr::from_ptr(buf.as_ptr()) };
cstr.to_string_lossy().into_owned()
}
fn close(&self) {
unsafe { ffi::lf2_device_close(self.ptr) };
}
}
impl Drop for Device {
fn drop(&mut self) {
self.close();
}
}
/// Factory-calibrated depth intrinsics: focal length and principal
/// point in pixels, plus the distortion model.
#[derive(Debug, Clone, Copy, Default, PartialEq)]
pub struct IrParams(ffi::IrParams);
impl IrParams {
pub fn fx(&self) -> f32 {
self.0.fx
}
pub fn fy(&self) -> f32 {
self.0.fy
}
pub fn cx(&self) -> f32 {
self.0.cx
}
pub fn cy(&self) -> f32 {
self.0.cy
}
}
/// A synchronized color+depth listener attached to a [`Device`].
///
/// Frames are borrowed: read them inside
/// [`Listener::with_frames`] and copy what you need; the frames are
/// released when the closure returns.
pub struct Listener {
ptr: *mut ffi::Listener,
}
impl Listener {
/// Create and attach a `Color | Depth` sync listener.
pub fn attach(device: &Device) -> anyhow::Result<Self> {
let listener = opt_call!(ffi::lf2_listener_new());
call!(ffi::lf2_listener_attach(device.ptr, listener));
Ok(Self { ptr: listener })
}
/// Wait up to `timeout_ms` for a synchronized color+depth pair and
/// run `f` with views into both, returning `f`'s value.
/// Returns `Ok(None)` on timeout.
pub fn with_frames<T>(
&mut self,
timeout_ms: i32,
mut f: impl FnMut(&FrameRef, &FrameRef) -> T,
) -> anyhow::Result<Option<T>> {
let mut color = ffi::FrameView {
frame: std::ptr::null_mut(),
width: 0,
height: 0,
bytes_per_pixel: 0,
data: std::ptr::null_mut(),
format: 0,
timestamp: 0,
};
let mut depth = color;
let got = unsafe { ffi::lf2_wait_for_frame(self.ptr, timeout_ms, &mut color, &mut depth) };
match got {
1 => {
let value = f(&FrameRef { view: color }, &FrameRef { view: depth });
unsafe { ffi::lf2_release(self.ptr) };
Ok(Some(value))
}
0 => Ok(None),
_ => anyhow::bail!("{}", last_error()),
}
}
}
impl Drop for Listener {
fn drop(&mut self) {
unsafe { ffi::lf2_listener_free(self.ptr) };
}
}
/// Registration of color onto depth (libfreenect2's reverse-engineered
/// factory-extrinsics mapping): produces the undistorted depth map
/// (float millimetres) and the color image resampled onto the depth
/// grid, both at 512x424.
pub struct Registration {
ptr: *mut ffi::Registration,
undistorted: *mut ffi::FrameView,
registered: *mut ffi::FrameView,
}
impl Registration {
/// From the device's factory calibration.
pub fn from_device(device: &Device) -> anyhow::Result<Self> {
let registration = opt_call!(ffi::lf2_registration_new(device.ptr));
let undistorted = opt_call!(ffi::lf2_frame_new(
DEPTH_WIDTH as i32,
DEPTH_HEIGHT as i32,
4
));
let registered = opt_call!(ffi::lf2_frame_new(
DEPTH_WIDTH as i32,
DEPTH_HEIGHT as i32,
4
));
Ok(Self {
ptr: registration,
undistorted,
registered,
})
}
/// Register one frame pair; the returned views are valid until the
/// next call or drop. The registered image's channel order matches
/// the color frame's (`FrameRef::format` of `color`).
pub fn apply(
&mut self,
color: &FrameRef,
depth: &FrameRef,
) -> anyhow::Result<(FrameRef, FrameRef)> {
let mut color_view = color.view;
let mut depth_view = depth.view;
call!(ffi::lf2_register(
self.ptr,
&mut color_view,
&mut depth_view,
self.undistorted,
self.registered
));
// SAFETY: the out-frames live as long as `self`; the returned
// views are plain copies pointing at their data, invalidated by
// the next `apply` or by dropping `self`.
unsafe { Ok(((*self.undistorted).borrow(), (*self.registered).borrow())) }
}
}
// SAFETY: libfreenect2 device/listener/registration handles are
// thread-safe across our usage (one worker thread drives them).
unsafe impl Send for Device {}
unsafe impl Send for Listener {}
unsafe impl Send for Registration {}
impl ffi::FrameView {
/// Reinterpret as a borrow of the frame data.
///
/// # Safety
/// The view must be alive for `'self`'s lifetime; the caller
/// guarantees it.
unsafe fn borrow(&self) -> FrameRef {
FrameRef { view: *self }
}
}
impl Drop for Registration {
fn drop(&mut self) {
unsafe {
ffi::lf2_frame_free(self.undistorted);
ffi::lf2_frame_free(self.registered);
ffi::lf2_registration_free(self.ptr);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn linkage_smoke_without_a_device() {
// The library links and its context works even without USB.
let count = Device::count().expect("enumerating devices works");
println!("{count} Kinect(s) connected");
}
#[test]
fn open_fails_clearly_without_a_device() {
// The container has no USB: opening must fail (or succeed only
// if a Kinect is actually attached, which it is not here).
if Device::count().unwrap() == 0 {
let err = Device::open_default().expect_err("no device");
assert!(!err.to_string().is_empty());
}
}
}