| Name | Message | Date |
|---|---|---|
| 📁 physical_monitor | 1 day ago | |
| 📄 camera_controller.rs | 11 hours ago | |
| 📄 face_detection.rs | 11 hours ago | |
| 📄 main.rs | 11 hours ago | |
| 📄 virtual_window.rs | 11 hours ago |
📄
src/face_detection.rs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
use async_channel::{Receiver, RecvError, Sender, TryRecvError}; use bevy::{ app::{App, Plugin, PreUpdate, Startup}, ecs::{ message::Message, resource::Resource, system::{Commands, ResMut}, }, log, math::Vec3, tasks::ComputeTaskPool, }; use nokhwa::{ Buffer, Camera, nokhwa_initialize, pixel_format::RgbFormat, utils::{CameraIndex, RequestedFormat, RequestedFormatType, Resolution}, }; use ort::{ inputs, session::{Session, SessionOutputs}, value::Tensor, }; pub struct FaceDetectionPlugin; impl Plugin for FaceDetectionPlugin { fn build(&self, app: &mut App) { app.add_message::<FaceMoved>() .add_systems(Startup, setup_face_detection) .add_systems(PreUpdate, publish_face_moved); } } #[derive(Message)] pub struct FaceMoved(pub Vec3); #[derive(Resource)] struct FaceDetectionReader(Receiver<Vec3>); fn setup_face_detection(mut commands: Commands) { let (frame_sender, frame_receiver) = async_channel::unbounded(); let (face_sender, face_receiver) = async_channel::unbounded(); ComputeTaskPool::get() .spawn(camera_loop(frame_sender)) .detach(); ComputeTaskPool::get() .spawn(detection_loop(frame_receiver, face_sender)) .detach(); commands.insert_resource(FaceDetectionReader(face_receiver)); } async fn camera_loop(sender: Sender<Buffer>) { { let (init_sender, init_receiver) = async_channel::unbounded(); nokhwa_initialize(move |success| { if let Err(err) = init_sender.send_blocking(success) { log::error!("Could not send nokhwa initialization ({success}): {err}"); } }); match init_receiver.recv().await { Ok(true) => {} Ok(false) => { log::error!("nokhwa failed to initialize"); return; } Err(err) => { log::error!("nokhwa failed to initialize: {err}"); return; } } } let format = RequestedFormat::new::<RgbFormat>(RequestedFormatType::None); let mut camera = match Camera::new(CameraIndex::Index(1), format) { Ok(camera) => camera, Err(err) => { log::error!("Failed to create camera: {err}"); return; } }; if let Err(err) = camera.open_stream() { log::error!("Failed to open camera stream: {err}"); return; } loop { let frame = match camera.frame() { Ok(frame) => frame, Err(err) => { log::error!("Could not get frame: {err}"); return; } }; if let Err(err) = sender.send(frame).await { log::error!("Could not send frame to face thread: {err}"); return; } } } async fn detection_loop(mut receiver: Receiver<Buffer>, sender: Sender<Vec3>) { let mut model = match Session::builder().and_then(|mut b| { b.commit_from_file("face_detection_yunet_2026may.onnx") }) { Ok(session) => session, Err(err) => { log::error!("Could not build ONNX session: {err}"); return; } }; loop { let frame = match recv_latest(&mut receiver).await { Ok(frame) => frame, Err(err) => { log::error!("Could not get frame from camera thread: {err}"); return; } }; let resolution = frame.resolution(); let resolution = Resolution::new( resolution.width().div_ceil(32) * 32, resolution.height().div_ceil(32) * 32, ); let image = match frame.decode_image::<RgbFormat>() { Ok(image) => image, Err(err) => { log::error!("Could not decode image: {err}"); return; } }; let mut tensor = match Tensor::from_array(( [ 1, 3, resolution.height() as usize, resolution.width() as usize, ], vec![0.0_f32; 1 * 3 * (resolution.height() * resolution.width()) as usize], )) { Ok(tensor) => tensor, Err(err) => { log::warn!("Could not create tensor: {err}"); continue; } }; for (y, row) in image.rows().enumerate() { for (x, pixel) in row.enumerate() { tensor[[0, 2, y as i64, x as i64]] = pixel[0] as f32; tensor[[0, 1, y as i64, x as i64]] = pixel[1] as f32; tensor[[0, 0, y as i64, x as i64]] = pixel[2] as f32; } } let outputs = match model.run(inputs!["input" => &tensor]) { Ok(output) => output, Err(err) => { log::error!("Model did not run to completion: {err}"); return; } }; let Some(face) = get_face_position(&outputs, &resolution, 32) .or_else(|| get_face_position(&outputs, &resolution, 16)) .or_else(|| get_face_position(&outputs, &resolution, 8)) else { log::info!("No face detected in frame"); continue; }; if let Err(err) = sender.send(face).await { log::warn!("Could not send face position: {err}"); } } } const THRESHOLD: f32 = 0.75; fn get_face_position( outputs: &SessionOutputs, resolution: &Resolution, stride: usize, ) -> Option<Vec3> { let cls = outputs.get(format!("cls_{stride}"))?; let obj = outputs.get(format!("obj_{stride}"))?; let (cls_shape, cls) = match cls.try_extract_tensor::<f32>() { Ok(cls) => cls, Err(err) => { log::warn!("cls data is not a float tensor: {err}"); return None; } }; let (obj_shape, obj) = match obj.try_extract_tensor::<f32>() { Ok(obj) => obj, Err(err) => { log::warn!("obj data is not a float tensor: {err}"); return None; } }; if cls_shape.len() != 3 || obj_shape.len() != 3 || cls_shape[1] != obj_shape[1] { log::warn!("Result cls and obj is on wrong format: {cls_shape} and {obj_shape}"); return None; } let (max_i, max_score) = cls .iter() .zip(obj.iter()) .enumerate() .map(|(i, (cls, obj))| (i, (cls.clamp(0.0, 1.0) * obj.clamp(0.0, 1.0)).sqrt())) .filter(|(_, s)| s.is_finite()) .max_by(|(_, a), (_, b)| a.total_cmp(b))?; if max_score < THRESHOLD { log::info!("No face above threshold"); return None; } let bbox = outputs.get(format!("bbox_{stride}"))?; let (bbox_shape, bbox) = match bbox.try_extract_tensor::<f32>() { Ok(bbox) => bbox, Err(err) => { log::warn!("bbox is not a float tensor: {err}"); return None; } }; if bbox_shape.len() != 3 || bbox_shape[1] <= max_i as i64 || bbox_shape[2] != 4 { log::warn!("Result bbox is on the wrong format: {bbox_shape}"); return None; } let s = stride as f32; let cols = (resolution.width().div_ceil(32) * 32) as usize / stride; let (col, row) = ((max_i % cols) as f32, (max_i / cols) as f32); let x_pixels = (col + bbox[max_i * 4 + 0]) * s; let y_pixels = (row + bbox[max_i * 4 + 1]) * s; let width_pixels = bbox[max_i * 4 + 2].exp() * s; const HFOV_DEG: f32 = 60.0; const FACE_WIDTH_METERS: f32 = 0.2; let (w, h) = (resolution.width() as f32, resolution.height() as f32); let f = (w / 2.0) / (HFOV_DEG.to_radians() / 2.0).tan(); let z = f * FACE_WIDTH_METERS / width_pixels; let pos = Vec3::new( (x_pixels - w / 2.0) * z / f, -(y_pixels - h / 2.0) * z / f, -z, ); Some(pos) } fn publish_face_moved( mut commands: Commands, mut face_detection_reader: ResMut<FaceDetectionReader>, ) { let Ok(face) = try_recv_latest(&mut face_detection_reader.0) else { return; }; commands.write_message(FaceMoved(face)); } async fn recv_latest<T>(receiver: &mut Receiver<T>) -> Result<T, RecvError> { match try_recv_latest(receiver) { Ok(msg) => Ok(msg), Err(TryRecvError::Closed) => Err(RecvError), Err(TryRecvError::Empty) => receiver.recv().await, } } fn try_recv_latest<T>(receiver: &mut Receiver<T>) -> Result<T, TryRecvError> { receiver.try_recv().map(|mut msg| { while let Ok(next) = receiver.try_recv() { msg = next; } msg }) }