cli: Add strobe-tolerant IR capture and bounded RGB retry · isaaccorbrey.com/pareidolia@118e727

+248 -20

2 changed files

Expand all

crates

cli

src

lib.rs

pipeline_runtime.rs

+14

crates/cli/src/lib.rs

··· 153 153 /// the generic pipeline-error wording. 154 154 #[error("no face detected in captured frame")] 155 155 NoFaceDetected, 156 + 157 + /// The IR camera produced only near-black frames across every 158 + /// warm-up and retry attempt, so no face could be detected. This is 159 + /// almost always the IR illuminator failing to power on (a known 160 + /// issue on some laptop UVC cameras, e.g. the 2018 Dell XPS 13) 161 + /// rather than a missing or misconfigured face — distinguished from 162 + /// [`Self::NoFaceDetected`] so the message can point at the real 163 + /// cause. `mean` is the brightest frame's mean pixel value (0–255). 164 + #[error( 165 + "IR camera frames were too dark to detect a face (brightest frame mean {mean:.1}/255 \ 166 + after {attempts} attempts). The IR illuminator likely did not turn on. Try re-running \ 167 + enrollment, or pass --no-ir to enroll RGB only." 168 + )] 169 + IrTooDark { mean: f32, attempts: u32 }, 156 170 }

+234 -20

crates/cli/src/pipeline_runtime.rs

··· 15 15 use std::path::{Path, PathBuf}; 16 16 17 17 use pareidolia_core::camera::convert::to_rgb8; 18 - use pareidolia_core::camera::v4l2::{V4l2Config, V4l2FrameSource}; 18 + use pareidolia_core::camera::v4l2::{V4l2Config, V4l2FrameSource, Warmup}; 19 19 use pareidolia_core::camera::{CameraKind, Frame, FrameSource}; 20 20 use pareidolia_core::enrollment::ModelFingerprint; 21 21 use pareidolia_core::pipeline::arcface::ArcfaceEmbedder; ··· 44 44 /// 45 45 /// # Stream priming 46 46 /// 47 - /// Two unrelated problems both want the first frame discarded: 48 - /// 49 - /// 1. The `v4l` crate's `Stream::next()` first-call path runs 50 - /// `VIDIOC_STREAMON` and returns the buffer slot *without* the 51 - /// QBUF/DQBUF cycle that fills it with captured data. The first 52 - /// `next()` therefore hands back whatever was in mmap'd memory at 53 - /// REQBUFS time — typically zeros on Linux, sometimes leftover 54 - /// bytes. Either way it's not a real captured frame. 55 - /// 2. IR illuminators on UVC cameras are commonly gated on the device 56 - /// entering streaming state and take a frame or two after STREAMON 57 - /// to reach full brightness. The first captured frame can be too 58 - /// dark for the detector even if the crate's behaviour were 59 - /// perfect. 47 + /// Both kinds use [`Warmup::Frames(1)`](Warmup::Frames): the `v4l` 48 + /// crate's first `Stream::next()` runs `VIDIOC_STREAMON` and returns the 49 + /// buffer slot *without* the QBUF/DQBUF cycle that fills it, so the very 50 + /// first frame is uninitialised mmap memory, not a capture. One discard 51 + /// covers that for every sensor. 60 52 /// 61 - /// Priming once after open sidesteps both. The ~33 ms cost is paid 62 - /// once per source, not per capture. 53 + /// Notably this does *not* try to wait for an IR illuminator: picking a 54 + /// usable frame out of a strobed/alternating IR stream is the capture 55 + /// layer's job (see [`capture_ir_face`]), not the open path's. 63 56 pub fn open_camera(path: &Path, kind: CameraKind) -> Result<V4l2FrameSource, CliError> { 64 57 let mut cfg = V4l2Config::new(path, "pareidolia camera", kind); 65 58 cfg.preferred_fourcc = Some(match kind { 66 59 CameraKind::Rgb => *b"YUYV", 67 60 CameraKind::Ir => *b"GREY", 68 61 }); 69 - let mut source = V4l2FrameSource::open(cfg).map_err(CliError::Camera)?; 70 - // Prime the stream: see fn-level doc. 71 - let _ = source.capture().map_err(CliError::Camera)?; 72 - Ok(source) 62 + cfg.warmup = Warmup::Frames(1); 63 + V4l2FrameSource::open(cfg).map_err(CliError::Camera) 64 + } 65 + 66 + /// Mean pixel value below which an RGB8 frame is considered "too dark to 67 + /// contain a detectable face". An IR capture lands here only when *every* 68 + /// frame in the burst was dark — i.e. the illuminator never fired at all 69 + /// (broken emitter, covered lens), as opposed to the normal strobed case 70 + /// where the burst contains both dark and lit frames and we keep the lit 71 + /// one. We use it to turn an opaque `NoFaceDetected` into an actionable 72 + /// "the IR frame was nearly black" message. A face-lit IR frame means out 73 + /// around 70–80; an unlit one sits in the low teens (see the XPS 13 74 + /// measurements). 75 + pub const DARK_FRAME_MEAN_THRESHOLD: f32 = 16.0; 76 + 77 + /// Mean of all bytes in an RGB8 frame. Cheap brightness proxy used to 78 + /// distinguish "no face in a normally-exposed frame" from "the frame 79 + /// was effectively black". Thin wrapper over 80 + /// [`pareidolia_core::camera::v4l2::frame_brightness`] for `Frame`s. 81 + pub fn frame_mean_brightness(frame: &Frame) -> f32 { 82 + pareidolia_core::camera::v4l2::frame_brightness(&frame.data) 73 83 } 74 84 75 85 /// Resolve a model path from either the explicit `--*-model` argument ··· 140 150 embed_best_face(&frame, detector, embedder) 141 151 } 142 152 153 + /// Outcome of a face-capture attempt ([`capture_one_face_retrying`], 154 + /// [`capture_ir_face`]): either we found a face, or we exhausted the 155 + /// capture budget without one — in which case we report how bright the 156 + /// *best* frame we saw was, so the caller can distinguish "no face in a 157 + /// well-lit frame" from "every frame was effectively black" (a dead IR 158 + /// illuminator). 159 + #[derive(Debug)] 160 + pub enum RetryCaptureOutcome { 161 + /// A face was detected. `attempt` is the 1-based frame index it was 162 + /// found on (for `capture_one_face_retrying`) or the burst length 163 + /// consumed (for `capture_ir_face`). 164 + Found { face: Box<FaceResult>, attempt: u32 }, 165 + /// No face across the whole budget. `brightest` is the highest mean 166 + /// brightness (0–255) of any frame captured. 167 + NoFace { attempts: u32, brightest: f32 }, 168 + } 169 + 170 + /// Capture from `source` up to `max_attempts` times, returning the first 171 + /// frame in which a face is detected. 172 + /// 173 + /// # Why retry at all 174 + /// 175 + /// A single capture is a coin-flip on hardware where the frame can be 176 + /// momentarily unusable: an IR illuminator still ramping after STREAMON, 177 + /// a blink, the user not yet settled in frame, or rolling-shutter 178 + /// artefacts. Retrying a *bounded* number of times turns those transient 179 + /// misses into a success without changing the security model — we still 180 + /// only accept a face that is actually in front of the lens *now*. 181 + /// 182 + /// # Why bounded (never "loop until a face appears") 183 + /// 184 + /// This helper is used on the auth/test hot path, not just enrollment. 185 + /// An unbounded loop would hang the login flow whenever no one is in 186 + /// front of the camera (the common case — you walked away), holding the 187 + /// camera exclusively and never falling through to the password prompt. 188 + /// The cap guarantees the call returns in bounded time whether or not a 189 + /// face ever shows up. Callers that want a *time* budget instead of an 190 + /// attempt count can size `max_attempts` against the frame rate. 191 + /// 192 + /// `on_frame` is invoked with every captured RGB8 frame and its 1-based 193 + /// attempt index *before* detection runs, so callers can dump frames for 194 + /// debugging regardless of whether detection later succeeds. Errors from 195 + /// it abort the capture (e.g. a failed debug write shouldn't be 196 + /// silently swallowed). 197 + /// 198 + /// Capture/conversion errors and non-"no face" pipeline errors abort 199 + /// immediately — only [`CliError::NoFaceDetected`] is treated as 200 + /// "retry". A `max_attempts` of 0 is treated as 1 (always at least one 201 + /// real attempt). 202 + pub fn capture_one_face_retrying( 203 + source: &mut V4l2FrameSource, 204 + detector: &mut ScrfdDetector, 205 + embedder: &mut ArcfaceEmbedder, 206 + max_attempts: u32, 207 + mut on_frame: impl FnMut(&Frame, u32) -> Result<(), CliError>, 208 + ) -> Result<RetryCaptureOutcome, CliError> { 209 + let max_attempts = max_attempts.max(1); 210 + let mut brightest = 0.0_f32; 211 + for attempt in 1..=max_attempts { 212 + let frame = capture_rgb8(source)?; 213 + brightest = brightest.max(frame_mean_brightness(&frame)); 214 + on_frame(&frame, attempt)?; 215 + match embed_best_face(&frame, detector, embedder) { 216 + Ok(face) => { 217 + return Ok(RetryCaptureOutcome::Found { 218 + face: Box::new(face), 219 + attempt, 220 + }) 221 + } 222 + Err(CliError::NoFaceDetected) => continue, 223 + Err(e) => return Err(e), 224 + } 225 + } 226 + Ok(RetryCaptureOutcome::NoFace { 227 + attempts: max_attempts, 228 + brightest, 229 + }) 230 + } 231 + 232 + /// Default attempt cap for a single modality on the auth/test path. At 233 + /// ~30 fps this is well under a second of capture, short enough not to 234 + /// stall a login but enough to ride out a blink or a marginal IR frame. 235 + pub const DEFAULT_CAPTURE_ATTEMPTS: u32 = 5; 236 + 237 + /// Burst length to capture from an IR camera when the emission mode is 238 + /// unknown (no calibration yet). 239 + /// 240 + /// An IR sensor with a strobed illuminator hands us alternating lit/dark 241 + /// frames (period 2 on the hardware measured so far). Capturing a burst 242 + /// and keeping the brightest guarantees a lit frame as long as the burst 243 + /// spans at least one full strobe period. We default to 4 — comfortably 244 + /// over period-2 alternation, with slack for a period-3 strobe or a 245 + /// dropped frame — without knowing the mode. A calibrated `Constant` 246 + /// sensor can drop this to 1; a calibrated `Strobed { period }` can use 247 + /// exactly `period`. See [`capture_ir_face`]. 248 + pub const DEFAULT_IR_BURST: u32 = 4; 249 + 250 + /// Capture a face from an IR camera, tolerant of a strobed illuminator. 251 + /// 252 + /// Unlike [`capture_one_face_retrying`], which runs detection on every 253 + /// frame, this captures a short *burst* first, picks the brightest frame 254 + /// (via [`select_brightest`]), and runs detection only on that. The 255 + /// reason is the IR emission pattern: a strobed sensor alternates 256 + /// lit/dark frames forever, so "the brightest frame in the burst" is the 257 + /// illuminated one regardless of where in the strobe cycle we started — 258 + /// no per-frame detection guesswork, no dependence on landing a lit 259 + /// frame by luck. 260 + /// 261 + /// `burst` is the number of frames to capture; size it from the 262 + /// calibrated [`IrCaptureMode`] when known (1 for `Constant`, `period` 263 + /// for `Strobed`), else [`DEFAULT_IR_BURST`]. It's clamped to ≥1. 264 + /// 265 + /// `on_frame` sees every captured frame with its 1-based index *before* 266 + /// selection, so debug dumps capture the whole burst (lit and dark 267 + /// frames alike), which is exactly what you want when diagnosing IR. 268 + /// 269 + /// Returns [`RetryCaptureOutcome::NoFace`] (not an error) when the 270 + /// brightest frame still has no detectable face, carrying the brightest 271 + /// brightness so the caller can tell "no face in a lit frame" from 272 + /// "every frame was black". 273 + pub fn capture_ir_face( 274 + source: &mut V4l2FrameSource, 275 + detector: &mut ScrfdDetector, 276 + embedder: &mut ArcfaceEmbedder, 277 + burst: u32, 278 + mut on_frame: impl FnMut(&Frame, u32) -> Result<(), CliError>, 279 + ) -> Result<RetryCaptureOutcome, CliError> { 280 + let burst = burst.max(1); 281 + let mut frames: Vec<Frame> = Vec::with_capacity(burst as usize); 282 + for i in 1..=burst { 283 + let frame = capture_rgb8(source)?; 284 + on_frame(&frame, i)?; 285 + frames.push(frame); 286 + } 287 + 288 + // Pick the brightest frame in the burst — the illuminated one on a 289 + // strobed sensor, and harmlessly "any of them" on a constant one. 290 + let brightest_idx = pareidolia_core::camera::v4l2::select_brightest( 291 + frames.iter().map(|f| f.data.as_slice()), 292 + ) 293 + .expect("burst is non-empty (burst >= 1)"); 294 + let brightest = frame_mean_brightness(&frames[brightest_idx]); 295 + 296 + match embed_best_face(&frames[brightest_idx], detector, embedder) { 297 + Ok(face) => Ok(RetryCaptureOutcome::Found { 298 + face: Box::new(face), 299 + attempt: burst, 300 + }), 301 + Err(CliError::NoFaceDetected) => Ok(RetryCaptureOutcome::NoFace { 302 + attempts: burst, 303 + brightest, 304 + }), 305 + Err(e) => Err(e), 306 + } 307 + } 308 + 143 309 /// Write `frame` to `path` as a binary PPM (P6) file. 144 310 /// 145 311 /// PPM is the simplest "just dump RGB bytes after a tiny text header" ··· 177 343 bytes.copy_from_slice(&hash); 178 344 Ok(ModelFingerprint::from_bytes(bytes)) 179 345 } 346 + 347 + #[cfg(test)] 348 + mod tests { 349 + use super::*; 350 + use pareidolia_core::camera::CameraKind; 351 + use std::time::Instant; 352 + 353 + fn rgb_frame(data: Vec<u8>) -> Frame { 354 + Frame { 355 + width: 1, 356 + height: (data.len() / 3) as u32, 357 + format: pareidolia_core::camera::PixelFormat::Rgb8, 358 + data, 359 + captured_at: Instant::now(), 360 + kind: CameraKind::Ir, 361 + } 362 + } 363 + 364 + #[test] 365 + fn empty_frame_has_zero_brightness() { 366 + assert_eq!(frame_mean_brightness(&rgb_frame(vec![])), 0.0); 367 + } 368 + 369 + #[test] 370 + fn uniform_frame_brightness_equals_its_value() { 371 + let f = rgb_frame(vec![200; 30]); 372 + assert!((frame_mean_brightness(&f) - 200.0).abs() < f32::EPSILON); 373 + } 374 + 375 + #[test] 376 + fn near_black_ir_frame_falls_below_dark_threshold() { 377 + // Mirrors the observed 2018 XPS 13 unlit-illuminator capture 378 + // (pixel range ~7–21, mean ~12): it must classify as "too dark". 379 + let f = rgb_frame(vec![12; 300]); 380 + assert!( 381 + frame_mean_brightness(&f) < DARK_FRAME_MEAN_THRESHOLD, 382 + "an unlit IR frame must register as dark", 383 + ); 384 + } 385 + 386 + #[test] 387 + fn well_exposed_frame_is_above_dark_threshold() { 388 + // A normally-lit frame (RGB capture mean was ~114) must not be 389 + // misclassified as a dark IR frame. 390 + let f = rgb_frame(vec![114; 300]); 391 + assert!(frame_mean_brightness(&f) > DARK_FRAME_MEAN_THRESHOLD); 392 + } 393 + }

Configure Feed

Configure Feed