A facial recognition login service for Linux.
core: Add ArcFace embedder implementation
Last real-model piece of the pipeline. Takes a 112×112 RGB8 chip produced
by `align_face`, runs a single ONNX forward pass through the ArcFace
glintr100 model, and returns a 512-d L2-normalised embedding ready for
the matching layer in M4.
Substantially simpler than the SCRFD detector: single input, single
output, no anchor decoding, no NMS. Preprocessing differs from SCRFD's
in one detail — ArcFace's training used `(pixel − 127.5) / 127.5`
(range [-1, 1]) where SCRFD used `/ 128.0` — and the embedder is strict
about input dimensions: 112×112 only, no rescaling on input. Wrong shapes
surface as `InvalidInputShape` before any inference runs.
The 512-d output goes straight through `Embedding::normalized` so cosine
similarity reduces to a dot product in M4's matching layer. The output
dimension is enforced after inference; a model returning the wrong
dimension surfaces as a Backend error rather than producing a malformed
embedding.
# Tests
4 pure unit tests covering the preprocessor (NCHW shape, the −1/+1
boundary mappings, per-channel separation into NCHW planes).
5 hardware-gated tests covering the embedder end-to-end via the real
ArcFace model:
- Non-RGB8 input is rejected before inference
- Wrong dimensions are rejected before inference
- Solid gray chip produces a 512-d unit-magnitude embedding
- Repeated inference is bit-for-bit deterministic (default CPU EP)
- Different inputs produce non-identical embeddings (sanity check)
Combined hardware suite now runs in ~16s (V4L2 capture dominates at ~11s;
all 5 ArcFace tests take ~5s combined including model load).
Isaac Corbrey
(May 27, 2026, 7:15 PM EDT)
8ee17ca5
8de0e808
+346
+3
crates/core/src/pipeline.rs
+3
crates/core/src/pipeline.rs
+343
crates/core/src/pipeline/arcface.rs
+343
crates/core/src/pipeline/arcface.rs
···
1
+
//! ArcFace face embedder via ONNX Runtime.
2
+
//!
3
+
//! Implements the [`Embedder`] trait against an [`OnnxSession`] loaded
4
+
//! with an ArcFace model (e.g. `glintr100` from the InsightFace buffalo_l
5
+
//! bundle, pinned in `nix/models.nix`).
6
+
//!
7
+
//! # Pipeline
8
+
//!
9
+
//! 1. Validate the input is a 112×112 RGB8 chip (matching what
10
+
//! [`align_face`] produces).
11
+
//! 2. Normalise with `(pixel − 127.5) / 127.5`, channel order RGB, NCHW
12
+
//! layout — the InsightFace recognition default. (Different from SCRFD,
13
+
//! which uses `/ 128.0`.)
14
+
//! 3. Single ONNX forward pass.
15
+
//! 4. L2-normalise the 512-d output via [`Embedding::normalized`] so the
16
+
//! matching layer can compare with a plain dot product later.
17
+
//!
18
+
//! [`align_face`]: super::align::align_face
19
+
20
+
use std::path::Path;
21
+
22
+
use ndarray::Array4;
23
+
use ort::inputs;
24
+
use ort::value::Value;
25
+
26
+
use crate::camera::{Frame, PixelFormat};
27
+
use crate::pipeline::align::ARCFACE_CHIP_SIZE;
28
+
use crate::pipeline::onnx::OnnxSession;
29
+
use crate::pipeline::{Embedder, Embedding, PipelineError};
30
+
31
+
/// ArcFace pixel normalisation: `(pixel − INPUT_MEAN) / INPUT_STD`. Both
32
+
/// values are 127.5, giving an output range of `[-1, 1]`. Do not change
33
+
/// without retraining.
34
+
const INPUT_MEAN: f32 = 127.5;
35
+
const INPUT_STD: f32 = 127.5;
36
+
37
+
/// Dimensionality of the embedding vector produced by glintr100 and the
38
+
/// other buffalo_l ArcFace variants.
39
+
pub const DEFAULT_OUTPUT_DIM: usize = 512;
40
+
41
+
/// Embedder configuration.
42
+
#[derive(Debug, Clone, Copy)]
43
+
pub struct ArcfaceConfig {
44
+
/// Square input edge in pixels.
45
+
pub input_size: u32,
46
+
/// Dimensionality of the output embedding. Enforced after inference;
47
+
/// a model that returns the wrong dim surfaces as
48
+
/// [`PipelineError::Backend`].
49
+
pub output_dim: usize,
50
+
}
51
+
52
+
impl Default for ArcfaceConfig {
53
+
fn default() -> Self {
54
+
Self {
55
+
input_size: ARCFACE_CHIP_SIZE,
56
+
output_dim: DEFAULT_OUTPUT_DIM,
57
+
}
58
+
}
59
+
}
60
+
61
+
/// ArcFace-backed face embedder.
62
+
#[derive(Debug)]
63
+
pub struct ArcfaceEmbedder {
64
+
session: OnnxSession,
65
+
config: ArcfaceConfig,
66
+
}
67
+
68
+
impl ArcfaceEmbedder {
69
+
/// Load an ArcFace ONNX model from `model_path`.
70
+
pub fn open(
71
+
model_path: impl AsRef<Path>,
72
+
config: ArcfaceConfig,
73
+
) -> Result<Self, PipelineError> {
74
+
let session = OnnxSession::from_file(model_path)?;
75
+
Ok(Self::from_session(session, config))
76
+
}
77
+
78
+
/// Wrap an existing [`OnnxSession`].
79
+
pub fn from_session(session: OnnxSession, config: ArcfaceConfig) -> Self {
80
+
Self { session, config }
81
+
}
82
+
83
+
pub fn config(&self) -> &ArcfaceConfig {
84
+
&self.config
85
+
}
86
+
}
87
+
88
+
impl Embedder for ArcfaceEmbedder {
89
+
fn embed(&mut self, chip: &Frame) -> Result<Embedding, PipelineError> {
90
+
if chip.format != PixelFormat::Rgb8 {
91
+
return Err(PipelineError::InvalidInputFormat {
92
+
expected: PixelFormat::Rgb8,
93
+
actual: chip.format,
94
+
});
95
+
}
96
+
let expected = self.config.input_size;
97
+
if chip.width != expected || chip.height != expected {
98
+
return Err(PipelineError::InvalidInputShape {
99
+
expected_w: expected,
100
+
expected_h: expected,
101
+
actual_w: chip.width,
102
+
actual_h: chip.height,
103
+
});
104
+
}
105
+
// Defensive: chip.data should be exactly expected² × 3 bytes if the
106
+
// header is honest, but checking explicitly avoids an indexing
107
+
// panic on a malformed input.
108
+
let expected_len = (expected as usize) * (expected as usize) * 3;
109
+
if chip.data.len() != expected_len {
110
+
return Err(PipelineError::Backend(format!(
111
+
"chip buffer length {} does not match {}×{}×3 = {}",
112
+
chip.data.len(),
113
+
expected,
114
+
expected,
115
+
expected_len
116
+
)));
117
+
}
118
+
119
+
let input_tensor = preprocess(&chip.data, expected as usize);
120
+
let input_value = Value::from_array(input_tensor).map_err(map_ort_err)?;
121
+
let outputs = self
122
+
.session
123
+
.session_mut()
124
+
.run(inputs![input_value])
125
+
.map_err(map_ort_err)?;
126
+
127
+
let (_shape, values) = outputs[0]
128
+
.try_extract_tensor::<f32>()
129
+
.map_err(map_ort_err)?;
130
+
if values.len() != self.config.output_dim {
131
+
return Err(PipelineError::Backend(format!(
132
+
"expected embedder output dim {}, got {}",
133
+
self.config.output_dim,
134
+
values.len()
135
+
)));
136
+
}
137
+
138
+
Ok(Embedding::normalized(values.to_vec()))
139
+
}
140
+
141
+
fn input_size(&self) -> u32 {
142
+
self.config.input_size
143
+
}
144
+
145
+
fn output_dim(&self) -> usize {
146
+
self.config.output_dim
147
+
}
148
+
}
149
+
150
+
/// Build a `[1, 3, size, size]` `f32` NCHW tensor from an RGB8 chip buffer
151
+
/// with the ArcFace normalisation baked in.
152
+
///
153
+
/// Public to the module so the unit tests can pin the exact numeric output.
154
+
fn preprocess(chip_data: &[u8], size: usize) -> Array4<f32> {
155
+
let mut tensor = vec![0f32; 3 * size * size];
156
+
for y in 0..size {
157
+
for x in 0..size {
158
+
for c in 0..3 {
159
+
let pixel = chip_data[(y * size + x) * 3 + c] as f32;
160
+
let normalised = (pixel - INPUT_MEAN) / INPUT_STD;
161
+
tensor[c * size * size + y * size + x] = normalised;
162
+
}
163
+
}
164
+
}
165
+
Array4::from_shape_vec((1, 3, size, size), tensor)
166
+
.expect("tensor data length matches NCHW dimensions")
167
+
}
168
+
169
+
fn map_ort_err<E: std::fmt::Display>(e: E) -> PipelineError {
170
+
PipelineError::Backend(e.to_string())
171
+
}
172
+
173
+
#[cfg(test)]
174
+
mod tests {
175
+
use super::*;
176
+
use crate::camera::CameraKind;
177
+
use std::time::Instant;
178
+
179
+
fn solid_chip(size: u32, color: [u8; 3]) -> Frame {
180
+
let s = size as usize;
181
+
let mut data = vec![0u8; s * s * 3];
182
+
for chunk in data.chunks_exact_mut(3) {
183
+
chunk.copy_from_slice(&color);
184
+
}
185
+
Frame {
186
+
width: size,
187
+
height: size,
188
+
format: PixelFormat::Rgb8,
189
+
data,
190
+
captured_at: Instant::now(),
191
+
kind: CameraKind::Rgb,
192
+
}
193
+
}
194
+
195
+
// ----- preprocess ---------------------------------------------------------
196
+
197
+
#[test]
198
+
fn preprocess_produces_nchw_shape() {
199
+
let chip = solid_chip(ARCFACE_CHIP_SIZE, [128, 128, 128]);
200
+
let tensor = preprocess(&chip.data, ARCFACE_CHIP_SIZE as usize);
201
+
assert_eq!(
202
+
tensor.dim(),
203
+
(1, 3, ARCFACE_CHIP_SIZE as usize, ARCFACE_CHIP_SIZE as usize)
204
+
);
205
+
}
206
+
207
+
#[test]
208
+
fn preprocess_zero_pixels_map_to_minus_one() {
209
+
// (0 - 127.5) / 127.5 = -1.0
210
+
let chip = solid_chip(ARCFACE_CHIP_SIZE, [0, 0, 0]);
211
+
let tensor = preprocess(&chip.data, ARCFACE_CHIP_SIZE as usize);
212
+
for &v in tensor.iter() {
213
+
assert!((v - (-1.0)).abs() < 1e-6, "expected -1.0, got {v}");
214
+
}
215
+
}
216
+
217
+
#[test]
218
+
fn preprocess_full_pixels_map_to_one_within_epsilon() {
219
+
// (255 - 127.5) / 127.5 = 1.0 exactly in f32.
220
+
let chip = solid_chip(ARCFACE_CHIP_SIZE, [255, 255, 255]);
221
+
let tensor = preprocess(&chip.data, ARCFACE_CHIP_SIZE as usize);
222
+
for &v in tensor.iter() {
223
+
assert!((v - 1.0).abs() < 1e-6, "expected 1.0, got {v}");
224
+
}
225
+
}
226
+
227
+
#[test]
228
+
fn preprocess_separates_channels_into_nchw_planes() {
229
+
// 2×2 chip with distinct per-channel values: R=10, G=128, B=200.
230
+
// After normalisation: R ≈ -0.922, G ≈ 0.004, B ≈ 0.569.
231
+
// The first plane (c=0) holds all R, second all G, third all B.
232
+
let size = 2usize;
233
+
let mut chip = vec![0u8; size * size * 3];
234
+
for pixel in chip.chunks_exact_mut(3) {
235
+
pixel[0] = 10; // R
236
+
pixel[1] = 128; // G
237
+
pixel[2] = 200; // B
238
+
}
239
+
let tensor = preprocess(&chip, size);
240
+
241
+
// Plane 0: R
242
+
for &v in tensor.slice(ndarray::s![0, 0, .., ..]).iter() {
243
+
assert!((v - ((10.0 - 127.5) / 127.5)).abs() < 1e-6);
244
+
}
245
+
// Plane 1: G
246
+
for &v in tensor.slice(ndarray::s![0, 1, .., ..]).iter() {
247
+
assert!((v - ((128.0 - 127.5) / 127.5)).abs() < 1e-6);
248
+
}
249
+
// Plane 2: B
250
+
for &v in tensor.slice(ndarray::s![0, 2, .., ..]).iter() {
251
+
assert!((v - ((200.0 - 127.5) / 127.5)).abs() < 1e-6);
252
+
}
253
+
}
254
+
255
+
// ----- ArcfaceEmbedder input validation ----------------------------------
256
+
257
+
/// Build an embedder backed by a never-loaded session, for tests that
258
+
/// only exercise the validation paths (no inference reached). We can't
259
+
/// construct an OnnxSession without a real file, so these tests are
260
+
/// hardware-gated too — they need PAREIDOLIA_TEST_ARCFACE to provide
261
+
/// any valid session, but rejection happens before inference runs.
262
+
fn open_for_validation() -> Option<ArcfaceEmbedder> {
263
+
let path = std::env::var("PAREIDOLIA_TEST_ARCFACE").ok()?;
264
+
Some(ArcfaceEmbedder::open(path, ArcfaceConfig::default()).expect("load arcface"))
265
+
}
266
+
267
+
#[test]
268
+
#[ignore = "requires PAREIDOLIA_TEST_ARCFACE env var (set by nix develop)"]
269
+
fn embed_rejects_non_rgb8_input() {
270
+
let mut embedder = open_for_validation().expect("env var");
271
+
let bad = Frame {
272
+
width: ARCFACE_CHIP_SIZE,
273
+
height: ARCFACE_CHIP_SIZE,
274
+
format: PixelFormat::Yuyv,
275
+
data: vec![128u8; (ARCFACE_CHIP_SIZE as usize) * (ARCFACE_CHIP_SIZE as usize) * 2],
276
+
captured_at: Instant::now(),
277
+
kind: CameraKind::Rgb,
278
+
};
279
+
let err = embedder.embed(&bad).expect_err("yuyv rejected");
280
+
assert!(matches!(err, PipelineError::InvalidInputFormat { .. }));
281
+
}
282
+
283
+
#[test]
284
+
#[ignore = "requires PAREIDOLIA_TEST_ARCFACE env var (set by nix develop)"]
285
+
fn embed_rejects_wrong_dimensions() {
286
+
let mut embedder = open_for_validation().expect("env var");
287
+
let bad = solid_chip(64, [128, 128, 128]); // not 112
288
+
let err = embedder.embed(&bad).expect_err("wrong size rejected");
289
+
assert!(matches!(err, PipelineError::InvalidInputShape { .. }));
290
+
}
291
+
292
+
#[test]
293
+
#[ignore = "requires PAREIDOLIA_TEST_ARCFACE env var (set by nix develop)"]
294
+
fn arcface_embeds_solid_gray_chip_to_unit_magnitude_vector() {
295
+
let mut embedder = open_for_validation().expect("env var");
296
+
let chip = solid_chip(ARCFACE_CHIP_SIZE, [128, 128, 128]);
297
+
let embedding = embedder.embed(&chip).expect("embed");
298
+
299
+
assert_eq!(embedding.dim(), DEFAULT_OUTPUT_DIM);
300
+
let mag = embedding.magnitude();
301
+
assert!(
302
+
(mag - 1.0).abs() < 1e-4,
303
+
"L2-normalised embedding magnitude should be ~1.0, got {mag}",
304
+
);
305
+
}
306
+
307
+
#[test]
308
+
#[ignore = "requires PAREIDOLIA_TEST_ARCFACE env var (set by nix develop)"]
309
+
fn arcface_is_deterministic_on_repeated_inputs() {
310
+
let mut embedder = open_for_validation().expect("env var");
311
+
let chip = solid_chip(ARCFACE_CHIP_SIZE, [200, 100, 50]);
312
+
let a = embedder.embed(&chip).expect("embed a");
313
+
let b = embedder.embed(&chip).expect("embed b");
314
+
// Same input → same output bit-for-bit (CPU inference is
315
+
// deterministic with the default execution provider).
316
+
assert_eq!(a.values(), b.values());
317
+
// Cosine self-similarity should be 1.0.
318
+
let cos = a.cosine_similarity(&b);
319
+
assert!(
320
+
(cos - 1.0).abs() < 1e-5,
321
+
"cos(same, same) should be ~1.0, got {cos}",
322
+
);
323
+
}
324
+
325
+
#[test]
326
+
#[ignore = "requires PAREIDOLIA_TEST_ARCFACE env var (set by nix develop)"]
327
+
fn arcface_different_inputs_produce_different_embeddings() {
328
+
let mut embedder = open_for_validation().expect("env var");
329
+
let a = embedder
330
+
.embed(&solid_chip(ARCFACE_CHIP_SIZE, [200, 100, 50]))
331
+
.unwrap();
332
+
let b = embedder
333
+
.embed(&solid_chip(ARCFACE_CHIP_SIZE, [50, 100, 200]))
334
+
.unwrap();
335
+
// Different chips should produce different embeddings — they may
336
+
// not be orthogonal, but they should not be byte-identical.
337
+
assert_ne!(
338
+
a.values(),
339
+
b.values(),
340
+
"different inputs produced identical embeddings",
341
+
);
342
+
}
343
+
}