mirror of
https://github.com/ruvnet/RuView
synced 2026-08-08 20:11:43 +00:00
feat(cog-pose): per-room LoRA calibration adapter in the Rust inference path
Ports the calibration mechanism (ADR-150 §3.5-3.6, reference impl in aether-arena/calibration/) into the real product pose engine. The Candle InferenceEngine now loads an optional per-room adapter safetensors and applies low-rank deltas (y + (x.A).B) on the fc1/fc2 head at inference. Architecture-agnostic LoRA; base behaviour unchanged when no adapter. New API: with_weights_and_adapter(), is_calibrated(). Tested: adapter detection + output-change integration test (6/6 pass). Co-Authored-By: claude-flow <ruv@ruv.net>
This commit is contained in:
@@ -46,6 +46,40 @@ impl PoseOutput {
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-room LoRA calibration adapter (ADR-150 §3.5–3.6). Low-rank deltas on the pose
|
||||
/// head: `delta = (x · A) · B`, with `A:[in,r]`, `B:[r,out]` (scale baked into `B` at
|
||||
/// save time). A handful of labeled in-room samples fit this ~few-KB adapter and recover
|
||||
/// SOTA-level pose for an unseen room/person, on top of the frozen shared base.
|
||||
/// Adapter safetensors keys: `fc1.a`, `fc1.b`, `fc2.a`, `fc2.b` (any subset).
|
||||
#[derive(Clone)]
|
||||
struct PoseLora {
|
||||
fc1: Option<(Tensor, Tensor)>,
|
||||
fc2: Option<(Tensor, Tensor)>,
|
||||
}
|
||||
|
||||
impl PoseLora {
|
||||
/// Load from an adapter safetensors. Missing layer keys are simply skipped.
|
||||
fn load(path: &Path, device: &Device) -> candle_core::Result<Self> {
|
||||
let t = candle_core::safetensors::load(path, device)?;
|
||||
let pair = |a: &str, b: &str| match (t.get(a), t.get(b)) {
|
||||
(Some(x), Some(y)) => Some((x.clone(), y.clone())),
|
||||
_ => None,
|
||||
};
|
||||
Ok(Self {
|
||||
fc1: pair("fc1.a", "fc1.b"),
|
||||
fc2: pair("fc2.a", "fc2.b"),
|
||||
})
|
||||
}
|
||||
|
||||
/// `y + (x · A) · B` when an adapter for this layer is present, else `y` unchanged.
|
||||
fn apply(slot: &Option<(Tensor, Tensor)>, x: &Tensor, y: Tensor) -> candle_core::Result<Tensor> {
|
||||
match slot {
|
||||
Some((a, b)) => y + x.matmul(a)?.matmul(b)?,
|
||||
None => Ok(y),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal model — mirrors the training script's `PoseModel` exactly.
|
||||
struct PoseNet {
|
||||
c1: Conv1d,
|
||||
@@ -53,6 +87,8 @@ struct PoseNet {
|
||||
c3: Conv1d,
|
||||
fc1: Linear,
|
||||
fc2: Linear,
|
||||
/// Optional per-room calibration adapter (none = shared base behaviour).
|
||||
adapter: Option<PoseLora>,
|
||||
}
|
||||
|
||||
impl PoseNet {
|
||||
@@ -108,20 +144,31 @@ impl PoseNet {
|
||||
c3,
|
||||
fc1,
|
||||
fc2,
|
||||
adapter: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Forward pass: `[B, 56, 20]` -> `[B, 34]` in `[0, 1]`.
|
||||
/// Forward pass: `[B, 56, 20]` -> `[B, 34]` in `[0, 1]`. Applies the per-room
|
||||
/// LoRA calibration adapter on the head layers when one is attached.
|
||||
fn forward(&self, x: &Tensor) -> candle_core::Result<Tensor> {
|
||||
let h = self.c1.forward(x)?.relu()?;
|
||||
let h = self.c2.forward(&h)?.relu()?;
|
||||
let h = self.c3.forward(&h)?.relu()?;
|
||||
// Global average pool over time dim (last dim) -> [B, 128]
|
||||
let h = h.mean(2)?;
|
||||
let h = self.fc1.forward(&h)?.relu()?;
|
||||
let h = self.fc2.forward(&h)?;
|
||||
let pooled = h.mean(2)?;
|
||||
// fc1 (+ adapter delta) -> ReLU
|
||||
let mut h1 = self.fc1.forward(&pooled)?;
|
||||
if let Some(ad) = &self.adapter {
|
||||
h1 = PoseLora::apply(&ad.fc1, &pooled, h1)?;
|
||||
}
|
||||
let h1 = h1.relu()?;
|
||||
// fc2 (+ adapter delta)
|
||||
let mut h2 = self.fc2.forward(&h1)?;
|
||||
if let Some(ad) = &self.adapter {
|
||||
h2 = PoseLora::apply(&ad.fc2, &h1, h2)?;
|
||||
}
|
||||
// sigmoid -> keep in [0, 1]
|
||||
candle_nn::ops::sigmoid(&h)
|
||||
candle_nn::ops::sigmoid(&h2)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,6 +195,17 @@ impl InferenceEngine {
|
||||
/// in `cog-pose-estimation run`). If `weights_path` is `None`, the
|
||||
/// stub fallback is used.
|
||||
pub fn with_weights(weights_path: Option<&Path>) -> Result<Self, Box<dyn std::error::Error>> {
|
||||
Self::with_weights_and_adapter(weights_path, None)
|
||||
}
|
||||
|
||||
/// Create an engine with a shared base **and an optional per-room calibration
|
||||
/// adapter** (ADR-150 §3.5). The adapter is a tiny LoRA safetensors fitted from a
|
||||
/// short labeled in-room capture (`aether-arena/calibration/calibrate.py`); attaching
|
||||
/// it recovers SOTA-level pose in an unseen room/person. `None` = uncalibrated base.
|
||||
pub fn with_weights_and_adapter(
|
||||
weights_path: Option<&Path>,
|
||||
adapter_path: Option<&Path>,
|
||||
) -> Result<Self, Box<dyn std::error::Error>> {
|
||||
let device = pick_device();
|
||||
let inner = match weights_path {
|
||||
Some(p) if p.exists() => {
|
||||
@@ -158,7 +216,12 @@ impl InferenceEngine {
|
||||
let vb = unsafe {
|
||||
VarBuilder::from_mmaped_safetensors(&[p.to_path_buf()], DType::F32, &device)?
|
||||
};
|
||||
let net = PoseNet::new(vb)?;
|
||||
let mut net = PoseNet::new(vb)?;
|
||||
if let Some(ap) = adapter_path {
|
||||
if ap.exists() {
|
||||
net.adapter = Some(PoseLora::load(ap, &device)?);
|
||||
}
|
||||
}
|
||||
Some(Arc::new(LoadedModel { net }))
|
||||
}
|
||||
_ => None,
|
||||
@@ -166,6 +229,14 @@ impl InferenceEngine {
|
||||
Ok(Self { inner, device })
|
||||
}
|
||||
|
||||
/// Whether a per-room calibration adapter is currently attached.
|
||||
pub fn is_calibrated(&self) -> bool {
|
||||
self.inner
|
||||
.as_ref()
|
||||
.map(|m| m.net.adapter.is_some())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Where the weights actually came from. Useful for the run.started event.
|
||||
pub fn backend(&self) -> &'static str {
|
||||
match (&self.inner, &self.device) {
|
||||
|
||||
Reference in New Issue
Block a user