mirror of
https://github.com/ruvnet/RuView
synced 2026-08-09 20:21:43 +00:00
feat: dual-modal WASM browser pose estimation demo (ADR-058)
Live webcam video + WiFi CSI fusion for real-time pose estimation. Two parallel CNN pipelines (ruvector-cnn-wasm) with attention-weighted fusion and dynamic confidence gating. Three modes: Dual, Video-only, CSI-only. Includes pre-built WASM package (~52KB) for browser deployment. - ADR-058: Dual-modal architecture design - ui/pose-fusion.html: Main demo page with dark theme UI - 7 JS modules: video-capture, csi-simulator, cnn-embedder, fusion-engine, pose-decoder, canvas-renderer, main orchestrator - Pre-built ruvector-cnn-wasm WASM package for browser - CSI heatmap, embedding space visualization, latency metrics - WebSocket support for live ESP32 CSI data - Navigation link added to main dashboard Co-Authored-By: claude-flow <ruv@ruv.net>
This commit is contained in:
ruv
@@ -0,0 +1,166 @@
|
||||
/**
|
||||
* FusionEngine — Attention-weighted dual-modal embedding fusion.
|
||||
*
|
||||
* Combines visual (camera) and CSI (WiFi) embeddings with dynamic
|
||||
* confidence gating based on signal quality.
|
||||
*/
|
||||
|
||||
export class FusionEngine {
|
||||
/**
|
||||
* @param {number} embeddingDim
|
||||
*/
|
||||
constructor(embeddingDim = 128) {
|
||||
this.embeddingDim = embeddingDim;
|
||||
|
||||
// Learnable attention weights (initialized to balanced 0.5)
|
||||
// In production, these would be loaded from trained JSON
|
||||
this.attentionWeights = new Float32Array(embeddingDim).fill(0.5);
|
||||
|
||||
// Dynamic modality confidence [0, 1]
|
||||
this.videoConfidence = 1.0;
|
||||
this.csiConfidence = 0.0;
|
||||
this.fusedConfidence = 0.5;
|
||||
|
||||
// Smoothing for confidence transitions
|
||||
this._smoothAlpha = 0.85;
|
||||
|
||||
// Embedding history for visualization
|
||||
this.recentVideoEmbeddings = [];
|
||||
this.recentCsiEmbeddings = [];
|
||||
this.recentFusedEmbeddings = [];
|
||||
this.maxHistory = 50;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update quality-based confidence scores
|
||||
* @param {number} videoBrightness - [0,1] video brightness quality
|
||||
* @param {number} videoMotion - [0,1] motion detected
|
||||
* @param {number} csiSnr - CSI signal-to-noise ratio in dB
|
||||
* @param {boolean} csiActive - Whether CSI source is connected
|
||||
*/
|
||||
updateConfidence(videoBrightness, videoMotion, csiSnr, csiActive) {
|
||||
// Video confidence: drops with low brightness, boosted by motion
|
||||
let vc = 0;
|
||||
if (videoBrightness > 0.05) {
|
||||
vc = Math.min(1, videoBrightness * 1.5) * 0.7 + Math.min(1, videoMotion * 3) * 0.3;
|
||||
}
|
||||
|
||||
// CSI confidence: based on SNR and connection status
|
||||
let cc = 0;
|
||||
if (csiActive) {
|
||||
cc = Math.min(1, csiSnr / 25); // 25dB = full confidence
|
||||
}
|
||||
|
||||
// Smooth transitions
|
||||
this.videoConfidence = this._smoothAlpha * this.videoConfidence + (1 - this._smoothAlpha) * vc;
|
||||
this.csiConfidence = this._smoothAlpha * this.csiConfidence + (1 - this._smoothAlpha) * cc;
|
||||
|
||||
// Fused confidence is the max of either (fusion can only help)
|
||||
this.fusedConfidence = Math.min(1, Math.sqrt(
|
||||
this.videoConfidence * this.videoConfidence + this.csiConfidence * this.csiConfidence
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* Fuse video and CSI embeddings
|
||||
* @param {Float32Array|null} videoEmb - Visual embedding (or null if video-off)
|
||||
* @param {Float32Array|null} csiEmb - CSI embedding (or null if CSI-off)
|
||||
* @param {string} mode - 'dual' | 'video' | 'csi'
|
||||
* @returns {Float32Array} Fused embedding
|
||||
*/
|
||||
fuse(videoEmb, csiEmb, mode = 'dual') {
|
||||
const dim = this.embeddingDim;
|
||||
const fused = new Float32Array(dim);
|
||||
|
||||
if (mode === 'video' || !csiEmb) {
|
||||
if (videoEmb) fused.set(videoEmb);
|
||||
this._recordEmbedding(videoEmb, null, fused);
|
||||
return fused;
|
||||
}
|
||||
|
||||
if (mode === 'csi' || !videoEmb) {
|
||||
if (csiEmb) fused.set(csiEmb);
|
||||
this._recordEmbedding(null, csiEmb, fused);
|
||||
return fused;
|
||||
}
|
||||
|
||||
// Dual mode: attention-weighted fusion with confidence gating
|
||||
const totalConf = this.videoConfidence + this.csiConfidence;
|
||||
const videoWeight = totalConf > 0 ? this.videoConfidence / totalConf : 0.5;
|
||||
|
||||
for (let i = 0; i < dim; i++) {
|
||||
const alpha = this.attentionWeights[i] * videoWeight +
|
||||
(1 - this.attentionWeights[i]) * (1 - videoWeight);
|
||||
fused[i] = alpha * videoEmb[i] + (1 - alpha) * csiEmb[i];
|
||||
}
|
||||
|
||||
// Re-normalize
|
||||
let norm = 0;
|
||||
for (let i = 0; i < dim; i++) norm += fused[i] * fused[i];
|
||||
norm = Math.sqrt(norm);
|
||||
if (norm > 1e-8) {
|
||||
for (let i = 0; i < dim; i++) fused[i] /= norm;
|
||||
}
|
||||
|
||||
this._recordEmbedding(videoEmb, csiEmb, fused);
|
||||
return fused;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get embedding pairs for 2D visualization (PCA projection)
|
||||
* @returns {{ video: Array, csi: Array, fused: Array }}
|
||||
*/
|
||||
getEmbeddingPoints() {
|
||||
// Simple 2D projection using first two principal components (approximated)
|
||||
const project = (emb) => {
|
||||
if (!emb || emb.length < 4) return null;
|
||||
// Use pairs of dimensions as crude 2D projection
|
||||
let x = 0, y = 0;
|
||||
for (let i = 0; i < emb.length; i += 2) {
|
||||
x += emb[i] * (i % 4 < 2 ? 1 : -1);
|
||||
if (i + 1 < emb.length) {
|
||||
y += emb[i + 1] * (i % 4 < 2 ? 1 : -1);
|
||||
}
|
||||
}
|
||||
return [x * 2, y * 2]; // Scale for visibility
|
||||
};
|
||||
|
||||
return {
|
||||
video: this.recentVideoEmbeddings.map(project).filter(Boolean),
|
||||
csi: this.recentCsiEmbeddings.map(project).filter(Boolean),
|
||||
fused: this.recentFusedEmbeddings.map(project).filter(Boolean)
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Cross-modal similarity score
|
||||
* @returns {number} Cosine similarity between latest video and CSI embeddings
|
||||
*/
|
||||
getCrossModalSimilarity() {
|
||||
const v = this.recentVideoEmbeddings[this.recentVideoEmbeddings.length - 1];
|
||||
const c = this.recentCsiEmbeddings[this.recentCsiEmbeddings.length - 1];
|
||||
if (!v || !c) return 0;
|
||||
|
||||
let dot = 0, na = 0, nb = 0;
|
||||
for (let i = 0; i < v.length; i++) {
|
||||
dot += v[i] * c[i];
|
||||
na += v[i] * v[i];
|
||||
nb += c[i] * c[i];
|
||||
}
|
||||
na = Math.sqrt(na); nb = Math.sqrt(nb);
|
||||
return (na > 1e-8 && nb > 1e-8) ? dot / (na * nb) : 0;
|
||||
}
|
||||
|
||||
_recordEmbedding(video, csi, fused) {
|
||||
if (video) {
|
||||
this.recentVideoEmbeddings.push(new Float32Array(video));
|
||||
if (this.recentVideoEmbeddings.length > this.maxHistory) this.recentVideoEmbeddings.shift();
|
||||
}
|
||||
if (csi) {
|
||||
this.recentCsiEmbeddings.push(new Float32Array(csi));
|
||||
if (this.recentCsiEmbeddings.length > this.maxHistory) this.recentCsiEmbeddings.shift();
|
||||
}
|
||||
this.recentFusedEmbeddings.push(new Float32Array(fused));
|
||||
if (this.recentFusedEmbeddings.length > this.maxHistory) this.recentFusedEmbeddings.shift();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user