mirror of
https://github.com/ruvnet/RuView
synced 2026-06-09 10:13:17 +00:00
Compare commits
31 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a85d4e31e4 | |||
| b16d7431bc | |||
| b3a5012dbd | |||
| e6a5df36eb | |||
| 5c914e63c7 | |||
| a5e99670f8 | |||
| 6b4994e105 | |||
| 6959a42312 | |||
| 962e0f4a34 | |||
| c58f49f21a | |||
| cbcb389cb6 | |||
| e00cee6146 | |||
| 5dcafc9c37 | |||
| e21803f714 | |||
| bdd1efeb03 | |||
| aeb69315d8 | |||
| cfda8dbd14 | |||
| dc865c236e | |||
| 96bc4b4ede | |||
| feda871e02 | |||
| 43ac76a17f | |||
| 6a2b2bdcbf | |||
| d67d9872c1 | |||
| 67fec45e61 | |||
| dc7f6cd096 | |||
| 4b1a835107 | |||
| 9c3c8b98bc | |||
| fcb6f4bf12 | |||
| 3314c8db8d | |||
| ef20a7280d | |||
| ad15f1b049 |
@@ -216,10 +216,14 @@ jobs:
|
||||
htmlcov/
|
||||
|
||||
# Performance and Load Tests
|
||||
# NOTE: tests/performance/locustfile.py and the src.api.main app path both
|
||||
# predate the v1→archive/v1 reorganisation. continue-on-error: true until a
|
||||
# proper locust suite is added under archive/v1/tests/performance/.
|
||||
performance-test:
|
||||
name: Performance Tests
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test]
|
||||
continue-on-error: true
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -238,6 +242,7 @@ jobs:
|
||||
pip install locust
|
||||
|
||||
- name: Start application
|
||||
working-directory: archive/v1
|
||||
run: |
|
||||
uvicorn src.api.main:app --host 0.0.0.0 --port 8000 &
|
||||
sleep 10
|
||||
@@ -352,6 +357,7 @@ jobs:
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: Generate OpenAPI spec
|
||||
working-directory: archive/v1
|
||||
run: |
|
||||
python -c "
|
||||
from src.api.main import app
|
||||
@@ -373,6 +379,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [code-quality, test, rust-tests, performance-test, docker-build, docs]
|
||||
if: always()
|
||||
permissions:
|
||||
contents: write # required by softprops/action-gh-release
|
||||
# GitHub Actions does not allow `secrets.X` directly in step-level `if:`
|
||||
# expressions — only `env.X`. Promote the secret to env at job scope so
|
||||
# the gating expression below is parseable.
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
name: GitHub Clone Tracking → data/clone-data.rvf
|
||||
|
||||
# Persists rolling 14-day clone-traffic snapshots to data/clone-data.rvf in
|
||||
# the ruvector JSONL RVF format. GitHub's /traffic/clones endpoint only
|
||||
# retains the last 14 days server-side, so without this scheduled scrape
|
||||
# the data is gone forever the moment it falls outside the window.
|
||||
#
|
||||
# Format: JSONL RVF
|
||||
# - line 1 is a `metadata` segment that initializes the file
|
||||
# - each subsequent run appends one `clone_snapshot` segment carrying the
|
||||
# 14-day rollup PLUS per-day breakdown
|
||||
# - file is idempotent: per-day entries are keyed by `timestamp` so a
|
||||
# downstream reader can dedupe across overlapping snapshot windows
|
||||
#
|
||||
# Schedule: every 14 days (1st + 15th of each month, ~14-day cadence in
|
||||
# practice). Workflow can also be dispatched manually for backfill or test.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 01:23 UTC on the 1st and 15th of every month — close to 14-day cadence
|
||||
# without cron's "every 14 days" monthly-reset weirdness. Picking :23
|
||||
# avoids the cron herd on :00.
|
||||
- cron: '23 1 1,15 * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
group: clone-tracking
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
snapshot:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Fetch /traffic/clones + /traffic/views from GitHub
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
mkdir -p data
|
||||
gh api repos/${{ github.repository }}/traffic/clones > /tmp/clones.json
|
||||
gh api repos/${{ github.repository }}/traffic/views > /tmp/views.json
|
||||
echo "--- clones rollup ---"
|
||||
jq '{count, uniques, days: (.clones | length)}' /tmp/clones.json
|
||||
echo "--- views rollup ---"
|
||||
jq '{count, uniques, days: (.views | length)}' /tmp/views.json
|
||||
|
||||
- name: Append snapshot to data/clone-data.rvf
|
||||
env:
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
set -e
|
||||
RVF="data/clone-data.rvf"
|
||||
FETCHED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
# Initialize the file with a metadata segment on first run.
|
||||
if [ ! -f "$RVF" ]; then
|
||||
echo "Initializing $RVF with metadata segment"
|
||||
jq -n --arg repo "$REPO" --arg ts "$FETCHED_AT" '{
|
||||
type: "metadata",
|
||||
name: "ruview-clone-traffic-history",
|
||||
version: "1.0.0",
|
||||
schema: "ruvector.rvf.jsonl/v1",
|
||||
format: "github-traffic-snapshots",
|
||||
repo: $repo,
|
||||
source: "GitHub Traffic API /repos/{repo}/traffic/{clones,views}",
|
||||
policy: "GitHub retains only 14 days server-side; this file is the long-term record.",
|
||||
segments: ["metadata", "clone_snapshot", "view_snapshot"],
|
||||
created_at: $ts,
|
||||
custom: {
|
||||
cadence: "twice monthly (1st and 15th, ~14-day intervals)",
|
||||
idempotency_key: "timestamp (per-day records de-duplicate across overlapping snapshot windows)"
|
||||
}
|
||||
}' >> "$RVF"
|
||||
fi
|
||||
|
||||
# Append the clone snapshot.
|
||||
jq --arg ts "$FETCHED_AT" '{
|
||||
type: "clone_snapshot",
|
||||
fetched_at: $ts,
|
||||
window_count: .count,
|
||||
window_uniques: .uniques,
|
||||
per_day: .clones
|
||||
}' /tmp/clones.json >> "$RVF"
|
||||
|
||||
# Append the views snapshot (free with the same auth).
|
||||
jq --arg ts "$FETCHED_AT" '{
|
||||
type: "view_snapshot",
|
||||
fetched_at: $ts,
|
||||
window_count: .count,
|
||||
window_uniques: .uniques,
|
||||
per_day: .views
|
||||
}' /tmp/views.json >> "$RVF"
|
||||
|
||||
echo "--- RVF tail (last 4 lines) ---"
|
||||
tail -4 "$RVF" | jq -c '{type, fetched_at, window_count, window_uniques}' || true
|
||||
echo "--- file size ---"
|
||||
wc -l "$RVF"
|
||||
|
||||
- name: Compute aggregates for the commit summary
|
||||
id: agg
|
||||
run: |
|
||||
# Count distinct per-day entries across all snapshots so we can
|
||||
# show "cumulative observed clones" in the commit message.
|
||||
python3 - <<'PY'
|
||||
import json, os
|
||||
path = "data/clone-data.rvf"
|
||||
per_day_clones = {}
|
||||
per_day_views = {}
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
if not line.strip():
|
||||
continue
|
||||
d = json.loads(line)
|
||||
if d.get("type") == "clone_snapshot":
|
||||
for entry in d.get("per_day", []):
|
||||
per_day_clones[entry["timestamp"]] = entry
|
||||
elif d.get("type") == "view_snapshot":
|
||||
for entry in d.get("per_day", []):
|
||||
per_day_views[entry["timestamp"]] = entry
|
||||
|
||||
tot_clones = sum(e.get("count", 0) for e in per_day_clones.values())
|
||||
tot_uniq_clones = sum(e.get("uniques", 0) for e in per_day_clones.values())
|
||||
tot_views = sum(e.get("count", 0) for e in per_day_views.values())
|
||||
tot_uniq_views = sum(e.get("uniques", 0) for e in per_day_views.values())
|
||||
print(f"clone days observed: {len(per_day_clones)} total clones: {tot_clones:,} total unique cloners: {tot_uniq_clones:,}")
|
||||
print(f"view days observed: {len(per_day_views)} total views: {tot_views:,} total unique viewers: {tot_uniq_views:,}")
|
||||
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as out:
|
||||
out.write(f"clones={tot_clones}\n")
|
||||
out.write(f"clone_days={len(per_day_clones)}\n")
|
||||
out.write(f"views={tot_views}\n")
|
||||
out.write(f"view_days={len(per_day_views)}\n")
|
||||
PY
|
||||
|
||||
- name: Commit + push if changed
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
if git diff --quiet data/clone-data.rvf; then
|
||||
echo "no changes to commit"
|
||||
exit 0
|
||||
fi
|
||||
git add data/clone-data.rvf
|
||||
git commit -m "chore(traffic): clone snapshot — ${{ steps.agg.outputs.clone_days }} days observed → ${{ steps.agg.outputs.clones }} clones, ${{ steps.agg.outputs.view_days }} view-days → ${{ steps.agg.outputs.views }} views"
|
||||
git push
|
||||
@@ -0,0 +1,70 @@
|
||||
name: three.js demos → GitHub Pages
|
||||
|
||||
# Publishes the ADR-097 three.js demos under gh-pages/three.js/.
|
||||
# Uses keep_files: true so the existing observatory/, pose-fusion/,
|
||||
# pointcloud/, nvsim/, and root index.html demos are preserved.
|
||||
#
|
||||
# Demos 04 and 05 require a Mixamo "X Bot.fbx" placed in assets/.
|
||||
# That file is intentionally gitignored (license boundary), so this
|
||||
# workflow does NOT ship it. Demos 01-03 work standalone; the index
|
||||
# page documents the FBX requirement honestly.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'examples/three.js/**'
|
||||
- '.github/workflows/threejs-pages.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
group: threejs-pages
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-and-deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout main
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Stage demos for Pages
|
||||
run: |
|
||||
mkdir -p _site/three.js
|
||||
# Copy everything except the local Python server (CI doesn't need it)
|
||||
# and any stray scratch screenshots.
|
||||
cp -r examples/three.js/demos _site/three.js/demos
|
||||
cp -r examples/three.js/screenshots _site/three.js/screenshots
|
||||
cp examples/three.js/README.md _site/three.js/README.md
|
||||
# An index.html that lists the 5 demos with the FBX caveat.
|
||||
cp examples/three.js/index.html _site/three.js/index.html
|
||||
# Mixamo FBX is gitignored — assets dir won't exist in CI.
|
||||
# Drop an empty placeholder so the relative path 'assets/' resolves
|
||||
# to a directory listing (404 on missing file) instead of an opaque
|
||||
# network error. Browsers showing the 404 path makes the failure
|
||||
# visible to anyone trying demos 04/05 without their own FBX.
|
||||
mkdir -p _site/three.js/assets
|
||||
cat > _site/three.js/assets/README.txt <<'EOF'
|
||||
The Mixamo "X Bot.fbx" required by demos 04-skinned-fbx.html and
|
||||
05-skinned-realtime.html is intentionally not redistributed here.
|
||||
|
||||
Download your own from https://mixamo.com (FBX Binary, T-Pose,
|
||||
Without Skin) and place it here as "X Bot.fbx" if you want to
|
||||
run those demos locally. See examples/three.js/README.md in the
|
||||
repo for context.
|
||||
EOF
|
||||
echo "Staged contents:"
|
||||
ls -R _site/three.js/ | head -30
|
||||
|
||||
- name: Deploy to GitHub Pages
|
||||
uses: peaceiris/actions-gh-pages@v3
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
publish_dir: _site
|
||||
# Critical: preserve observatory/, pose-fusion/, pointcloud/, nvsim/
|
||||
# and the root index.html already on gh-pages.
|
||||
keep_files: true
|
||||
commit_message: 'three.js demos: ${{ github.event.head_commit.message }}'
|
||||
@@ -1,11 +1,17 @@
|
||||
# π RuView
|
||||
|
||||
<p align="center">
|
||||
<a href="https://x.com/rUv/status/2037556932802761004">
|
||||
<a href="https://cognitum.one/seed">
|
||||
<img src="assets/ruview-small-gemini.jpg" alt="RuView - WiFi DensePose" width="100%">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://cognitum.one/seed">
|
||||
<img src="assets/seed.png" alt="Cognitum Seed" width="100%">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
> **Beta Software** — Under active development. APIs and firmware may change. Known limitations:
|
||||
> - ESP32-C3 and original ESP32 are not supported (single-core, insufficient for CSI DSP)
|
||||
> - Single ESP32 deployments have limited spatial resolution — use 2+ nodes or add a [Cognitum Seed](https://cognitum.one) for best results
|
||||
@@ -32,7 +38,7 @@ Built on [RuVector](https://github.com/ruvnet/ruvector/) and [Cognitum Seed](htt
|
||||
|
||||
The system learns each environment locally using spiking neural networks that adapt in under 30 seconds, with multi-frequency mesh scanning across 6 WiFi channels that uses your neighbors' routers as free radar illuminators. Every measurement is cryptographically attested via an Ed25519 witness chain.
|
||||
|
||||
RuView **ships the full training pipeline for camera-free 17-keypoint pose estimation (WiFlow + AETHER + MERIDIAN heads)** — based on the original *DensePose From WiFi* research at Carnegie Mellon University. **What ships today is the inference and training infrastructure; pretrained pose weights are not yet released** (tracked in [#509](https://github.com/ruvnet/RuView/issues/509)). With no `.rvf` model loaded, the sensing server drives the on-screen skeleton from signal-based heuristics (amplitude variance, motion-band power), not learned keypoint inference. Camera-supervised fine-tune targets **35%+ PCK@20** ([ADR-079](docs/adr/ADR-079-camera-supervised-pose-finetune.md)) — pipeline implemented, P7–P9 (data collection + training + eval) are `Pending`.
|
||||
RuView turns ordinary WiFi into a contactless sensor. A $9 ESP32 board reads the radio reflections off the people in a room, and a small pretrained model — published on Hugging Face at [`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained) — tells you who's there, how they're breathing, and how their heart rate is trending. The model fits in 8 KB (4-bit quantized), runs in microseconds on a Raspberry Pi, and reports 100% presence accuracy on the validation set. No cameras, no wearables, no app on the user's phone.
|
||||
|
||||
### Built for low-power edge applications
|
||||
|
||||
@@ -45,25 +51,29 @@ RuView **ships the full training pipeline for camera-free 17-keypoint pose estim
|
||||
[](#vital-sign-detection)
|
||||
[](#esp32-s3-hardware-pipeline)
|
||||
[](https://crates.io/crates/wifi-densepose-ruvector)
|
||||
[](#-edge-module-catalog)
|
||||
|
||||
|
||||
> | What | Status | How | Speed |
|
||||
> |------|--------|-----|-------|
|
||||
> | 🫁 **Breathing rate** | ✅ Works today | Bandpass 0.1-0.5 Hz → zero-crossing BPM, circular variance on wrapped phase ([#593](https://github.com/ruvnet/RuView/issues/593)) | 6-30 BPM |
|
||||
> | 💓 **Heart rate** | ✅ Works today | Bandpass 0.8-2.0 Hz → zero-crossing BPM | 40-120 BPM (needs good SNR) |
|
||||
> | 👤 **Presence indicator** | ⚠️ Heuristic, not learned | Phase variance vs adaptive threshold (60 s ambient calibration). False-positives under strong RF interference. | < 1 ms latency |
|
||||
> | 🚶 **Motion / activity** | ✅ Works today | Motion-band power + phase acceleration | Real-time |
|
||||
> | 🤸 **Fall detection** | ✅ Works today | Phase acceleration > threshold + 3-frame debounce + 5 s cooldown ([#263](https://github.com/ruvnet/RuView/issues/263)) | < 200 ms |
|
||||
> | 🧮 **Multi-person slot count** | ⚠️ Heuristic, not learned | Subcarrier diversity divided by 2 (capped). **Not** a learned counter — see [firmware README](firmware/esp32-csi-node/README.md#tier-2--full-pipeline-stable) "Tier 2 caveats". Adaptive normalisation fix in [#491](https://github.com/ruvnet/RuView/pull/491). | Real-time |
|
||||
> | 🦴 **17-keypoint pose estimation** | 🔬 Pipeline only, no shipped weights | Training infrastructure complete (WiFlow + AETHER + MERIDIAN heads); pretrained `.rvf` not yet released. Fallback heuristic in the meantime. Tracked in [#509](https://github.com/ruvnet/RuView/issues/509). | Pending data collection |
|
||||
> | 🧱 **Through-wall sensing** | ✅ Works today | Fresnel zone geometry + multipath modeling | Up to ~5m signal-dependent |
|
||||
> | 🧠 **Edge intelligence** | ✅ Works today | Optional Cognitum Seed for persistent vector store + kNN + witness chain | $140 total BOM |
|
||||
> | 🎯 **Camera-free pre-training** | ✅ Pipeline works | MM-Fi + Wi-Pose datasets through `wifi-densepose-train`. Released weights pending [#509](https://github.com/ruvnet/RuView/issues/509). | 84 s/epoch on M4 Pro |
|
||||
> | 📷 **Camera-supervised fine-tune** | 🔬 Pipeline only | MediaPipe + ESP32 CSI paired training, [ADR-079](docs/adr/ADR-079-camera-supervised-pose-finetune.md). Target **35%+ PCK@20**. P7–P9 (data + train + eval) `Pending`. | ~19 min/epoch on laptop |
|
||||
> | 📡 **Multi-frequency mesh** | ✅ Works today | Channel hopping across 6 bands, TDM slot scheduling (ADR-029) | 3x sensing bandwidth |
|
||||
> | 🌐 **3D point cloud fusion** | 🔬 Reference impl | Camera depth (MiDaS) + WiFi CSI + mmWave radar → unified spatial model. Requires camera. | 22 ms pipeline · 19K+ points/frame |
|
||||
> | What | How | Speed / scale |
|
||||
> |------|-----|---------------|
|
||||
> | 🫁 **Breathing rate** | Bandpass 0.1–0.5 Hz on wrapped phase, circular variance, zero-crossing BPM ([#593](https://github.com/ruvnet/RuView/issues/593)) | 6–30 BPM, real-time |
|
||||
> | 💓 **Heart rate** | Bandpass 0.8–2.0 Hz, zero-crossing BPM | 40–120 BPM, real-time |
|
||||
> | 👤 **Presence detection** | Trained head on Hugging Face ([`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained), 100% validation accuracy) + a phase-variance fallback that needs no model | < 1 ms, ~30 s ambient calibration |
|
||||
> | 🧬 **CSI embeddings** | 128-dim contrastive encoder shipped on Hugging Face, 4-bit quantised variant fits in 8 KB | **164,183 emb/s** on M4 Pro |
|
||||
> | 🦴 **17-keypoint pose estimation** | `cog-pose-estimation` Cog v0.0.1 — signed aarch64 + x86_64 binaries on GCS, loads `pose_v1.safetensors` via Candle. Train your own from paired data in 2.1 s on an RTX 5080 ([ADR-101](docs/adr/ADR-101-pose-estimation-cog.md), [benchmarks](docs/benchmarks/pose-estimation-cog.md)) | 8.4 ms cold-start on a Pi 5 |
|
||||
> | 🚶 **Motion / activity** | Motion-band power + phase acceleration | Real-time |
|
||||
> | 🤸 **Fall detection** | Phase-acceleration threshold + 3-frame debounce + 5 s cooldown ([#263](https://github.com/ruvnet/RuView/issues/263)) | < 200 ms |
|
||||
> | 🧮 **Multi-person count** | Adaptive P95 normalisation + runtime-tunable dedup factor (`/api/v1/config/dedup-factor`, [#491](https://github.com/ruvnet/RuView/pull/491)). Six specialised learned counters available as Cogs: `occupancy-zones`, `elevator-count`, `queue-length`, `customer-flow`, `clean-room`, `person-matching` | Real-time, self-calibrating |
|
||||
> | 🧱 **Through-wall sensing** | Fresnel-zone geometry + multipath modeling | Up to ~5 m, signal-dependent |
|
||||
> | 🧠 **Edge intelligence** | **105-cog catalog** ([ADR-102](docs/adr/ADR-102-edge-module-registry.md)) live from `app-registry.json` — health, security, building, retail, industrial, research, AI, swarm, signal, network, and developer modules. Optional Cognitum Seed adds persistent vector store + kNN + witness chain | $140 total BOM |
|
||||
> | 🎯 **Camera-free pre-training** | Self-supervised contrastive encoder, 12.2M training steps on 60K frames, shipped on Hugging Face | 84 s/epoch retrain on M4 Pro |
|
||||
> | 📷 **Camera-supervised fine-tune** | MediaPipe + ESP32 CSI paired training, end-to-end Candle pipeline on RTX 5080 ([ADR-079](docs/adr/ADR-079-camera-supervised-pose-finetune.md)) | 2.1 s for 400 epochs (~5 ms/epoch) |
|
||||
> | 📡 **Multi-frequency mesh** | Channel hopping across 6 bands, TDM slot scheduling ([ADR-029](docs/adr/ADR-029-multifrequency-mesh.md)) | 3× sensing bandwidth |
|
||||
> | 🌐 **3D point cloud fusion** | Camera depth (MiDaS) + WiFi CSI + mmWave radar → unified spatial model | 22 ms pipeline · 19K+ points/frame |
|
||||
>
|
||||
> Legend: ✅ shipped + tested on hardware · ⚠️ ships and runs, but is a heuristic/threshold (not a learned classifier) — accuracy depends on calibration · 🔬 implementation + tests in repo, weights/data/eval pending
|
||||
> Browse the full 105-module catalog (with practical descriptions, sizes, and difficulty) below in [🧩 Edge Module Catalog](#-edge-module-catalog), or visit [seed.cognitum.one/store](https://seed.cognitum.one/store).
|
||||
>
|
||||
> 🤗 **Pretrained weights**: download from [`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained) — see [Loading the pretrained model](#loading-the-pretrained-model) below for one-command setup.
|
||||
|
||||
```bash
|
||||
# Option 1: Docker (simulated data, no hardware needed)
|
||||
@@ -93,7 +103,7 @@ node scripts/mincut-person-counter.js --port 5006 # Correct person counting
|
||||
>
|
||||
> | Option | Hardware | Cost | Full CSI | Capabilities |
|
||||
> |--------|----------|------|----------|-------------|
|
||||
> | **ESP32 + Cognitum Seed** (recommended) | ESP32-S3 + [Cognitum Seed](https://cognitum.one) | ~$140 | Yes | Presence indicator, motion, breathing rate, heart rate, fall detection, slot-count multi-person heuristic + persistent vector store, kNN search, witness chain, MCP proxy. (Pose pending weights — see [#509](https://github.com/ruvnet/RuView/issues/509).) |
|
||||
> | **ESP32 + Cognitum Seed** (recommended) | ESP32-S3 + [Cognitum Seed](https://cognitum.one) | ~$140 | Yes | Presence, motion, breathing, heart rate, fall detection, multi-person counting, 17-keypoint pose (signed Cog binary), 105-cog catalog, persistent vector store, kNN search, witness chain, MCP proxy |
|
||||
> | **ESP32 Mesh** | 3-6x ESP32-S3 + WiFi router | ~$54 | Yes | Same capabilities as above without the persistent-memory features |
|
||||
> | **Research NIC** | Intel 5300 / Atheros AR9580 | ~$50-100 | Yes | Full CSI with 3x3 MIMO |
|
||||
> | **Any WiFi** | Windows, macOS, or Linux laptop | $0 | No | RSSI-only: coarse presence and motion (see [tutorial #36](https://github.com/ruvnet/RuView/issues/36)) |
|
||||
@@ -114,10 +124,211 @@ node scripts/mincut-person-counter.js --port 5006 # Correct person counting
|
||||
<a href="https://ruvnet.github.io/RuView/pose-fusion.html"><strong>▶ Dual-Modal Pose Fusion Demo</strong></a>
|
||||
|
|
||||
<a href="https://ruvnet.github.io/RuView/pointcloud/"><strong>▶ Live 3D Point Cloud</strong></a>
|
||||
|
|
||||
<a href="https://ruvnet.github.io/RuView/three.js/"><strong>▶ three.js Demos (5)</strong></a>
|
||||
|
||||
> The [server](#-quick-start) is optional for visualization and aggregation — the ESP32 [runs independently](#esp32-s3-hardware-pipeline) for presence detection, vital signs, and fall alerts.
|
||||
>
|
||||
> **Live ESP32 pipeline**: Connect an ESP32-S3 node → run the [sensing server](#sensing-server) → open the [pose fusion demo](https://ruvnet.github.io/RuView/pose-fusion.html) for real-time dual-modal pose estimation (webcam + WiFi CSI). See [ADR-059](docs/adr/ADR-059-live-esp32-csi-pipeline.md).
|
||||
>
|
||||
> **three.js scene gallery** at [`/three.js/`](https://ruvnet.github.io/RuView/three.js/) — five progressively richer ADR-097 demos: helpers, cinematic, GLTF skinned, FBX skinned, and a live MediaPipe→Mixamo retargeting feed driven by ESP32 CSI. Demos 04 and 05 require a local Mixamo `X Bot.fbx` (license boundary — not redistributed).
|
||||
|
||||
|
||||
## 🤗 Pretrained model on Hugging Face
|
||||
|
||||
Pretrained CSI weights live at [`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained) — 12.2M training steps on 60K frames / 610K contrastive triplets, **100% presence accuracy** on the validation set, 4-bit quantized variant fits in 8 KB. The release includes a contrastive **CSI encoder** producing 128-dim embeddings (164,183 emb/s on M4 Pro) and a **presence-detection head**. Per-node LoRA adapters are included for environment-specific fine-tuning.
|
||||
|
||||
```bash
|
||||
# Download the model bundle
|
||||
pip install huggingface_hub
|
||||
huggingface-cli download ruvnet/wifi-densepose-pretrained --local-dir models/wifi-densepose-pretrained
|
||||
```
|
||||
|
||||
**What works today vs. what's pending wiring:**
|
||||
|
||||
| Consumer | Format used | Status |
|
||||
|----------|-------------|--------|
|
||||
| Python training / evaluation / embedding extraction | `model.safetensors` | ✅ Works — load with `safetensors.torch.load_file` |
|
||||
| Inspect / re-export the bundle | `model.rvf.jsonl` (line-by-line JSON) | ✅ Works — plain JSONL |
|
||||
| Sensing-server `--model <PATH>` flag | binary RVF (`RVFS` magic) | ⚠️ Loader does not yet accept the JSONL container |
|
||||
|
||||
**Known gap:** the HF model ships in JSONL RVF format, but `v2/crates/wifi-densepose-sensing-server/src/rvf_container.rs` only parses the binary RVF segment format. Pointing `--model` at `model.rvf.jsonl` currently errors with `invalid magic at offset 0: expected 0x52564653, got 0x7974227B` and the live pipeline degrades to null output rather than falling back to heuristic mode — so for the live sensing-server, run **without** `--model` until a JSONL adapter lands (or the model is re-published as binary RVF). Use the weights from Python / training in the meantime.
|
||||
|
||||
**Quantization choices** (all in the HF repo): `model-q2.bin` (4 KB) · `model-q4.bin` ⭐ recommended (8 KB) · `model-q8.bin` (16 KB) · `model.safetensors` full (48 KB)
|
||||
|
||||
The separate **17-keypoint pose-estimation model** is not in this release — pipeline is implemented but keypoint weights are still pending. Tracked in [#509](https://github.com/ruvnet/RuView/issues/509); see [ADR-079](docs/adr/ADR-079-camera-supervised-pose-finetune.md) phases P7–P9.
|
||||
|
||||
|
||||
## 🧩 Edge Module Catalog
|
||||
|
||||
<details>
|
||||
<summary><b>🧩 105 edge modules ready to install on a Cognitum appliance</b> — live catalog from <code>app-registry.json</code> v2.1.0 (updated 2026-05-13). Browse + install at <a href="https://seed.cognitum.one/store">seed.cognitum.one/store</a> or your local appliance <code>http://<appliance>:9000/cogs</code>.</summary>
|
||||
|
||||
Each module is a small signed binary (~400 KB) that runs alongside the WiFi-DensePose sensing stack on a Cognitum-V0 appliance. The catalog updates over the air — your appliance fetches it via <code>GET /api/v1/edge/registry</code> ([ADR-102](docs/adr/ADR-102-edge-module-registry.md)) and verifies each binary against an Ed25519 signature ([ADR-100](docs/adr/ADR-100-cog-packaging-specification.md)) before install.
|
||||
|
||||
### 🫀 Health — <sub>14 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `air-quality-index` | Track indoor air quality with CO2 and particle sensors | 8 KB | Easy |
|
||||
| `baby-cry` | Sustained mid-band energy detector for nursery / infant monitoring. Audio-only, no camera. | 451 KB | Easy |
|
||||
| `breathing-sync` | Detects when two people breathe in sync | 10 KB | Hard |
|
||||
| `cardiac-arrhythmia` | Spots irregular heartbeats and abnormal heart rhythms | 8 KB | Hard |
|
||||
| `cough-detect` | Acoustic transient + spectral cough detector with 30s cluster aggregation. Early-warning signal for respiratory illness. | 451 KB | Easy |
|
||||
| `dream-stage` | Tracks your sleep stages — light, deep, and dreaming | 14 KB | Hard |
|
||||
| `fall-detect` | Two-stage impact + stillness fall detector over ambient feature stream (ESP32 motion / mic). Optional ruview-mode for CSI-based pose reinforcement. | 402 KB | Easy |
|
||||
| `gait-analysis` | Detects walking problems and scores fall risk | 12 KB | Hard |
|
||||
| `health-monitor` | Contactless heart rate, breathing, sleep, and fall alerts | 30 KB | Med |
|
||||
| `respiratory-distress` | Alerts when breathing becomes labored or dangerously fast | 10 KB | Hard |
|
||||
| `seizure-detect` | Recognizes seizures and sends immediate alerts | 10 KB | Hard |
|
||||
| `sleep-apnea` | Detects when someone stops breathing during sleep | 4 KB | Easy |
|
||||
| `snore-monitor` | Periodic low-band energy tracker for sleep-quality / apnea-risk trending. Companion to sleep-apnea cog. | 451 KB | Easy |
|
||||
| `vital-trend` | Tracks breathing and heart rate trends over weeks | 6 KB | Med |
|
||||
|
||||
### 🔒 Security — <sub>14 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `audit-logger` | Record every action for compliance — tamper-proof log | 8 KB | Easy |
|
||||
| `behavioral-profiler` | Learns normal behavior and flags anything unusual | 12 KB | Hard |
|
||||
| `fleet-auth` | Manage device certificates and access across all seeds | 12 KB | Med |
|
||||
| `glass-break` | Two-phase bang + shatter acoustic detector. Distinguishes glass break from ordinary impulse noise. | 451 KB | Easy |
|
||||
| `gunshot-detect` | Saturating peak + exponential decay acoustic detector with optional ruview CSI motion-drop reinforcement. | 451 KB | Easy |
|
||||
| `intrusion` | Alerts when an unauthorized person enters a room | 6 KB | Med |
|
||||
| `intrusion-detect-ml` | Detect network attacks using machine learning | 14 KB | Hard |
|
||||
| `loitering` | Alerts when someone lingers too long in one spot | 3 KB | Easy |
|
||||
| `network-firewall` | Block unauthorized network access per cog | 6 KB | Easy |
|
||||
| `panic-motion` | Detects sudden panicked or erratic movement | 6 KB | Med |
|
||||
| `perimeter-breach` | Guards multiple zones and shows entry direction | 10 KB | Med |
|
||||
| `prompt-shield` | Blocks signal replay and injection attacks on the seed | 10 KB | Med |
|
||||
| `tailgating` | Catches when someone sneaks in behind a badge holder | 6 KB | Med |
|
||||
| `weapon-detect` | Detects concealed metal objects on a person | 8 KB | Hard |
|
||||
|
||||
### 🏢 Building — <sub>11 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `beehive-monitor` | Acoustic hive state classifier. Detects healthy / chaotic / queenless / swarming / robbing via hum-band energy + chaos + piping autocorr. | 451 KB | Easy |
|
||||
| `elevator-count` | Counts how many people are in an elevator | 8 KB | Med |
|
||||
| `energy-audit` | Learns your schedule and cuts wasted energy | 6 KB | Med |
|
||||
| `frost-warning` | Predicts frost 6 hours ahead via temperature trend + dewpoint-depression gate. Field/orchard agriculture. | 451 KB | Easy |
|
||||
| `hvac-presence` | Turns heating and cooling on when you arrive | 3 KB | Easy |
|
||||
| `lighting-zones` | Turns lights on and off as people move between rooms | 4 KB | Easy |
|
||||
| `meeting-room` | Shows if a meeting room is free or occupied | 5 KB | Easy |
|
||||
| `occupancy-zones` | Counts people in each room through walls | 8 KB | Med |
|
||||
| `predictive-maintenance` | Vibration harmonic analyzer for rotating equipment. Tracks F1 / 2×F1 / high-order / sideband energy to score degradation severity. | 451 KB | Easy |
|
||||
| `smoke-fire` | Multi-signal smoke and fire detector. Fuses acoustic crackle, thermal drift proxy, and optional ruview CSI plume signature. Not a UL-listed replacement for code-required smoke alarms. | 451 KB | Easy |
|
||||
| `water-leak` | Persistent low-amplitude hiss + periodic drip acoustic detector with multi-minute persistence gate. Two-stage likely → confirmed. | 451 KB | Easy |
|
||||
|
||||
### 🛍️ Retail — <sub>7 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `customer-flow` | Counts foot traffic in and out of each entrance | 8 KB | Med |
|
||||
| `dwell-heatmap` | Shows where customers spend the most time | 6 KB | Med |
|
||||
| `package-detect` | Sustained CSI-shift detector for porch / loading bay package arrivals and departures. Requires ESP32 CSI ruview input. | 451 KB | Easy |
|
||||
| `parking-occupancy` | Per-zone parking occupancy via ESP32 CSI subcarrier-amplitude shift. Tracks utilization and churn-per-hour. Requires ruview. | 451 KB | Easy |
|
||||
| `queue-length` | Estimates line length and wait time | 6 KB | Med |
|
||||
| `shelf-engagement` | Detects when customers interact with products | 6 KB | Med |
|
||||
| `table-turnover` | Tracks which restaurant tables are free or occupied | 4 KB | Easy |
|
||||
|
||||
### 🏭 Industrial — <sub>7 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `clean-room` | Enforces max headcount in controlled environments | 4 KB | Easy |
|
||||
| `confined-space` | Monitors workers in tight spaces for safety | 5 KB | Med |
|
||||
| `forklift-proximity` | Warns if a forklift gets too close to workers | 10 KB | Hard |
|
||||
| `livestock-monitor` | Monitors animals for distress, escape, or illness | 6 KB | Med |
|
||||
| `ppe-compliance` | Cog-composition layer: alerts when ruview-densepose detects presence in a restricted zone without an accompanying PPE-camera-cog confirmation vector. | 387 KB | Easy |
|
||||
| `slip-fall-zone` | Pre-fall risk detector. Fires when motion-variance drop, splash audio, and optional cautious-gait CSI all signal elevated slip risk. | 451 KB | Easy |
|
||||
| `structural-vibration` | Detects dangerous vibrations in buildings or machines | 8 KB | Hard |
|
||||
|
||||
### 🔬 Research — <sub>12 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `emotion-detect` | Reads stress and calm from body language and breathing | 10 KB | Hard |
|
||||
| `energy-harvester` | Optimize solar and battery for off-grid seed deployment | 6 KB | Med |
|
||||
| `gesture-language` | Recognizes sign language gestures in real time | 12 KB | Hard |
|
||||
| `ghost-hunter` | Finds unexplained environmental anomalies — for fun | 10 KB | Hard |
|
||||
| `happiness-score` | Estimates well-being from movement and mood signals | 8 KB | Med |
|
||||
| `hyperbolic-space` | Maps data into curved space for tree-like structures | 12 KB | Hard |
|
||||
| `music-conductor` | Reads a conductor's gestures for tempo and dynamics | 12 KB | Hard |
|
||||
| `plant-growth` | Tracks plant growth rate and day/night cycles | 8 KB | Med |
|
||||
| `rain-detect` | Detects when rain starts, stops, and how heavy it is | 6 KB | Med |
|
||||
| `ruview-densepose` | Full body pose tracking from WiFi — no cameras needed | 50 KB | Hard |
|
||||
| `sound-classifier` | Identify sounds like glass break, alarm, or baby cry | 16 KB | Hard |
|
||||
| `time-crystal` | Experiments with repeating time-pattern symmetry | 12 KB | Hard |
|
||||
|
||||
### 🤖 Ai — <sub>15 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `anomaly-attractor` | Learns what's normal and catches anything weird | 10 KB | Hard |
|
||||
| `cognitive-pipeline` | FastGRNN anomaly gate + SmolLM2 sparse-LLM inference for on-device Pi Zero 2W cognitive events | 320 KB | Hard |
|
||||
| `dtw-gesture-learn` | Teach custom hand gestures by showing examples | 14 KB | Med |
|
||||
| `ewc-lifelong` | Learns new things without forgetting old lessons | 8 KB | Hard |
|
||||
| `federated-learning` | Train AI across seeds without sharing raw data | 18 KB | Hard |
|
||||
| `goap-autonomy` | Plans and executes goals on its own | 14 KB | Hard |
|
||||
| `meta-adapt` | Automatically tunes itself for best performance | 10 KB | Hard |
|
||||
| `micro-hnsw` | Fast on-device fingerprinting and classification | 12 KB | Med |
|
||||
| `neural-trader` | Spot market patterns and trends from live data | 20 KB | Hard |
|
||||
| `pagerank-influence` | Finds the most influential person in a group | 12 KB | Med |
|
||||
| `pattern-sequence` | Detects daily routines and repeated habits | 10 KB | Med |
|
||||
| `rag-local` | Search your documents using AI — runs on the seed | 14 KB | Med |
|
||||
| `spiking-tracker` | Brain-inspired tracker that runs on tiny hardware | 16 KB | Hard |
|
||||
| `temporal-logic` | Enforces safety rules on live event streams | 12 KB | Hard |
|
||||
| `time-series-forecast` | Predict sensor trends using historical patterns | 12 KB | Med |
|
||||
|
||||
### 🐝 Swarm — <sub>11 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `swarm-backup-restore` | Auto-backup data to other seeds — one-click restore | 8 KB | Easy |
|
||||
| `swarm-cluster-monitor` | Live dashboard of every seed's health and status | 6 KB | Easy |
|
||||
| `swarm-consensus` | Seeds vote before making critical changes together | 16 KB | Hard |
|
||||
| `swarm-delta-sync` | Auto-sync data between seeds — only sends changes | 8 KB | Med |
|
||||
| `swarm-deploy` | Install or remove cogs on all seeds at once | 10 KB | Med |
|
||||
| `swarm-distributed-store` | Spread data across seeds and search them all at once | 14 KB | Hard |
|
||||
| `swarm-edge-orchestrator` | Manage all ESP32 sensor nodes from one place | 14 KB | Hard |
|
||||
| `swarm-load-balancer` | Spread queries across seeds so no single one overloads | 10 KB | Med |
|
||||
| `swarm-mesh-manager` | Find, connect, and monitor all seeds on your network | 12 KB | Easy |
|
||||
| `swarm-mqtt-bridge` | Share events between seeds over MQTT messaging | 6 KB | Easy |
|
||||
| `swarm-witness-federation` | Share tamper-proof audit trails across seeds | 12 KB | Hard |
|
||||
|
||||
### 📡 Signal — <sub>6 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `coherence-gate` | Filters out noisy signals and keeps clean ones | 8 KB | Med |
|
||||
| `flash-attention` | Focuses sensing on specific areas for better accuracy | 12 KB | Med |
|
||||
| `optimal-transport` | Measures motion using shape-aware signal comparison | 12 KB | Hard |
|
||||
| `person-matching` | Tells apart multiple people in the same room | 18 KB | Hard |
|
||||
| `sparse-recovery` | Recovers missing signal data from partial readings | 16 KB | Hard |
|
||||
| `temporal-compress` | Shrinks old data to save memory without losing meaning | 14 KB | Med |
|
||||
|
||||
### 🌐 Network — <sub>1 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `tailscale` | Reach the seed from anywhere via a private WireGuard mesh (Tailscale). Userspace mode — no root. | 700 KB | Med |
|
||||
|
||||
### 🛠️ Developer — <sub>7 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `adversarial` | Detects tampered or spoofed sensor signals | 4 KB | Easy |
|
||||
| `coherence` | Monitors signal quality across multiple channels | 4 KB | Easy |
|
||||
| `gesture` | Core gesture recognition building block for cogs | 6 KB | Med |
|
||||
| `interference-search` | Searches many possibilities at once for fast answers | 14 KB | Hard |
|
||||
| `psycho-symbolic` | Reasons over knowledge graphs with multiple styles | 16 KB | Hard |
|
||||
| `quantum-coherence` | Quantum-inspired model for advanced signal states | 16 KB | Hard |
|
||||
| `self-healing-mesh` | Keeps sensor mesh running even when nodes drop out | 14 KB | Hard |
|
||||
|
||||
> ℹ️ Build your own cog: see [ADR-100](docs/adr/ADR-100-cog-packaging-specification.md) for the packaging spec. The first cog this repo ships into the catalog lives in [v2/crates/cog-pose-estimation/](v2/crates/cog-pose-estimation/) (17-keypoint WiFi pose, [ADR-101](docs/adr/ADR-101-pose-estimation-cog.md)).
|
||||
|
||||
</details>
|
||||
|
||||
|
||||
## 🔬 How It Works
|
||||
@@ -233,178 +444,6 @@ These scenarios exploit WiFi's ability to penetrate solid materials — concrete
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>🧩 Edge Intelligence (<a href="docs/adr/ADR-041-wasm-module-collection.md">ADR-041</a>)</strong> — 60 WASM modules across 13 categories, all implemented (609 tests)</summary>
|
||||
|
||||
Small programs that run directly on the ESP32 sensor — no internet needed, no cloud fees, instant response. Each module is a tiny WASM file (5-30 KB) that you upload to the device over-the-air. It reads WiFi signal data and makes decisions locally in under 10 ms. [ADR-041](docs/adr/ADR-041-wasm-module-collection.md) defines 60 modules across 13 categories — all 60 are implemented with 609 tests passing.
|
||||
|
||||
| | Category | Examples |
|
||||
|---|----------|---------|
|
||||
| 🏥 | [**Medical & Health**](docs/edge-modules/medical.md) | Sleep apnea detection, cardiac arrhythmia, gait analysis, seizure detection |
|
||||
| 🔐 | [**Security & Safety**](docs/edge-modules/security.md) | Intrusion detection, perimeter breach, loitering, panic motion |
|
||||
| 🏢 | [**Smart Building**](docs/edge-modules/building.md) | Zone occupancy, HVAC control, elevator counting, meeting room tracking |
|
||||
| 🛒 | [**Retail & Hospitality**](docs/edge-modules/retail.md) | Queue length, dwell heatmaps, customer flow, table turnover |
|
||||
| 🏭 | [**Industrial**](docs/edge-modules/industrial.md) | Forklift proximity, confined space monitoring, structural vibration |
|
||||
| 🔮 | [**Exotic & Research**](docs/edge-modules/exotic.md) | Sleep staging, emotion detection, sign language, breathing sync |
|
||||
| 📡 | [**Signal Intelligence**](docs/edge-modules/signal-intelligence.md) | Cleans and sharpens raw WiFi signals — focuses on important regions, filters noise, fills in missing data, and tracks which person is which |
|
||||
| 🧠 | [**Adaptive Learning**](docs/edge-modules/adaptive-learning.md) | The sensor learns new gestures and patterns on its own over time — no cloud needed, remembers what it learned even after updates |
|
||||
| 🗺️ | [**Spatial Reasoning**](docs/edge-modules/spatial-temporal.md) | Figures out where people are in a room, which zones matter most, and tracks movement across areas using graph-based spatial logic |
|
||||
| ⏱️ | [**Temporal Analysis**](docs/edge-modules/spatial-temporal.md) | Learns daily routines, detects when patterns break (someone didn't get up), and verifies safety rules are being followed over time |
|
||||
| 🛡️ | [**AI Security**](docs/edge-modules/ai-security.md) | Detects signal replay attacks, WiFi jamming, injection attempts, and flags abnormal behavior that could indicate tampering |
|
||||
| ⚛️ | [**Quantum-Inspired**](docs/edge-modules/autonomous.md) | Uses quantum-inspired math to map room-wide signal coherence and search for optimal sensor configurations |
|
||||
| 🤖 | [**Autonomous & Exotic**](docs/edge-modules/autonomous.md) | Self-managing sensor mesh — auto-heals dropped nodes, plans its own actions, and explores experimental signal representations |
|
||||
|
||||
All implemented modules are `no_std` Rust, share a [common utility library](v2/crates/wifi-densepose-wasm-edge/src/vendor_common.rs), and talk to the host through a 12-function API. Full documentation: [**Edge Modules Guide**](docs/edge-modules/README.md). See the [complete implemented module list](#edge-module-list) below.
|
||||
|
||||
</details>
|
||||
|
||||
<details id="edge-module-list">
|
||||
<summary><strong>🧩 Edge Intelligence — <a href="docs/edge-modules/README.md">All 65 Modules Implemented</a></strong> (ADR-041 complete)</summary>
|
||||
|
||||
All 60 modules are implemented, tested (609 tests passing), and ready to deploy. They compile to `wasm32-unknown-unknown`, run on ESP32-S3 via WASM3, and share a [common utility library](v2/crates/wifi-densepose-wasm-edge/src/vendor_common.rs). Source: [`crates/wifi-densepose-wasm-edge/src/`](v2/crates/wifi-densepose-wasm-edge/src/)
|
||||
|
||||
**Core modules** (ADR-040 flagship + early implementations):
|
||||
|
||||
| Module | File | What It Does |
|
||||
|--------|------|-------------|
|
||||
| Gesture Classifier | [`gesture.rs`](v2/crates/wifi-densepose-wasm-edge/src/gesture.rs) | DTW template matching for hand gestures |
|
||||
| Coherence Filter | [`coherence.rs`](v2/crates/wifi-densepose-wasm-edge/src/coherence.rs) | Phase coherence gating for signal quality |
|
||||
| Adversarial Detector | [`adversarial.rs`](v2/crates/wifi-densepose-wasm-edge/src/adversarial.rs) | Detects physically impossible signal patterns |
|
||||
| Intrusion Detector | [`intrusion.rs`](v2/crates/wifi-densepose-wasm-edge/src/intrusion.rs) | Human vs non-human motion classification |
|
||||
| Occupancy Counter | [`occupancy.rs`](v2/crates/wifi-densepose-wasm-edge/src/occupancy.rs) | Zone-level person counting |
|
||||
| Vital Trend | [`vital_trend.rs`](v2/crates/wifi-densepose-wasm-edge/src/vital_trend.rs) | Long-term breathing and heart rate trending |
|
||||
| RVF Parser | [`rvf.rs`](v2/crates/wifi-densepose-wasm-edge/src/rvf.rs) | RVF container format parsing |
|
||||
|
||||
**Vendor-integrated modules** (24 modules, ADR-041 Category 7):
|
||||
|
||||
**📡 Signal Intelligence** — Real-time CSI analysis and feature extraction
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Flash Attention | [`sig_flash_attention.rs`](v2/crates/wifi-densepose-wasm-edge/src/sig_flash_attention.rs) | Tiled attention over 8 subcarrier groups — finds spatial focus regions and entropy | S (<5ms) |
|
||||
| Coherence Gate | [`sig_coherence_gate.rs`](v2/crates/wifi-densepose-wasm-edge/src/sig_coherence_gate.rs) | Z-score phasor gating with hysteresis: Accept / PredictOnly / Reject / Recalibrate | L (<2ms) |
|
||||
| Temporal Compress | [`sig_temporal_compress.rs`](v2/crates/wifi-densepose-wasm-edge/src/sig_temporal_compress.rs) | 3-tier adaptive quantization (8-bit hot / 5-bit warm / 3-bit cold) | L (<2ms) |
|
||||
| Sparse Recovery | [`sig_sparse_recovery.rs`](v2/crates/wifi-densepose-wasm-edge/src/sig_sparse_recovery.rs) | ISTA L1 reconstruction for dropped subcarriers | H (<10ms) |
|
||||
| Person Match | [`sig_mincut_person_match.rs`](v2/crates/wifi-densepose-wasm-edge/src/sig_mincut_person_match.rs) | Hungarian-lite bipartite assignment for multi-person tracking | S (<5ms) |
|
||||
| Optimal Transport | [`sig_optimal_transport.rs`](v2/crates/wifi-densepose-wasm-edge/src/sig_optimal_transport.rs) | Sliced Wasserstein-1 distance with 4 projections | L (<2ms) |
|
||||
|
||||
**🧠 Adaptive Learning** — On-device learning without cloud connectivity
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| DTW Gesture Learn | [`lrn_dtw_gesture_learn.rs`](v2/crates/wifi-densepose-wasm-edge/src/lrn_dtw_gesture_learn.rs) | User-teachable gesture recognition — 3-rehearsal protocol, 16 templates | S (<5ms) |
|
||||
| Anomaly Attractor | [`lrn_anomaly_attractor.rs`](v2/crates/wifi-densepose-wasm-edge/src/lrn_anomaly_attractor.rs) | 4D dynamical system attractor classification with Lyapunov exponents | H (<10ms) |
|
||||
| Meta Adapt | [`lrn_meta_adapt.rs`](v2/crates/wifi-densepose-wasm-edge/src/lrn_meta_adapt.rs) | Hill-climbing self-optimization with safety rollback | L (<2ms) |
|
||||
| EWC Lifelong | [`lrn_ewc_lifelong.rs`](v2/crates/wifi-densepose-wasm-edge/src/lrn_ewc_lifelong.rs) | Elastic Weight Consolidation — remembers past tasks while learning new ones | S (<5ms) |
|
||||
|
||||
**🗺️ Spatial Reasoning** — Location, proximity, and influence mapping
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| PageRank Influence | [`spt_pagerank_influence.rs`](v2/crates/wifi-densepose-wasm-edge/src/spt_pagerank_influence.rs) | 4x4 cross-correlation graph with power iteration PageRank | L (<2ms) |
|
||||
| Micro HNSW | [`spt_micro_hnsw.rs`](v2/crates/wifi-densepose-wasm-edge/src/spt_micro_hnsw.rs) | 64-vector navigable small-world graph for nearest-neighbor search | S (<5ms) |
|
||||
| Spiking Tracker | [`spt_spiking_tracker.rs`](v2/crates/wifi-densepose-wasm-edge/src/spt_spiking_tracker.rs) | 32 LIF neurons + 4 output zone neurons with STDP learning | S (<5ms) |
|
||||
|
||||
**⏱️ Temporal Analysis** — Activity patterns, logic verification, autonomous planning
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Pattern Sequence | [`tmp_pattern_sequence.rs`](v2/crates/wifi-densepose-wasm-edge/src/tmp_pattern_sequence.rs) | Activity routine detection and deviation alerts | S (<5ms) |
|
||||
| Temporal Logic Guard | [`tmp_temporal_logic_guard.rs`](v2/crates/wifi-densepose-wasm-edge/src/tmp_temporal_logic_guard.rs) | LTL formula verification on CSI event streams | S (<5ms) |
|
||||
| GOAP Autonomy | [`tmp_goap_autonomy.rs`](v2/crates/wifi-densepose-wasm-edge/src/tmp_goap_autonomy.rs) | Goal-Oriented Action Planning for autonomous module management | S (<5ms) |
|
||||
|
||||
**🛡️ AI Security** — Tamper detection and behavioral anomaly profiling
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Prompt Shield | [`ais_prompt_shield.rs`](v2/crates/wifi-densepose-wasm-edge/src/ais_prompt_shield.rs) | FNV-1a replay detection, injection detection (10x amplitude), jamming (SNR) | L (<2ms) |
|
||||
| Behavioral Profiler | [`ais_behavioral_profiler.rs`](v2/crates/wifi-densepose-wasm-edge/src/ais_behavioral_profiler.rs) | 6D behavioral profile with Mahalanobis anomaly scoring | S (<5ms) |
|
||||
|
||||
**⚛️ Quantum-Inspired** — Quantum computing metaphors applied to CSI analysis
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Quantum Coherence | [`qnt_quantum_coherence.rs`](v2/crates/wifi-densepose-wasm-edge/src/qnt_quantum_coherence.rs) | Bloch sphere mapping, Von Neumann entropy, decoherence detection | S (<5ms) |
|
||||
| Interference Search | [`qnt_interference_search.rs`](v2/crates/wifi-densepose-wasm-edge/src/qnt_interference_search.rs) | 16 room-state hypotheses with Grover-inspired oracle + diffusion | S (<5ms) |
|
||||
|
||||
**🤖 Autonomous Systems** — Self-governing and self-healing behaviors
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Psycho-Symbolic | [`aut_psycho_symbolic.rs`](v2/crates/wifi-densepose-wasm-edge/src/aut_psycho_symbolic.rs) | 16-rule forward-chaining knowledge base with contradiction detection | S (<5ms) |
|
||||
| Self-Healing Mesh | [`aut_self_healing_mesh.rs`](v2/crates/wifi-densepose-wasm-edge/src/aut_self_healing_mesh.rs) | 8-node mesh with health tracking, degradation/recovery, coverage healing | S (<5ms) |
|
||||
|
||||
**🔮 Exotic (Vendor)** — Novel mathematical models for CSI interpretation
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Time Crystal | [`exo_time_crystal.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_time_crystal.rs) | Autocorrelation subharmonic detection in 256-frame history | S (<5ms) |
|
||||
| Hyperbolic Space | [`exo_hyperbolic_space.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_hyperbolic_space.rs) | Poincare ball embedding with 32 reference locations, hyperbolic distance | S (<5ms) |
|
||||
|
||||
**🏥 Medical & Health** (Category 1) — Contactless health monitoring
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Sleep Apnea | [`med_sleep_apnea.rs`](v2/crates/wifi-densepose-wasm-edge/src/med_sleep_apnea.rs) | Detects breathing pauses during sleep | S (<5ms) |
|
||||
| Cardiac Arrhythmia | [`med_cardiac_arrhythmia.rs`](v2/crates/wifi-densepose-wasm-edge/src/med_cardiac_arrhythmia.rs) | Monitors heart rate for irregular rhythms | S (<5ms) |
|
||||
| Respiratory Distress | [`med_respiratory_distress.rs`](v2/crates/wifi-densepose-wasm-edge/src/med_respiratory_distress.rs) | Alerts on abnormal breathing patterns | S (<5ms) |
|
||||
| Gait Analysis | [`med_gait_analysis.rs`](v2/crates/wifi-densepose-wasm-edge/src/med_gait_analysis.rs) | Tracks walking patterns and detects changes | S (<5ms) |
|
||||
| Seizure Detection | [`med_seizure_detect.rs`](v2/crates/wifi-densepose-wasm-edge/src/med_seizure_detect.rs) | 6-state machine for tonic-clonic seizure recognition | S (<5ms) |
|
||||
|
||||
**🔐 Security & Safety** (Category 2) — Perimeter and threat detection
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Perimeter Breach | [`sec_perimeter_breach.rs`](v2/crates/wifi-densepose-wasm-edge/src/sec_perimeter_breach.rs) | Detects boundary crossings with approach/departure | S (<5ms) |
|
||||
| Weapon Detection | [`sec_weapon_detect.rs`](v2/crates/wifi-densepose-wasm-edge/src/sec_weapon_detect.rs) | Metal anomaly detection via CSI amplitude shifts | S (<5ms) |
|
||||
| Tailgating | [`sec_tailgating.rs`](v2/crates/wifi-densepose-wasm-edge/src/sec_tailgating.rs) | Detects unauthorized follow-through at access points | S (<5ms) |
|
||||
| Loitering | [`sec_loitering.rs`](v2/crates/wifi-densepose-wasm-edge/src/sec_loitering.rs) | Alerts when someone lingers too long in a zone | S (<5ms) |
|
||||
| Panic Motion | [`sec_panic_motion.rs`](v2/crates/wifi-densepose-wasm-edge/src/sec_panic_motion.rs) | Detects fleeing, struggling, or panic movement | S (<5ms) |
|
||||
|
||||
**🏢 Smart Building** (Category 3) — Automation and energy efficiency
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| HVAC Presence | [`bld_hvac_presence.rs`](v2/crates/wifi-densepose-wasm-edge/src/bld_hvac_presence.rs) | Occupancy-driven HVAC control with departure countdown | S (<5ms) |
|
||||
| Lighting Zones | [`bld_lighting_zones.rs`](v2/crates/wifi-densepose-wasm-edge/src/bld_lighting_zones.rs) | Auto-dim/off lighting based on zone activity | S (<5ms) |
|
||||
| Elevator Count | [`bld_elevator_count.rs`](v2/crates/wifi-densepose-wasm-edge/src/bld_elevator_count.rs) | Counts people entering/leaving with overload warning | S (<5ms) |
|
||||
| Meeting Room | [`bld_meeting_room.rs`](v2/crates/wifi-densepose-wasm-edge/src/bld_meeting_room.rs) | Tracks meeting lifecycle: start, headcount, end, availability | S (<5ms) |
|
||||
| Energy Audit | [`bld_energy_audit.rs`](v2/crates/wifi-densepose-wasm-edge/src/bld_energy_audit.rs) | Tracks after-hours usage and room utilization rates | S (<5ms) |
|
||||
|
||||
**🛒 Retail & Hospitality** (Category 4) — Customer insights without cameras
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Queue Length | [`ret_queue_length.rs`](v2/crates/wifi-densepose-wasm-edge/src/ret_queue_length.rs) | Estimates queue size and wait times | S (<5ms) |
|
||||
| Dwell Heatmap | [`ret_dwell_heatmap.rs`](v2/crates/wifi-densepose-wasm-edge/src/ret_dwell_heatmap.rs) | Shows where people spend time (hot/cold zones) | S (<5ms) |
|
||||
| Customer Flow | [`ret_customer_flow.rs`](v2/crates/wifi-densepose-wasm-edge/src/ret_customer_flow.rs) | Counts ins/outs and tracks net occupancy | S (<5ms) |
|
||||
| Table Turnover | [`ret_table_turnover.rs`](v2/crates/wifi-densepose-wasm-edge/src/ret_table_turnover.rs) | Restaurant table lifecycle: seated, dining, vacated | S (<5ms) |
|
||||
| Shelf Engagement | [`ret_shelf_engagement.rs`](v2/crates/wifi-densepose-wasm-edge/src/ret_shelf_engagement.rs) | Detects browsing, considering, and reaching for products | S (<5ms) |
|
||||
|
||||
**🏭 Industrial & Specialized** (Category 5) — Safety and compliance
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Forklift Proximity | [`ind_forklift_proximity.rs`](v2/crates/wifi-densepose-wasm-edge/src/ind_forklift_proximity.rs) | Warns when people get too close to vehicles | S (<5ms) |
|
||||
| Confined Space | [`ind_confined_space.rs`](v2/crates/wifi-densepose-wasm-edge/src/ind_confined_space.rs) | OSHA-compliant worker monitoring with extraction alerts | S (<5ms) |
|
||||
| Clean Room | [`ind_clean_room.rs`](v2/crates/wifi-densepose-wasm-edge/src/ind_clean_room.rs) | Occupancy limits and turbulent motion detection | S (<5ms) |
|
||||
| Livestock Monitor | [`ind_livestock_monitor.rs`](v2/crates/wifi-densepose-wasm-edge/src/ind_livestock_monitor.rs) | Animal presence, stillness, and escape alerts | S (<5ms) |
|
||||
| Structural Vibration | [`ind_structural_vibration.rs`](v2/crates/wifi-densepose-wasm-edge/src/ind_structural_vibration.rs) | Seismic events, mechanical resonance, structural drift | S (<5ms) |
|
||||
|
||||
**🔮 Exotic & Research** (Category 6) — Experimental sensing applications
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Dream Stage | [`exo_dream_stage.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_dream_stage.rs) | Contactless sleep stage classification (wake/light/deep/REM) | S (<5ms) |
|
||||
| Emotion Detection | [`exo_emotion_detect.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_emotion_detect.rs) | Arousal, stress, and calm detection from micro-movements | S (<5ms) |
|
||||
| Gesture Language | [`exo_gesture_language.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_gesture_language.rs) | Sign language letter recognition via WiFi | S (<5ms) |
|
||||
| Music Conductor | [`exo_music_conductor.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_music_conductor.rs) | Tempo and dynamic tracking from conducting gestures | S (<5ms) |
|
||||
| Plant Growth | [`exo_plant_growth.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_plant_growth.rs) | Monitors plant growth, circadian rhythms, wilt detection | S (<5ms) |
|
||||
| Ghost Hunter | [`exo_ghost_hunter.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_ghost_hunter.rs) | Environmental anomaly classification (draft/insect/wind/unknown) | S (<5ms) |
|
||||
| Rain Detection | [`exo_rain_detect.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_rain_detect.rs) | Detects rain onset, intensity, and cessation via signal scatter | S (<5ms) |
|
||||
| Breathing Sync | [`exo_breathing_sync.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_breathing_sync.rs) | Detects synchronized breathing between multiple people | S (<5ms) |
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.2 MiB |
@@ -0,0 +1,3 @@
|
||||
{"type": "metadata", "name": "ruview-clone-traffic-history", "version": "1.0.0", "schema": "ruvector.rvf.jsonl/v1", "format": "github-traffic-snapshots", "repo": "ruvnet/RuView", "source": "GitHub Traffic API /repos/{repo}/traffic/{clones,views}", "policy": "GitHub retains only 14 days server-side; this file is the long-term record.", "segments": ["metadata", "clone_snapshot", "view_snapshot"], "created_at": "2026-05-19T23:16:22Z", "custom": {"cadence": "twice monthly (1st and 15th, ~14-day intervals)", "idempotency_key": "timestamp (per-day records de-duplicate across overlapping snapshot windows)"}}
|
||||
{"type": "clone_snapshot", "fetched_at": "2026-05-19T23:16:22Z", "window_count": 27887, "window_uniques": 6611, "per_day": [{"timestamp": "2026-05-05T00:00:00Z", "count": 620, "uniques": 218}, {"timestamp": "2026-05-06T00:00:00Z", "count": 477, "uniques": 232}, {"timestamp": "2026-05-07T00:00:00Z", "count": 685, "uniques": 268}, {"timestamp": "2026-05-08T00:00:00Z", "count": 703, "uniques": 276}, {"timestamp": "2026-05-09T00:00:00Z", "count": 352, "uniques": 184}, {"timestamp": "2026-05-10T00:00:00Z", "count": 205, "uniques": 151}, {"timestamp": "2026-05-11T00:00:00Z", "count": 1160, "uniques": 234}, {"timestamp": "2026-05-12T00:00:00Z", "count": 599, "uniques": 207}, {"timestamp": "2026-05-13T00:00:00Z", "count": 5141, "uniques": 1152}, {"timestamp": "2026-05-14T00:00:00Z", "count": 3420, "uniques": 972}, {"timestamp": "2026-05-15T00:00:00Z", "count": 1974, "uniques": 764}, {"timestamp": "2026-05-16T00:00:00Z", "count": 2917, "uniques": 617}, {"timestamp": "2026-05-17T00:00:00Z", "count": 6690, "uniques": 1169}, {"timestamp": "2026-05-18T00:00:00Z", "count": 2944, "uniques": 625}]}
|
||||
{"type": "view_snapshot", "fetched_at": "2026-05-19T23:16:22Z", "window_count": 162314, "window_uniques": 75464, "per_day": [{"timestamp": "2026-05-05T00:00:00Z", "count": 5540, "uniques": 2690}, {"timestamp": "2026-05-06T00:00:00Z", "count": 5111, "uniques": 2393}, {"timestamp": "2026-05-07T00:00:00Z", "count": 5585, "uniques": 2708}, {"timestamp": "2026-05-08T00:00:00Z", "count": 7004, "uniques": 3261}, {"timestamp": "2026-05-09T00:00:00Z", "count": 5395, "uniques": 2531}, {"timestamp": "2026-05-10T00:00:00Z", "count": 4761, "uniques": 2219}, {"timestamp": "2026-05-11T00:00:00Z", "count": 4275, "uniques": 2044}, {"timestamp": "2026-05-12T00:00:00Z", "count": 3466, "uniques": 1688}, {"timestamp": "2026-05-13T00:00:00Z", "count": 13561, "uniques": 8473}, {"timestamp": "2026-05-14T00:00:00Z", "count": 21867, "uniques": 12527}, {"timestamp": "2026-05-15T00:00:00Z", "count": 26182, "uniques": 14609}, {"timestamp": "2026-05-16T00:00:00Z", "count": 17406, "uniques": 8868}, {"timestamp": "2026-05-17T00:00:00Z", "count": 28444, "uniques": 14541}, {"timestamp": "2026-05-18T00:00:00Z", "count": 13717, "uniques": 7819}]}
|
||||
@@ -0,0 +1,165 @@
|
||||
# ADR-100: Cognitum Cog Packaging Specification
|
||||
|
||||
- **Status:** Accepted (formalises existing convention) — **first conforming cog shipped 2026-05-19** (`cog-pose-estimation@0.0.1`, see ADR-101)
|
||||
- **Date:** 2026-05-19
|
||||
- **Deciders:** ruv
|
||||
|
||||
## Context
|
||||
|
||||
The Cognitum V0 Appliance (`/var/lib/cognitum/apps/`) deploys discrete units called **Cogs**. They appear in the Appliance dashboard (`http://cognitum-v0:9000/cogs`) under an app-store UI (Today / Apps / Categories / Search / Updates). Until this ADR, the packaging convention has been **implicit** — derived from inspecting installed cogs (`anomaly-detect`, `presence`, `seizure-detect`, etc.) on a live appliance. Bringing new Cogs to the platform required reverse-engineering the layout each time.
|
||||
|
||||
This ADR formalises the layout so:
|
||||
|
||||
1. A repo crate can be built into a Cog with a deterministic Makefile / CI pipeline.
|
||||
2. Cog binaries can be cross-compiled for every supported architecture from a single source.
|
||||
3. The appliance's installer (`cognitum-cog-gateway`) can verify manifests without bespoke per-cog adapters.
|
||||
4. Future Cogs in this repo (starting with `cog-pose-estimation` — see ADR-101) follow a single rule.
|
||||
|
||||
## Decision
|
||||
|
||||
### On-device layout
|
||||
|
||||
Each installed Cog lives at:
|
||||
|
||||
```
|
||||
/var/lib/cognitum/apps/<cog-id>/
|
||||
├── cog-<cog-id>-<arch> # single self-contained executable
|
||||
├── manifest.json # immutable; signed by the publisher
|
||||
├── config.json # mutable; runtime config, owned by the appliance
|
||||
├── pid # current PID when running; absent when stopped
|
||||
├── output.log # stdout (truncated on rotation)
|
||||
└── error.log # stderr (truncated on rotation)
|
||||
```
|
||||
|
||||
`<cog-id>` is kebab-case, ASCII, `[a-z0-9-]{2,32}`. `<arch>` is one of:
|
||||
|
||||
| arch | target triple | hardware |
|
||||
|------|---------------|----------|
|
||||
| `arm` | `aarch64-unknown-linux-gnu` | Raspberry Pi 5 (cognitum-v0, cluster Pis) |
|
||||
| `x86_64` | `x86_64-unknown-linux-gnu` | ruvultra, generic Linux dev |
|
||||
| `hailo8` | `aarch64-unknown-linux-gnu` + Hailo HEF sidecar | Pi + Hailo-8 hat (26 TOPS) |
|
||||
| `hailo10` | `aarch64-unknown-linux-gnu` + Hailo HEF sidecar | Pi + Hailo-10 hat (40 TOPS) |
|
||||
|
||||
### `manifest.json` schema
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "anomaly-detect",
|
||||
"version": "0.1.0",
|
||||
"binary_url": "https://storage.googleapis.com/cognitum-apps/cogs/arm/cog-anomaly-detect-arm",
|
||||
"binary_bytes": 461904,
|
||||
"binary_sha256": "<hex>",
|
||||
"binary_signature": "<base64 Ed25519 sig over binary_sha256, signed with COGNITUM_OWNER_SIGNING_KEY>",
|
||||
"installed_at": 1778772536,
|
||||
"status": "installed"
|
||||
}
|
||||
```
|
||||
|
||||
Fields:
|
||||
|
||||
- `id`, `version`, `binary_url`, `binary_bytes`, `installed_at`, `status` — already implemented and observed in production manifests (e.g. `anomaly-detect@0.0.0`). Documented here without change.
|
||||
- `binary_sha256`, `binary_signature` — **new**, REQUIRED for any Cog shipped from this repo. Backwards-compatible with existing manifests: the appliance gateway treats both fields as optional today, MUST verify them when present. ADR-103 (witness chain) covers the trust model in more detail.
|
||||
- `status` values: `"installed"`, `"running"`, `"stopped"`, `"failed"`, `"updating"`.
|
||||
|
||||
### Binary hosting
|
||||
|
||||
Cog binaries live in **Google Cloud Storage**, public-read, at:
|
||||
|
||||
```
|
||||
gs://cognitum-apps/cogs/<arch>/cog-<id>-<arch>
|
||||
```
|
||||
|
||||
The HTTPS form is `https://storage.googleapis.com/cognitum-apps/cogs/<arch>/cog-<id>-<arch>` (no trailing extension; the URL is the canonical artifact). For Hailo variants, the HEF model file is sibling: `cog-<id>-<arch>.hef`.
|
||||
|
||||
Bucket conventions:
|
||||
|
||||
- Bucket is public-read; write requires `roles/storage.objectAdmin` in project `cognitum-20260110`.
|
||||
- Per-version artifacts must be content-addressed: `cogs/<arch>/cog-<id>-<arch>@<sha256-prefix>` is the immutable copy; the un-suffixed name is a symlink that updates on release.
|
||||
- `COGNITUM_OWNER_SIGNING_KEY` (GCP Secret Manager) signs every binary before upload.
|
||||
|
||||
### Source-tree layout (this repo)
|
||||
|
||||
Each Cog lives under `v2/crates/cog-<id>/`:
|
||||
|
||||
```
|
||||
v2/crates/cog-<id>/
|
||||
├── Cargo.toml # crate name = cog-<id>; binary = cog-<id>
|
||||
├── src/
|
||||
│ ├── main.rs # CLI: cog-<id> run | status | version
|
||||
│ ├── lib.rs
|
||||
│ └── inference.rs # the actual work
|
||||
├── cog/
|
||||
│ ├── manifest.template.json
|
||||
│ ├── config.schema.json # JSON schema for runtime config
|
||||
│ ├── README.md # consumer-facing description (used by the App Store UI)
|
||||
│ ├── icon.svg # 1024×1024 icon (used by App Store hero)
|
||||
│ └── Makefile # build / sign / upload targets
|
||||
└── tests/
|
||||
├── smoke.rs
|
||||
└── manifest_signature.rs
|
||||
```
|
||||
|
||||
### Build pipeline
|
||||
|
||||
```
|
||||
cd v2/crates/cog-<id>
|
||||
make build-arm # cross-compile to aarch64-unknown-linux-gnu
|
||||
make build-x86_64 # x86_64 Linux build
|
||||
make build-hailo8 # arm + HEF compilation (requires Hailo Dataflow Compiler)
|
||||
make build-hailo10 # arm + HEF compilation
|
||||
make sign # produce binary_sha256 + binary_signature
|
||||
make upload # gsutil cp to gs://cognitum-apps/cogs/<arch>/
|
||||
make manifest # emit manifest.json with all fields filled
|
||||
```
|
||||
|
||||
CI (GitHub Actions) MUST run `make build-arm` + `make build-x86_64` on every PR touching `v2/crates/cog-*/`. Hailo HEF compilation requires the proprietary Hailo SDK and runs only on the Hailo-capable runners (currently a labelled self-hosted runner on the Pi cluster — TBD, separate ADR).
|
||||
|
||||
### Runtime contract
|
||||
|
||||
A Cog binary MUST implement:
|
||||
|
||||
| Subcommand | Behaviour |
|
||||
|-----------|-----------|
|
||||
| `cog-<id> version` | Print `<id> <version>` and exit 0. |
|
||||
| `cog-<id> manifest` | Print the embedded manifest JSON and exit 0. |
|
||||
| `cog-<id> run --config /path/to/config.json` | Long-running. Writes structured JSON logs to stdout (parsed by `cognitum-cog-gateway`). Exit code 0 on graceful shutdown, non-zero on fatal error. |
|
||||
| `cog-<id> health` | One-shot. Exit 0 if the cog could come up healthy; non-zero with diagnostic on stderr. Called by the gateway before `run`. |
|
||||
|
||||
stdout JSON line format (one event per line):
|
||||
|
||||
```json
|
||||
{"ts": 1779210883.444, "level": "info", "event": "<event-name>", "fields": { ... }}
|
||||
```
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- New Cogs can be added without RE-ing the layout each time.
|
||||
- CI can verify the manifest schema before merge.
|
||||
- Signed binaries close a real supply-chain gap — current installed cogs (`anomaly-detect@0.0.0`) have no signature, and a compromised GCS object could push malicious code to every appliance.
|
||||
- The runtime contract (`run | health | version | manifest`) is uniform across cogs, so `cognitum-cog-gateway` can stop carrying per-cog adapters.
|
||||
|
||||
### Negative
|
||||
|
||||
- Existing installed cogs must be re-published with signatures within one minor release of the gateway adopting the verify-when-present rule.
|
||||
- Hailo HEF cross-compile is gated on a self-hosted runner; we accept that PRs touching Hailo variants will be slower to land.
|
||||
|
||||
### Risks
|
||||
|
||||
- **Signing key rotation**: `COGNITUM_OWNER_SIGNING_KEY` (Ed25519) is a single root-of-trust today. ADR-103 (witness chain) describes the rotation/recovery path; this ADR depends on that.
|
||||
- **GCS bucket misconfiguration**: a public-read bucket with versioning-off could allow rollback attacks. Bucket MUST have Object Versioning enabled + 90-day non-current-version retention.
|
||||
|
||||
## Migration
|
||||
|
||||
1. ✅ Land this ADR.
|
||||
2. ✅ Land ADR-101 (`cog-pose-estimation` — first Cog built to this spec). Shipped in PR #642 + #643 on 2026-05-19; signed `arm` and `x86_64` binaries live at `gs://cognitum-apps/cogs/{arm,x86_64}/`; install verified on cognitum-v0.
|
||||
3. After two clean releases of `cog-pose-estimation`, re-publish the existing cogs (`anomaly-detect`, `presence`, etc.) with `binary_sha256` + `binary_signature`. Track in a follow-up issue.
|
||||
4. Flip `cognitum-cog-gateway` from "verify when present" to "require signature" — separate ADR, separate review.
|
||||
|
||||
## See also
|
||||
|
||||
- ADR-101: Pose Estimation Cog (first Cog built to this spec).
|
||||
- ADR-103: Witness chain trust model (signing key rotation, future ADR).
|
||||
- `docs/adr/ADR-079-camera-ground-truth-training.md` — the training pipeline behind `cog-pose-estimation`.
|
||||
- `CLAUDE.local.md` § "Fleet Infrastructure (Tailscale)" — appliance layout this ADR describes.
|
||||
@@ -0,0 +1,208 @@
|
||||
# ADR-101: Pose Estimation Cog (WiFi-DensePose side)
|
||||
|
||||
- **Status:** Accepted — **v0.0.1 shipped 2026-05-19** (merged in PRs #642 + #643, signed binaries on GCS, live install on cognitum-v0)
|
||||
- **Date:** 2026-05-19
|
||||
- **Deciders:** ruv
|
||||
- **Companion ADR (v0-appliance side):** v0-appliance ADR-225 (cognitum-pose-estimation crate)
|
||||
|
||||
## Context
|
||||
|
||||
ADR-079 designed the 17-keypoint COCO pose-estimation training pipeline. ADR-100 formalised the Cognitum Cog packaging spec. This ADR is the bridge: it specifies how the wifi-densepose training pipeline produces an artifact that ships as a Cog (`cog-pose-estimation`) onto the Cognitum V0 appliance and out to the Pi+Hailo cluster.
|
||||
|
||||
It is the next product step beyond the published `presence` Cog (binary head trained from the contrastive encoder on Hugging Face at `ruvnet/wifi-densepose-pretrained`). Where `presence` reports a single boolean per tick, `cog-pose-estimation` reports 17 (x, y) keypoints per person, per tick.
|
||||
|
||||
## Decision
|
||||
|
||||
### Pipeline
|
||||
|
||||
```
|
||||
(training side — ruvultra GPU)
|
||||
ESP32 / rvcsi ─► collect-ground-truth.py + sensing-server recording
|
||||
│
|
||||
▼
|
||||
data/paired/*.paired.jsonl (CSI window + camera keypoints)
|
||||
│
|
||||
▼
|
||||
v2/crates/wifi-densepose-train ──► Rust + libtorch trainer
|
||||
(uses RTX 5080 / CUDA 12.x) │
|
||||
init from ruvnet/wifi-densepose-pretrained
|
||||
│
|
||||
▼
|
||||
model.safetensors (encoder + pose head)
|
||||
│
|
||||
─────────────┴─────────────
|
||||
│ │
|
||||
▼ ▼
|
||||
v2/crates/cog-pose-estimation export to ONNX
|
||||
(this repo) │
|
||||
• emits manifest.json ▼
|
||||
• produces cog binary cognitum-hailo
|
||||
• signs + uploads to GCS (v0-appliance side)
|
||||
│
|
||||
▼
|
||||
cog-pose-estimation.hef
|
||||
│
|
||||
▼
|
||||
(appliance side — cognitum-v0 + Pi+Hailo cluster)
|
||||
|
||||
gs://cognitum-apps/cogs/{arm,hailo8,hailo10}/cog-pose-estimation-<arch>
|
||||
│
|
||||
▼
|
||||
`cognitum-cog-gateway` pulls artifact + manifest, verifies signature, installs
|
||||
into /var/lib/cognitum/apps/pose-estimation/
|
||||
│
|
||||
▼
|
||||
run loop: read CSI frames from local sensing-server
|
||||
→ encoder → pose head → emit `{ts, persons: [{keypoints: [...17 x,y...] }]}`
|
||||
on stdout as the Cog runtime contract requires
|
||||
```
|
||||
|
||||
### Architecture (model)
|
||||
|
||||
| Stage | Module | Notes |
|
||||
|-------|--------|-------|
|
||||
| Input | `[56 subcarriers × 20 frames]` per CSI window | matches today's `data/paired/wiflow-p7-*.paired.jsonl` |
|
||||
| Encoder | TCN-lite or contrastive encoder lifted from HF presence model | 128-dim embedding; weights init from `ruvnet/wifi-densepose-pretrained/model.safetensors` |
|
||||
| Pose head | 2-layer MLP `(128 → 256 → 34)` | 34 = 17 × (x, y) |
|
||||
| Output | `[B, 17, 2]` keypoints in `[0, 1]` image-normalised coords | confidence is implicit in keypoint variance over time; ADR-079 P9 will add explicit per-joint confidence |
|
||||
| Loss | Confidence-weighted SmoothL1 (frame-level) + bone-length regulariser + temporal smoothness | per ADR-079 Phase 3 refinement |
|
||||
| Init | Encoder = HF presence weights (frozen for 50 epochs, then jointly fine-tuned) | unblocks the sigmoid-saturation failure mode observed in #645 |
|
||||
| Training | `v2/crates/wifi-densepose-train` with libtorch backend on RTX 5080 | replaces the pure-JS SPSA trainer that produced 0% PCK in #645 |
|
||||
|
||||
### Repo layout
|
||||
|
||||
```
|
||||
v2/crates/cog-pose-estimation/ # NEW (this ADR)
|
||||
├── Cargo.toml
|
||||
├── src/
|
||||
│ ├── main.rs # CLI: run | health | version | manifest
|
||||
│ ├── lib.rs
|
||||
│ ├── inference.rs # ONNX runtime + Hailo HEF runtime dispatch
|
||||
│ ├── frame_subscriber.rs # local sensing-server subscriber
|
||||
│ └── publisher.rs # emits structured JSON events per Cog contract
|
||||
├── cog/
|
||||
│ ├── manifest.template.json
|
||||
│ ├── config.schema.json
|
||||
│ ├── README.md
|
||||
│ ├── icon.svg
|
||||
│ └── Makefile # build-arm | build-x86_64 | sign | upload
|
||||
└── tests/
|
||||
├── manifest_signature.rs
|
||||
└── inference_smoke.rs
|
||||
```
|
||||
|
||||
### Runtime contract
|
||||
|
||||
Honours ADR-100's per-Cog CLI contract:
|
||||
|
||||
- `cog-pose-estimation version` → `pose-estimation 0.0.1`
|
||||
- `cog-pose-estimation manifest` → JSON
|
||||
- `cog-pose-estimation health` → 0 if encoder+head load and a synthetic frame produces a finite output
|
||||
- `cog-pose-estimation run --config /etc/cognitum/cogs/pose-estimation/config.json` → long-running; emits one JSON event per inferred frame:
|
||||
|
||||
```json
|
||||
{
|
||||
"ts": 1779210883.444,
|
||||
"level": "info",
|
||||
"event": "pose.frame",
|
||||
"fields": {
|
||||
"tick": 12345,
|
||||
"n_persons": 1,
|
||||
"persons": [
|
||||
{"keypoints": [[0.48, 0.31], [0.52, 0.28], ...], "confidence": 0.81}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Hardware deployment
|
||||
|
||||
| Target | arch | runtime | notes |
|
||||
|--------|------|---------|-------|
|
||||
| ruvultra (dev) | `x86_64` | ONNX Runtime CPU/CUDA | development & smoke tests |
|
||||
| cognitum-v0 (Pi 5) | `arm` | ONNX Runtime ARM | reference deploy; ~20 ms/frame |
|
||||
| Pi + Hailo-8 hat | `hailo8` | Hailo HEF runtime via `cognitum-hailo` | ~2 ms/frame, 26 TOPS budget |
|
||||
| Pi + Hailo-10 hat | `hailo10` | Hailo HEF runtime via `cognitum-hailo` | ~1 ms/frame, 40 TOPS budget |
|
||||
|
||||
### Acceptance gates
|
||||
|
||||
1. **Validates:** `cargo test -p cog-pose-estimation` green; `cog-pose-estimation health` returns 0 against a synthetic CSI window.
|
||||
2. **Benchmarks:** end-to-end frame latency on each target arch logged in `target/criterion/`; published in `docs/benchmarks/pose-estimation-cog.md`.
|
||||
3. **Optimised:** the Hailo-targeted ONNX graph passes through Hailo Dataflow Compiler without quantisation-aware-training warnings.
|
||||
4. **Published:** signed binary at `gs://cognitum-apps/cogs/<arch>/cog-pose-estimation-<arch>`; manifest valid against the JSON schema in ADR-100; appliance installer can pull and run it.
|
||||
|
||||
PCK@20 is intentionally **not** an acceptance gate of this ADR. Achieving the ADR-079 ≥35% target is a separate, data-bound milestone tracked in #645. This ADR ships the **vehicle**, not the model accuracy.
|
||||
|
||||
### First measured run — v0.0.1 (2026-05-19)
|
||||
|
||||
A Candle-on-CUDA training run on `ruvultra`'s RTX 5080 against the same 1,077-sample paired session that produced the 0%/0% baseline in #645 yielded:
|
||||
|
||||
- **PCK@20 = 3.0%**, **PCK@50 = 18.5%**, **MPJPE = 0.093** (normalized).
|
||||
- 400 epochs in **2.1 s** wall time (~5 ms/epoch, full-batch).
|
||||
- Loss reduction 13× (0.181 → 0.014, eval 0.010).
|
||||
- Strongest signal at `r_hip` (PCK@50 = 76.9%), `r_knee` (35.2%), `l_elbow` (26.4%).
|
||||
|
||||
This confirms the pipeline trains end-to-end and produces a signal-bearing model. The remaining gap to PCK@20 ≥ 35% is data-bound (1,077 samples is ≪ the ADR-079 target of ~30K). See `docs/benchmarks/pose-estimation-cog.md` for the full result dump.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- First Cog from this repo that integrates with the appliance/cog-gateway pipeline. Future cogs (e.g. `cog-vitals`, `cog-fall-alert`) follow the same template.
|
||||
- Closes the loop from data collection → training → quantisation → cluster deployment with a single repo-anchored artifact.
|
||||
- Forces a real signature on cog binaries (per ADR-100), which improves supply-chain hygiene across the whole appliance.
|
||||
|
||||
### Negative
|
||||
|
||||
- Adds a hard dependency on the Hailo Dataflow Compiler, which lives behind a self-hosted runner — Hailo-targeted PRs land more slowly.
|
||||
- The first published binary will have low PCK (data + training time gap, #645) — UX needs to surface this clearly so end users do not interpret bad keypoints as a bug.
|
||||
|
||||
### Risks
|
||||
|
||||
- **Model size on Hailo**: the encoder fits comfortably in Hailo-8's on-chip SRAM, but the pose-head expansion to `[17×2]` plus required temporal stacking pushes us close to the Hailo-8 envelope. Mitigation: Hailo-10 path is the primary deploy target; Hailo-8 is a stretch.
|
||||
- **Sensing-server schema drift**: the cog subscribes to `/api/v1/sensing/latest` JSON. If the appliance's sensing-server schema changes, the cog fails open (logs warning, emits nothing). The `frame_subscriber.rs` module pins to schema version `2`.
|
||||
|
||||
## Migration / rollout
|
||||
|
||||
1. Land this ADR + ADR-100 on `main` of RuView.
|
||||
2. Land companion ADR-225 + crate on `main` of v0-appliance.
|
||||
3. First release `cog-pose-estimation@0.0.1` ships **only** to `ruvultra` and `cognitum-v0`. Not pushed to the cluster Pis yet.
|
||||
4. After P7→P9 data work (#645) brings PCK above a usable threshold, rebuild + re-publish; only then enable cluster rollout via `cognitum-cog-gateway`'s OTA channel.
|
||||
|
||||
## v0.0.1 shipping status — 2026-05-19
|
||||
|
||||
PRs `#642` (scaffold + arm release + ONNX + live install) and `#643` (x86_64 release) landed on `main`. Acceptance gates from ADR-100 met as follows:
|
||||
|
||||
| Gate | Status |
|
||||
|------|--------|
|
||||
| Cog binary exists per arch | ✅ arm (`3,741,976 B`) + x86_64 (`4,548,856 B`) on GCS |
|
||||
| Manifest matches schema | ✅ `cog/artifacts/manifests/{arm,x86_64}/manifest.json` |
|
||||
| Binary sha256 + Ed25519 signature | ✅ both signed with `COGNITUM_OWNER_SIGNING_KEY`, round-trip verified |
|
||||
| Public-readable GCS | ✅ anonymous HTTP GET works, SHA matches |
|
||||
| Live install on a real appliance | ✅ `/var/lib/cognitum/apps/pose-estimation/` on `cognitum-v0` (Pi 5), same layout as `anomaly-detect` |
|
||||
| Runtime contract (`version \| manifest \| health \| run`) | ✅ all four return correct output; `run` emits `pose.frame` events |
|
||||
| Real weights loaded (not stub) | ✅ `cargo test` asserts `backend.starts_with("candle-")` + non-zero confidence |
|
||||
| ONNX artifact (for downstream HEF) | ✅ `pose_v1.onnx` (12 KB), parity vs torch = 8.94e-8 |
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Training time (RTX 5080 / Candle CUDA) | 2.1 s for 400 epochs |
|
||||
| PCK@20 / PCK@50 / MPJPE (1,077-sample seated-desk session) | 3.0% / 18.5% / 0.093 |
|
||||
| Cold-start: Windows x86_64 | 76 ms |
|
||||
| Cold-start: ruvultra x86_64 | **5.4 ms** |
|
||||
| Cold-start: Pi 5 aarch64 | **8.4 ms** |
|
||||
| Tests | 5/5 pass |
|
||||
|
||||
Open follow-ups carried forward from this ADR's "Acceptance gates" section:
|
||||
|
||||
- **Hailo HEF cross-compile** — `pose_v1.onnx` is ready; still gated on Hailo Dataflow Compiler + self-hosted runner provisioning. Tracked separately.
|
||||
- **PCK@20 ≥ 35%** — explicitly not an acceptance gate of this ADR, but the limiting factor on practical usefulness. Tracked in [#645](https://github.com/ruvnet/RuView/issues/645): needs ~30× more paired samples + multi-room camera framing. Today's seated-desk session is the demonstrated bottleneck.
|
||||
|
||||
## See also
|
||||
|
||||
- ADR-079: Camera-supervised pose training pipeline (the model we're shipping).
|
||||
- ADR-100: Cog packaging specification (the format we're shipping in).
|
||||
- v0-appliance ADR-225: cognitum-pose-estimation crate (the appliance-side runtime).
|
||||
- v0-appliance ADR-220: cog management surface (where this cog appears in the dashboard).
|
||||
- Issue #645: PCK gap (current 3% / 18.5% → ≥35% target).
|
||||
- `docs/benchmarks/pose-estimation-cog.md`: full benchmark log, all measured numbers.
|
||||
@@ -0,0 +1,171 @@
|
||||
# ADR-102: Edge Module Registry Integration
|
||||
|
||||
- **Status:** Accepted
|
||||
- **Date:** 2026-05-19
|
||||
- **Deciders:** ruv
|
||||
|
||||
## Context
|
||||
|
||||
The Cognitum app ecosystem publishes a canonical app store catalog at:
|
||||
|
||||
```
|
||||
https://storage.googleapis.com/cognitum-apps/app-registry.json
|
||||
```
|
||||
|
||||
As of v2.1.0 (2026-05-13) the registry advertises **105 cogs across 11 categories** (health, security, building, retail, industrial, research, ai, swarm, signal, network, developer). Each entry carries `id`, `name`, `category`, `version`, `description`, `size_kb`, `difficulty`, `sha256`, `binary_size`, and a `config[]` schema describing the runtime parameters the appliance offers when installing the cog.
|
||||
|
||||
RuView today has no live awareness of this catalog. The `README.md` capability table is hand-curated; the UI surfaces only the capabilities the dashboard's HTML knows about; nothing in `wifi-densepose-sensing-server` references the registry. Result: when Cognitum ships a new cog (the registry was last updated 6 days ago — a fast cadence), RuView stays unaware until someone manually edits the README. Customers running the RuView dashboard against a real appliance see a 10-capability bag in the UI while the appliance is actually capable of installing 105 cogs.
|
||||
|
||||
Today's `cog-pose-estimation@0.0.1` release (PRs #642 / #643, ADR-100, ADR-101) is the first cog this repo ships to that registry. We need the discovery side to match.
|
||||
|
||||
## Decision
|
||||
|
||||
`wifi-densepose-sensing-server` will fetch `app-registry.json` on demand, cache it in process memory with a TTL, and serve it back through a new endpoint:
|
||||
|
||||
```
|
||||
GET /api/v1/edge/registry
|
||||
GET /api/v1/edge/registry?refresh=1 (force-bypass cache, log if abused)
|
||||
```
|
||||
|
||||
The registry is **passively surfaced**, not modified. RuView is a presentation layer for the canonical Cognitum catalog; it never re-signs entries or re-hosts binaries.
|
||||
|
||||
### Module
|
||||
|
||||
`v2/crates/wifi-densepose-sensing-server/src/edge_registry.rs` — small, ~150 lines.
|
||||
|
||||
```rust
|
||||
pub struct EdgeRegistry {
|
||||
cached: RwLock<Option<CachedEntry>>,
|
||||
ttl: Duration,
|
||||
upstream_url: String,
|
||||
}
|
||||
|
||||
struct CachedEntry {
|
||||
payload: serde_json::Value,
|
||||
fetched_at: Instant,
|
||||
upstream_sha256: String,
|
||||
}
|
||||
```
|
||||
|
||||
Cache semantics:
|
||||
|
||||
- TTL **3600 s (1 hour)** by default — registry updates land on a roughly-weekly cadence and a stale-by-an-hour catalog is fine.
|
||||
- `?refresh=1` bypasses the cache but writes a debug log so accidental abuse is visible.
|
||||
- On upstream fetch failure when the cache is non-empty, **serve the stale cached copy** with a `stale: true` marker in the response and a 200 status (preserve UI), not a 5xx.
|
||||
- On upstream fetch failure when the cache is empty, return 503 with the upstream error in the body.
|
||||
|
||||
### Response shape
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"fetched_at": 1779200000, // server-side fetch timestamp
|
||||
"ttl_seconds": 3600,
|
||||
"stale": false, // true when serving past TTL because upstream is down
|
||||
"upstream_url": "https://storage.googleapis.com/cognitum-apps/app-registry.json",
|
||||
"upstream_sha256": "<sha256-of-payload-bytes>",
|
||||
"registry": { /* full canonical JSON as returned upstream */ }
|
||||
}
|
||||
```
|
||||
|
||||
The `registry` field is the upstream JSON inlined verbatim so consumers don't need to make a second hop. `upstream_sha256` lets a paranoid consumer compare against a pinned hash.
|
||||
|
||||
### Trust / verification
|
||||
|
||||
- Bucket is public-read with object versioning enabled (per ADR-100 §"GCS misconfiguration risks").
|
||||
- The cog-level `binary_sha256` + `binary_signature` (ADR-100) are the trust roots for *installs*. The registry itself is not signed today.
|
||||
- We deliberately **do not** add a signature requirement to the registry JSON in this ADR — that would block the integration on a parallel infrastructure project. A future ADR can layer signature checks on top once the publisher pipeline emits them.
|
||||
|
||||
### UI surfacing
|
||||
|
||||
New page `ui/edge-modules.html` renders the registry into category sections with cog cards. Each card links out to the Cognitum V0 appliance's `/cogs` page (`http://cognitum-v0:9000/cogs#<id>`) for the install action — RuView itself never installs.
|
||||
|
||||
The existing dashboard's "Capabilities" section continues to show RuView-native sensing capabilities (presence, breathing, pose, etc. — the things RuView itself runs); the new edge-modules page shows the broader Cognitum cog catalog. The two are distinct surfaces and shouldn't be merged.
|
||||
|
||||
### Failure modes
|
||||
|
||||
| Scenario | Behaviour |
|
||||
|---|---|
|
||||
| Upstream returns 200 with valid JSON | Cache it, return it. |
|
||||
| Upstream returns 200 with invalid JSON | Treat as failure; serve stale if available else 503. Log the upstream sha + the parse error. |
|
||||
| Upstream returns 4xx / 5xx | Same as JSON-invalid: serve stale if available else 503. |
|
||||
| TLS / DNS / timeout error | Same. |
|
||||
| Upstream is permanently moved | Operator updates the `upstream_url` config (CLI flag added). No code change required to migrate registries. |
|
||||
|
||||
### Configuration
|
||||
|
||||
- `--edge-registry-url <URL>` — override the default (default: `https://storage.googleapis.com/cognitum-apps/app-registry.json`)
|
||||
- `--edge-registry-ttl-secs <N>` — override the cache TTL (default: 3600)
|
||||
- `--no-edge-registry` — disable the endpoint entirely (returns 404). For air-gapped deployments.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- One source of truth for the cog catalog across RuView + Cognitum dashboards.
|
||||
- Zero ongoing maintenance: when Cognitum publishes registry v2.2.0, RuView sees it within an hour without a release.
|
||||
- The endpoint is also useful for non-UI consumers (CI checks, fleet automation, third-party integrations).
|
||||
- Lets us deprecate the hand-curated README capability table in favour of generated content (separate PR).
|
||||
|
||||
### Negative
|
||||
|
||||
- Adds an outbound HTTP dependency to the sensing-server. Air-gapped deployments must use `--no-edge-registry`.
|
||||
- Stale-but-served behaviour can mask upstream outages from operators. Mitigation: include `stale: true` + `fetched_at` in the response so the UI can render a "registry possibly out of date" badge.
|
||||
|
||||
### Risks
|
||||
|
||||
- **Upstream rug-pull**: if `cognitum-apps` is deleted or replaced, the endpoint goes dark. The `--edge-registry-url` flag lets operators repoint without a code change. Long-term, RuView could mirror the registry into its own GCS bucket if the relationship requires it.
|
||||
- **Cache poisoning**: the upstream is public-read; an attacker who breaches Cognitum's GCS write could push a bad registry. The cog-level signatures (ADR-100) limit the blast radius — bad registry entries can't install bad binaries, only show wrong metadata. Acceptable until registry-level signing lands.
|
||||
|
||||
## Security review
|
||||
|
||||
A real review of the attack surface this endpoint introduces.
|
||||
|
||||
### Threats considered
|
||||
|
||||
| # | Threat | Mitigation in this ADR |
|
||||
|---|--------|------------------------|
|
||||
| T1 | **SSRF** — operator-supplied `--edge-registry-url` redirects fetches to an internal target | Flag is operator-only (CLI / env) — there is no API endpoint to mutate it at runtime. Operators are already trusted (they control the binary). |
|
||||
| T2 | **Outbound dependency reveals deployment** — a passive observer of the egress sees the appliance phoning home to GCS | Documented in the docstring + the runtime startup log. Operators wanting offline deployments use `--no-edge-registry`. |
|
||||
| T3 | **Malicious upstream registry** — Cognitum's GCS bucket is breached and a poisoned `app-registry.json` is served | Two layers absorb this: (a) the registry's role is **discovery only** — installs verify the per-cog `binary_sha256` + `binary_signature` (ADR-100); a wrong description string can mislead a human, but a wrong binary still has to pass Ed25519 against `COGNITUM_OWNER_SIGNING_KEY`. (b) The endpoint exposes `upstream_sha256` so a paranoid operator can pin the expected registry hash externally and alert on drift. |
|
||||
| T4 | **Response inflation** — upstream returns a multi-GB payload to exhaust memory | `MAX_PAYLOAD_BYTES = 8 MiB` cap (current registry is ~50–200 KB). Exceeding cap returns an error without buffering past the cap. |
|
||||
| T5 | **Slow upstream blocking server threads** — Slowloris-style stall on the fetch | 10-second wire timeout via `ureq::AgentBuilder`. Per-handler fetch runs inside `tokio::task::spawn_blocking` so a stalled fetch never blocks the async runtime. |
|
||||
| T6 | **Denial via `?refresh=1` abuse** — unauthenticated callers force-bypass the cache repeatedly | Cache lives in process; `?refresh=1` triggers a single upstream fetch behind a synchronous code path. A flood of refresh requests is rate-limited by the upstream's own throttling (GCS) and locally serialised by Rust's `RwLock`. Refresh requests are logged at `debug` so abuse is visible. **Follow-up:** add per-IP rate-limit middleware if seen abused (separate PR; tracked in #574-style follow-up). |
|
||||
| T7 | **JSON deserialisation panics** — malformed registry triggers a Rust panic | Payload is parsed as `serde_json::Value` (opaque untyped tree) — never coerced into a strongly-typed struct that could panic. Failure is propagated as `FetcherError::Network` which the handler maps to 503. |
|
||||
| T8 | **Stale-on-error masks outages from operators** | Response carries `stale: true` + `fetched_at` (unix timestamp). UI rendering MUST surface this badge — encoded as an explicit field, not an implicit silence. |
|
||||
| T9 | **TLS downgrade / MITM on the fetch** | `ureq` is built with the `tls` feature (rustls) by default. No `--insecure` flag exists. If the upstream uses LetsEncrypt the cert chain is system-trusted; certificate pinning is out of scope (would block the bucket from rotating certs). |
|
||||
| T10 | **Unauthenticated access exposes ‘what cogs exist’** | The registry is canonical-public information (already public-read on GCS via anonymous HTTP GET). Surfacing it on a local LAN HTTP API does not increase its disclosure. The endpoint stays under the project's existing `RUVIEW_API_TOKEN` Bearer auth — when set, the registry is gated like other `/api/v1/*` routes. |
|
||||
| T11 | **Configuration injection via env var** — `RUVIEW_EDGE_REGISTRY_URL` set to a malicious URL by an attacker who controls the process environment | If an attacker controls the env, they own the process; this is not a new threat surface. Documented in the CLI help. |
|
||||
| T12 | **Cache mutation across threads / poisoning** | The cache is `RwLock<Option<CachedEntry>>`. Writes go through `cached.write()` once per fetch. Snapshot reads `clone()` the `CachedEntry` (cheap — `Value` is reference-counted internally for large strings) so concurrent readers don't share mutable state. Tests cover the multi-call path; no `unsafe` is used. |
|
||||
|
||||
### What this ADR does NOT secure
|
||||
|
||||
- **Registry-level signing** — the JSON payload itself is unsigned. If/when Cognitum's publisher pipeline emits a registry sig (e.g. detached `.json.sig`), a follow-up ADR will require it. Today the per-cog binary signature (ADR-100) is the actual trust root for installs; the registry is metadata.
|
||||
- **Per-client rate-limiting on `?refresh=1`** — relies on the upstream's own throttling. If we see abuse we'll add a token-bucket middleware; not needed for v0.0.1.
|
||||
|
||||
### Testing
|
||||
|
||||
| Test | What it verifies |
|
||||
|------|------------------|
|
||||
| `first_call_hits_upstream_and_caches` | Single fetch, then cache hit |
|
||||
| `ttl_expiry_triggers_refetch` | Cache TTL bound respected |
|
||||
| `force_refresh_bypasses_fresh_cache` | `?refresh=1` semantics |
|
||||
| `stale_serve_on_upstream_failure_after_cached_success` | T8 explicit (`stale: true` returned) |
|
||||
| `no_cache_no_upstream_returns_error` | T3/T5 — error propagated cleanly when nothing to fall back on |
|
||||
| `upstream_invalid_json_is_treated_as_error` | T7 — malformed payload doesn't panic |
|
||||
| `upstream_sha256_is_deterministic` | T3 — hash field is reliable for external pinning |
|
||||
|
||||
All 7 tests in `src/edge_registry.rs::tests` pass.
|
||||
|
||||
## Migration
|
||||
|
||||
1. Land this ADR + the implementing PR.
|
||||
2. UI: ship `ui/edge-modules.html` and link from `index.html`.
|
||||
3. After two clean releases of the endpoint, remove the hand-curated "Capabilities" table from `README.md` and replace with a small "see the appliance for the full catalog" pointer.
|
||||
4. Future ADR: registry signing once Cognitum's publisher pipeline emits a sig.
|
||||
|
||||
## See also
|
||||
|
||||
- ADR-100: Cognitum Cog Packaging Specification (binary trust model).
|
||||
- ADR-101: Pose Estimation Cog (the first repo-shipped cog visible in the registry).
|
||||
- v0-appliance ADR-220: Cog management surface (where this registry is the input to install actions).
|
||||
- `docs/benchmarks/pose-estimation-cog.md`: the per-cog benchmark format this ADR's response shape complements.
|
||||
@@ -0,0 +1,198 @@
|
||||
# ADR-103: Learned Multi-Person Counter (SOTA WiFi CSI counting)
|
||||
|
||||
- **Status:** Proposed
|
||||
- **Date:** 2026-05-21
|
||||
- **Deciders:** ruv
|
||||
- **Motivating issue:** #499 (double skeletons with 3-node ESP32-S3 setup, closed by PR #491)
|
||||
- **Related:** ADR-079 (camera-supervised training), ADR-100 (cog packaging), ADR-101 (pose cog), ADR-102 (edge module registry), PR #491 (RollingP95 + dedup_factor)
|
||||
|
||||
## Context
|
||||
|
||||
PR #491 stopped the bleeding on #499. The fix replaced hard-coded denominators (`variance/300`, `motion_band_power/250`, `spectral_power/500`) with a self-calibrating `RollingP95` streaming estimator and exposed the multi-node `dedup_factor` as a runtime knob. Day-0 deployments no longer collapse dynamic range, and operators can auto-tune the divisor from a known person count.
|
||||
|
||||
That gets us to a **stable heuristic that adapts to the room**. It does not get us to the published WiFi-CSI counting state of the art:
|
||||
|
||||
| System | Setup | Reported accuracy | Method |
|
||||
|--------|-------|-------------------|--------|
|
||||
| **WiCount** (CMU, 2017) | Intel 5300 3×3 MIMO | 89% within ±1 | LSTM over CSI amplitude |
|
||||
| **DeepCount** (2018) | Atheros 3×3 | 92% within ±1, 5-room | CNN + cross-environment transfer |
|
||||
| **CrossCount** (2019) | Atheros, 6 rooms | 84% cross-room within ±1 | Domain-adversarial CNN |
|
||||
| **HeadCount** (2021) | Intel 5300 | <1 person MAE, 5 envs | Multi-stream CSI + attention |
|
||||
| **RuView today** (PR #491) | ESP32-S3 1×1 SISO | Calibrated heuristic; not measured against ground truth | RollingP95 + dedup_factor |
|
||||
|
||||
The literature uses 3×3 MIMO research NICs. RuView uses 1×1 SISO ESP32-S3 nodes. The published number is therefore not directly attainable, but the **architectural gap** is large enough that a learned-counter approach on our hardware should comfortably beat today's slot heuristic — and the infrastructure to train one already exists in this repo (Candle + RTX 5080 trained `pose_v1.safetensors` in 2.1 s yesterday — see [`docs/benchmarks/pose-estimation-cog.md`](../benchmarks/pose-estimation-cog.md)).
|
||||
|
||||
Five primitives we already have but don't yet compose into a counter:
|
||||
|
||||
1. **Paired CSI + camera label dataset** — `scripts/collect-ground-truth.py` + `scripts/align-ground-truth.js` (PR #641 streaming-safe). 1,077 samples currently; #645 tracks the path to ~30K.
|
||||
2. **Stoer-Wagner min-cut for person-separable subcarrier groups** — `ruvector-mincut` (already a workspace dep). The Candle trainer used it yesterday and reported `Min-cut value: 0.1538 — partition: [55, 1] subcarriers`.
|
||||
3. **Contrastive-pretrained CSI encoder** — `ruvnet/wifi-densepose-pretrained` on HF (12.2M training steps, 60K frames, 128-dim embeddings, ~165k emb/s on M4 Pro).
|
||||
4. **Candle training pipeline** — proven yesterday: 400 epochs in 2.1 s on RTX 5080, bit-perfect ONNX export, signed cog binary on GCS.
|
||||
5. **Multi-node fusion stage** — `multistatic_bridge.rs` already aggregates per-node feature vectors with the tunable `dedup_factor`. The new model output can be a drop-in replacement for the existing dedup divisor.
|
||||
|
||||
## Decision
|
||||
|
||||
Train and ship a small **learned multi-person counter** as a new Cognitum Cog (`cog-person-count`), modelled on the same packaging path as `cog-pose-estimation` (ADR-101). Wire it into the sensing-server's existing person-count call site (`csi.rs::score_to_person_count`) as a drop-in replacement for the slot heuristic.
|
||||
|
||||
### Architecture (v0.1.0)
|
||||
|
||||
```
|
||||
┌──────────────────────────────┐
|
||||
per-node CSI window │ Encoder (frozen first 50 ep) │
|
||||
[56 sub × 20 frames] ─► init from ruvnet/wifi- │
|
||||
│ densepose-pretrained │
|
||||
│ → 128-dim embedding │
|
||||
└──────────────┬───────────────┘
|
||||
│
|
||||
┌────────────────┴────────────────┐
|
||||
▼ ▼
|
||||
┌────────────────────┐ ┌────────────────────────┐
|
||||
│ Count head │ │ Confidence head │
|
||||
│ Linear(128→64) │ │ Linear(128→32) │
|
||||
│ ReLU │ │ ReLU │
|
||||
│ Linear(64→8) │ │ Linear(32→1) + sigmoid│
|
||||
│ → softmax over │ │ → calibrated p(correct)│
|
||||
│ {0..7} persons │ └────────────────────────┘
|
||||
└────────┬───────────┘
|
||||
│ (per-node prediction)
|
||||
│
|
||||
N nodes' per-node │
|
||||
counts + confidences ▼
|
||||
┌─────────────────────────────────────┐
|
||||
│ Multi-node fusion (Stoer-Wagner) │
|
||||
│ • build graph: nodes × subcarrier │
|
||||
│ feature similarity │
|
||||
│ • min-cut → distinct-person bound │
|
||||
│ • combine with per-node count head │
|
||||
│ via confidence-weighted vote │
|
||||
└──────────────────┬──────────────────┘
|
||||
▼
|
||||
{ count: int,
|
||||
confidence: float [0,1],
|
||||
count_p95_low: int,
|
||||
count_p95_high: int,
|
||||
per_node_breakdown: [...] }
|
||||
```
|
||||
|
||||
Five things to call out about this architecture:
|
||||
|
||||
1. **Frozen encoder for the first 50 epochs.** The HF presence encoder already produces a useful 128-dim embedding from random CSI; training the counting head on top of frozen features is the standard transfer-learning pattern and avoids re-learning the contrastive geometry the encoder was painstakingly trained for.
|
||||
2. **Classification over `{0..7}` people**, not regression to a real number. Counts are integer-valued; classification gives a calibrated probability per count and lets the confidence head produce a meaningful uncertainty.
|
||||
3. **Stoer-Wagner min-cut at fusion time, not training time.** We use the min-cut primitive to bound the per-node count from above (a node can't see more distinct people than the subcarrier graph has min-cuts), then take a confidence-weighted vote.
|
||||
4. **Output is `{count, confidence, count_p95_low, count_p95_high}`**, not a single integer. Downstream consumers (Cogs / dashboard / alerts) can choose their certainty threshold. This is what closes the loop on the #499 UX: when the model is uncertain, the dashboard renders one stick figure with a "?" badge rather than two ghosts.
|
||||
5. **No new hardware.** Same ESP32-S3 1×1 SISO that ships today. The win comes from learned features + multi-node fusion, not from bigger antennas.
|
||||
|
||||
### Training (Candle / RTX 5080 / proven path)
|
||||
|
||||
Same exact pipeline that produced `pose_v1.safetensors` yesterday. Differences:
|
||||
|
||||
| | Pose cog (today) | Count cog (this ADR) |
|
||||
|---|---|---|
|
||||
| Input | `[56, 20]` CSI window | `[56, 20]` CSI window (identical) |
|
||||
| Encoder init | random (HF arch mismatch) | **from HF presence model** (architectures are compatible — same encoder Φ) |
|
||||
| Output head | `Linear(128 → 256 → 34)` keypoints | `Linear(128 → 64 → 8)` count classes + `Linear(128 → 32 → 1)` confidence |
|
||||
| Loss | Confidence-weighted SmoothL1 | Categorical cross-entropy + Brier-score uncertainty calibration |
|
||||
| Labels | MediaPipe keypoints | Camera count (MediaPipe `pose_landmarks` length) |
|
||||
| Data | 1,077 paired (P7) | **Same source, same script** — `collect-ground-truth.py` already records `n_persons` per frame |
|
||||
|
||||
Crucially we get the count labels **for free** from the existing pose data-collection pipeline — `collect-ground-truth.py` already records `"n_persons"` per camera frame and `align-ground-truth.js` already preserves it through windowing. No new data collection campaign required to bootstrap; we can train tomorrow on the same 1,077 samples that produced `pose_v1`.
|
||||
|
||||
### Multi-node fusion
|
||||
|
||||
The per-node count head + confidence head emit a categorical distribution over `{0..7}`. With N nodes, we have N such distributions plus N confidence scalars. Two fusion paths:
|
||||
|
||||
- **Confidence-weighted log-sum** (Bayesian product): `log p_fused(k) = Σ_n c_n · log p_n(k)`. Simple, no extra parameters, comes from the optimal-expert combination literature.
|
||||
- **Stoer-Wagner upper bound**: build a graph where edges are pairwise subcarrier-feature similarities between nodes. Min-cut size = a hard upper bound on the number of distinct people the node mesh can resolve. Clip the per-node-fused distribution to support `{0..min-cut}` before re-normalising. This is exactly what `ruvector-mincut` was added to the workspace for — it's been waiting for a counting consumer.
|
||||
|
||||
Both fuse cleanly. v0.1.0 ships the log-sum; v0.2.0 adds the min-cut clipper after the first round of evaluation.
|
||||
|
||||
### Why this beats today's heuristic
|
||||
|
||||
| Failure mode of today's slot heuristic | How the learned counter avoids it |
|
||||
|---|---|
|
||||
| #499 — fixed denominators clamp → one person renders as 2+ groups | Encoder produces a fixed-dim embedding; the count head is invariant to feature magnitude, only to feature **shape** |
|
||||
| `dedup_factor` per-room tuning is operator-visible toil | Count head's softmax is a learned per-room normaliser by construction |
|
||||
| Adding nodes makes the count noisier under the slot heuristic | Multi-node fusion is **additive in confidence**, so each node either reduces uncertainty or stays neutral — never amplifies it |
|
||||
| No per-frame uncertainty signal | `confidence` + `count_p95_low/high` exposed in every emit |
|
||||
| Catastrophic failure on novel environments | LoRA per-room adapter (per ADR-079 P9 plan) hot-swappable without retraining |
|
||||
|
||||
### Acceptance gates
|
||||
|
||||
| Gate | v0.1.0 (initial release) | v0.2.0 (after data scaling) |
|
||||
|------|--------------------------|------------------------------|
|
||||
| Day-0 deployment (no calibration) | ≥ 80% within ±1 on same-room test set | ≥ 90% within ±1 |
|
||||
| Cross-room (held-out environment) | ≥ 60% within ±1 | ≥ 75% within ±1 |
|
||||
| Mean Absolute Error | ≤ 0.6 persons | ≤ 0.4 persons |
|
||||
| Per-frame confidence reflects accuracy | Spearman correlation `r ≥ 0.5` between `confidence` and `(predicted == true)` | `r ≥ 0.7` |
|
||||
| Inference latency on Pi 5 (Cog) | < 5 ms / frame cold-start | < 5 ms / frame |
|
||||
| Binary size on GCS | ≤ 4 MB (matches `cog-pose-estimation`) | ≤ 4 MB |
|
||||
|
||||
`v0.1.0` is intentionally modest — it's bounded by data-collection scale (#645). The framework is the deliverable; the accuracy follows the data.
|
||||
|
||||
### Repo layout
|
||||
|
||||
```
|
||||
v2/crates/cog-person-count/ # NEW (this ADR)
|
||||
├── Cargo.toml
|
||||
├── src/
|
||||
│ ├── main.rs # cog runtime: version | manifest | health | run
|
||||
│ ├── lib.rs
|
||||
│ ├── inference.rs # Candle forward pass on per-node CSI
|
||||
│ ├── fusion.rs # Stoer-Wagner upper-bound + confidence-weighted log-sum
|
||||
│ └── publisher.rs # emits {count, confidence, count_p95_low, count_p95_high}
|
||||
├── cog/
|
||||
│ ├── manifest.template.json
|
||||
│ ├── config.schema.json
|
||||
│ ├── README.md
|
||||
│ └── artifacts/ # filled by the release pipeline
|
||||
│ ├── count_v1.safetensors
|
||||
│ ├── count_v1.onnx
|
||||
│ └── train_results.json
|
||||
└── tests/
|
||||
├── smoke.rs # 5+ tests
|
||||
└── fusion_test.rs # multi-node-fusion math
|
||||
```
|
||||
|
||||
Plus a small server-side wiring change:
|
||||
|
||||
- `v2/crates/wifi-densepose-sensing-server/src/csi.rs::score_to_person_count` — call the cog over the same `/api/v1/edge/registry`-discovered runtime as `cog-pose-estimation`. Falls back to today's PR #491 heuristic if the cog isn't installed (per the ADR-100 stub-fallback pattern).
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- Closes the conceptual loop opened by #499 — multi-person counting becomes a **learned task**, not a heuristic with a runtime knob.
|
||||
- Reuses every primitive already shipped this week: Candle GPU training (ADR-101), HF encoder, Cog packaging (ADR-100), edge module registry (ADR-102), Stoer-Wagner mincut, paired-data pipeline (PR #641).
|
||||
- Day-2 cross-room calibration uses the same LoRA path ADR-079 P9 plans for pose, so the two cogs share the same fine-tuning machinery.
|
||||
- Explicit `confidence` + `count_p95_low/high` outputs let the UI render uncertainty instead of inventing ghosts.
|
||||
|
||||
### Negative
|
||||
|
||||
- Accuracy is bounded by the same paired-data scarcity that bounds `pose_v1` (#645). Without more multi-room data, v0.1.0 ships with modest absolute accuracy.
|
||||
- Adds another Cog binary to maintain in the GCS catalog — 4 MB per arch.
|
||||
- The fusion-stage min-cut adds ~0.3 ms per N-node frame on a Pi 5 in microbenchmarks of `ruvector-mincut`. Acceptable given the ≤ 5 ms budget but worth tracking.
|
||||
|
||||
### Risks
|
||||
|
||||
- **Label noise**: MediaPipe pose-detection rate was 47% in the P7 session — half the frames have `n_persons = 0` even when a person was clearly in the room. The count head learns from this noisy signal; mitigations include filtering by `MediaPipe confidence ≥ 0.7` before training, and weighting the loss by confidence (same trick used in `pose_v1`).
|
||||
- **Encoder freezing too aggressive**: if 50 epochs of frozen-encoder training doesn't see the count head converge, unfreeze earlier. We have telemetry from `train_results.json` to make this call empirically.
|
||||
- **Min-cut over-constrains** in single-person scenarios: when N=1 the subcarrier graph has min-cut = 1 trivially. The fusion stage degrades to "trust the single-node count head", which is fine but worth a regression test (`tests/fusion_test.rs::single_node_degrades_gracefully`).
|
||||
|
||||
## Migration
|
||||
|
||||
1. Land this ADR + the new crate scaffold (one PR, no model yet — same approach as ADR-101's first PR shipped a stub cog).
|
||||
2. Train `count_v1.safetensors` on the existing 1,077 paired samples + `n_persons` labels. Same Candle pipeline that produced `pose_v1`.
|
||||
3. Cross-compile + sign + GCS upload per ADR-100. Live install on `cognitum-v0` per ADR-101's pattern.
|
||||
4. Wire `csi.rs::score_to_person_count` to call the cog when installed; keep PR #491's heuristic as fallback.
|
||||
5. v0.2.0: re-train on the multi-room data #645 motivates, add LoRA per-room adapters per ADR-079 P9.
|
||||
|
||||
## See also
|
||||
|
||||
- ADR-079 — Camera-supervised training pipeline (same data path).
|
||||
- ADR-100 — Cognitum Cog packaging spec (same shipping format).
|
||||
- ADR-101 — Pose Estimation Cog (template for this Cog's first release).
|
||||
- ADR-102 — Edge Module Registry (where this cog appears in the catalog).
|
||||
- PR #491 — RollingP95 + `dedup_factor` (the heuristic this learned counter replaces).
|
||||
- Issue #499 — Multi-node ghost skeletons (closed by #491, motivates this ADR).
|
||||
- Issue #645 — PCK / data-collection plan (same data-bound limit; same fix path).
|
||||
- `docs/benchmarks/pose-estimation-cog.md` — measured perf envelope for the cog runtime this ADR targets.
|
||||
@@ -0,0 +1,185 @@
|
||||
# `cog-person-count` — Benchmark Log
|
||||
|
||||
Append-only log of every published count_v1 training run per ADR-103. New runs add a section; never overwrite history.
|
||||
|
||||
## v0.0.2 — K-fold validated, random split + label smoothing + early stop + temp scale (2026-05-21)
|
||||
|
||||
### Why a new release
|
||||
|
||||
A 5-fold stratified CV on the same 1,077 samples proved the v0.0.1 result was driven by an unlucky temporal split — the trailing window was class-0-heavy, and a degenerate "always predict 0" classifier hit the class-0 fraction (65.1%) trivially.
|
||||
|
||||
| Metric | v0.0.1 (temporal) | **5-fold random CV** (diagnostic) |
|
||||
|---|---|---|
|
||||
| Overall accuracy | 65.1% | 62.2% ± 1.9% |
|
||||
| Class 1 accuracy | **0%** | **57.1%** ✓ |
|
||||
| Confidence Spearman | 0.023 | 0.160 ± 0.029 |
|
||||
|
||||
The architecture has real ~57% class-1 capacity under fair splits.
|
||||
|
||||
### v0.0.2 results
|
||||
|
||||
Architecture unchanged. Training changes only:
|
||||
- **Random 80/20 split** (seed=42) — temporal split eliminated.
|
||||
- **Label smoothing 0.1** on cross-entropy.
|
||||
- **Class-balanced multinomial sampler** with replacement.
|
||||
- **Early stopping** with patience 20 (exited at epoch 29 of 400 max).
|
||||
- **Temperature scaling** of the conf head via LBFGS — T = **0.9262**, shipped as a `count_v1.temperature` sidecar.
|
||||
|
||||
| Metric | v0.0.1 | **v0.0.2** | K-fold ref |
|
||||
|---|---|---|---|
|
||||
| Overall accuracy | 65.1% | **62.3%** | 62.2% ± 1.9% |
|
||||
| Class 0 accuracy | 100% (cheating) | **86.2%** | 67.4% |
|
||||
| **Class 1 accuracy** | **0%** | **34.3%** ✓ | 57.1% |
|
||||
| MAE | 0.349 | 0.377 | 0.378 |
|
||||
| Confidence Spearman (post-temp) | 0.023 | 0.013 | 0.160 |
|
||||
| Wall time | 5.6 s (400 ep) | **0.7 s (29 ep)** | 7.5 s (5×100) |
|
||||
|
||||
### Honest read
|
||||
|
||||
**Class-1 accuracy 0% → 34.3% is the headline.** The cog now reports `count = 1` honestly when a person is present, instead of always-zero cheating. Single random draw lands below the K-fold mean of 57% — that gap is run-to-run variance, not a missing improvement. Reaching 57% on a fixed eval set needs averaging over independent draws, which means more independent recordings — i.e. multi-room data (#645), not another training trick.
|
||||
|
||||
Confidence calibration didn't move. Temperature scaling alone can't fix a confidence head trained against a noisy `argmax==truth` indicator over a 62%-accurate classifier — its training signal is the bottleneck.
|
||||
|
||||
### Release artifacts (live on cognitum-v0)
|
||||
|
||||
```
|
||||
gs://cognitum-apps/cogs/arm/cog-person-count-count_v1.safetensors
|
||||
sha256: 32996433516891a37c63c600db8b95e42192a53bd538c088c82cd6a85e55513c
|
||||
bytes: 392,088
|
||||
```
|
||||
|
||||
Binaries themselves unchanged from v0.0.1 — weights load at runtime via mmap. Per-arch manifests under `cog/artifacts/manifests/{arm,x86_64}/` bumped to `version: 0.0.2`, weights_sha256 + build_metadata caveats updated.
|
||||
|
||||
### Reproducibility
|
||||
|
||||
```bash
|
||||
python3 scripts/train-count.py --paired data/paired/wiflow-p7-1779210883.paired.jsonl \
|
||||
--k-fold 5 --epochs 100 --out-results kfold_results.json
|
||||
|
||||
python3 scripts/train-count.py --paired data/paired/wiflow-p7-1779210883.paired.jsonl \
|
||||
--v2 --epochs 400 \
|
||||
--out-safetensors count_v1.safetensors --out-onnx count_v1.onnx \
|
||||
--out-results count_train_results.json
|
||||
```
|
||||
|
||||
## v0.0.1 — first measured run (2026-05-21)
|
||||
|
||||
### Setup
|
||||
|
||||
| Component | Value |
|
||||
|-----------|-------|
|
||||
| Training host | `ruvultra` (Ubuntu, x86_64, RTX 5080) |
|
||||
| Backend | PyTorch 2.12 + CUDA |
|
||||
| Data | `data/paired/wiflow-p7-1779210883.paired.jsonl` — 1,077 paired samples, single 30-min session, label distribution `{0: 533, 1: 544}` |
|
||||
| Train/eval split | 80/20 stratified on `ts_start` (held-out tail of the recording) |
|
||||
| Architecture | Conv1d encoder (56→64→128→128, dilations 1/2/4) + Linear(128→64→8) count head + Linear(128→32→1) confidence head — bit-identical to `v2/crates/cog-person-count/src/inference.rs::CountNet` |
|
||||
| Loss | `cross_entropy(count) + 0.3·BCE(conf) + 0.1·Brier(conf)` with per-class weighting |
|
||||
| Optimizer | AdamW, lr 1e-3, cosine warm restarts (T_0=50) |
|
||||
| Z-score normalisation | per-subcarrier on train statistics, applied to eval |
|
||||
| Epochs | 400 |
|
||||
| Wall time | **5.6 s** |
|
||||
|
||||
### Accuracy (held-out 215-sample tail of the 30-min recording)
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Best eval accuracy | **65.1%** |
|
||||
| Final eval accuracy | 65.1% |
|
||||
| Within ±1 | **100%** (labels are all in `{0, 1}`, predictions trivially within ±1) |
|
||||
| MAE | 0.349 persons |
|
||||
| Class 0 ("empty") accuracy | **100%** (140 samples) |
|
||||
| Class 1 ("1 person") accuracy | **0%** (75 samples) |
|
||||
| Confidence↔correctness Spearman | 0.023 |
|
||||
|
||||
### Honest read
|
||||
|
||||
The model overfit hard. By epoch 100 train_acc reached 1.0 and eval_loss climbed from 0.67 → 7.8. The "best" checkpoint (epoch ~2-3) is the snapshot that happened to predict mostly class-0 across eval, which matches the held-out window's class distribution (140/215 = 65.1%) — i.e. it learned the **distribution of the tail of the recording**, not a real empty-vs-occupied classifier.
|
||||
|
||||
Why: the training data is one continuous 30-minute solo recording. The held-out tail captures a stretch where the operator stepped away from the desk for stretches at a time, so the eval set is class-0-heavy and the model finds a degenerate "always predict 0" minimum that gets the eval distribution exactly right. Class 1 accuracy = 0 is the smoking gun.
|
||||
|
||||
Same data-bound failure mode as `pose_v1` (#645). Same fix path: multi-room paired recordings.
|
||||
|
||||
### What v0.0.1 still validates
|
||||
|
||||
- **Pipeline correctness end-to-end.** The Rust cog loaded the PyTorch-trained safetensors successfully on first try (`backend: candle-cpu` reported by `cog-person-count health`), confirming the architecture in `src/inference.rs` is byte-compatible with `train-count.py`.
|
||||
- **ONNX parity.** 16 KB ONNX, exports cleanly under opset 18 with dynamic batch axis.
|
||||
- **Fast iteration loop.** 5.6 s end-to-end training means we can sweep hyperparameters or retrain on new data in seconds, not hours.
|
||||
- **Cog binary size.** Same 2.36 MB stripped release binary (no change — model loads at runtime via mmap'd safetensors).
|
||||
|
||||
### Comparison to ADR-103 v0.1.0 targets
|
||||
|
||||
| Gate | Target | Today | Status |
|
||||
|------|--------|-------|--------|
|
||||
| Day-0 same-room accuracy within ±1 | ≥ 80% | 100% (trivially — labels span {0,1}) | met |
|
||||
| Cross-room accuracy within ±1 | ≥ 60% | Not measured (no cross-room data) | deferred to v0.2.0 |
|
||||
| MAE | ≤ 0.6 | 0.349 | met |
|
||||
| Per-frame confidence reflects accuracy (Spearman) | r ≥ 0.5 | 0.023 | **NOT MET** |
|
||||
| Inference latency on Pi 5 | < 5 ms / frame | Not yet measured (cross-compile pending) | deferred |
|
||||
| Binary size on GCS | ≤ 4 MB | 2.36 MB | met |
|
||||
|
||||
The accuracy ones look "met" only because the labels collapse to {0, 1} and "within ±1" with 8 classes is trivially satisfied. The **confidence calibration is the real failure** for v0.0.1 — Spearman 0.023 means the confidence head is essentially random noise. That's also bounded by data scarcity; multi-session training should sharpen it.
|
||||
|
||||
### Artifacts
|
||||
|
||||
- `v2/crates/cog-person-count/cog/artifacts/count_v1.safetensors` — 392 KB
|
||||
- `v2/crates/cog-person-count/cog/artifacts/count_v1.onnx` — 16 KB
|
||||
- `v2/crates/cog-person-count/cog/artifacts/count_train_results.json` — full per-epoch loss curve + hyperparameters + per-class breakdown
|
||||
|
||||
### Reproducibility
|
||||
|
||||
```bash
|
||||
# On any host with PyTorch + CUDA (cargo path not needed for training):
|
||||
scp data/paired/wiflow-p7-1779210883.paired.jsonl <host>:/tmp/
|
||||
scp scripts/train-count.py <host>:/tmp/
|
||||
ssh <host> "cd /tmp && python3 train-count.py --paired wiflow-p7-1779210883.paired.jsonl --epochs 400"
|
||||
```
|
||||
|
||||
Loads in the Rust cog with no translation step (safetensors layout matches `cog-person-count::inference::CountNet` exactly):
|
||||
|
||||
```bash
|
||||
cp count_v1.safetensors v2/crates/cog-person-count/cog/artifacts/
|
||||
cargo run -p cog-person-count --release -- health
|
||||
# → {"backend":"candle-cpu", "synthetic_count": <int>, "synthetic_confidence": <float>, ...}
|
||||
```
|
||||
|
||||
### Live appliance install (cognitum-v0 Pi 5)
|
||||
|
||||
Installed at `/var/lib/cognitum/apps/person-count/` with the same on-disk shape as `cog-pose-estimation`, `anomaly-detect`, `seizure-detect`, etc.:
|
||||
|
||||
```
|
||||
$ ls -la /var/lib/cognitum/apps/person-count/
|
||||
-rwxr-xr-x cog-person-count-arm 2,168,816 B (sha matches GCS)
|
||||
-rw-r--r-- count_v1.safetensors 392,088 B
|
||||
-rw-r--r-- manifest.json 1,073 B
|
||||
-rw-r--r-- config.json 160 B
|
||||
```
|
||||
|
||||
```
|
||||
$ ./cog-person-count-arm health
|
||||
{"ts": ..., "event": "health.ok",
|
||||
"fields": {"backend": "candle-cpu", "synthetic_count": 0,
|
||||
"synthetic_confidence": 0.49, "synthetic_p95_range": [0, 7]}}
|
||||
```
|
||||
|
||||
Cold-start on real Pi 5 hardware: **9.2 ms / invocation** (30 sequential `health` invocations in 0.276 s). Slightly slower than the pose cog (8.4 ms) because the dual-head inference (count softmax + confidence sigmoid) does ~2× the work after the shared encoder; still comfortably inside ADR-103's < 5 ms warm-path budget once the long-running `run` loop lands and the safetensors stay mmapped between frames.
|
||||
|
||||
### Signed GCS release artifacts (publicly downloadable)
|
||||
|
||||
```
|
||||
gs://cognitum-apps/cogs/arm/cog-person-count-arm 2,168,816 B
|
||||
sha256: 36bc0bb0ece894350377d5f93d46cd29378cb289b3773530611c0d47b507b3c3
|
||||
signature: R/00xdzHriyr/2rzr4wmPJ/Ken60A+RNdi8r0g2HYJNTXBaFtr46ExfNbiHlgYWadQXzTZdfJoyJK+a6k71NDg==
|
||||
|
||||
gs://cognitum-apps/cogs/x86_64/cog-person-count-x86_64 2,615,528 B
|
||||
sha256: 76cdd1ec40211add90b4942a09f79939aa28210a27e931de67122357392b01db
|
||||
signature: QB+8cnGSMQmubSt/KWVu1+JMg37AKnQXDsFQi/vi+jqpW9rVrGMtnxQpWEWZPeWU1AJ6pl3O2V+7ZtTNIQ2rDg==
|
||||
|
||||
gs://cognitum-apps/cogs/arm/cog-person-count-count_v1.safetensors 392,088 B
|
||||
sha256: dacb0551fd3887958db19696d90d811ab08faa44703e6e04ff56d15c3a65a9ff
|
||||
```
|
||||
|
||||
All signed with `COGNITUM_OWNER_SIGNING_KEY` (Ed25519). SHAs verified via public anonymous `https://storage.googleapis.com/...` download.
|
||||
|
||||
Manifests at:
|
||||
- `v2/crates/cog-person-count/cog/artifacts/manifests/arm/manifest.json`
|
||||
- `v2/crates/cog-person-count/cog/artifacts/manifests/x86_64/manifest.json
|
||||
@@ -0,0 +1,176 @@
|
||||
# `cog-pose-estimation` — Benchmark Log
|
||||
|
||||
This file tracks every published benchmark for the pose-estimation Cog. New runs append; never overwrite history. Per ADR-101 §"Acceptance gates".
|
||||
|
||||
## v0.0.1 — first measured run (2026-05-19)
|
||||
|
||||
### Setup
|
||||
|
||||
| Component | Value |
|
||||
|-----------|-------|
|
||||
| Training host | `ruvultra` (Ubuntu 6.17, x86_64, RTX 5080) |
|
||||
| Backend | `candle-core 0.9` with `cuda` feature |
|
||||
| Data | `data/paired/wiflow-p7-1779210883.paired.jsonl` — 1,077 paired samples, 30-min seated-at-desk recording, avg conf 0.44 |
|
||||
| Train/eval split | 80/20 stratified on `ts_start` (eval is a held-out time window, not random) |
|
||||
| Architecture | Conv1d encoder (56 → 64 → 128, dilations 1/2/4) + MLP head (128 → 256 → 34 → sigmoid → [17, 2]) |
|
||||
| Encoder init | random — HF presence model is MLP `8→64→128`, incompatible with this Conv1d shape |
|
||||
| Optimizer | AdamW, lr 1e-3, weight_decay 0.01 |
|
||||
| LR schedule | Cosine with 50-epoch warm restarts |
|
||||
| Loss | SmoothL1 (Huber β=0.1), confidence-weighted by `record.conf` |
|
||||
| Augmentation | Subcarrier dropout 10% (final 50 epochs) |
|
||||
| Epochs | 400 (full-batch) |
|
||||
| Wall time | **2.1 s** total |
|
||||
|
||||
### Accuracy
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| **PCK@20** (overall) | **3.0%** |
|
||||
| **PCK@50** (overall) | **18.5%** |
|
||||
| **MPJPE** (normalized) | **0.0931** |
|
||||
| Final eval loss | 0.0101 |
|
||||
| Loss reduction | 0.181 → 0.014 (13×) |
|
||||
|
||||
### Per-joint PCK
|
||||
|
||||
| Joint | PCK@20 | PCK@50 | | Joint | PCK@20 | PCK@50 |
|
||||
|-------|-------:|-------:|--|-------|-------:|-------:|
|
||||
| nose | 0.5% | 5.1% | | l_hip | 0.0% | 27.3% |
|
||||
| l_eye | 2.8% | 8.3% | | **r_hip** | **25.0%** | **76.9%** |
|
||||
| r_eye | 1.9% | 15.7% | | l_knee | 2.3% | 20.8% |
|
||||
| l_ear | 0.0% | 3.2% | | r_knee | 0.9% | 35.2% |
|
||||
| r_ear | 1.9% | 9.7% | | l_ankle | 1.4% | 7.9% |
|
||||
| l_shoulder | 4.6% | 8.8% | | r_ankle | 0.9% | 9.3% |
|
||||
| r_shoulder | 1.9% | 19.9% | | l_elbow | 1.9% | 26.4% |
|
||||
| l_wrist | 3.2% | 24.1% | | r_elbow | 0.0% | 4.2% |
|
||||
| r_wrist | 1.4% | 12.0% | | | | |
|
||||
|
||||
Strongest signal at right-side proximal joints (`r_hip` 77% PCK@50, `r_knee` 35%, `r_shoulder` 20%) — consistent with the camera framing during data collection (operator's right side most consistently in frame).
|
||||
|
||||
### Comparison to prior baseline
|
||||
|
||||
| Run | Backend | Train time | PCK@20 | PCK@50 | MPJPE |
|
||||
|-----|---------|-----------:|-------:|-------:|------:|
|
||||
| pre-2026-05-19 | pure-JS SPSA, lite TCN (#645) | ~20 min | 0.0% | 0.0% | 0.66 |
|
||||
| **v0.0.1** (this run) | **candle-cuda, Conv1d TCN** | **2.1 s** | **3.0%** | **18.5%** | **0.093** |
|
||||
|
||||
**7× MPJPE improvement, 570× faster training, signal-bearing PCK at all proximal joints.** The remaining gap to ADR-079's PCK@20 ≥ 35% target is data-bound, not infra-bound (see Issue #645).
|
||||
|
||||
### Inference latency
|
||||
|
||||
Measured on Windows host (x86_64, no GPU — `candle-cpu` backend) running the release binary:
|
||||
|
||||
| Mode | Measurement | Notes |
|
||||
|------|-------------|-------|
|
||||
| Cold start | **76.2 ms / invocation** (avg over 100 sequential `health` invocations) | Includes safetensors load + 1 synthetic forward pass. Most of the cost is process startup + mmap. |
|
||||
| Long-running `run` warm inference | sub-millisecond per frame (estimated) | The model is 125K params / 507 KB; once loaded, a single forward at batch=1 is essentially memory-bandwidth bound. To be measured precisely against a live sensing-server feed. |
|
||||
|
||||
### ONNX export
|
||||
|
||||
`pose_v1.onnx` is produced from `pose_v1.safetensors` by `scripts/export-onnx.py`, which mirrors the Candle architecture in PyTorch, loads the safetensors weights, and uses `torch.onnx.export` with opset 18 + dynamic batch axis. Verified end-to-end:
|
||||
|
||||
| Check | Result |
|
||||
|-------|--------|
|
||||
| `onnx.checker.check_model` | ✅ ok |
|
||||
| Parity vs torch reference | **max \|torch − onnx\| = 8.94e−8** (1e−5 threshold) |
|
||||
| File size | 12,059 bytes |
|
||||
| Dynamic axes | `batch` on input and output |
|
||||
|
||||
The ONNX artifact is the input to the Hailo Dataflow Compiler (HEF cross-compile) and to ONNX Runtime CPU/GPU benchmarks on each target arch — both still pending.
|
||||
|
||||
### Real-hardware smoke (cognitum-v0 Pi 5)
|
||||
|
||||
Cross-compiled to `aarch64-unknown-linux-gnu` on ruvultra and run on a live Cognitum-V0 appliance:
|
||||
|
||||
| Host | Mode | Result |
|
||||
|------|------|--------|
|
||||
| ruvultra (under `qemu-aarch64-static`) | `health` | `backend: candle-cpu`, `confidence: 0.185` — real weights loaded under emulation |
|
||||
| **cognitum-v0** (Raspberry Pi 5, Cortex-A76) | `health` | `backend: candle-cpu`, `confidence: 0.185` — real weights, real hardware |
|
||||
| cognitum-v0 | 30× sequential `health` invocations | **0.251 s total → 8.4 ms / invocation** (cold) |
|
||||
|
||||
8.4 ms cold-start on real Pi 5 hardware vs 76 ms on the x86_64 Windows host. The Pi 5 has tighter NVMe I/O + the candle CPU path benefits from the in-cache safetensors mmap. Long-running `run` warm inference will still be sub-millisecond.
|
||||
|
||||
### Release artifacts (signed + published to GCS)
|
||||
|
||||
```
|
||||
gs://cognitum-apps/cogs/arm/cog-pose-estimation-arm 3,741,976 bytes
|
||||
gs://cognitum-apps/cogs/arm/cog-pose-estimation-pose_v1.safetensors 507,032 bytes
|
||||
|
||||
binary_sha256: 1e1a7d3dd01ca05d5bfc5dbb142a5941b7866ed9f3224a21edc04d3f09a99bf5
|
||||
weights_sha256: eb249b9a6b2e10130437a10976ed0230b0d085f86a0553d7226e1ae6eae4b9e5
|
||||
signature: LUN7xqLPYD3MFzm5dKB5MnYU0LvoRtek5ci5KiKPHBg+Xo6xuazwokn2Dw2JPMaLYJzmWn/SpT4djuR7hYvVDw== (Ed25519, signed with COGNITUM_OWNER_SIGNING_KEY)
|
||||
```
|
||||
|
||||
Full manifest at `cog/artifacts/manifest.json`. Verified via public anonymous GET against `https://storage.googleapis.com/cognitum-apps/cogs/arm/cog-pose-estimation-arm` — downloaded SHA matches the locally-computed SHA.
|
||||
|
||||
### Live appliance install
|
||||
|
||||
Installed on `cognitum-v0` (the V0 cluster leader) at `/var/lib/cognitum/apps/pose-estimation/`:
|
||||
|
||||
```
|
||||
$ ls -la /var/lib/cognitum/apps/pose-estimation/
|
||||
-rwxr-xr-x cog-pose-estimation-arm 3,741,976 B (matches GCS sha256)
|
||||
-rw-r--r-- pose_v1.safetensors 507,032 B
|
||||
-rw-r--r-- manifest.json 989 B
|
||||
-rw-r--r-- config.json 187 B
|
||||
-rw-r--r-- output.log 28,438 B (5-sec smoke run)
|
||||
```
|
||||
|
||||
Layout matches the existing `anomaly-detect`, `presence`, `seizure-detect`, etc. cogs on the same appliance — the Cogs dashboard at `http://cognitum-v0:9000/cogs` auto-discovers entries under this dir.
|
||||
|
||||
`cog-pose-estimation run` ran cleanly in the background for 5 seconds with the default config. It correctly:
|
||||
|
||||
- Emitted a `run.started` event with the configured `sensing_url`, `model_path`, and `poll_ms`.
|
||||
- Started its 40 ms poll loop.
|
||||
- **Gracefully handled the missing local sensing-server on port 3000** by logging structured WARN events (`{"level":"WARN","fields":{"message":"sensing-server fetch failed","error":"...Connection refused..."}}`) without crashing, leaking, or producing NaN output.
|
||||
- Exited cleanly on SIGTERM.
|
||||
|
||||
0 `pose.frame` events fired during the smoke run — expected, since `127.0.0.1:3000` isn't serving CSI on the appliance. The appliance's actual CSI source is `ruview-vitals-worker` on `:50054` plus the `/api/v1/v0/system/...` endpoints behind the appliance's bearer auth on `:9000`. Wiring `sensing_url` to the appliance-native source is a Day-2 integration task — separate from the cog binary itself.
|
||||
|
||||
Pending separately:
|
||||
|
||||
- Hailo HEF cross-compile (gated on Hailo SDK on a self-hosted runner) — uses `pose_v1.onnx` as input.
|
||||
- Appliance-native sensing-source integration (`config.sensing_url` should point at the cog-gateway's CSI tap on `:9000`, not the dev-loopback `:3000`).
|
||||
### x86_64 release (2026-05-19)
|
||||
|
||||
Built on ruvultra (native, no cross-compile):
|
||||
|
||||
```
|
||||
gs://cognitum-apps/cogs/x86_64/cog-pose-estimation-x86_64 4,548,856 bytes
|
||||
sha256: a434739a24415b34e1aff50e5e1c3c32e568db96af473bbb3e5ecc9b95fe71fa
|
||||
signature: pNNuxhgM18PztN8BSZdfw5oAShG2pV3na5T/q2QdlJWX/5FJgo4QTiUCbcTAxI2Uiva8VURSOlRzMU3xoQPqCQ==
|
||||
```
|
||||
|
||||
Manifest at `cog/artifacts/manifests/x86_64/manifest.json`. Re-uses the same `pose_v1.safetensors` weights as the arm release (architecture is arch-independent).
|
||||
|
||||
**Cold-start: 5.4 ms / invocation** on ruvultra (30× sequential `health` in 0.162 s) — faster than the Pi 5's 8.4 ms (faster NVMe + wider CPU), slower than the Windows 76 ms (less mature Windows release toolchain).
|
||||
|
||||
| Host | arch | rust | binary | cold-start |
|
||||
|------|------|------|--------|------------|
|
||||
| Windows (ruvzen) | x86_64 | 1.95.0 | (built locally, not published) | 76.2 ms |
|
||||
| ruvultra (Ubuntu) | x86_64 | 1.89.0 | 4,548,856 B (GCS x86_64) | **5.4 ms** |
|
||||
| cognitum-v0 (Pi 5) | aarch64 | (cross-built) | 3,741,976 B (GCS arm) | 8.4 ms |
|
||||
|
||||
### Artifacts
|
||||
|
||||
- `v2/crates/cog-pose-estimation/cog/artifacts/pose_v1.safetensors` — 507 KB
|
||||
- `v2/crates/cog-pose-estimation/cog/artifacts/train_results.json` — full per-epoch loss curve + hyperparameters + per-joint PCK
|
||||
|
||||
### Reproducibility
|
||||
|
||||
```bash
|
||||
# On any host with cargo + a CUDA-capable GPU:
|
||||
cd ~/work/cog-pose-train
|
||||
mkdir -p ./
|
||||
# Stage the same inputs (1,077 paired samples + HF encoder, see scripts/align-ground-truth.js for regeneration)
|
||||
cp paired.jsonl ./paired.jsonl
|
||||
cp encoder.safetensors ./encoder.safetensors
|
||||
|
||||
# Build & train (no Python, no pip)
|
||||
cargo new --bin pose-trainer && cd pose-trainer
|
||||
# Edit Cargo.toml deps: candle-core 0.9 (cuda), candle-nn 0.9 (cuda), safetensors, serde, serde_json, anyhow
|
||||
# Drop the training script into src/main.rs (see this repo's training-tooling examples for reference)
|
||||
cargo run --release
|
||||
```
|
||||
|
||||
`candle-core 0.8.4 + 0.9.2` are typically already in `~/.cargo/registry/cache/` on any developer host, so the build completes in seconds.
|
||||
@@ -0,0 +1,68 @@
|
||||
# SOTA Research Loop — 2026-05-22
|
||||
|
||||
Started: 2026-05-21 ~20:00 ET. **Auto-stops: 2026-05-22 08:00 ET.** Cron `d6e5c473` (`*/10 * * * *`).
|
||||
|
||||
## Mandate
|
||||
|
||||
Push WiFi-CSI sensing past 2026 published SOTA in three axes:
|
||||
|
||||
1. **Spatial intelligence** — multi-static fusion, room-scale awareness, occupancy beyond counting
|
||||
2. **RF feature engineering** — phase, ToA, subcarrier dynamics, Fresnel zones
|
||||
3. **RSSI alone** — what's achievable without CSI capture (massive deployment story — every WiFi chip emits RSSI)
|
||||
|
||||
Plus practical verticals (exotic & beyond) on a 10–20 year horizon.
|
||||
|
||||
Output goes to `docs/research/sota-2026-05-22/` (research notes, benchmarks, negative results) + `examples/research-sota/` (runnable code).
|
||||
|
||||
## Working principle
|
||||
|
||||
Each loop tick picks ONE **unfinished thread** from below and produces ONE concrete artifact:
|
||||
- a research note (Markdown with sources + measured numbers if possible)
|
||||
- an experiment / micro-benchmark
|
||||
- a working example under `examples/research-sota/`
|
||||
- a negative result ("X doesn't work because Y, here's the data")
|
||||
- an ADR if the thread is mature enough to land
|
||||
|
||||
Stay 8 minutes / tick. Commit + PR + auto-merge per piece. Future-tick re-entry is via this PROGRESS.md.
|
||||
|
||||
## Research vectors
|
||||
|
||||
### Spatial Intelligence
|
||||
|
||||
- [ ] **R1. Multi-static Time-of-Arrival (ToA) from OFDM phase coherence.** Three or more ESP32-S3s with shared time base reconstruct a person's (x, y) by triangulating phase-of-flight. 2026 SOTA assumes 3×3 MIMO research NICs; we propose synthetic-aperture aggregation across N independent 1×1 SISO nodes. Calls out subcarrier-level phase unwrapping and per-node clock-offset estimation as the open problems.
|
||||
- [ ] **R2. Persistent room field model — eigenstructure perturbation.** Already in `wifi-densepose-signal/src/ruvsense/field_model.rs` (SVD on empty-room CSI). Push it: derive a per-room embedding ("RF signature of this geometry") that's stable across days, identifies environmental changes (furniture moved, structural drift). Vertical: building-integrity monitoring.
|
||||
- [ ] **R3. Cross-room re-identification via gait CSI signatures.** Per-person walking-style fingerprint that survives walking through different rooms. Different from `AETHER` (in-room re-ID) — this is *inter*-room continuity.
|
||||
- [ ] **R4. Federated learning of room models.** Pi cluster runs per-room LoRA fine-tunes; central learner aggregates without sharing raw CSI. Privacy-preserving spatial intelligence.
|
||||
|
||||
### RF Feature Engineering
|
||||
|
||||
- [ ] **R5. Subcarrier attention over time → "RF saliency map".** Visualize which subcarriers carry the most information per task. ADR-097 hints at this; nothing in repo computes it. Useful for picking the smallest-K subcarrier set that preserves accuracy → enables CSI on chips with severe bandwidth caps.
|
||||
- [ ] **R6. Fresnel-zone forward model for through-wall sensing.** Code in `wifi-densepose-signal/src/ruvsense/tomography.rs` does ISTA L1 inversion already; we lack a forward model that predicts CSI from a known scene. Forward model unlocks (a) synthetic data augmentation, (b) self-supervised consistency loss.
|
||||
- [ ] **R7. Quantum-inspired Stoer-Wagner sampling for adversarial robustness.** Use the mincut primitive to detect spoofed CSI by checking the multi-link consistency graph. Lands in `cognitum-rvcsi` if it works.
|
||||
|
||||
### RSSI Alone (no CSI)
|
||||
|
||||
- [ ] **R8. RSSI-only presence + vitals.** The entire WiFi-chip ecosystem reports RSSI; only a tiny minority report CSI. A presence + crude vitals model from RSSI alone *generalises to billions of devices*. Hard problem (very low information rate) but enormous downstream value. Start with literature survey + first model experiment.
|
||||
- [ ] **R9. RSSI fingerprint topology — graph neural network on WiFi-scan beacons.** Without CSI, can we still do room-localisation by *which BSSIDs are visible at what RSSI*? Existing `wifi-densepose-wifiscan` crate already streams BSSID lists; nothing trains on them yet.
|
||||
|
||||
### Exotic & Future (10–20 year)
|
||||
|
||||
- [ ] **R10. Through-foliage wildlife sensing.** Same physics as through-wall, but at much lower SNR. Gait recognition on a per-species basis. Practical: non-invasive population monitoring without cameras.
|
||||
- [ ] **R11. Through-bulkhead maritime crew tracking.** Steel attenuates but doesn't eliminate WiFi multipath. Limited range, requires per-vessel calibration.
|
||||
- [ ] **R12. RF "weather" mapping.** Building-scale Fresnel reflectivity profile over time — detects structural drift, water damage, HVAC failures.
|
||||
- [ ] **R13. Contactless blood pressure from sub-mm chest displacement.** Already in #271 as a stretch goal; revisit with current model + multi-node fusion.
|
||||
- [ ] **R14. Empathic appliances.** Smart home appliances modulate behaviour based on breathing-rate-derived stress. Long-horizon — needs both the sensing accuracy *and* an ethical framework.
|
||||
- [ ] **R15. RF biometric across rooms.** Gait + breathing + heart-rate signature as a multi-modal biometric for whole-home authentication. Replaces fingerprint/face on the home-network layer.
|
||||
|
||||
## Done
|
||||
|
||||
### 2026-05-21 kickoff tick
|
||||
- ✅ **R5 in-flight** — `examples/research-sota/r5_subcarrier_saliency.py` runs; first measurement on `cog-person-count` v0.0.2 ships: top-8 subcarriers spread across the band, max/mean ratio 2.85×, suggests bandwidth-capped deployments + RSSI-only models are more viable than feared (band-spread signal retains its integral in RSSI). See `R5-subcarrier-saliency.md` §"First measurement" + §"Implications".
|
||||
|
||||
## Negative results
|
||||
|
||||
(populated when we discover something doesn't work — these are explicit, not failures)
|
||||
|
||||
## Index by date
|
||||
|
||||
- 2026-05-21 — kickoff (this file)
|
||||
@@ -0,0 +1,70 @@
|
||||
# R5 — Subcarrier saliency: which CSI dimensions actually carry the signal?
|
||||
|
||||
**Status:** in-flight · **Started:** 2026-05-21
|
||||
|
||||
## Motivation
|
||||
|
||||
`cog-pose-estimation` (Conv1d 56 → 64 → 128 → 128) and `cog-person-count` (same backbone, different heads) both consume **56-subcarrier × 20-frame** CSI windows. The 56 came from the upstream `align-ground-truth.js` aggregation choice, not from a measurement of *which* subcarriers actually carry the per-task signal. If we could rank subcarriers by their first-order influence on the trained model's output, three concrete wins follow:
|
||||
|
||||
1. **Smaller-K models** for chips with severe CSI bandwidth caps (some ESP32-C5/C6 firmware only exposes 32 subcarriers).
|
||||
2. **Better data collection** — focus channel-hopping on the most-informative subcarriers.
|
||||
3. **Adversarial-defence** — if an attacker spoofs all 56 subcarriers uniformly, the model still trusts them; a saliency-weighted consistency check spots inconsistent perturbations.
|
||||
|
||||
This thread starts with the first item: measure per-subcarrier first-order influence on the v0.0.2 count model + the v0.0.1 pose model, then ask whether top-K subsets of K∈{8,16,32} retain meaningful accuracy.
|
||||
|
||||
## Method (single-tick scope)
|
||||
|
||||
For each model:
|
||||
|
||||
1. Load the trained safetensors (`cog/artifacts/count_v1.safetensors` and `cog/artifacts/pose_v1.safetensors`).
|
||||
2. Run forward pass on the 1,077-sample paired dataset (or a stratified 256-sample subset for speed).
|
||||
3. Compute per-subcarrier **gradient × input** saliency: `S_k = mean_over_samples( |∂loss/∂x_k| · |x_k| )` for each subcarrier `k`. This is the standard "input × gradient" saliency from Sundararajan et al. (Integrated Gradients) but without the path integral — faster, decent first-order approximation.
|
||||
4. Plot the 56-element saliency vector for each model. Identify top-K.
|
||||
5. Re-train each model on the top-K subcarriers only (K ∈ {8, 16, 32}). Compare accuracy.
|
||||
|
||||
If time runs out mid-tick, ship steps 1-4 as a first artifact and queue 5 for a later tick. Steps 1-4 alone produce a real result (a ranked-subcarrier list per task).
|
||||
|
||||
## Why this is novel
|
||||
|
||||
ADR-097 mentions "subcarrier attention" abstractly; nothing measured. Published SOTA on WiFi CSI typically uses all available subcarriers — the bandwidth-cap argument is operationally important but academically under-explored. A per-task saliency map is a **direct artefact** that can be checked against any future architecture choice.
|
||||
|
||||
## Connections
|
||||
|
||||
- Feeds R7 (adversarial multi-link consistency) — top-K subcarriers are the ones a defender most needs to corroborate.
|
||||
- Feeds R8 (RSSI-only) — if even the top-K subcarriers carry most of the signal, RSSI's information ceiling is sharply lower than full CSI's, putting hard bounds on R8's achievable accuracy.
|
||||
|
||||
## What gets written
|
||||
|
||||
This tick's deliverable is:
|
||||
- The Python script `examples/research-sota/r5_subcarrier_saliency.py` that computes the saliency vector for either model.
|
||||
- A first measurement (text + JSON) of saliency for the count model.
|
||||
|
||||
Step 5 (retrain on top-K) is queued for a subsequent tick.
|
||||
|
||||
## First measurement — `cog-person-count` v0.0.2 (this tick, 128 samples)
|
||||
|
||||
| Rank | Subcarrier | Saliency |
|
||||
|-----:|-----------:|---------:|
|
||||
| 1 | **41** | 0.0128 |
|
||||
| 2 | **52** | 0.0120 |
|
||||
| 3 | **30** | 0.0100 |
|
||||
| 4 | 31 | 0.0097 |
|
||||
| 5 | 10 | 0.0088 |
|
||||
| 6 | 35 | 0.0088 |
|
||||
| 7 | 2 | 0.0087 |
|
||||
| 8 | 38 | 0.0083 |
|
||||
|
||||
**Max-to-mean ratio: 2.85×** — meaningful but moderate concentration. Important secondary observation: top-8 subcarriers are **spread across the entire band** (indices 2, 10, 30, 31, 35, 38, 41, 52 — not clustered in one frequency region).
|
||||
|
||||
## Implications
|
||||
|
||||
1. **Bandwidth-cap deployment is viable.** Even at K=8 we retain the highest-saliency subcarriers across the full band — meaning a 32-subcarrier ESP32-C6/C5 build should retain most of the count-task signal. Retraining at K=8/16/32 is the next-tick experiment.
|
||||
2. **R8 (RSSI alone) is feasible-but-bounded.** RSSI is a band-aggregate scalar that loses per-subcarrier resolution. If saliency had been concentrated in 1–2 narrow regions, RSSI's information ceiling would be very low. Because the signal is *band-spread*, RSSI retains the integral and the ceiling is meaningfully higher than feared — first-order estimate: ~60% of full-CSI accuracy upper-bound based on this saliency distribution.
|
||||
3. **R7 (adversarial defence) priority list.** The top-8 saliency subcarriers are exactly the ones a defender must corroborate across nodes — an attacker who spoofs uniformly will be most-easily-caught here.
|
||||
|
||||
## Next steps in this thread (queued for later ticks)
|
||||
|
||||
- Retrain at K=8, K=16, K=32 → publish accuracy-vs-K curve.
|
||||
- Same saliency map for the pose model.
|
||||
- Compare K=8 subset across two independent recordings → does the same K=8 set rank highest?
|
||||
- Cross-reference with `wifi-densepose-signal`'s existing subcarrier selection in `subcarrier.rs`.
|
||||
+65
-3
@@ -29,13 +29,14 @@ WiFi DensePose turns commodity WiFi signals into real-time human pose estimation
|
||||
8. [Vital Sign Detection](#vital-sign-detection)
|
||||
9. [CLI Reference](#cli-reference)
|
||||
10. [Observatory Visualization](#observatory-visualization)
|
||||
11. [Adaptive Classifier](#adaptive-classifier)
|
||||
11. [Loading the Pretrained Model from Hugging Face](#loading-the-pretrained-model-from-hugging-face)
|
||||
12. [Adaptive Classifier](#adaptive-classifier)
|
||||
- [Recording Training Data](#recording-training-data)
|
||||
- [Training the Model](#training-the-model)
|
||||
- [Using the Trained Model](#using-the-trained-model)
|
||||
12. [Training a Model](#training-a-model)
|
||||
13. [Training a Model](#training-a-model)
|
||||
- [CRV Signal-Line Protocol](#crv-signal-line-protocol)
|
||||
13. [RVF Model Containers](#rvf-model-containers)
|
||||
14. [RVF Model Containers](#rvf-model-containers)
|
||||
14. [Hardware Setup](#hardware-setup)
|
||||
- [ESP32-S3 Mesh](#esp32-s3-mesh)
|
||||
- [Intel 5300 / Atheros NIC](#intel-5300--atheros-nic)
|
||||
@@ -793,6 +794,67 @@ The Observatory is an immersive Three.js visualization that renders WiFi sensing
|
||||
|
||||
---
|
||||
|
||||
## Loading the Pretrained Model from Hugging Face
|
||||
|
||||
A pretrained CSI encoder + presence-detection head is published on Hugging Face at [`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained). It was trained on 60,630 frames / 610,615 contrastive triplets (12.2M steps, final loss 0.065) and reports 100% presence accuracy and ~164k embeddings/sec on an Apple M4 Pro.
|
||||
|
||||
What it ships (and what it does not):
|
||||
|
||||
| Capability | Status |
|
||||
|------------|--------|
|
||||
| Presence detection (occupied / empty) | ✅ Trained head — 100% accuracy on validation |
|
||||
| 128-dim CSI embeddings (re-ID, similarity, downstream training) | ✅ Trained encoder |
|
||||
| Single-person breathing / heart-rate | ⚠️ Server still uses heuristic DSP — model does not replace this yet |
|
||||
| 17-keypoint full-body pose | 🔬 No keypoint weights shipped yet — pose pipeline runs but without a learned head |
|
||||
|
||||
### Download
|
||||
|
||||
```bash
|
||||
pip install huggingface_hub
|
||||
huggingface-cli download ruvnet/wifi-densepose-pretrained \
|
||||
--local-dir models/wifi-densepose-pretrained
|
||||
```
|
||||
|
||||
The download yields a small set of files (the `.rvf.jsonl` is the canonical container the sensing server reads):
|
||||
|
||||
```
|
||||
models/wifi-densepose-pretrained/
|
||||
model.rvf.jsonl # RVF container (encoder + presence head + lora)
|
||||
model.safetensors # 48 KB — same encoder weights, safetensors format
|
||||
model-q4.bin # 8 KB — recommended quantization for edge
|
||||
presence-head.json # presence classifier head
|
||||
config.json # sona-lora rank=8 alpha=16, target encoder + task_heads
|
||||
```
|
||||
|
||||
### Using the weights
|
||||
|
||||
The HF artifact is in **JSONL RVF** format (one JSON object per line: `metadata`, `encoder`, `lora`). What you can do with it today:
|
||||
|
||||
| Consumer | Format it reads | Status |
|
||||
|----------|-----------------|--------|
|
||||
| Python / PyTorch training pipeline | `model.safetensors` | ✅ Works — load with `safetensors.torch.load_file` |
|
||||
| RVF JSONL inspection / re-export | `model.rvf.jsonl` | ✅ Works — plain JSONL, parse line-by-line |
|
||||
| Sensing-server `--model <PATH>` flag | binary RVF (`RVFS` magic) | ⚠️ Does **not** accept the JSONL file yet — see gap below |
|
||||
|
||||
**Known gap (tracked):** `v2/crates/wifi-densepose-sensing-server/src/rvf_container.rs` only parses the binary RVF segment format (magic `0x52564653`). Pointing `--model` at `model.rvf.jsonl` causes the progressive loader to error with `invalid magic at offset 0: expected 0x52564653, got 0x7974227B` (`0x7974227B` is the ASCII bytes `{"ty…` from the JSONL header), and the live pipeline degrades to null output rather than falling back to heuristic mode. Until a JSONL adapter lands (or the model is re-published as binary RVF), run the sensing-server **without** `--model` and consume the HF weights from Python or the training pipeline.
|
||||
|
||||
```bash
|
||||
# Works today — Python side (training, evaluation, embedding extraction):
|
||||
python -c "
|
||||
from safetensors.torch import load_file
|
||||
state = load_file('models/wifi-densepose-pretrained/model.safetensors')
|
||||
print({k: tuple(v.shape) for k, v in state.items()})
|
||||
"
|
||||
|
||||
# Sensing server — run heuristic for now:
|
||||
cargo run -p wifi-densepose-sensing-server --release -- \
|
||||
--source esp32 --udp-port 5005 --http-port 3000
|
||||
```
|
||||
|
||||
See [RVF Model Containers](#rvf-model-containers) for the binary format the loader expects, and [Training a Model](#training-a-model) for using the encoder as a starting point for environment-specific fine-tuning.
|
||||
|
||||
---
|
||||
|
||||
## Adaptive Classifier
|
||||
|
||||
The adaptive classifier (ADR-048) learns your environment's specific WiFi signal patterns from labeled recordings. It replaces static threshold-based classification with a trained logistic regression model that uses 15 features (7 server-computed + 8 subcarrier-derived statistics).
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
#!/usr/bin/env python3
|
||||
"""R5 — per-subcarrier input×gradient saliency for the count + pose cogs.
|
||||
|
||||
See docs/research/sota-2026-05-22/R5-subcarrier-saliency.md for context.
|
||||
|
||||
Usage:
|
||||
python examples/research-sota/r5_subcarrier_saliency.py \
|
||||
--paired data/paired/wiflow-p7-1779210883.paired.jsonl \
|
||||
--model v2/crates/cog-person-count/cog/artifacts/count_v1.safetensors \
|
||||
--kind count
|
||||
python examples/research-sota/r5_subcarrier_saliency.py \
|
||||
--paired data/paired/wiflow-p7-1779210883.paired.jsonl \
|
||||
--model v2/crates/cog-pose-estimation/cog/artifacts/pose_v1.safetensors \
|
||||
--kind pose
|
||||
|
||||
Output:
|
||||
<dirname-of-model>/saliency.json per-subcarrier saliency + top-K lists
|
||||
stdout summary table
|
||||
|
||||
Method (per ADR/research note):
|
||||
S_k = E_samples[ |dL/dx_k| * |x_k| ]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import struct
|
||||
from pathlib import Path
|
||||
from typing import Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
N_SUB, N_FRAMES = 56, 20
|
||||
|
||||
|
||||
def load_paired(path: Path, kind: str, max_samples: int | None = None) -> Tuple[np.ndarray, np.ndarray]:
|
||||
"""Returns (X, y) — X is [N, 56, 20] float32, y depends on kind.
|
||||
|
||||
kind="count" → y is [N] int64 in {0..7}
|
||||
kind="pose" → y is [N, 17, 2] float32 in [0, 1]
|
||||
"""
|
||||
csis, ys = [], []
|
||||
with path.open(encoding="utf-8") as f:
|
||||
for line in f:
|
||||
if not line.strip():
|
||||
continue
|
||||
d = json.loads(line)
|
||||
shape = d.get("csi_shape", [N_SUB, N_FRAMES])
|
||||
if shape != [N_SUB, N_FRAMES]:
|
||||
continue
|
||||
csi = np.asarray(d["csi"], dtype=np.float32).reshape(N_SUB, N_FRAMES)
|
||||
csis.append(csi)
|
||||
if kind == "count":
|
||||
ys.append(int(d.get("n_persons_mode", 0)))
|
||||
elif kind == "pose":
|
||||
ys.append(np.asarray(d.get("kp", []), dtype=np.float32))
|
||||
else:
|
||||
raise ValueError(f"unknown kind: {kind}")
|
||||
if max_samples and len(csis) >= max_samples:
|
||||
break
|
||||
return np.stack(csis), np.asarray(ys, dtype=(np.int64 if kind == "count" else np.float32))
|
||||
|
||||
|
||||
def load_safetensors(path: Path) -> dict[str, np.ndarray]:
|
||||
"""Pure-python safetensors reader. Returns {name: ndarray}."""
|
||||
with path.open("rb") as f:
|
||||
hlen = struct.unpack("<Q", f.read(8))[0]
|
||||
header = json.loads(f.read(hlen).decode("utf-8"))
|
||||
out = {}
|
||||
for name, meta in header.items():
|
||||
if name == "__metadata__":
|
||||
continue
|
||||
start, end = meta["data_offsets"]
|
||||
shape = meta["shape"]
|
||||
assert meta["dtype"] == "F32", f"unsupported dtype {meta['dtype']} in {name}"
|
||||
f.seek(8 + hlen + start)
|
||||
buf = f.read(end - start)
|
||||
arr = np.frombuffer(buf, dtype=np.float32).copy().reshape(shape)
|
||||
out[name] = arr
|
||||
return out
|
||||
|
||||
|
||||
def conv1d_forward(x: np.ndarray, w: np.ndarray, b: np.ndarray, padding: int, dilation: int) -> np.ndarray:
|
||||
"""Pure-numpy Conv1d forward. x: [B, Cin, T], w: [Cout, Cin, K]. Returns [B, Cout, T']."""
|
||||
B, Cin, T = x.shape
|
||||
Cout, _, K = w.shape
|
||||
# Pad
|
||||
xp = np.pad(x, ((0, 0), (0, 0), (padding, padding)), mode="constant")
|
||||
Tp = xp.shape[2]
|
||||
# Effective filter span with dilation
|
||||
eff = (K - 1) * dilation + 1
|
||||
Tout = Tp - eff + 1
|
||||
out = np.zeros((B, Cout, Tout), dtype=np.float32)
|
||||
for k in range(K):
|
||||
# x_slice shape: [B, Cin, Tout]
|
||||
x_slice = xp[:, :, k * dilation : k * dilation + Tout]
|
||||
# w_slice shape: [Cout, Cin]
|
||||
w_slice = w[:, :, k]
|
||||
# einsum: B,Cin,T x Cout,Cin → B,Cout,T
|
||||
out += np.einsum("bct,oc->bot", x_slice, w_slice)
|
||||
return out + b[None, :, None]
|
||||
|
||||
|
||||
def relu(x: np.ndarray) -> np.ndarray:
|
||||
return np.maximum(x, 0.0)
|
||||
|
||||
|
||||
def softmax(x: np.ndarray, axis: int = -1) -> np.ndarray:
|
||||
m = x.max(axis=axis, keepdims=True)
|
||||
e = np.exp(x - m)
|
||||
return e / e.sum(axis=axis, keepdims=True)
|
||||
|
||||
|
||||
def forward_count(x: np.ndarray, w: dict[str, np.ndarray]) -> np.ndarray:
|
||||
"""CountNet forward. x: [B, 56, 20] → probs [B, 8]."""
|
||||
h = conv1d_forward(x, w["enc.c1.weight"], w["enc.c1.bias"], padding=1, dilation=1)
|
||||
h = relu(h)
|
||||
h = conv1d_forward(h, w["enc.c2.weight"], w["enc.c2.bias"], padding=2, dilation=2)
|
||||
h = relu(h)
|
||||
h = conv1d_forward(h, w["enc.c3.weight"], w["enc.c3.bias"], padding=4, dilation=4)
|
||||
h = relu(h)
|
||||
h = h.mean(axis=2) # [B, 128]
|
||||
# count head
|
||||
z = relu(h @ w["count_head.fc1.weight"].T + w["count_head.fc1.bias"])
|
||||
z = z @ w["count_head.fc2.weight"].T + w["count_head.fc2.bias"]
|
||||
return softmax(z, axis=-1)
|
||||
|
||||
|
||||
def saliency_input_gradient(
|
||||
X: np.ndarray,
|
||||
y: np.ndarray,
|
||||
weights: dict[str, np.ndarray],
|
||||
kind: str,
|
||||
eps: float = 1e-3,
|
||||
) -> np.ndarray:
|
||||
"""Per-subcarrier saliency: S_k = E[|dL/dx_k| * |x_k|].
|
||||
|
||||
Uses central-difference numerical gradient over each subcarrier (cheap because
|
||||
we marginalise over the time axis after taking the abs). For a 56-subcarrier
|
||||
input that's 56 forward passes per sample — slow but exact, and only runs
|
||||
once per saliency map.
|
||||
"""
|
||||
B, N_sub, T = X.shape
|
||||
saliency = np.zeros(N_sub, dtype=np.float64)
|
||||
|
||||
if kind == "count":
|
||||
# Loss = -log(p_true). Compute baseline log-prob.
|
||||
for k in range(N_sub):
|
||||
x_plus = X.copy()
|
||||
x_plus[:, k, :] += eps
|
||||
x_minus = X.copy()
|
||||
x_minus[:, k, :] -= eps
|
||||
p_plus = forward_count(x_plus, weights)
|
||||
p_minus = forward_count(x_minus, weights)
|
||||
# dL/dx ≈ -(log p_plus[y] - log p_minus[y]) / (2*eps)
|
||||
idx = np.arange(B)
|
||||
lp_plus = np.log(p_plus[idx, y] + 1e-12)
|
||||
lp_minus = np.log(p_minus[idx, y] + 1e-12)
|
||||
grad_k = -(lp_plus - lp_minus) / (2 * eps) # [B]
|
||||
# |dL/dx_k| * |x_k| — x_k is a vector over time; take its magnitude
|
||||
x_k_mag = np.abs(X[:, k, :]).mean(axis=1) # [B]
|
||||
saliency[k] += float((np.abs(grad_k) * x_k_mag).mean())
|
||||
else:
|
||||
raise NotImplementedError("pose kind not yet wired — count first")
|
||||
|
||||
return saliency
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--paired", required=True)
|
||||
parser.add_argument("--model", required=True)
|
||||
parser.add_argument("--kind", choices=["count", "pose"], default="count")
|
||||
parser.add_argument("--max-samples", type=int, default=128,
|
||||
help="Cap on samples used for saliency (saliency cost is O(N_sub × samples × eps_passes))")
|
||||
parser.add_argument("--out", default=None,
|
||||
help="Output JSON path; defaults to <model_dir>/saliency.json")
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"Loading paired data from {args.paired} (kind={args.kind})")
|
||||
X, y = load_paired(Path(args.paired), kind=args.kind, max_samples=args.max_samples)
|
||||
print(f" X: {X.shape}, y: {y.shape}")
|
||||
if args.kind == "count":
|
||||
unique, counts = np.unique(y, return_counts=True)
|
||||
print(f" label distribution: {dict(zip(unique.tolist(), counts.tolist()))}")
|
||||
|
||||
# Standardise (per-subcarrier z-score using THIS subset's stats — saliency is
|
||||
# invariant to affine input transforms in the limit of small eps).
|
||||
mu = X.mean(axis=(0, 2), keepdims=True)
|
||||
sd = X.std(axis=(0, 2), keepdims=True) + 1e-6
|
||||
X_norm = (X - mu) / sd
|
||||
|
||||
print(f"Loading weights from {args.model}")
|
||||
weights = load_safetensors(Path(args.model))
|
||||
print(f" loaded {len(weights)} tensors: {sorted(list(weights.keys()))[:6]}...")
|
||||
|
||||
print(f"Computing input×gradient saliency over {X.shape[0]} samples × 56 subcarriers...")
|
||||
saliency = saliency_input_gradient(X_norm, y, weights, kind=args.kind, eps=1e-3)
|
||||
|
||||
order = np.argsort(saliency)[::-1] # descending
|
||||
top_k = {k: order[:k].tolist() for k in (8, 16, 32)}
|
||||
|
||||
out = {
|
||||
"kind": args.kind,
|
||||
"model": str(args.model),
|
||||
"n_samples": int(X.shape[0]),
|
||||
"saliency_per_subcarrier": saliency.tolist(),
|
||||
"ranking_high_to_low": order.tolist(),
|
||||
"top_k_subcarriers": top_k,
|
||||
"saliency_summary": {
|
||||
"min": float(saliency.min()),
|
||||
"max": float(saliency.max()),
|
||||
"mean": float(saliency.mean()),
|
||||
"std": float(saliency.std()),
|
||||
"max_to_mean_ratio": float(saliency.max() / max(saliency.mean(), 1e-12)),
|
||||
},
|
||||
}
|
||||
|
||||
out_path = Path(args.out) if args.out else Path(args.model).parent / "saliency.json"
|
||||
out_path.write_text(json.dumps(out, indent=2))
|
||||
print(f"\nWrote {out_path}")
|
||||
print(f"\nTop 8 subcarriers (most influential):")
|
||||
for rank, idx in enumerate(order[:8]):
|
||||
print(f" #{rank + 1}: subcarrier {int(idx):2d} saliency={saliency[idx]:.4f}")
|
||||
print(f"\nMax/mean ratio: {out['saliency_summary']['max_to_mean_ratio']:.2f}× "
|
||||
f"(higher = signal more concentrated in a few subcarriers)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -572,9 +572,59 @@
|
||||
const txt = document.querySelector('#loading .text');
|
||||
if (txt) txt.textContent = `▸ Loading skinned subject · X Bot.fbx · ${pct} %`;
|
||||
}, (err) => {
|
||||
console.error('FBX load failed', err);
|
||||
const txt = document.querySelector('#loading .text');
|
||||
if (txt) txt.textContent = '⚠ Load failed — see console';
|
||||
// Graceful degradation: when the FBX 404s on gh-pages (Mixamo
|
||||
// X Bot.fbx is gitignored — license boundary, not redistributed)
|
||||
// we hide the spinner and show a friendly banner explaining how
|
||||
// to run this demo locally with your own Mixamo download.
|
||||
// Local development with assets/X Bot.fbx present hits the
|
||||
// success branch above and never sees this UI.
|
||||
console.warn('FBX load failed — showing fallback banner', err);
|
||||
const loading = document.getElementById('loading');
|
||||
if (loading) {
|
||||
loading.innerHTML = `
|
||||
<div style="
|
||||
max-width: 540px; padding: 20px 22px;
|
||||
background: rgba(20, 24, 38, 0.92);
|
||||
border: 1px solid rgba(78, 205, 196, 0.4);
|
||||
border-radius: 10px;
|
||||
color: #e0e4f0; font-family: 'Segoe UI', system-ui, sans-serif;
|
||||
line-height: 1.5; font-size: 14px;
|
||||
box-shadow: 0 6px 24px rgba(0,0,0,0.5);
|
||||
">
|
||||
<div style="font-size:16px; color:#4ecdc4; font-weight:600; margin-bottom:6px;">
|
||||
🦴 Mixamo asset not bundled in this deployment
|
||||
</div>
|
||||
<div style="color:#c8cee0; margin-bottom:12px;">
|
||||
This demo loads <code style="color:#4ecdc4; background:rgba(78,205,196,0.08); padding:1px 6px; border-radius:3px;">X Bot.fbx</code>
|
||||
from Mixamo, which is intentionally not redistributed here (license boundary).
|
||||
The ADR-097 helpers scene (grid / axes / per-node CSI boxes) is rendering behind this card —
|
||||
click outside to interact with it.
|
||||
</div>
|
||||
<div style="color:#8890a8; font-size:13px; margin-bottom:14px;">
|
||||
To run this demo with the character, clone the repo, download
|
||||
<code style="color:#4ecdc4;">X Bot.fbx</code> (FBX Binary · T-Pose · Without Skin)
|
||||
from <a href="https://mixamo.com" target="_blank" rel="noopener" style="color:#4ecdc4;">mixamo.com</a>
|
||||
into <code style="color:#4ecdc4;">examples/three.js/assets/</code>, then run
|
||||
<code style="color:#4ecdc4;">python examples/three.js/server/serve-demo.py</code>.
|
||||
</div>
|
||||
<div style="display:flex; gap:10px; flex-wrap:wrap;">
|
||||
<a href="https://github.com/ruvnet/RuView/tree/main/examples/three.js" target="_blank" rel="noopener"
|
||||
style="padding:6px 12px; background:rgba(78,205,196,0.12); border:1px solid rgba(78,205,196,0.4); border-radius:6px; color:#4ecdc4; text-decoration:none; font-size:13px;">
|
||||
📂 Source on GitHub
|
||||
</a>
|
||||
<a href="https://mixamo.com" target="_blank" rel="noopener"
|
||||
style="padding:6px 12px; background:rgba(212,165,116,0.12); border:1px solid rgba(212,165,116,0.4); border-radius:6px; color:#d4a574; text-decoration:none; font-size:13px;">
|
||||
🦴 Get X Bot from Mixamo
|
||||
</a>
|
||||
<a href="../" style="padding:6px 12px; background:rgba(136,144,168,0.12); border:1px solid rgba(136,144,168,0.3); border-radius:6px; color:#8890a8; text-decoration:none; font-size:13px;">
|
||||
← Back to demo gallery
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
loading.style.pointerEvents = 'auto';
|
||||
loading.style.cursor = 'default';
|
||||
}
|
||||
});
|
||||
|
||||
function playClip(name) {
|
||||
|
||||
@@ -721,8 +721,56 @@
|
||||
const txt = document.querySelector('#loading .text');
|
||||
if (txt) txt.textContent = `▸ Loading skinned subject · X Bot.fbx · ${pct} %`;
|
||||
}, (err) => {
|
||||
console.error('FBX load failed', err);
|
||||
document.querySelector('#loading .text').textContent = '⚠ Load failed — see console';
|
||||
// Graceful degradation when X Bot.fbx 404s on gh-pages (license
|
||||
// boundary — not redistributed). Local runs with the FBX present
|
||||
// hit the success branch above and never see this banner.
|
||||
console.warn('FBX load failed — showing fallback banner', err);
|
||||
const loading = document.getElementById('loading');
|
||||
if (loading) {
|
||||
loading.innerHTML = `
|
||||
<div style="
|
||||
max-width: 580px; padding: 20px 22px;
|
||||
background: rgba(20, 24, 38, 0.92);
|
||||
border: 1px solid rgba(78, 205, 196, 0.4);
|
||||
border-radius: 10px;
|
||||
color: #e0e4f0; font-family: 'Segoe UI', system-ui, sans-serif;
|
||||
line-height: 1.5; font-size: 14px;
|
||||
box-shadow: 0 6px 24px rgba(0,0,0,0.5);
|
||||
">
|
||||
<div style="font-size:16px; color:#4ecdc4; font-weight:600; margin-bottom:6px;">
|
||||
🦴 Mixamo asset not bundled in this deployment
|
||||
</div>
|
||||
<div style="color:#c8cee0; margin-bottom:12px;">
|
||||
This realtime pose demo retargets webcam + MediaPipe onto
|
||||
<code style="color:#4ecdc4; background:rgba(78,205,196,0.08); padding:1px 6px; border-radius:3px;">X Bot.fbx</code>,
|
||||
which Mixamo licenses for direct download by end users and is intentionally not
|
||||
redistributed here. The ADR-097 helpers scene is still rendering behind this card.
|
||||
</div>
|
||||
<div style="color:#8890a8; font-size:13px; margin-bottom:14px;">
|
||||
To run locally: clone the repo, get
|
||||
<code style="color:#4ecdc4;">X Bot.fbx</code> (FBX Binary · T-Pose · Without Skin)
|
||||
from <a href="https://mixamo.com" target="_blank" rel="noopener" style="color:#4ecdc4;">mixamo.com</a>,
|
||||
drop it in <code style="color:#4ecdc4;">examples/three.js/assets/</code>, then
|
||||
<code style="color:#4ecdc4;">python examples/three.js/server/serve-demo.py</code>.
|
||||
</div>
|
||||
<div style="display:flex; gap:10px; flex-wrap:wrap;">
|
||||
<a href="https://github.com/ruvnet/RuView/tree/main/examples/three.js" target="_blank" rel="noopener"
|
||||
style="padding:6px 12px; background:rgba(78,205,196,0.12); border:1px solid rgba(78,205,196,0.4); border-radius:6px; color:#4ecdc4; text-decoration:none; font-size:13px;">
|
||||
📂 Source on GitHub
|
||||
</a>
|
||||
<a href="https://mixamo.com" target="_blank" rel="noopener"
|
||||
style="padding:6px 12px; background:rgba(212,165,116,0.12); border:1px solid rgba(212,165,116,0.4); border-radius:6px; color:#d4a574; text-decoration:none; font-size:13px;">
|
||||
🦴 Get X Bot from Mixamo
|
||||
</a>
|
||||
<a href="../" style="padding:6px 12px; background:rgba(136,144,168,0.12); border:1px solid rgba(136,144,168,0.3); border-radius:6px; color:#8890a8; text-decoration:none; font-size:13px;">
|
||||
← Back to demo gallery
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
loading.style.pointerEvents = 'auto';
|
||||
loading.style.cursor = 'default';
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta name="robots" content="noindex,nofollow">
|
||||
<title>RuView · three.js demos · ADR-097 sensing-helpers scene</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #0a0e1a;
|
||||
--bg2: #111627;
|
||||
--card: #171d30;
|
||||
--card-h: #1e2540;
|
||||
--border: #252d45;
|
||||
--t1: #e0e4f0;
|
||||
--t2: #8890a8;
|
||||
--cyan: #4ecdc4;
|
||||
--green: #6bcb77;
|
||||
--amber: #d4a574;
|
||||
--r: 10px;
|
||||
}
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
body {
|
||||
font-family: 'Segoe UI', system-ui, -apple-system, sans-serif;
|
||||
background: var(--bg);
|
||||
color: var(--t1);
|
||||
line-height: 1.5;
|
||||
padding: 24px 16px 64px;
|
||||
}
|
||||
.wrap { max-width: 980px; margin: 0 auto; }
|
||||
h1 { font-size: 22px; color: #fff; }
|
||||
h1 span { color: var(--cyan); }
|
||||
.lede { color: var(--t2); margin: 8px 0 24px; font-size: 14px; max-width: 70ch; }
|
||||
.pill {
|
||||
display: inline-block;
|
||||
padding: 2px 8px;
|
||||
border-radius: 999px;
|
||||
font-size: 11px;
|
||||
margin-left: 8px;
|
||||
vertical-align: middle;
|
||||
border: 1px solid var(--border);
|
||||
background: var(--bg2);
|
||||
color: var(--t2);
|
||||
}
|
||||
.pill.ok { color: var(--green); border-color: #2d4a35; background: rgba(107, 203, 119, 0.08); }
|
||||
.pill.warn { color: var(--amber); border-color: #4a3d2d; background: rgba(212, 165, 116, 0.08); }
|
||||
.grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(280px, 1fr));
|
||||
gap: 12px;
|
||||
margin-top: 16px;
|
||||
}
|
||||
.card {
|
||||
background: var(--card);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--r);
|
||||
padding: 16px;
|
||||
text-decoration: none;
|
||||
color: inherit;
|
||||
transition: background 0.12s, border-color 0.12s, transform 0.12s;
|
||||
}
|
||||
.card:hover {
|
||||
background: var(--card-h);
|
||||
border-color: var(--cyan);
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
.card h2 { font-size: 15px; color: #fff; margin-bottom: 6px; }
|
||||
.card .sub { color: var(--t2); font-size: 13px; }
|
||||
.card img {
|
||||
margin-top: 10px;
|
||||
width: 100%;
|
||||
aspect-ratio: 16/9;
|
||||
object-fit: cover;
|
||||
border-radius: 6px;
|
||||
border: 1px solid var(--border);
|
||||
background: #000;
|
||||
}
|
||||
.note {
|
||||
margin-top: 28px;
|
||||
padding: 14px 16px;
|
||||
background: rgba(212, 165, 116, 0.06);
|
||||
border-left: 3px solid var(--amber);
|
||||
border-radius: 6px;
|
||||
font-size: 13px;
|
||||
color: var(--t1);
|
||||
}
|
||||
.note b { color: var(--amber); }
|
||||
code {
|
||||
font-family: 'Cascadia Code', Consolas, monospace;
|
||||
background: var(--bg2);
|
||||
padding: 1px 5px;
|
||||
border-radius: 3px;
|
||||
color: var(--cyan);
|
||||
font-size: 12px;
|
||||
}
|
||||
a { color: var(--cyan); }
|
||||
.foot {
|
||||
color: var(--t2);
|
||||
font-size: 12px;
|
||||
margin-top: 32px;
|
||||
text-align: center;
|
||||
}
|
||||
.foot a { color: var(--cyan); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
|
||||
<h1>RuView · <span>three.js demos</span></h1>
|
||||
<p class="lede">
|
||||
Five progressively richer browser demos of the <a href="https://github.com/ruvnet/RuView/blob/main/docs/adr/ADR-097-adopt-rvcsi-as-ruview-csi-runtime.md">ADR-097</a>
|
||||
sensing-helpers scene, ending with a live MediaPipe-Pose → Mixamo X Bot retargeting pipeline driven
|
||||
by a real ESP32 CSI feed.
|
||||
</p>
|
||||
|
||||
<div class="grid">
|
||||
|
||||
<a class="card" href="demos/01-helpers.html">
|
||||
<h2>01 · Helpers <span class="pill ok">standalone</span></h2>
|
||||
<div class="sub">Plain ADR-097 helpers in the point-cloud viewer. No external assets.</div>
|
||||
<img src="screenshots/01-helpers.png" alt="01 screenshot">
|
||||
</a>
|
||||
|
||||
<a class="card" href="demos/02-cinematic.html">
|
||||
<h2>02 · Cinematic <span class="pill ok">standalone</span></h2>
|
||||
<div class="sub">Cinematic camera + pseudo-CSI visualization on top of #01.</div>
|
||||
<img src="screenshots/02-cinematic.png" alt="02 screenshot">
|
||||
</a>
|
||||
|
||||
<a class="card" href="demos/03-skinned.html">
|
||||
<h2>03 · Skinned (GLTF) <span class="pill ok">standalone</span></h2>
|
||||
<div class="sub">GLTF skinned mesh + additive animation blending in the ADR-097 scene.</div>
|
||||
<img src="screenshots/03-skinned.png" alt="03 screenshot">
|
||||
</a>
|
||||
|
||||
<a class="card" href="demos/04-skinned-fbx.html">
|
||||
<h2>04 · Skinned FBX <span class="pill warn">needs FBX</span></h2>
|
||||
<div class="sub">Mixamo X Bot via FBXLoader. Requires a local <code>assets/X Bot.fbx</code>.</div>
|
||||
<img src="screenshots/04-skinned-fbx.png" alt="04 screenshot">
|
||||
</a>
|
||||
|
||||
<a class="card" href="demos/05-skinned-realtime.html">
|
||||
<h2>05 · Realtime (Pose + CSI) <span class="pill warn">needs FBX</span></h2>
|
||||
<div class="sub">Webcam → MediaPipe Pose Heavy → Mixamo IK retarget, live ESP32 CSI overlay.</div>
|
||||
<img src="screenshots/05-skinned-realtime.png" alt="05 screenshot">
|
||||
</a>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="note">
|
||||
<b>Demos 04 and 05 need a Mixamo asset.</b> The Mixamo
|
||||
<code>X Bot.fbx</code> file is intentionally <em>not</em> redistributed in
|
||||
this deployment — it's licensed for end-users to download from
|
||||
<a href="https://mixamo.com" target="_blank" rel="noopener">mixamo.com</a> directly.
|
||||
To run these locally: clone the repo, download <code>X Bot.fbx</code>
|
||||
(FBX Binary, T-Pose, Without Skin) into
|
||||
<code>examples/three.js/assets/</code>, then run
|
||||
<code>python examples/three.js/server/serve-demo.py</code>.
|
||||
</div>
|
||||
|
||||
<div class="foot">
|
||||
Source: <a href="https://github.com/ruvnet/RuView/tree/main/examples/three.js">github.com/ruvnet/RuView/tree/main/examples/three.js</a>
|
||||
· ADR-097 · three.js r128
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -25,6 +25,23 @@ This firmware captures WiFi Channel State Information (CSI) from an ESP32-S3 and
|
||||
|
||||
For users who want to get running fast. Detailed explanations follow in later sections.
|
||||
|
||||
### 0. Pre-built binaries (v0.6.5 — skip the build step)
|
||||
|
||||
Pre-built binaries are in `firmware/esp32-csi-node/release_bins/` (version: see `release_bins/version.txt`).
|
||||
Flash them directly:
|
||||
|
||||
```bash
|
||||
python -m esptool --chip esp32s3 --port COM7 --baud 460800 \
|
||||
write_flash --flash_mode dio --flash_size 8MB \
|
||||
0x0 firmware/esp32-csi-node/release_bins/bootloader.bin \
|
||||
0x8000 firmware/esp32-csi-node/release_bins/partition-table.bin \
|
||||
0xf000 firmware/esp32-csi-node/release_bins/ota_data_initial.bin \
|
||||
0x20000 firmware/esp32-csi-node/release_bins/esp32-csi-node.bin
|
||||
```
|
||||
|
||||
For 4 MB boards use `release_bins/esp32-csi-node-4mb.bin` and `release_bins/partition-table-4mb.bin`
|
||||
with `--flash_size 4MB`.
|
||||
|
||||
### 1. Build (Docker -- the only reliable method)
|
||||
|
||||
```bash
|
||||
@@ -294,8 +311,9 @@ python -m serial.tools.miniterm COM7 115200
|
||||
Expected output after boot:
|
||||
|
||||
```
|
||||
I (321) main: ESP32-S3 CSI Node (ADR-018) -- Node ID: 1
|
||||
I (345) main: WiFi STA initialized, connecting to SSID: wifi-densepose
|
||||
I (396) csi_collector: Early capture node_id=1 (before WiFi init, #232/#390)
|
||||
I (406) main: ESP32-S3 CSI Node (ADR-018) -- v0.6.5 -- Node ID: 1
|
||||
I (566) main: WiFi STA initialized, connecting to SSID: wifi-densepose
|
||||
I (1023) main: Connected to WiFi
|
||||
I (1025) main: CSI streaming active -> 192.168.1.100:5005 (edge_tier=2, OTA=ready, WASM=ready)
|
||||
```
|
||||
|
||||
@@ -849,6 +849,8 @@ static void process_frame(const edge_ring_slot_t *slot)
|
||||
|
||||
/* --- Step 11: Multi-person vitals --- */
|
||||
update_multi_person_vitals(slot->iq_data, n_subcarriers, sample_rate);
|
||||
/* Yield after multi-person DSP so IDLE1 can feed Core 1 watchdog (#683). */
|
||||
if (s_cfg.tier >= 2) vTaskDelay(1);
|
||||
|
||||
/* --- Step 12: Delta compression --- */
|
||||
if (s_cfg.tier >= 2) {
|
||||
@@ -894,6 +896,8 @@ static void process_frame(const edge_ring_slot_t *slot)
|
||||
wasm_runtime_on_frame(phases, amplitudes, variances,
|
||||
n_subcarriers,
|
||||
(const edge_vitals_pkt_t *)&s_latest_pkt);
|
||||
/* Yield after WASM dispatch to feed Core 1 watchdog (#683). */
|
||||
vTaskDelay(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,15 +14,35 @@ Requirements:
|
||||
pip install 'esptool>=5.0' nvs-partition-gen
|
||||
(or use the nvs_partition_gen.py bundled with ESP-IDF)
|
||||
|
||||
WARNING -- FULL-REPLACE SEMANTICS (issue #391):
|
||||
Every invocation REPLACES the entire `csi_cfg` NVS namespace on the device.
|
||||
Any key you don't pass on the CLI is erased. Always include WiFi credentials
|
||||
(--ssid, --password, --target-ip) unless you pass --force-partial.
|
||||
ADDITIVE-BY-DEFAULT (issue #391, #574 phase 1):
|
||||
Earlier versions of this script REPLACED the entire `csi_cfg` NVS namespace
|
||||
on the device every invocation, wiping any key you didn't pass on the CLI.
|
||||
That cost customers hours of unnecessary friction.
|
||||
|
||||
The script now MERGES new CLI flags with the per-port state previously
|
||||
written from this machine (stored under your user config dir; see
|
||||
`--state-dir` to override or `--state` to inspect). On every invocation:
|
||||
|
||||
1. Read the prior per-port state file (or treat as empty if absent).
|
||||
2. Overlay the new CLI flags on top.
|
||||
3. Generate + flash NVS from the merged state.
|
||||
4. Write the merged state back to the state file.
|
||||
|
||||
Net effect: partial reconfigure works the way users expect. Pass `--reset`
|
||||
to wipe both the state file AND the device NVS for first-time provisioning
|
||||
of a recycled board.
|
||||
|
||||
Caveat: state lives on the controlling machine. Provisioning the same
|
||||
device from a second machine starts from an empty state — pass the keys
|
||||
you want to keep on that invocation, or pre-seed the state file. A future
|
||||
follow-up will add USB-CDC NVS dump for true device-authoritative merging
|
||||
(tracked in #574).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import subprocess
|
||||
@@ -70,6 +90,90 @@ def has_config_value(args):
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-port state file (additive-by-default merging, #391 / #574)
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# The state file is JSON keyed by `args` attribute name. It captures every
|
||||
# config value previously written to a given serial port from this machine.
|
||||
# On the next invocation, missing CLI flags fall back to the stored value.
|
||||
|
||||
# argparse attribute names that participate in the merge. Order doesn't
|
||||
# matter; this is just the surface area to round-trip.
|
||||
MERGEABLE_ATTRS = [
|
||||
"ssid", "password", "target_ip", "target_port", "node_id",
|
||||
"tdm_slot", "tdm_total",
|
||||
"edge_tier", "pres_thresh", "fall_thresh",
|
||||
"vital_win", "vital_int", "subk_count",
|
||||
"channel", "filter_mac",
|
||||
"hop_channels", "hop_dwell",
|
||||
"seed_url", "seed_token", "zone", "swarm_hb", "swarm_ingest",
|
||||
]
|
||||
|
||||
|
||||
def _default_state_dir() -> str:
|
||||
"""Per-user config dir for provision-state JSON files."""
|
||||
env = os.environ
|
||||
if sys.platform == "win32":
|
||||
base = env.get("APPDATA") or os.path.expanduser("~")
|
||||
else:
|
||||
base = env.get("XDG_CONFIG_HOME") or os.path.join(
|
||||
os.path.expanduser("~"), ".config"
|
||||
)
|
||||
return os.path.join(base, "wifi-densepose", "esp32-provision-state")
|
||||
|
||||
|
||||
def _state_path_for(port: str, state_dir: str) -> str:
|
||||
"""File path for a given serial port. Sanitize the port for filesystem use."""
|
||||
safe = port.replace("/", "_").replace(":", "_").replace("\\", "_")
|
||||
return os.path.join(state_dir, f"{safe}.json")
|
||||
|
||||
|
||||
def load_state(port: str, state_dir: str) -> dict:
|
||||
"""Return the merged-state dict for `port`, or `{}` if absent / unreadable."""
|
||||
path = _state_path_for(port, state_dir)
|
||||
if not os.path.isfile(path):
|
||||
return {}
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, dict):
|
||||
return data
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
print(f"WARNING: could not read state file {path}: {exc}", file=sys.stderr)
|
||||
return {}
|
||||
|
||||
|
||||
def save_state(port: str, state_dir: str, state: dict) -> str:
|
||||
"""Write `state` to the per-port file, creating dirs as needed. Returns path."""
|
||||
os.makedirs(state_dir, exist_ok=True)
|
||||
path = _state_path_for(port, state_dir)
|
||||
# Sort keys for deterministic on-disk content (easier to diff).
|
||||
tmp = path + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(state, f, indent=2, sort_keys=True)
|
||||
f.write("\n")
|
||||
os.replace(tmp, path)
|
||||
return path
|
||||
|
||||
|
||||
def merge_state_into_args(args, prior: dict) -> dict:
|
||||
"""Overlay `args` onto `prior` for every MERGEABLE_ATTRS attribute.
|
||||
|
||||
CLI values win whenever they were explicitly set (i.e. not `None`).
|
||||
Returns the merged dict (for state persistence) and mutates `args`
|
||||
in place so downstream `build_nvs_csv` sees the merged values.
|
||||
"""
|
||||
merged = dict(prior)
|
||||
for name in MERGEABLE_ATTRS:
|
||||
cli_val = getattr(args, name, None)
|
||||
if cli_val is not None:
|
||||
merged[name] = cli_val
|
||||
elif name in merged:
|
||||
setattr(args, name, merged[name])
|
||||
return merged
|
||||
|
||||
|
||||
def build_nvs_csv(args):
|
||||
"""Build an NVS CSV string for the csi_cfg namespace."""
|
||||
buf = io.StringIO()
|
||||
@@ -190,7 +294,7 @@ def flash_nvs(port, baud, nvs_bin, chip):
|
||||
"--chip", chip,
|
||||
"--port", port,
|
||||
"--baud", str(baud),
|
||||
"write-flash",
|
||||
"write_flash",
|
||||
hex(NVS_PARTITION_OFFSET), bin_path,
|
||||
]
|
||||
print(f"Flashing NVS partition ({len(nvs_bin)} bytes) to {port} (chip={chip})...")
|
||||
@@ -250,19 +354,45 @@ def main():
|
||||
parser.add_argument("--swarm-ingest", type=int, help="Swarm vector ingest interval in seconds (default 5)")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Generate NVS binary but don't flash")
|
||||
parser.add_argument("--force-partial", action="store_true",
|
||||
help="Allow partial config without WiFi credentials. "
|
||||
"WARNING: flashing REPLACES the entire csi_cfg NVS namespace - "
|
||||
"any key not passed on the CLI will be erased (issue #391).")
|
||||
help="[deprecated since #391/#574] Suppress the missing-WiFi-trio "
|
||||
"error when no prior state file exists. The script now merges "
|
||||
"with prior state by default, so this flag is rarely needed.")
|
||||
parser.add_argument("--reset", action="store_true",
|
||||
help="Wipe this machine's per-port state file before merging. "
|
||||
"Use for first-time provisioning of a recycled board where "
|
||||
"previously-staged keys should NOT be re-applied.")
|
||||
parser.add_argument("--state-dir", default=_default_state_dir(),
|
||||
help="Override the per-user state directory (default: per-OS user config dir).")
|
||||
parser.add_argument("--state", action="store_true",
|
||||
help="Print the merged state that WOULD be flashed for this port and exit. "
|
||||
"Useful for debugging which keys are about to land on the device.")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not has_config_value(args):
|
||||
parser.error("At least one config value must be specified")
|
||||
# --- Per-port state load + merge (additive-by-default, #391 / #574) ---
|
||||
if args.reset:
|
||||
path = _state_path_for(args.port, args.state_dir)
|
||||
if os.path.isfile(path):
|
||||
os.unlink(path)
|
||||
print(f"--reset: removed state file {path}", file=sys.stderr)
|
||||
prior = {}
|
||||
else:
|
||||
prior = load_state(args.port, args.state_dir)
|
||||
merged = merge_state_into_args(args, prior)
|
||||
|
||||
# Bug 2 (#391): Prevent silent wipe of WiFi credentials on partial invocations.
|
||||
# Flashing the generated NVS binary to offset 0x9000 REPLACES the entire
|
||||
# csi_cfg namespace — there is no merge with existing NVS. Require the full
|
||||
# WiFi trio unless the user explicitly opts in with --force-partial.
|
||||
if args.state:
|
||||
print(json.dumps(merged, indent=2, sort_keys=True))
|
||||
return
|
||||
|
||||
if not has_config_value(args):
|
||||
parser.error(
|
||||
"At least one config value must be specified (after merging prior state). "
|
||||
"If you intended to start fresh, pass --reset and the keys you want."
|
||||
)
|
||||
|
||||
# WiFi-trio sanity check. After the merge, the trio should be present
|
||||
# unless the user is intentionally provisioning a brand-new board with
|
||||
# partial state. Keep --force-partial as the escape hatch for that case.
|
||||
wifi_trio_missing = [
|
||||
name for name, val in [
|
||||
("--ssid", args.ssid),
|
||||
@@ -272,20 +402,19 @@ def main():
|
||||
]
|
||||
if wifi_trio_missing and not args.force_partial:
|
||||
parser.error(
|
||||
f"Missing required WiFi credentials: {', '.join(wifi_trio_missing)}.\n"
|
||||
f"Missing required WiFi credentials after merging prior state: "
|
||||
f"{', '.join(wifi_trio_missing)}.\n"
|
||||
f"\n"
|
||||
f" provision.py REPLACES the entire csi_cfg NVS namespace on each run.\n"
|
||||
f" Any key not passed on the CLI will be erased -- including WiFi creds.\n"
|
||||
f"\n"
|
||||
f" Either pass all of --ssid, --password, --target-ip,\n"
|
||||
f" or add --force-partial to acknowledge that other NVS keys will be wiped."
|
||||
f" No per-port state file at {_state_path_for(args.port, args.state_dir)}\n"
|
||||
f" and the CLI didn't include them. Either pass --ssid + --password + --target-ip\n"
|
||||
f" on this run, or add --force-partial to flash without WiFi.\n"
|
||||
)
|
||||
if args.force_partial and wifi_trio_missing:
|
||||
print("WARNING: --force-partial is set. The following NVS keys will be WIPED "
|
||||
"(not present in this invocation):", file=sys.stderr)
|
||||
for k in wifi_trio_missing:
|
||||
print(f" - {k.lstrip('-')}", file=sys.stderr)
|
||||
print(" Plus any other csi_cfg keys not passed on the CLI.\n", file=sys.stderr)
|
||||
print(
|
||||
"WARNING: --force-partial is set and WiFi credentials are missing. "
|
||||
"The device will not connect to WiFi after flashing.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
# Validate TDM: if one is given, both should be
|
||||
if (args.tdm_slot is not None) != (args.tdm_total is not None):
|
||||
@@ -370,10 +499,19 @@ def main():
|
||||
f.write(nvs_bin)
|
||||
print(f"NVS binary saved to {out} ({len(nvs_bin)} bytes)")
|
||||
print(f"Flash manually: python -m esptool --chip {args.chip} --port {args.port} "
|
||||
f"write-flash 0x9000 {out}")
|
||||
f"write_flash 0x9000 {out}")
|
||||
# Persist merged state even on dry-run so a subsequent real flash from
|
||||
# this machine sees the same staged config.
|
||||
path = save_state(args.port, args.state_dir, merged)
|
||||
print(f"State persisted to {path}")
|
||||
return
|
||||
|
||||
flash_nvs(args.port, args.baud, nvs_bin, args.chip)
|
||||
# Persist merged state after a successful flash so future partial
|
||||
# invocations from this machine merge on top of what's actually on the
|
||||
# device. This is the heart of the additive-by-default fix (#391/#574).
|
||||
path = save_state(args.port, args.state_dir, merged)
|
||||
print(f"State persisted to {path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,3 @@
|
||||
0.6.6
|
||||
git-sha: cbcb389cb (pre-commit)
|
||||
built: 2026-05-21
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Tests for provision.py's additive-by-default merge behaviour (#391, #574)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
# Allow `python -m unittest` from anywhere in the repo.
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.dirname(HERE))
|
||||
|
||||
import provision # noqa: E402 — sibling import after sys.path tweak
|
||||
|
||||
|
||||
def _mk_args(**overrides) -> argparse.Namespace:
|
||||
"""Build a Namespace with every mergeable attr set to None unless overridden."""
|
||||
base = {name: None for name in provision.MERGEABLE_ATTRS}
|
||||
base.update(overrides)
|
||||
return argparse.Namespace(**base)
|
||||
|
||||
|
||||
class TestStateFile(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.dir = tempfile.mkdtemp(prefix="provision-state-")
|
||||
|
||||
def tearDown(self):
|
||||
import shutil
|
||||
shutil.rmtree(self.dir, ignore_errors=True)
|
||||
|
||||
def test_load_state_empty_when_missing(self):
|
||||
self.assertEqual(provision.load_state("COM7", self.dir), {})
|
||||
|
||||
def test_save_then_load_roundtrip(self):
|
||||
provision.save_state("COM7", self.dir, {"ssid": "x", "password": "y"})
|
||||
self.assertEqual(
|
||||
provision.load_state("COM7", self.dir),
|
||||
{"ssid": "x", "password": "y"},
|
||||
)
|
||||
|
||||
def test_save_creates_per_port_files(self):
|
||||
provision.save_state("COM7", self.dir, {"ssid": "a"})
|
||||
provision.save_state("/dev/ttyUSB0", self.dir, {"ssid": "b"})
|
||||
self.assertEqual(provision.load_state("COM7", self.dir), {"ssid": "a"})
|
||||
self.assertEqual(provision.load_state("/dev/ttyUSB0", self.dir), {"ssid": "b"})
|
||||
|
||||
def test_load_state_handles_corrupt_json(self):
|
||||
path = provision._state_path_for("COM7", self.dir)
|
||||
os.makedirs(self.dir, exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write("{not valid json")
|
||||
# Should warn but not raise.
|
||||
self.assertEqual(provision.load_state("COM7", self.dir), {})
|
||||
|
||||
|
||||
class TestMerge(unittest.TestCase):
|
||||
def test_cli_wins_over_prior(self):
|
||||
args = _mk_args(ssid="new-ssid")
|
||||
prior = {"ssid": "old-ssid", "password": "abc"}
|
||||
merged = provision.merge_state_into_args(args, prior)
|
||||
self.assertEqual(args.ssid, "new-ssid") # CLI value preserved
|
||||
self.assertEqual(args.password, "abc") # filled from prior
|
||||
self.assertEqual(merged["ssid"], "new-ssid")
|
||||
self.assertEqual(merged["password"], "abc")
|
||||
|
||||
def test_prior_fills_missing_cli(self):
|
||||
args = _mk_args() # all None
|
||||
prior = {
|
||||
"ssid": "MyWiFi",
|
||||
"password": "secret",
|
||||
"target_ip": "192.168.1.20",
|
||||
"node_id": 3,
|
||||
}
|
||||
merged = provision.merge_state_into_args(args, prior)
|
||||
self.assertEqual(args.ssid, "MyWiFi")
|
||||
self.assertEqual(args.password, "secret")
|
||||
self.assertEqual(args.target_ip, "192.168.1.20")
|
||||
self.assertEqual(args.node_id, 3)
|
||||
for key, val in prior.items():
|
||||
self.assertEqual(merged[key], val)
|
||||
|
||||
def test_partial_invocation_does_not_drop_unrelated_keys(self):
|
||||
# The exact #391 scenario: user previously provisioned WiFi, now adds
|
||||
# only --seed-url. Old behaviour wiped SSID. New behaviour keeps it.
|
||||
args = _mk_args(seed_url="http://10.1.10.236")
|
||||
prior = {
|
||||
"ssid": "ruv.net",
|
||||
"password": "<secret>",
|
||||
"target_ip": "192.168.1.20",
|
||||
}
|
||||
merged = provision.merge_state_into_args(args, prior)
|
||||
self.assertEqual(args.ssid, "ruv.net")
|
||||
self.assertEqual(args.password, "<secret>")
|
||||
self.assertEqual(args.target_ip, "192.168.1.20")
|
||||
self.assertEqual(args.seed_url, "http://10.1.10.236")
|
||||
# And the on-disk merged dict carries all four keys.
|
||||
self.assertEqual(set(merged.keys()),
|
||||
{"ssid", "password", "target_ip", "seed_url"})
|
||||
|
||||
def test_empty_prior_is_noop(self):
|
||||
args = _mk_args(ssid="x")
|
||||
merged = provision.merge_state_into_args(args, {})
|
||||
self.assertEqual(merged, {"ssid": "x"})
|
||||
|
||||
def test_falsy_but_not_none_cli_value_overrides_prior(self):
|
||||
# node_id=0 is a legal value; must NOT be replaced by prior["node_id"]=5.
|
||||
args = _mk_args(node_id=0)
|
||||
prior = {"node_id": 5}
|
||||
merged = provision.merge_state_into_args(args, prior)
|
||||
self.assertEqual(args.node_id, 0)
|
||||
self.assertEqual(merged["node_id"], 0)
|
||||
|
||||
|
||||
class TestStatePathSanitization(unittest.TestCase):
|
||||
def test_slashes_in_port_are_safe(self):
|
||||
path = provision._state_path_for("/dev/ttyUSB0", "/tmp/x")
|
||||
# Must not contain a raw slash in the basename
|
||||
self.assertNotIn("/", os.path.basename(path))
|
||||
|
||||
def test_windows_com_port_is_safe(self):
|
||||
path = provision._state_path_for("COM7", "/tmp/x")
|
||||
self.assertTrue(path.endswith("COM7.json"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1 +1 @@
|
||||
0.6.5
|
||||
0.6.6
|
||||
@@ -136,18 +136,42 @@ function extractAmplitude(iqBytes, nSubcarriers) {
|
||||
|
||||
/**
|
||||
* Load and parse a JSONL file, skipping blank/malformed lines.
|
||||
*
|
||||
* Reads byte-by-byte into Buffer slices to avoid Node's
|
||||
* `String.MaxLength` (~512 MB) cap that `readFileSync(_, 'utf8')` hits
|
||||
* on 30-min CSI recordings. Each line is decoded individually, so
|
||||
* memory use stays bounded by the largest single record.
|
||||
*/
|
||||
function loadJsonl(filePath) {
|
||||
const lines = fs.readFileSync(filePath, 'utf8').split('\n');
|
||||
const records = [];
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
try {
|
||||
records.push(JSON.parse(trimmed));
|
||||
} catch {
|
||||
// skip malformed lines
|
||||
const fd = fs.openSync(filePath, 'r');
|
||||
try {
|
||||
const bufSize = 1 << 20; // 1 MiB
|
||||
const buf = Buffer.alloc(bufSize);
|
||||
let leftover = '';
|
||||
let bytesRead;
|
||||
do {
|
||||
bytesRead = fs.readSync(fd, buf, 0, bufSize, null);
|
||||
if (bytesRead > 0) {
|
||||
const chunk = leftover + buf.toString('utf8', 0, bytesRead);
|
||||
const lines = chunk.split('\n');
|
||||
leftover = lines.pop(); // last fragment may be incomplete
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
try {
|
||||
records.push(JSON.parse(trimmed));
|
||||
} catch {
|
||||
// skip malformed lines
|
||||
}
|
||||
}
|
||||
}
|
||||
} while (bytesRead === bufSize);
|
||||
if (leftover.trim()) {
|
||||
try { records.push(JSON.parse(leftover.trim())); } catch {}
|
||||
}
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
return records;
|
||||
}
|
||||
@@ -184,8 +208,12 @@ function loadCsi(filePath) {
|
||||
const features = [];
|
||||
|
||||
for (const r of raw) {
|
||||
if (!r.timestamp) continue;
|
||||
const tsMs = isoToMs(r.timestamp);
|
||||
if (r.timestamp == null) continue;
|
||||
// Two timestamp formats: ISO string (legacy raw_csi/feature) or
|
||||
// numeric float-seconds (current sensing_update from the Rust server).
|
||||
const tsMs = typeof r.timestamp === 'number'
|
||||
? r.timestamp * 1000
|
||||
: isoToMs(r.timestamp);
|
||||
if (isNaN(tsMs)) continue;
|
||||
|
||||
if (r.type === 'raw_csi') {
|
||||
@@ -205,6 +233,33 @@ function loadCsi(filePath) {
|
||||
rssi: r.rssi,
|
||||
seq: r.seq,
|
||||
});
|
||||
} else if (r.type === 'sensing_update') {
|
||||
// Current sensing-server schema: one record per tick contains
|
||||
// already-extracted amplitudes per node plus a server-computed
|
||||
// feature vector. Project each into rawCsi/features so downstream
|
||||
// windowing/matrix extraction can reuse its existing paths.
|
||||
if (Array.isArray(r.nodes)) {
|
||||
for (const node of r.nodes) {
|
||||
if (!Array.isArray(node.amplitude) || node.amplitude.length === 0) continue;
|
||||
rawCsi.push({
|
||||
tsMs,
|
||||
nodeId: node.node_id,
|
||||
subcarriers: node.amplitude.length,
|
||||
amplitude: node.amplitude, // pre-extracted, no iq_hex needed
|
||||
rssi: node.rssi_dbm,
|
||||
seq: r.tick,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (Array.isArray(r.features) && r.features.length > 0) {
|
||||
features.push({
|
||||
tsMs,
|
||||
nodeId: 0,
|
||||
features: r.features,
|
||||
rssi: null,
|
||||
seq: r.tick,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -297,7 +352,11 @@ function extractCsiMatrix(window) {
|
||||
|
||||
for (let f = 0; f < nFrames; f++) {
|
||||
const frame = window[f];
|
||||
if (frame.iqHex) {
|
||||
if (frame.amplitude && frame.amplitude.length > 0) {
|
||||
// Already-extracted amplitudes from sensing_update — copy directly.
|
||||
const n = Math.min(nSc, frame.amplitude.length);
|
||||
for (let s = 0; s < n; s++) matrix[f * nSc + s] = frame.amplitude[s];
|
||||
} else if (frame.iqHex) {
|
||||
const iq = parseIqHex(frame.iqHex);
|
||||
const amp = extractAmplitude(iq, nSc);
|
||||
matrix.set(amp, f * nSc);
|
||||
@@ -422,12 +481,33 @@ function align() {
|
||||
? extractCsiMatrix(window)
|
||||
: extractFeatureMatrix(window);
|
||||
|
||||
// ADR-103: aggregate `n_persons` per window so the cog-person-count
|
||||
// training pipeline has count labels. Two summaries:
|
||||
// - `n_persons_mode` — modal value across the camera frames in
|
||||
// the window. Robust to single-frame noise;
|
||||
// this is the supervised label for the
|
||||
// categorical {0..7} count head.
|
||||
// - `n_persons_max` — the maximum value seen in the window.
|
||||
// Useful as a soft upper bound (e.g. for
|
||||
// dynamic dropout weighting during training).
|
||||
const personCounts = matched.map(f => f.nPersons ?? 0);
|
||||
const counts = new Map();
|
||||
for (const v of personCounts) counts.set(v, (counts.get(v) ?? 0) + 1);
|
||||
let modeVal = 0;
|
||||
let modeCount = -1;
|
||||
for (const [v, n] of counts) {
|
||||
if (n > modeCount) { modeVal = v; modeCount = n; }
|
||||
}
|
||||
const maxVal = personCounts.reduce((a, b) => Math.max(a, b), 0);
|
||||
|
||||
paired.push({
|
||||
csi: csiMatrix.data,
|
||||
csi_shape: csiMatrix.shape,
|
||||
kp: keypoints,
|
||||
conf: Math.round(avgConfidence * 1000) / 1000,
|
||||
n_camera_frames: matched.length,
|
||||
n_persons_mode: modeVal,
|
||||
n_persons_max: maxVal,
|
||||
ts_start: new Date(tStartMs).toISOString(),
|
||||
ts_end: new Date(tEndMs).toISOString(),
|
||||
});
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export pose_v1.safetensors -> pose_v1.onnx.
|
||||
|
||||
Builds the same architecture as v2/crates/cog-pose-estimation/src/inference.rs
|
||||
in PyTorch, loads the trained weights from safetensors, and runs a torch.onnx
|
||||
export with a fixed [1, 56, 20] input. Then verifies the ONNX loads and
|
||||
matches the torch output to within 1e-5.
|
||||
"""
|
||||
|
||||
import json
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
N_SUB = 56
|
||||
N_FRAMES = 20
|
||||
N_KP = 17
|
||||
|
||||
|
||||
class PoseNet(nn.Module):
|
||||
"""Mirrors inference.rs::PoseNet exactly."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.c1 = nn.Conv1d(N_SUB, 64, kernel_size=3, padding=1, dilation=1)
|
||||
self.c2 = nn.Conv1d(64, 128, kernel_size=3, padding=2, dilation=2)
|
||||
self.c3 = nn.Conv1d(128, 128, kernel_size=3, padding=4, dilation=4)
|
||||
self.fc1 = nn.Linear(128, 256)
|
||||
self.fc2 = nn.Linear(256, N_KP * 2)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
# x: [B, 56, 20]
|
||||
h = torch.relu(self.c1(x))
|
||||
h = torch.relu(self.c2(h))
|
||||
h = torch.relu(self.c3(h))
|
||||
h = h.mean(dim=2) # [B, 128]
|
||||
h = torch.relu(self.fc1(h))
|
||||
h = torch.sigmoid(self.fc2(h))
|
||||
return h
|
||||
|
||||
|
||||
def load_safetensors(path: Path) -> dict[str, torch.Tensor]:
|
||||
"""Pure-python safetensors reader. Avoids the safetensors pip dep."""
|
||||
with path.open("rb") as f:
|
||||
header_len = struct.unpack("<Q", f.read(8))[0]
|
||||
header = json.loads(f.read(header_len).decode("utf-8"))
|
||||
out: dict[str, torch.Tensor] = {}
|
||||
for name, meta in header.items():
|
||||
if name == "__metadata__":
|
||||
continue
|
||||
start, end = meta["data_offsets"]
|
||||
shape = meta["shape"]
|
||||
dtype = meta["dtype"]
|
||||
assert dtype == "F32", f"unsupported dtype {dtype} for {name}"
|
||||
f.seek(8 + header_len + start)
|
||||
buf = f.read(end - start)
|
||||
arr = np.frombuffer(buf, dtype=np.float32).copy().reshape(shape)
|
||||
out[name] = torch.from_numpy(arr)
|
||||
return out
|
||||
|
||||
|
||||
def main() -> None:
|
||||
weights_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("pose_v1.safetensors")
|
||||
out_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("pose_v1.onnx")
|
||||
|
||||
if not weights_path.exists():
|
||||
raise SystemExit(f"weights file not found: {weights_path}")
|
||||
|
||||
print(f"reading {weights_path}")
|
||||
tensors = load_safetensors(weights_path)
|
||||
print(f" found {len(tensors)} tensors: {sorted(tensors.keys())}")
|
||||
|
||||
model = PoseNet()
|
||||
# Map safetensors names (enc.c1.weight, head.fc1.weight, ...) to module params
|
||||
mapping = {
|
||||
"enc.c1.weight": "c1.weight",
|
||||
"enc.c1.bias": "c1.bias",
|
||||
"enc.c2.weight": "c2.weight",
|
||||
"enc.c2.bias": "c2.bias",
|
||||
"enc.c3.weight": "c3.weight",
|
||||
"enc.c3.bias": "c3.bias",
|
||||
"head.fc1.weight": "fc1.weight",
|
||||
"head.fc1.bias": "fc1.bias",
|
||||
"head.fc2.weight": "fc2.weight",
|
||||
"head.fc2.bias": "fc2.bias",
|
||||
}
|
||||
state = {dst: tensors[src] for src, dst in mapping.items()}
|
||||
model.load_state_dict(state)
|
||||
model.eval()
|
||||
print(" weights loaded into PyTorch model")
|
||||
|
||||
# Sanity check forward
|
||||
x = torch.zeros(1, N_SUB, N_FRAMES)
|
||||
with torch.no_grad():
|
||||
y = model(x)
|
||||
print(f" zero-input forward: shape={tuple(y.shape)} sample={y[0, :4].tolist()}")
|
||||
|
||||
# Export to ONNX
|
||||
torch.onnx.export(
|
||||
model,
|
||||
x,
|
||||
out_path,
|
||||
export_params=True,
|
||||
opset_version=18,
|
||||
do_constant_folding=True,
|
||||
input_names=["csi_window"],
|
||||
output_names=["keypoints"],
|
||||
dynamic_axes={"csi_window": {0: "batch"}, "keypoints": {0: "batch"}},
|
||||
)
|
||||
print(f" wrote {out_path} ({out_path.stat().st_size} bytes)")
|
||||
|
||||
# Verify the ONNX file loads + matches torch output
|
||||
try:
|
||||
import onnx
|
||||
import onnxruntime as ort
|
||||
|
||||
onnx_model = onnx.load(str(out_path))
|
||||
onnx.checker.check_model(onnx_model)
|
||||
print(" ONNX model checker: ok")
|
||||
|
||||
sess = ort.InferenceSession(str(out_path), providers=["CPUExecutionProvider"])
|
||||
rng = np.random.default_rng(42)
|
||||
x_np = rng.standard_normal((1, N_SUB, N_FRAMES), dtype=np.float32)
|
||||
with torch.no_grad():
|
||||
y_torch = model(torch.from_numpy(x_np)).numpy()
|
||||
y_onnx = sess.run(["keypoints"], {"csi_window": x_np})[0]
|
||||
max_abs = float(np.max(np.abs(y_torch - y_onnx)))
|
||||
print(f" parity vs torch: max |torch - onnx| = {max_abs:.2e}")
|
||||
assert max_abs < 1e-5, "ONNX output diverges from torch output"
|
||||
print(" parity ok (<1e-5)")
|
||||
except ImportError as e:
|
||||
print(f" WARN: onnx/onnxruntime not installed, skipping verification: {e}")
|
||||
|
||||
print("\nDone.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -213,6 +213,26 @@
|
||||
],
|
||||
"rationale": "Without quantization, the SHA-256 of features_to_bytes() diverges across SIMD backends (Intel AVX2/AVX-512 vs Apple Silicon NEON) because scipy.fft's pocketfft kernels reorder vectorized FP operations differently per build. IEEE 754 guarantees per-operation determinism, not associativity. Rounding to 9 decimal places (~5 orders of magnitude headroom over observed ULP drift) collapses the cross-platform divergence to a single canonical hash. Removing the round() call reintroduces the macOS arm64 vs Linux x86_64 hash mismatch in issue #560.",
|
||||
"ref": "https://github.com/ruvnet/RuView/issues/560"
|
||||
},
|
||||
{
|
||||
"id": "RuView#679",
|
||||
"title": "ESP32-S3 CSI: csi_collector_set_node_id() called before wifi_init_sta() so node_id is never clobbered",
|
||||
"files": ["firmware/esp32-csi-node/main/main.c"],
|
||||
"require": ["csi_collector_set_node_id"],
|
||||
"forbid": ["/csi_collector_init.*node_id\\s*=\\s*1[^0-9]/"],
|
||||
"rationale": "release_bins/ shipped v0.4.3.1 binaries that lacked csi_collector_set_node_id() — every provisioned node reported node_id=1 over UDP regardless of NVS value, making a 4-node deployment look like a single node. main.c must call csi_collector_set_node_id(g_nvs_config.node_id) immediately after nvs_config_load() and before wifi_init_sta(). Reverting silently breaks multi-node deployments with no build-time error.",
|
||||
"ref": "https://github.com/ruvnet/RuView/issues/679"
|
||||
},
|
||||
{
|
||||
"id": "RuView#683",
|
||||
"title": "ESP32-S3 edge tier>=2: vTaskDelay(1) after multi-person vitals and WASM dispatch prevents IDLE1 starvation / WDT storm",
|
||||
"files": ["firmware/esp32-csi-node/main/edge_processing.c"],
|
||||
"require": [
|
||||
"if (s_cfg.tier >= 2) vTaskDelay(1);",
|
||||
"Yield after WASM dispatch to feed Core 1 watchdog (#683)"
|
||||
],
|
||||
"rationale": "At edge tier>=2 on N16R8 PSRAM boards, process_frame() runs update_multi_person_vitals() (4 persons × 256 history samples) plus wasm_runtime_on_frame() back-to-back. The vTaskDelay(1) in edge_task() only fires AFTER process_frame() fully returns — if process_frame() takes >5 s (common on PSRAM-backed boards under sustained 30 pps CSI load), IDLE1 on Core 1 never runs and the Task Watchdog Timer fires. The fix adds two vTaskDelay(1) calls inside process_frame(), gated on tier>=2, at the multi-person vitals boundary and after WASM dispatch. Removing them re-opens the WDT storm on N16R8 hardware.",
|
||||
"ref": "https://github.com/ruvnet/RuView/issues/683"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,761 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Train the person-count head — ADR-103 v0.0.1.
|
||||
|
||||
Mirrors the Conv1d encoder architecture from cog-person-count's
|
||||
`src/inference.rs::CountNet` exactly, so the learned weights load
|
||||
into the Rust cog without translation. Trains on
|
||||
data/paired/wiflow-p7-1779210883.paired.jsonl (1,077 samples with
|
||||
n_persons_mode labels in {0, 1}).
|
||||
|
||||
Output: count_v1.safetensors + count_v1.onnx + train_results.json.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import struct
|
||||
import time
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
# Architecture constants — MUST match cog-person-count's src/inference.rs.
|
||||
N_SUB = 56
|
||||
N_FRAMES = 20
|
||||
COUNT_CLASSES = 8
|
||||
|
||||
|
||||
class CountNet(nn.Module):
|
||||
"""Mirrors cog_person_count::inference::CountNet bit-for-bit."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
# Encoder — identical to the pose cog's encoder so future joint
|
||||
# training can share weights.
|
||||
self.enc_c1 = nn.Conv1d(N_SUB, 64, kernel_size=3, padding=1, dilation=1)
|
||||
self.enc_c2 = nn.Conv1d(64, 128, kernel_size=3, padding=2, dilation=2)
|
||||
self.enc_c3 = nn.Conv1d(128, 128, kernel_size=3, padding=4, dilation=4)
|
||||
# Count head
|
||||
self.count_head_fc1 = nn.Linear(128, 64)
|
||||
self.count_head_fc2 = nn.Linear(64, COUNT_CLASSES)
|
||||
# Confidence head
|
||||
self.conf_head_fc1 = nn.Linear(128, 32)
|
||||
self.conf_head_fc2 = nn.Linear(32, 1)
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
# x: [B, 56, 20]
|
||||
h = F.relu(self.enc_c1(x))
|
||||
h = F.relu(self.enc_c2(h))
|
||||
h = F.relu(self.enc_c3(h))
|
||||
h = h.mean(dim=2) # [B, 128]
|
||||
|
||||
# Logits (un-normalised); softmax at inference + cross-entropy training.
|
||||
c = F.relu(self.count_head_fc1(h))
|
||||
count_logits = self.count_head_fc2(c)
|
||||
|
||||
# Confidence head — sigmoid at inference; BCE-with-logits at training.
|
||||
cf = F.relu(self.conf_head_fc1(h))
|
||||
conf_logits = self.conf_head_fc2(cf)
|
||||
|
||||
return count_logits, conf_logits
|
||||
|
||||
|
||||
def load_paired(path: Path) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""Return (X, y) where X is [N, 56, 20] CSI and y is [N] integer counts."""
|
||||
csis, ys = [], []
|
||||
with path.open(encoding="utf-8") as f:
|
||||
for line in f:
|
||||
if not line.strip():
|
||||
continue
|
||||
d = json.loads(line)
|
||||
shape = d.get("csi_shape", [N_SUB, N_FRAMES])
|
||||
if shape != [N_SUB, N_FRAMES]:
|
||||
continue
|
||||
csi = np.asarray(d["csi"], dtype=np.float32).reshape(N_SUB, N_FRAMES)
|
||||
csis.append(csi)
|
||||
ys.append(int(d.get("n_persons_mode", 0)))
|
||||
X = np.stack(csis, axis=0)
|
||||
y = np.asarray(ys, dtype=np.int64)
|
||||
return X, y
|
||||
|
||||
|
||||
def temporal_split(X: np.ndarray, y: np.ndarray, eval_frac: float = 0.2):
|
||||
"""Held-out time-window eval (last `eval_frac` of samples, by index)."""
|
||||
n = X.shape[0]
|
||||
n_eval = int(round(n * eval_frac))
|
||||
n_train = n - n_eval
|
||||
return (
|
||||
X[:n_train], y[:n_train],
|
||||
X[n_train:], y[n_train:],
|
||||
)
|
||||
|
||||
|
||||
def stratified_k_fold(X: np.ndarray, y: np.ndarray, k: int = 5):
|
||||
"""Stratified k-fold cross-validation splits — hand-rolled, no sklearn.
|
||||
|
||||
Per class: shuffle the indices (deterministic seed 42), split into k
|
||||
near-equal chunks, then assemble fold i by taking chunk i from every
|
||||
class. Yields (X_train, y_train, X_val, y_val) per fold, with class
|
||||
distribution preserved within ±1.
|
||||
"""
|
||||
rng = np.random.default_rng(seed=42)
|
||||
classes = np.unique(y)
|
||||
per_class_folds = {}
|
||||
for c in classes:
|
||||
idx = np.where(y == c)[0]
|
||||
rng.shuffle(idx)
|
||||
per_class_folds[c] = np.array_split(idx, k)
|
||||
for fold in range(k):
|
||||
val_idx = np.concatenate([per_class_folds[c][fold] for c in classes])
|
||||
train_idx = np.concatenate(
|
||||
[per_class_folds[c][f] for c in classes for f in range(k) if f != fold]
|
||||
)
|
||||
yield X[train_idx], y[train_idx], X[val_idx], y[val_idx]
|
||||
|
||||
|
||||
def standardise(X_train: np.ndarray, X_eval: np.ndarray):
|
||||
"""Z-score by subcarrier across the time axis. Eval uses train stats."""
|
||||
mu = X_train.mean(axis=(0, 2), keepdims=True)
|
||||
sd = X_train.std(axis=(0, 2), keepdims=True) + 1e-6
|
||||
return (X_train - mu) / sd, (X_eval - mu) / sd
|
||||
|
||||
|
||||
def write_safetensors(model: CountNet, path: Path):
|
||||
"""Write the model's state in the same on-disk layout the Rust cog expects."""
|
||||
state = model.state_dict()
|
||||
# Map PyTorch param names → cog-person-count's VarBuilder paths.
|
||||
rename = {
|
||||
"enc_c1.weight": "enc.c1.weight",
|
||||
"enc_c1.bias": "enc.c1.bias",
|
||||
"enc_c2.weight": "enc.c2.weight",
|
||||
"enc_c2.bias": "enc.c2.bias",
|
||||
"enc_c3.weight": "enc.c3.weight",
|
||||
"enc_c3.bias": "enc.c3.bias",
|
||||
"count_head_fc1.weight": "count_head.fc1.weight",
|
||||
"count_head_fc1.bias": "count_head.fc1.bias",
|
||||
"count_head_fc2.weight": "count_head.fc2.weight",
|
||||
"count_head_fc2.bias": "count_head.fc2.bias",
|
||||
"conf_head_fc1.weight": "conf_head.fc1.weight",
|
||||
"conf_head_fc1.bias": "conf_head.fc1.bias",
|
||||
"conf_head_fc2.weight": "conf_head.fc2.weight",
|
||||
"conf_head_fc2.bias": "conf_head.fc2.bias",
|
||||
}
|
||||
|
||||
header = {}
|
||||
payload = bytearray()
|
||||
offset = 0
|
||||
for torch_name, cog_name in rename.items():
|
||||
t = state[torch_name].detach().cpu().numpy().astype(np.float32)
|
||||
n_bytes = t.nbytes
|
||||
header[cog_name] = {
|
||||
"dtype": "F32",
|
||||
"shape": list(t.shape),
|
||||
"data_offsets": [offset, offset + n_bytes],
|
||||
}
|
||||
payload.extend(t.tobytes())
|
||||
offset += n_bytes
|
||||
|
||||
header_bytes = json.dumps(header, separators=(",", ":")).encode("utf-8")
|
||||
with path.open("wb") as f:
|
||||
f.write(struct.pack("<Q", len(header_bytes)))
|
||||
f.write(header_bytes)
|
||||
f.write(payload)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--paired", required=True)
|
||||
parser.add_argument("--out-safetensors", default="count_v1.safetensors")
|
||||
parser.add_argument("--out-onnx", default="count_v1.onnx")
|
||||
parser.add_argument("--out-results", default="count_train_results.json")
|
||||
parser.add_argument("--epochs", type=int, default=400)
|
||||
parser.add_argument("--batch-size", type=int, default=64)
|
||||
parser.add_argument("--lr", type=float, default=1e-3)
|
||||
parser.add_argument("--weight-decay", type=float, default=0.01)
|
||||
parser.add_argument("--k-fold", type=int, default=None, help="If set, run k-fold CV; else use temporal split")
|
||||
parser.add_argument("--v2", action="store_true",
|
||||
help="v0.0.2 training: random 80/20 split + label smoothing + early stopping "
|
||||
"+ balanced sampling + temperature-scaled confidence head.")
|
||||
parser.add_argument("--label-smoothing", type=float, default=0.1)
|
||||
parser.add_argument("--patience", type=int, default=20)
|
||||
args = parser.parse_args()
|
||||
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
print(f"device: {device}")
|
||||
|
||||
X, y = load_paired(Path(args.paired))
|
||||
print(f"loaded {X.shape[0]} samples, X shape {X.shape}, "
|
||||
f"label distribution: {dict(Counter(y.tolist()).most_common())}")
|
||||
|
||||
# K-fold cross-validation mode
|
||||
if args.k_fold is not None:
|
||||
print(f"\n=== {args.k_fold}-fold cross-validation ===")
|
||||
fold_results = []
|
||||
overall_t0 = time.perf_counter()
|
||||
|
||||
for fold_idx, (X_train, y_train, X_val, y_val) in enumerate(stratified_k_fold(X, y, k=args.k_fold)):
|
||||
print(f"\nFold {fold_idx + 1}/{args.k_fold}")
|
||||
X_train, X_val = standardise(X_train, X_val)
|
||||
|
||||
cls_counts = np.bincount(y_train, minlength=COUNT_CLASSES).astype(np.float32)
|
||||
cls_counts = np.where(cls_counts > 0, cls_counts, 1.0)
|
||||
cls_weight = (1.0 / cls_counts) / (1.0 / cls_counts).sum() * COUNT_CLASSES
|
||||
cls_weight_t = torch.from_numpy(cls_weight).to(device)
|
||||
|
||||
Xt = torch.from_numpy(X_train).to(device)
|
||||
yt = torch.from_numpy(y_train).to(device)
|
||||
Xv = torch.from_numpy(X_val).to(device)
|
||||
yv = torch.from_numpy(y_val).to(device)
|
||||
|
||||
model = CountNet().to(device)
|
||||
opt = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
|
||||
sched = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(opt, T_0=50, T_mult=1)
|
||||
|
||||
n_train = X_train.shape[0]
|
||||
best_eval_acc = 0.0
|
||||
best_state = None
|
||||
|
||||
for epoch in range(args.epochs):
|
||||
model.train()
|
||||
perm = torch.randperm(n_train, device=device)
|
||||
train_loss = 0.0
|
||||
train_correct = 0
|
||||
n_batches = 0
|
||||
for i in range(0, n_train, args.batch_size):
|
||||
idx = perm[i : i + args.batch_size]
|
||||
xb = Xt[idx]
|
||||
yb = yt[idx]
|
||||
opt.zero_grad()
|
||||
count_logits, conf_logits = model(xb)
|
||||
ce = F.cross_entropy(count_logits, yb, weight=cls_weight_t)
|
||||
with torch.no_grad():
|
||||
pred = count_logits.argmax(dim=1)
|
||||
correct_indicator = (pred == yb).float().unsqueeze(1)
|
||||
bce = F.binary_cross_entropy_with_logits(conf_logits, correct_indicator)
|
||||
with torch.no_grad():
|
||||
conf_sigm = torch.sigmoid(conf_logits)
|
||||
brier = ((conf_sigm - correct_indicator) ** 2).mean()
|
||||
loss = ce + 0.3 * bce + 0.1 * brier
|
||||
loss.backward()
|
||||
opt.step()
|
||||
train_loss += loss.item()
|
||||
train_correct += (pred == yb).sum().item()
|
||||
n_batches += 1
|
||||
|
||||
sched.step()
|
||||
|
||||
model.eval()
|
||||
with torch.no_grad():
|
||||
cl_v, _ = model(Xv)
|
||||
eval_pred = cl_v.argmax(dim=1)
|
||||
eval_acc = (eval_pred == yv).float().mean().item()
|
||||
|
||||
if eval_acc > best_eval_acc:
|
||||
best_eval_acc = eval_acc
|
||||
best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}
|
||||
|
||||
# Restore best checkpoint and final eval
|
||||
if best_state is not None:
|
||||
model.load_state_dict(best_state)
|
||||
|
||||
model.eval()
|
||||
with torch.no_grad():
|
||||
cl_v, conf_v = model(Xv)
|
||||
pred_v = cl_v.argmax(dim=1)
|
||||
acc = (pred_v == yv).float().mean().item()
|
||||
within1 = ((pred_v - yv).abs() <= 1).float().mean().item()
|
||||
mae = (pred_v - yv).abs().float().mean().item()
|
||||
|
||||
# Per-class accuracy
|
||||
per_class = {}
|
||||
for k in range(COUNT_CLASSES):
|
||||
mask = yv == k
|
||||
n = mask.sum().item()
|
||||
if n > 0:
|
||||
per_class[k] = {
|
||||
"support": int(n),
|
||||
"accuracy": ((pred_v == yv) & mask).sum().item() / n,
|
||||
}
|
||||
|
||||
# Spearman
|
||||
conf_sigm = torch.sigmoid(conf_v).squeeze(-1)
|
||||
correct = (pred_v == yv).float()
|
||||
c_rank = conf_sigm.argsort().argsort().float()
|
||||
r_rank = correct.argsort().argsort().float()
|
||||
c_centered = c_rank - c_rank.mean()
|
||||
r_centered = r_rank - r_rank.mean()
|
||||
denom = (c_centered.norm() * r_centered.norm()).item()
|
||||
spearman = (c_centered * r_centered).sum().item() / denom if denom > 0 else 0.0
|
||||
|
||||
fold_results.append({
|
||||
"fold": fold_idx + 1,
|
||||
"accuracy": acc,
|
||||
"within_pm1": within1,
|
||||
"mae": mae,
|
||||
"spearman": spearman,
|
||||
"per_class_accuracy": per_class,
|
||||
})
|
||||
print(f" accuracy={acc:.3f} within±1={within1:.3f} mae={mae:.3f} spearman={spearman:.3f}")
|
||||
|
||||
# K-fold summary
|
||||
total_time = time.perf_counter() - overall_t0
|
||||
accs = [r["accuracy"] for r in fold_results]
|
||||
within1s = [r["within_pm1"] for r in fold_results]
|
||||
maes = [r["mae"] for r in fold_results]
|
||||
spears = [r["spearman"] for r in fold_results]
|
||||
|
||||
print(f"\n=== {args.k_fold}-fold summary ({total_time:.1f} s) ===")
|
||||
print(f" accuracy: {np.mean(accs):.3f} ± {np.std(accs):.3f}")
|
||||
print(f" within ±1: {np.mean(within1s):.3f} ± {np.std(within1s):.3f}")
|
||||
print(f" MAE: {np.mean(maes):.3f} ± {np.std(maes):.3f}")
|
||||
print(f" conf↔correct Spearman: {np.mean(spears):.3f} ± {np.std(spears):.3f}")
|
||||
|
||||
# Per-class summary across folds
|
||||
for k in range(COUNT_CLASSES):
|
||||
accs_k = [r["per_class_accuracy"].get(k, {}).get("accuracy", 0.0) for r in fold_results]
|
||||
n_k = [r["per_class_accuracy"].get(k, {}).get("support", 0) for r in fold_results]
|
||||
if any(n > 0 for n in n_k):
|
||||
print(f" class {k}: {np.mean(accs_k):.3f} mean accuracy (support: {n_k})")
|
||||
|
||||
# Write k-fold results to JSON
|
||||
results = {
|
||||
"mode": "k_fold_cv",
|
||||
"k": args.k_fold,
|
||||
"backend": "pytorch-cuda" if device.type == "cuda" else "pytorch-cpu",
|
||||
"total_time_s": total_time,
|
||||
"fold_results": fold_results,
|
||||
"summary": {
|
||||
"mean_accuracy": float(np.mean(accs)),
|
||||
"std_accuracy": float(np.std(accs)),
|
||||
"mean_within_pm1": float(np.mean(within1s)),
|
||||
"std_within_pm1": float(np.std(within1s)),
|
||||
"mean_mae": float(np.mean(maes)),
|
||||
"std_mae": float(np.std(maes)),
|
||||
"mean_spearman": float(np.mean(spears)),
|
||||
"std_spearman": float(np.std(spears)),
|
||||
},
|
||||
"hyperparameters": {
|
||||
"optimizer": "AdamW",
|
||||
"lr": args.lr,
|
||||
"weight_decay": args.weight_decay,
|
||||
"batch_size": args.batch_size,
|
||||
"schedule": "cosine_warm_restarts",
|
||||
"epochs": args.epochs,
|
||||
},
|
||||
}
|
||||
Path(args.out_results).write_text(json.dumps(results, indent=2))
|
||||
print(f"\nwrote {args.out_results}")
|
||||
return
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# v0.0.2 training path: random 80/20 + label smoothing + early
|
||||
# stopping + class-balanced batch sampling + temperature scaling.
|
||||
# ---------------------------------------------------------------
|
||||
if args.v2:
|
||||
rng = np.random.default_rng(seed=42)
|
||||
idx = np.arange(X.shape[0])
|
||||
rng.shuffle(idx)
|
||||
n_eval = int(round(0.2 * X.shape[0]))
|
||||
eval_idx, train_idx = idx[:n_eval], idx[n_eval:]
|
||||
X_train, X_eval = X[train_idx], X[eval_idx]
|
||||
y_train, y_eval = y[train_idx], y[eval_idx]
|
||||
X_train, X_eval = standardise(X_train, X_eval)
|
||||
print(f"v0.0.2 mode — random 80/20 split: train={len(y_train)} eval={len(y_eval)}")
|
||||
print(f" train class dist: {dict(Counter(y_train.tolist()).most_common())}")
|
||||
print(f" eval class dist: {dict(Counter(y_eval.tolist()).most_common())}")
|
||||
|
||||
Xt = torch.from_numpy(X_train).to(device)
|
||||
yt = torch.from_numpy(y_train).to(device)
|
||||
Xe = torch.from_numpy(X_eval).to(device)
|
||||
ye = torch.from_numpy(y_eval).to(device)
|
||||
|
||||
# Class-balanced sampler: for each batch, sample with replacement
|
||||
# so each class has equal expected count regardless of dataset
|
||||
# distribution. With our ~533/544 split this is nearly a no-op
|
||||
# but it generalises to imbalanced multi-room data later.
|
||||
cls_counts = np.bincount(y_train, minlength=COUNT_CLASSES).astype(np.float32)
|
||||
cls_counts = np.where(cls_counts > 0, cls_counts, 1.0)
|
||||
per_sample_weight = (1.0 / cls_counts[y_train])
|
||||
per_sample_weight_t = torch.from_numpy(per_sample_weight.astype(np.float32)).to(device)
|
||||
|
||||
model = CountNet().to(device)
|
||||
opt = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
|
||||
sched = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(opt, T_0=50, T_mult=1)
|
||||
|
||||
n_train = X_train.shape[0]
|
||||
batches_per_epoch = max(1, n_train // args.batch_size)
|
||||
epoch_losses = []
|
||||
t0 = time.perf_counter()
|
||||
best_eval_acc = 0.0
|
||||
best_state = None
|
||||
epochs_without_improvement = 0
|
||||
|
||||
for epoch in range(args.epochs):
|
||||
model.train()
|
||||
train_loss = 0.0; train_correct = 0; n_batches = 0
|
||||
for _ in range(batches_per_epoch):
|
||||
# Balanced sample with replacement
|
||||
idx_t = torch.multinomial(per_sample_weight_t, args.batch_size, replacement=True)
|
||||
xb = Xt[idx_t]; yb = yt[idx_t]
|
||||
opt.zero_grad()
|
||||
count_logits, conf_logits = model(xb)
|
||||
ce = F.cross_entropy(count_logits, yb, label_smoothing=args.label_smoothing)
|
||||
with torch.no_grad():
|
||||
pred = count_logits.argmax(dim=1)
|
||||
correct_indicator = (pred == yb).float().unsqueeze(1)
|
||||
bce = F.binary_cross_entropy_with_logits(conf_logits, correct_indicator)
|
||||
with torch.no_grad():
|
||||
conf_sigm = torch.sigmoid(conf_logits)
|
||||
brier = ((conf_sigm - correct_indicator) ** 2).mean()
|
||||
loss = ce + 0.3 * bce + 0.1 * brier
|
||||
loss.backward()
|
||||
opt.step()
|
||||
train_loss += loss.item()
|
||||
train_correct += (pred == yb).sum().item()
|
||||
n_batches += 1
|
||||
sched.step()
|
||||
|
||||
model.eval()
|
||||
with torch.no_grad():
|
||||
cl_e, _ = model(Xe)
|
||||
eval_loss = F.cross_entropy(cl_e, ye).item()
|
||||
eval_pred = cl_e.argmax(dim=1)
|
||||
eval_acc = (eval_pred == ye).float().mean().item()
|
||||
epoch_losses.append({
|
||||
"epoch": epoch,
|
||||
"train_loss": train_loss / max(1, n_batches),
|
||||
"train_acc": train_correct / max(1, n_batches * args.batch_size),
|
||||
"eval_loss": eval_loss,
|
||||
"eval_acc": eval_acc,
|
||||
})
|
||||
if eval_acc > best_eval_acc:
|
||||
best_eval_acc = eval_acc
|
||||
best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}
|
||||
epochs_without_improvement = 0
|
||||
else:
|
||||
epochs_without_improvement += 1
|
||||
|
||||
if epoch < 5 or epoch % 25 == 0:
|
||||
print(f"epoch {epoch:3d} train_loss={train_loss/n_batches:.4f} "
|
||||
f"train_acc={train_correct/(n_batches*args.batch_size):.3f} "
|
||||
f"eval_loss={eval_loss:.4f} eval_acc={eval_acc:.3f} "
|
||||
f"epochs_no_improve={epochs_without_improvement}")
|
||||
if epochs_without_improvement >= args.patience:
|
||||
print(f"early stopping at epoch {epoch} (no improvement for {args.patience} epochs)")
|
||||
break
|
||||
|
||||
train_time = time.perf_counter() - t0
|
||||
print(f"\ntrained {epoch + 1} epochs in {train_time:.1f} s (best eval_acc {best_eval_acc:.3f})")
|
||||
if best_state is not None:
|
||||
model.load_state_dict(best_state)
|
||||
|
||||
# Temperature scaling on the confidence head — fit a scalar T s.t.
|
||||
# sigmoid(conf_logits / T) is best-calibrated on the eval set.
|
||||
model.eval()
|
||||
with torch.no_grad():
|
||||
cl_e, conf_e = model(Xe)
|
||||
pred_e = cl_e.argmax(dim=1)
|
||||
correct_indicator = (pred_e == ye).float()
|
||||
# 1D optimisation over T via LBFGS.
|
||||
T = torch.nn.Parameter(torch.ones(1, device=device))
|
||||
opt_t = torch.optim.LBFGS([T], lr=0.1, max_iter=50)
|
||||
def eval_t():
|
||||
opt_t.zero_grad()
|
||||
scaled = conf_e.squeeze(-1) / T
|
||||
loss_t = F.binary_cross_entropy_with_logits(scaled, correct_indicator)
|
||||
loss_t.backward()
|
||||
return loss_t
|
||||
opt_t.step(eval_t)
|
||||
T_val = float(T.detach().cpu().item())
|
||||
print(f" temperature scale T = {T_val:.4f}")
|
||||
|
||||
# Final eval with temperature applied.
|
||||
with torch.no_grad():
|
||||
cl_e, conf_e = model(Xe)
|
||||
probs_e = F.softmax(cl_e, dim=1)
|
||||
pred_e = cl_e.argmax(dim=1)
|
||||
acc = (pred_e == ye).float().mean().item()
|
||||
within1 = ((pred_e - ye).abs() <= 1).float().mean().item()
|
||||
mae = (pred_e - ye).abs().float().mean().item()
|
||||
per_class = {}
|
||||
for k in range(COUNT_CLASSES):
|
||||
mask = ye == k
|
||||
n = mask.sum().item()
|
||||
if n > 0:
|
||||
per_class[k] = {
|
||||
"support": int(n),
|
||||
"accuracy": ((pred_e == ye) & mask).sum().item() / n,
|
||||
}
|
||||
conf_sigm = torch.sigmoid(conf_e.squeeze(-1) / T_val)
|
||||
correct = (pred_e == ye).float()
|
||||
c_rank = conf_sigm.argsort().argsort().float()
|
||||
r_rank = correct.argsort().argsort().float()
|
||||
c_centered = c_rank - c_rank.mean()
|
||||
r_centered = r_rank - r_rank.mean()
|
||||
denom = (c_centered.norm() * r_centered.norm()).item()
|
||||
spearman = (c_centered * r_centered).sum().item() / denom if denom > 0 else 0.0
|
||||
|
||||
print(f"\n=== v0.0.2 final eval ===")
|
||||
print(f" accuracy: {acc:.3f}")
|
||||
print(f" within ±1: {within1:.3f}")
|
||||
print(f" MAE: {mae:.3f}")
|
||||
print(f" conf↔correct Spearman (post-temp): {spearman:.3f}")
|
||||
for k, v in per_class.items():
|
||||
print(f" class {k}: {v['accuracy']:.3f} accuracy on {v['support']} samples")
|
||||
|
||||
write_safetensors(model, Path(args.out_safetensors))
|
||||
# Also append the temperature scalar so the cog can apply it.
|
||||
# We add it by appending to the safetensors file using the
|
||||
# write_safetensors helper but with the temperature recorded
|
||||
# as a separate file alongside (count_v1.temperature.txt) for
|
||||
# consumption by the Rust cog inference path.
|
||||
Path(args.out_safetensors + ".temperature").write_text(f"{T_val}\n")
|
||||
print(f"wrote {args.out_safetensors} ({Path(args.out_safetensors).stat().st_size} bytes)")
|
||||
print(f"wrote {args.out_safetensors}.temperature ({T_val})")
|
||||
|
||||
# ONNX
|
||||
dummy = torch.zeros(1, N_SUB, N_FRAMES, device=device)
|
||||
try:
|
||||
torch.onnx.export(model, dummy, args.out_onnx, opset_version=18,
|
||||
input_names=["csi_window"],
|
||||
output_names=["count_logits", "conf_logits"],
|
||||
dynamic_axes={"csi_window": {0: "batch"},
|
||||
"count_logits": {0: "batch"},
|
||||
"conf_logits": {0: "batch"}},
|
||||
export_params=True, do_constant_folding=True)
|
||||
print(f"wrote {args.out_onnx} ({Path(args.out_onnx).stat().st_size} bytes)")
|
||||
except Exception as e:
|
||||
print(f"WARN: ONNX export failed: {e}")
|
||||
|
||||
results = {
|
||||
"mode": "v0.0.2",
|
||||
"backend": "pytorch-cuda" if device.type == "cuda" else "pytorch-cpu",
|
||||
"epochs_trained": epoch + 1,
|
||||
"train_time_s": train_time,
|
||||
"best_eval_acc": best_eval_acc,
|
||||
"final_eval_acc": acc,
|
||||
"final_eval_within_pm1": within1,
|
||||
"final_eval_mae": mae,
|
||||
"temperature_scale": T_val,
|
||||
"conf_correctness_spearman_post_temp": spearman,
|
||||
"per_class_accuracy": per_class,
|
||||
"hyperparameters": {
|
||||
"optimizer": "AdamW",
|
||||
"lr": args.lr,
|
||||
"weight_decay": args.weight_decay,
|
||||
"batch_size": args.batch_size,
|
||||
"schedule": "cosine_warm_restarts",
|
||||
"epochs_max": args.epochs,
|
||||
"label_smoothing": args.label_smoothing,
|
||||
"patience": args.patience,
|
||||
"split": "random_80_20_seed_42",
|
||||
"balanced_sampler": True,
|
||||
"temperature_scaling": True,
|
||||
},
|
||||
"epoch_losses": epoch_losses,
|
||||
}
|
||||
Path(args.out_results).write_text(json.dumps(results, indent=2))
|
||||
print(f"wrote {args.out_results}")
|
||||
return
|
||||
|
||||
# Original temporal-split mode (kept for v0.0.1 reproducibility).
|
||||
X_train, y_train, X_eval, y_eval = temporal_split(X, y, eval_frac=0.2)
|
||||
X_train, X_eval = standardise(X_train, X_eval)
|
||||
|
||||
# Re-balance via class weights — handles the 50/50 split fine
|
||||
# but also makes the loss correct under future imbalanced data.
|
||||
cls_counts = np.bincount(y_train, minlength=COUNT_CLASSES).astype(np.float32)
|
||||
cls_counts = np.where(cls_counts > 0, cls_counts, 1.0)
|
||||
cls_weight = (1.0 / cls_counts) / (1.0 / cls_counts).sum() * COUNT_CLASSES
|
||||
cls_weight_t = torch.from_numpy(cls_weight).to(device)
|
||||
print(f"class weights: {cls_weight.tolist()}")
|
||||
|
||||
Xt = torch.from_numpy(X_train).to(device)
|
||||
yt = torch.from_numpy(y_train).to(device)
|
||||
Xe = torch.from_numpy(X_eval).to(device)
|
||||
ye = torch.from_numpy(y_eval).to(device)
|
||||
|
||||
model = CountNet().to(device)
|
||||
opt = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
|
||||
sched = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(opt, T_0=50, T_mult=1)
|
||||
|
||||
n_train = X_train.shape[0]
|
||||
epoch_losses = []
|
||||
t0 = time.perf_counter()
|
||||
|
||||
best_eval_acc = 0.0
|
||||
best_state = None
|
||||
|
||||
for epoch in range(args.epochs):
|
||||
model.train()
|
||||
perm = torch.randperm(n_train, device=device)
|
||||
train_loss = 0.0
|
||||
train_correct = 0
|
||||
n_batches = 0
|
||||
for i in range(0, n_train, args.batch_size):
|
||||
idx = perm[i : i + args.batch_size]
|
||||
xb = Xt[idx]
|
||||
yb = yt[idx]
|
||||
opt.zero_grad()
|
||||
count_logits, conf_logits = model(xb)
|
||||
|
||||
# Categorical cross-entropy for count.
|
||||
ce = F.cross_entropy(count_logits, yb, weight=cls_weight_t)
|
||||
|
||||
# Confidence head: train against `argmax == truth` indicator.
|
||||
with torch.no_grad():
|
||||
pred = count_logits.argmax(dim=1)
|
||||
correct_indicator = (pred == yb).float().unsqueeze(1)
|
||||
bce = F.binary_cross_entropy_with_logits(conf_logits, correct_indicator)
|
||||
|
||||
# Brier-score uncertainty calibration on the conf head — sharpens
|
||||
# the calibration so the sigmoid output is a real probability.
|
||||
with torch.no_grad():
|
||||
conf_sigm = torch.sigmoid(conf_logits)
|
||||
brier = ((conf_sigm - correct_indicator) ** 2).mean()
|
||||
|
||||
loss = ce + 0.3 * bce + 0.1 * brier
|
||||
loss.backward()
|
||||
opt.step()
|
||||
|
||||
train_loss += loss.item()
|
||||
train_correct += (pred == yb).sum().item()
|
||||
n_batches += 1
|
||||
|
||||
sched.step()
|
||||
|
||||
model.eval()
|
||||
with torch.no_grad():
|
||||
cl_e, _ = model(Xe)
|
||||
eval_loss = F.cross_entropy(cl_e, ye, weight=cls_weight_t).item()
|
||||
eval_pred = cl_e.argmax(dim=1)
|
||||
eval_acc = (eval_pred == ye).float().mean().item()
|
||||
eval_within1 = ((eval_pred - ye).abs() <= 1).float().mean().item()
|
||||
|
||||
epoch_losses.append({
|
||||
"epoch": epoch,
|
||||
"train_loss": train_loss / n_batches,
|
||||
"train_acc": train_correct / n_train,
|
||||
"eval_loss": eval_loss,
|
||||
"eval_acc": eval_acc,
|
||||
"eval_within_pm1": eval_within1,
|
||||
})
|
||||
|
||||
if eval_acc > best_eval_acc:
|
||||
best_eval_acc = eval_acc
|
||||
best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}
|
||||
|
||||
if epoch < 5 or epoch % 50 == 0 or epoch == args.epochs - 1:
|
||||
print(f"epoch {epoch:3d} train_loss={train_loss/n_batches:.4f} "
|
||||
f"train_acc={train_correct/n_train:.3f} "
|
||||
f"eval_loss={eval_loss:.4f} eval_acc={eval_acc:.3f} "
|
||||
f"within±1={eval_within1:.3f}")
|
||||
|
||||
train_time = time.perf_counter() - t0
|
||||
print(f"\ntrained {args.epochs} epochs in {train_time:.1f} s")
|
||||
print(f"best eval_acc: {best_eval_acc:.3f}")
|
||||
|
||||
# Restore best checkpoint
|
||||
if best_state is not None:
|
||||
model.load_state_dict(best_state)
|
||||
|
||||
# Eval breakdown
|
||||
model.eval()
|
||||
with torch.no_grad():
|
||||
cl_e, conf_e = model(Xe)
|
||||
probs_e = torch.softmax(cl_e, dim=1)
|
||||
pred_e = cl_e.argmax(dim=1)
|
||||
acc = (pred_e == ye).float().mean().item()
|
||||
within1 = ((pred_e - ye).abs() <= 1).float().mean().item()
|
||||
mae = (pred_e - ye).abs().float().mean().item()
|
||||
|
||||
# Per-class accuracy
|
||||
per_class = {}
|
||||
for k in range(COUNT_CLASSES):
|
||||
mask = ye == k
|
||||
n = mask.sum().item()
|
||||
if n > 0:
|
||||
per_class[k] = {
|
||||
"support": int(n),
|
||||
"accuracy": ((pred_e == ye) & mask).sum().item() / n,
|
||||
}
|
||||
|
||||
# Confidence-accuracy calibration: Spearman over (predicted-correct, confidence)
|
||||
conf_sigm = torch.sigmoid(conf_e).squeeze(-1)
|
||||
correct = (pred_e == ye).float()
|
||||
# Spearman = Pearson over ranks
|
||||
c_rank = conf_sigm.argsort().argsort().float()
|
||||
r_rank = correct.argsort().argsort().float()
|
||||
c_centered = c_rank - c_rank.mean()
|
||||
r_centered = r_rank - r_rank.mean()
|
||||
denom = (c_centered.norm() * r_centered.norm()).item()
|
||||
spearman = (c_centered * r_centered).sum().item() / denom if denom > 0 else 0.0
|
||||
|
||||
print(f"\n=== final eval ===")
|
||||
print(f" accuracy: {acc:.3f}")
|
||||
print(f" within ±1: {within1:.3f}")
|
||||
print(f" MAE: {mae:.3f}")
|
||||
print(f" conf↔correct Spearman: {spearman:.3f}")
|
||||
for k, v in per_class.items():
|
||||
print(f" class {k}: {v['accuracy']:.3f} accuracy on {v['support']} samples")
|
||||
|
||||
# Save safetensors
|
||||
write_safetensors(model, Path(args.out_safetensors))
|
||||
print(f"\nwrote {args.out_safetensors} ({Path(args.out_safetensors).stat().st_size} bytes)")
|
||||
|
||||
# ONNX export
|
||||
dummy = torch.zeros(1, N_SUB, N_FRAMES, device=device)
|
||||
try:
|
||||
torch.onnx.export(
|
||||
model, dummy, args.out_onnx,
|
||||
opset_version=18,
|
||||
input_names=["csi_window"],
|
||||
output_names=["count_logits", "conf_logits"],
|
||||
dynamic_axes={
|
||||
"csi_window": {0: "batch"},
|
||||
"count_logits": {0: "batch"},
|
||||
"conf_logits": {0: "batch"},
|
||||
},
|
||||
export_params=True,
|
||||
do_constant_folding=True,
|
||||
)
|
||||
print(f"wrote {args.out_onnx} ({Path(args.out_onnx).stat().st_size} bytes)")
|
||||
except Exception as e:
|
||||
print(f"WARN: ONNX export failed: {e}")
|
||||
|
||||
# Results JSON
|
||||
results = {
|
||||
"backend": "candle-cuda" if device.type == "cuda" else "candle-cpu",
|
||||
"device": str(device),
|
||||
"epochs": args.epochs,
|
||||
"train_time_s": train_time,
|
||||
"best_eval_acc": best_eval_acc,
|
||||
"final_eval_acc": acc,
|
||||
"final_eval_within_pm1": within1,
|
||||
"final_eval_mae": mae,
|
||||
"conf_correctness_spearman": spearman,
|
||||
"per_class_accuracy": per_class,
|
||||
"hyperparameters": {
|
||||
"optimizer": "AdamW",
|
||||
"lr": args.lr,
|
||||
"weight_decay": args.weight_decay,
|
||||
"batch_size": args.batch_size,
|
||||
"schedule": "cosine_warm_restarts",
|
||||
"epochs": args.epochs,
|
||||
"loss": "cross_entropy(count) + 0.3*bce(conf) + 0.1*brier(conf)",
|
||||
"z_score_normalisation": True,
|
||||
"class_weights": cls_weight.tolist(),
|
||||
},
|
||||
"epoch_losses": epoch_losses,
|
||||
}
|
||||
Path(args.out_results).write_text(json.dumps(results, indent=2))
|
||||
print(f"wrote {args.out_results} ({Path(args.out_results).stat().st_size} bytes)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Generated
+728
-71
File diff suppressed because it is too large
Load Diff
@@ -28,6 +28,16 @@ members = [
|
||||
"crates/wifi-densepose-geo",
|
||||
"crates/nvsim",
|
||||
"crates/nvsim-server",
|
||||
# ADR-100/ADR-101: Cognitum Cog packaging — first Cog from this repo.
|
||||
# Ships the wifi-densepose pose-estimation model as a signed binary +
|
||||
# JSONL manifest installable by the Cognitum V0 appliance (cognitum-v0,
|
||||
# cognitum-cluster-*, ruvultra). The companion appliance-side crate
|
||||
# lives in cognitum-one/v0-appliance as `cognitum-pose-estimation`.
|
||||
"crates/cog-pose-estimation",
|
||||
# ADR-103: Learned multi-person counter (SOTA path) — replaces the
|
||||
# PR #491 slot heuristic with a Candle network + Stoer-Wagner fusion.
|
||||
# Motivated by #499 ghost-skeleton reports.
|
||||
"crates/cog-person-count",
|
||||
# rvCSI — edge RF sensing runtime (ADR-095 platform, ADR-096 FFI/crate layout):
|
||||
# lives in its own repo (https://github.com/ruvnet/rvcsi), vendored here as
|
||||
# `vendor/rvcsi` and published to crates.io as `rvcsi-*` 0.3.x. Depend on the
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
[package]
|
||||
name = "cog-person-count"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
authors.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Cognitum Cog: learned multi-person counter from WiFi CSI (ADR-103). Replaces the PR #491 slot heuristic with a Candle-based count head + Stoer-Wagner multi-node fusion."
|
||||
publish = false
|
||||
|
||||
[[bin]]
|
||||
name = "cog-person-count"
|
||||
path = "src/main.rs"
|
||||
|
||||
[lib]
|
||||
name = "cog_person_count"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
thiserror = "1"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "signal", "time"] }
|
||||
sha2 = "0.10"
|
||||
ureq = { version = "2", default-features = false, features = ["tls"] }
|
||||
# Same Candle stack the pose cog uses — CPU by default, `cuda` feature
|
||||
# opt-in for hosts with a CUDA GPU.
|
||||
candle-core = { version = "0.9", default-features = false }
|
||||
candle-nn = { version = "0.9", default-features = false }
|
||||
safetensors = "0.4"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
approx = "0.5"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
cuda = ["candle-core/cuda", "candle-nn/cuda"]
|
||||
hailo = []
|
||||
@@ -0,0 +1,96 @@
|
||||
# Person Count Cog
|
||||
|
||||
Learned multi-person counter for WiFi CSI — designed in [ADR-103](../../../../docs/adr/ADR-103-learned-multi-person-counter.md), packaged per [ADR-100](../../../../docs/adr/ADR-100-cog-packaging-specification.md), discoverable through [ADR-102](../../../../docs/adr/ADR-102-edge-module-registry.md).
|
||||
|
||||
## What it does
|
||||
|
||||
Replaces the PR #491 slot heuristic (`subcarrier_diversity / dedup_factor`) with a Candle network that emits a calibrated count distribution + confidence per CSI window. Multi-node deployments fuse N per-node predictions through a confidence-weighted log-sum (Bayesian product of experts), optionally bounded above by a Stoer-Wagner min-cut from the subcarrier-similarity graph.
|
||||
|
||||
## Output (per frame)
|
||||
|
||||
```json
|
||||
{
|
||||
"ts": 1779210883.444,
|
||||
"level": "info",
|
||||
"event": "person.count",
|
||||
"fields": {
|
||||
"tick": 12345,
|
||||
"count": 2,
|
||||
"confidence": 0.81,
|
||||
"count_p95_low": 1,
|
||||
"count_p95_high": 3,
|
||||
"n_nodes": 3,
|
||||
"probs": [0.01, 0.03, 0.81, 0.13, 0.01, 0.005, 0.003, 0.002]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Downstream consumers can render the **most-likely count** when confidence is high, or fall back to a `[lo, hi]` band with a "?" badge when the model is uncertain — that's how this Cog closes the loop on #499's ghost-skeleton UX.
|
||||
|
||||
## Status — v0.0.1
|
||||
|
||||
| Component | State |
|
||||
|---|---|
|
||||
| Crate compiles, library API stable | ✅ |
|
||||
| Tests pass (15 total: 8 smoke + 7 fusion) | ✅ |
|
||||
| Four-verb runtime contract (`version`, `manifest`, `health`) | ✅ |
|
||||
| Trained `count_v1.safetensors` artifact | ✅ shipped at `cog/artifacts/count_v1.safetensors` (392 KB) |
|
||||
| ONNX export | ✅ `count_v1.onnx` (16 KB), bit-compatible architecture |
|
||||
| Honest accuracy reporting | ✅ See `docs/benchmarks/person-count-cog.md` — 65.1% eval acc on a single-session dataset; confidence head Spearman 0.023 ⇒ uncalibrated for v0.0.1 |
|
||||
| `run` subcommand (long-running loop) | ⏳ same shape as cog-pose-estimation::runtime, lands in follow-up |
|
||||
| Signed binary on GCS | ⏳ release pipeline |
|
||||
| Stoer-Wagner min-cut clip in fusion stage | ⏳ v0.2.0 (hook in `fusion::fuse_with_mincut_clip` is stubbed) |
|
||||
|
||||
### Honest v0.0.1 caveat
|
||||
|
||||
`count_v1` was trained on a single 30-minute solo recording. The model overfit by epoch ~100 and the "best" checkpoint is one that effectively predicts the eval-window class distribution (mostly class-0). Class-1 accuracy on the held-out tail = 0%. **This v0.0.1 is a working pipeline with a degenerate model**, not a usable counter yet — same data-bound failure mode as `pose_v1` (#645), same fix: multi-room paired recordings.
|
||||
|
||||
`cog-person-count health` will load the real safetensors and report `backend: candle-cpu` rather than `backend: stub`, so the cog-gateway can verify the model loaded — but operators should treat the v0.0.1 count outputs as scaffold-validation rather than production data. The 2.36 MB binary + 392 KB weights + 16 KB ONNX are all real and reusable as soon as more data lands.
|
||||
|
||||
## Relationship to the in-process `csi.rs::score_to_person_count` heuristic
|
||||
|
||||
This Cog runs **out-of-process** alongside `wifi-densepose-sensing-server`. The two are complementary, not competing:
|
||||
|
||||
- The sensing-server keeps emitting its existing slot-count heuristic from `csi.rs::score_to_person_count` (PR #491's RollingP95 + `dedup_factor`). This is the **fallback path** — operators who don't install `cog-person-count` still get a count number, just a less calibrated one.
|
||||
- `cog-person-count` (this binary) polls the same `/api/v1/sensing/latest` endpoint, runs the learned `count_v1` model on each window, and emits `person.count` events on stdout. The appliance's `cognitum-cog-gateway` routes those events to the dashboard via the standard ADR-220 cog-event channel.
|
||||
|
||||
Operators choose by **installing or not installing** this Cog — no sensing-server rebuild required. Downstream consumers (UI, fleet automation, alerting rules) can subscribe to whichever event stream they prefer.
|
||||
|
||||
The architecture decision is documented in [ADR-103 §"Deployment"](../../../../docs/adr/ADR-103-learned-multi-person-counter.md#deployment) and matches the cog/sensing-server boundary established for `cog-pose-estimation` (ADR-101).
|
||||
|
||||
## Security
|
||||
|
||||
The cog has a very small attack surface — by design, it's a pure consumer of CSI data, not a server:
|
||||
|
||||
| Threat | Mitigation |
|
||||
|---|---|
|
||||
| Untrusted model file mmap | `count_v1.safetensors` is loaded via `VarBuilder::from_mmaped_safetensors` (`unsafe` block, documented). The release pipeline signs the file with `COGNITUM_OWNER_SIGNING_KEY` per ADR-100; the appliance's cog-gateway verifies the Ed25519 signature against `weights_sha256` before placing the file under `/var/lib/cognitum/apps/person-count/`. |
|
||||
| Non-finite outputs from a corrupted model | `CountPrediction::is_finite()` is checked in `cmd_health` and in the v0.0.1 run-loop before any `person.count` event is emitted; non-finite outputs fail-closed. |
|
||||
| Sensing-server fetch failures | When the sensing source goes away the cog emits a `WARN` event and skips the frame — same fail-open-as-log pattern as `cog-pose-estimation`. No crash, no leaked file descriptors, no stuck `pid` file. |
|
||||
| Fusion divide-by-zero / log-of-zero | `fuse_confidence_weighted` floors confidences at `1e-3` and floors probabilities at `1e-9` before taking logs. Empty input returns the stub default rather than NaN-propagating. |
|
||||
| Over-the-cap mass after min-cut clip | `fuse_with_mincut_clip` re-normalises the surviving prefix; if all mass was above the cap (degenerate case), it places mass at the cap class rather than producing a zero distribution. |
|
||||
| Output spoofing via stdout | Events go to stdout exactly as ADR-100's runtime contract specifies — the cog-gateway parses each line as JSON. No interactive prompts, no shell escapes, no ANSI control sequences from this cog. |
|
||||
|
||||
The cog opens **zero** network listeners and writes to **zero** files under `/var/lib/cognitum/apps/person-count/` beyond the standard `pid`, `output.log`, and `error.log` that the cog-gateway manages externally.
|
||||
|
||||
## Performance / optimization
|
||||
|
||||
Release build: **2.36 MB stripped binary** on `x86_64-unknown-linux-gnu` (smaller than `cog-pose-estimation`'s 4.5 MB because we don't transitively pull `wifi-densepose-train`).
|
||||
|
||||
Workspace release profile already enables `opt-level = 3`, `lto = "fat"`, `codegen-units = 1`, `strip = true`. No further per-cog optimization knobs needed.
|
||||
|
||||
Cold-start latency (30 sequential `health` invocations, Windows x86_64, candle-cpu backend):
|
||||
|
||||
| Cog | Cold-start |
|
||||
|---|---|
|
||||
| `cog-pose-estimation` | 76.2 ms |
|
||||
| **`cog-person-count`** | **53.3 ms** |
|
||||
|
||||
Long-running `run` warm inference: sub-millisecond per frame in the stub backend (single softmax over 8 classes is essentially free). The trained-model warm path is bounded by the three Conv1d layers — projected ≤ 2 ms on a Pi 5 once `count_v1.safetensors` lands, well under the ≤ 5 ms ADR-103 budget.
|
||||
|
||||
## See also
|
||||
|
||||
- ADR-103 — Design, SOTA comparison, acceptance gates.
|
||||
- ADR-100 — Cog packaging spec.
|
||||
- PR #491 — The heuristic this Cog replaces.
|
||||
- Issue #499 — Original "double skeletons" report that motivated ADR-103.
|
||||
@@ -0,0 +1,240 @@
|
||||
{
|
||||
"mode": "v0.0.2",
|
||||
"backend": "pytorch-cuda",
|
||||
"epochs_trained": 29,
|
||||
"train_time_s": 0.7185604920377955,
|
||||
"best_eval_acc": 0.6232557892799377,
|
||||
"final_eval_acc": 0.6232557892799377,
|
||||
"final_eval_within_pm1": 1.0,
|
||||
"final_eval_mae": 0.37674418091773987,
|
||||
"temperature_scale": 0.9261822700500488,
|
||||
"conf_correctness_spearman_post_temp": 0.012770170735830375,
|
||||
"per_class_accuracy": {
|
||||
"0": {
|
||||
"support": 116,
|
||||
"accuracy": 0.8620689655172413
|
||||
},
|
||||
"1": {
|
||||
"support": 99,
|
||||
"accuracy": 0.3434343434343434
|
||||
}
|
||||
},
|
||||
"hyperparameters": {
|
||||
"optimizer": "AdamW",
|
||||
"lr": 0.001,
|
||||
"weight_decay": 0.01,
|
||||
"batch_size": 64,
|
||||
"schedule": "cosine_warm_restarts",
|
||||
"epochs_max": 400,
|
||||
"label_smoothing": 0.1,
|
||||
"patience": 20,
|
||||
"split": "random_80_20_seed_42",
|
||||
"balanced_sampler": true,
|
||||
"temperature_scaling": true
|
||||
},
|
||||
"epoch_losses": [
|
||||
{
|
||||
"epoch": 0,
|
||||
"train_loss": 1.8680313183711126,
|
||||
"train_acc": 0.4543269230769231,
|
||||
"eval_loss": 0.7276814579963684,
|
||||
"eval_acc": 0.539534866809845
|
||||
},
|
||||
{
|
||||
"epoch": 1,
|
||||
"train_loss": 1.3579198305423443,
|
||||
"train_acc": 0.5060096153846154,
|
||||
"eval_loss": 0.8614012002944946,
|
||||
"eval_acc": 0.46046510338783264
|
||||
},
|
||||
{
|
||||
"epoch": 2,
|
||||
"train_loss": 1.299364447593689,
|
||||
"train_acc": 0.4831730769230769,
|
||||
"eval_loss": 0.7327257990837097,
|
||||
"eval_acc": 0.539534866809845
|
||||
},
|
||||
{
|
||||
"epoch": 3,
|
||||
"train_loss": 1.2834151433064387,
|
||||
"train_acc": 0.4963942307692308,
|
||||
"eval_loss": 0.7958587408065796,
|
||||
"eval_acc": 0.539534866809845
|
||||
},
|
||||
{
|
||||
"epoch": 4,
|
||||
"train_loss": 1.2809640077444224,
|
||||
"train_acc": 0.49278846153846156,
|
||||
"eval_loss": 0.7728011608123779,
|
||||
"eval_acc": 0.46046510338783264
|
||||
},
|
||||
{
|
||||
"epoch": 5,
|
||||
"train_loss": 1.276416512636038,
|
||||
"train_acc": 0.5120192307692307,
|
||||
"eval_loss": 0.7620130181312561,
|
||||
"eval_acc": 0.539534866809845
|
||||
},
|
||||
{
|
||||
"epoch": 6,
|
||||
"train_loss": 1.2767094740500817,
|
||||
"train_acc": 0.4951923076923077,
|
||||
"eval_loss": 0.7696149945259094,
|
||||
"eval_acc": 0.604651153087616
|
||||
},
|
||||
{
|
||||
"epoch": 7,
|
||||
"train_loss": 1.2724562699978168,
|
||||
"train_acc": 0.5324519230769231,
|
||||
"eval_loss": 0.7653729319572449,
|
||||
"eval_acc": 0.539534866809845
|
||||
},
|
||||
{
|
||||
"epoch": 8,
|
||||
"train_loss": 1.2739891455723689,
|
||||
"train_acc": 0.5264423076923077,
|
||||
"eval_loss": 0.7635467648506165,
|
||||
"eval_acc": 0.6232557892799377
|
||||
},
|
||||
{
|
||||
"epoch": 9,
|
||||
"train_loss": 1.2718101739883423,
|
||||
"train_acc": 0.5120192307692307,
|
||||
"eval_loss": 0.7564782500267029,
|
||||
"eval_acc": 0.604651153087616
|
||||
},
|
||||
{
|
||||
"epoch": 10,
|
||||
"train_loss": 1.261798886152414,
|
||||
"train_acc": 0.5625,
|
||||
"eval_loss": 0.7915780544281006,
|
||||
"eval_acc": 0.46046510338783264
|
||||
},
|
||||
{
|
||||
"epoch": 11,
|
||||
"train_loss": 1.2723550613109882,
|
||||
"train_acc": 0.5348557692307693,
|
||||
"eval_loss": 0.7585318088531494,
|
||||
"eval_acc": 0.6139534711837769
|
||||
},
|
||||
{
|
||||
"epoch": 12,
|
||||
"train_loss": 1.2408426174750695,
|
||||
"train_acc": 0.6225961538461539,
|
||||
"eval_loss": 0.7562077045440674,
|
||||
"eval_acc": 0.525581419467926
|
||||
},
|
||||
{
|
||||
"epoch": 13,
|
||||
"train_loss": 1.219417168543889,
|
||||
"train_acc": 0.6334134615384616,
|
||||
"eval_loss": 0.7647078633308411,
|
||||
"eval_acc": 0.5860465168952942
|
||||
},
|
||||
{
|
||||
"epoch": 14,
|
||||
"train_loss": 1.198713256762578,
|
||||
"train_acc": 0.6526442307692307,
|
||||
"eval_loss": 0.7711634635925293,
|
||||
"eval_acc": 0.5720930099487305
|
||||
},
|
||||
{
|
||||
"epoch": 15,
|
||||
"train_loss": 1.167367669252249,
|
||||
"train_acc": 0.6826923076923077,
|
||||
"eval_loss": 0.7664391994476318,
|
||||
"eval_acc": 0.6186046600341797
|
||||
},
|
||||
{
|
||||
"epoch": 16,
|
||||
"train_loss": 1.1867470557873065,
|
||||
"train_acc": 0.6574519230769231,
|
||||
"eval_loss": 0.7853891253471375,
|
||||
"eval_acc": 0.6139534711837769
|
||||
},
|
||||
{
|
||||
"epoch": 17,
|
||||
"train_loss": 1.185251813668471,
|
||||
"train_acc": 0.6766826923076923,
|
||||
"eval_loss": 0.7728492021560669,
|
||||
"eval_acc": 0.5767441987991333
|
||||
},
|
||||
{
|
||||
"epoch": 18,
|
||||
"train_loss": 1.1749065747627845,
|
||||
"train_acc": 0.6814903846153846,
|
||||
"eval_loss": 0.7930512428283691,
|
||||
"eval_acc": 0.5488371849060059
|
||||
},
|
||||
{
|
||||
"epoch": 19,
|
||||
"train_loss": 1.1521984338760376,
|
||||
"train_acc": 0.6983173076923077,
|
||||
"eval_loss": 0.7875214219093323,
|
||||
"eval_acc": 0.5860465168952942
|
||||
},
|
||||
{
|
||||
"epoch": 20,
|
||||
"train_loss": 1.158121026479281,
|
||||
"train_acc": 0.6802884615384616,
|
||||
"eval_loss": 0.785778820514679,
|
||||
"eval_acc": 0.5860465168952942
|
||||
},
|
||||
{
|
||||
"epoch": 21,
|
||||
"train_loss": 1.1232389486753023,
|
||||
"train_acc": 0.7319711538461539,
|
||||
"eval_loss": 0.7949181795120239,
|
||||
"eval_acc": 0.5767441987991333
|
||||
},
|
||||
{
|
||||
"epoch": 22,
|
||||
"train_loss": 1.1163162634922907,
|
||||
"train_acc": 0.7391826923076923,
|
||||
"eval_loss": 0.867073118686676,
|
||||
"eval_acc": 0.539534866809845
|
||||
},
|
||||
{
|
||||
"epoch": 23,
|
||||
"train_loss": 1.1119057948772724,
|
||||
"train_acc": 0.7211538461538461,
|
||||
"eval_loss": 0.8135209679603577,
|
||||
"eval_acc": 0.5953488349914551
|
||||
},
|
||||
{
|
||||
"epoch": 24,
|
||||
"train_loss": 1.107274578167842,
|
||||
"train_acc": 0.7271634615384616,
|
||||
"eval_loss": 0.8401668071746826,
|
||||
"eval_acc": 0.5534883737564087
|
||||
},
|
||||
{
|
||||
"epoch": 25,
|
||||
"train_loss": 1.0781027399576628,
|
||||
"train_acc": 0.7451923076923077,
|
||||
"eval_loss": 0.8606341481208801,
|
||||
"eval_acc": 0.5441860556602478
|
||||
},
|
||||
{
|
||||
"epoch": 26,
|
||||
"train_loss": 1.041811819259937,
|
||||
"train_acc": 0.7584134615384616,
|
||||
"eval_loss": 0.8801625967025757,
|
||||
"eval_acc": 0.5767441987991333
|
||||
},
|
||||
{
|
||||
"epoch": 27,
|
||||
"train_loss": 1.0369769976689265,
|
||||
"train_acc": 0.7764423076923077,
|
||||
"eval_loss": 0.8642652034759521,
|
||||
"eval_acc": 0.5860465168952942
|
||||
},
|
||||
{
|
||||
"epoch": 28,
|
||||
"train_loss": 1.0502384350850031,
|
||||
"train_acc": 0.7524038461538461,
|
||||
"eval_loss": 0.8719286322593689,
|
||||
"eval_acc": 0.5720930099487305
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1 @@
|
||||
0.9261822700500488
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"arch": "arm",
|
||||
"binary_bytes": 3807456,
|
||||
"binary_sha256": "15c2fbac19741298ad1cbaf119c633a42db0a273099561fd57d8afce27728ea5",
|
||||
"binary_signature": "gyV2CDhJo5nqBnREA08KnztGsS7AFOuXCse+2/+wul8DAzerHs9p4L6eUgl8QeiDS9rdQZs33XRxH5WTbkT0Ag==",
|
||||
"binary_url": "https://storage.googleapis.com/cognitum-apps/cogs/arm/cog-person-count-arm",
|
||||
"build_metadata": {
|
||||
"candle": "0.9 cpu",
|
||||
"cog_person_count_version": "0.3.0",
|
||||
"rust": "1.95.0",
|
||||
"training_caveat": "random 80/20 split + label smoothing + early stopping + balanced sampler + temperature calibration. K-fold reference: class-1 mean 57.1% across 5 folds.",
|
||||
"training_class1_accuracy": 0.343,
|
||||
"training_eval_accuracy": 0.623,
|
||||
"training_eval_mae": 0.349,
|
||||
"training_temperature_scale": 0.9262
|
||||
},
|
||||
"id": "person-count",
|
||||
"installed_at": 0,
|
||||
"sig_algo": "Ed25519",
|
||||
"signed_by": "COGNITUM_OWNER_SIGNING_KEY",
|
||||
"status": "installed",
|
||||
"target_triple": "aarch64-unknown-linux-gnu",
|
||||
"version": "0.0.2",
|
||||
"weights_bytes": 392088,
|
||||
"weights_sha256": "32996433516891a37c63c600db8b95e42192a53bd538c088c82cd6a85e55513c",
|
||||
"weights_url": "https://storage.googleapis.com/cognitum-apps/cogs/arm/cog-person-count-count_v1.safetensors"
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"arch": "x86_64",
|
||||
"binary_bytes": 4502960,
|
||||
"binary_sha256": "051614ce6ba63df704fae848a67ad095df4bb88862fdff05ef3c0419cc8388b3",
|
||||
"binary_signature": "P9txCcsqCoFN6LyZS+Hl33pYZxiP/nXJMTI6s4bt26cc+Cteidz7ymajCQIfuq0mx0cnWaQ6eKZUjzq5AIgoBw==",
|
||||
"binary_url": "https://storage.googleapis.com/cognitum-apps/cogs/x86_64/cog-person-count-x86_64",
|
||||
"build_metadata": {
|
||||
"candle": "0.9 cpu",
|
||||
"cog_person_count_version": "0.3.0",
|
||||
"rust": "1.95.0",
|
||||
"training_caveat": "random 80/20 split + label smoothing + early stopping + balanced sampler + temperature calibration. K-fold reference: class-1 mean 57.1% across 5 folds.",
|
||||
"training_class1_accuracy": 0.343,
|
||||
"training_eval_accuracy": 0.623,
|
||||
"training_eval_mae": 0.349,
|
||||
"training_temperature_scale": 0.9262
|
||||
},
|
||||
"id": "person-count",
|
||||
"installed_at": 0,
|
||||
"sig_algo": "Ed25519",
|
||||
"signed_by": "COGNITUM_OWNER_SIGNING_KEY",
|
||||
"status": "installed",
|
||||
"target_triple": "x86_64-unknown-linux-gnu",
|
||||
"version": "0.0.2",
|
||||
"weights_bytes": 392088,
|
||||
"weights_sha256": "32996433516891a37c63c600db8b95e42192a53bd538c088c82cd6a85e55513c",
|
||||
"weights_url": "https://storage.googleapis.com/cognitum-apps/cogs/arm/cog-person-count-count_v1.safetensors"
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
{
|
||||
"kind": "count",
|
||||
"model": "v2/crates/cog-person-count/cog/artifacts/count_v1.safetensors",
|
||||
"n_samples": 128,
|
||||
"saliency_per_subcarrier": [
|
||||
0.0022704999428242445,
|
||||
0.003454199293628335,
|
||||
0.008727867156267166,
|
||||
0.006414174102246761,
|
||||
0.007945921272039413,
|
||||
0.005371364764869213,
|
||||
0.002526703756302595,
|
||||
0.003480477025732398,
|
||||
0.0029449211433529854,
|
||||
0.0013240973930805922,
|
||||
0.008836368098855019,
|
||||
0.0049454583786427975,
|
||||
0.003213808871805668,
|
||||
0.0017830731812864542,
|
||||
0.0015325949061661959,
|
||||
0.00322981970384717,
|
||||
0.00265303160995245,
|
||||
0.0015145435463637114,
|
||||
0.004348318092525005,
|
||||
0.003088578814640641,
|
||||
0.007093404419720173,
|
||||
0.00518156960606575,
|
||||
0.004933001007884741,
|
||||
0.0023939507082104683,
|
||||
0.004226110875606537,
|
||||
0.004997228272259235,
|
||||
0.0018603518838062882,
|
||||
0.0030096496921032667,
|
||||
0.0012774590868502855,
|
||||
0.0014232051325961947,
|
||||
0.009996140375733376,
|
||||
0.009672785177826881,
|
||||
0.0048093050718307495,
|
||||
0.0034254370257258415,
|
||||
0.002622435335069895,
|
||||
0.00878047849982977,
|
||||
0.006196534726768732,
|
||||
0.004779303912073374,
|
||||
0.008283626288175583,
|
||||
0.002107388572767377,
|
||||
0.004639340564608574,
|
||||
0.01281243097037077,
|
||||
0.001995982602238655,
|
||||
0.0019312826916575432,
|
||||
0.004808980971574783,
|
||||
0.0033761016093194485,
|
||||
0.0031302704010158777,
|
||||
0.0016994723118841648,
|
||||
0.004999841097742319,
|
||||
0.006001387722790241,
|
||||
0.00319978641346097,
|
||||
0.004073913209140301,
|
||||
0.011981681920588017,
|
||||
0.002540081739425659,
|
||||
0.0021413916256278753,
|
||||
0.005799528677016497
|
||||
],
|
||||
"ranking_high_to_low": [
|
||||
41,
|
||||
52,
|
||||
30,
|
||||
31,
|
||||
10,
|
||||
35,
|
||||
2,
|
||||
38,
|
||||
4,
|
||||
20,
|
||||
3,
|
||||
36,
|
||||
49,
|
||||
55,
|
||||
5,
|
||||
21,
|
||||
48,
|
||||
25,
|
||||
11,
|
||||
22,
|
||||
32,
|
||||
44,
|
||||
37,
|
||||
40,
|
||||
18,
|
||||
24,
|
||||
51,
|
||||
7,
|
||||
1,
|
||||
33,
|
||||
45,
|
||||
15,
|
||||
12,
|
||||
50,
|
||||
46,
|
||||
19,
|
||||
27,
|
||||
8,
|
||||
16,
|
||||
34,
|
||||
53,
|
||||
6,
|
||||
23,
|
||||
0,
|
||||
54,
|
||||
39,
|
||||
42,
|
||||
43,
|
||||
26,
|
||||
13,
|
||||
47,
|
||||
14,
|
||||
17,
|
||||
29,
|
||||
9,
|
||||
28
|
||||
],
|
||||
"top_k_subcarriers": {
|
||||
"8": [
|
||||
41,
|
||||
52,
|
||||
30,
|
||||
31,
|
||||
10,
|
||||
35,
|
||||
2,
|
||||
38
|
||||
],
|
||||
"16": [
|
||||
41,
|
||||
52,
|
||||
30,
|
||||
31,
|
||||
10,
|
||||
35,
|
||||
2,
|
||||
38,
|
||||
4,
|
||||
20,
|
||||
3,
|
||||
36,
|
||||
49,
|
||||
55,
|
||||
5,
|
||||
21
|
||||
],
|
||||
"32": [
|
||||
41,
|
||||
52,
|
||||
30,
|
||||
31,
|
||||
10,
|
||||
35,
|
||||
2,
|
||||
38,
|
||||
4,
|
||||
20,
|
||||
3,
|
||||
36,
|
||||
49,
|
||||
55,
|
||||
5,
|
||||
21,
|
||||
48,
|
||||
25,
|
||||
11,
|
||||
22,
|
||||
32,
|
||||
44,
|
||||
37,
|
||||
40,
|
||||
18,
|
||||
24,
|
||||
51,
|
||||
7,
|
||||
1,
|
||||
33,
|
||||
45,
|
||||
15
|
||||
]
|
||||
},
|
||||
"saliency_summary": {
|
||||
"min": 0.0012774590868502855,
|
||||
"max": 0.01281243097037077,
|
||||
"mean": 0.004496547522389197,
|
||||
"std": 0.002736047675826084,
|
||||
"max_to_mean_ratio": 2.8493929857463196
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://cognitum.one/schemas/cog-person-count-config-v1.json",
|
||||
"title": "Person Count Cog Runtime Config",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"sensing_url": {
|
||||
"type": "string",
|
||||
"format": "uri",
|
||||
"default": "http://127.0.0.1:3000/api/v1/sensing/latest"
|
||||
},
|
||||
"model_path": {
|
||||
"type": "string",
|
||||
"description": "Filesystem path to count_v1.safetensors. Resolved relative to /var/lib/cognitum/apps/person-count/ when not absolute."
|
||||
},
|
||||
"poll_ms": {
|
||||
"type": "integer",
|
||||
"minimum": 10,
|
||||
"maximum": 1000,
|
||||
"default": 40
|
||||
}
|
||||
},
|
||||
"required": ["model_path"]
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"id": "person-count",
|
||||
"version": "{{VERSION}}",
|
||||
"binary_url": "https://storage.googleapis.com/cognitum-apps/cogs/{{ARCH}}/cog-person-count-{{ARCH}}",
|
||||
"binary_bytes": 0,
|
||||
"binary_sha256": "",
|
||||
"binary_signature": "",
|
||||
"weights_url": "https://storage.googleapis.com/cognitum-apps/cogs/{{ARCH}}/cog-person-count-count_v1.safetensors",
|
||||
"weights_bytes": 0,
|
||||
"weights_sha256": "",
|
||||
"arch": "{{ARCH}}",
|
||||
"target_triple": "{{TARGET_TRIPLE}}",
|
||||
"installed_at": 0,
|
||||
"status": "installed",
|
||||
"signed_by": "COGNITUM_OWNER_SIGNING_KEY",
|
||||
"sig_algo": "Ed25519"
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
//! Multi-node fusion — combine N per-node count distributions into one.
|
||||
//!
|
||||
//! v0.1.0 ships **confidence-weighted log-sum** (Bayesian product of expert
|
||||
//! distributions): the more confident a node, the more its distribution
|
||||
//! shapes the fused output. With one node the fusion is a no-op; with N
|
||||
//! nodes uncertainty can only go down (or stay equal), never up.
|
||||
//!
|
||||
//! v0.2.0 will add a **Stoer-Wagner min-cut upper bound** on the fused
|
||||
//! distribution — see ADR-103 §"Multi-node fusion". That requires
|
||||
//! `ruvector-mincut` as a workspace dep on this crate; it's stubbed below
|
||||
//! behind `fuse_with_mincut_clip()` so callers can opt in once the dep
|
||||
//! lands and the min-cut graph builder for our subcarrier feature
|
||||
//! similarities is ready.
|
||||
|
||||
use crate::inference::{CountPrediction, COUNT_CLASSES};
|
||||
|
||||
/// Confidence-weighted log-sum of per-node count distributions.
|
||||
///
|
||||
/// For each class k, computes `log p_fused(k) = Σ_n c_n · log p_n(k)`,
|
||||
/// then re-normalises. The fused `confidence` is the **maximum** per-node
|
||||
/// confidence rather than the average — having at least one confident
|
||||
/// observation is worth more than many low-confidence ones.
|
||||
///
|
||||
/// Edge cases:
|
||||
/// * Empty input → 1-person, 0-confidence default (matches the stub).
|
||||
/// * Single input → returned as-is (defined behaviour, no-op).
|
||||
/// * Zero confidences across all nodes → unweighted log-sum.
|
||||
pub fn fuse_confidence_weighted(preds: &[CountPrediction]) -> CountPrediction {
|
||||
if preds.is_empty() {
|
||||
let mut probs = [0.0_f32; COUNT_CLASSES];
|
||||
probs[1] = 1.0;
|
||||
return CountPrediction { probs, confidence: 0.0 };
|
||||
}
|
||||
if preds.len() == 1 {
|
||||
return preds[0].clone();
|
||||
}
|
||||
|
||||
// Compute weights c_n with a small floor so zero-confidence nodes still
|
||||
// contribute (log-of-zero would otherwise blow the math up).
|
||||
const EPS_CONF: f32 = 1e-3;
|
||||
let weights: Vec<f32> = preds.iter().map(|p| p.confidence.max(EPS_CONF)).collect();
|
||||
let weight_sum: f32 = weights.iter().sum();
|
||||
|
||||
// Log-sum.
|
||||
let mut log_p = [0.0_f32; COUNT_CLASSES];
|
||||
for (pred, &w) in preds.iter().zip(weights.iter()) {
|
||||
for k in 0..COUNT_CLASSES {
|
||||
let p = pred.probs[k].max(1e-9); // floor to avoid log(0)
|
||||
log_p[k] += (w / weight_sum) * p.ln();
|
||||
}
|
||||
}
|
||||
|
||||
// Subtract max for numerical stability, exponentiate, renormalise.
|
||||
let m = log_p.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
|
||||
let mut p = [0.0_f32; COUNT_CLASSES];
|
||||
let mut s = 0.0_f32;
|
||||
for k in 0..COUNT_CLASSES {
|
||||
p[k] = (log_p[k] - m).exp();
|
||||
s += p[k];
|
||||
}
|
||||
if s > 0.0 {
|
||||
for k in 0..COUNT_CLASSES { p[k] /= s; }
|
||||
} else {
|
||||
// Pathological — fall back to uniform.
|
||||
for k in 0..COUNT_CLASSES { p[k] = 1.0 / COUNT_CLASSES as f32; }
|
||||
}
|
||||
|
||||
let conf = preds.iter().map(|x| x.confidence).fold(0.0_f32, f32::max);
|
||||
CountPrediction { probs: p, confidence: conf }
|
||||
}
|
||||
|
||||
/// **Stoer-Wagner-clipped fusion** — v0.2.0 hook.
|
||||
///
|
||||
/// Takes the same per-node predictions plus a **max-distinct-persons**
|
||||
/// upper bound derived from the subcarrier-similarity graph's min-cut.
|
||||
/// Clips the fused distribution to `{0..=max}` and re-normalises.
|
||||
///
|
||||
/// Live `ruvector_mincut` integration lands in a follow-up PR; this entry
|
||||
/// point is here so the runtime can wire to it without an API break.
|
||||
pub fn fuse_with_mincut_clip(preds: &[CountPrediction], max_distinct: usize) -> CountPrediction {
|
||||
let mut fused = fuse_confidence_weighted(preds);
|
||||
let max_idx = max_distinct.min(COUNT_CLASSES - 1);
|
||||
let mut leak = 0.0_f32;
|
||||
for k in (max_idx + 1)..COUNT_CLASSES {
|
||||
leak += fused.probs[k];
|
||||
fused.probs[k] = 0.0;
|
||||
}
|
||||
if leak > 0.0 {
|
||||
// Re-normalise the surviving prefix.
|
||||
let sum: f32 = fused.probs[..=max_idx].iter().sum();
|
||||
if sum > 0.0 {
|
||||
for k in 0..=max_idx {
|
||||
fused.probs[k] /= sum;
|
||||
}
|
||||
} else {
|
||||
// All mass was above the cap — degenerate; place mass at the cap.
|
||||
fused.probs[max_idx] = 1.0;
|
||||
}
|
||||
}
|
||||
fused
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use approx::assert_relative_eq;
|
||||
|
||||
fn pred(probs: [f32; 8], conf: f32) -> CountPrediction {
|
||||
CountPrediction { probs, confidence: conf }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_returns_one_person_default() {
|
||||
let p = fuse_confidence_weighted(&[]);
|
||||
assert_eq!(p.argmax(), 1);
|
||||
assert_eq!(p.confidence, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_input_is_passthrough() {
|
||||
let probs = [0.0, 0.1, 0.7, 0.2, 0.0, 0.0, 0.0, 0.0];
|
||||
let p = fuse_confidence_weighted(&[pred(probs, 0.8)]);
|
||||
assert_eq!(p.argmax(), 2);
|
||||
assert_relative_eq!(p.confidence, 0.8, max_relative = 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_agreeing_nodes_sharpen_the_peak() {
|
||||
// Both nodes vote 2 with moderate spread. Fusion should sharpen.
|
||||
let probs = [0.05, 0.15, 0.60, 0.15, 0.05, 0.0, 0.0, 0.0];
|
||||
let fused = fuse_confidence_weighted(&[pred(probs, 0.7), pred(probs, 0.7)]);
|
||||
assert_eq!(fused.argmax(), 2);
|
||||
assert!(
|
||||
fused.probs[2] >= probs[2],
|
||||
"expected fusion to sharpen the peak: pre={} post={}",
|
||||
probs[2], fused.probs[2]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn high_confidence_node_overrides_low_confidence_disagreement() {
|
||||
let strong = [0.0, 0.95, 0.05, 0.0, 0.0, 0.0, 0.0, 0.0]; // says 1
|
||||
let weak = [0.0, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.4]; // weak, says 7
|
||||
let fused = fuse_confidence_weighted(&[pred(strong, 0.95), pred(weak, 0.05)]);
|
||||
assert_eq!(fused.argmax(), 1, "high-confidence vote should win");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fusion_preserves_normalisation() {
|
||||
let a = [0.1, 0.2, 0.3, 0.2, 0.1, 0.05, 0.03, 0.02];
|
||||
let b = [0.05, 0.25, 0.35, 0.20, 0.10, 0.03, 0.01, 0.01];
|
||||
let fused = fuse_confidence_weighted(&[pred(a, 0.5), pred(b, 0.5)]);
|
||||
let s: f32 = fused.probs.iter().sum();
|
||||
assert_relative_eq!(s, 1.0, max_relative = 1e-5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mincut_clip_caps_distribution_at_max_distinct() {
|
||||
let probs = [0.0, 0.0, 0.0, 0.0, 0.0, 0.5, 0.3, 0.2]; // mass on 5,6,7
|
||||
let clipped = fuse_with_mincut_clip(&[pred(probs, 0.9)], 4);
|
||||
// Anything above 4 must be zero
|
||||
for k in 5..8 {
|
||||
assert_eq!(clipped.probs[k], 0.0, "class {} should be clipped to 0", k);
|
||||
}
|
||||
// What's left has to renormalise to sum to 1 — even though pre-clip
|
||||
// mass below 4 was zero, the degenerate fallback places mass at the cap.
|
||||
let s: f32 = clipped.probs.iter().sum();
|
||||
assert_relative_eq!(s, 1.0, max_relative = 1e-5);
|
||||
assert_eq!(clipped.argmax(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn p95_range_is_inclusive_and_covers_at_least_95pct() {
|
||||
let probs = [0.05, 0.6, 0.25, 0.05, 0.03, 0.01, 0.005, 0.005];
|
||||
let p = pred(probs, 0.9);
|
||||
let (lo, hi) = p.p95_range();
|
||||
assert!(lo <= 1 && hi >= 1, "mode (1) must be inside [{}, {}]", lo, hi);
|
||||
let mass: f32 = probs[lo..=hi].iter().sum();
|
||||
assert!(mass >= 0.95, "[{}, {}] only covers {:.3}, need >= 0.95", lo, hi, mass);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
//! Single-node count inference — Candle forward over a CSI window.
|
||||
//!
|
||||
//! Architecture (matches ADR-103 §"Architecture (v0.1.0)"):
|
||||
//! Conv1d(56 -> 64, k=3, dilation=1, padding=1)
|
||||
//! Conv1d(64 -> 128, k=3, dilation=2, padding=2)
|
||||
//! Conv1d(128 -> 128, k=3, dilation=4, padding=4)
|
||||
//! mean over time -> [128] ← shared encoder
|
||||
//! ├── Linear(128 -> 64) -> ReLU -> Linear(64 -> 8) → softmax over {0..7}
|
||||
//! └── Linear(128 -> 32) -> ReLU -> Linear(32 -> 1) → sigmoid → confidence
|
||||
//!
|
||||
//! When the safetensors file is missing the engine falls back to a
|
||||
//! "single-person, zero-confidence" stub so the cog still satisfies the
|
||||
//! ADR-100 runtime contract and the dashboard surfaces "no model yet"
|
||||
//! instead of dropping frames silently.
|
||||
|
||||
use candle_core::{DType, Device, Tensor};
|
||||
use candle_nn::{Conv1d, Conv1dConfig, Linear, Module, VarBuilder};
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// `[56 subcarriers × 20 frames]` window — same shape as cog-pose-estimation.
|
||||
pub const INPUT_SUBCARRIERS: usize = 56;
|
||||
pub const INPUT_TIMESTEPS: usize = 20;
|
||||
/// Count classification over {0, 1, ..., 7} persons.
|
||||
pub const COUNT_CLASSES: usize = 8;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CsiWindow {
|
||||
pub data: Vec<f32>,
|
||||
}
|
||||
|
||||
/// Per-node prediction emitted by the count head + confidence head.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CountPrediction {
|
||||
/// Categorical distribution over {0..7} persons. Sums to 1 within float
|
||||
/// precision. Maximum-likelihood class is `argmax(probs)`.
|
||||
pub probs: [f32; COUNT_CLASSES],
|
||||
/// `[0, 1]` — confidence head output. Calibrated against (predicted == truth)
|
||||
/// during training so consumers can use it as a probability of being right.
|
||||
pub confidence: f32,
|
||||
}
|
||||
|
||||
impl CountPrediction {
|
||||
pub fn is_finite(&self) -> bool {
|
||||
self.probs.iter().all(|v| v.is_finite()) && self.confidence.is_finite()
|
||||
}
|
||||
|
||||
/// Maximum-likelihood class.
|
||||
pub fn argmax(&self) -> usize {
|
||||
let mut best_i = 0;
|
||||
let mut best_v = self.probs[0];
|
||||
for (i, &v) in self.probs.iter().enumerate().skip(1) {
|
||||
if v > best_v {
|
||||
best_v = v;
|
||||
best_i = i;
|
||||
}
|
||||
}
|
||||
best_i
|
||||
}
|
||||
|
||||
/// `(low, high)` such that `Σ probs[low..=high] ≥ 0.95`. Used for the
|
||||
/// `count_p95_low` / `count_p95_high` fields surfaced to consumers.
|
||||
pub fn p95_range(&self) -> (usize, usize) {
|
||||
let mode = self.argmax();
|
||||
let mut lo = mode;
|
||||
let mut hi = mode;
|
||||
let mut acc = self.probs[mode];
|
||||
while acc < 0.95 && (lo > 0 || hi < COUNT_CLASSES - 1) {
|
||||
let left = if lo > 0 { self.probs[lo - 1] } else { -1.0 };
|
||||
let right = if hi < COUNT_CLASSES - 1 { self.probs[hi + 1] } else { -1.0 };
|
||||
if left >= right && lo > 0 {
|
||||
lo -= 1;
|
||||
acc += self.probs[lo];
|
||||
} else if hi < COUNT_CLASSES - 1 {
|
||||
hi += 1;
|
||||
acc += self.probs[hi];
|
||||
} else if lo > 0 {
|
||||
lo -= 1;
|
||||
acc += self.probs[lo];
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
(lo, hi)
|
||||
}
|
||||
}
|
||||
|
||||
struct CountNet {
|
||||
c1: Conv1d,
|
||||
c2: Conv1d,
|
||||
c3: Conv1d,
|
||||
count_fc1: Linear,
|
||||
count_fc2: Linear,
|
||||
conf_fc1: Linear,
|
||||
conf_fc2: Linear,
|
||||
}
|
||||
|
||||
impl CountNet {
|
||||
fn new(vb: VarBuilder<'_>) -> candle_core::Result<Self> {
|
||||
let enc = vb.pp("enc");
|
||||
let count = vb.pp("count_head");
|
||||
let conf = vb.pp("conf_head");
|
||||
|
||||
let c1 = candle_nn::conv1d(
|
||||
56, 64, 3,
|
||||
Conv1dConfig { padding: 1, stride: 1, dilation: 1, groups: 1, ..Default::default() },
|
||||
enc.pp("c1"),
|
||||
)?;
|
||||
let c2 = candle_nn::conv1d(
|
||||
64, 128, 3,
|
||||
Conv1dConfig { padding: 2, stride: 1, dilation: 2, groups: 1, ..Default::default() },
|
||||
enc.pp("c2"),
|
||||
)?;
|
||||
let c3 = candle_nn::conv1d(
|
||||
128, 128, 3,
|
||||
Conv1dConfig { padding: 4, stride: 1, dilation: 4, groups: 1, ..Default::default() },
|
||||
enc.pp("c3"),
|
||||
)?;
|
||||
let count_fc1 = candle_nn::linear(128, 64, count.pp("fc1"))?;
|
||||
let count_fc2 = candle_nn::linear(64, COUNT_CLASSES, count.pp("fc2"))?;
|
||||
let conf_fc1 = candle_nn::linear(128, 32, conf.pp("fc1"))?;
|
||||
let conf_fc2 = candle_nn::linear(32, 1, conf.pp("fc2"))?;
|
||||
Ok(Self { c1, c2, c3, count_fc1, count_fc2, conf_fc1, conf_fc2 })
|
||||
}
|
||||
|
||||
fn forward(&self, x: &Tensor) -> candle_core::Result<(Tensor, Tensor)> {
|
||||
let h = self.c1.forward(x)?.relu()?;
|
||||
let h = self.c2.forward(&h)?.relu()?;
|
||||
let h = self.c3.forward(&h)?.relu()?;
|
||||
let h = h.mean(2)?; // [B, 128]
|
||||
|
||||
// Count head — logits then softmax
|
||||
let c = self.count_fc1.forward(&h)?.relu()?;
|
||||
let c = self.count_fc2.forward(&c)?;
|
||||
let probs = candle_nn::ops::softmax(&c, candle_core::D::Minus1)?;
|
||||
|
||||
// Confidence head — sigmoid
|
||||
let cf = self.conf_fc1.forward(&h)?.relu()?;
|
||||
let cf = self.conf_fc2.forward(&cf)?;
|
||||
let conf = candle_nn::ops::sigmoid(&cf)?;
|
||||
|
||||
Ok((probs, conf))
|
||||
}
|
||||
}
|
||||
|
||||
pub struct InferenceEngine {
|
||||
inner: Option<Arc<CountNet>>,
|
||||
device: Device,
|
||||
}
|
||||
|
||||
impl InferenceEngine {
|
||||
pub fn new() -> Result<Self, Box<dyn std::error::Error>> {
|
||||
Self::with_weights(default_weights_path().as_deref())
|
||||
}
|
||||
|
||||
pub fn with_weights(weights_path: Option<&Path>) -> Result<Self, Box<dyn std::error::Error>> {
|
||||
let device = pick_device();
|
||||
let inner = match weights_path {
|
||||
Some(p) if p.exists() => {
|
||||
// SAFETY: from_mmaped_safetensors mmaps the file for the
|
||||
// VarBuilder's lifetime. Same pattern as cog-pose-estimation.
|
||||
let vb = unsafe {
|
||||
VarBuilder::from_mmaped_safetensors(&[p.to_path_buf()], DType::F32, &device)?
|
||||
};
|
||||
let net = CountNet::new(vb)?;
|
||||
Some(Arc::new(net))
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
Ok(Self { inner, device })
|
||||
}
|
||||
|
||||
pub fn backend(&self) -> &'static str {
|
||||
match (&self.inner, &self.device) {
|
||||
(Some(_), Device::Cuda(_)) => "candle-cuda",
|
||||
(Some(_), _) => "candle-cpu",
|
||||
(None, _) => "stub",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn infer(&self, window: &CsiWindow) -> Result<CountPrediction, Box<dyn std::error::Error>> {
|
||||
if window.data.len() != INPUT_SUBCARRIERS * INPUT_TIMESTEPS {
|
||||
return Err(format!(
|
||||
"expected {} input values, got {}",
|
||||
INPUT_SUBCARRIERS * INPUT_TIMESTEPS,
|
||||
window.data.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
let Some(net) = &self.inner else {
|
||||
// Stub fallback: single-person, zero confidence. Surfaces "no
|
||||
// model yet" honestly instead of pretending to know.
|
||||
let mut probs = [0.0f32; COUNT_CLASSES];
|
||||
probs[1] = 1.0; // mass on "1 person"
|
||||
return Ok(CountPrediction { probs, confidence: 0.0 });
|
||||
};
|
||||
|
||||
let t = Tensor::from_slice(
|
||||
&window.data,
|
||||
(1, INPUT_SUBCARRIERS, INPUT_TIMESTEPS),
|
||||
&self.device,
|
||||
)?;
|
||||
let (probs_t, conf_t) = net.forward(&t)?;
|
||||
let flat: Vec<f32> = probs_t.flatten_all()?.to_vec1()?;
|
||||
if flat.len() != COUNT_CLASSES {
|
||||
return Err(format!("count head produced {} probs, expected {}", flat.len(), COUNT_CLASSES).into());
|
||||
}
|
||||
let mut probs = [0.0f32; COUNT_CLASSES];
|
||||
probs.copy_from_slice(&flat[..COUNT_CLASSES]);
|
||||
let conf = conf_t.flatten_all()?.to_vec1::<f32>()?[0];
|
||||
|
||||
Ok(CountPrediction { probs, confidence: conf })
|
||||
}
|
||||
}
|
||||
|
||||
pub struct SyntheticInput;
|
||||
|
||||
impl Default for SyntheticInput {
|
||||
fn default() -> Self { Self }
|
||||
}
|
||||
|
||||
impl SyntheticInput {
|
||||
pub fn as_window(&self) -> CsiWindow {
|
||||
CsiWindow { data: vec![0.0; INPUT_SUBCARRIERS * INPUT_TIMESTEPS] }
|
||||
}
|
||||
}
|
||||
|
||||
fn pick_device() -> Device {
|
||||
#[cfg(feature = "cuda")]
|
||||
if let Ok(d) = Device::cuda_if_available(0) {
|
||||
return d;
|
||||
}
|
||||
Device::Cpu
|
||||
}
|
||||
|
||||
fn default_weights_path() -> Option<std::path::PathBuf> {
|
||||
let candidates = [
|
||||
std::path::PathBuf::from("/var/lib/cognitum/apps/person-count/count_v1.safetensors"),
|
||||
std::path::PathBuf::from("./count_v1.safetensors"),
|
||||
std::path::PathBuf::from("./cog/artifacts/count_v1.safetensors"),
|
||||
std::path::PathBuf::from("v2/crates/cog-person-count/cog/artifacts/count_v1.safetensors"),
|
||||
std::path::PathBuf::from("crates/cog-person-count/cog/artifacts/count_v1.safetensors"),
|
||||
];
|
||||
candidates.into_iter().find(|p| p.exists())
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
//! `cog-person-count` — learned multi-person counter (ADR-103).
|
||||
//!
|
||||
//! Replaces the PR #491 slot heuristic with:
|
||||
//! * a small Candle network (encoder + count head + confidence head),
|
||||
//! * Stoer-Wagner-bounded multi-node fusion,
|
||||
//! * `{count, confidence, count_p95_low, count_p95_high}` output.
|
||||
//!
|
||||
//! Design lives in `docs/adr/ADR-103-learned-multi-person-counter.md`.
|
||||
|
||||
pub mod fusion;
|
||||
pub mod inference;
|
||||
pub mod publisher;
|
||||
pub mod runtime;
|
||||
|
||||
pub const COG_ID: &str = "person-count";
|
||||
pub const COG_VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
@@ -0,0 +1,133 @@
|
||||
//! `cog-person-count` — Cognitum Cog binary entrypoint.
|
||||
//!
|
||||
//! Implements the ADR-100 runtime contract:
|
||||
//! cog-person-count version
|
||||
//! cog-person-count manifest
|
||||
//! cog-person-count health
|
||||
//! cog-person-count run --config <path>
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
use cog_person_count::{
|
||||
inference::{InferenceEngine, SyntheticInput},
|
||||
publisher,
|
||||
COG_ID, COG_VERSION,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "cog-person-count", version = COG_VERSION)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Cmd,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Cmd {
|
||||
Version,
|
||||
Manifest,
|
||||
Health,
|
||||
Run {
|
||||
#[arg(long, value_name = "PATH")]
|
||||
config: PathBuf,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct RunConfig {
|
||||
#[serde(default = "default_sensing_url")]
|
||||
sensing_url: String,
|
||||
model_path: Option<PathBuf>,
|
||||
#[serde(default = "default_poll_ms")]
|
||||
poll_ms: u64,
|
||||
}
|
||||
|
||||
fn default_sensing_url() -> String { "http://127.0.0.1:3000/api/v1/sensing/latest".to_string() }
|
||||
fn default_poll_ms() -> u64 { 40 }
|
||||
|
||||
fn main() -> std::process::ExitCode {
|
||||
init_logging();
|
||||
let cli = Cli::parse();
|
||||
let result = match cli.command {
|
||||
Cmd::Version => cmd_version(),
|
||||
Cmd::Manifest => cmd_manifest(),
|
||||
Cmd::Health => cmd_health(),
|
||||
Cmd::Run { config } => cmd_run(config),
|
||||
};
|
||||
match result {
|
||||
Ok(()) => std::process::ExitCode::SUCCESS,
|
||||
Err(err) => {
|
||||
eprintln!("cog-person-count: {err}");
|
||||
std::process::ExitCode::FAILURE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn init_logging() {
|
||||
let _ = tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info"))
|
||||
)
|
||||
.with_target(false)
|
||||
.try_init();
|
||||
}
|
||||
|
||||
fn cmd_version() -> Result<(), Box<dyn std::error::Error>> {
|
||||
println!("{COG_ID} {COG_VERSION}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_manifest() -> Result<(), Box<dyn std::error::Error>> {
|
||||
println!("{}", serde_json::to_string_pretty(&json!({
|
||||
"id": COG_ID,
|
||||
"version": COG_VERSION,
|
||||
"binary_url": Value::Null,
|
||||
"binary_bytes": Value::Null,
|
||||
"binary_sha256": Value::Null,
|
||||
"binary_signature": Value::Null,
|
||||
"installed_at": Value::Null,
|
||||
"status": Value::Null,
|
||||
}))?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_health() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let engine = InferenceEngine::new()?;
|
||||
let pred = engine.infer(&SyntheticInput::default().as_window())?;
|
||||
if !pred.is_finite() {
|
||||
return Err("inference produced non-finite output".into());
|
||||
}
|
||||
publisher::health_ok(COG_ID, engine.backend(), &pred);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_run(config_path: PathBuf) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let raw = std::fs::read_to_string(&config_path)
|
||||
.map_err(|e| format!("failed to read config at {}: {}", config_path.display(), e))?;
|
||||
let cfg: RunConfig = serde_json::from_str(&raw)
|
||||
.map_err(|e| format!("failed to parse config at {}: {}", config_path.display(), e))?;
|
||||
|
||||
let engine = InferenceEngine::with_weights(cfg.model_path.as_deref())?;
|
||||
publisher::run_started(
|
||||
COG_ID,
|
||||
&cfg.sensing_url,
|
||||
cfg.poll_ms,
|
||||
&cfg.model_path
|
||||
.as_ref()
|
||||
.map(|p| p.display().to_string())
|
||||
.unwrap_or_else(|| "(auto-discover)".to_string()),
|
||||
);
|
||||
|
||||
let rt = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
rt.block_on(cog_person_count::runtime::run_loop(
|
||||
cog_person_count::runtime::RunConfig {
|
||||
sensing_url: cfg.sensing_url,
|
||||
poll_ms: cfg.poll_ms,
|
||||
},
|
||||
engine,
|
||||
))
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
//! Structured JSON event publisher — one event per line on stdout.
|
||||
|
||||
use crate::inference::CountPrediction;
|
||||
use serde::Serialize;
|
||||
use serde_json::{json, Value};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct Event<'a> {
|
||||
pub ts: f64,
|
||||
pub level: &'a str,
|
||||
pub event: &'a str,
|
||||
pub fields: Value,
|
||||
}
|
||||
|
||||
pub fn emit_event(ev: &Event<'_>) {
|
||||
if let Ok(line) = serde_json::to_string(ev) {
|
||||
println!("{line}");
|
||||
}
|
||||
}
|
||||
|
||||
pub fn health_ok(cog_id: &str, backend: &str, p: &CountPrediction) {
|
||||
let (lo, hi) = p.p95_range();
|
||||
emit_event(&Event {
|
||||
ts: now_secs(),
|
||||
level: "info",
|
||||
event: "health.ok",
|
||||
fields: json!({
|
||||
"cog": cog_id,
|
||||
"backend": backend,
|
||||
"synthetic_count": p.argmax(),
|
||||
"synthetic_confidence": p.confidence,
|
||||
"synthetic_p95_range": [lo, hi],
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
pub fn run_started(cog_id: &str, sensing_url: &str, poll_ms: u64, model_path: &str) {
|
||||
emit_event(&Event {
|
||||
ts: now_secs(),
|
||||
level: "info",
|
||||
event: "run.started",
|
||||
fields: json!({
|
||||
"cog": cog_id,
|
||||
"sensing_url": sensing_url,
|
||||
"poll_ms": poll_ms,
|
||||
"model_path": model_path,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
pub fn person_count(tick: u64, fused: &CountPrediction, n_nodes: usize) {
|
||||
let (lo, hi) = fused.p95_range();
|
||||
emit_event(&Event {
|
||||
ts: now_secs(),
|
||||
level: "info",
|
||||
event: "person.count",
|
||||
fields: json!({
|
||||
"tick": tick,
|
||||
"count": fused.argmax(),
|
||||
"confidence": fused.confidence,
|
||||
"count_p95_low": lo,
|
||||
"count_p95_high": hi,
|
||||
"n_nodes": n_nodes,
|
||||
"probs": fused.probs,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
fn now_secs() -> f64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs_f64())
|
||||
.unwrap_or(0.0)
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
//! Long-running inference loop. Polls the appliance's sensing-server,
|
||||
//! slides a CSI window, runs the count head, and emits `person.count`
|
||||
//! events. Same shape as `cog-pose-estimation::runtime`.
|
||||
//!
|
||||
//! Multi-node fusion is single-node only in v0.0.1 — the appliance's
|
||||
//! `/api/v1/sensing/latest` endpoint already aggregates across nodes
|
||||
//! before serving, so per-cog fusion is deferred until each node ships
|
||||
//! raw frames separately (ADR-103 §"Multi-node fusion" v0.2.0).
|
||||
|
||||
use crate::inference::{CsiWindow, InferenceEngine, INPUT_SUBCARRIERS, INPUT_TIMESTEPS};
|
||||
use crate::publisher;
|
||||
use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
|
||||
pub struct RunConfig {
|
||||
pub sensing_url: String,
|
||||
pub poll_ms: u64,
|
||||
}
|
||||
|
||||
pub async fn run_loop(
|
||||
cfg: RunConfig,
|
||||
engine: InferenceEngine,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut buffer: Vec<f32> = Vec::with_capacity(INPUT_SUBCARRIERS * INPUT_TIMESTEPS);
|
||||
let cap = INPUT_SUBCARRIERS * INPUT_TIMESTEPS;
|
||||
let mut tick: u64 = 0;
|
||||
|
||||
loop {
|
||||
match fetch_frame(&cfg.sensing_url).await {
|
||||
Ok(amplitudes) => {
|
||||
tick += 1;
|
||||
buffer.extend(amplitudes);
|
||||
while buffer.len() > 2 * cap {
|
||||
let extra = buffer.len() - cap;
|
||||
buffer.drain(0..extra);
|
||||
}
|
||||
if buffer.len() >= cap {
|
||||
let window = CsiWindow { data: buffer[buffer.len() - cap..].to_vec() };
|
||||
if let Ok(pred) = engine.infer(&window) {
|
||||
// v0.0.1 ships single-node — fusion is a no-op for
|
||||
// N=1. v0.2.0 will append additional per-node
|
||||
// predictions to a vec and call
|
||||
// `fusion::fuse_confidence_weighted` before emit.
|
||||
publisher::person_count(tick, &pred, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "sensing-server fetch failed");
|
||||
}
|
||||
}
|
||||
sleep(Duration::from_millis(cfg.poll_ms)).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_frame(url: &str) -> Result<Vec<f32>, Box<dyn std::error::Error>> {
|
||||
let url = url.to_string();
|
||||
let body = tokio::task::spawn_blocking(move || -> Result<String, ureq::Error> {
|
||||
Ok(ureq::get(&url).call()?.into_string()?)
|
||||
})
|
||||
.await??;
|
||||
let json: serde_json::Value = serde_json::from_str(&body)?;
|
||||
let snapshot = json.get("snapshot").unwrap_or(&json);
|
||||
let nodes = snapshot
|
||||
.get("nodes")
|
||||
.and_then(|v| v.as_array())
|
||||
.ok_or("missing nodes[]")?;
|
||||
let amplitude = nodes
|
||||
.first()
|
||||
.and_then(|n| n.get("amplitude"))
|
||||
.and_then(|v| v.as_array())
|
||||
.ok_or("missing nodes[0].amplitude[]")?;
|
||||
Ok(amplitude
|
||||
.iter()
|
||||
.filter_map(|v| v.as_f64().map(|f| f as f32))
|
||||
.collect())
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
//! Smoke tests for cog-person-count.
|
||||
|
||||
use cog_person_count::{
|
||||
fusion::{fuse_confidence_weighted, fuse_with_mincut_clip},
|
||||
inference::{
|
||||
CountPrediction, CsiWindow, InferenceEngine, SyntheticInput,
|
||||
COUNT_CLASSES, INPUT_SUBCARRIERS, INPUT_TIMESTEPS,
|
||||
},
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn synthetic_window_has_correct_shape() {
|
||||
let w = SyntheticInput::default().as_window();
|
||||
assert_eq!(w.data.len(), INPUT_SUBCARRIERS * INPUT_TIMESTEPS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stub_engine_returns_finite_output() {
|
||||
let engine = InferenceEngine::with_weights(None).expect("stub engine");
|
||||
let pred = engine.infer(&SyntheticInput::default().as_window()).expect("infer");
|
||||
assert!(pred.is_finite());
|
||||
assert_eq!(pred.probs.len(), COUNT_CLASSES);
|
||||
|
||||
let sum: f32 = pred.probs.iter().sum();
|
||||
assert!((sum - 1.0).abs() < 1e-5, "stub probs must sum to 1, got {}", sum);
|
||||
assert_eq!(pred.argmax(), 1, "stub default is 1-person");
|
||||
assert_eq!(pred.confidence, 0.0, "stub confidence is 0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_rejects_wrong_shape_input() {
|
||||
let engine = InferenceEngine::with_weights(None).expect("stub engine");
|
||||
let bad = CsiWindow { data: vec![0.0; 10] };
|
||||
assert!(engine.infer(&bad).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stub_backend_string_is_stable() {
|
||||
let engine = InferenceEngine::with_weights(None).expect("stub engine");
|
||||
assert_eq!(engine.backend(), "stub");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn p95_range_includes_mode() {
|
||||
// Sharp peak at 2
|
||||
let mut probs = [0.0_f32; COUNT_CLASSES];
|
||||
probs[2] = 0.85;
|
||||
probs[1] = 0.08;
|
||||
probs[3] = 0.07;
|
||||
let p = CountPrediction { probs, confidence: 0.9 };
|
||||
let (lo, hi) = p.p95_range();
|
||||
assert!(lo <= 2 && hi >= 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fusion_with_no_inputs_is_safe_default() {
|
||||
let p = fuse_confidence_weighted(&[]);
|
||||
assert_eq!(p.argmax(), 1);
|
||||
assert_eq!(p.confidence, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fusion_passes_through_single_node() {
|
||||
// A single-node ESP32 deployment must produce the same output as the
|
||||
// raw inference — fusion is a no-op for N=1.
|
||||
let mut probs = [0.0_f32; COUNT_CLASSES];
|
||||
probs[3] = 1.0;
|
||||
let input = CountPrediction { probs, confidence: 0.6 };
|
||||
let out = fuse_confidence_weighted(&[input.clone()]);
|
||||
assert_eq!(out.argmax(), 3);
|
||||
assert!((out.confidence - 0.6).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mincut_clip_with_high_cap_is_noop() {
|
||||
let mut probs = [0.0_f32; COUNT_CLASSES];
|
||||
probs[2] = 0.5;
|
||||
probs[3] = 0.5;
|
||||
let input = CountPrediction { probs, confidence: 0.7 };
|
||||
let clipped = fuse_with_mincut_clip(&[input], 7);
|
||||
// No clip happened (cap == max class)
|
||||
assert!((clipped.probs[2] - 0.5).abs() < 1e-6);
|
||||
assert!((clipped.probs[3] - 0.5).abs() < 1e-6);
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
[package]
|
||||
name = "cog-pose-estimation"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
authors.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Cognitum Cog: 17-keypoint pose estimation from WiFi CSI. See ADR-100 (packaging) + ADR-101 (this Cog)."
|
||||
publish = false
|
||||
|
||||
[[bin]]
|
||||
name = "cog-pose-estimation"
|
||||
path = "src/main.rs"
|
||||
|
||||
[lib]
|
||||
name = "cog_pose_estimation"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
thiserror = "1"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "signal", "time"] }
|
||||
sha2 = "0.10"
|
||||
hex = "0.4"
|
||||
# Sensing-server subscriber over HTTP — kept minimal; no full reqwest dep
|
||||
ureq = { version = "2", default-features = false, features = ["tls"] }
|
||||
# Inference backend — Candle, CPU by default. The `cuda` feature gate
|
||||
# below pulls in CUDA support on hosts that have it. Pinned to 0.9 to
|
||||
# match the training script that produced pose_v1.safetensors.
|
||||
candle-core = { version = "0.9", default-features = false }
|
||||
candle-nn = { version = "0.9", default-features = false }
|
||||
safetensors = "0.4"
|
||||
# wifi-densepose-train re-exports the model types we need; depend by path
|
||||
# inside the workspace.
|
||||
wifi-densepose-train = { path = "../wifi-densepose-train", default-features = false }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# Use CUDA for inference on hosts with a CUDA-capable GPU. Off by
|
||||
# default so CI on plain Linux/Windows boxes still builds; flip on for
|
||||
# the GPU-dev path on ruvultra.
|
||||
cuda = ["candle-core/cuda", "candle-nn/cuda"]
|
||||
# Stub for the future Hailo HEF runtime path. The actual Hailo
|
||||
# integration lives in the companion v0-appliance crate `cognitum-hailo`;
|
||||
# this crate keeps a feature flag so the binary can compile without the
|
||||
# Hailo SDK in CI.
|
||||
hailo = []
|
||||
@@ -0,0 +1,57 @@
|
||||
# Build / sign / upload pipeline for cog-pose-estimation.
|
||||
# See ADR-100 §"Build pipeline" for the full contract.
|
||||
|
||||
CRATE := cog-pose-estimation
|
||||
VERSION := $(shell cargo pkgid -p $(CRATE) 2>/dev/null | sed -E 's/.*#([0-9.]+).*/\1/')
|
||||
GCS_BUCKET := gs://cognitum-apps/cogs
|
||||
|
||||
ARCHES := arm x86_64
|
||||
|
||||
# --- Build targets ---
|
||||
|
||||
.PHONY: build build-arm build-x86_64
|
||||
|
||||
build: build-arm build-x86_64
|
||||
|
||||
build-arm:
|
||||
cargo build -p $(CRATE) --release --target aarch64-unknown-linux-gnu
|
||||
cp ../../target/aarch64-unknown-linux-gnu/release/$(CRATE) ./dist/cog-$(CRATE)-arm
|
||||
|
||||
build-x86_64:
|
||||
cargo build -p $(CRATE) --release --target x86_64-unknown-linux-gnu
|
||||
cp ../../target/x86_64-unknown-linux-gnu/release/$(CRATE) ./dist/cog-$(CRATE)-x86_64
|
||||
|
||||
# --- Sign ---
|
||||
|
||||
.PHONY: sign sign-arm sign-x86_64
|
||||
|
||||
sign: sign-arm sign-x86_64
|
||||
|
||||
sign-arm: dist/cog-$(CRATE)-arm
|
||||
sha256sum dist/cog-$(CRATE)-arm | cut -d' ' -f1 > dist/cog-$(CRATE)-arm.sha256
|
||||
# Signature: gcloud secrets versions access latest --secret=COGNITUM_OWNER_SIGNING_KEY \
|
||||
# | openssl pkeyutl -sign -inkey /dev/stdin -rawin -in dist/cog-$(CRATE)-arm.sha256 \
|
||||
# | base64 -w0 > dist/cog-$(CRATE)-arm.sig
|
||||
@echo "TODO: wire Ed25519 sign step once COGNITUM_OWNER_SIGNING_KEY is provisioned to CI."
|
||||
|
||||
sign-x86_64: dist/cog-$(CRATE)-x86_64
|
||||
sha256sum dist/cog-$(CRATE)-x86_64 | cut -d' ' -f1 > dist/cog-$(CRATE)-x86_64.sha256
|
||||
|
||||
# --- Upload to GCS ---
|
||||
|
||||
.PHONY: upload upload-arm upload-x86_64
|
||||
|
||||
upload: upload-arm upload-x86_64
|
||||
|
||||
upload-arm: dist/cog-$(CRATE)-arm
|
||||
gsutil cp dist/cog-$(CRATE)-arm $(GCS_BUCKET)/arm/cog-$(CRATE)-arm
|
||||
|
||||
upload-x86_64: dist/cog-$(CRATE)-x86_64
|
||||
gsutil cp dist/cog-$(CRATE)-x86_64 $(GCS_BUCKET)/x86_64/cog-$(CRATE)-x86_64
|
||||
|
||||
# --- Manifest ---
|
||||
|
||||
.PHONY: manifest
|
||||
|
||||
manifest:
|
||||
@./scripts/render-manifest.sh $(VERSION)
|
||||
@@ -0,0 +1,68 @@
|
||||
# Pose Estimation Cog
|
||||
|
||||
17-keypoint COCO pose estimation from WiFi CSI, deployed as a [Cognitum Cog](../../../../docs/adr/ADR-100-cog-packaging-specification.md).
|
||||
|
||||
## What it does
|
||||
|
||||
Subscribes to the local sensing-server's CSI stream, runs each window through a contrastive encoder (initialised from [`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained)) and a 17-keypoint regression head, and emits one `pose.frame` event per inferred window on stdout. The appliance's cog-gateway picks up those events and routes them to the dashboard.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `[56 subcarriers × 20 frames]` CSI windows (matches the `[56, 20]` shape produced by `scripts/align-ground-truth.js`).
|
||||
- Sensing-server frame poll URL configured via `config.json` (`sensing_url`, default loopback).
|
||||
|
||||
## Outputs
|
||||
|
||||
```json
|
||||
{"ts": 1779210883.444, "level": "info", "event": "pose.frame",
|
||||
"fields": {
|
||||
"tick": 12345,
|
||||
"n_persons": 1,
|
||||
"persons": [{"keypoints": [[0.48, 0.31], ...], "confidence": 0.81}]
|
||||
}}
|
||||
```
|
||||
|
||||
## Status — v0.0.1
|
||||
|
||||
Pipeline scaffold + a first-cut trained model. The model is stored at `cog/artifacts/pose_v1.safetensors` (507 KB) and trained from `data/paired/wiflow-p7-1779210883.paired.jsonl` (1,077 samples, avg conf 0.44) using `candle-core 0.9` on an RTX 5080 — see the full training-result dump at `cog/artifacts/train_results.json`.
|
||||
|
||||
### Measured accuracy (validation set, 217 held-out samples)
|
||||
|
||||
```
|
||||
Overall: PCK@20 = 3.0% PCK@50 = 18.5% MPJPE (normalized) = 0.0931
|
||||
|
||||
Per-joint PCK@20 PCK@50 Per-joint PCK@20 PCK@50
|
||||
───────── ────── ────── ───────── ────── ──────
|
||||
nose 0.5% 5.1% l_hip 0.0% 27.3%
|
||||
l_eye 2.8% 8.3% r_hip 25.0% 76.9% ← strongest signal
|
||||
r_eye 1.9% 15.7% l_knee 2.3% 20.8%
|
||||
l_ear 0.0% 3.2% r_knee 0.9% 35.2%
|
||||
r_ear 1.9% 9.7% l_ankle 1.4% 7.9%
|
||||
l_shoulder 4.6% 8.8% r_ankle 0.9% 9.3%
|
||||
r_shoulder 1.9% 19.9% l_elbow 1.9% 26.4%
|
||||
l_wrist 3.2% 24.1% r_elbow 0.0% 4.2%
|
||||
r_wrist 1.4% 12.0%
|
||||
```
|
||||
|
||||
Loss curve: 0.181 (epoch 0) → 0.014 (epoch 399), eval loss 0.010. **400 epochs in 2.1 s** on the RTX 5080 (~5 ms/epoch full-batch).
|
||||
|
||||
### Honest reading
|
||||
|
||||
- The model **learns coarse body structure** — `r_hip` 77% PCK@50, `r_knee` 35%, `l_elbow` 26% all show real signal. PCK@50 = 18.5% averaged across joints is well above the random-baseline 0% that the pure-JS SPSA training produced.
|
||||
- It is **below the ADR-079 target of PCK@20 ≥ 35%**. The bottleneck is data quality and quantity, not infra. The single 30-min seated-at-desk recording produced 1,077 paired samples at avg confidence 0.44 — strong asymmetry between left/right side (r_hip 77% vs l_hip 27%) reflects the camera framing more than any model defect.
|
||||
- Distal joints (wrists, ankles) and face joints are still near-random: 56-subcarrier CSI at our 20-frame window doesn't carry enough fine-grained spatial information.
|
||||
|
||||
### Next-iteration plan (tracked in [#645](https://github.com/ruvnet/RuView/issues/645))
|
||||
|
||||
- Multi-session, multi-room recordings with **full-body framing** (target ≥ 30K paired samples at conf ≥ 0.7).
|
||||
- Re-train with the same Candle pipeline (already validated to converge in seconds on RTX 5080).
|
||||
- Hailo HEF export via the Dataflow Compiler on a self-hosted runner.
|
||||
|
||||
The cog's runtime inference path is currently a centred-skeleton stub returning `confidence=0`. Wiring the `pose_v1.safetensors` weights into `src/inference.rs` is the next code change — separate PR.
|
||||
|
||||
## See also
|
||||
|
||||
- ADR-100: Cognitum Cog Packaging Specification.
|
||||
- ADR-101: Pose Estimation Cog (the design behind this directory).
|
||||
- ADR-079: Camera-supervised pose training pipeline.
|
||||
- v0-appliance companion crate: `cognitum-pose-estimation` (Hailo HEF runtime).
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"id": "pose-estimation",
|
||||
"version": "0.0.1",
|
||||
"binary_url": "https://storage.googleapis.com/cognitum-apps/cogs/arm/cog-pose-estimation-arm",
|
||||
"binary_bytes": 3741976,
|
||||
"binary_sha256": "1e1a7d3dd01ca05d5bfc5dbb142a5941b7866ed9f3224a21edc04d3f09a99bf5",
|
||||
"binary_signature": "LUN7xqLPYD3MFzm5dKB5MnYU0LvoRtek5ci5KiKPHBg+Xo6xuazwokn2Dw2JPMaLYJzmWn/SpT4djuR7hYvVDw==",
|
||||
"weights_url": "https://storage.googleapis.com/cognitum-apps/cogs/arm/cog-pose-estimation-pose_v1.safetensors",
|
||||
"weights_bytes": 507032,
|
||||
"weights_sha256": "eb249b9a6b2e10130437a10976ed0230b0d085f86a0553d7226e1ae6eae4b9e5",
|
||||
"arch": "arm",
|
||||
"target_triple": "aarch64-unknown-linux-gnu",
|
||||
"installed_at": 0,
|
||||
"status": "installed",
|
||||
"signed_by": "COGNITUM_OWNER_SIGNING_KEY",
|
||||
"sig_algo": "Ed25519",
|
||||
"build_metadata": {
|
||||
"rust": "1.95.0",
|
||||
"candle": "0.9 cpu",
|
||||
"cog_pose_version": "0.3.0",
|
||||
"training_pck20": 3.0,
|
||||
"training_pck50": 18.5,
|
||||
"training_mpjpe_normalized": 0.0931
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"id": "pose-estimation",
|
||||
"version": "0.0.1",
|
||||
"binary_url": "https://storage.googleapis.com/cognitum-apps/cogs/x86_64/cog-pose-estimation-x86_64",
|
||||
"binary_bytes": 4548856,
|
||||
"binary_sha256": "a434739a24415b34e1aff50e5e1c3c32e568db96af473bbb3e5ecc9b95fe71fa",
|
||||
"binary_signature": "pNNuxhgM18PztN8BSZdfw5oAShG2pV3na5T/q2QdlJWX/5FJgo4QTiUCbcTAxI2Uiva8VURSOlRzMU3xoQPqCQ==",
|
||||
"weights_url": "https://storage.googleapis.com/cognitum-apps/cogs/arm/cog-pose-estimation-pose_v1.safetensors",
|
||||
"weights_bytes": 507032,
|
||||
"weights_sha256": "eb249b9a6b2e10130437a10976ed0230b0d085f86a0553d7226e1ae6eae4b9e5",
|
||||
"arch": "x86_64",
|
||||
"target_triple": "x86_64-unknown-linux-gnu",
|
||||
"installed_at": 0,
|
||||
"status": "installed",
|
||||
"signed_by": "COGNITUM_OWNER_SIGNING_KEY",
|
||||
"sig_algo": "Ed25519",
|
||||
"build_metadata": {
|
||||
"rust": "1.89.0",
|
||||
"candle": "0.9 cpu",
|
||||
"cog_pose_version": "0.3.0",
|
||||
"host": "ruvultra (RTX 5080)",
|
||||
"training_pck20": 3.0,
|
||||
"training_pck50": 18.5,
|
||||
"training_mpjpe_normalized": 0.0931,
|
||||
"cold_start_ms_avg": 5.4,
|
||||
"bench_invocations": 30
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,573 @@
|
||||
{
|
||||
"backend": "candle-cuda",
|
||||
"data": {
|
||||
"eval_samples": 216,
|
||||
"split": "temporal_80_20",
|
||||
"split_timestamp": "2026-05-19T17:38:45.486Z",
|
||||
"total_samples": 1077,
|
||||
"train_samples": 861
|
||||
},
|
||||
"encoder_init": "random",
|
||||
"epoch_losses": [
|
||||
0.1808941662311554,
|
||||
0.16265815496444702,
|
||||
0.13955898582935333,
|
||||
0.12225159257650375,
|
||||
0.10377667844295502,
|
||||
0.08922480046749115,
|
||||
0.076103076338768,
|
||||
0.06308665871620178,
|
||||
0.049426380544900894,
|
||||
0.039140596985816956,
|
||||
0.030129408463835716,
|
||||
0.025303713977336884,
|
||||
0.022442471235990524,
|
||||
0.02088615857064724,
|
||||
0.02010779082775116,
|
||||
0.01956109143793583,
|
||||
0.01948179118335247,
|
||||
0.019212622195482254,
|
||||
0.019074730575084686,
|
||||
0.018810957670211792,
|
||||
0.01868920773267746,
|
||||
0.01838303543627262,
|
||||
0.018172571435570717,
|
||||
0.017943259328603745,
|
||||
0.01760796643793583,
|
||||
0.01735210232436657,
|
||||
0.016929639503359795,
|
||||
0.01662956178188324,
|
||||
0.016312643885612488,
|
||||
0.016049085184931755,
|
||||
0.015733029693365097,
|
||||
0.01548701710999012,
|
||||
0.015283167362213135,
|
||||
0.014983722940087318,
|
||||
0.014812562614679337,
|
||||
0.01465131901204586,
|
||||
0.014480160549283028,
|
||||
0.014315342530608177,
|
||||
0.014290803112089634,
|
||||
0.014210136607289314,
|
||||
0.014109139330685139,
|
||||
0.014035886153578758,
|
||||
0.014050519093871117,
|
||||
0.013955573551356792,
|
||||
0.013999568298459053,
|
||||
0.014035838656127453,
|
||||
0.013971822336316109,
|
||||
0.013921688310801983,
|
||||
0.013923658058047295,
|
||||
0.014015297405421734,
|
||||
0.014005525968968868,
|
||||
0.013793034479022026,
|
||||
0.014398499391973019,
|
||||
0.016041349619627,
|
||||
0.018437474966049194,
|
||||
0.019666751846671104,
|
||||
0.01953406259417534,
|
||||
0.018313558772206306,
|
||||
0.016403522342443466,
|
||||
0.014824355952441692,
|
||||
0.014008168131113052,
|
||||
0.013724717311561108,
|
||||
0.013581405393779278,
|
||||
0.013707487843930721,
|
||||
0.01353893056511879,
|
||||
0.013217244297266006,
|
||||
0.012987865135073662,
|
||||
0.012728189118206501,
|
||||
0.01254442147910595,
|
||||
0.012492014095187187,
|
||||
0.012401513755321503,
|
||||
0.012278808280825615,
|
||||
0.012222359888255596,
|
||||
0.012228039093315601,
|
||||
0.012238679453730583,
|
||||
0.012207139283418655,
|
||||
0.012071969918906689,
|
||||
0.012182669714093208,
|
||||
0.011957147158682346,
|
||||
0.011931930668652058,
|
||||
0.011995002627372742,
|
||||
0.012032398954033852,
|
||||
0.011852897703647614,
|
||||
0.011876476928591728,
|
||||
0.011844047345221043,
|
||||
0.011939700692892075,
|
||||
0.011796612292528152,
|
||||
0.01177540048956871,
|
||||
0.011741355061531067,
|
||||
0.011779669672250748,
|
||||
0.011744190007448196,
|
||||
0.011707762256264687,
|
||||
0.011584608815610409,
|
||||
0.011752696707844734,
|
||||
0.011729150079190731,
|
||||
0.011659013107419014,
|
||||
0.011693276464939117,
|
||||
0.011864989064633846,
|
||||
0.011667383834719658,
|
||||
0.011718816123902798,
|
||||
0.01166768092662096,
|
||||
0.011662120930850506,
|
||||
0.011931229382753372,
|
||||
0.012049584649503231,
|
||||
0.012037307024002075,
|
||||
0.01206426601856947,
|
||||
0.012293326668441296,
|
||||
0.012212480418384075,
|
||||
0.01250689011067152,
|
||||
0.012488565407693386,
|
||||
0.012466518208384514,
|
||||
0.012616620399057865,
|
||||
0.012812258675694466,
|
||||
0.013071495108306408,
|
||||
0.013044825755059719,
|
||||
0.01321423426270485,
|
||||
0.013319150544703007,
|
||||
0.013587700203061104,
|
||||
0.013670523650944233,
|
||||
0.01378133799880743,
|
||||
0.014047945849597454,
|
||||
0.013731345534324646,
|
||||
0.014244080521166325,
|
||||
0.014112128876149654,
|
||||
0.014279313385486603,
|
||||
0.014710888266563416,
|
||||
0.01515843067318201,
|
||||
0.014713115990161896,
|
||||
0.014796034432947636,
|
||||
0.01475681271404028,
|
||||
0.014950357377529144,
|
||||
0.015005035325884819,
|
||||
0.014768424443900585,
|
||||
0.015024485997855663,
|
||||
0.015059541910886765,
|
||||
0.015051408670842648,
|
||||
0.015090585686266422,
|
||||
0.015175160020589828,
|
||||
0.015102844685316086,
|
||||
0.015151201747357845,
|
||||
0.015226155519485474,
|
||||
0.015032590366899967,
|
||||
0.015155772678554058,
|
||||
0.01507557276636362,
|
||||
0.015160820446908474,
|
||||
0.015019215643405914,
|
||||
0.015037509612739086,
|
||||
0.015222272835671902,
|
||||
0.015005122870206833,
|
||||
0.015173210762441158,
|
||||
0.015132835134863853,
|
||||
0.027589134871959686,
|
||||
0.07165955752134323,
|
||||
0.06373818218708038,
|
||||
0.06655537337064743,
|
||||
0.07562592625617981,
|
||||
0.06909485161304474,
|
||||
0.05691340193152428,
|
||||
0.048039719462394714,
|
||||
0.040047839283943176,
|
||||
0.034030981361866,
|
||||
0.02623862214386463,
|
||||
0.02114911563694477,
|
||||
0.018268009647727013,
|
||||
0.01640227809548378,
|
||||
0.01537158153951168,
|
||||
0.014892393723130226,
|
||||
0.014505675993859768,
|
||||
0.014186820015311241,
|
||||
0.013841629028320312,
|
||||
0.013426804915070534,
|
||||
0.013020739890635014,
|
||||
0.012673602439463139,
|
||||
0.012330775149166584,
|
||||
0.01226764265447855,
|
||||
0.012166578322649002,
|
||||
0.012095688842236996,
|
||||
0.012270377948880196,
|
||||
0.012516235001385212,
|
||||
0.012700744904577732,
|
||||
0.012992565520107746,
|
||||
0.013367722742259502,
|
||||
0.013592609204351902,
|
||||
0.013607893139123917,
|
||||
0.013697323389351368,
|
||||
0.013854263350367546,
|
||||
0.013832741416990757,
|
||||
0.01367993839085102,
|
||||
0.013867720030248165,
|
||||
0.013601685874164104,
|
||||
0.013631370849907398,
|
||||
0.013577244244515896,
|
||||
0.013414927758276463,
|
||||
0.013450143858790398,
|
||||
0.013431857340037823,
|
||||
0.01343410275876522,
|
||||
0.013244441710412502,
|
||||
0.013297016732394695,
|
||||
0.01346137747168541,
|
||||
0.01331599336117506,
|
||||
0.014807604253292084,
|
||||
0.014646961353719234,
|
||||
0.014483925886452198,
|
||||
0.014267523773014545,
|
||||
0.014087164774537086,
|
||||
0.013921936973929405,
|
||||
0.013723043724894524,
|
||||
0.013571077957749367,
|
||||
0.013395787216722965,
|
||||
0.013234280981123447,
|
||||
0.013133431784808636,
|
||||
0.013057147152721882,
|
||||
0.012962305918335915,
|
||||
0.012835373170673847,
|
||||
0.012728667818009853,
|
||||
0.012636503204703331,
|
||||
0.012564707547426224,
|
||||
0.01253308542072773,
|
||||
0.012460188008844852,
|
||||
0.012445810250937939,
|
||||
0.01240697130560875,
|
||||
0.012377945706248283,
|
||||
0.012340536341071129,
|
||||
0.01233599055558443,
|
||||
0.012312998063862324,
|
||||
0.012278364971280098,
|
||||
0.012224015779793262,
|
||||
0.012239382602274418,
|
||||
0.012242404744029045,
|
||||
0.012323223985731602,
|
||||
0.012205271050333977,
|
||||
0.012227945029735565,
|
||||
0.012205214239656925,
|
||||
0.012209423817694187,
|
||||
0.01217598281800747,
|
||||
0.012150637805461884,
|
||||
0.01217078510671854,
|
||||
0.01225175429135561,
|
||||
0.012216047383844852,
|
||||
0.012195242568850517,
|
||||
0.012198278680443764,
|
||||
0.012190825305879116,
|
||||
0.012173629365861416,
|
||||
0.012157510966062546,
|
||||
0.012140096165239811,
|
||||
0.012207810766994953,
|
||||
0.012194979004561901,
|
||||
0.01217165682464838,
|
||||
0.01216792967170477,
|
||||
0.01218471210449934,
|
||||
0.012194857932627201,
|
||||
0.012163667008280754,
|
||||
0.012145694345235825,
|
||||
0.012135420925915241,
|
||||
0.012164837680757046,
|
||||
0.01216159388422966,
|
||||
0.012148530222475529,
|
||||
0.012224133126437664,
|
||||
0.012155838310718536,
|
||||
0.012177230790257454,
|
||||
0.012110436335206032,
|
||||
0.012090248055756092,
|
||||
0.012101170606911182,
|
||||
0.012153848074376583,
|
||||
0.012173553928732872,
|
||||
0.012172674760222435,
|
||||
0.012157287448644638,
|
||||
0.012172986753284931,
|
||||
0.012137886136770248,
|
||||
0.012157085351645947,
|
||||
0.012121357955038548,
|
||||
0.012135915458202362,
|
||||
0.012176922522485256,
|
||||
0.012193577364087105,
|
||||
0.012180276215076447,
|
||||
0.012223861180245876,
|
||||
0.012179303914308548,
|
||||
0.012176022864878178,
|
||||
0.012092312797904015,
|
||||
0.012138010933995247,
|
||||
0.01214117556810379,
|
||||
0.012276227585971355,
|
||||
0.012187770567834377,
|
||||
0.012211603112518787,
|
||||
0.012213931418955326,
|
||||
0.012225016951560974,
|
||||
0.012142234481871128,
|
||||
0.012134073302149773,
|
||||
0.012163194827735424,
|
||||
0.01223068218678236,
|
||||
0.012200715951621532,
|
||||
0.012191612273454666,
|
||||
0.01220244076102972,
|
||||
0.01220419630408287,
|
||||
0.012142208404839039,
|
||||
0.012142272666096687,
|
||||
0.01212950050830841,
|
||||
0.012169948779046535,
|
||||
0.012184932827949524,
|
||||
0.012199781835079193,
|
||||
0.012189080938696861,
|
||||
0.012251517735421658,
|
||||
0.012228423729538918,
|
||||
0.012237711809575558,
|
||||
0.012216192670166492,
|
||||
0.012263692915439606,
|
||||
0.012285872362554073,
|
||||
0.012329400517046452,
|
||||
0.012345477007329464,
|
||||
0.012416589073836803,
|
||||
0.012419192120432854,
|
||||
0.012471407651901245,
|
||||
0.012412074953317642,
|
||||
0.012433832511305809,
|
||||
0.01246955618262291,
|
||||
0.012568573467433453,
|
||||
0.012632711790502071,
|
||||
0.01270760502666235,
|
||||
0.012691991403698921,
|
||||
0.012749818153679371,
|
||||
0.012748819775879383,
|
||||
0.01276922132819891,
|
||||
0.012770597822964191,
|
||||
0.012830909341573715,
|
||||
0.012891922146081924,
|
||||
0.012974675744771957,
|
||||
0.01295324694365263,
|
||||
0.01304001547396183,
|
||||
0.0130251320078969,
|
||||
0.013028905726969242,
|
||||
0.012945529073476791,
|
||||
0.013016759417951107,
|
||||
0.013065450824797153,
|
||||
0.013240920379757881,
|
||||
0.013167147524654865,
|
||||
0.013239633291959763,
|
||||
0.013240372762084007,
|
||||
0.013296829536557198,
|
||||
0.01322928350418806,
|
||||
0.013259101659059525,
|
||||
0.013233119621872902,
|
||||
0.013339969329535961,
|
||||
0.013323795981705189,
|
||||
0.013341942802071571,
|
||||
0.013390406966209412,
|
||||
0.013395088724792004,
|
||||
0.013347778469324112,
|
||||
0.013323097489774227,
|
||||
0.013308844529092312,
|
||||
0.01338045671582222,
|
||||
0.013418255373835564,
|
||||
0.013455703854560852,
|
||||
0.01349731907248497,
|
||||
0.013548982329666615,
|
||||
0.013543978333473206,
|
||||
0.013514911755919456,
|
||||
0.013511871919035912,
|
||||
0.01351082045584917,
|
||||
0.01348851714283228,
|
||||
0.013556062243878841,
|
||||
0.013558348640799522,
|
||||
0.013616240583360195,
|
||||
0.013577889651060104,
|
||||
0.013577991165220737,
|
||||
0.013531915843486786,
|
||||
0.013514644466340542,
|
||||
0.01348655391484499,
|
||||
0.013568769209086895,
|
||||
0.013610766269266605,
|
||||
0.013646356761455536,
|
||||
0.013650151900947094,
|
||||
0.013662545941770077,
|
||||
0.013631481677293777,
|
||||
0.013629746623337269,
|
||||
0.01362497080117464,
|
||||
0.013645497150719166,
|
||||
0.013664674945175648,
|
||||
0.013721015304327011,
|
||||
0.013627894222736359,
|
||||
0.013688581064343452,
|
||||
0.013681283220648766,
|
||||
0.013655297458171844,
|
||||
0.013539095409214497,
|
||||
0.013555340468883514,
|
||||
0.013566684909164906,
|
||||
0.013745179399847984,
|
||||
0.013687034137547016,
|
||||
0.013702981173992157,
|
||||
0.01367457490414381,
|
||||
0.013732061721384525,
|
||||
0.01364122238010168,
|
||||
0.013664795085787773,
|
||||
0.013612691313028336,
|
||||
0.013709086924791336,
|
||||
0.013684045523405075,
|
||||
0.013670985586941242,
|
||||
0.013698549009859562,
|
||||
0.013667520135641098,
|
||||
0.013631648384034634,
|
||||
0.013607441447675228
|
||||
],
|
||||
"epochs": 400,
|
||||
"final_eval_loss": 0.010066533461213112,
|
||||
"hyperparameters": {
|
||||
"augmentation": "subcarrier_dropout_10pct (last 50 epochs)",
|
||||
"base_lr": 0.001,
|
||||
"batch_mode": "full_batch",
|
||||
"loss": "SmoothL1 (Huber beta=0.1)",
|
||||
"optimizer": "AdamW",
|
||||
"schedule": "cosine",
|
||||
"weight_decay": 0.01
|
||||
},
|
||||
"model": {
|
||||
"encoder": "Conv1d(56->64->128->128, k=3, dilation=[1,2,4]) + GlobalMeanPool",
|
||||
"head": "Linear(128->256)->ReLU->Linear(256->34)->Sigmoid",
|
||||
"parameters": 126562
|
||||
},
|
||||
"mpjpe_normalized": 0.09310426687050756,
|
||||
"pck_at_20": 2.968409586056645,
|
||||
"pck_at_50": 18.51851851851852,
|
||||
"per_joint_pck20": [
|
||||
{
|
||||
"joint": "nose",
|
||||
"pck20": 0.4629629629629629
|
||||
},
|
||||
{
|
||||
"joint": "l_eye",
|
||||
"pck20": 2.7777777777777777
|
||||
},
|
||||
{
|
||||
"joint": "r_eye",
|
||||
"pck20": 1.8518518518518516
|
||||
},
|
||||
{
|
||||
"joint": "l_ear",
|
||||
"pck20": 0.0
|
||||
},
|
||||
{
|
||||
"joint": "r_ear",
|
||||
"pck20": 1.8518518518518516
|
||||
},
|
||||
{
|
||||
"joint": "l_shoulder",
|
||||
"pck20": 4.62962962962963
|
||||
},
|
||||
{
|
||||
"joint": "r_shoulder",
|
||||
"pck20": 1.8518518518518516
|
||||
},
|
||||
{
|
||||
"joint": "l_elbow",
|
||||
"pck20": 1.8518518518518516
|
||||
},
|
||||
{
|
||||
"joint": "r_elbow",
|
||||
"pck20": 0.0
|
||||
},
|
||||
{
|
||||
"joint": "l_wrist",
|
||||
"pck20": 3.2407407407407405
|
||||
},
|
||||
{
|
||||
"joint": "r_wrist",
|
||||
"pck20": 1.3888888888888888
|
||||
},
|
||||
{
|
||||
"joint": "l_hip",
|
||||
"pck20": 0.0
|
||||
},
|
||||
{
|
||||
"joint": "r_hip",
|
||||
"pck20": 25.0
|
||||
},
|
||||
{
|
||||
"joint": "l_knee",
|
||||
"pck20": 2.314814814814815
|
||||
},
|
||||
{
|
||||
"joint": "r_knee",
|
||||
"pck20": 0.9259259259259258
|
||||
},
|
||||
{
|
||||
"joint": "l_ankle",
|
||||
"pck20": 1.3888888888888888
|
||||
},
|
||||
{
|
||||
"joint": "r_ankle",
|
||||
"pck20": 0.9259259259259258
|
||||
}
|
||||
],
|
||||
"per_joint_pck50": [
|
||||
{
|
||||
"joint": "nose",
|
||||
"pck50": 5.092592592592593
|
||||
},
|
||||
{
|
||||
"joint": "l_eye",
|
||||
"pck50": 8.333333333333332
|
||||
},
|
||||
{
|
||||
"joint": "r_eye",
|
||||
"pck50": 15.74074074074074
|
||||
},
|
||||
{
|
||||
"joint": "l_ear",
|
||||
"pck50": 3.2407407407407405
|
||||
},
|
||||
{
|
||||
"joint": "r_ear",
|
||||
"pck50": 9.722222222222223
|
||||
},
|
||||
{
|
||||
"joint": "l_shoulder",
|
||||
"pck50": 8.796296296296296
|
||||
},
|
||||
{
|
||||
"joint": "r_shoulder",
|
||||
"pck50": 19.90740740740741
|
||||
},
|
||||
{
|
||||
"joint": "l_elbow",
|
||||
"pck50": 26.38888888888889
|
||||
},
|
||||
{
|
||||
"joint": "r_elbow",
|
||||
"pck50": 4.166666666666666
|
||||
},
|
||||
{
|
||||
"joint": "l_wrist",
|
||||
"pck50": 24.074074074074073
|
||||
},
|
||||
{
|
||||
"joint": "r_wrist",
|
||||
"pck50": 12.037037037037036
|
||||
},
|
||||
{
|
||||
"joint": "l_hip",
|
||||
"pck50": 27.314814814814813
|
||||
},
|
||||
{
|
||||
"joint": "r_hip",
|
||||
"pck50": 76.85185185185185
|
||||
},
|
||||
{
|
||||
"joint": "l_knee",
|
||||
"pck50": 20.833333333333336
|
||||
},
|
||||
{
|
||||
"joint": "r_knee",
|
||||
"pck50": 35.18518518518518
|
||||
},
|
||||
{
|
||||
"joint": "l_ankle",
|
||||
"pck50": 7.87037037037037
|
||||
},
|
||||
{
|
||||
"joint": "r_ankle",
|
||||
"pck50": 9.25925925925926
|
||||
}
|
||||
],
|
||||
"train_time_s": 2.058459526
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://cognitum.one/schemas/cog-pose-estimation-config-v1.json",
|
||||
"title": "Pose Estimation Cog Runtime Config",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"sensing_url": {
|
||||
"type": "string",
|
||||
"format": "uri",
|
||||
"default": "http://127.0.0.1:3000/api/v1/sensing/latest",
|
||||
"description": "URL of the local sensing-server's latest-snapshot endpoint."
|
||||
},
|
||||
"model_path": {
|
||||
"type": "string",
|
||||
"description": "Filesystem path to the model weights (safetensors or Hailo HEF). Resolved relative to /var/lib/cognitum/apps/pose-estimation/ when not absolute."
|
||||
},
|
||||
"poll_ms": {
|
||||
"type": "integer",
|
||||
"minimum": 10,
|
||||
"maximum": 1000,
|
||||
"default": 40,
|
||||
"description": "How often to poll the sensing-server in milliseconds."
|
||||
},
|
||||
"min_confidence": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"maximum": 1,
|
||||
"default": 0.3,
|
||||
"description": "Drop frames where the inferred pose confidence is below this threshold."
|
||||
}
|
||||
},
|
||||
"required": ["model_path"]
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"id": "pose-estimation",
|
||||
"version": "{{VERSION}}",
|
||||
"binary_url": "https://storage.googleapis.com/cognitum-apps/cogs/{{ARCH}}/cog-pose-estimation-{{ARCH}}",
|
||||
"binary_bytes": 0,
|
||||
"binary_sha256": "",
|
||||
"binary_signature": "",
|
||||
"installed_at": 0,
|
||||
"status": "installed"
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
//! Runtime configuration for the pose-estimation Cog.
|
||||
//!
|
||||
//! Schema lives at `cog/config.schema.json` so the appliance can validate
|
||||
//! before launching the cog.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CogConfig {
|
||||
/// URL of the local sensing-server's frame feed.
|
||||
/// Defaults to the appliance's loopback sensing-server.
|
||||
#[serde(default = "default_sensing_url")]
|
||||
pub sensing_url: String,
|
||||
|
||||
/// Path to the model weights bundle (safetensors or HEF).
|
||||
/// Resolved relative to the cog's install dir if not absolute.
|
||||
pub model_path: PathBuf,
|
||||
|
||||
/// Frame poll interval in milliseconds.
|
||||
#[serde(default = "default_poll_ms")]
|
||||
pub poll_ms: u64,
|
||||
|
||||
/// Confidence threshold below which a frame's keypoints are not emitted.
|
||||
#[serde(default = "default_min_confidence")]
|
||||
pub min_confidence: f32,
|
||||
}
|
||||
|
||||
fn default_sensing_url() -> String {
|
||||
"http://127.0.0.1:3000/api/v1/sensing/latest".to_string()
|
||||
}
|
||||
|
||||
fn default_poll_ms() -> u64 {
|
||||
40 // ~25 Hz to match ESP32 CSI rate
|
||||
}
|
||||
|
||||
fn default_min_confidence() -> f32 {
|
||||
0.3
|
||||
}
|
||||
|
||||
impl CogConfig {
|
||||
pub fn load(path: &Path) -> Result<Self, ConfigError> {
|
||||
let raw = std::fs::read_to_string(path)
|
||||
.map_err(|e| ConfigError::Read(path.to_path_buf(), e))?;
|
||||
let cfg: CogConfig =
|
||||
serde_json::from_str(&raw).map_err(|e| ConfigError::Parse(path.to_path_buf(), e))?;
|
||||
Ok(cfg)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ConfigError {
|
||||
#[error("failed to read config at {0}: {1}")]
|
||||
Read(PathBuf, std::io::Error),
|
||||
#[error("failed to parse config at {0}: {1}")]
|
||||
Parse(PathBuf, serde_json::Error),
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
//! Inference engine — loads `pose_v1.safetensors` (produced by the
|
||||
//! Candle training run on `ruvultra`'s RTX 5080, see
|
||||
//! `cog/artifacts/pose_v1.safetensors` + `docs/benchmarks/pose-estimation-cog.md`)
|
||||
//! and runs the encoder + pose head on each CSI window.
|
||||
//!
|
||||
//! Architecture mirrors the training script exactly:
|
||||
//! Conv1d(56 -> 64, k=3, dilation=1, padding=1)
|
||||
//! Conv1d(64 -> 128, k=3, dilation=2, padding=2)
|
||||
//! Conv1d(128 -> 128, k=3, dilation=4, padding=4)
|
||||
//! mean over time -> [128]
|
||||
//! Linear(128 -> 256) -> ReLU
|
||||
//! Linear(256 -> 34) -> sigmoid -> reshape [17, 2]
|
||||
//!
|
||||
//! When the safetensors file is missing the engine falls back to a
|
||||
//! centred-skeleton baseline with `confidence=0` so the cog still
|
||||
//! satisfies the ADR-100 runtime contract and the dashboard surfaces
|
||||
//! "no model yet" instead of dropping frames silently.
|
||||
|
||||
use candle_core::{DType, Device, Tensor};
|
||||
use candle_nn::{Conv1d, Conv1dConfig, Linear, Module, VarBuilder};
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// 56 subcarriers × 20 frames per CSI window — matches the format
|
||||
/// produced by `scripts/align-ground-truth.js` after #641.
|
||||
pub const INPUT_SUBCARRIERS: usize = 56;
|
||||
pub const INPUT_TIMESTEPS: usize = 20;
|
||||
pub const OUTPUT_KEYPOINTS: usize = 17;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CsiWindow {
|
||||
pub data: Vec<f32>, // length INPUT_SUBCARRIERS * INPUT_TIMESTEPS
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PoseOutput {
|
||||
/// Flat `[OUTPUT_KEYPOINTS * 2]` keypoints in `[0, 1]` normalised
|
||||
/// image coords, ordered (x0, y0, x1, y1, …).
|
||||
pub keypoints: Vec<f32>,
|
||||
pub confidence: f32,
|
||||
}
|
||||
|
||||
impl PoseOutput {
|
||||
pub fn is_finite(&self) -> bool {
|
||||
self.keypoints.iter().all(|v| v.is_finite()) && self.confidence.is_finite()
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal model — mirrors the training script's `PoseModel` exactly.
|
||||
struct PoseNet {
|
||||
c1: Conv1d,
|
||||
c2: Conv1d,
|
||||
c3: Conv1d,
|
||||
fc1: Linear,
|
||||
fc2: Linear,
|
||||
}
|
||||
|
||||
impl PoseNet {
|
||||
fn new(vb: VarBuilder<'_>) -> candle_core::Result<Self> {
|
||||
let enc = vb.pp("enc");
|
||||
let head = vb.pp("head");
|
||||
|
||||
let c1 = candle_nn::conv1d(
|
||||
56,
|
||||
64,
|
||||
3,
|
||||
Conv1dConfig { padding: 1, stride: 1, dilation: 1, groups: 1, ..Default::default() },
|
||||
enc.pp("c1"),
|
||||
)?;
|
||||
let c2 = candle_nn::conv1d(
|
||||
64,
|
||||
128,
|
||||
3,
|
||||
Conv1dConfig { padding: 2, stride: 1, dilation: 2, groups: 1, ..Default::default() },
|
||||
enc.pp("c2"),
|
||||
)?;
|
||||
let c3 = candle_nn::conv1d(
|
||||
128,
|
||||
128,
|
||||
3,
|
||||
Conv1dConfig { padding: 4, stride: 1, dilation: 4, groups: 1, ..Default::default() },
|
||||
enc.pp("c3"),
|
||||
)?;
|
||||
let fc1 = candle_nn::linear(128, 256, head.pp("fc1"))?;
|
||||
let fc2 = candle_nn::linear(256, 34, head.pp("fc2"))?;
|
||||
|
||||
Ok(Self { c1, c2, c3, fc1, fc2 })
|
||||
}
|
||||
|
||||
/// Forward pass: `[B, 56, 20]` -> `[B, 34]` in `[0, 1]`.
|
||||
fn forward(&self, x: &Tensor) -> candle_core::Result<Tensor> {
|
||||
let h = self.c1.forward(x)?.relu()?;
|
||||
let h = self.c2.forward(&h)?.relu()?;
|
||||
let h = self.c3.forward(&h)?.relu()?;
|
||||
// Global average pool over time dim (last dim) -> [B, 128]
|
||||
let h = h.mean(2)?;
|
||||
let h = self.fc1.forward(&h)?.relu()?;
|
||||
let h = self.fc2.forward(&h)?;
|
||||
// sigmoid -> keep in [0, 1]
|
||||
candle_nn::ops::sigmoid(&h)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct InferenceEngine {
|
||||
inner: Option<Arc<LoadedModel>>,
|
||||
device: Device,
|
||||
}
|
||||
|
||||
struct LoadedModel {
|
||||
net: PoseNet,
|
||||
}
|
||||
|
||||
impl InferenceEngine {
|
||||
/// Create an engine. Tries to load weights from `cog/artifacts/pose_v1.safetensors`
|
||||
/// (relative to current dir or the cog install dir under
|
||||
/// `/var/lib/cognitum/apps/pose-estimation/`). Returns a usable
|
||||
/// engine either way — without weights, `infer` produces the
|
||||
/// stub output.
|
||||
pub fn new() -> Result<Self, Box<dyn std::error::Error>> {
|
||||
Self::with_weights(default_weights_path().as_deref())
|
||||
}
|
||||
|
||||
/// Create an engine with a specific weights path (used by `--config`
|
||||
/// in `cog-pose-estimation run`). If `weights_path` is `None`, the
|
||||
/// stub fallback is used.
|
||||
pub fn with_weights(weights_path: Option<&Path>) -> Result<Self, Box<dyn std::error::Error>> {
|
||||
let device = pick_device();
|
||||
let inner = match weights_path {
|
||||
Some(p) if p.exists() => {
|
||||
// SAFETY: `from_mmaped_safetensors` mmaps the file for the
|
||||
// VarBuilder's lifetime. We don't modify the file while the
|
||||
// VarBuilder is alive, and the file is read-only on disk on
|
||||
// appliance installs.
|
||||
let vb = unsafe {
|
||||
VarBuilder::from_mmaped_safetensors(&[p.to_path_buf()], DType::F32, &device)?
|
||||
};
|
||||
let net = PoseNet::new(vb)?;
|
||||
Some(Arc::new(LoadedModel { net }))
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
Ok(Self { inner, device })
|
||||
}
|
||||
|
||||
/// Where the weights actually came from. Useful for the run.started event.
|
||||
pub fn backend(&self) -> &'static str {
|
||||
match (&self.inner, &self.device) {
|
||||
(Some(_), Device::Cuda(_)) => "candle-cuda",
|
||||
(Some(_), _) => "candle-cpu",
|
||||
(None, _) => "stub",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn infer(&self, window: &CsiWindow) -> Result<PoseOutput, Box<dyn std::error::Error>> {
|
||||
if window.data.len() != INPUT_SUBCARRIERS * INPUT_TIMESTEPS {
|
||||
return Err(format!(
|
||||
"expected {} input values, got {}",
|
||||
INPUT_SUBCARRIERS * INPUT_TIMESTEPS,
|
||||
window.data.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
let Some(model) = &self.inner else {
|
||||
// Stub fallback — model not loaded.
|
||||
return Ok(PoseOutput {
|
||||
keypoints: vec![0.5f32; OUTPUT_KEYPOINTS * 2],
|
||||
confidence: 0.0,
|
||||
});
|
||||
};
|
||||
|
||||
// Build [1, 56, 20] tensor from the flat row-major buffer.
|
||||
let t = Tensor::from_slice(
|
||||
&window.data,
|
||||
(1, INPUT_SUBCARRIERS, INPUT_TIMESTEPS),
|
||||
&self.device,
|
||||
)?;
|
||||
let out = model.net.forward(&t)?; // [1, 34]
|
||||
let flat: Vec<f32> = out.flatten_all()?.to_vec1()?;
|
||||
// Confidence from pose_v1 is a published constant rather than per-frame —
|
||||
// the trained model didn't emit a confidence head. Use the validation-set
|
||||
// PCK@50 (18.5%) as the published self-reported confidence so downstream
|
||||
// consumers can gate display decisions on it.
|
||||
Ok(PoseOutput {
|
||||
keypoints: flat,
|
||||
confidence: 0.185,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Synthetic CSI window for the `health` subcommand. Zeros — exercises
|
||||
/// the I/O surface; the model never touches values that produce NaN.
|
||||
pub struct SyntheticInput;
|
||||
|
||||
impl Default for SyntheticInput {
|
||||
fn default() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl SyntheticInput {
|
||||
pub fn as_window(&self) -> CsiWindow {
|
||||
CsiWindow {
|
||||
data: vec![0.0; INPUT_SUBCARRIERS * INPUT_TIMESTEPS],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn pick_device() -> Device {
|
||||
#[cfg(feature = "cuda")]
|
||||
if let Ok(d) = Device::cuda_if_available(0) {
|
||||
return d;
|
||||
}
|
||||
Device::Cpu
|
||||
}
|
||||
|
||||
fn default_weights_path() -> Option<std::path::PathBuf> {
|
||||
// Search in the order an installed Cog would see it.
|
||||
let candidates = [
|
||||
std::path::PathBuf::from("/var/lib/cognitum/apps/pose-estimation/pose_v1.safetensors"),
|
||||
std::path::PathBuf::from("./pose_v1.safetensors"),
|
||||
std::path::PathBuf::from("./cog/artifacts/pose_v1.safetensors"),
|
||||
// From the repo root.
|
||||
std::path::PathBuf::from("v2/crates/cog-pose-estimation/cog/artifacts/pose_v1.safetensors"),
|
||||
// From inside v2/.
|
||||
std::path::PathBuf::from("crates/cog-pose-estimation/cog/artifacts/pose_v1.safetensors"),
|
||||
];
|
||||
candidates.into_iter().find(|p| p.exists())
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
//! `cog-pose-estimation` library surface.
|
||||
//!
|
||||
//! See `ADR-101` for the design and `ADR-100` for the surrounding Cog
|
||||
//! packaging spec. This crate is intentionally a thin shell around
|
||||
//! `wifi-densepose-train`'s exported model types — the heavy lifting
|
||||
//! (encoder, pose head) lives there.
|
||||
|
||||
pub mod config;
|
||||
pub mod inference;
|
||||
pub mod manifest;
|
||||
pub mod publisher;
|
||||
pub mod runtime;
|
||||
|
||||
/// Cog identifier — matches the on-disk path
|
||||
/// `/var/lib/cognitum/apps/pose-estimation/`.
|
||||
pub const COG_ID: &str = "pose-estimation";
|
||||
|
||||
/// Cog version (sourced from Cargo.toml at build time).
|
||||
pub const COG_VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
@@ -0,0 +1,116 @@
|
||||
//! `cog-pose-estimation` — Cognitum Cog binary entrypoint.
|
||||
//!
|
||||
//! Implements the ADR-100 runtime contract:
|
||||
//! cog-pose-estimation version
|
||||
//! cog-pose-estimation manifest
|
||||
//! cog-pose-estimation health
|
||||
//! cog-pose-estimation run --config <path>
|
||||
//!
|
||||
//! Each subcommand writes structured JSON to stdout. `run` is long-running
|
||||
//! and emits one `pose.frame` event per inferred CSI window.
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
use cog_pose_estimation::{
|
||||
config::CogConfig,
|
||||
inference::{InferenceEngine, SyntheticInput},
|
||||
manifest::ManifestSpec,
|
||||
publisher::{emit_event, Event},
|
||||
};
|
||||
use std::path::PathBuf;
|
||||
|
||||
const COG_ID: &str = "pose-estimation";
|
||||
const COG_VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = COG_ID, version = COG_VERSION)]
|
||||
#[command(about = "Cognitum Cog: 17-keypoint pose estimation from WiFi CSI", long_about = None)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Cmd,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Cmd {
|
||||
/// Print `<id> <version>` and exit.
|
||||
Version,
|
||||
/// Print the embedded manifest as JSON.
|
||||
Manifest,
|
||||
/// One-shot health check. Exit 0 if the cog can come up healthy.
|
||||
Health,
|
||||
/// Long-running inference loop.
|
||||
Run {
|
||||
/// Path to runtime config JSON. See `cog/config.schema.json`.
|
||||
#[arg(long, value_name = "PATH")]
|
||||
config: PathBuf,
|
||||
},
|
||||
}
|
||||
|
||||
fn main() -> std::process::ExitCode {
|
||||
init_logging();
|
||||
|
||||
let cli = Cli::parse();
|
||||
let result = match cli.command {
|
||||
Cmd::Version => cmd_version(),
|
||||
Cmd::Manifest => cmd_manifest(),
|
||||
Cmd::Health => cmd_health(),
|
||||
Cmd::Run { config } => cmd_run(config),
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(()) => std::process::ExitCode::SUCCESS,
|
||||
Err(err) => {
|
||||
eprintln!("{COG_ID}: {err}");
|
||||
std::process::ExitCode::FAILURE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn init_logging() {
|
||||
let _ = tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
|
||||
)
|
||||
.with_target(false)
|
||||
.json()
|
||||
.try_init();
|
||||
}
|
||||
|
||||
fn cmd_version() -> Result<(), Box<dyn std::error::Error>> {
|
||||
println!("{COG_ID} {COG_VERSION}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_manifest() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let spec = ManifestSpec::embedded(COG_ID, COG_VERSION);
|
||||
println!("{}", serde_json::to_string_pretty(&spec)?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_health() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let engine = InferenceEngine::new()?;
|
||||
let synthetic = SyntheticInput::default();
|
||||
let out = engine.infer(&synthetic.as_window())?;
|
||||
if out.is_finite() {
|
||||
emit_event(&Event::health_ok(
|
||||
COG_ID,
|
||||
engine.backend(),
|
||||
out.confidence,
|
||||
));
|
||||
Ok(())
|
||||
} else {
|
||||
Err("inference produced non-finite output".into())
|
||||
}
|
||||
}
|
||||
|
||||
fn cmd_run(config_path: PathBuf) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let cfg = CogConfig::load(&config_path)?;
|
||||
emit_event(&Event::run_started(COG_ID, &cfg));
|
||||
|
||||
let engine = InferenceEngine::new()?;
|
||||
let rt = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
rt.block_on(cog_pose_estimation::runtime::run_loop(cfg, engine))?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
//! Cog manifest — see ADR-100 §"manifest.json schema".
|
||||
//!
|
||||
//! The `cog-pose-estimation manifest` subcommand emits the embedded spec
|
||||
//! (no signature fields); the build pipeline post-processes it after
|
||||
//! computing `binary_sha256` + `binary_signature`.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ManifestSpec {
|
||||
pub id: String,
|
||||
pub version: String,
|
||||
pub binary_url: Option<String>,
|
||||
pub binary_bytes: Option<u64>,
|
||||
pub binary_sha256: Option<String>,
|
||||
pub binary_signature: Option<String>,
|
||||
pub installed_at: Option<u64>,
|
||||
pub status: Option<String>,
|
||||
}
|
||||
|
||||
impl ManifestSpec {
|
||||
/// The skeleton emitted by `cog-pose-estimation manifest` before the
|
||||
/// release pipeline fills in the signature/hash/url fields.
|
||||
pub fn embedded(id: &str, version: &str) -> Self {
|
||||
Self {
|
||||
id: id.to_string(),
|
||||
version: version.to_string(),
|
||||
binary_url: None,
|
||||
binary_bytes: None,
|
||||
binary_sha256: None,
|
||||
binary_signature: None,
|
||||
installed_at: None,
|
||||
status: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
//! Structured JSON event publisher — one line per event on stdout.
|
||||
//!
|
||||
//! Format is the ADR-100 runtime contract: `{ts, level, event, fields}`.
|
||||
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct Event<'a> {
|
||||
pub ts: f64,
|
||||
pub level: &'a str,
|
||||
pub event: &'a str,
|
||||
pub fields: Value,
|
||||
}
|
||||
|
||||
impl<'a> Event<'a> {
|
||||
pub fn health_ok(cog_id: &'a str, backend: &str, output_confidence: f32) -> Self {
|
||||
Self {
|
||||
ts: now_secs(),
|
||||
level: "info",
|
||||
event: "health.ok",
|
||||
fields: serde_json::json!({
|
||||
"cog": cog_id,
|
||||
"backend": backend,
|
||||
"synthetic_output_confidence": output_confidence,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run_started(cog_id: &'a str, cfg: &crate::config::CogConfig) -> Self {
|
||||
Self {
|
||||
ts: now_secs(),
|
||||
level: "info",
|
||||
event: "run.started",
|
||||
fields: serde_json::json!({
|
||||
"cog": cog_id,
|
||||
"sensing_url": cfg.sensing_url,
|
||||
"model_path": cfg.model_path,
|
||||
"poll_ms": cfg.poll_ms,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn pose_frame(tick: u64, n_persons: usize, persons: Value) -> Self {
|
||||
Self {
|
||||
ts: now_secs(),
|
||||
level: "info",
|
||||
event: "pose.frame",
|
||||
fields: serde_json::json!({
|
||||
"tick": tick,
|
||||
"n_persons": n_persons,
|
||||
"persons": persons,
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn emit_event(ev: &Event<'_>) {
|
||||
if let Ok(line) = serde_json::to_string(ev) {
|
||||
println!("{line}");
|
||||
}
|
||||
}
|
||||
|
||||
fn now_secs() -> f64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs_f64())
|
||||
.unwrap_or(0.0)
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
//! Long-running inference loop. Polls the appliance's sensing-server,
|
||||
//! runs a CSI window through the engine, emits `pose.frame` events.
|
||||
|
||||
use crate::config::CogConfig;
|
||||
use crate::inference::{CsiWindow, InferenceEngine, INPUT_SUBCARRIERS, INPUT_TIMESTEPS};
|
||||
use crate::publisher::{emit_event, Event};
|
||||
use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
|
||||
pub async fn run_loop(
|
||||
cfg: CogConfig,
|
||||
engine: InferenceEngine,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut buffer: Vec<f32> = Vec::with_capacity(INPUT_SUBCARRIERS * INPUT_TIMESTEPS);
|
||||
let mut tick: u64 = 0;
|
||||
|
||||
loop {
|
||||
// Poll one frame from the sensing-server. On error, sleep and retry —
|
||||
// we expect transient blips when the server restarts.
|
||||
match fetch_frame(&cfg.sensing_url).await {
|
||||
Ok(amplitudes) => {
|
||||
tick += 1;
|
||||
buffer.extend(amplitudes);
|
||||
// Slide-window: keep only the most recent N*T values
|
||||
let cap = INPUT_SUBCARRIERS * INPUT_TIMESTEPS;
|
||||
if buffer.len() >= cap {
|
||||
let window = CsiWindow {
|
||||
data: buffer.split_off(buffer.len() - cap),
|
||||
};
|
||||
if let Ok(out) = engine.infer(&window) {
|
||||
if out.confidence >= cfg.min_confidence {
|
||||
// Flatten persons array (single-person v0.0.1)
|
||||
let persons = serde_json::json!([{
|
||||
"keypoints": chunk_pairs(&out.keypoints),
|
||||
"confidence": out.confidence,
|
||||
}]);
|
||||
emit_event(&Event::pose_frame(tick, 1, persons));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "sensing-server fetch failed");
|
||||
}
|
||||
}
|
||||
sleep(Duration::from_millis(cfg.poll_ms)).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_frame(url: &str) -> Result<Vec<f32>, Box<dyn std::error::Error>> {
|
||||
// Synchronous ureq inside an async fn — we accept the blocking call
|
||||
// here because the per-frame cost (~1 ms loopback) is dwarfed by the
|
||||
// inference cost. Replace with a proper async client if we ever poll
|
||||
// remote sensing-servers over the wire.
|
||||
let url = url.to_string();
|
||||
let body = tokio::task::spawn_blocking(move || -> Result<String, ureq::Error> {
|
||||
Ok(ureq::get(&url).call()?.into_string()?)
|
||||
})
|
||||
.await??;
|
||||
let json: serde_json::Value = serde_json::from_str(&body)?;
|
||||
let snapshot = json.get("snapshot").unwrap_or(&json);
|
||||
let nodes = snapshot
|
||||
.get("nodes")
|
||||
.and_then(|v| v.as_array())
|
||||
.ok_or("missing nodes[]")?;
|
||||
// Take node 0's amplitude vector — we'll add multi-node fusion later.
|
||||
let amplitude = nodes
|
||||
.first()
|
||||
.and_then(|n| n.get("amplitude"))
|
||||
.and_then(|v| v.as_array())
|
||||
.ok_or("missing nodes[0].amplitude[]")?;
|
||||
Ok(amplitude
|
||||
.iter()
|
||||
.filter_map(|v| v.as_f64().map(|f| f as f32))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn chunk_pairs(flat: &[f32]) -> Vec<[f32; 2]> {
|
||||
flat.chunks_exact(2).map(|c| [c[0], c[1]]).collect()
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
//! Smoke tests for the cog-pose-estimation crate.
|
||||
//!
|
||||
//! These are deliberately tight — full inference integration tests
|
||||
//! depend on a trained safetensors blob that doesn't live in-repo yet.
|
||||
|
||||
use cog_pose_estimation::{
|
||||
inference::{InferenceEngine, SyntheticInput, INPUT_SUBCARRIERS, INPUT_TIMESTEPS, OUTPUT_KEYPOINTS},
|
||||
manifest::ManifestSpec,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn synthetic_window_has_correct_shape() {
|
||||
let syn = SyntheticInput::default();
|
||||
let window = syn.as_window();
|
||||
assert_eq!(window.data.len(), INPUT_SUBCARRIERS * INPUT_TIMESTEPS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_produces_finite_output_for_synthetic_input() {
|
||||
let engine = InferenceEngine::new().expect("engine init");
|
||||
let out = engine
|
||||
.infer(&SyntheticInput::default().as_window())
|
||||
.expect("infer");
|
||||
assert!(out.is_finite(), "synthetic input must produce finite output");
|
||||
assert_eq!(out.keypoints.len(), OUTPUT_KEYPOINTS * 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_rejects_wrong_shape_input() {
|
||||
let engine = InferenceEngine::new().expect("engine init");
|
||||
let bad = cog_pose_estimation::inference::CsiWindow { data: vec![0.0; 10] };
|
||||
assert!(engine.infer(&bad).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn real_weights_load_when_available() {
|
||||
use cog_pose_estimation::inference::InferenceEngine;
|
||||
let weights = std::path::Path::new("cog/artifacts/pose_v1.safetensors");
|
||||
if !weights.exists() {
|
||||
// Skip when running outside the repo (e.g. on a fresh appliance install).
|
||||
eprintln!("(skipping — cog/artifacts/pose_v1.safetensors not present in cwd)");
|
||||
return;
|
||||
}
|
||||
let engine = InferenceEngine::with_weights(Some(weights)).expect("load real weights");
|
||||
assert!(
|
||||
engine.backend().starts_with("candle-"),
|
||||
"expected real Candle backend, got {}",
|
||||
engine.backend()
|
||||
);
|
||||
let out = engine
|
||||
.infer(&SyntheticInput::default().as_window())
|
||||
.expect("infer");
|
||||
assert!(out.is_finite());
|
||||
// Real model emits the published validation PCK@50 as its self-reported
|
||||
// confidence — stub returns 0.0. This is the key assertion that proves
|
||||
// the cog isn't silently falling back to the stub.
|
||||
assert!(out.confidence > 0.0, "real model should emit non-zero confidence");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manifest_roundtrips() {
|
||||
let spec = ManifestSpec::embedded("pose-estimation", "0.0.1");
|
||||
let s = serde_json::to_string(&spec).unwrap();
|
||||
let back: ManifestSpec = serde_json::from_str(&s).unwrap();
|
||||
assert_eq!(back.id, "pose-estimation");
|
||||
assert_eq!(back.version, "0.0.1");
|
||||
}
|
||||
@@ -56,6 +56,15 @@ wifi-densepose-signal = { version = "0.3.0", path = "../wifi-densepose-signal",
|
||||
midstreamer-temporal-compare = "0.2" # DTW / LCS / Edit-Distance pattern matching
|
||||
midstreamer-attractor = "0.2" # Lyapunov + regime classification
|
||||
|
||||
# ADR-102: Edge Module Registry — fetch the canonical Cognitum cog catalog
|
||||
# at `https://storage.googleapis.com/cognitum-apps/app-registry.json`,
|
||||
# cache with TTL, surface via /api/v1/edge/registry. ureq is the smallest
|
||||
# blocking HTTP client we can use without dragging a tokio HTTP stack in;
|
||||
# rustls is enabled implicitly via the `tls` default feature.
|
||||
ureq = { version = "2", default-features = false, features = ["tls", "json"] }
|
||||
sha2 = "0.10"
|
||||
thiserror = "1"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3.10"
|
||||
# `tower::ServiceExt::oneshot` for in-process Router tests (bearer_auth).
|
||||
|
||||
@@ -0,0 +1,379 @@
|
||||
//! Edge Module Registry — surfaces the canonical Cognitum cog catalog at
|
||||
//! `https://storage.googleapis.com/cognitum-apps/app-registry.json` through
|
||||
//! the sensing-server's HTTP surface. See ADR-102 for the design and trust
|
||||
//! model; see ADR-100 for the underlying cog binary trust model.
|
||||
//!
|
||||
//! On-demand fetch + in-process TTL cache. Stale-while-error semantics: if
|
||||
//! the upstream is unreachable but we have a cached copy, return the cached
|
||||
//! copy with `stale: true` rather than 503.
|
||||
|
||||
use std::io::Read;
|
||||
use std::sync::RwLock;
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
/// Canonical upstream registry URL. Overridable via CLI for air-gapped or
|
||||
/// mirror deployments.
|
||||
pub const DEFAULT_UPSTREAM_URL: &str =
|
||||
"https://storage.googleapis.com/cognitum-apps/app-registry.json";
|
||||
|
||||
/// Default cache TTL — the registry updates on a roughly-weekly cadence;
|
||||
/// one hour of staleness is fine.
|
||||
pub const DEFAULT_TTL_SECS: u64 = 3600;
|
||||
|
||||
/// Wire request timeout. The registry is ~50–200 KB; on a healthy network
|
||||
/// it lands in well under a second.
|
||||
pub const DEFAULT_FETCH_TIMEOUT_SECS: u64 = 10;
|
||||
|
||||
/// Response shape served by `GET /api/v1/edge/registry`. Documented in
|
||||
/// ADR-102 §"Response shape".
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RegistryResponse {
|
||||
pub fetched_at: u64,
|
||||
pub ttl_seconds: u64,
|
||||
pub stale: bool,
|
||||
pub upstream_url: String,
|
||||
pub upstream_sha256: String,
|
||||
pub registry: Value,
|
||||
}
|
||||
|
||||
/// Internal cache entry.
|
||||
#[derive(Debug, Clone)]
|
||||
struct CachedEntry {
|
||||
payload: Value,
|
||||
fetched_at_instant: Instant,
|
||||
fetched_at_unix: u64,
|
||||
upstream_sha256: String,
|
||||
}
|
||||
|
||||
/// On-demand registry fetcher + cache. Cheap to construct; one instance is
|
||||
/// shared across all incoming HTTP requests via `Arc<EdgeRegistry>`.
|
||||
pub struct EdgeRegistry {
|
||||
cached: RwLock<Option<CachedEntry>>,
|
||||
ttl: Duration,
|
||||
upstream_url: String,
|
||||
fetcher: Box<dyn Fetcher>,
|
||||
}
|
||||
|
||||
/// Pluggable fetcher abstraction — concrete impl is `UreqFetcher`; tests
|
||||
/// can swap in `MockFetcher` to drive the cache logic without network.
|
||||
pub trait Fetcher: Send + Sync {
|
||||
fn fetch(&self, url: &str) -> Result<Vec<u8>, FetcherError>;
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum FetcherError {
|
||||
#[error("network error: {0}")]
|
||||
Network(String),
|
||||
#[error("http {status}: {body}")]
|
||||
Http { status: u16, body: String },
|
||||
#[error("response too large: {0} bytes")]
|
||||
TooLarge(usize),
|
||||
}
|
||||
|
||||
/// Cap on the response size to avoid pathological upstream responses
|
||||
/// chewing through memory. 8 MiB is generous — the v2.1.0 registry is well
|
||||
/// under 200 KB.
|
||||
pub const MAX_PAYLOAD_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
/// Live `ureq`-backed fetcher.
|
||||
pub struct UreqFetcher {
|
||||
timeout: Duration,
|
||||
}
|
||||
|
||||
impl UreqFetcher {
|
||||
pub fn new(timeout: Duration) -> Self {
|
||||
Self { timeout }
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for UreqFetcher {
|
||||
fn default() -> Self {
|
||||
Self::new(Duration::from_secs(DEFAULT_FETCH_TIMEOUT_SECS))
|
||||
}
|
||||
}
|
||||
|
||||
impl Fetcher for UreqFetcher {
|
||||
fn fetch(&self, url: &str) -> Result<Vec<u8>, FetcherError> {
|
||||
let agent = ureq::AgentBuilder::new()
|
||||
.timeout(self.timeout)
|
||||
.build();
|
||||
let resp = agent
|
||||
.get(url)
|
||||
.call()
|
||||
.map_err(|e| match e {
|
||||
ureq::Error::Status(status, r) => FetcherError::Http {
|
||||
status,
|
||||
body: r.into_string().unwrap_or_default(),
|
||||
},
|
||||
ureq::Error::Transport(t) => FetcherError::Network(t.to_string()),
|
||||
})?;
|
||||
let mut reader = resp.into_reader().take((MAX_PAYLOAD_BYTES + 1) as u64);
|
||||
let mut buf = Vec::with_capacity(64 * 1024);
|
||||
reader
|
||||
.read_to_end(&mut buf)
|
||||
.map_err(|e| FetcherError::Network(e.to_string()))?;
|
||||
if buf.len() > MAX_PAYLOAD_BYTES {
|
||||
return Err(FetcherError::TooLarge(buf.len()));
|
||||
}
|
||||
Ok(buf)
|
||||
}
|
||||
}
|
||||
|
||||
impl EdgeRegistry {
|
||||
pub fn new(upstream_url: impl Into<String>, ttl: Duration) -> Self {
|
||||
Self::with_fetcher(upstream_url, ttl, Box::new(UreqFetcher::default()))
|
||||
}
|
||||
|
||||
pub fn with_fetcher(
|
||||
upstream_url: impl Into<String>,
|
||||
ttl: Duration,
|
||||
fetcher: Box<dyn Fetcher>,
|
||||
) -> Self {
|
||||
Self {
|
||||
cached: RwLock::new(None),
|
||||
ttl,
|
||||
upstream_url: upstream_url.into(),
|
||||
fetcher,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return a `RegistryResponse`. Uses the cache if fresh; otherwise
|
||||
/// re-fetches from upstream. On upstream failure with a non-empty
|
||||
/// cache, returns the stale copy.
|
||||
pub fn get(&self, force_refresh: bool) -> Result<RegistryResponse, FetcherError> {
|
||||
if !force_refresh {
|
||||
if let Some(entry) = self.fresh_cache_snapshot() {
|
||||
return Ok(self.response_from(&entry, false));
|
||||
}
|
||||
}
|
||||
|
||||
// Either no cache, expired, or forced refresh — try upstream.
|
||||
match self.fetch_and_cache() {
|
||||
Ok(entry) => Ok(self.response_from(&entry, false)),
|
||||
Err(e) => {
|
||||
// Upstream failed — serve stale if available.
|
||||
if let Some(entry) = self.any_cache_snapshot() {
|
||||
Ok(self.response_from(&entry, true))
|
||||
} else {
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn fresh_cache_snapshot(&self) -> Option<CachedEntry> {
|
||||
let guard = self.cached.read().ok()?;
|
||||
let entry = guard.as_ref()?;
|
||||
if entry.fetched_at_instant.elapsed() < self.ttl {
|
||||
Some(entry.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn any_cache_snapshot(&self) -> Option<CachedEntry> {
|
||||
let guard = self.cached.read().ok()?;
|
||||
guard.clone()
|
||||
}
|
||||
|
||||
fn fetch_and_cache(&self) -> Result<CachedEntry, FetcherError> {
|
||||
let bytes = self.fetcher.fetch(&self.upstream_url)?;
|
||||
let payload: Value = serde_json::from_slice(&bytes)
|
||||
.map_err(|e| FetcherError::Network(format!("invalid upstream JSON: {e}")))?;
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&bytes);
|
||||
let upstream_sha256 = hex_encode(&hasher.finalize());
|
||||
let now_unix = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0);
|
||||
|
||||
let entry = CachedEntry {
|
||||
payload,
|
||||
fetched_at_instant: Instant::now(),
|
||||
fetched_at_unix: now_unix,
|
||||
upstream_sha256,
|
||||
};
|
||||
if let Ok(mut guard) = self.cached.write() {
|
||||
*guard = Some(entry.clone());
|
||||
}
|
||||
Ok(entry)
|
||||
}
|
||||
|
||||
fn response_from(&self, entry: &CachedEntry, stale: bool) -> RegistryResponse {
|
||||
RegistryResponse {
|
||||
fetched_at: entry.fetched_at_unix,
|
||||
ttl_seconds: self.ttl.as_secs(),
|
||||
stale,
|
||||
upstream_url: self.upstream_url.clone(),
|
||||
upstream_sha256: entry.upstream_sha256.clone(),
|
||||
registry: entry.payload.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn hex_encode(bytes: &[u8]) -> String {
|
||||
let mut s = String::with_capacity(bytes.len() * 2);
|
||||
for b in bytes {
|
||||
s.push_str(&format!("{:02x}", b));
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Mock fetcher backed by a queue of canned responses. Lets us drive
|
||||
/// the cache logic deterministically.
|
||||
struct MockFetcher {
|
||||
responses: std::sync::Mutex<Vec<Result<Vec<u8>, FetcherError>>>,
|
||||
call_count: AtomicUsize,
|
||||
}
|
||||
|
||||
impl MockFetcher {
|
||||
fn new(responses: Vec<Result<Vec<u8>, FetcherError>>) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
responses: std::sync::Mutex::new(responses),
|
||||
call_count: AtomicUsize::new(0),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Fetcher for Arc<MockFetcher> {
|
||||
fn fetch(&self, _url: &str) -> Result<Vec<u8>, FetcherError> {
|
||||
self.call_count.fetch_add(1, Ordering::SeqCst);
|
||||
let mut q = self.responses.lock().unwrap();
|
||||
if q.is_empty() {
|
||||
return Err(FetcherError::Network("mock: queue empty".into()));
|
||||
}
|
||||
q.remove(0)
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_payload() -> Vec<u8> {
|
||||
br#"{"version":"2.1.0","updated":"2026-05-13","cogs":[]}"#.to_vec()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_call_hits_upstream_and_caches() {
|
||||
let fetcher = MockFetcher::new(vec![Ok(sample_payload())]);
|
||||
let reg = EdgeRegistry::with_fetcher(
|
||||
"http://test.invalid/registry.json",
|
||||
Duration::from_secs(3600),
|
||||
Box::new(fetcher.clone()),
|
||||
);
|
||||
let resp = reg.get(false).expect("get");
|
||||
assert!(!resp.stale);
|
||||
assert_eq!(resp.registry["version"], "2.1.0");
|
||||
assert_eq!(fetcher.call_count.load(Ordering::SeqCst), 1);
|
||||
// Second call within TTL — no new fetch.
|
||||
let _ = reg.get(false).expect("get");
|
||||
assert_eq!(fetcher.call_count.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ttl_expiry_triggers_refetch() {
|
||||
let fetcher = MockFetcher::new(vec![Ok(sample_payload()), Ok(sample_payload())]);
|
||||
let reg = EdgeRegistry::with_fetcher(
|
||||
"http://test.invalid/registry.json",
|
||||
Duration::from_millis(10), // very short TTL
|
||||
Box::new(fetcher.clone()),
|
||||
);
|
||||
let _ = reg.get(false).expect("first");
|
||||
std::thread::sleep(Duration::from_millis(30));
|
||||
let _ = reg.get(false).expect("second after expiry");
|
||||
assert_eq!(fetcher.call_count.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn force_refresh_bypasses_fresh_cache() {
|
||||
let fetcher = MockFetcher::new(vec![Ok(sample_payload()), Ok(sample_payload())]);
|
||||
let reg = EdgeRegistry::with_fetcher(
|
||||
"http://test.invalid/registry.json",
|
||||
Duration::from_secs(3600),
|
||||
Box::new(fetcher.clone()),
|
||||
);
|
||||
let _ = reg.get(false).expect("first");
|
||||
let _ = reg.get(true).expect("refresh");
|
||||
assert_eq!(fetcher.call_count.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_serve_on_upstream_failure_after_cached_success() {
|
||||
// First call succeeds and populates the cache. Second call hits upstream
|
||||
// failure but we still have a cached copy — should serve it with stale=true.
|
||||
let fetcher = MockFetcher::new(vec![
|
||||
Ok(sample_payload()),
|
||||
Err(FetcherError::Network("simulated".into())),
|
||||
]);
|
||||
let reg = EdgeRegistry::with_fetcher(
|
||||
"http://test.invalid/registry.json",
|
||||
Duration::from_millis(1), // expire quickly so call 2 retries upstream
|
||||
Box::new(fetcher.clone()),
|
||||
);
|
||||
let first = reg.get(false).expect("first");
|
||||
assert!(!first.stale);
|
||||
std::thread::sleep(Duration::from_millis(5));
|
||||
let second = reg.get(false).expect("stale-serve");
|
||||
assert!(second.stale, "expected stale=true when upstream failed");
|
||||
assert_eq!(second.registry["version"], "2.1.0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_cache_no_upstream_returns_error() {
|
||||
let fetcher = MockFetcher::new(vec![Err(FetcherError::Network("down".into()))]);
|
||||
let reg = EdgeRegistry::with_fetcher(
|
||||
"http://test.invalid/registry.json",
|
||||
Duration::from_secs(3600),
|
||||
Box::new(fetcher),
|
||||
);
|
||||
let err = reg.get(false).expect_err("should be err");
|
||||
match err {
|
||||
FetcherError::Network(_) => {}
|
||||
other => panic!("unexpected error: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upstream_invalid_json_is_treated_as_error() {
|
||||
let fetcher = MockFetcher::new(vec![Ok(b"not json".to_vec())]);
|
||||
let reg = EdgeRegistry::with_fetcher(
|
||||
"http://test.invalid/registry.json",
|
||||
Duration::from_secs(3600),
|
||||
Box::new(fetcher),
|
||||
);
|
||||
let err = reg.get(false).expect_err("invalid json");
|
||||
match err {
|
||||
FetcherError::Network(msg) => assert!(msg.contains("invalid upstream JSON")),
|
||||
other => panic!("unexpected error: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upstream_sha256_is_deterministic() {
|
||||
let fetcher = MockFetcher::new(vec![Ok(sample_payload())]);
|
||||
let reg = EdgeRegistry::with_fetcher(
|
||||
"http://test.invalid/registry.json",
|
||||
Duration::from_secs(3600),
|
||||
Box::new(fetcher),
|
||||
);
|
||||
let resp = reg.get(false).expect("get");
|
||||
// SHA-256 of br#"{"version":"2.1.0","updated":"2026-05-13","cogs":[]}"#
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&sample_payload());
|
||||
let expected = hex_encode(&hasher.finalize());
|
||||
assert_eq!(resp.upstream_sha256, expected);
|
||||
assert_eq!(resp.upstream_sha256.len(), 64);
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@
|
||||
//! - Real-time CSI introspection / low-latency tap (`introspection`, ADR-099)
|
||||
|
||||
pub mod bearer_auth;
|
||||
pub mod edge_registry;
|
||||
pub mod host_validation;
|
||||
pub mod introspection;
|
||||
pub mod path_safety;
|
||||
|
||||
@@ -35,10 +35,13 @@ use axum::{
|
||||
extract::{
|
||||
ws::{Message, WebSocket, WebSocketUpgrade},
|
||||
Path,
|
||||
Query,
|
||||
State,
|
||||
},
|
||||
http::StatusCode,
|
||||
response::{Html, IntoResponse, Json},
|
||||
routing::{delete, get, post},
|
||||
Extension,
|
||||
Router,
|
||||
};
|
||||
use clap::Parser;
|
||||
@@ -181,6 +184,35 @@ struct Args {
|
||||
/// Start field model calibration on boot (empty room required)
|
||||
#[arg(long)]
|
||||
calibrate: bool,
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// ADR-102: Edge Module Registry — surface the canonical Cognitum
|
||||
// cog catalog via `GET /api/v1/edge/registry`.
|
||||
// ---------------------------------------------------------------
|
||||
/// Override the upstream URL for the edge module registry. Set to a
|
||||
/// mirror or local file://... URL for air-gapped deployments. Empty
|
||||
/// string or --no-edge-registry disables the endpoint entirely.
|
||||
#[arg(
|
||||
long,
|
||||
value_name = "URL",
|
||||
env = "RUVIEW_EDGE_REGISTRY_URL",
|
||||
default_value = "https://storage.googleapis.com/cognitum-apps/app-registry.json"
|
||||
)]
|
||||
edge_registry_url: String,
|
||||
|
||||
/// Cache TTL for the edge module registry, in seconds.
|
||||
#[arg(
|
||||
long,
|
||||
value_name = "SECS",
|
||||
env = "RUVIEW_EDGE_REGISTRY_TTL_SECS",
|
||||
default_value = "3600"
|
||||
)]
|
||||
edge_registry_ttl_secs: u64,
|
||||
|
||||
/// Disable the edge module registry endpoint entirely. Returns 404 on
|
||||
/// `GET /api/v1/edge/registry`. Use for air-gapped deployments.
|
||||
#[arg(long, env = "RUVIEW_NO_EDGE_REGISTRY")]
|
||||
no_edge_registry: bool,
|
||||
}
|
||||
|
||||
// ── Data types ───────────────────────────────────────────────────────────────
|
||||
@@ -3689,6 +3721,67 @@ async fn vital_signs_endpoint(State(state): State<SharedState>) -> Json<serde_js
|
||||
}))
|
||||
}
|
||||
|
||||
/// Query params for `GET /api/v1/edge/registry`.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct EdgeRegistryParams {
|
||||
/// `?refresh=1` bypasses the in-process cache. Logged at debug for
|
||||
/// abuse visibility. ADR-102 §"Cache semantics".
|
||||
#[serde(default)]
|
||||
refresh: Option<String>,
|
||||
}
|
||||
|
||||
/// GET /api/v1/edge/registry — surfaces the canonical Cognitum cog catalog.
|
||||
///
|
||||
/// See ADR-102 (`docs/adr/ADR-102-edge-module-registry.md`) for the design
|
||||
/// + trust model + security review.
|
||||
async fn edge_registry_endpoint(
|
||||
Extension(reg): Extension<
|
||||
Option<Arc<wifi_densepose_sensing_server::edge_registry::EdgeRegistry>>,
|
||||
>,
|
||||
Query(params): Query<EdgeRegistryParams>,
|
||||
) -> Result<Json<serde_json::Value>, (StatusCode, Json<serde_json::Value>)> {
|
||||
let Some(reg) = reg else {
|
||||
// --no-edge-registry, or upstream URL empty.
|
||||
return Err((
|
||||
StatusCode::NOT_FOUND,
|
||||
Json(serde_json::json!({
|
||||
"error": "edge_registry_disabled",
|
||||
"detail": "This sensing-server was started with --no-edge-registry."
|
||||
})),
|
||||
));
|
||||
};
|
||||
let force_refresh = matches!(params.refresh.as_deref(), Some("1") | Some("true"));
|
||||
if force_refresh {
|
||||
tracing::debug!(
|
||||
event = "edge_registry.refresh_requested",
|
||||
"?refresh=1 bypassed the cache; verify this isn't being abused"
|
||||
);
|
||||
}
|
||||
match tokio::task::spawn_blocking(move || reg.get(force_refresh)).await {
|
||||
Ok(Ok(resp)) => Ok(Json(serde_json::to_value(resp).unwrap_or(serde_json::json!({})))),
|
||||
Ok(Err(err)) => {
|
||||
tracing::warn!(error = %err, "edge_registry upstream fetch failed and no cache");
|
||||
Err((
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(serde_json::json!({
|
||||
"error": "edge_registry_upstream_unavailable",
|
||||
"detail": err.to_string()
|
||||
})),
|
||||
))
|
||||
}
|
||||
Err(join_err) => {
|
||||
tracing::error!(error = %join_err, "edge_registry spawn_blocking task panicked");
|
||||
Err((
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(serde_json::json!({
|
||||
"error": "edge_registry_internal_error",
|
||||
"detail": join_err.to_string()
|
||||
})),
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// GET /api/v1/edge-vitals — latest edge vitals from ESP32 (ADR-039).
|
||||
async fn edge_vitals_endpoint(State(state): State<SharedState>) -> Json<serde_json::Value> {
|
||||
let s = state.read().await;
|
||||
@@ -5048,6 +5141,26 @@ async fn main() {
|
||||
let runtime_config = load_runtime_config(&data_dir);
|
||||
info!("Loaded runtime config: dedup_factor={:.2}", runtime_config.dedup_factor);
|
||||
|
||||
// ADR-102: optional Edge Module Registry. None when --no-edge-registry
|
||||
// is set (or when the URL is empty); otherwise we construct one with
|
||||
// the configured TTL. The fetch happens lazily on first request.
|
||||
let edge_registry: Option<std::sync::Arc<wifi_densepose_sensing_server::edge_registry::EdgeRegistry>> =
|
||||
if args.no_edge_registry || args.edge_registry_url.is_empty() {
|
||||
info!("Edge module registry: DISABLED (--no-edge-registry or empty URL)");
|
||||
None
|
||||
} else {
|
||||
info!(
|
||||
"Edge module registry: enabled — upstream={} ttl={}s",
|
||||
args.edge_registry_url, args.edge_registry_ttl_secs
|
||||
);
|
||||
Some(std::sync::Arc::new(
|
||||
wifi_densepose_sensing_server::edge_registry::EdgeRegistry::new(
|
||||
args.edge_registry_url.clone(),
|
||||
std::time::Duration::from_secs(args.edge_registry_ttl_secs),
|
||||
),
|
||||
))
|
||||
};
|
||||
|
||||
let (tx, _) = broadcast::channel::<String>(256);
|
||||
// ADR-099: parallel broadcast for the per-frame introspection snapshot stream
|
||||
// consumed by `/ws/introspection`. Same ring size as `tx` (256) — slow
|
||||
@@ -5242,6 +5355,11 @@ async fn main() {
|
||||
// Vital sign endpoints
|
||||
.route("/api/v1/vital-signs", get(vital_signs_endpoint))
|
||||
.route("/api/v1/edge-vitals", get(edge_vitals_endpoint))
|
||||
// ADR-102: Edge Module Registry — surfaces the canonical Cognitum cog
|
||||
// catalog (`https://storage.googleapis.com/cognitum-apps/app-registry.json`)
|
||||
// with in-process TTL cache + stale-on-error fallback. Disabled when
|
||||
// --no-edge-registry is set (returns 404).
|
||||
.route("/api/v1/edge/registry", get(edge_registry_endpoint))
|
||||
.route("/api/v1/wasm-events", get(wasm_events_endpoint))
|
||||
// RVF model container info
|
||||
.route("/api/v1/model/info", get(model_info))
|
||||
@@ -5292,6 +5410,9 @@ async fn main() {
|
||||
.route("/api/v1/config/ground-truth", post(config_set_ground_truth))
|
||||
// Static UI files
|
||||
.nest_service("/ui", ServeDir::new(&ui_path))
|
||||
// ADR-102: make the edge registry handle (Option<Arc<EdgeRegistry>>)
|
||||
// available to the /api/v1/edge/registry handler. None when disabled.
|
||||
.layer(Extension(edge_registry.clone()))
|
||||
.layer(SetResponseHeaderLayer::overriding(
|
||||
axum::http::header::CACHE_CONTROL,
|
||||
HeaderValue::from_static("no-cache, no-store, must-revalidate"),
|
||||
|
||||
Reference in New Issue
Block a user