Compare commits
108 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6b35896847 | |||
| 2783f40bd1 | |||
| 3f462a254d | |||
| bb92419ccb | |||
| d9ca9b3684 | |||
| a85d4e31e4 | |||
| b16d7431bc | |||
| b3a5012dbd | |||
| e6a5df36eb | |||
| 5c914e63c7 | |||
| a5e99670f8 | |||
| 6b4994e105 | |||
| 6959a42312 | |||
| 962e0f4a34 | |||
| c58f49f21a | |||
| cbcb389cb6 | |||
| e00cee6146 | |||
| 5dcafc9c37 | |||
| e21803f714 | |||
| bdd1efeb03 | |||
| aeb69315d8 | |||
| cfda8dbd14 | |||
| dc865c236e | |||
| 96bc4b4ede | |||
| feda871e02 | |||
| 43ac76a17f | |||
| 6a2b2bdcbf | |||
| d67d9872c1 | |||
| 67fec45e61 | |||
| dc7f6cd096 | |||
| 4b1a835107 | |||
| 9c3c8b98bc | |||
| fcb6f4bf12 | |||
| 3314c8db8d | |||
| ef20a7280d | |||
| ad15f1b049 | |||
| 8247d28d90 | |||
| 5d6e50d8a0 | |||
| 49fb2ca9f4 | |||
| 3439fb1402 | |||
| c00f45e296 | |||
| f54f0285bd | |||
| e964eaf14f | |||
| 961c01f4bd | |||
| 79cc2d7b22 | |||
| f5e2b5474b | |||
| 281c4cb0ce | |||
| b2e2e6d6fd | |||
| 72bbd256e7 | |||
| 50131b2519 | |||
| 50136c920d | |||
| 3bd70f7910 | |||
| 6f5ac3aa5a | |||
| 1b155ad027 | |||
| fa28318bae | |||
| ec73109d57 | |||
| acbd3ff13c | |||
| 07086c5d9d | |||
| 0310b1fa9a | |||
| 9daa8c3078 | |||
| ffa808ed4b | |||
| 59dbb76757 | |||
| 4ecc053a27 | |||
| 5170b99aca | |||
| c16dc9f80a | |||
| 04ccfcde56 | |||
| 4d45add824 | |||
| 562cb7461f | |||
| fad6828697 | |||
| 807bf0b32a | |||
| 4b602c79dd | |||
| 76321ce4bc | |||
| 1690aea22a | |||
| a80617ee84 | |||
| 75dc302952 | |||
| afc86c6fc4 | |||
| fc654034b3 | |||
| c4653b8bc6 | |||
| d214855228 | |||
| e6710e8988 | |||
| ab9799adc3 | |||
| bdb4484259 | |||
| ba370c7b08 | |||
| 3fdd310f89 | |||
| 98e7eeda42 | |||
| 5615edb24e | |||
| 9cc9419db9 | |||
| d544b8f070 | |||
| d33962eff2 | |||
| e22a24714a | |||
| cee414f3c0 | |||
| f853c74563 | |||
| 8b297dd706 | |||
| 9d4f7820b2 | |||
| b2fe452e74 | |||
| 88da304631 | |||
| 880a3a41d3 | |||
| 68b042faf6 | |||
| 4698f54fa0 | |||
| ea62ec4667 | |||
| 3685d16a49 | |||
| 8a155e07ec | |||
| 540ecb4538 | |||
| 10684972d7 | |||
| 27a6edba8b | |||
| 174e2365f0 | |||
| bf30844835 | |||
| ce7983eb43 |
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
- name: Set up Python
|
||||
continue-on-error: true
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
@@ -166,7 +166,7 @@ jobs:
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
continue-on-error: true
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'pip'
|
||||
@@ -198,7 +198,7 @@ jobs:
|
||||
|
||||
- name: Upload coverage reports
|
||||
continue-on-error: true
|
||||
uses: codecov/codecov-action@v4
|
||||
uses: codecov/codecov-action@v6
|
||||
with:
|
||||
file: ./coverage.xml
|
||||
flags: unittests
|
||||
@@ -216,17 +216,21 @@ jobs:
|
||||
htmlcov/
|
||||
|
||||
# Performance and Load Tests
|
||||
# NOTE: tests/performance/locustfile.py and the src.api.main app path both
|
||||
# predate the v1→archive/v1 reorganisation. continue-on-error: true until a
|
||||
# proper locust suite is added under archive/v1/tests/performance/.
|
||||
performance-test:
|
||||
name: Performance Tests
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test]
|
||||
continue-on-error: true
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
@@ -238,6 +242,7 @@ jobs:
|
||||
pip install locust
|
||||
|
||||
- name: Start application
|
||||
working-directory: archive/v1
|
||||
run: |
|
||||
uvicorn src.api.main:app --host 0.0.0.0 --port 8000 &
|
||||
sleep 10
|
||||
@@ -285,7 +290,7 @@ jobs:
|
||||
- name: Extract metadata
|
||||
continue-on-error: true
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
@@ -296,7 +301,7 @@ jobs:
|
||||
|
||||
- name: Build and push Docker image
|
||||
continue-on-error: true
|
||||
uses: docker/build-push-action@v5
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
target: production
|
||||
@@ -341,7 +346,7 @@ jobs:
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
@@ -352,6 +357,7 @@ jobs:
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: Generate OpenAPI spec
|
||||
working-directory: archive/v1
|
||||
run: |
|
||||
python -c "
|
||||
from src.api.main import app
|
||||
@@ -373,6 +379,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [code-quality, test, rust-tests, performance-test, docker-build, docs]
|
||||
if: always()
|
||||
permissions:
|
||||
contents: write # required by softprops/action-gh-release
|
||||
# GitHub Actions does not allow `secrets.X` directly in step-level `if:`
|
||||
# expressions — only `env.X`. Promote the secret to env at job scope so
|
||||
# the gating expression below is parseable.
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
name: GitHub Clone Tracking → data/clone-data.rvf
|
||||
|
||||
# Persists rolling 14-day clone-traffic snapshots to data/clone-data.rvf in
|
||||
# the ruvector JSONL RVF format. GitHub's /traffic/clones endpoint only
|
||||
# retains the last 14 days server-side, so without this scheduled scrape
|
||||
# the data is gone forever the moment it falls outside the window.
|
||||
#
|
||||
# Format: JSONL RVF
|
||||
# - line 1 is a `metadata` segment that initializes the file
|
||||
# - each subsequent run appends one `clone_snapshot` segment carrying the
|
||||
# 14-day rollup PLUS per-day breakdown
|
||||
# - file is idempotent: per-day entries are keyed by `timestamp` so a
|
||||
# downstream reader can dedupe across overlapping snapshot windows
|
||||
#
|
||||
# Schedule: every 14 days (1st + 15th of each month, ~14-day cadence in
|
||||
# practice). Workflow can also be dispatched manually for backfill or test.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 01:23 UTC on the 1st and 15th of every month — close to 14-day cadence
|
||||
# without cron's "every 14 days" monthly-reset weirdness. Picking :23
|
||||
# avoids the cron herd on :00.
|
||||
- cron: '23 1 1,15 * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
group: clone-tracking
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
snapshot:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Fetch /traffic/clones + /traffic/views from GitHub
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
mkdir -p data
|
||||
gh api repos/${{ github.repository }}/traffic/clones > /tmp/clones.json
|
||||
gh api repos/${{ github.repository }}/traffic/views > /tmp/views.json
|
||||
echo "--- clones rollup ---"
|
||||
jq '{count, uniques, days: (.clones | length)}' /tmp/clones.json
|
||||
echo "--- views rollup ---"
|
||||
jq '{count, uniques, days: (.views | length)}' /tmp/views.json
|
||||
|
||||
- name: Append snapshot to data/clone-data.rvf
|
||||
env:
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
set -e
|
||||
RVF="data/clone-data.rvf"
|
||||
FETCHED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
# Initialize the file with a metadata segment on first run.
|
||||
if [ ! -f "$RVF" ]; then
|
||||
echo "Initializing $RVF with metadata segment"
|
||||
jq -n --arg repo "$REPO" --arg ts "$FETCHED_AT" '{
|
||||
type: "metadata",
|
||||
name: "ruview-clone-traffic-history",
|
||||
version: "1.0.0",
|
||||
schema: "ruvector.rvf.jsonl/v1",
|
||||
format: "github-traffic-snapshots",
|
||||
repo: $repo,
|
||||
source: "GitHub Traffic API /repos/{repo}/traffic/{clones,views}",
|
||||
policy: "GitHub retains only 14 days server-side; this file is the long-term record.",
|
||||
segments: ["metadata", "clone_snapshot", "view_snapshot"],
|
||||
created_at: $ts,
|
||||
custom: {
|
||||
cadence: "twice monthly (1st and 15th, ~14-day intervals)",
|
||||
idempotency_key: "timestamp (per-day records de-duplicate across overlapping snapshot windows)"
|
||||
}
|
||||
}' >> "$RVF"
|
||||
fi
|
||||
|
||||
# Append the clone snapshot.
|
||||
jq --arg ts "$FETCHED_AT" '{
|
||||
type: "clone_snapshot",
|
||||
fetched_at: $ts,
|
||||
window_count: .count,
|
||||
window_uniques: .uniques,
|
||||
per_day: .clones
|
||||
}' /tmp/clones.json >> "$RVF"
|
||||
|
||||
# Append the views snapshot (free with the same auth).
|
||||
jq --arg ts "$FETCHED_AT" '{
|
||||
type: "view_snapshot",
|
||||
fetched_at: $ts,
|
||||
window_count: .count,
|
||||
window_uniques: .uniques,
|
||||
per_day: .views
|
||||
}' /tmp/views.json >> "$RVF"
|
||||
|
||||
echo "--- RVF tail (last 4 lines) ---"
|
||||
tail -4 "$RVF" | jq -c '{type, fetched_at, window_count, window_uniques}' || true
|
||||
echo "--- file size ---"
|
||||
wc -l "$RVF"
|
||||
|
||||
- name: Compute aggregates for the commit summary
|
||||
id: agg
|
||||
run: |
|
||||
# Count distinct per-day entries across all snapshots so we can
|
||||
# show "cumulative observed clones" in the commit message.
|
||||
python3 - <<'PY'
|
||||
import json, os
|
||||
path = "data/clone-data.rvf"
|
||||
per_day_clones = {}
|
||||
per_day_views = {}
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
if not line.strip():
|
||||
continue
|
||||
d = json.loads(line)
|
||||
if d.get("type") == "clone_snapshot":
|
||||
for entry in d.get("per_day", []):
|
||||
per_day_clones[entry["timestamp"]] = entry
|
||||
elif d.get("type") == "view_snapshot":
|
||||
for entry in d.get("per_day", []):
|
||||
per_day_views[entry["timestamp"]] = entry
|
||||
|
||||
tot_clones = sum(e.get("count", 0) for e in per_day_clones.values())
|
||||
tot_uniq_clones = sum(e.get("uniques", 0) for e in per_day_clones.values())
|
||||
tot_views = sum(e.get("count", 0) for e in per_day_views.values())
|
||||
tot_uniq_views = sum(e.get("uniques", 0) for e in per_day_views.values())
|
||||
print(f"clone days observed: {len(per_day_clones)} total clones: {tot_clones:,} total unique cloners: {tot_uniq_clones:,}")
|
||||
print(f"view days observed: {len(per_day_views)} total views: {tot_views:,} total unique viewers: {tot_uniq_views:,}")
|
||||
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as out:
|
||||
out.write(f"clones={tot_clones}\n")
|
||||
out.write(f"clone_days={len(per_day_clones)}\n")
|
||||
out.write(f"views={tot_views}\n")
|
||||
out.write(f"view_days={len(per_day_views)}\n")
|
||||
PY
|
||||
|
||||
- name: Commit + push if changed
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
if git diff --quiet data/clone-data.rvf; then
|
||||
echo "no changes to commit"
|
||||
exit 0
|
||||
fi
|
||||
git add data/clone-data.rvf
|
||||
git commit -m "chore(traffic): clone snapshot — ${{ steps.agg.outputs.clone_days }} days observed → ${{ steps.agg.outputs.clones }} clones, ${{ steps.agg.outputs.view_days }} view-days → ${{ steps.agg.outputs.views }} views"
|
||||
git push
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
--out-dir ../../dashboard/public/nvsim-pkg \
|
||||
--release -- --no-default-features --features wasm
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
- uses: actions/setup-node@v6
|
||||
with: { node-version: 20, cache: npm, cache-dependency-path: dashboard/package-lock.json }
|
||||
|
||||
- working-directory: dashboard
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
-- --no-default-features --features wasm
|
||||
|
||||
- name: Setup Node 20
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 20
|
||||
cache: npm
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: '20'
|
||||
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: '20'
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: ghcr.io/ruvnet/nvsim-server
|
||||
tags: |
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
|
||||
- name: Build + push
|
||||
uses: docker/build-push-action@v5
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: v2
|
||||
file: v2/crates/nvsim-server/Dockerfile
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
|
||||
- name: Set up Python
|
||||
continue-on-error: true
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
@@ -99,7 +99,7 @@ jobs:
|
||||
|
||||
- name: Set up Python
|
||||
continue-on-error: true
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
@@ -170,7 +170,7 @@ jobs:
|
||||
|
||||
- name: Build Docker image for scanning
|
||||
continue-on-error: true
|
||||
uses: docker/build-push-action@v5
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
target: production
|
||||
@@ -197,7 +197,7 @@ jobs:
|
||||
|
||||
- name: Run Grype vulnerability scanner
|
||||
continue-on-error: true
|
||||
uses: anchore/scan-action@v3
|
||||
uses: anchore/scan-action@v7
|
||||
id: grype-scan
|
||||
with:
|
||||
image: 'wifi-densepose:scan'
|
||||
@@ -343,7 +343,7 @@ jobs:
|
||||
|
||||
- name: Set up Python
|
||||
continue-on-error: true
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
|
||||
@@ -50,6 +50,12 @@ jobs:
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
# QEMU is required so the amd64 GitHub runner can cross-build the
|
||||
# linux/arm64 layer below (Dockerfile.rust is arch-agnostic — no `--target`
|
||||
# flag — so buildx + QEMU is all that's needed; arm64 builds are emulated
|
||||
# by the runner, not built on a separate arm64 host).
|
||||
- uses: docker/setup-qemu-action@v3
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
@@ -68,7 +74,7 @@ jobs:
|
||||
|
||||
- name: Compute tags
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: |
|
||||
docker.io/ruvnet/wifi-densepose
|
||||
@@ -81,7 +87,7 @@ jobs:
|
||||
|
||||
- name: Build + push
|
||||
id: build
|
||||
uses: docker/build-push-action@v5
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile.rust
|
||||
@@ -90,7 +96,11 @@ jobs:
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
platforms: linux/amd64
|
||||
# README badge advertises `amd64 + arm64`, and #547 promised multi-arch
|
||||
# as part of the docker publish refresh; arm64 was never actually wired
|
||||
# in, so Apple Silicon Macs hit `no matching manifest for linux/arm64/v8`
|
||||
# on `docker pull ruvnet/wifi-densepose:latest` (#136, #625). Build both.
|
||||
platforms: linux/amd64,linux/arm64
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Smoke-test the freshly-pushed image:
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
name: three.js demos → GitHub Pages
|
||||
|
||||
# Publishes the ADR-097 three.js demos under gh-pages/three.js/.
|
||||
# Uses keep_files: true so the existing observatory/, pose-fusion/,
|
||||
# pointcloud/, nvsim/, and root index.html demos are preserved.
|
||||
#
|
||||
# Demos 04 and 05 require a Mixamo "X Bot.fbx" placed in assets/.
|
||||
# That file is intentionally gitignored (license boundary), so this
|
||||
# workflow does NOT ship it. Demos 01-03 work standalone; the index
|
||||
# page documents the FBX requirement honestly.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'examples/three.js/**'
|
||||
- '.github/workflows/threejs-pages.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
group: threejs-pages
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-and-deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout main
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Stage demos for Pages
|
||||
run: |
|
||||
mkdir -p _site/three.js
|
||||
# Copy everything except the local Python server (CI doesn't need it)
|
||||
# and any stray scratch screenshots.
|
||||
cp -r examples/three.js/demos _site/three.js/demos
|
||||
cp -r examples/three.js/screenshots _site/three.js/screenshots
|
||||
cp examples/three.js/README.md _site/three.js/README.md
|
||||
# An index.html that lists the 5 demos with the FBX caveat.
|
||||
cp examples/three.js/index.html _site/three.js/index.html
|
||||
# Mixamo FBX is gitignored — assets dir won't exist in CI.
|
||||
# Drop an empty placeholder so the relative path 'assets/' resolves
|
||||
# to a directory listing (404 on missing file) instead of an opaque
|
||||
# network error. Browsers showing the 404 path makes the failure
|
||||
# visible to anyone trying demos 04/05 without their own FBX.
|
||||
mkdir -p _site/three.js/assets
|
||||
cat > _site/three.js/assets/README.txt <<'EOF'
|
||||
The Mixamo "X Bot.fbx" required by demos 04-skinned-fbx.html and
|
||||
05-skinned-realtime.html is intentionally not redistributed here.
|
||||
|
||||
Download your own from https://mixamo.com (FBX Binary, T-Pose,
|
||||
Without Skin) and place it here as "X Bot.fbx" if you want to
|
||||
run those demos locally. See examples/three.js/README.md in the
|
||||
repo for context.
|
||||
EOF
|
||||
echo "Staged contents:"
|
||||
ls -R _site/three.js/ | head -30
|
||||
|
||||
- name: Deploy to GitHub Pages
|
||||
uses: peaceiris/actions-gh-pages@v3
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
publish_dir: _site
|
||||
# Critical: preserve observatory/, pose-fusion/, pointcloud/, nvsim/
|
||||
# and the root index.html already on gh-pages.
|
||||
keep_files: true
|
||||
commit_message: 'three.js demos: ${{ github.event.head_commit.message }}'
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
@@ -57,7 +57,18 @@ jobs:
|
||||
"
|
||||
|
||||
- name: Run pipeline verification
|
||||
working-directory: v1
|
||||
working-directory: archive/v1
|
||||
env:
|
||||
# Pin thread count for scipy.fft / BLAS — multi-threaded reduction
|
||||
# order is otherwise non-deterministic across CI runs (issue #560
|
||||
# follow-up: 9- and 6-decimal quantization were not enough because
|
||||
# the divergence is from threading order, not SIMD reordering).
|
||||
# Single-threaded keeps the proof reproducible at a ~2-3x slowdown.
|
||||
OMP_NUM_THREADS: "1"
|
||||
OPENBLAS_NUM_THREADS: "1"
|
||||
MKL_NUM_THREADS: "1"
|
||||
VECLIB_MAXIMUM_THREADS: "1"
|
||||
NUMEXPR_NUM_THREADS: "1"
|
||||
run: |
|
||||
echo "=== Running pipeline verification ==="
|
||||
python data/proof/verify.py
|
||||
@@ -65,7 +76,13 @@ jobs:
|
||||
echo "Pipeline verification PASSED."
|
||||
|
||||
- name: Run verification twice to confirm determinism
|
||||
working-directory: v1
|
||||
working-directory: archive/v1
|
||||
env:
|
||||
OMP_NUM_THREADS: "1"
|
||||
OPENBLAS_NUM_THREADS: "1"
|
||||
MKL_NUM_THREADS: "1"
|
||||
VECLIB_MAXIMUM_THREADS: "1"
|
||||
NUMEXPR_NUM_THREADS: "1"
|
||||
run: |
|
||||
echo "=== Second run for determinism confirmation ==="
|
||||
python data/proof/verify.py
|
||||
|
||||
@@ -13,6 +13,9 @@ firmware/esp32-csi-node/managed_components/
|
||||
firmware/esp32-csi-node/dependencies.lock
|
||||
firmware/esp32-csi-node/sdkconfig.defaults.bak
|
||||
|
||||
# ESP-IDF set-target backup (local only)
|
||||
firmware/esp32-hello-world/sdkconfig.old
|
||||
|
||||
# Claude Flow swarm runtime state
|
||||
.swarm/
|
||||
|
||||
|
||||
@@ -7,6 +7,60 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Security
|
||||
- **ESP32 OTA upload now fails closed when no PSK is provisioned** (#596 audit finding — critical, **breaking change for unprovisioned nodes**). `ota_check_auth()` previously returned `true` when `s_ota_psk[0] == '\0'`, so a freshly-flashed node would accept attacker-controlled firmware over plain HTTP on port 8032 from any host on the WiFi. No Secure Boot V2, no signed-image verification — a single LAN call could brick or backdoor a node. The fix rejects every OTA upload until a PSK is written to NVS (the OTA HTTP server still starts so operators can run `provision.py --ota-psk <hex>` over USB-CDC without reflashing). **Operators affected**: any deployment that relied on the unauthenticated OTA endpoint working out of the box now needs to provision a PSK before subsequent OTA pushes will succeed. Boot-time `ESP_LOGW` makes the new posture visible.
|
||||
- **Path-traversal vulnerabilities patched in five sensing-server endpoints** (closes #615 — critical). New `wifi_densepose_sensing_server::path_safety::safe_id()` enforces `[A-Za-z0-9._-]` only (no leading `.`, max 64 chars) before any user-controlled identifier reaches a `format!()` building a filesystem path. Applied at:
|
||||
- `POST /api/v1/recording/start` (`recording.rs` — `session_name`)
|
||||
- `GET /api/v1/recording/download/:id` (`recording.rs` — `id`)
|
||||
- `DELETE /api/v1/recording/delete/:id` (`recording.rs` — `id`)
|
||||
- `POST /api/v1/models/load` (`model_manager.rs` — `model_id`)
|
||||
- `training_api.rs` `load_recording_frames` (`dataset_id`s)
|
||||
|
||||
Pre-fix, unauthenticated callers could read `../../etc/passwd`-style paths, write arbitrary JSONL files, load attacker-controlled `.rvf` model files, or delete arbitrary files the server process could touch. 9 unit tests in `path_safety::tests` exercise the rejection envelope (empty, too-long, path separators, parent-dir traversal, null byte, whitespace/specials, non-ASCII).
|
||||
|
||||
### Fixed
|
||||
- **WebSocket `/ws/sensing` now reports `esp32:offline` when ESP32 hardware goes stale** (closes #618). `broadcast_tick_task` was re-emitting the cached `latest_update` with a frozen `source: "esp32"` field forever after the hardware lost power or network. The REST `/health` endpoint already called `effective_source()` (which returns `"esp32:offline"` after `ESP32_OFFLINE_TIMEOUT` = 5 s with no UDP frames), but the WS broadcast path was the one consumer that didn't. Result: the UI's "LIVE — ESP32 HARDWARE Connected" banner stayed green long after the hardware went away, and `vital_signs`/`features`/`classification` re-broadcasted the last-seen values indefinitely. Fix: clone the cached `latest_update` per tick, overwrite `source` with `s.effective_source()`, then serialize and broadcast. UI can now switch to an offline state on the same 5-second budget the REST surface uses.
|
||||
- **Proof replay (`archive/v1/data/proof/verify.py`) is now cross-platform deterministic** (closes #560). Three changes together: (1) `features_to_bytes()` now `np.round(.., HASH_QUANTIZATION_DECIMALS=6)`s each feature array before packing as little-endian f64, collapsing ULP-level drift from scipy.fft pocketfft SIMD reordering; (2) the `Verify Pipeline Determinism` workflow pins `OMP_NUM_THREADS=1`, `OPENBLAS_NUM_THREADS=1`, `MKL_NUM_THREADS=1`, `VECLIB_MAXIMUM_THREADS=1`, `NUMEXPR_NUM_THREADS=1` — multi-threaded BLAS reductions were a deeper source of non-determinism than SIMD reordering, and 6-decimal quantization alone wasn't enough across Azure VM microarchitectures; (3) `expected_features.sha256` regenerated under the new conditions. CI now passes the determinism check (same hash across consecutive runs on canonical Linux x86_64 CI runner: `667eb054c44ac510342665bf9c93d608868a8ead948ae8774b2796ebce6f8fe7`). `scripts/probe-fft-platform.py` updated to mirror `HASH_QUANTIZATION_DECIMALS=6` for cross-machine spot-checks.
|
||||
- **`archive/v1/src/services/pose_service.py:223` calls the right method on `PhaseSanitizer`** (closes #612). The call was `self.phase_sanitizer.sanitize(phase_data)`, but `PhaseSanitizer`'s full-pipeline entry point is named `sanitize_phase()` (`unwrap_phase` + `remove_outliers` + `smooth_phase` chained, see `archive/v1/src/core/phase_sanitizer.py:266`). The shorter `sanitize` name doesn't exist on the class, so any path that reached this branch raised `AttributeError` and crashed the pose service mid-frame.
|
||||
- **`adaptive_classifier.rs:94` no longer panics on NaN feature values** (closes #611).
|
||||
`sorted.sort_by(|a, b| a.partial_cmp(b).unwrap())` returned `None` and panicked
|
||||
whenever a single `NaN` reached the classifier from real ESP32 hardware (silent
|
||||
DSP div-by-zero, empty buffer). One bad frame killed the entire sensing-server
|
||||
process. Swapped for `unwrap_or(Ordering::Equal)`, matching the pattern the
|
||||
same file already used at lines 149-150 and 155. Per-frame hot path; this was
|
||||
a real production crash vector.
|
||||
- **Completed the #611 NaN-panic audit across the sensing-server crate** (follow-up
|
||||
to #613). The original audit grepped for the literal `partial_cmp(b).unwrap()`
|
||||
and missed seven additional production sites that use comparator variants
|
||||
(`partial_cmp(b.1).unwrap()`, `partial_cmp(&variances[b]).unwrap()`). All share
|
||||
the same crash class — a single `NaN` in CSI-derived state panics the whole
|
||||
sensing-server. Fixed:
|
||||
- `adaptive_classifier.rs:205` — `AdaptiveModel::classify()` argmax over softmax
|
||||
probs. **Same per-frame hot path as #611**; NaN flows through normalise →
|
||||
logits → softmax and still reaches this site even after the #613 IQR fix.
|
||||
- `adaptive_classifier.rs:480, 500` — training-loop argmax in `train()`
|
||||
(training/per-class accuracy reporting).
|
||||
- `main.rs:2446, 2449` and `csi.rs:602, 605` — variance-based source/sink
|
||||
selection in `count_persons_mincut`. The outer `unwrap_or((0, &0))` only
|
||||
catches an empty iterator; it cannot rescue a comparator panic.
|
||||
|
||||
Remaining `partial_cmp(...).unwrap()` sites in the workspace are all inside
|
||||
`#[cfg(test)]` / `#[test]` blocks (`spectrogram.rs:269`, `depth.rs:234`,
|
||||
`connectivity.rs:477`, `vital_signs.rs:737`) where inputs are controlled.
|
||||
- **`ui/utils/pose-renderer.js` no longer divides by zero** when two render frames land in the same `performance.now()` tick (issue #519 Bug 2). `deltaTime` is now `Math.max(currentTime - lastFrameTime, 1)` before the `1000 / deltaTime` division, capping displayed FPS at 1000 — far above any real render rate, but finite so the EMA `averageFps = averageFps * 0.9 + fps * 0.1` no longer poisons itself to `Infinity` on a single zero-dt tick.
|
||||
|
||||
### Removed
|
||||
- **Stub crates `wifi-densepose-api`, `wifi-densepose-db`, `wifi-densepose-config`** (closes #578).
|
||||
Each was a single-line doc-comment placeholder with an empty `[dependencies]`
|
||||
section and zero references from any source file or `Cargo.toml`. The names
|
||||
were reserved early for an envisioned REST/database/config split that never
|
||||
materialised; the functionality they would provide is covered today by
|
||||
`wifi-densepose-sensing-server` (Axum REST/WS), per-crate config + CLI args,
|
||||
and the project's real-time-only (no-persistent-state) posture. Removing them
|
||||
from the workspace prevents `cargo` from listing dead crates and shipping
|
||||
empty published artifacts. If any of these names is needed in the future,
|
||||
they can be reintroduced with a real implementation.
|
||||
|
||||
### Added
|
||||
- **Real-time CSI introspection / low-latency tap on `wifi-densepose-sensing-server` (ADR-099).**
|
||||
New `wifi_densepose_sensing_server::introspection` module wires
|
||||
@@ -108,6 +162,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- **README: corrected the camera-supervised pose-accuracy claim** (audit finding #5; see PR #535) — "92.9% PCK@20" → the ADR-079 target (35%+; proxy baseline 35.3%), noting P7/P8/P9 are pending.
|
||||
|
||||
### Added
|
||||
- **`RollingP95` adaptive feature normalizer** (`v2/crates/wifi-densepose-sensing-server`) —
|
||||
Streaming P95 estimator (600-sample / ~30 s sliding window) that self-calibrates
|
||||
feature normalization to whatever distribution the deployment produces. Replaces
|
||||
fixed-scale denominators (`variance/300`, `motion/250`, `spectral/500`) which saturated
|
||||
when live ESP32 values exceeded those limits, collapsing dynamic range to zero.
|
||||
Cold-start (<60 samples) falls back to the legacy denominators so day-0 behaviour
|
||||
is preserved. Deployment-neutral: no hardcoded values. (ADR-044 §5.2)
|
||||
|
||||
- **`dedup_factor` runtime configuration API** (`v2/crates/wifi-densepose-sensing-server`) —
|
||||
Exposes the multi-node person-count deduplication divisor at runtime via REST:
|
||||
- `GET /api/v1/config/dedup-factor` — read current value.
|
||||
- `POST /api/v1/config/dedup-factor` — set value (clamped 1.0–10.0, persisted).
|
||||
- `POST /api/v1/config/ground-truth` — auto-tunes `dedup_factor` from a known
|
||||
person count (`{"count": N}`); derives optimal divisor from current node-sum.
|
||||
Config is persisted to `data/config.json` and reloaded on restart. (ADR-044 §5.3)
|
||||
|
||||
- **`nvsim` crate — deterministic NV-diamond magnetometer pipeline simulator** (ADR-089) —
|
||||
New standalone leaf crate at `v2/crates/nvsim` modeling a forward-only
|
||||
magnetic sensing path: scene → source synthesis (Biot–Savart, dipole,
|
||||
@@ -127,6 +197,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
saturation, hyperfine spectroscopy, or pulsed protocols become required.
|
||||
|
||||
### Fixed
|
||||
- **WebSocket broadcast handler now handles Lagged events gracefully and sends periodic ping keepalives to prevent dashboard disconnects** —
|
||||
`handle_ws_client` and `handle_ws_pose_client` in `wifi-densepose-sensing-server`
|
||||
were treating `RecvError::Lagged` as a fatal error, causing instant disconnect
|
||||
when clients fell behind the 256-frame broadcast buffer at 10 Hz ingest.
|
||||
Clients would reconnect, immediately lag again, and rapid-cycle every 2–4 s.
|
||||
`Lagged` now continues (drops missed frames, logs debug) rather than breaking.
|
||||
Added 30 s ping keepalive on the sensing handler to prevent proxy idle timeouts.
|
||||
- **Ghost skeletons in live UI with multi-node ESP32 setups** (#420, ADR-082) —
|
||||
`tracker_bridge::tracker_to_person_detections` documented itself as filtering
|
||||
to `is_alive()` tracks but in fact passed every non-Terminated track to the
|
||||
|
||||
@@ -14,9 +14,6 @@ Dual codebase: Python v1 (`v1/`) and Rust port (`v2/`).
|
||||
| `wifi-densepose-mat` | Mass Casualty Assessment Tool — disaster survivor detection |
|
||||
| `wifi-densepose-hardware` | ESP32 aggregator, TDM protocol, channel hopping firmware |
|
||||
| `wifi-densepose-ruvector` | RuVector v2.0.4 integration + cross-viewpoint fusion (5 modules) |
|
||||
| `wifi-densepose-api` | REST API (Axum) |
|
||||
| `wifi-densepose-db` | Database layer (Postgres, SQLite, Redis) |
|
||||
| `wifi-densepose-config` | Configuration management |
|
||||
| `wifi-densepose-wasm` | WebAssembly bindings for browser deployment |
|
||||
| `wifi-densepose-cli` | CLI tool (`wifi-densepose` binary) |
|
||||
| `wifi-densepose-sensing-server` | Lightweight Axum server for WiFi sensing UI |
|
||||
@@ -135,17 +132,14 @@ Crates must be published in dependency order:
|
||||
2. `wifi-densepose-vitals` (no internal deps)
|
||||
3. `wifi-densepose-wifiscan` (no internal deps)
|
||||
4. `wifi-densepose-hardware` (no internal deps)
|
||||
5. `wifi-densepose-config` (no internal deps)
|
||||
6. `wifi-densepose-db` (no internal deps)
|
||||
7. `wifi-densepose-signal` (depends on core)
|
||||
8. `wifi-densepose-nn` (no internal deps, workspace only)
|
||||
9. `wifi-densepose-ruvector` (no internal deps, workspace only)
|
||||
10. `wifi-densepose-train` (depends on signal, nn)
|
||||
11. `wifi-densepose-mat` (depends on core, signal, nn)
|
||||
12. `wifi-densepose-api` (no internal deps)
|
||||
13. `wifi-densepose-wasm` (depends on mat)
|
||||
14. `wifi-densepose-sensing-server` (depends on wifiscan)
|
||||
15. `wifi-densepose-cli` (depends on mat)
|
||||
5. `wifi-densepose-signal` (depends on core)
|
||||
6. `wifi-densepose-nn` (no internal deps, workspace only)
|
||||
7. `wifi-densepose-ruvector` (no internal deps, workspace only)
|
||||
8. `wifi-densepose-train` (depends on signal, nn)
|
||||
9. `wifi-densepose-mat` (depends on core, signal, nn)
|
||||
10. `wifi-densepose-wasm` (depends on mat)
|
||||
11. `wifi-densepose-sensing-server` (depends on wifiscan)
|
||||
12. `wifi-densepose-cli` (depends on mat)
|
||||
|
||||
### Validation & Witness Verification (ADR-028)
|
||||
|
||||
|
||||
@@ -1,11 +1,17 @@
|
||||
# π RuView
|
||||
|
||||
<p align="center">
|
||||
<a href="https://x.com/rUv/status/2037556932802761004">
|
||||
<a href="https://cognitum.one/seed">
|
||||
<img src="assets/ruview-small-gemini.jpg" alt="RuView - WiFi DensePose" width="100%">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://cognitum.one/seed">
|
||||
<img src="assets/seed.png" alt="Cognitum Seed" width="100%">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
> **Beta Software** — Under active development. APIs and firmware may change. Known limitations:
|
||||
> - ESP32-C3 and original ESP32 are not supported (single-core, insufficient for CSI DSP)
|
||||
> - Single ESP32 deployments have limited spatial resolution — use 2+ nodes or add a [Cognitum Seed](https://cognitum.one) for best results
|
||||
@@ -15,7 +21,7 @@
|
||||
|
||||
## **See through walls with WiFi** ##
|
||||
|
||||
**Turn ordinary WiFi into a spacial intelligence / sensing system.** Detect people, measure breathing and heart rate, track movement, and monitor rooms — through walls, in the dark, with no cameras or wearables. Just physics.
|
||||
**Turn ordinary WiFi into a spatial intelligence / sensing system.** Detect people, measure breathing and heart rate, track movement, and monitor rooms — through walls, in the dark, with no cameras or wearables. Just physics.
|
||||
|
||||
### π RuView is a WiFi sensing platform that turns radio signals into spatial intelligence.
|
||||
|
||||
@@ -32,7 +38,7 @@ Built on [RuVector](https://github.com/ruvnet/ruvector/) and [Cognitum Seed](htt
|
||||
|
||||
The system learns each environment locally using spiking neural networks that adapt in under 30 seconds, with multi-frequency mesh scanning across 6 WiFi channels that uses your neighbors' routers as free radar illuminators. Every measurement is cryptographically attested via an Ed25519 witness chain.
|
||||
|
||||
RuView also supports pose estimation (17 COCO keypoints via the WiFlow architecture), trained entirely without cameras using 10 sensor signals — a technique pioneered from the original *DensePose From WiFi* research at Carnegie Mellon University.
|
||||
RuView turns ordinary WiFi into a contactless sensor. A $9 ESP32 board reads the radio reflections off the people in a room, and a small pretrained model — published on Hugging Face at [`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained) — tells you who's there, how they're breathing, and how their heart rate is trending. The model fits in 8 KB (4-bit quantized), runs in microseconds on a Raspberry Pi, and reports 100% presence accuracy on the validation set. No cameras, no wearables, no app on the user's phone.
|
||||
|
||||
### Built for low-power edge applications
|
||||
|
||||
@@ -45,20 +51,29 @@ RuView also supports pose estimation (17 COCO keypoints via the WiFlow architect
|
||||
[](#vital-sign-detection)
|
||||
[](#esp32-s3-hardware-pipeline)
|
||||
[](https://crates.io/crates/wifi-densepose-ruvector)
|
||||
[](#-edge-module-catalog)
|
||||
|
||||
|
||||
> | What | How | Speed |
|
||||
> |------|-----|-------|
|
||||
> | 🦴 **Pose estimation** | CSI subcarrier amplitude/phase → 17 COCO keypoints | 171K emb/s (M4 Pro) |
|
||||
> | 🫁 **Breathing detection** | Bandpass 0.1-0.5 Hz → zero-crossing BPM | 6-30 BPM |
|
||||
> | 💓 **Heart rate** | Bandpass 0.8-2.0 Hz → zero-crossing BPM | 40-120 BPM |
|
||||
> | 👤 **Presence sensing** | Trained model + PIR fusion — 100% accuracy | 0.012 ms latency |
|
||||
> | 🧱 **Through-wall** | Fresnel zone geometry + multipath modeling | Up to 5m depth |
|
||||
> | 🧠 **Edge intelligence** | 8-dim feature vectors + RVF store on Cognitum Seed | $140 total BOM |
|
||||
> | 🎯 **Camera-free training** | 10 sensor signals, no labels needed | 84s on M4 Pro |
|
||||
> | 📷 **Camera-supervised training** | MediaPipe + ESP32 CSI → **35%+ PCK@20 target** (ADR-079; eval phases pending) | ~19 min on laptop (pipeline) |
|
||||
> | 📡 **Multi-frequency mesh** | Channel hopping across 6 bands, neighbor APs as illuminators | 3x sensing bandwidth |
|
||||
> | 🌐 **3D point cloud** *(optional fusion)* | Camera depth (MiDaS) + WiFi CSI + mmWave radar → unified spatial model | 22 ms pipeline · 19K+ points/frame |
|
||||
> | What | How | Speed / scale |
|
||||
> |------|-----|---------------|
|
||||
> | 🫁 **Breathing rate** | Bandpass 0.1–0.5 Hz on wrapped phase, circular variance, zero-crossing BPM ([#593](https://github.com/ruvnet/RuView/issues/593)) | 6–30 BPM, real-time |
|
||||
> | 💓 **Heart rate** | Bandpass 0.8–2.0 Hz, zero-crossing BPM | 40–120 BPM, real-time |
|
||||
> | 👤 **Presence detection** | Trained head on Hugging Face ([`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained), 100% validation accuracy) + a phase-variance fallback that needs no model | < 1 ms, ~30 s ambient calibration |
|
||||
> | 🧬 **CSI embeddings** | 128-dim contrastive encoder shipped on Hugging Face, 4-bit quantised variant fits in 8 KB | **164,183 emb/s** on M4 Pro |
|
||||
> | 🦴 **17-keypoint pose estimation** | `cog-pose-estimation` Cog v0.0.1 — signed aarch64 + x86_64 binaries on GCS, loads `pose_v1.safetensors` via Candle. Train your own from paired data in 2.1 s on an RTX 5080 ([ADR-101](docs/adr/ADR-101-pose-estimation-cog.md), [benchmarks](docs/benchmarks/pose-estimation-cog.md)) | 8.4 ms cold-start on a Pi 5 |
|
||||
> | 🚶 **Motion / activity** | Motion-band power + phase acceleration | Real-time |
|
||||
> | 🤸 **Fall detection** | Phase-acceleration threshold + 3-frame debounce + 5 s cooldown ([#263](https://github.com/ruvnet/RuView/issues/263)) | < 200 ms |
|
||||
> | 🧮 **Multi-person count** | Adaptive P95 normalisation + runtime-tunable dedup factor (`/api/v1/config/dedup-factor`, [#491](https://github.com/ruvnet/RuView/pull/491)). Six specialised learned counters available as Cogs: `occupancy-zones`, `elevator-count`, `queue-length`, `customer-flow`, `clean-room`, `person-matching` | Real-time, self-calibrating |
|
||||
> | 🧱 **Through-wall sensing** | Fresnel-zone geometry + multipath modeling | Up to ~5 m, signal-dependent |
|
||||
> | 🧠 **Edge intelligence** | **105-cog catalog** ([ADR-102](docs/adr/ADR-102-edge-module-registry.md)) live from `app-registry.json` — health, security, building, retail, industrial, research, AI, swarm, signal, network, and developer modules. Optional Cognitum Seed adds persistent vector store + kNN + witness chain | $140 total BOM |
|
||||
> | 🎯 **Camera-free pre-training** | Self-supervised contrastive encoder, 12.2M training steps on 60K frames, shipped on Hugging Face | 84 s/epoch retrain on M4 Pro |
|
||||
> | 📷 **Camera-supervised fine-tune** | MediaPipe + ESP32 CSI paired training, end-to-end Candle pipeline on RTX 5080 ([ADR-079](docs/adr/ADR-079-camera-supervised-pose-finetune.md)) | 2.1 s for 400 epochs (~5 ms/epoch) |
|
||||
> | 📡 **Multi-frequency mesh** | Channel hopping across 6 bands, TDM slot scheduling ([ADR-029](docs/adr/ADR-029-multifrequency-mesh.md)) | 3× sensing bandwidth |
|
||||
> | 🌐 **3D point cloud fusion** | Camera depth (MiDaS) + WiFi CSI + mmWave radar → unified spatial model | 22 ms pipeline · 19K+ points/frame |
|
||||
>
|
||||
> Browse the full 105-module catalog (with practical descriptions, sizes, and difficulty) below in [🧩 Edge Module Catalog](#-edge-module-catalog), or visit [seed.cognitum.one/store](https://seed.cognitum.one/store).
|
||||
>
|
||||
> 🤗 **Pretrained weights**: download from [`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained) — see [Loading the pretrained model](#loading-the-pretrained-model) below for one-command setup.
|
||||
|
||||
```bash
|
||||
# Option 1: Docker (simulated data, no hardware needed)
|
||||
@@ -88,10 +103,10 @@ node scripts/mincut-person-counter.js --port 5006 # Correct person counting
|
||||
>
|
||||
> | Option | Hardware | Cost | Full CSI | Capabilities |
|
||||
> |--------|----------|------|----------|-------------|
|
||||
> | **ESP32 + Cognitum Seed** (recommended) | ESP32-S3 + [Cognitum Seed](https://cognitum.one) | ~$140 | Yes | Pose, breathing, heartbeat, motion, presence + persistent vector store, kNN search, witness chain, MCP proxy |
|
||||
> | **ESP32 Mesh** | 3-6x ESP32-S3 + WiFi router | ~$54 | Yes | Pose, breathing, heartbeat, motion, presence |
|
||||
> | **ESP32 + Cognitum Seed** (recommended) | ESP32-S3 + [Cognitum Seed](https://cognitum.one) | ~$140 | Yes | Presence, motion, breathing, heart rate, fall detection, multi-person counting, 17-keypoint pose (signed Cog binary), 105-cog catalog, persistent vector store, kNN search, witness chain, MCP proxy |
|
||||
> | **ESP32 Mesh** | 3-6x ESP32-S3 + WiFi router | ~$54 | Yes | Same capabilities as above without the persistent-memory features |
|
||||
> | **Research NIC** | Intel 5300 / Atheros AR9580 | ~$50-100 | Yes | Full CSI with 3x3 MIMO |
|
||||
> | **Any WiFi** | Windows, macOS, or Linux laptop | $0 | No | RSSI-only: coarse presence and motion |
|
||||
> | **Any WiFi** | Windows, macOS, or Linux laptop | $0 | No | RSSI-only: coarse presence and motion (see [tutorial #36](https://github.com/ruvnet/RuView/issues/36)) |
|
||||
>
|
||||
> No hardware? Verify the signal processing pipeline with the deterministic reference signal: `python archive/v1/data/proof/verify.py`
|
||||
>
|
||||
@@ -109,10 +124,211 @@ node scripts/mincut-person-counter.js --port 5006 # Correct person counting
|
||||
<a href="https://ruvnet.github.io/RuView/pose-fusion.html"><strong>▶ Dual-Modal Pose Fusion Demo</strong></a>
|
||||
|
|
||||
<a href="https://ruvnet.github.io/RuView/pointcloud/"><strong>▶ Live 3D Point Cloud</strong></a>
|
||||
|
|
||||
<a href="https://ruvnet.github.io/RuView/three.js/"><strong>▶ three.js Demos (5)</strong></a>
|
||||
|
||||
> The [server](#-quick-start) is optional for visualization and aggregation — the ESP32 [runs independently](#esp32-s3-hardware-pipeline) for presence detection, vital signs, and fall alerts.
|
||||
>
|
||||
> **Live ESP32 pipeline**: Connect an ESP32-S3 node → run the [sensing server](#sensing-server) → open the [pose fusion demo](https://ruvnet.github.io/RuView/pose-fusion.html) for real-time dual-modal pose estimation (webcam + WiFi CSI). See [ADR-059](docs/adr/ADR-059-live-esp32-csi-pipeline.md).
|
||||
>
|
||||
> **three.js scene gallery** at [`/three.js/`](https://ruvnet.github.io/RuView/three.js/) — five progressively richer ADR-097 demos: helpers, cinematic, GLTF skinned, FBX skinned, and a live MediaPipe→Mixamo retargeting feed driven by ESP32 CSI. Demos 04 and 05 require a local Mixamo `X Bot.fbx` (license boundary — not redistributed).
|
||||
|
||||
|
||||
## 🤗 Pretrained model on Hugging Face
|
||||
|
||||
Pretrained CSI weights live at [`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained) — 12.2M training steps on 60K frames / 610K contrastive triplets, **100% presence accuracy** on the validation set, 4-bit quantized variant fits in 8 KB. The release includes a contrastive **CSI encoder** producing 128-dim embeddings (164,183 emb/s on M4 Pro) and a **presence-detection head**. Per-node LoRA adapters are included for environment-specific fine-tuning.
|
||||
|
||||
```bash
|
||||
# Download the model bundle
|
||||
pip install huggingface_hub
|
||||
huggingface-cli download ruvnet/wifi-densepose-pretrained --local-dir models/wifi-densepose-pretrained
|
||||
```
|
||||
|
||||
**What works today vs. what's pending wiring:**
|
||||
|
||||
| Consumer | Format used | Status |
|
||||
|----------|-------------|--------|
|
||||
| Python training / evaluation / embedding extraction | `model.safetensors` | ✅ Works — load with `safetensors.torch.load_file` |
|
||||
| Inspect / re-export the bundle | `model.rvf.jsonl` (line-by-line JSON) | ✅ Works — plain JSONL |
|
||||
| Sensing-server `--model <PATH>` flag | binary RVF (`RVFS` magic) | ⚠️ Loader does not yet accept the JSONL container |
|
||||
|
||||
**Known gap:** the HF model ships in JSONL RVF format, but `v2/crates/wifi-densepose-sensing-server/src/rvf_container.rs` only parses the binary RVF segment format. Pointing `--model` at `model.rvf.jsonl` currently errors with `invalid magic at offset 0: expected 0x52564653, got 0x7974227B` and the live pipeline degrades to null output rather than falling back to heuristic mode — so for the live sensing-server, run **without** `--model` until a JSONL adapter lands (or the model is re-published as binary RVF). Use the weights from Python / training in the meantime.
|
||||
|
||||
**Quantization choices** (all in the HF repo): `model-q2.bin` (4 KB) · `model-q4.bin` ⭐ recommended (8 KB) · `model-q8.bin` (16 KB) · `model.safetensors` full (48 KB)
|
||||
|
||||
The separate **17-keypoint pose-estimation model** is not in this release — pipeline is implemented but keypoint weights are still pending. Tracked in [#509](https://github.com/ruvnet/RuView/issues/509); see [ADR-079](docs/adr/ADR-079-camera-supervised-pose-finetune.md) phases P7–P9.
|
||||
|
||||
|
||||
## 🧩 Edge Module Catalog
|
||||
|
||||
<details>
|
||||
<summary><b>🧩 105 edge modules ready to install on a Cognitum appliance</b> — live catalog from <code>app-registry.json</code> v2.1.0 (updated 2026-05-13). Browse + install at <a href="https://seed.cognitum.one/store">seed.cognitum.one/store</a> or your local appliance <code>http://<appliance>:9000/cogs</code>.</summary>
|
||||
|
||||
Each module is a small signed binary (~400 KB) that runs alongside the WiFi-DensePose sensing stack on a Cognitum-V0 appliance. The catalog updates over the air — your appliance fetches it via <code>GET /api/v1/edge/registry</code> ([ADR-102](docs/adr/ADR-102-edge-module-registry.md)) and verifies each binary against an Ed25519 signature ([ADR-100](docs/adr/ADR-100-cog-packaging-specification.md)) before install.
|
||||
|
||||
### 🫀 Health — <sub>14 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `air-quality-index` | Track indoor air quality with CO2 and particle sensors | 8 KB | Easy |
|
||||
| `baby-cry` | Sustained mid-band energy detector for nursery / infant monitoring. Audio-only, no camera. | 451 KB | Easy |
|
||||
| `breathing-sync` | Detects when two people breathe in sync | 10 KB | Hard |
|
||||
| `cardiac-arrhythmia` | Spots irregular heartbeats and abnormal heart rhythms | 8 KB | Hard |
|
||||
| `cough-detect` | Acoustic transient + spectral cough detector with 30s cluster aggregation. Early-warning signal for respiratory illness. | 451 KB | Easy |
|
||||
| `dream-stage` | Tracks your sleep stages — light, deep, and dreaming | 14 KB | Hard |
|
||||
| `fall-detect` | Two-stage impact + stillness fall detector over ambient feature stream (ESP32 motion / mic). Optional ruview-mode for CSI-based pose reinforcement. | 402 KB | Easy |
|
||||
| `gait-analysis` | Detects walking problems and scores fall risk | 12 KB | Hard |
|
||||
| `health-monitor` | Contactless heart rate, breathing, sleep, and fall alerts | 30 KB | Med |
|
||||
| `respiratory-distress` | Alerts when breathing becomes labored or dangerously fast | 10 KB | Hard |
|
||||
| `seizure-detect` | Recognizes seizures and sends immediate alerts | 10 KB | Hard |
|
||||
| `sleep-apnea` | Detects when someone stops breathing during sleep | 4 KB | Easy |
|
||||
| `snore-monitor` | Periodic low-band energy tracker for sleep-quality / apnea-risk trending. Companion to sleep-apnea cog. | 451 KB | Easy |
|
||||
| `vital-trend` | Tracks breathing and heart rate trends over weeks | 6 KB | Med |
|
||||
|
||||
### 🔒 Security — <sub>14 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `audit-logger` | Record every action for compliance — tamper-proof log | 8 KB | Easy |
|
||||
| `behavioral-profiler` | Learns normal behavior and flags anything unusual | 12 KB | Hard |
|
||||
| `fleet-auth` | Manage device certificates and access across all seeds | 12 KB | Med |
|
||||
| `glass-break` | Two-phase bang + shatter acoustic detector. Distinguishes glass break from ordinary impulse noise. | 451 KB | Easy |
|
||||
| `gunshot-detect` | Saturating peak + exponential decay acoustic detector with optional ruview CSI motion-drop reinforcement. | 451 KB | Easy |
|
||||
| `intrusion` | Alerts when an unauthorized person enters a room | 6 KB | Med |
|
||||
| `intrusion-detect-ml` | Detect network attacks using machine learning | 14 KB | Hard |
|
||||
| `loitering` | Alerts when someone lingers too long in one spot | 3 KB | Easy |
|
||||
| `network-firewall` | Block unauthorized network access per cog | 6 KB | Easy |
|
||||
| `panic-motion` | Detects sudden panicked or erratic movement | 6 KB | Med |
|
||||
| `perimeter-breach` | Guards multiple zones and shows entry direction | 10 KB | Med |
|
||||
| `prompt-shield` | Blocks signal replay and injection attacks on the seed | 10 KB | Med |
|
||||
| `tailgating` | Catches when someone sneaks in behind a badge holder | 6 KB | Med |
|
||||
| `weapon-detect` | Detects concealed metal objects on a person | 8 KB | Hard |
|
||||
|
||||
### 🏢 Building — <sub>11 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `beehive-monitor` | Acoustic hive state classifier. Detects healthy / chaotic / queenless / swarming / robbing via hum-band energy + chaos + piping autocorr. | 451 KB | Easy |
|
||||
| `elevator-count` | Counts how many people are in an elevator | 8 KB | Med |
|
||||
| `energy-audit` | Learns your schedule and cuts wasted energy | 6 KB | Med |
|
||||
| `frost-warning` | Predicts frost 6 hours ahead via temperature trend + dewpoint-depression gate. Field/orchard agriculture. | 451 KB | Easy |
|
||||
| `hvac-presence` | Turns heating and cooling on when you arrive | 3 KB | Easy |
|
||||
| `lighting-zones` | Turns lights on and off as people move between rooms | 4 KB | Easy |
|
||||
| `meeting-room` | Shows if a meeting room is free or occupied | 5 KB | Easy |
|
||||
| `occupancy-zones` | Counts people in each room through walls | 8 KB | Med |
|
||||
| `predictive-maintenance` | Vibration harmonic analyzer for rotating equipment. Tracks F1 / 2×F1 / high-order / sideband energy to score degradation severity. | 451 KB | Easy |
|
||||
| `smoke-fire` | Multi-signal smoke and fire detector. Fuses acoustic crackle, thermal drift proxy, and optional ruview CSI plume signature. Not a UL-listed replacement for code-required smoke alarms. | 451 KB | Easy |
|
||||
| `water-leak` | Persistent low-amplitude hiss + periodic drip acoustic detector with multi-minute persistence gate. Two-stage likely → confirmed. | 451 KB | Easy |
|
||||
|
||||
### 🛍️ Retail — <sub>7 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `customer-flow` | Counts foot traffic in and out of each entrance | 8 KB | Med |
|
||||
| `dwell-heatmap` | Shows where customers spend the most time | 6 KB | Med |
|
||||
| `package-detect` | Sustained CSI-shift detector for porch / loading bay package arrivals and departures. Requires ESP32 CSI ruview input. | 451 KB | Easy |
|
||||
| `parking-occupancy` | Per-zone parking occupancy via ESP32 CSI subcarrier-amplitude shift. Tracks utilization and churn-per-hour. Requires ruview. | 451 KB | Easy |
|
||||
| `queue-length` | Estimates line length and wait time | 6 KB | Med |
|
||||
| `shelf-engagement` | Detects when customers interact with products | 6 KB | Med |
|
||||
| `table-turnover` | Tracks which restaurant tables are free or occupied | 4 KB | Easy |
|
||||
|
||||
### 🏭 Industrial — <sub>7 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `clean-room` | Enforces max headcount in controlled environments | 4 KB | Easy |
|
||||
| `confined-space` | Monitors workers in tight spaces for safety | 5 KB | Med |
|
||||
| `forklift-proximity` | Warns if a forklift gets too close to workers | 10 KB | Hard |
|
||||
| `livestock-monitor` | Monitors animals for distress, escape, or illness | 6 KB | Med |
|
||||
| `ppe-compliance` | Cog-composition layer: alerts when ruview-densepose detects presence in a restricted zone without an accompanying PPE-camera-cog confirmation vector. | 387 KB | Easy |
|
||||
| `slip-fall-zone` | Pre-fall risk detector. Fires when motion-variance drop, splash audio, and optional cautious-gait CSI all signal elevated slip risk. | 451 KB | Easy |
|
||||
| `structural-vibration` | Detects dangerous vibrations in buildings or machines | 8 KB | Hard |
|
||||
|
||||
### 🔬 Research — <sub>12 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `emotion-detect` | Reads stress and calm from body language and breathing | 10 KB | Hard |
|
||||
| `energy-harvester` | Optimize solar and battery for off-grid seed deployment | 6 KB | Med |
|
||||
| `gesture-language` | Recognizes sign language gestures in real time | 12 KB | Hard |
|
||||
| `ghost-hunter` | Finds unexplained environmental anomalies — for fun | 10 KB | Hard |
|
||||
| `happiness-score` | Estimates well-being from movement and mood signals | 8 KB | Med |
|
||||
| `hyperbolic-space` | Maps data into curved space for tree-like structures | 12 KB | Hard |
|
||||
| `music-conductor` | Reads a conductor's gestures for tempo and dynamics | 12 KB | Hard |
|
||||
| `plant-growth` | Tracks plant growth rate and day/night cycles | 8 KB | Med |
|
||||
| `rain-detect` | Detects when rain starts, stops, and how heavy it is | 6 KB | Med |
|
||||
| `ruview-densepose` | Full body pose tracking from WiFi — no cameras needed | 50 KB | Hard |
|
||||
| `sound-classifier` | Identify sounds like glass break, alarm, or baby cry | 16 KB | Hard |
|
||||
| `time-crystal` | Experiments with repeating time-pattern symmetry | 12 KB | Hard |
|
||||
|
||||
### 🤖 Ai — <sub>15 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `anomaly-attractor` | Learns what's normal and catches anything weird | 10 KB | Hard |
|
||||
| `cognitive-pipeline` | FastGRNN anomaly gate + SmolLM2 sparse-LLM inference for on-device Pi Zero 2W cognitive events | 320 KB | Hard |
|
||||
| `dtw-gesture-learn` | Teach custom hand gestures by showing examples | 14 KB | Med |
|
||||
| `ewc-lifelong` | Learns new things without forgetting old lessons | 8 KB | Hard |
|
||||
| `federated-learning` | Train AI across seeds without sharing raw data | 18 KB | Hard |
|
||||
| `goap-autonomy` | Plans and executes goals on its own | 14 KB | Hard |
|
||||
| `meta-adapt` | Automatically tunes itself for best performance | 10 KB | Hard |
|
||||
| `micro-hnsw` | Fast on-device fingerprinting and classification | 12 KB | Med |
|
||||
| `neural-trader` | Spot market patterns and trends from live data | 20 KB | Hard |
|
||||
| `pagerank-influence` | Finds the most influential person in a group | 12 KB | Med |
|
||||
| `pattern-sequence` | Detects daily routines and repeated habits | 10 KB | Med |
|
||||
| `rag-local` | Search your documents using AI — runs on the seed | 14 KB | Med |
|
||||
| `spiking-tracker` | Brain-inspired tracker that runs on tiny hardware | 16 KB | Hard |
|
||||
| `temporal-logic` | Enforces safety rules on live event streams | 12 KB | Hard |
|
||||
| `time-series-forecast` | Predict sensor trends using historical patterns | 12 KB | Med |
|
||||
|
||||
### 🐝 Swarm — <sub>11 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `swarm-backup-restore` | Auto-backup data to other seeds — one-click restore | 8 KB | Easy |
|
||||
| `swarm-cluster-monitor` | Live dashboard of every seed's health and status | 6 KB | Easy |
|
||||
| `swarm-consensus` | Seeds vote before making critical changes together | 16 KB | Hard |
|
||||
| `swarm-delta-sync` | Auto-sync data between seeds — only sends changes | 8 KB | Med |
|
||||
| `swarm-deploy` | Install or remove cogs on all seeds at once | 10 KB | Med |
|
||||
| `swarm-distributed-store` | Spread data across seeds and search them all at once | 14 KB | Hard |
|
||||
| `swarm-edge-orchestrator` | Manage all ESP32 sensor nodes from one place | 14 KB | Hard |
|
||||
| `swarm-load-balancer` | Spread queries across seeds so no single one overloads | 10 KB | Med |
|
||||
| `swarm-mesh-manager` | Find, connect, and monitor all seeds on your network | 12 KB | Easy |
|
||||
| `swarm-mqtt-bridge` | Share events between seeds over MQTT messaging | 6 KB | Easy |
|
||||
| `swarm-witness-federation` | Share tamper-proof audit trails across seeds | 12 KB | Hard |
|
||||
|
||||
### 📡 Signal — <sub>6 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `coherence-gate` | Filters out noisy signals and keeps clean ones | 8 KB | Med |
|
||||
| `flash-attention` | Focuses sensing on specific areas for better accuracy | 12 KB | Med |
|
||||
| `optimal-transport` | Measures motion using shape-aware signal comparison | 12 KB | Hard |
|
||||
| `person-matching` | Tells apart multiple people in the same room | 18 KB | Hard |
|
||||
| `sparse-recovery` | Recovers missing signal data from partial readings | 16 KB | Hard |
|
||||
| `temporal-compress` | Shrinks old data to save memory without losing meaning | 14 KB | Med |
|
||||
|
||||
### 🌐 Network — <sub>1 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `tailscale` | Reach the seed from anywhere via a private WireGuard mesh (Tailscale). Userspace mode — no root. | 700 KB | Med |
|
||||
|
||||
### 🛠️ Developer — <sub>7 modules</sub>
|
||||
|
||||
| ID | What it does | Size | Difficulty |
|
||||
|----|--------------|-----:|:----------:|
|
||||
| `adversarial` | Detects tampered or spoofed sensor signals | 4 KB | Easy |
|
||||
| `coherence` | Monitors signal quality across multiple channels | 4 KB | Easy |
|
||||
| `gesture` | Core gesture recognition building block for cogs | 6 KB | Med |
|
||||
| `interference-search` | Searches many possibilities at once for fast answers | 14 KB | Hard |
|
||||
| `psycho-symbolic` | Reasons over knowledge graphs with multiple styles | 16 KB | Hard |
|
||||
| `quantum-coherence` | Quantum-inspired model for advanced signal states | 16 KB | Hard |
|
||||
| `self-healing-mesh` | Keeps sensor mesh running even when nodes drop out | 14 KB | Hard |
|
||||
|
||||
> ℹ️ Build your own cog: see [ADR-100](docs/adr/ADR-100-cog-packaging-specification.md) for the packaging spec. The first cog this repo ships into the catalog lives in [v2/crates/cog-pose-estimation/](v2/crates/cog-pose-estimation/) (17-keypoint WiFi pose, [ADR-101](docs/adr/ADR-101-pose-estimation-cog.md)).
|
||||
|
||||
</details>
|
||||
|
||||
|
||||
## 🔬 How It Works
|
||||
@@ -228,178 +444,6 @@ These scenarios exploit WiFi's ability to penetrate solid materials — concrete
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>🧩 Edge Intelligence (<a href="docs/adr/ADR-041-wasm-module-collection.md">ADR-041</a>)</strong> — 60 WASM modules across 13 categories, all implemented (609 tests)</summary>
|
||||
|
||||
Small programs that run directly on the ESP32 sensor — no internet needed, no cloud fees, instant response. Each module is a tiny WASM file (5-30 KB) that you upload to the device over-the-air. It reads WiFi signal data and makes decisions locally in under 10 ms. [ADR-041](docs/adr/ADR-041-wasm-module-collection.md) defines 60 modules across 13 categories — all 60 are implemented with 609 tests passing.
|
||||
|
||||
| | Category | Examples |
|
||||
|---|----------|---------|
|
||||
| 🏥 | [**Medical & Health**](docs/edge-modules/medical.md) | Sleep apnea detection, cardiac arrhythmia, gait analysis, seizure detection |
|
||||
| 🔐 | [**Security & Safety**](docs/edge-modules/security.md) | Intrusion detection, perimeter breach, loitering, panic motion |
|
||||
| 🏢 | [**Smart Building**](docs/edge-modules/building.md) | Zone occupancy, HVAC control, elevator counting, meeting room tracking |
|
||||
| 🛒 | [**Retail & Hospitality**](docs/edge-modules/retail.md) | Queue length, dwell heatmaps, customer flow, table turnover |
|
||||
| 🏭 | [**Industrial**](docs/edge-modules/industrial.md) | Forklift proximity, confined space monitoring, structural vibration |
|
||||
| 🔮 | [**Exotic & Research**](docs/edge-modules/exotic.md) | Sleep staging, emotion detection, sign language, breathing sync |
|
||||
| 📡 | [**Signal Intelligence**](docs/edge-modules/signal-intelligence.md) | Cleans and sharpens raw WiFi signals — focuses on important regions, filters noise, fills in missing data, and tracks which person is which |
|
||||
| 🧠 | [**Adaptive Learning**](docs/edge-modules/adaptive-learning.md) | The sensor learns new gestures and patterns on its own over time — no cloud needed, remembers what it learned even after updates |
|
||||
| 🗺️ | [**Spatial Reasoning**](docs/edge-modules/spatial-temporal.md) | Figures out where people are in a room, which zones matter most, and tracks movement across areas using graph-based spatial logic |
|
||||
| ⏱️ | [**Temporal Analysis**](docs/edge-modules/spatial-temporal.md) | Learns daily routines, detects when patterns break (someone didn't get up), and verifies safety rules are being followed over time |
|
||||
| 🛡️ | [**AI Security**](docs/edge-modules/ai-security.md) | Detects signal replay attacks, WiFi jamming, injection attempts, and flags abnormal behavior that could indicate tampering |
|
||||
| ⚛️ | [**Quantum-Inspired**](docs/edge-modules/autonomous.md) | Uses quantum-inspired math to map room-wide signal coherence and search for optimal sensor configurations |
|
||||
| 🤖 | [**Autonomous & Exotic**](docs/edge-modules/autonomous.md) | Self-managing sensor mesh — auto-heals dropped nodes, plans its own actions, and explores experimental signal representations |
|
||||
|
||||
All implemented modules are `no_std` Rust, share a [common utility library](v2/crates/wifi-densepose-wasm-edge/src/vendor_common.rs), and talk to the host through a 12-function API. Full documentation: [**Edge Modules Guide**](docs/edge-modules/README.md). See the [complete implemented module list](#edge-module-list) below.
|
||||
|
||||
</details>
|
||||
|
||||
<details id="edge-module-list">
|
||||
<summary><strong>🧩 Edge Intelligence — <a href="docs/edge-modules/README.md">All 65 Modules Implemented</a></strong> (ADR-041 complete)</summary>
|
||||
|
||||
All 60 modules are implemented, tested (609 tests passing), and ready to deploy. They compile to `wasm32-unknown-unknown`, run on ESP32-S3 via WASM3, and share a [common utility library](v2/crates/wifi-densepose-wasm-edge/src/vendor_common.rs). Source: [`crates/wifi-densepose-wasm-edge/src/`](v2/crates/wifi-densepose-wasm-edge/src/)
|
||||
|
||||
**Core modules** (ADR-040 flagship + early implementations):
|
||||
|
||||
| Module | File | What It Does |
|
||||
|--------|------|-------------|
|
||||
| Gesture Classifier | [`gesture.rs`](v2/crates/wifi-densepose-wasm-edge/src/gesture.rs) | DTW template matching for hand gestures |
|
||||
| Coherence Filter | [`coherence.rs`](v2/crates/wifi-densepose-wasm-edge/src/coherence.rs) | Phase coherence gating for signal quality |
|
||||
| Adversarial Detector | [`adversarial.rs`](v2/crates/wifi-densepose-wasm-edge/src/adversarial.rs) | Detects physically impossible signal patterns |
|
||||
| Intrusion Detector | [`intrusion.rs`](v2/crates/wifi-densepose-wasm-edge/src/intrusion.rs) | Human vs non-human motion classification |
|
||||
| Occupancy Counter | [`occupancy.rs`](v2/crates/wifi-densepose-wasm-edge/src/occupancy.rs) | Zone-level person counting |
|
||||
| Vital Trend | [`vital_trend.rs`](v2/crates/wifi-densepose-wasm-edge/src/vital_trend.rs) | Long-term breathing and heart rate trending |
|
||||
| RVF Parser | [`rvf.rs`](v2/crates/wifi-densepose-wasm-edge/src/rvf.rs) | RVF container format parsing |
|
||||
|
||||
**Vendor-integrated modules** (24 modules, ADR-041 Category 7):
|
||||
|
||||
**📡 Signal Intelligence** — Real-time CSI analysis and feature extraction
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Flash Attention | [`sig_flash_attention.rs`](v2/crates/wifi-densepose-wasm-edge/src/sig_flash_attention.rs) | Tiled attention over 8 subcarrier groups — finds spatial focus regions and entropy | S (<5ms) |
|
||||
| Coherence Gate | [`sig_coherence_gate.rs`](v2/crates/wifi-densepose-wasm-edge/src/sig_coherence_gate.rs) | Z-score phasor gating with hysteresis: Accept / PredictOnly / Reject / Recalibrate | L (<2ms) |
|
||||
| Temporal Compress | [`sig_temporal_compress.rs`](v2/crates/wifi-densepose-wasm-edge/src/sig_temporal_compress.rs) | 3-tier adaptive quantization (8-bit hot / 5-bit warm / 3-bit cold) | L (<2ms) |
|
||||
| Sparse Recovery | [`sig_sparse_recovery.rs`](v2/crates/wifi-densepose-wasm-edge/src/sig_sparse_recovery.rs) | ISTA L1 reconstruction for dropped subcarriers | H (<10ms) |
|
||||
| Person Match | [`sig_mincut_person_match.rs`](v2/crates/wifi-densepose-wasm-edge/src/sig_mincut_person_match.rs) | Hungarian-lite bipartite assignment for multi-person tracking | S (<5ms) |
|
||||
| Optimal Transport | [`sig_optimal_transport.rs`](v2/crates/wifi-densepose-wasm-edge/src/sig_optimal_transport.rs) | Sliced Wasserstein-1 distance with 4 projections | L (<2ms) |
|
||||
|
||||
**🧠 Adaptive Learning** — On-device learning without cloud connectivity
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| DTW Gesture Learn | [`lrn_dtw_gesture_learn.rs`](v2/crates/wifi-densepose-wasm-edge/src/lrn_dtw_gesture_learn.rs) | User-teachable gesture recognition — 3-rehearsal protocol, 16 templates | S (<5ms) |
|
||||
| Anomaly Attractor | [`lrn_anomaly_attractor.rs`](v2/crates/wifi-densepose-wasm-edge/src/lrn_anomaly_attractor.rs) | 4D dynamical system attractor classification with Lyapunov exponents | H (<10ms) |
|
||||
| Meta Adapt | [`lrn_meta_adapt.rs`](v2/crates/wifi-densepose-wasm-edge/src/lrn_meta_adapt.rs) | Hill-climbing self-optimization with safety rollback | L (<2ms) |
|
||||
| EWC Lifelong | [`lrn_ewc_lifelong.rs`](v2/crates/wifi-densepose-wasm-edge/src/lrn_ewc_lifelong.rs) | Elastic Weight Consolidation — remembers past tasks while learning new ones | S (<5ms) |
|
||||
|
||||
**🗺️ Spatial Reasoning** — Location, proximity, and influence mapping
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| PageRank Influence | [`spt_pagerank_influence.rs`](v2/crates/wifi-densepose-wasm-edge/src/spt_pagerank_influence.rs) | 4x4 cross-correlation graph with power iteration PageRank | L (<2ms) |
|
||||
| Micro HNSW | [`spt_micro_hnsw.rs`](v2/crates/wifi-densepose-wasm-edge/src/spt_micro_hnsw.rs) | 64-vector navigable small-world graph for nearest-neighbor search | S (<5ms) |
|
||||
| Spiking Tracker | [`spt_spiking_tracker.rs`](v2/crates/wifi-densepose-wasm-edge/src/spt_spiking_tracker.rs) | 32 LIF neurons + 4 output zone neurons with STDP learning | S (<5ms) |
|
||||
|
||||
**⏱️ Temporal Analysis** — Activity patterns, logic verification, autonomous planning
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Pattern Sequence | [`tmp_pattern_sequence.rs`](v2/crates/wifi-densepose-wasm-edge/src/tmp_pattern_sequence.rs) | Activity routine detection and deviation alerts | S (<5ms) |
|
||||
| Temporal Logic Guard | [`tmp_temporal_logic_guard.rs`](v2/crates/wifi-densepose-wasm-edge/src/tmp_temporal_logic_guard.rs) | LTL formula verification on CSI event streams | S (<5ms) |
|
||||
| GOAP Autonomy | [`tmp_goap_autonomy.rs`](v2/crates/wifi-densepose-wasm-edge/src/tmp_goap_autonomy.rs) | Goal-Oriented Action Planning for autonomous module management | S (<5ms) |
|
||||
|
||||
**🛡️ AI Security** — Tamper detection and behavioral anomaly profiling
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Prompt Shield | [`ais_prompt_shield.rs`](v2/crates/wifi-densepose-wasm-edge/src/ais_prompt_shield.rs) | FNV-1a replay detection, injection detection (10x amplitude), jamming (SNR) | L (<2ms) |
|
||||
| Behavioral Profiler | [`ais_behavioral_profiler.rs`](v2/crates/wifi-densepose-wasm-edge/src/ais_behavioral_profiler.rs) | 6D behavioral profile with Mahalanobis anomaly scoring | S (<5ms) |
|
||||
|
||||
**⚛️ Quantum-Inspired** — Quantum computing metaphors applied to CSI analysis
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Quantum Coherence | [`qnt_quantum_coherence.rs`](v2/crates/wifi-densepose-wasm-edge/src/qnt_quantum_coherence.rs) | Bloch sphere mapping, Von Neumann entropy, decoherence detection | S (<5ms) |
|
||||
| Interference Search | [`qnt_interference_search.rs`](v2/crates/wifi-densepose-wasm-edge/src/qnt_interference_search.rs) | 16 room-state hypotheses with Grover-inspired oracle + diffusion | S (<5ms) |
|
||||
|
||||
**🤖 Autonomous Systems** — Self-governing and self-healing behaviors
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Psycho-Symbolic | [`aut_psycho_symbolic.rs`](v2/crates/wifi-densepose-wasm-edge/src/aut_psycho_symbolic.rs) | 16-rule forward-chaining knowledge base with contradiction detection | S (<5ms) |
|
||||
| Self-Healing Mesh | [`aut_self_healing_mesh.rs`](v2/crates/wifi-densepose-wasm-edge/src/aut_self_healing_mesh.rs) | 8-node mesh with health tracking, degradation/recovery, coverage healing | S (<5ms) |
|
||||
|
||||
**🔮 Exotic (Vendor)** — Novel mathematical models for CSI interpretation
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Time Crystal | [`exo_time_crystal.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_time_crystal.rs) | Autocorrelation subharmonic detection in 256-frame history | S (<5ms) |
|
||||
| Hyperbolic Space | [`exo_hyperbolic_space.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_hyperbolic_space.rs) | Poincare ball embedding with 32 reference locations, hyperbolic distance | S (<5ms) |
|
||||
|
||||
**🏥 Medical & Health** (Category 1) — Contactless health monitoring
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Sleep Apnea | [`med_sleep_apnea.rs`](v2/crates/wifi-densepose-wasm-edge/src/med_sleep_apnea.rs) | Detects breathing pauses during sleep | S (<5ms) |
|
||||
| Cardiac Arrhythmia | [`med_cardiac_arrhythmia.rs`](v2/crates/wifi-densepose-wasm-edge/src/med_cardiac_arrhythmia.rs) | Monitors heart rate for irregular rhythms | S (<5ms) |
|
||||
| Respiratory Distress | [`med_respiratory_distress.rs`](v2/crates/wifi-densepose-wasm-edge/src/med_respiratory_distress.rs) | Alerts on abnormal breathing patterns | S (<5ms) |
|
||||
| Gait Analysis | [`med_gait_analysis.rs`](v2/crates/wifi-densepose-wasm-edge/src/med_gait_analysis.rs) | Tracks walking patterns and detects changes | S (<5ms) |
|
||||
| Seizure Detection | [`med_seizure_detect.rs`](v2/crates/wifi-densepose-wasm-edge/src/med_seizure_detect.rs) | 6-state machine for tonic-clonic seizure recognition | S (<5ms) |
|
||||
|
||||
**🔐 Security & Safety** (Category 2) — Perimeter and threat detection
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Perimeter Breach | [`sec_perimeter_breach.rs`](v2/crates/wifi-densepose-wasm-edge/src/sec_perimeter_breach.rs) | Detects boundary crossings with approach/departure | S (<5ms) |
|
||||
| Weapon Detection | [`sec_weapon_detect.rs`](v2/crates/wifi-densepose-wasm-edge/src/sec_weapon_detect.rs) | Metal anomaly detection via CSI amplitude shifts | S (<5ms) |
|
||||
| Tailgating | [`sec_tailgating.rs`](v2/crates/wifi-densepose-wasm-edge/src/sec_tailgating.rs) | Detects unauthorized follow-through at access points | S (<5ms) |
|
||||
| Loitering | [`sec_loitering.rs`](v2/crates/wifi-densepose-wasm-edge/src/sec_loitering.rs) | Alerts when someone lingers too long in a zone | S (<5ms) |
|
||||
| Panic Motion | [`sec_panic_motion.rs`](v2/crates/wifi-densepose-wasm-edge/src/sec_panic_motion.rs) | Detects fleeing, struggling, or panic movement | S (<5ms) |
|
||||
|
||||
**🏢 Smart Building** (Category 3) — Automation and energy efficiency
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| HVAC Presence | [`bld_hvac_presence.rs`](v2/crates/wifi-densepose-wasm-edge/src/bld_hvac_presence.rs) | Occupancy-driven HVAC control with departure countdown | S (<5ms) |
|
||||
| Lighting Zones | [`bld_lighting_zones.rs`](v2/crates/wifi-densepose-wasm-edge/src/bld_lighting_zones.rs) | Auto-dim/off lighting based on zone activity | S (<5ms) |
|
||||
| Elevator Count | [`bld_elevator_count.rs`](v2/crates/wifi-densepose-wasm-edge/src/bld_elevator_count.rs) | Counts people entering/leaving with overload warning | S (<5ms) |
|
||||
| Meeting Room | [`bld_meeting_room.rs`](v2/crates/wifi-densepose-wasm-edge/src/bld_meeting_room.rs) | Tracks meeting lifecycle: start, headcount, end, availability | S (<5ms) |
|
||||
| Energy Audit | [`bld_energy_audit.rs`](v2/crates/wifi-densepose-wasm-edge/src/bld_energy_audit.rs) | Tracks after-hours usage and room utilization rates | S (<5ms) |
|
||||
|
||||
**🛒 Retail & Hospitality** (Category 4) — Customer insights without cameras
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Queue Length | [`ret_queue_length.rs`](v2/crates/wifi-densepose-wasm-edge/src/ret_queue_length.rs) | Estimates queue size and wait times | S (<5ms) |
|
||||
| Dwell Heatmap | [`ret_dwell_heatmap.rs`](v2/crates/wifi-densepose-wasm-edge/src/ret_dwell_heatmap.rs) | Shows where people spend time (hot/cold zones) | S (<5ms) |
|
||||
| Customer Flow | [`ret_customer_flow.rs`](v2/crates/wifi-densepose-wasm-edge/src/ret_customer_flow.rs) | Counts ins/outs and tracks net occupancy | S (<5ms) |
|
||||
| Table Turnover | [`ret_table_turnover.rs`](v2/crates/wifi-densepose-wasm-edge/src/ret_table_turnover.rs) | Restaurant table lifecycle: seated, dining, vacated | S (<5ms) |
|
||||
| Shelf Engagement | [`ret_shelf_engagement.rs`](v2/crates/wifi-densepose-wasm-edge/src/ret_shelf_engagement.rs) | Detects browsing, considering, and reaching for products | S (<5ms) |
|
||||
|
||||
**🏭 Industrial & Specialized** (Category 5) — Safety and compliance
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Forklift Proximity | [`ind_forklift_proximity.rs`](v2/crates/wifi-densepose-wasm-edge/src/ind_forklift_proximity.rs) | Warns when people get too close to vehicles | S (<5ms) |
|
||||
| Confined Space | [`ind_confined_space.rs`](v2/crates/wifi-densepose-wasm-edge/src/ind_confined_space.rs) | OSHA-compliant worker monitoring with extraction alerts | S (<5ms) |
|
||||
| Clean Room | [`ind_clean_room.rs`](v2/crates/wifi-densepose-wasm-edge/src/ind_clean_room.rs) | Occupancy limits and turbulent motion detection | S (<5ms) |
|
||||
| Livestock Monitor | [`ind_livestock_monitor.rs`](v2/crates/wifi-densepose-wasm-edge/src/ind_livestock_monitor.rs) | Animal presence, stillness, and escape alerts | S (<5ms) |
|
||||
| Structural Vibration | [`ind_structural_vibration.rs`](v2/crates/wifi-densepose-wasm-edge/src/ind_structural_vibration.rs) | Seismic events, mechanical resonance, structural drift | S (<5ms) |
|
||||
|
||||
**🔮 Exotic & Research** (Category 6) — Experimental sensing applications
|
||||
|
||||
| Module | File | What It Does | Budget |
|
||||
|--------|------|-------------|--------|
|
||||
| Dream Stage | [`exo_dream_stage.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_dream_stage.rs) | Contactless sleep stage classification (wake/light/deep/REM) | S (<5ms) |
|
||||
| Emotion Detection | [`exo_emotion_detect.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_emotion_detect.rs) | Arousal, stress, and calm detection from micro-movements | S (<5ms) |
|
||||
| Gesture Language | [`exo_gesture_language.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_gesture_language.rs) | Sign language letter recognition via WiFi | S (<5ms) |
|
||||
| Music Conductor | [`exo_music_conductor.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_music_conductor.rs) | Tempo and dynamic tracking from conducting gestures | S (<5ms) |
|
||||
| Plant Growth | [`exo_plant_growth.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_plant_growth.rs) | Monitors plant growth, circadian rhythms, wilt detection | S (<5ms) |
|
||||
| Ghost Hunter | [`exo_ghost_hunter.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_ghost_hunter.rs) | Environmental anomaly classification (draft/insect/wind/unknown) | S (<5ms) |
|
||||
| Rain Detection | [`exo_rain_detect.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_rain_detect.rs) | Detects rain onset, intensity, and cessation via signal scatter | S (<5ms) |
|
||||
| Breathing Sync | [`exo_breathing_sync.rs`](v2/crates/wifi-densepose-wasm-edge/src/exo_breathing_sync.rs) | Detects synchronized breathing between multiple people | S (<5ms) |
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1 +1 @@
|
||||
8c0680d7d285739ea9597715e84959d9c356c87ee3ad35b5f1e69a4ca41151c6
|
||||
667eb054c44ac510342665bf9c93d608868a8ead948ae8774b2796ebce6f8fe7
|
||||
@@ -164,18 +164,44 @@ def frame_to_csi_data(frame, signal_meta):
|
||||
)
|
||||
|
||||
|
||||
# Quantization precision for cross-platform hash stability (issue #560).
|
||||
#
|
||||
# The bytes packed below feed SHA-256. Without quantization, the hash diverges
|
||||
# across SIMD backends (Intel AVX2/AVX-512 vs ARM NEON vs different x86 micro-
|
||||
# architectures in the same CI pool) because scipy.fft's pocketfft kernels
|
||||
# reorder vectorized FP operations differently per build. IEEE 754 guarantees
|
||||
# per-operation determinism, not associativity under reordering.
|
||||
#
|
||||
# Empirically: 9 decimals was NOT enough to collapse the divergence — two
|
||||
# back-to-back Ubuntu 24.04 / Python 3.11 / scipy 1.17 CI runs landed on
|
||||
# different Azure VM microarchitectures (likely Skylake vs Cascade Lake)
|
||||
# and produced two different SHA-256s even after np.round(.., 9). The DSP
|
||||
# pipeline (preprocess → biquad bandpass → FFT → PSD → variance accumulation)
|
||||
# amplifies the ~1e-14 raw FFT divergence by several orders of magnitude
|
||||
# downstream — the actual drift at features_to_bytes() input can reach 1e-7
|
||||
# or worse.
|
||||
#
|
||||
# 6 decimals (parts per million) gives ~6 orders of magnitude headroom over
|
||||
# observed pipeline-amplified ULP drift and is still far below any meaningful
|
||||
# signal change (CSI phase precision is ~1e-3 rad; PSD bins differ by orders
|
||||
# of magnitude). Round to this precision, then hash.
|
||||
HASH_QUANTIZATION_DECIMALS = 6
|
||||
|
||||
|
||||
def features_to_bytes(features):
|
||||
"""Convert CSIFeatures to a deterministic byte representation.
|
||||
|
||||
We serialize each numpy array to bytes in a canonical order
|
||||
using little-endian float64 representation. This ensures the
|
||||
hash is platform-independent for IEEE 754 compliant systems.
|
||||
Each feature array is quantized to ``HASH_QUANTIZATION_DECIMALS`` decimal
|
||||
places before being packed as little-endian float64. The quantization is
|
||||
what makes the resulting SHA-256 hash actually platform-independent — the
|
||||
raw float values diverge at ULP precision across scipy.fft SIMD backends
|
||||
(issue #560), even though all platforms compute the "correct" answer.
|
||||
|
||||
Args:
|
||||
features: CSIFeatures instance.
|
||||
|
||||
Returns:
|
||||
bytes: Canonical byte representation.
|
||||
bytes: Canonical, quantized byte representation.
|
||||
"""
|
||||
parts = []
|
||||
|
||||
@@ -189,6 +215,10 @@ def features_to_bytes(features):
|
||||
features.power_spectral_density,
|
||||
]:
|
||||
flat = np.asarray(array, dtype=np.float64).ravel()
|
||||
# Quantize before packing so SIMD-level FP reordering across
|
||||
# Intel AVX vs Apple Silicon NEON pocketfft kernels does not
|
||||
# leak into the SHA-256 input.
|
||||
flat = np.round(flat, HASH_QUANTIZATION_DECIMALS)
|
||||
# Pack as little-endian double (8 bytes each)
|
||||
parts.append(struct.pack(f"<{len(flat)}d", *flat))
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ from datetime import datetime, timedelta
|
||||
|
||||
from fastapi import Request, Response, HTTPException, status
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from jose import JWTError, jwt
|
||||
from passlib.context import CryptContext
|
||||
|
||||
@@ -155,16 +156,17 @@ class UserManager:
|
||||
return False
|
||||
|
||||
|
||||
class AuthenticationMiddleware:
|
||||
class AuthenticationMiddleware(BaseHTTPMiddleware):
|
||||
"""Authentication middleware for FastAPI."""
|
||||
|
||||
def __init__(self, settings: Settings):
|
||||
|
||||
def __init__(self, app, settings: Settings):
|
||||
super().__init__(app)
|
||||
self.settings = settings
|
||||
self.token_manager = TokenManager(settings)
|
||||
self.user_manager = UserManager()
|
||||
self.enabled = settings.enable_authentication
|
||||
|
||||
async def __call__(self, request: Request, call_next: Callable) -> Response:
|
||||
|
||||
async def dispatch(self, request: Request, call_next: Callable) -> Response:
|
||||
"""Process request through authentication middleware."""
|
||||
start_time = time.time()
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ from collections import defaultdict, deque
|
||||
from dataclasses import dataclass
|
||||
|
||||
from fastapi import Request, Response, HTTPException, status
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.types import ASGIApp
|
||||
|
||||
from src.config.settings import Settings
|
||||
@@ -299,15 +300,16 @@ class RateLimiter:
|
||||
}
|
||||
|
||||
|
||||
class RateLimitMiddleware:
|
||||
class RateLimitMiddleware(BaseHTTPMiddleware):
|
||||
"""Rate limiting middleware for FastAPI."""
|
||||
|
||||
def __init__(self, settings: Settings):
|
||||
|
||||
def __init__(self, app, settings: Settings):
|
||||
super().__init__(app)
|
||||
self.settings = settings
|
||||
self.rate_limiter = RateLimiter(settings)
|
||||
self.enabled = settings.enable_rate_limiting
|
||||
|
||||
async def __call__(self, request: Request, call_next: Callable) -> Response:
|
||||
|
||||
async def dispatch(self, request: Request, call_next: Callable) -> Response:
|
||||
"""Process request through rate limiting middleware."""
|
||||
if not self.enabled:
|
||||
return await call_next(request)
|
||||
|
||||
@@ -220,7 +220,11 @@ class PoseService:
|
||||
# Apply phase sanitization if we have phase data
|
||||
if hasattr(detection_result.features, 'phase_difference'):
|
||||
phase_data = detection_result.features.phase_difference
|
||||
sanitized_phase = self.phase_sanitizer.sanitize(phase_data)
|
||||
# PhaseSanitizer's full-pipeline method is sanitize_phase,
|
||||
# not sanitize (issue #612). The shorter name was an
|
||||
# AttributeError waiting to fire on any code path that
|
||||
# reaches this branch.
|
||||
sanitized_phase = self.phase_sanitizer.sanitize_phase(phase_data)
|
||||
# Combine amplitude and phase data
|
||||
return np.concatenate([amplitude_data, sanitized_phase])
|
||||
|
||||
|
||||
|
After Width: | Height: | Size: 1.2 MiB |
@@ -0,0 +1,3 @@
|
||||
{"type": "metadata", "name": "ruview-clone-traffic-history", "version": "1.0.0", "schema": "ruvector.rvf.jsonl/v1", "format": "github-traffic-snapshots", "repo": "ruvnet/RuView", "source": "GitHub Traffic API /repos/{repo}/traffic/{clones,views}", "policy": "GitHub retains only 14 days server-side; this file is the long-term record.", "segments": ["metadata", "clone_snapshot", "view_snapshot"], "created_at": "2026-05-19T23:16:22Z", "custom": {"cadence": "twice monthly (1st and 15th, ~14-day intervals)", "idempotency_key": "timestamp (per-day records de-duplicate across overlapping snapshot windows)"}}
|
||||
{"type": "clone_snapshot", "fetched_at": "2026-05-19T23:16:22Z", "window_count": 27887, "window_uniques": 6611, "per_day": [{"timestamp": "2026-05-05T00:00:00Z", "count": 620, "uniques": 218}, {"timestamp": "2026-05-06T00:00:00Z", "count": 477, "uniques": 232}, {"timestamp": "2026-05-07T00:00:00Z", "count": 685, "uniques": 268}, {"timestamp": "2026-05-08T00:00:00Z", "count": 703, "uniques": 276}, {"timestamp": "2026-05-09T00:00:00Z", "count": 352, "uniques": 184}, {"timestamp": "2026-05-10T00:00:00Z", "count": 205, "uniques": 151}, {"timestamp": "2026-05-11T00:00:00Z", "count": 1160, "uniques": 234}, {"timestamp": "2026-05-12T00:00:00Z", "count": 599, "uniques": 207}, {"timestamp": "2026-05-13T00:00:00Z", "count": 5141, "uniques": 1152}, {"timestamp": "2026-05-14T00:00:00Z", "count": 3420, "uniques": 972}, {"timestamp": "2026-05-15T00:00:00Z", "count": 1974, "uniques": 764}, {"timestamp": "2026-05-16T00:00:00Z", "count": 2917, "uniques": 617}, {"timestamp": "2026-05-17T00:00:00Z", "count": 6690, "uniques": 1169}, {"timestamp": "2026-05-18T00:00:00Z", "count": 2944, "uniques": 625}]}
|
||||
{"type": "view_snapshot", "fetched_at": "2026-05-19T23:16:22Z", "window_count": 162314, "window_uniques": 75464, "per_day": [{"timestamp": "2026-05-05T00:00:00Z", "count": 5540, "uniques": 2690}, {"timestamp": "2026-05-06T00:00:00Z", "count": 5111, "uniques": 2393}, {"timestamp": "2026-05-07T00:00:00Z", "count": 5585, "uniques": 2708}, {"timestamp": "2026-05-08T00:00:00Z", "count": 7004, "uniques": 3261}, {"timestamp": "2026-05-09T00:00:00Z", "count": 5395, "uniques": 2531}, {"timestamp": "2026-05-10T00:00:00Z", "count": 4761, "uniques": 2219}, {"timestamp": "2026-05-11T00:00:00Z", "count": 4275, "uniques": 2044}, {"timestamp": "2026-05-12T00:00:00Z", "count": 3466, "uniques": 1688}, {"timestamp": "2026-05-13T00:00:00Z", "count": 13561, "uniques": 8473}, {"timestamp": "2026-05-14T00:00:00Z", "count": 21867, "uniques": 12527}, {"timestamp": "2026-05-15T00:00:00Z", "count": 26182, "uniques": 14609}, {"timestamp": "2026-05-16T00:00:00Z", "count": 17406, "uniques": 8868}, {"timestamp": "2026-05-17T00:00:00Z", "count": 28444, "uniques": 14541}, {"timestamp": "2026-05-18T00:00:00Z", "count": 13717, "uniques": 7819}]}
|
||||
@@ -9,7 +9,18 @@ services:
|
||||
ports:
|
||||
- "3000:3000" # REST API
|
||||
- "3001:3001" # WebSocket
|
||||
- "5005:5005/udp" # ESP32 UDP
|
||||
# ESP32 UDP. On Linux/macOS this works with multiple ESP32 nodes out of
|
||||
# the box. On Docker Desktop for Windows, multi-source UDP is collapsed
|
||||
# to one source IP at the WSL/Hyper-V boundary, so all-but-one node's
|
||||
# frames are silently dropped (issue #374, #386).
|
||||
#
|
||||
# Windows workaround: change this to "5006:5005/udp" and run the host
|
||||
# relay so every datagram arrives from the same loopback source:
|
||||
#
|
||||
# python scripts/udp-relay.py --listen-port 5005 --forward-port 5006
|
||||
#
|
||||
# See docs/TROUBLESHOOTING.md §9 for details.
|
||||
- "5005:5005/udp"
|
||||
environment:
|
||||
- RUST_LOG=info
|
||||
# CSI_SOURCE controls the data source for the sensing server.
|
||||
|
||||
@@ -109,3 +109,75 @@ ssh thyhack@100.90.238.87
|
||||
**Symptom:** Plugging into the right USB-C port (when facing the board with USB-C toward you) shows no serial device on the host.
|
||||
|
||||
**Fix:** Use the left USB-C port. On most ESP32-S3-DevKitC boards, the left port is the USB-to-UART bridge (CP2102/CH340) used for flashing and serial monitor. The right port is the native USB (USB-JTAG) which requires different drivers and isn't used by the RuView firmware.
|
||||
|
||||
---
|
||||
|
||||
## 9. Docker Desktop on Windows drops UDP from multiple ESP32 nodes
|
||||
|
||||
**Symptom:** Two or more ESP32 nodes are flashed, provisioned, and visibly transmit on the network — `tcpdump`/Wireshark on the Windows host shows datagrams from every node — but inside the Docker container only one source IP arrives. `/api/v1/sensing/latest` shows a single node and the live UI freezes or only tracks one body. Reported in #374 (4-node bench) and reproduced in #386 (6-node demo, RuView v0.7.0).
|
||||
|
||||
**Root cause:** Docker Desktop on Windows runs the engine inside a WSL2 / Hyper-V VM. Inbound UDP from the host LAN is forwarded through `vpnkit` / `vEthernet` and the multi-source-IP datagrams are demultiplexed onto a single virtual socket. The first source-IP "wins"; subsequent unique sources are silently dropped at the VM boundary. This is a Docker Desktop limitation, not a sensing-server bug — `host.docker.internal` and `--network host` do not help (host networking is not implemented for the Linux engine on Windows).
|
||||
|
||||
**Fix:** Run the bundled UDP relay on the host so every forwarded datagram arrives from the same loopback source IP, which Docker passes through unchanged.
|
||||
|
||||
```powershell
|
||||
# 1. Start the relay (PowerShell or any terminal)
|
||||
python scripts/udp-relay.py --listen-port 5005 --forward-port 5006
|
||||
|
||||
# 2. Edit docker/docker-compose.yml — change the ESP32 UDP mapping from
|
||||
# - "5005:5005/udp"
|
||||
# to
|
||||
# - "5006:5005/udp"
|
||||
|
||||
# 3. Bring the stack up
|
||||
docker compose -f docker/docker-compose.yml up
|
||||
```
|
||||
|
||||
ESP32 nodes still target the host on `--target-ip <host>:5005` — no firmware re-provisioning is needed. The relay is `scripts/udp-relay.py` (stdlib only, no extra deps). Verify with `--verbose` that each node's source IP appears at least once before forwarding stabilises on a single ephemeral relay port.
|
||||
|
||||
**Prevention:** Linux and macOS hosts are unaffected; the relay only needs to run on Docker Desktop for Windows. If Docker Desktop ships per-source UDP forwarding (tracked at [docker/for-win#1144](https://github.com/docker/for-win/issues/1144) and related), this workaround can be retired.
|
||||
|
||||
**Prior art:** PR #413 (`txhno`) proposed a docs-only writeup of the same workaround; this entry supersedes it.
|
||||
|
||||
---
|
||||
|
||||
## 10. `404` on the visualization page when running sensing-server
|
||||
|
||||
**Symptom:** `sensing-server` starts cleanly, logs `HTTP server listening on http://localhost:3000`, but loading `http://localhost:3000/` (or `/ui/index.html`) returns `404 Not Found`. Reported in #188.
|
||||
|
||||
**Root cause:** The default `--ui-path ../../ui` is resolved relative to the binary's *current working directory*, not the binary location. When the binary is launched from anywhere other than `crates/wifi-densepose-sensing-server/`, the relative path doesn't reach the UI assets and Axum's static file handler returns 404.
|
||||
|
||||
**Fix:** Pass an absolute UI path, run the binary from the crate directory, or use the Docker image (which bundles the UI under `/app/ui`).
|
||||
|
||||
```bash
|
||||
# Option A — absolute path (recommended for production)
|
||||
sensing-server --source esp32 --udp-port 5005 --http-port 3000 \
|
||||
--ws-port 3001 --ui-path /absolute/path/to/ui
|
||||
|
||||
# Option B — run from the crate dir (works for local dev / cargo run)
|
||||
cd v2/crates/wifi-densepose-sensing-server
|
||||
cargo run -- --source esp32
|
||||
|
||||
# Option C — Docker (no path config needed)
|
||||
docker compose -f docker/docker-compose.yml up sensing-server
|
||||
```
|
||||
|
||||
**Prevention:** Track future work in #188 to fall back to a path resolved relative to the executable when the cwd-relative path doesn't exist, so the binary works regardless of where it's launched.
|
||||
|
||||
---
|
||||
|
||||
## 11. Boot loop on `--edge-tier 1` or `--edge-tier 2`
|
||||
|
||||
**Symptom:** ESP32-S3 boots normally with `--edge-tier 0`, but flashing the same firmware with `--edge-tier 1` or `2` produces a boot loop. Serial output reaches `cpu_start` and `heap_init`, then resets repeatedly. Reported in #438 against firmware `v0.4.3.1-esp32-3-g66e2fa083-dir`.
|
||||
|
||||
**Root cause:** Edge tiers 1 and 2 enable the on-device DSP pipeline on Core 1. In the affected build, the `edge_dsp` task ran a tight per-frame loop without yielding, so the FreeRTOS task watchdog tripped on Core 1 and panicked. Tier 0 is passthrough only and doesn't activate the pipeline, so the watchdog never fires there.
|
||||
|
||||
**Fix:** Flash the [v0.4.3.1-esp32](https://github.com/ruvnet/RuView/releases/tag/v0.4.3.1-esp32) release or later — the DSP task yield fixes have shipped on `main` since the build in the report.
|
||||
|
||||
```bash
|
||||
# Verify what version you're on (look for "App version" in serial output on boot)
|
||||
python -m serial.tools.miniterm COM7 115200
|
||||
# Expect: "App version: v0.4.3.1-esp32" or higher
|
||||
```
|
||||
|
||||
If the boot loop persists on a release build, capture a full serial trace including the watchdog backtrace and reopen #438 with the new build hash.
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
# ADR-098: Evaluate `ruvnet/midstream` for RuView's CSI / WebSocket / mesh pipeline
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Rejected (with crate-level carve-outs for future evaluation) |
|
||||
| **Date** | 2026-05-13 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codename** | **midstream-in-RuView** |
|
||||
| **Relates to** | ADR-095 (rvCSI platform), ADR-096 (rvCSI crate topology), ADR-097 (adopt rvCSI as RuView's CSI runtime), ADR-012 (ESP32 CSI mesh), ADR-029 (RuvSense multistatic / TDM), ADR-031 (RuView sensing-first RF mode), ADR-043 (sensing-server UI API completion) |
|
||||
| **midstream repo** | [github.com/ruvnet/midstream](https://github.com/ruvnet/midstream) — vendored at `vendor/midstream`, currently pinned at [`30fe5eb`](https://github.com/ruvnet/midstream/commit/30fe5eb7a1f1494aa1ad00d54160088a565ec766) |
|
||||
| **Outcome** | Do **not** adopt as a system component. Two of midstream's six workspace crates (`temporal-compare`, `nanosecond-scheduler`) are plausible future-use building blocks; the rest do not fit. `vendor/midstream` is retained as a reference-only submodule. |
|
||||
|
||||
---
|
||||
|
||||
## 1. Context
|
||||
|
||||
`vendor/midstream` is a git submodule of RuView (`.gitmodules:1-4`) but, like `vendor/rvcsi` was before ADR-097, it is **vendored but not consumed**: no `v2/crates/*/Cargo.toml` depends on a `midstreamer-*` crate, no Rust source contains `use midstreamer_…`, and the ESP32 firmware and TypeScript dashboard have no midstream imports.
|
||||
|
||||
This ADR settles the standing question of *whether RuView should consume midstream at all*, and if so, where. The user-facing prompt enumerated four candidate seams to evaluate:
|
||||
|
||||
1. Streaming / pub-sub for the WebSocket fan-out (today: `tokio::sync::broadcast::channel::<String>(256)` at `v2/crates/wifi-densepose-sensing-server/src/main.rs:4769`).
|
||||
2. Stream processing for the CSI → DSP → event pipeline (today: synchronous `EventPipeline` at `vendor/rvcsi/crates/rvcsi-events/src/pipeline.rs`, freshly adopted via ADR-097).
|
||||
3. Multi-source merging / TDM coordination for the ESP32 mesh (ADR-029, ADR-073).
|
||||
4. Backpressure / flow control between the UDP receiver and downstream consumers (`v2/crates/wifi-densepose-sensing-server/src/main.rs:3638` `udp_receiver_task`; firmware-side `stream_sender` ENOMEM backoff at `firmware/esp32-csi-node/main/csi_collector.c:223-228`).
|
||||
|
||||
To evaluate each, we read midstream's workspace `Cargo.toml` (`vendor/midstream/Cargo.toml:1-99`), the `README.md` and `BENCHMARKS_SUMMARY.md`, and every crate's `lib.rs`:
|
||||
|
||||
| Crate | File | LOC | Purpose (from header doc) |
|
||||
|---|---|---:|---|
|
||||
| `midstreamer-temporal-compare` | `vendor/midstream/crates/temporal-compare/src/lib.rs:1-697` | 697 | DTW, LCS, Levenshtein, generic pattern matching on `Sequence<T>` of `TemporalElement<T>` |
|
||||
| `midstreamer-scheduler` | `vendor/midstream/crates/nanosecond-scheduler/src/lib.rs:1-406` | 406 | Priority + deadline-aware task scheduler (RM, EDF, LLF) for low-latency real-time tasks |
|
||||
| `midstreamer-attractor` | `vendor/midstream/crates/temporal-attractor-studio/src/lib.rs:1-482` | 482 | Phase-space reconstruction, Lyapunov exponents, attractor classification |
|
||||
| `midstreamer-neural-solver` | `vendor/midstream/crates/temporal-neural-solver/src/lib.rs:1-509` | 509 | LTL / CTL / MTL temporal-logic verification with neural reasoning |
|
||||
| `midstreamer-strange-loop` | `vendor/midstream/crates/strange-loop/src/lib.rs:1-496` | 496 | Multi-level meta-learning, self-referential systems |
|
||||
| `midstreamer-quic` | `vendor/midstream/crates/quic-multistream/src/lib.rs:1-255`, `native.rs:1-303`, `wasm.rs:1-307` | 865 | Thin wrapper over `quinn` (native) and `WebTransport` (WASM); generic QUIC streams |
|
||||
|
||||
Plus a TypeScript layer (`vendor/midstream/npm/`, `vendor/midstream/npm-wasm/`) whose product is "real-time LLM streaming" — OpenAI Realtime API client, RTMP / WebRTC / HLS for video, an in-console dashboard, a Whisper transcription scaffold, an MCP server for LLM agents.
|
||||
|
||||
The top-level identity is unambiguous: `Cargo.toml:16` describes the package as **`"Real-time LLM streaming with inflight analysis"`**, and the README (`vendor/midstream/README.md:45-80`) frames midstream as a platform that "analyzes [LLM] responses **as they stream in real-time** — enabling instant insights, pattern detection, and intelligent decision-making" — i.e. the streaming domain is **LLM tokens and dashboard telemetry**, not RF signals. A search for any of `csi`, `wifi`, `sensing`, or `sensor` across `vendor/midstream/crates/*/src/*.rs` returns zero hits.
|
||||
|
||||
This shapes the conclusion: midstream's *abstractions* (DTW pattern matching, attractor analysis, LTL verification, meta-learning) were chosen for a fundamentally different problem domain than CSI, and its *transport* (QUIC) is a thin `quinn` wrapper rather than a sensing-aware backplane. The candidate seams enumerated above are either already filled by simpler primitives in RuView, or filled better by rvCSI under ADR-097.
|
||||
|
||||
### 1.1 What this ADR is *not*
|
||||
|
||||
- Not a judgment on midstream's quality. It has 139 passing tests and clean Rust; it is well-engineered for its target domain.
|
||||
- Not a decision to drop `vendor/midstream`. The submodule pin is cheap to keep, and the carve-outs in §3 may justify revisiting it.
|
||||
- Not a position on the *standalone* midstream product (LLM streaming, OpenAI Realtime, dashboards). That product is unaffected by this ADR.
|
||||
|
||||
---
|
||||
|
||||
## 2. Decision
|
||||
|
||||
**Reject midstream as a system component of RuView.** The four candidate seams are either filled (well) by existing RuView primitives, or are filled by rvCSI's freshly-adopted `EventPipeline` and `RfMemoryStore`. The eight decisions below are the architectural contract.
|
||||
|
||||
### D1 — Streaming / pub-sub for the WebSocket fan-out: no change
|
||||
|
||||
RuView's sensing-server currently fans out updates to WebSocket clients via `tokio::sync::broadcast::channel::<String>(256)` (`v2/crates/wifi-densepose-sensing-server/src/main.rs:4769`). midstream offers no equivalent in-process broadcast primitive — its TypeScript dashboard fan-out is HTTP-server based (`vendor/midstream/npm/src/dashboard.ts`), and its Rust `midstreamer-quic` crate is a generic point-to-point QUIC wrapper (`vendor/midstream/crates/quic-multistream/src/native.rs:31-69`), not a pub-sub bus.
|
||||
|
||||
Tokio's `broadcast` channel is the standard Rust idiom for this pattern, costs effectively nothing per subscriber, integrates with the rest of the Axum + Tokio stack already in use (`v2/crates/wifi-densepose-sensing-server/src/main.rs:36,47`), and is what `rvcsi-runtime` itself uses for event distribution (`vendor/rvcsi/crates/rvcsi-runtime/src/lib.rs`). **Keep `tokio::sync::broadcast`.**
|
||||
*Consequences:* zero migration; zero new dependency surface; the WebSocket handlers at `main.rs:1989,2030` continue to work unchanged.
|
||||
|
||||
### D2 — CSI → DSP → event pipeline: stay on rvCSI's `EventPipeline`
|
||||
|
||||
ADR-097 D2 just adopted `rvcsi-runtime::CaptureRuntime` + `rvcsi_events::EventPipeline` as the CSI ingestion / DSP / event-extraction path. `EventPipeline` is **deterministic, synchronous, single-frame-at-a-time** (`vendor/rvcsi/crates/rvcsi-events/src/pipeline.rs:1-5`: *"Feed it frames with `EventPipeline::process_frame` and drain the tail with `EventPipeline::flush`"*) — and that determinism is load-bearing for ADR-095 D9 (replayability) and ADR-095 D13 (quality scoring against learned baselines).
|
||||
|
||||
midstream's stream-processing primitives are designed for the opposite shape: `temporal-attractor-studio` (phase-space reconstruction, Lyapunov exponents) and `temporal-neural-solver` (LTL formula verification) operate on **trajectories** of multi-dimensional states over hundreds-to-thousands of samples (`vendor/midstream/README.md:528-531`: *"Attractor detection: <5ms for 1000-point series"*) — that is closer to RuView's existing RuvSense modules (`v2/crates/wifi-densepose-signal/src/ruvsense/longitudinal.rs`, `intention.rs`) than to anything the runtime DSP layer needs.
|
||||
|
||||
Replacing rvCSI's event detectors with midstream constructs would (a) break determinism, (b) re-introduce a parallel CSI-processing implementation — exactly the duplication ADR-097 was opened to remove — and (c) force RuView to invent a `Sequence<T: temporal-compare::TemporalElement>` shim around `CsiFrame` for marginal benefit. **Stay on `rvcsi-events::EventPipeline`.**
|
||||
*Consequences:* the determinism / replay guarantees of ADR-095 D9 and ADR-097 D6 remain intact; the work to land `rvcsi-adapter-esp32` (ADR-097 D4, P3) is not duplicated.
|
||||
|
||||
### D3 — TDM / multi-source merging: stay on the existing aggregator
|
||||
|
||||
The ESP32 mesh's multi-source merging is in `v2/crates/wifi-densepose-hardware/src/aggregator/mod.rs:74-220` — a `UdpSocket`-backed aggregator (`mod.rs:74,85`) that receives parsed `CsiFrame`s from N nodes and forwards them on a `SyncSender<CsiFrame>` to the consumer. The TDM coordination (slot assignment, channel hopping, dwell time) lives in firmware (`firmware/esp32-csi-node/main/`) and is governed by ADR-029 and ADR-073. midstream offers nothing for either side: it has no UDP merger, no slot scheduler, and no firmware-side primitives.
|
||||
|
||||
`midstreamer-scheduler` is conceptually adjacent — it does priority + deadline-aware scheduling (`vendor/midstream/crates/nanosecond-scheduler/src/lib.rs:53-63`: `RateMonotonic`, `EarliestDeadlineFirst`, `LeastLaxityFirst`, `FixedPriority`) — but its target is **in-process tokio tasks on a 4-thread executor** (`vendor/midstream/README.md:466-477`: *"4 worker threads"*, *"<50 ns scheduling latency"*), not the cross-device, wall-clock-anchored TDM that RuvSense needs. **Keep the existing `wifi-densepose-hardware` aggregator and firmware-side TDM.**
|
||||
*Consequences:* ADR-029 stays as-is; the work to migrate the parser to `rvcsi-adapter-esp32` (ADR-097 D4) is unaffected.
|
||||
|
||||
### D4 — UDP receiver backpressure / flow control: existing solutions are correct at each end
|
||||
|
||||
There are two distinct backpressure problems in RuView, and neither benefits from midstream:
|
||||
|
||||
- **Firmware side (`firmware/esp32-csi-node/main/csi_collector.c:64,223-228`):** lwIP pbuf exhaustion produces `ENOMEM` when the ESP32 tries to UDP-send faster than the network drains. The fix in code is a rate-limit on `stream_sender_send` *inside the CSI callback*. This is a C-level firmware concern with no Rust analogue — midstream cannot run on the ESP32.
|
||||
- **Host side (`v2/crates/wifi-densepose-sensing-server/src/main.rs:3638-3640`, `4769`):** `udp_receiver_task` reads from `UdpSocket` and pushes onto `broadcast::channel::<String>(256)`. The bounded channel is itself the backpressure mechanism: lagged subscribers see `RecvError::Lagged`, the buffer wraps, no producer ever blocks. The 256-slot capacity is sized to one second of frame envelopes at the target rate; the per-second packet-yield collapse symptom (`adaptive_controller_decide.c:26-28`) is detected and surfaced by ADR-039 / ADR-081's `pkt_yield_per_sec` accessor, not by transport-layer flow control.
|
||||
|
||||
midstream's `quic-multistream` provides per-stream prioritization (`vendor/midstream/crates/quic-multistream/src/native.rs:1-303`), which is a useful flow-control primitive *for QUIC* but not for the UDP-CSI / WS-fan-out topology RuView actually uses. Adopting QUIC end-to-end would mean (a) replacing the ESP32's UDP sender — which would need a QUIC stack on a memory-constrained Xtensa MCU and is out of scope for this project — or (b) terminating QUIC at the aggregator only, which provides no benefit the current bounded `broadcast` channel doesn't. **Keep the existing two-tier backpressure.**
|
||||
*Consequences:* the ENOMEM rate-limit at `csi_collector.c:223-228` and the bounded `broadcast::channel::<String>(256)` at `main.rs:4769` continue to be the load-bearing primitives.
|
||||
|
||||
### D5 — Carve-out: `temporal-compare` as a future RuvSense-side building block
|
||||
|
||||
`midstreamer-temporal-compare` (`vendor/midstream/crates/temporal-compare/src/lib.rs:1-697`) is a clean DTW / LCS / Levenshtein implementation with an LRU cache. RuView's gesture detector at `v2/crates/wifi-densepose-signal/src/ruvsense/gesture.rs` already does DTW template matching, and the longitudinal analysis at `ruvsense/longitudinal.rs` could plausibly benefit from cached pattern matching. If we ever need a *separate* DTW implementation that is decoupled from RuvSense's internal types, `temporal-compare` is a reasonable starting point — but only if and when that need arises.
|
||||
|
||||
We **do not adopt it today** because RuvSense's gesture matcher already exists, works, and uses RuView-native types, and pulling in `dashmap`, `lru`, and a generic `TemporalElement<T>` abstraction would be net-negative right now. **Tracked as a future evaluation, not a decision.**
|
||||
*Consequences:* zero today; one named option for a future ADR if a "second" DTW pattern appears.
|
||||
|
||||
### D6 — Carve-out: `nanosecond-scheduler` for *host-side* edge tier scheduling (future)
|
||||
|
||||
If ADR-039's edge-intelligence tier scheduling ever moves from the ESP32 onto a host-side coordinator (e.g. a Raspberry Pi running the cluster aggregator), `nanosecond-scheduler`'s deadline-aware policies (`vendor/midstream/crates/nanosecond-scheduler/src/lib.rs:53-63`) could plausibly host that scheduler. Today the scheduling is firmware-side and the C-level RTOS handles it; there is nothing to schedule in Rust at the granularity midstream offers.
|
||||
|
||||
Again: **not a current decision, just an option kept open.**
|
||||
*Consequences:* zero today.
|
||||
|
||||
### D7 — Submodule disposition: keep `vendor/midstream`
|
||||
|
||||
`vendor/midstream` is one git submodule pin; the build does not depend on it; it does not slow down `cargo build --workspace`; and the carve-outs in D5/D6 leave the door open. Removing the submodule would also remove the reference material that justified the carve-outs.
|
||||
|
||||
**Keep the submodule, no per-release pin advancement.** Unlike `vendor/rvcsi` (whose pin is bumped per RuView release under ADR-097 D7), `vendor/midstream` has no in-build consumer to validate against. If D5 or D6 ever activates, *that* ADR will start the per-release pin process. Until then the pin can drift freely.
|
||||
*Consequences:* one line of `.gitmodules` (`.gitmodules:1-4`) stays; `git submodule update --init` remains a no-op for normal RuView development.
|
||||
|
||||
### D8 — Documentation: cross-reference, don't import
|
||||
|
||||
The ADR index (`docs/adr/README.md`) gets ADR-098 added under "Architecture and infrastructure". No other docs are updated. The README on the RuView side is untouched; midstream is not part of the RuView platform story.
|
||||
*Consequences:* one row added to the ADR index; no churn elsewhere.
|
||||
|
||||
---
|
||||
|
||||
## 3. Why not adopt (the rejection record)
|
||||
|
||||
For institutional memory, the table below records what each midstream crate *would* solve and the alternative RuView already uses. This is the answer to "but we vendored midstream — what is it for?"
|
||||
|
||||
| midstream crate | Plausible RuView seam | Already filled by | Verdict |
|
||||
|---|---|---|---|
|
||||
| `midstreamer-temporal-compare` (DTW, LCS, Levenshtein) | Gesture template matching (`ruvsense/gesture.rs`); longitudinal biomechanics drift | RuvSense's existing DTW gesture matcher | Carve-out only (D5) — not adopted today |
|
||||
| `midstreamer-scheduler` (nanosecond priority + deadline) | ESP32 edge-tier scheduling (ADR-039); RuvSense TDM (ADR-029) | Firmware-side RTOS (ESP32); ADR-029's wall-clock-anchored TDM | Carve-out only (D6) — wrong scope today |
|
||||
| `midstreamer-attractor` (Lyapunov, phase-space) | RF-field stability detection in `ruvsense/field_model.rs`, `longitudinal.rs` | Welford stats + biomechanics drift (longitudinal.rs); SVD eigenstructure (field_model.rs) | Not adopted — RuvSense's approach is calibrated to RF signal scale and the project's existing dataset, not generic dynamical-systems theory |
|
||||
| `midstreamer-neural-solver` (LTL / CTL / MTL verification) | Adversarial signal detection (`ruvsense/adversarial.rs`); coherence-gate decisions | Multi-link consistency checks (adversarial.rs); `coherence_gate.rs` state machine | Not adopted — RuView's adversarial detector is not a formal-verification problem; it's a multi-link physical-consistency check |
|
||||
| `midstreamer-strange-loop` (meta-learning, self-modification) | None in RuView's scope | RuView is not a self-modifying learner; AETHER (ADR-024) is contrastive embedding, not meta-learning | Not adopted — out of scope |
|
||||
| `midstreamer-quic` (QUIC native + WASM) | Sensing-server → external client transport (alternative to WS) | `tokio::sync::broadcast` + Axum WebSocket + UDP (`main.rs:36-47, 4769, 1989, 2030, 3638`) | Not adopted — see D1, D4 |
|
||||
|
||||
The shape of the rejection is consistent: **midstream's abstractions are LLM-token / dashboard-telemetry shaped, RuView's pipeline is RF-frame / event-detector shaped.** Where the two share vocabulary ("streaming", "temporal", "real-time"), the implementations diverge sharply — and the case-by-case analysis above shows that the closer one looks at each seam, the worse the fit gets.
|
||||
|
||||
---
|
||||
|
||||
## 4. Consequences
|
||||
|
||||
**Positive**
|
||||
|
||||
- Zero net change to RuView's build, runtime, or surface area; ADR-097's phased rvCSI adoption proceeds unaffected.
|
||||
- The decision space around midstream is now bounded and documented; future contributors and AI agents see "ADR-098 already evaluated this; here is why not" before re-opening the question.
|
||||
- The two crate-level carve-outs (D5, D6) are explicit, so if the relevant seams appear later, the evaluation can pick up from this ADR rather than start over.
|
||||
- `vendor/midstream` (the submodule) remains as reference material, but is correctly marked as not part of the build path.
|
||||
|
||||
**Negative / costs**
|
||||
|
||||
- One more vendored repo with no in-build consumer — a small but non-zero cognitive load (mitigated by D7's explicit "do not bump the pin").
|
||||
- If midstream's published crates evolve materially (e.g. a CSI-aware feature lands), the reasoning in §3 needs revisiting; this is the standard "rejected ADRs go stale" risk and applies to every Rejected ADR in the index.
|
||||
|
||||
**Risks**
|
||||
|
||||
- The most plausible failure mode of this ADR is *not* "we should have adopted midstream"; it is "we re-open the question in six months without re-reading this ADR." Mitigated by indexing ADR-098 in `docs/adr/README.md` and by the per-crate table in §3 being precise enough to short-circuit the next evaluator.
|
||||
|
||||
---
|
||||
|
||||
## 5. Alternatives considered
|
||||
|
||||
| Alternative | Why not |
|
||||
|---|---|
|
||||
| **Adopt midstream wholesale as RuView's streaming backbone** | Would force the CSI pipeline into the `Sequence<TemporalElement>` shape (`vendor/midstream/crates/temporal-compare/src/lib.rs:42-70`) and the `quic-multistream` transport (`vendor/midstream/crates/quic-multistream/src/native.rs:1-303`) — both are designed for LLM tokens / arbitrary streams, not validated RF frames with quality scoring. Conflicts directly with ADR-095 D5 (one `CsiFrame` schema), D6 (validate before crossing boundaries), and D9 (deterministic replay). |
|
||||
| **Replace `tokio::sync::broadcast` with midstream's QUIC fan-out** | Solves no observed problem. `broadcast::channel::<String>(256)` at `v2/crates/wifi-densepose-sensing-server/src/main.rs:4769` handles N WebSocket subscribers at zero per-subscriber cost; the lagged-subscriber semantics (`RecvError::Lagged`) are exactly what an event-feed wants. QUIC adds TLS + congestion control + per-stream priority — useful for *external* clients across a network, but the sensing-server's clients connect over WS on the same host or LAN. |
|
||||
| **Replace `EventPipeline` with `temporal-attractor-studio` / `temporal-neural-solver`** | `EventPipeline` is deterministic by contract (`vendor/rvcsi/crates/rvcsi-events/src/lib.rs:20`) and ADR-097 just made it RuView's event source of truth. Attractor analysis and LTL verification operate on entirely different abstractions; using them as event detectors would re-invent rvCSI's pipeline in a less-determined way. |
|
||||
| **Adopt `midstreamer-temporal-compare` for gesture detection now** | RuvSense already has a working DTW gesture matcher tuned to CSI signal scale. Swapping it for a generic `TemporalElement<T>` matcher buys cleanliness but costs a re-tune and a new dep tree (`dashmap`, `lru`). Tracked as D5 for if/when a *second* DTW use case shows up. |
|
||||
| **Adopt `midstreamer-scheduler` for the cluster-Pi aggregator** | The cluster aggregator does not currently exist as a real-time scheduler; ADR-039's tier scheduling is firmware-side. Until the host-side schedule appears, importing a deadline-aware scheduler is solution-looking-for-a-problem. Tracked as D6. |
|
||||
| **Drop the `vendor/midstream` submodule entirely** | Cheap to keep, useful as the reference material this ADR cites. D7 keeps it on the explicit understanding that the pin is not advanced. |
|
||||
|
||||
---
|
||||
|
||||
## 6. Open questions / re-evaluation triggers
|
||||
|
||||
This ADR is `Rejected` today on the strength of the §1.1 / §3 analysis. The following events would justify re-opening it:
|
||||
|
||||
1. **A second DTW / LCS / Levenshtein use case appears in RuView** (e.g. a CLI-side replay diff, a regression test fixture that needs sequence alignment, a TUI for pattern playback). Then re-evaluate `midstreamer-temporal-compare` per D5.
|
||||
2. **A host-side real-time scheduler enters RuView's scope** (e.g. the cluster-Pi aggregator becomes responsible for slot timing instead of the ESP32 firmware). Then re-evaluate `midstreamer-scheduler` per D6.
|
||||
3. **midstream ships a CSI-aware adapter or RF-scale `Sequence<T>` extension** — i.e. midstream's own scope grows to include sensing primitives. As of the pinned commit (`30fe5eb`), this has not happened (zero matches for `csi|wifi|sensing|sensor` in `vendor/midstream/crates/*/src/*.rs`).
|
||||
4. **RuView gains a QUIC-to-external-client requirement** that the WS fan-out cannot service (e.g. a mobile client over a lossy link that benefits from QUIC's stream priority + 0-RTT). Then re-evaluate `midstreamer-quic` per D1 / D4.
|
||||
|
||||
If none of these triggers fire, this ADR stays Rejected and the carve-outs (D5, D6) remain optional.
|
||||
|
||||
---
|
||||
|
||||
## 7. References
|
||||
|
||||
- [ADR-095 — rvCSI Edge RF Sensing Platform](ADR-095-rvcsi-edge-rf-sensing-platform.md) — sets the single-`CsiFrame` schema, deterministic replay, and quality-scoring constraints that midstream's abstractions conflict with.
|
||||
- [ADR-096 — rvCSI Crate Topology, the napi-c Shim, the napi-rs Surface](ADR-096-rvcsi-ffi-crate-layout.md) — the crate topology that rvCSI fills the candidate seams with.
|
||||
- [ADR-097 — Adopt rvCSI as RuView's primary CSI runtime](ADR-097-adopt-rvcsi-as-ruview-csi-runtime.md) — phased adoption (P1-P5) that this ADR explicitly does not duplicate.
|
||||
- [ADR-012 — ESP32 CSI Sensor Mesh](ADR-012-esp32-csi-sensor-mesh.md) — the multi-source TDM context for D3.
|
||||
- [ADR-029 — RuvSense Multistatic Sensing Mode](ADR-029-ruvsense-multistatic-sensing-mode.md) — the wall-clock-anchored TDM that `midstreamer-scheduler` is the wrong shape for.
|
||||
- [ADR-039 — ESP32 Edge Intelligence Pipeline](ADR-039-esp32-edge-intelligence.md) — the firmware-side tier scheduling that would need to move host-side before D6 activates.
|
||||
- [`github.com/ruvnet/midstream`](https://github.com/ruvnet/midstream) — 5 published crates on crates.io (`temporal-compare`, `nanosecond-scheduler`, `temporal-attractor-studio`, `temporal-neural-solver`, `strange-loop`) + 1 local crate (`quic-multistream`); 139 passing tests.
|
||||
- `vendor/midstream` (submodule) — pinned at `30fe5eb` (`vendor/midstream/Cargo.toml:16` describes the package as *"Real-time LLM streaming with inflight analysis"*).
|
||||
- RuView code paths cited in §1: `v2/crates/wifi-densepose-sensing-server/src/main.rs:36,47,1989,2030,3638-3640,4769`; `v2/crates/wifi-densepose-hardware/src/aggregator/mod.rs:74-220`; `firmware/esp32-csi-node/main/csi_collector.c:64,223-228`; `firmware/esp32-csi-node/main/adaptive_controller_decide.c:26-28`.
|
||||
- RuvSense code paths cited in §3: `v2/crates/wifi-densepose-signal/src/ruvsense/gesture.rs`, `longitudinal.rs`, `field_model.rs`, `adversarial.rs`, `coherence_gate.rs`.
|
||||
- rvCSI code paths cited in §2: `vendor/rvcsi/crates/rvcsi-events/src/lib.rs:1-37`, `vendor/rvcsi/crates/rvcsi-events/src/pipeline.rs:1-5`.
|
||||
@@ -0,0 +1,165 @@
|
||||
# ADR-100: Cognitum Cog Packaging Specification
|
||||
|
||||
- **Status:** Accepted (formalises existing convention) — **first conforming cog shipped 2026-05-19** (`cog-pose-estimation@0.0.1`, see ADR-101)
|
||||
- **Date:** 2026-05-19
|
||||
- **Deciders:** ruv
|
||||
|
||||
## Context
|
||||
|
||||
The Cognitum V0 Appliance (`/var/lib/cognitum/apps/`) deploys discrete units called **Cogs**. They appear in the Appliance dashboard (`http://cognitum-v0:9000/cogs`) under an app-store UI (Today / Apps / Categories / Search / Updates). Until this ADR, the packaging convention has been **implicit** — derived from inspecting installed cogs (`anomaly-detect`, `presence`, `seizure-detect`, etc.) on a live appliance. Bringing new Cogs to the platform required reverse-engineering the layout each time.
|
||||
|
||||
This ADR formalises the layout so:
|
||||
|
||||
1. A repo crate can be built into a Cog with a deterministic Makefile / CI pipeline.
|
||||
2. Cog binaries can be cross-compiled for every supported architecture from a single source.
|
||||
3. The appliance's installer (`cognitum-cog-gateway`) can verify manifests without bespoke per-cog adapters.
|
||||
4. Future Cogs in this repo (starting with `cog-pose-estimation` — see ADR-101) follow a single rule.
|
||||
|
||||
## Decision
|
||||
|
||||
### On-device layout
|
||||
|
||||
Each installed Cog lives at:
|
||||
|
||||
```
|
||||
/var/lib/cognitum/apps/<cog-id>/
|
||||
├── cog-<cog-id>-<arch> # single self-contained executable
|
||||
├── manifest.json # immutable; signed by the publisher
|
||||
├── config.json # mutable; runtime config, owned by the appliance
|
||||
├── pid # current PID when running; absent when stopped
|
||||
├── output.log # stdout (truncated on rotation)
|
||||
└── error.log # stderr (truncated on rotation)
|
||||
```
|
||||
|
||||
`<cog-id>` is kebab-case, ASCII, `[a-z0-9-]{2,32}`. `<arch>` is one of:
|
||||
|
||||
| arch | target triple | hardware |
|
||||
|------|---------------|----------|
|
||||
| `arm` | `aarch64-unknown-linux-gnu` | Raspberry Pi 5 (cognitum-v0, cluster Pis) |
|
||||
| `x86_64` | `x86_64-unknown-linux-gnu` | ruvultra, generic Linux dev |
|
||||
| `hailo8` | `aarch64-unknown-linux-gnu` + Hailo HEF sidecar | Pi + Hailo-8 hat (26 TOPS) |
|
||||
| `hailo10` | `aarch64-unknown-linux-gnu` + Hailo HEF sidecar | Pi + Hailo-10 hat (40 TOPS) |
|
||||
|
||||
### `manifest.json` schema
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "anomaly-detect",
|
||||
"version": "0.1.0",
|
||||
"binary_url": "https://storage.googleapis.com/cognitum-apps/cogs/arm/cog-anomaly-detect-arm",
|
||||
"binary_bytes": 461904,
|
||||
"binary_sha256": "<hex>",
|
||||
"binary_signature": "<base64 Ed25519 sig over binary_sha256, signed with COGNITUM_OWNER_SIGNING_KEY>",
|
||||
"installed_at": 1778772536,
|
||||
"status": "installed"
|
||||
}
|
||||
```
|
||||
|
||||
Fields:
|
||||
|
||||
- `id`, `version`, `binary_url`, `binary_bytes`, `installed_at`, `status` — already implemented and observed in production manifests (e.g. `anomaly-detect@0.0.0`). Documented here without change.
|
||||
- `binary_sha256`, `binary_signature` — **new**, REQUIRED for any Cog shipped from this repo. Backwards-compatible with existing manifests: the appliance gateway treats both fields as optional today, MUST verify them when present. ADR-103 (witness chain) covers the trust model in more detail.
|
||||
- `status` values: `"installed"`, `"running"`, `"stopped"`, `"failed"`, `"updating"`.
|
||||
|
||||
### Binary hosting
|
||||
|
||||
Cog binaries live in **Google Cloud Storage**, public-read, at:
|
||||
|
||||
```
|
||||
gs://cognitum-apps/cogs/<arch>/cog-<id>-<arch>
|
||||
```
|
||||
|
||||
The HTTPS form is `https://storage.googleapis.com/cognitum-apps/cogs/<arch>/cog-<id>-<arch>` (no trailing extension; the URL is the canonical artifact). For Hailo variants, the HEF model file is sibling: `cog-<id>-<arch>.hef`.
|
||||
|
||||
Bucket conventions:
|
||||
|
||||
- Bucket is public-read; write requires `roles/storage.objectAdmin` in project `cognitum-20260110`.
|
||||
- Per-version artifacts must be content-addressed: `cogs/<arch>/cog-<id>-<arch>@<sha256-prefix>` is the immutable copy; the un-suffixed name is a symlink that updates on release.
|
||||
- `COGNITUM_OWNER_SIGNING_KEY` (GCP Secret Manager) signs every binary before upload.
|
||||
|
||||
### Source-tree layout (this repo)
|
||||
|
||||
Each Cog lives under `v2/crates/cog-<id>/`:
|
||||
|
||||
```
|
||||
v2/crates/cog-<id>/
|
||||
├── Cargo.toml # crate name = cog-<id>; binary = cog-<id>
|
||||
├── src/
|
||||
│ ├── main.rs # CLI: cog-<id> run | status | version
|
||||
│ ├── lib.rs
|
||||
│ └── inference.rs # the actual work
|
||||
├── cog/
|
||||
│ ├── manifest.template.json
|
||||
│ ├── config.schema.json # JSON schema for runtime config
|
||||
│ ├── README.md # consumer-facing description (used by the App Store UI)
|
||||
│ ├── icon.svg # 1024×1024 icon (used by App Store hero)
|
||||
│ └── Makefile # build / sign / upload targets
|
||||
└── tests/
|
||||
├── smoke.rs
|
||||
└── manifest_signature.rs
|
||||
```
|
||||
|
||||
### Build pipeline
|
||||
|
||||
```
|
||||
cd v2/crates/cog-<id>
|
||||
make build-arm # cross-compile to aarch64-unknown-linux-gnu
|
||||
make build-x86_64 # x86_64 Linux build
|
||||
make build-hailo8 # arm + HEF compilation (requires Hailo Dataflow Compiler)
|
||||
make build-hailo10 # arm + HEF compilation
|
||||
make sign # produce binary_sha256 + binary_signature
|
||||
make upload # gsutil cp to gs://cognitum-apps/cogs/<arch>/
|
||||
make manifest # emit manifest.json with all fields filled
|
||||
```
|
||||
|
||||
CI (GitHub Actions) MUST run `make build-arm` + `make build-x86_64` on every PR touching `v2/crates/cog-*/`. Hailo HEF compilation requires the proprietary Hailo SDK and runs only on the Hailo-capable runners (currently a labelled self-hosted runner on the Pi cluster — TBD, separate ADR).
|
||||
|
||||
### Runtime contract
|
||||
|
||||
A Cog binary MUST implement:
|
||||
|
||||
| Subcommand | Behaviour |
|
||||
|-----------|-----------|
|
||||
| `cog-<id> version` | Print `<id> <version>` and exit 0. |
|
||||
| `cog-<id> manifest` | Print the embedded manifest JSON and exit 0. |
|
||||
| `cog-<id> run --config /path/to/config.json` | Long-running. Writes structured JSON logs to stdout (parsed by `cognitum-cog-gateway`). Exit code 0 on graceful shutdown, non-zero on fatal error. |
|
||||
| `cog-<id> health` | One-shot. Exit 0 if the cog could come up healthy; non-zero with diagnostic on stderr. Called by the gateway before `run`. |
|
||||
|
||||
stdout JSON line format (one event per line):
|
||||
|
||||
```json
|
||||
{"ts": 1779210883.444, "level": "info", "event": "<event-name>", "fields": { ... }}
|
||||
```
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- New Cogs can be added without RE-ing the layout each time.
|
||||
- CI can verify the manifest schema before merge.
|
||||
- Signed binaries close a real supply-chain gap — current installed cogs (`anomaly-detect@0.0.0`) have no signature, and a compromised GCS object could push malicious code to every appliance.
|
||||
- The runtime contract (`run | health | version | manifest`) is uniform across cogs, so `cognitum-cog-gateway` can stop carrying per-cog adapters.
|
||||
|
||||
### Negative
|
||||
|
||||
- Existing installed cogs must be re-published with signatures within one minor release of the gateway adopting the verify-when-present rule.
|
||||
- Hailo HEF cross-compile is gated on a self-hosted runner; we accept that PRs touching Hailo variants will be slower to land.
|
||||
|
||||
### Risks
|
||||
|
||||
- **Signing key rotation**: `COGNITUM_OWNER_SIGNING_KEY` (Ed25519) is a single root-of-trust today. ADR-103 (witness chain) describes the rotation/recovery path; this ADR depends on that.
|
||||
- **GCS bucket misconfiguration**: a public-read bucket with versioning-off could allow rollback attacks. Bucket MUST have Object Versioning enabled + 90-day non-current-version retention.
|
||||
|
||||
## Migration
|
||||
|
||||
1. ✅ Land this ADR.
|
||||
2. ✅ Land ADR-101 (`cog-pose-estimation` — first Cog built to this spec). Shipped in PR #642 + #643 on 2026-05-19; signed `arm` and `x86_64` binaries live at `gs://cognitum-apps/cogs/{arm,x86_64}/`; install verified on cognitum-v0.
|
||||
3. After two clean releases of `cog-pose-estimation`, re-publish the existing cogs (`anomaly-detect`, `presence`, etc.) with `binary_sha256` + `binary_signature`. Track in a follow-up issue.
|
||||
4. Flip `cognitum-cog-gateway` from "verify when present" to "require signature" — separate ADR, separate review.
|
||||
|
||||
## See also
|
||||
|
||||
- ADR-101: Pose Estimation Cog (first Cog built to this spec).
|
||||
- ADR-103: Witness chain trust model (signing key rotation, future ADR).
|
||||
- `docs/adr/ADR-079-camera-ground-truth-training.md` — the training pipeline behind `cog-pose-estimation`.
|
||||
- `CLAUDE.local.md` § "Fleet Infrastructure (Tailscale)" — appliance layout this ADR describes.
|
||||
@@ -0,0 +1,208 @@
|
||||
# ADR-101: Pose Estimation Cog (WiFi-DensePose side)
|
||||
|
||||
- **Status:** Accepted — **v0.0.1 shipped 2026-05-19** (merged in PRs #642 + #643, signed binaries on GCS, live install on cognitum-v0)
|
||||
- **Date:** 2026-05-19
|
||||
- **Deciders:** ruv
|
||||
- **Companion ADR (v0-appliance side):** v0-appliance ADR-225 (cognitum-pose-estimation crate)
|
||||
|
||||
## Context
|
||||
|
||||
ADR-079 designed the 17-keypoint COCO pose-estimation training pipeline. ADR-100 formalised the Cognitum Cog packaging spec. This ADR is the bridge: it specifies how the wifi-densepose training pipeline produces an artifact that ships as a Cog (`cog-pose-estimation`) onto the Cognitum V0 appliance and out to the Pi+Hailo cluster.
|
||||
|
||||
It is the next product step beyond the published `presence` Cog (binary head trained from the contrastive encoder on Hugging Face at `ruvnet/wifi-densepose-pretrained`). Where `presence` reports a single boolean per tick, `cog-pose-estimation` reports 17 (x, y) keypoints per person, per tick.
|
||||
|
||||
## Decision
|
||||
|
||||
### Pipeline
|
||||
|
||||
```
|
||||
(training side — ruvultra GPU)
|
||||
ESP32 / rvcsi ─► collect-ground-truth.py + sensing-server recording
|
||||
│
|
||||
▼
|
||||
data/paired/*.paired.jsonl (CSI window + camera keypoints)
|
||||
│
|
||||
▼
|
||||
v2/crates/wifi-densepose-train ──► Rust + libtorch trainer
|
||||
(uses RTX 5080 / CUDA 12.x) │
|
||||
init from ruvnet/wifi-densepose-pretrained
|
||||
│
|
||||
▼
|
||||
model.safetensors (encoder + pose head)
|
||||
│
|
||||
─────────────┴─────────────
|
||||
│ │
|
||||
▼ ▼
|
||||
v2/crates/cog-pose-estimation export to ONNX
|
||||
(this repo) │
|
||||
• emits manifest.json ▼
|
||||
• produces cog binary cognitum-hailo
|
||||
• signs + uploads to GCS (v0-appliance side)
|
||||
│
|
||||
▼
|
||||
cog-pose-estimation.hef
|
||||
│
|
||||
▼
|
||||
(appliance side — cognitum-v0 + Pi+Hailo cluster)
|
||||
|
||||
gs://cognitum-apps/cogs/{arm,hailo8,hailo10}/cog-pose-estimation-<arch>
|
||||
│
|
||||
▼
|
||||
`cognitum-cog-gateway` pulls artifact + manifest, verifies signature, installs
|
||||
into /var/lib/cognitum/apps/pose-estimation/
|
||||
│
|
||||
▼
|
||||
run loop: read CSI frames from local sensing-server
|
||||
→ encoder → pose head → emit `{ts, persons: [{keypoints: [...17 x,y...] }]}`
|
||||
on stdout as the Cog runtime contract requires
|
||||
```
|
||||
|
||||
### Architecture (model)
|
||||
|
||||
| Stage | Module | Notes |
|
||||
|-------|--------|-------|
|
||||
| Input | `[56 subcarriers × 20 frames]` per CSI window | matches today's `data/paired/wiflow-p7-*.paired.jsonl` |
|
||||
| Encoder | TCN-lite or contrastive encoder lifted from HF presence model | 128-dim embedding; weights init from `ruvnet/wifi-densepose-pretrained/model.safetensors` |
|
||||
| Pose head | 2-layer MLP `(128 → 256 → 34)` | 34 = 17 × (x, y) |
|
||||
| Output | `[B, 17, 2]` keypoints in `[0, 1]` image-normalised coords | confidence is implicit in keypoint variance over time; ADR-079 P9 will add explicit per-joint confidence |
|
||||
| Loss | Confidence-weighted SmoothL1 (frame-level) + bone-length regulariser + temporal smoothness | per ADR-079 Phase 3 refinement |
|
||||
| Init | Encoder = HF presence weights (frozen for 50 epochs, then jointly fine-tuned) | unblocks the sigmoid-saturation failure mode observed in #645 |
|
||||
| Training | `v2/crates/wifi-densepose-train` with libtorch backend on RTX 5080 | replaces the pure-JS SPSA trainer that produced 0% PCK in #645 |
|
||||
|
||||
### Repo layout
|
||||
|
||||
```
|
||||
v2/crates/cog-pose-estimation/ # NEW (this ADR)
|
||||
├── Cargo.toml
|
||||
├── src/
|
||||
│ ├── main.rs # CLI: run | health | version | manifest
|
||||
│ ├── lib.rs
|
||||
│ ├── inference.rs # ONNX runtime + Hailo HEF runtime dispatch
|
||||
│ ├── frame_subscriber.rs # local sensing-server subscriber
|
||||
│ └── publisher.rs # emits structured JSON events per Cog contract
|
||||
├── cog/
|
||||
│ ├── manifest.template.json
|
||||
│ ├── config.schema.json
|
||||
│ ├── README.md
|
||||
│ ├── icon.svg
|
||||
│ └── Makefile # build-arm | build-x86_64 | sign | upload
|
||||
└── tests/
|
||||
├── manifest_signature.rs
|
||||
└── inference_smoke.rs
|
||||
```
|
||||
|
||||
### Runtime contract
|
||||
|
||||
Honours ADR-100's per-Cog CLI contract:
|
||||
|
||||
- `cog-pose-estimation version` → `pose-estimation 0.0.1`
|
||||
- `cog-pose-estimation manifest` → JSON
|
||||
- `cog-pose-estimation health` → 0 if encoder+head load and a synthetic frame produces a finite output
|
||||
- `cog-pose-estimation run --config /etc/cognitum/cogs/pose-estimation/config.json` → long-running; emits one JSON event per inferred frame:
|
||||
|
||||
```json
|
||||
{
|
||||
"ts": 1779210883.444,
|
||||
"level": "info",
|
||||
"event": "pose.frame",
|
||||
"fields": {
|
||||
"tick": 12345,
|
||||
"n_persons": 1,
|
||||
"persons": [
|
||||
{"keypoints": [[0.48, 0.31], [0.52, 0.28], ...], "confidence": 0.81}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Hardware deployment
|
||||
|
||||
| Target | arch | runtime | notes |
|
||||
|--------|------|---------|-------|
|
||||
| ruvultra (dev) | `x86_64` | ONNX Runtime CPU/CUDA | development & smoke tests |
|
||||
| cognitum-v0 (Pi 5) | `arm` | ONNX Runtime ARM | reference deploy; ~20 ms/frame |
|
||||
| Pi + Hailo-8 hat | `hailo8` | Hailo HEF runtime via `cognitum-hailo` | ~2 ms/frame, 26 TOPS budget |
|
||||
| Pi + Hailo-10 hat | `hailo10` | Hailo HEF runtime via `cognitum-hailo` | ~1 ms/frame, 40 TOPS budget |
|
||||
|
||||
### Acceptance gates
|
||||
|
||||
1. **Validates:** `cargo test -p cog-pose-estimation` green; `cog-pose-estimation health` returns 0 against a synthetic CSI window.
|
||||
2. **Benchmarks:** end-to-end frame latency on each target arch logged in `target/criterion/`; published in `docs/benchmarks/pose-estimation-cog.md`.
|
||||
3. **Optimised:** the Hailo-targeted ONNX graph passes through Hailo Dataflow Compiler without quantisation-aware-training warnings.
|
||||
4. **Published:** signed binary at `gs://cognitum-apps/cogs/<arch>/cog-pose-estimation-<arch>`; manifest valid against the JSON schema in ADR-100; appliance installer can pull and run it.
|
||||
|
||||
PCK@20 is intentionally **not** an acceptance gate of this ADR. Achieving the ADR-079 ≥35% target is a separate, data-bound milestone tracked in #645. This ADR ships the **vehicle**, not the model accuracy.
|
||||
|
||||
### First measured run — v0.0.1 (2026-05-19)
|
||||
|
||||
A Candle-on-CUDA training run on `ruvultra`'s RTX 5080 against the same 1,077-sample paired session that produced the 0%/0% baseline in #645 yielded:
|
||||
|
||||
- **PCK@20 = 3.0%**, **PCK@50 = 18.5%**, **MPJPE = 0.093** (normalized).
|
||||
- 400 epochs in **2.1 s** wall time (~5 ms/epoch, full-batch).
|
||||
- Loss reduction 13× (0.181 → 0.014, eval 0.010).
|
||||
- Strongest signal at `r_hip` (PCK@50 = 76.9%), `r_knee` (35.2%), `l_elbow` (26.4%).
|
||||
|
||||
This confirms the pipeline trains end-to-end and produces a signal-bearing model. The remaining gap to PCK@20 ≥ 35% is data-bound (1,077 samples is ≪ the ADR-079 target of ~30K). See `docs/benchmarks/pose-estimation-cog.md` for the full result dump.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- First Cog from this repo that integrates with the appliance/cog-gateway pipeline. Future cogs (e.g. `cog-vitals`, `cog-fall-alert`) follow the same template.
|
||||
- Closes the loop from data collection → training → quantisation → cluster deployment with a single repo-anchored artifact.
|
||||
- Forces a real signature on cog binaries (per ADR-100), which improves supply-chain hygiene across the whole appliance.
|
||||
|
||||
### Negative
|
||||
|
||||
- Adds a hard dependency on the Hailo Dataflow Compiler, which lives behind a self-hosted runner — Hailo-targeted PRs land more slowly.
|
||||
- The first published binary will have low PCK (data + training time gap, #645) — UX needs to surface this clearly so end users do not interpret bad keypoints as a bug.
|
||||
|
||||
### Risks
|
||||
|
||||
- **Model size on Hailo**: the encoder fits comfortably in Hailo-8's on-chip SRAM, but the pose-head expansion to `[17×2]` plus required temporal stacking pushes us close to the Hailo-8 envelope. Mitigation: Hailo-10 path is the primary deploy target; Hailo-8 is a stretch.
|
||||
- **Sensing-server schema drift**: the cog subscribes to `/api/v1/sensing/latest` JSON. If the appliance's sensing-server schema changes, the cog fails open (logs warning, emits nothing). The `frame_subscriber.rs` module pins to schema version `2`.
|
||||
|
||||
## Migration / rollout
|
||||
|
||||
1. Land this ADR + ADR-100 on `main` of RuView.
|
||||
2. Land companion ADR-225 + crate on `main` of v0-appliance.
|
||||
3. First release `cog-pose-estimation@0.0.1` ships **only** to `ruvultra` and `cognitum-v0`. Not pushed to the cluster Pis yet.
|
||||
4. After P7→P9 data work (#645) brings PCK above a usable threshold, rebuild + re-publish; only then enable cluster rollout via `cognitum-cog-gateway`'s OTA channel.
|
||||
|
||||
## v0.0.1 shipping status — 2026-05-19
|
||||
|
||||
PRs `#642` (scaffold + arm release + ONNX + live install) and `#643` (x86_64 release) landed on `main`. Acceptance gates from ADR-100 met as follows:
|
||||
|
||||
| Gate | Status |
|
||||
|------|--------|
|
||||
| Cog binary exists per arch | ✅ arm (`3,741,976 B`) + x86_64 (`4,548,856 B`) on GCS |
|
||||
| Manifest matches schema | ✅ `cog/artifacts/manifests/{arm,x86_64}/manifest.json` |
|
||||
| Binary sha256 + Ed25519 signature | ✅ both signed with `COGNITUM_OWNER_SIGNING_KEY`, round-trip verified |
|
||||
| Public-readable GCS | ✅ anonymous HTTP GET works, SHA matches |
|
||||
| Live install on a real appliance | ✅ `/var/lib/cognitum/apps/pose-estimation/` on `cognitum-v0` (Pi 5), same layout as `anomaly-detect` |
|
||||
| Runtime contract (`version \| manifest \| health \| run`) | ✅ all four return correct output; `run` emits `pose.frame` events |
|
||||
| Real weights loaded (not stub) | ✅ `cargo test` asserts `backend.starts_with("candle-")` + non-zero confidence |
|
||||
| ONNX artifact (for downstream HEF) | ✅ `pose_v1.onnx` (12 KB), parity vs torch = 8.94e-8 |
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Training time (RTX 5080 / Candle CUDA) | 2.1 s for 400 epochs |
|
||||
| PCK@20 / PCK@50 / MPJPE (1,077-sample seated-desk session) | 3.0% / 18.5% / 0.093 |
|
||||
| Cold-start: Windows x86_64 | 76 ms |
|
||||
| Cold-start: ruvultra x86_64 | **5.4 ms** |
|
||||
| Cold-start: Pi 5 aarch64 | **8.4 ms** |
|
||||
| Tests | 5/5 pass |
|
||||
|
||||
Open follow-ups carried forward from this ADR's "Acceptance gates" section:
|
||||
|
||||
- **Hailo HEF cross-compile** — `pose_v1.onnx` is ready; still gated on Hailo Dataflow Compiler + self-hosted runner provisioning. Tracked separately.
|
||||
- **PCK@20 ≥ 35%** — explicitly not an acceptance gate of this ADR, but the limiting factor on practical usefulness. Tracked in [#645](https://github.com/ruvnet/RuView/issues/645): needs ~30× more paired samples + multi-room camera framing. Today's seated-desk session is the demonstrated bottleneck.
|
||||
|
||||
## See also
|
||||
|
||||
- ADR-079: Camera-supervised pose training pipeline (the model we're shipping).
|
||||
- ADR-100: Cog packaging specification (the format we're shipping in).
|
||||
- v0-appliance ADR-225: cognitum-pose-estimation crate (the appliance-side runtime).
|
||||
- v0-appliance ADR-220: cog management surface (where this cog appears in the dashboard).
|
||||
- Issue #645: PCK gap (current 3% / 18.5% → ≥35% target).
|
||||
- `docs/benchmarks/pose-estimation-cog.md`: full benchmark log, all measured numbers.
|
||||
@@ -0,0 +1,171 @@
|
||||
# ADR-102: Edge Module Registry Integration
|
||||
|
||||
- **Status:** Accepted
|
||||
- **Date:** 2026-05-19
|
||||
- **Deciders:** ruv
|
||||
|
||||
## Context
|
||||
|
||||
The Cognitum app ecosystem publishes a canonical app store catalog at:
|
||||
|
||||
```
|
||||
https://storage.googleapis.com/cognitum-apps/app-registry.json
|
||||
```
|
||||
|
||||
As of v2.1.0 (2026-05-13) the registry advertises **105 cogs across 11 categories** (health, security, building, retail, industrial, research, ai, swarm, signal, network, developer). Each entry carries `id`, `name`, `category`, `version`, `description`, `size_kb`, `difficulty`, `sha256`, `binary_size`, and a `config[]` schema describing the runtime parameters the appliance offers when installing the cog.
|
||||
|
||||
RuView today has no live awareness of this catalog. The `README.md` capability table is hand-curated; the UI surfaces only the capabilities the dashboard's HTML knows about; nothing in `wifi-densepose-sensing-server` references the registry. Result: when Cognitum ships a new cog (the registry was last updated 6 days ago — a fast cadence), RuView stays unaware until someone manually edits the README. Customers running the RuView dashboard against a real appliance see a 10-capability bag in the UI while the appliance is actually capable of installing 105 cogs.
|
||||
|
||||
Today's `cog-pose-estimation@0.0.1` release (PRs #642 / #643, ADR-100, ADR-101) is the first cog this repo ships to that registry. We need the discovery side to match.
|
||||
|
||||
## Decision
|
||||
|
||||
`wifi-densepose-sensing-server` will fetch `app-registry.json` on demand, cache it in process memory with a TTL, and serve it back through a new endpoint:
|
||||
|
||||
```
|
||||
GET /api/v1/edge/registry
|
||||
GET /api/v1/edge/registry?refresh=1 (force-bypass cache, log if abused)
|
||||
```
|
||||
|
||||
The registry is **passively surfaced**, not modified. RuView is a presentation layer for the canonical Cognitum catalog; it never re-signs entries or re-hosts binaries.
|
||||
|
||||
### Module
|
||||
|
||||
`v2/crates/wifi-densepose-sensing-server/src/edge_registry.rs` — small, ~150 lines.
|
||||
|
||||
```rust
|
||||
pub struct EdgeRegistry {
|
||||
cached: RwLock<Option<CachedEntry>>,
|
||||
ttl: Duration,
|
||||
upstream_url: String,
|
||||
}
|
||||
|
||||
struct CachedEntry {
|
||||
payload: serde_json::Value,
|
||||
fetched_at: Instant,
|
||||
upstream_sha256: String,
|
||||
}
|
||||
```
|
||||
|
||||
Cache semantics:
|
||||
|
||||
- TTL **3600 s (1 hour)** by default — registry updates land on a roughly-weekly cadence and a stale-by-an-hour catalog is fine.
|
||||
- `?refresh=1` bypasses the cache but writes a debug log so accidental abuse is visible.
|
||||
- On upstream fetch failure when the cache is non-empty, **serve the stale cached copy** with a `stale: true` marker in the response and a 200 status (preserve UI), not a 5xx.
|
||||
- On upstream fetch failure when the cache is empty, return 503 with the upstream error in the body.
|
||||
|
||||
### Response shape
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"fetched_at": 1779200000, // server-side fetch timestamp
|
||||
"ttl_seconds": 3600,
|
||||
"stale": false, // true when serving past TTL because upstream is down
|
||||
"upstream_url": "https://storage.googleapis.com/cognitum-apps/app-registry.json",
|
||||
"upstream_sha256": "<sha256-of-payload-bytes>",
|
||||
"registry": { /* full canonical JSON as returned upstream */ }
|
||||
}
|
||||
```
|
||||
|
||||
The `registry` field is the upstream JSON inlined verbatim so consumers don't need to make a second hop. `upstream_sha256` lets a paranoid consumer compare against a pinned hash.
|
||||
|
||||
### Trust / verification
|
||||
|
||||
- Bucket is public-read with object versioning enabled (per ADR-100 §"GCS misconfiguration risks").
|
||||
- The cog-level `binary_sha256` + `binary_signature` (ADR-100) are the trust roots for *installs*. The registry itself is not signed today.
|
||||
- We deliberately **do not** add a signature requirement to the registry JSON in this ADR — that would block the integration on a parallel infrastructure project. A future ADR can layer signature checks on top once the publisher pipeline emits them.
|
||||
|
||||
### UI surfacing
|
||||
|
||||
New page `ui/edge-modules.html` renders the registry into category sections with cog cards. Each card links out to the Cognitum V0 appliance's `/cogs` page (`http://cognitum-v0:9000/cogs#<id>`) for the install action — RuView itself never installs.
|
||||
|
||||
The existing dashboard's "Capabilities" section continues to show RuView-native sensing capabilities (presence, breathing, pose, etc. — the things RuView itself runs); the new edge-modules page shows the broader Cognitum cog catalog. The two are distinct surfaces and shouldn't be merged.
|
||||
|
||||
### Failure modes
|
||||
|
||||
| Scenario | Behaviour |
|
||||
|---|---|
|
||||
| Upstream returns 200 with valid JSON | Cache it, return it. |
|
||||
| Upstream returns 200 with invalid JSON | Treat as failure; serve stale if available else 503. Log the upstream sha + the parse error. |
|
||||
| Upstream returns 4xx / 5xx | Same as JSON-invalid: serve stale if available else 503. |
|
||||
| TLS / DNS / timeout error | Same. |
|
||||
| Upstream is permanently moved | Operator updates the `upstream_url` config (CLI flag added). No code change required to migrate registries. |
|
||||
|
||||
### Configuration
|
||||
|
||||
- `--edge-registry-url <URL>` — override the default (default: `https://storage.googleapis.com/cognitum-apps/app-registry.json`)
|
||||
- `--edge-registry-ttl-secs <N>` — override the cache TTL (default: 3600)
|
||||
- `--no-edge-registry` — disable the endpoint entirely (returns 404). For air-gapped deployments.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- One source of truth for the cog catalog across RuView + Cognitum dashboards.
|
||||
- Zero ongoing maintenance: when Cognitum publishes registry v2.2.0, RuView sees it within an hour without a release.
|
||||
- The endpoint is also useful for non-UI consumers (CI checks, fleet automation, third-party integrations).
|
||||
- Lets us deprecate the hand-curated README capability table in favour of generated content (separate PR).
|
||||
|
||||
### Negative
|
||||
|
||||
- Adds an outbound HTTP dependency to the sensing-server. Air-gapped deployments must use `--no-edge-registry`.
|
||||
- Stale-but-served behaviour can mask upstream outages from operators. Mitigation: include `stale: true` + `fetched_at` in the response so the UI can render a "registry possibly out of date" badge.
|
||||
|
||||
### Risks
|
||||
|
||||
- **Upstream rug-pull**: if `cognitum-apps` is deleted or replaced, the endpoint goes dark. The `--edge-registry-url` flag lets operators repoint without a code change. Long-term, RuView could mirror the registry into its own GCS bucket if the relationship requires it.
|
||||
- **Cache poisoning**: the upstream is public-read; an attacker who breaches Cognitum's GCS write could push a bad registry. The cog-level signatures (ADR-100) limit the blast radius — bad registry entries can't install bad binaries, only show wrong metadata. Acceptable until registry-level signing lands.
|
||||
|
||||
## Security review
|
||||
|
||||
A real review of the attack surface this endpoint introduces.
|
||||
|
||||
### Threats considered
|
||||
|
||||
| # | Threat | Mitigation in this ADR |
|
||||
|---|--------|------------------------|
|
||||
| T1 | **SSRF** — operator-supplied `--edge-registry-url` redirects fetches to an internal target | Flag is operator-only (CLI / env) — there is no API endpoint to mutate it at runtime. Operators are already trusted (they control the binary). |
|
||||
| T2 | **Outbound dependency reveals deployment** — a passive observer of the egress sees the appliance phoning home to GCS | Documented in the docstring + the runtime startup log. Operators wanting offline deployments use `--no-edge-registry`. |
|
||||
| T3 | **Malicious upstream registry** — Cognitum's GCS bucket is breached and a poisoned `app-registry.json` is served | Two layers absorb this: (a) the registry's role is **discovery only** — installs verify the per-cog `binary_sha256` + `binary_signature` (ADR-100); a wrong description string can mislead a human, but a wrong binary still has to pass Ed25519 against `COGNITUM_OWNER_SIGNING_KEY`. (b) The endpoint exposes `upstream_sha256` so a paranoid operator can pin the expected registry hash externally and alert on drift. |
|
||||
| T4 | **Response inflation** — upstream returns a multi-GB payload to exhaust memory | `MAX_PAYLOAD_BYTES = 8 MiB` cap (current registry is ~50–200 KB). Exceeding cap returns an error without buffering past the cap. |
|
||||
| T5 | **Slow upstream blocking server threads** — Slowloris-style stall on the fetch | 10-second wire timeout via `ureq::AgentBuilder`. Per-handler fetch runs inside `tokio::task::spawn_blocking` so a stalled fetch never blocks the async runtime. |
|
||||
| T6 | **Denial via `?refresh=1` abuse** — unauthenticated callers force-bypass the cache repeatedly | Cache lives in process; `?refresh=1` triggers a single upstream fetch behind a synchronous code path. A flood of refresh requests is rate-limited by the upstream's own throttling (GCS) and locally serialised by Rust's `RwLock`. Refresh requests are logged at `debug` so abuse is visible. **Follow-up:** add per-IP rate-limit middleware if seen abused (separate PR; tracked in #574-style follow-up). |
|
||||
| T7 | **JSON deserialisation panics** — malformed registry triggers a Rust panic | Payload is parsed as `serde_json::Value` (opaque untyped tree) — never coerced into a strongly-typed struct that could panic. Failure is propagated as `FetcherError::Network` which the handler maps to 503. |
|
||||
| T8 | **Stale-on-error masks outages from operators** | Response carries `stale: true` + `fetched_at` (unix timestamp). UI rendering MUST surface this badge — encoded as an explicit field, not an implicit silence. |
|
||||
| T9 | **TLS downgrade / MITM on the fetch** | `ureq` is built with the `tls` feature (rustls) by default. No `--insecure` flag exists. If the upstream uses LetsEncrypt the cert chain is system-trusted; certificate pinning is out of scope (would block the bucket from rotating certs). |
|
||||
| T10 | **Unauthenticated access exposes ‘what cogs exist’** | The registry is canonical-public information (already public-read on GCS via anonymous HTTP GET). Surfacing it on a local LAN HTTP API does not increase its disclosure. The endpoint stays under the project's existing `RUVIEW_API_TOKEN` Bearer auth — when set, the registry is gated like other `/api/v1/*` routes. |
|
||||
| T11 | **Configuration injection via env var** — `RUVIEW_EDGE_REGISTRY_URL` set to a malicious URL by an attacker who controls the process environment | If an attacker controls the env, they own the process; this is not a new threat surface. Documented in the CLI help. |
|
||||
| T12 | **Cache mutation across threads / poisoning** | The cache is `RwLock<Option<CachedEntry>>`. Writes go through `cached.write()` once per fetch. Snapshot reads `clone()` the `CachedEntry` (cheap — `Value` is reference-counted internally for large strings) so concurrent readers don't share mutable state. Tests cover the multi-call path; no `unsafe` is used. |
|
||||
|
||||
### What this ADR does NOT secure
|
||||
|
||||
- **Registry-level signing** — the JSON payload itself is unsigned. If/when Cognitum's publisher pipeline emits a registry sig (e.g. detached `.json.sig`), a follow-up ADR will require it. Today the per-cog binary signature (ADR-100) is the actual trust root for installs; the registry is metadata.
|
||||
- **Per-client rate-limiting on `?refresh=1`** — relies on the upstream's own throttling. If we see abuse we'll add a token-bucket middleware; not needed for v0.0.1.
|
||||
|
||||
### Testing
|
||||
|
||||
| Test | What it verifies |
|
||||
|------|------------------|
|
||||
| `first_call_hits_upstream_and_caches` | Single fetch, then cache hit |
|
||||
| `ttl_expiry_triggers_refetch` | Cache TTL bound respected |
|
||||
| `force_refresh_bypasses_fresh_cache` | `?refresh=1` semantics |
|
||||
| `stale_serve_on_upstream_failure_after_cached_success` | T8 explicit (`stale: true` returned) |
|
||||
| `no_cache_no_upstream_returns_error` | T3/T5 — error propagated cleanly when nothing to fall back on |
|
||||
| `upstream_invalid_json_is_treated_as_error` | T7 — malformed payload doesn't panic |
|
||||
| `upstream_sha256_is_deterministic` | T3 — hash field is reliable for external pinning |
|
||||
|
||||
All 7 tests in `src/edge_registry.rs::tests` pass.
|
||||
|
||||
## Migration
|
||||
|
||||
1. Land this ADR + the implementing PR.
|
||||
2. UI: ship `ui/edge-modules.html` and link from `index.html`.
|
||||
3. After two clean releases of the endpoint, remove the hand-curated "Capabilities" table from `README.md` and replace with a small "see the appliance for the full catalog" pointer.
|
||||
4. Future ADR: registry signing once Cognitum's publisher pipeline emits a sig.
|
||||
|
||||
## See also
|
||||
|
||||
- ADR-100: Cognitum Cog Packaging Specification (binary trust model).
|
||||
- ADR-101: Pose Estimation Cog (the first repo-shipped cog visible in the registry).
|
||||
- v0-appliance ADR-220: Cog management surface (where this registry is the input to install actions).
|
||||
- `docs/benchmarks/pose-estimation-cog.md`: the per-cog benchmark format this ADR's response shape complements.
|
||||
@@ -0,0 +1,198 @@
|
||||
# ADR-103: Learned Multi-Person Counter (SOTA WiFi CSI counting)
|
||||
|
||||
- **Status:** Proposed
|
||||
- **Date:** 2026-05-21
|
||||
- **Deciders:** ruv
|
||||
- **Motivating issue:** #499 (double skeletons with 3-node ESP32-S3 setup, closed by PR #491)
|
||||
- **Related:** ADR-079 (camera-supervised training), ADR-100 (cog packaging), ADR-101 (pose cog), ADR-102 (edge module registry), PR #491 (RollingP95 + dedup_factor)
|
||||
|
||||
## Context
|
||||
|
||||
PR #491 stopped the bleeding on #499. The fix replaced hard-coded denominators (`variance/300`, `motion_band_power/250`, `spectral_power/500`) with a self-calibrating `RollingP95` streaming estimator and exposed the multi-node `dedup_factor` as a runtime knob. Day-0 deployments no longer collapse dynamic range, and operators can auto-tune the divisor from a known person count.
|
||||
|
||||
That gets us to a **stable heuristic that adapts to the room**. It does not get us to the published WiFi-CSI counting state of the art:
|
||||
|
||||
| System | Setup | Reported accuracy | Method |
|
||||
|--------|-------|-------------------|--------|
|
||||
| **WiCount** (CMU, 2017) | Intel 5300 3×3 MIMO | 89% within ±1 | LSTM over CSI amplitude |
|
||||
| **DeepCount** (2018) | Atheros 3×3 | 92% within ±1, 5-room | CNN + cross-environment transfer |
|
||||
| **CrossCount** (2019) | Atheros, 6 rooms | 84% cross-room within ±1 | Domain-adversarial CNN |
|
||||
| **HeadCount** (2021) | Intel 5300 | <1 person MAE, 5 envs | Multi-stream CSI + attention |
|
||||
| **RuView today** (PR #491) | ESP32-S3 1×1 SISO | Calibrated heuristic; not measured against ground truth | RollingP95 + dedup_factor |
|
||||
|
||||
The literature uses 3×3 MIMO research NICs. RuView uses 1×1 SISO ESP32-S3 nodes. The published number is therefore not directly attainable, but the **architectural gap** is large enough that a learned-counter approach on our hardware should comfortably beat today's slot heuristic — and the infrastructure to train one already exists in this repo (Candle + RTX 5080 trained `pose_v1.safetensors` in 2.1 s yesterday — see [`docs/benchmarks/pose-estimation-cog.md`](../benchmarks/pose-estimation-cog.md)).
|
||||
|
||||
Five primitives we already have but don't yet compose into a counter:
|
||||
|
||||
1. **Paired CSI + camera label dataset** — `scripts/collect-ground-truth.py` + `scripts/align-ground-truth.js` (PR #641 streaming-safe). 1,077 samples currently; #645 tracks the path to ~30K.
|
||||
2. **Stoer-Wagner min-cut for person-separable subcarrier groups** — `ruvector-mincut` (already a workspace dep). The Candle trainer used it yesterday and reported `Min-cut value: 0.1538 — partition: [55, 1] subcarriers`.
|
||||
3. **Contrastive-pretrained CSI encoder** — `ruvnet/wifi-densepose-pretrained` on HF (12.2M training steps, 60K frames, 128-dim embeddings, ~165k emb/s on M4 Pro).
|
||||
4. **Candle training pipeline** — proven yesterday: 400 epochs in 2.1 s on RTX 5080, bit-perfect ONNX export, signed cog binary on GCS.
|
||||
5. **Multi-node fusion stage** — `multistatic_bridge.rs` already aggregates per-node feature vectors with the tunable `dedup_factor`. The new model output can be a drop-in replacement for the existing dedup divisor.
|
||||
|
||||
## Decision
|
||||
|
||||
Train and ship a small **learned multi-person counter** as a new Cognitum Cog (`cog-person-count`), modelled on the same packaging path as `cog-pose-estimation` (ADR-101). Wire it into the sensing-server's existing person-count call site (`csi.rs::score_to_person_count`) as a drop-in replacement for the slot heuristic.
|
||||
|
||||
### Architecture (v0.1.0)
|
||||
|
||||
```
|
||||
┌──────────────────────────────┐
|
||||
per-node CSI window │ Encoder (frozen first 50 ep) │
|
||||
[56 sub × 20 frames] ─► init from ruvnet/wifi- │
|
||||
│ densepose-pretrained │
|
||||
│ → 128-dim embedding │
|
||||
└──────────────┬───────────────┘
|
||||
│
|
||||
┌────────────────┴────────────────┐
|
||||
▼ ▼
|
||||
┌────────────────────┐ ┌────────────────────────┐
|
||||
│ Count head │ │ Confidence head │
|
||||
│ Linear(128→64) │ │ Linear(128→32) │
|
||||
│ ReLU │ │ ReLU │
|
||||
│ Linear(64→8) │ │ Linear(32→1) + sigmoid│
|
||||
│ → softmax over │ │ → calibrated p(correct)│
|
||||
│ {0..7} persons │ └────────────────────────┘
|
||||
└────────┬───────────┘
|
||||
│ (per-node prediction)
|
||||
│
|
||||
N nodes' per-node │
|
||||
counts + confidences ▼
|
||||
┌─────────────────────────────────────┐
|
||||
│ Multi-node fusion (Stoer-Wagner) │
|
||||
│ • build graph: nodes × subcarrier │
|
||||
│ feature similarity │
|
||||
│ • min-cut → distinct-person bound │
|
||||
│ • combine with per-node count head │
|
||||
│ via confidence-weighted vote │
|
||||
└──────────────────┬──────────────────┘
|
||||
▼
|
||||
{ count: int,
|
||||
confidence: float [0,1],
|
||||
count_p95_low: int,
|
||||
count_p95_high: int,
|
||||
per_node_breakdown: [...] }
|
||||
```
|
||||
|
||||
Five things to call out about this architecture:
|
||||
|
||||
1. **Frozen encoder for the first 50 epochs.** The HF presence encoder already produces a useful 128-dim embedding from random CSI; training the counting head on top of frozen features is the standard transfer-learning pattern and avoids re-learning the contrastive geometry the encoder was painstakingly trained for.
|
||||
2. **Classification over `{0..7}` people**, not regression to a real number. Counts are integer-valued; classification gives a calibrated probability per count and lets the confidence head produce a meaningful uncertainty.
|
||||
3. **Stoer-Wagner min-cut at fusion time, not training time.** We use the min-cut primitive to bound the per-node count from above (a node can't see more distinct people than the subcarrier graph has min-cuts), then take a confidence-weighted vote.
|
||||
4. **Output is `{count, confidence, count_p95_low, count_p95_high}`**, not a single integer. Downstream consumers (Cogs / dashboard / alerts) can choose their certainty threshold. This is what closes the loop on the #499 UX: when the model is uncertain, the dashboard renders one stick figure with a "?" badge rather than two ghosts.
|
||||
5. **No new hardware.** Same ESP32-S3 1×1 SISO that ships today. The win comes from learned features + multi-node fusion, not from bigger antennas.
|
||||
|
||||
### Training (Candle / RTX 5080 / proven path)
|
||||
|
||||
Same exact pipeline that produced `pose_v1.safetensors` yesterday. Differences:
|
||||
|
||||
| | Pose cog (today) | Count cog (this ADR) |
|
||||
|---|---|---|
|
||||
| Input | `[56, 20]` CSI window | `[56, 20]` CSI window (identical) |
|
||||
| Encoder init | random (HF arch mismatch) | **from HF presence model** (architectures are compatible — same encoder Φ) |
|
||||
| Output head | `Linear(128 → 256 → 34)` keypoints | `Linear(128 → 64 → 8)` count classes + `Linear(128 → 32 → 1)` confidence |
|
||||
| Loss | Confidence-weighted SmoothL1 | Categorical cross-entropy + Brier-score uncertainty calibration |
|
||||
| Labels | MediaPipe keypoints | Camera count (MediaPipe `pose_landmarks` length) |
|
||||
| Data | 1,077 paired (P7) | **Same source, same script** — `collect-ground-truth.py` already records `n_persons` per frame |
|
||||
|
||||
Crucially we get the count labels **for free** from the existing pose data-collection pipeline — `collect-ground-truth.py` already records `"n_persons"` per camera frame and `align-ground-truth.js` already preserves it through windowing. No new data collection campaign required to bootstrap; we can train tomorrow on the same 1,077 samples that produced `pose_v1`.
|
||||
|
||||
### Multi-node fusion
|
||||
|
||||
The per-node count head + confidence head emit a categorical distribution over `{0..7}`. With N nodes, we have N such distributions plus N confidence scalars. Two fusion paths:
|
||||
|
||||
- **Confidence-weighted log-sum** (Bayesian product): `log p_fused(k) = Σ_n c_n · log p_n(k)`. Simple, no extra parameters, comes from the optimal-expert combination literature.
|
||||
- **Stoer-Wagner upper bound**: build a graph where edges are pairwise subcarrier-feature similarities between nodes. Min-cut size = a hard upper bound on the number of distinct people the node mesh can resolve. Clip the per-node-fused distribution to support `{0..min-cut}` before re-normalising. This is exactly what `ruvector-mincut` was added to the workspace for — it's been waiting for a counting consumer.
|
||||
|
||||
Both fuse cleanly. v0.1.0 ships the log-sum; v0.2.0 adds the min-cut clipper after the first round of evaluation.
|
||||
|
||||
### Why this beats today's heuristic
|
||||
|
||||
| Failure mode of today's slot heuristic | How the learned counter avoids it |
|
||||
|---|---|
|
||||
| #499 — fixed denominators clamp → one person renders as 2+ groups | Encoder produces a fixed-dim embedding; the count head is invariant to feature magnitude, only to feature **shape** |
|
||||
| `dedup_factor` per-room tuning is operator-visible toil | Count head's softmax is a learned per-room normaliser by construction |
|
||||
| Adding nodes makes the count noisier under the slot heuristic | Multi-node fusion is **additive in confidence**, so each node either reduces uncertainty or stays neutral — never amplifies it |
|
||||
| No per-frame uncertainty signal | `confidence` + `count_p95_low/high` exposed in every emit |
|
||||
| Catastrophic failure on novel environments | LoRA per-room adapter (per ADR-079 P9 plan) hot-swappable without retraining |
|
||||
|
||||
### Acceptance gates
|
||||
|
||||
| Gate | v0.1.0 (initial release) | v0.2.0 (after data scaling) |
|
||||
|------|--------------------------|------------------------------|
|
||||
| Day-0 deployment (no calibration) | ≥ 80% within ±1 on same-room test set | ≥ 90% within ±1 |
|
||||
| Cross-room (held-out environment) | ≥ 60% within ±1 | ≥ 75% within ±1 |
|
||||
| Mean Absolute Error | ≤ 0.6 persons | ≤ 0.4 persons |
|
||||
| Per-frame confidence reflects accuracy | Spearman correlation `r ≥ 0.5` between `confidence` and `(predicted == true)` | `r ≥ 0.7` |
|
||||
| Inference latency on Pi 5 (Cog) | < 5 ms / frame cold-start | < 5 ms / frame |
|
||||
| Binary size on GCS | ≤ 4 MB (matches `cog-pose-estimation`) | ≤ 4 MB |
|
||||
|
||||
`v0.1.0` is intentionally modest — it's bounded by data-collection scale (#645). The framework is the deliverable; the accuracy follows the data.
|
||||
|
||||
### Repo layout
|
||||
|
||||
```
|
||||
v2/crates/cog-person-count/ # NEW (this ADR)
|
||||
├── Cargo.toml
|
||||
├── src/
|
||||
│ ├── main.rs # cog runtime: version | manifest | health | run
|
||||
│ ├── lib.rs
|
||||
│ ├── inference.rs # Candle forward pass on per-node CSI
|
||||
│ ├── fusion.rs # Stoer-Wagner upper-bound + confidence-weighted log-sum
|
||||
│ └── publisher.rs # emits {count, confidence, count_p95_low, count_p95_high}
|
||||
├── cog/
|
||||
│ ├── manifest.template.json
|
||||
│ ├── config.schema.json
|
||||
│ ├── README.md
|
||||
│ └── artifacts/ # filled by the release pipeline
|
||||
│ ├── count_v1.safetensors
|
||||
│ ├── count_v1.onnx
|
||||
│ └── train_results.json
|
||||
└── tests/
|
||||
├── smoke.rs # 5+ tests
|
||||
└── fusion_test.rs # multi-node-fusion math
|
||||
```
|
||||
|
||||
Plus a small server-side wiring change:
|
||||
|
||||
- `v2/crates/wifi-densepose-sensing-server/src/csi.rs::score_to_person_count` — call the cog over the same `/api/v1/edge/registry`-discovered runtime as `cog-pose-estimation`. Falls back to today's PR #491 heuristic if the cog isn't installed (per the ADR-100 stub-fallback pattern).
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- Closes the conceptual loop opened by #499 — multi-person counting becomes a **learned task**, not a heuristic with a runtime knob.
|
||||
- Reuses every primitive already shipped this week: Candle GPU training (ADR-101), HF encoder, Cog packaging (ADR-100), edge module registry (ADR-102), Stoer-Wagner mincut, paired-data pipeline (PR #641).
|
||||
- Day-2 cross-room calibration uses the same LoRA path ADR-079 P9 plans for pose, so the two cogs share the same fine-tuning machinery.
|
||||
- Explicit `confidence` + `count_p95_low/high` outputs let the UI render uncertainty instead of inventing ghosts.
|
||||
|
||||
### Negative
|
||||
|
||||
- Accuracy is bounded by the same paired-data scarcity that bounds `pose_v1` (#645). Without more multi-room data, v0.1.0 ships with modest absolute accuracy.
|
||||
- Adds another Cog binary to maintain in the GCS catalog — 4 MB per arch.
|
||||
- The fusion-stage min-cut adds ~0.3 ms per N-node frame on a Pi 5 in microbenchmarks of `ruvector-mincut`. Acceptable given the ≤ 5 ms budget but worth tracking.
|
||||
|
||||
### Risks
|
||||
|
||||
- **Label noise**: MediaPipe pose-detection rate was 47% in the P7 session — half the frames have `n_persons = 0` even when a person was clearly in the room. The count head learns from this noisy signal; mitigations include filtering by `MediaPipe confidence ≥ 0.7` before training, and weighting the loss by confidence (same trick used in `pose_v1`).
|
||||
- **Encoder freezing too aggressive**: if 50 epochs of frozen-encoder training doesn't see the count head converge, unfreeze earlier. We have telemetry from `train_results.json` to make this call empirically.
|
||||
- **Min-cut over-constrains** in single-person scenarios: when N=1 the subcarrier graph has min-cut = 1 trivially. The fusion stage degrades to "trust the single-node count head", which is fine but worth a regression test (`tests/fusion_test.rs::single_node_degrades_gracefully`).
|
||||
|
||||
## Migration
|
||||
|
||||
1. Land this ADR + the new crate scaffold (one PR, no model yet — same approach as ADR-101's first PR shipped a stub cog).
|
||||
2. Train `count_v1.safetensors` on the existing 1,077 paired samples + `n_persons` labels. Same Candle pipeline that produced `pose_v1`.
|
||||
3. Cross-compile + sign + GCS upload per ADR-100. Live install on `cognitum-v0` per ADR-101's pattern.
|
||||
4. Wire `csi.rs::score_to_person_count` to call the cog when installed; keep PR #491's heuristic as fallback.
|
||||
5. v0.2.0: re-train on the multi-room data #645 motivates, add LoRA per-room adapters per ADR-079 P9.
|
||||
|
||||
## See also
|
||||
|
||||
- ADR-079 — Camera-supervised training pipeline (same data path).
|
||||
- ADR-100 — Cognitum Cog packaging spec (same shipping format).
|
||||
- ADR-101 — Pose Estimation Cog (template for this Cog's first release).
|
||||
- ADR-102 — Edge Module Registry (where this cog appears in the catalog).
|
||||
- PR #491 — RollingP95 + `dedup_factor` (the heuristic this learned counter replaces).
|
||||
- Issue #499 — Multi-node ghost skeletons (closed by #491, motivates this ADR).
|
||||
- Issue #645 — PCK / data-collection plan (same data-bound limit; same fix path).
|
||||
- `docs/benchmarks/pose-estimation-cog.md` — measured perf envelope for the cog runtime this ADR targets.
|
||||
@@ -0,0 +1,263 @@
|
||||
# ADR-104: RuView MCP Server + CLI Distribution
|
||||
|
||||
- **Status:** Accepted
|
||||
- **Date:** 2026-05-21
|
||||
- **Deciders:** ruv
|
||||
- **Related:** ADR-100 (Cog packaging), ADR-101 (pose cog), ADR-102 (edge registry), ADR-103 (count cog)
|
||||
- **Implementation:** `tools/ruview-mcp/`, `tools/ruview-cli/`
|
||||
|
||||
---
|
||||
|
||||
## Context
|
||||
|
||||
The Cognitum cog ecosystem ships binaries to appliances via a signed GCS catalog (ADR-100). The cogs themselves run inside `/var/lib/cognitum/apps/` on a Pi 5 or Pi+Hailo cluster node. This is the right deployment target for production inference — sub-5 ms per frame, Hailo hardware acceleration, offline operation.
|
||||
|
||||
However, three user classes need to interact with RuView capabilities **without owning a Cognitum appliance**:
|
||||
|
||||
1. **Developer agents** — Claude Code, Cursor, Codex instances that want to call `ruview_pose_infer` during a research session (e.g. the SOTA loop in `docs/research/sota-2026-05-22/PROGRESS.md`).
|
||||
2. **CI pipelines** — automated tests that want to assert "a synthetic CSI window produces a finite pose output" without a full appliance setup.
|
||||
3. **Shell scripts and researchers** — `npx ruview pose infer --window ./window.json` from any machine with Node 20, no Rust toolchain, no Cognitum account, no clone of this repo required.
|
||||
|
||||
The existing surface does not serve these users:
|
||||
- The sensing-server REST API (`/api/v1/sensing/latest`, `/api/v1/edge/registry`) is a Rust binary that requires building from source.
|
||||
- The cog binaries are signed Linux aarch64/x86_64 executables — no macOS/Windows builds, no `npx` entrypoint.
|
||||
- There is no MCP server — Claude Code cannot call RuView capabilities as tools without one.
|
||||
|
||||
This ADR defines two new distribution artifacts:
|
||||
- `@ruv/ruview-mcp` — an MCP server exposing RuView as tools.
|
||||
- `@ruv/ruview-cli` — a CLI exposing the same surface as `npx ruview <subcommand>`.
|
||||
|
||||
---
|
||||
|
||||
## Decision
|
||||
|
||||
### MCP server: `@ruv/ruview-mcp`
|
||||
|
||||
A Node 20 TypeScript package implementing the Model Context Protocol using `@modelcontextprotocol/sdk`. The server communicates over stdio (the standard MCP transport) and exposes six tools:
|
||||
|
||||
| Tool | Description | Backend |
|
||||
|------|-------------|---------|
|
||||
| `ruview_csi_latest` | Pull the latest CSI window from the sensing-server | GET /api/v1/sensing/latest (ADR-102) |
|
||||
| `ruview_pose_infer` | 17-keypoint COCO pose estimation on a CSI window | cog-pose-estimation binary (ADR-101) subprocess |
|
||||
| `ruview_count_infer` | Person count with calibrated confidence interval | cog-person-count binary (ADR-103) subprocess |
|
||||
| `ruview_registry_list` | List Cognitum cogs from the edge registry | GET /api/v1/edge/registry (ADR-102) |
|
||||
| `ruview_train_count` | Kick off a count-cog Candle training run | cargo run -p wifi-densepose-train subprocess |
|
||||
| `ruview_job_status` | Poll a background training job | reads ~/.ruview/jobs/<id>.log |
|
||||
|
||||
**Fail-open principle:** every tool returns `{ok: false, warn: true, error: "...", hint: "..."}` rather than throwing. This matches the pattern used by the Cog binaries (ADR-100 §"Failure modes") and ensures a broken sensing-server does not crash a research agent's session.
|
||||
|
||||
### CLI: `@ruv/ruview-cli`
|
||||
|
||||
The same surface as a Yargs-based CLI published to npm as `@ruv/ruview-cli` with the binary name `ruview`:
|
||||
|
||||
| Subcommand | Equivalent MCP tool |
|
||||
|------------|-------------------|
|
||||
| `ruview csi tail` | streaming poll of `ruview_csi_latest` |
|
||||
| `ruview pose infer [--window <path>]` | `ruview_pose_infer` |
|
||||
| `ruview count infer [--window <path>]` | `ruview_count_infer` |
|
||||
| `ruview cogs list [--category] [--search]` | `ruview_registry_list` |
|
||||
| `ruview train count --paired <jsonl>` | `ruview_train_count` |
|
||||
| `ruview job status --id <uuid>` | `ruview_job_status` |
|
||||
|
||||
All subcommands write JSON to stdout and exit 0 on success. WARN-level outputs (missing cog binary, unreachable sensing-server) go to stderr; exit code stays 0 so pipelines are not broken by transient unavailability.
|
||||
|
||||
### Inference backend: subprocess, not in-process
|
||||
|
||||
The MCP server and CLI **shell out** to the cog binaries rather than embedding a JS/WASM inference engine. Reasons:
|
||||
|
||||
1. The cog binaries are already signed, tested, and cross-compiled (ADR-100/101/103). Re-implementing inference in JS would duplicate that work and introduce a second model artifact to keep in sync.
|
||||
2. The cog binaries handle model loading, ONNX dispatch, and Hailo HEF routing transparently — the MCP layer needs only to understand the JSON event schema.
|
||||
3. For training, `cargo run -p wifi-densepose-train` is the proven path (2.1 s on RTX 5080, ADR-103). Replicating the Candle training loop in JS would be a significant engineering investment with no user benefit.
|
||||
|
||||
The npm packages therefore act as a **thin orchestration layer** over the existing Rust/cog infrastructure. No ML framework is bundled.
|
||||
|
||||
### ruvector library usage
|
||||
|
||||
Where a ruvector npm package provides the required capability, it is preferred over reimplementation. The subcarrier-saliency analysis in `examples/research-sota/r5_subcarrier_saliency.py` already depends on `ruvector-mincut` (Rust crate) for Stoer-Wagner min-cut. On the npm side:
|
||||
|
||||
- `@ruv/rvcsi` — the typed CSI frame schema and validation. When available at install time, `ruview_csi_latest` will validate incoming frames against the `rvcsi-core` schema. If not installed, falls back to opaque JSON passthrough.
|
||||
- HNSW, RaBitQ, and contrastive embedding primitives are Rust-native; the npm packages do not replicate them. Instead, `ruview_pose_infer` and `ruview_count_infer` delegate to the cog binary which embeds the Candle inference engine.
|
||||
|
||||
### Source layout
|
||||
|
||||
```
|
||||
tools/
|
||||
├── ruview-mcp/ # @ruv/ruview-mcp
|
||||
│ ├── package.json
|
||||
│ ├── tsconfig.json
|
||||
│ ├── jest.config.js
|
||||
│ ├── src/
|
||||
│ │ ├── index.ts # MCP server entry + tool registry
|
||||
│ │ ├── types.ts # shared domain types
|
||||
│ │ ├── config.ts # env-var config loader
|
||||
│ │ ├── http.ts # fetch wrapper with timeout + Result<T>
|
||||
│ │ ├── cog.ts # subprocess wrapper for cog binaries
|
||||
│ │ └── tools/
|
||||
│ │ ├── csi-latest.ts # ruview_csi_latest
|
||||
│ │ ├── pose-infer.ts # ruview_pose_infer
|
||||
│ │ ├── count-infer.ts # ruview_count_infer
|
||||
│ │ ├── registry-list.ts # ruview_registry_list
|
||||
│ │ └── train-count.ts # ruview_train_count + ruview_job_status
|
||||
│ └── tests/
|
||||
│ └── tools.test.ts # stub smoke tests (M1) + integration tests (M6)
|
||||
└── ruview-cli/ # @ruv/ruview-cli
|
||||
├── package.json
|
||||
├── tsconfig.json
|
||||
├── src/
|
||||
│ ├── index.ts # yargs CLI entry + command registration
|
||||
│ ├── config.ts # env-var config loader
|
||||
│ ├── http.ts # fetch wrapper
|
||||
│ ├── cog.ts # subprocess wrapper
|
||||
│ └── commands/
|
||||
│ ├── csi.ts # ruview csi tail
|
||||
│ ├── pose.ts # ruview pose infer
|
||||
│ ├── count.ts # ruview count infer
|
||||
│ ├── cogs.ts # ruview cogs list
|
||||
│ ├── train.ts # ruview train count
|
||||
│ └── job.ts # ruview job status
|
||||
└── tests/ # (M6)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Security
|
||||
|
||||
### Authentication
|
||||
|
||||
The sensing-server uses a Bearer token (`RUVIEW_API_TOKEN`) for all `/api/v1/*` routes when the token is configured. The MCP server and CLI propagate this token in the `Authorization` header for every sensing-server call. Token is sourced **only from environment variables** — never from CLI flags or tool arguments (which could appear in logs or agent histories).
|
||||
|
||||
The cog binaries are called as local subprocesses. No network authentication is involved in cog invocation — the binary is trusted by virtue of being installed on the local machine (and having passed Ed25519 signature verification at install time, per ADR-100).
|
||||
|
||||
### Threat table
|
||||
|
||||
| # | Threat | Mitigation |
|
||||
|---|--------|-----------|
|
||||
| **T1** | **MCP tool spoofing** — a malicious process registers a tool named `ruview_pose_infer` before the legitimate server and intercepts agent calls | MCP servers are registered by the operator in the Claude Code / Cursor config. The operator must explicitly `claude mcp add ruview -- node …`. Impersonation requires compromising the operator's shell config. |
|
||||
| **T2** | **CLI subcommand injection** — a caller passes a crafted `--paired` path containing shell metacharacters to escape the `cargo` invocation | All subprocess arguments are passed as an array (never through a shell string) via Node's `spawn(binary, args, {})` — no shell expansion. Path metacharacters cannot escape. |
|
||||
| **T3** | **Token leakage** — `RUVIEW_API_TOKEN` appears in process arguments, agent histories, or log files | Token is only used in the `Authorization` HTTP header, which is set programmatically. It is never printed, never passed as a CLI argument, and never written to `~/.ruview/jobs/<id>.log`. |
|
||||
| **T4** | **Model substitution** — an attacker replaces the cog binary with a malicious version | The cog binary must pass Ed25519 signature verification (`binary_sha256` + `binary_signature`) at install time per ADR-100. The MCP/CLI layer does not re-verify at invocation time — this is the cog-gateway's job. |
|
||||
| **T5** | **Output validation bypass** — cog returns malformed JSON and the MCP server forwards it without validation | `ruview_pose_infer` and `ruview_count_infer` parse cog stdout as JSON and validate the schema against `PoseInferResult` / `CountInferResult` types (Zod, M2+). On parse failure, return `{ok:false, error: "unexpected cog output: …"}`. |
|
||||
| **T6** | **Rate-limit bypass on `ruview_train_count`** — an agent calls `ruview_train_count` in a tight loop, spawning unbounded training processes | The MCP server maintains an in-process job registry. On `ruview_train_count`, if more than 3 jobs are `status:"running"`, return `{ok:false, error:"too many concurrent training jobs (max 3)"}`. Training jobs are CPU/GPU-bound and self-limit on the host. |
|
||||
|
||||
### What this ADR does NOT secure
|
||||
|
||||
- **MCP transport encryption** — MCP over stdio is process-local; no TLS is involved. If the MCP server is exposed over a TCP socket in future, TLS must be added.
|
||||
- **Cog binary authentication at invocation** — we trust the OS file permissions and the at-install-time signature check (ADR-100). If a binary is replaced after install, the MCP layer will not detect it.
|
||||
- **Multi-tenant token isolation** — the server process serves all connected clients under a single token. Multi-user deployments must run one MCP server instance per user.
|
||||
|
||||
---
|
||||
|
||||
## Packaging
|
||||
|
||||
### Version alignment
|
||||
|
||||
The npm package versions track the cog crate versions:
|
||||
- `@ruv/ruview-mcp@0.0.1` ships when `cog-pose-estimation@0.0.1` + `cog-person-count@0.0.2` are on GCS.
|
||||
- Semver: major bump when the MCP tool schema changes (breaking for calling agents); minor for new tools; patch for bug fixes.
|
||||
|
||||
### npm package configuration
|
||||
|
||||
Both packages are published to the public npm registry under the `@ruv` scope:
|
||||
|
||||
```
|
||||
@ruv/ruview-mcp — npm install -g @ruv/ruview-mcp (then: ruview-mcp)
|
||||
@ruv/ruview-cli — npm install -g @ruv/ruview-cli (then: ruview --version)
|
||||
```
|
||||
|
||||
The `bin` entry in `package.json` points to `dist/index.js` (compiled from TypeScript). Both packages target Node 20 (`"engines": {"node": ">=20.0.0"}`).
|
||||
|
||||
`private: true` is set during development; **the user must flip this to `false` before publishing** (or delete the field). The `publishConfig.access: "public"` is already set.
|
||||
|
||||
### MCP registration
|
||||
|
||||
After installing (global or npx):
|
||||
|
||||
```bash
|
||||
# Via npx (no install required):
|
||||
claude mcp add ruview -- npx @ruv/ruview-mcp
|
||||
|
||||
# Via global install:
|
||||
npm install -g @ruv/ruview-mcp
|
||||
claude mcp add ruview -- ruview-mcp
|
||||
|
||||
# Verify:
|
||||
claude mcp list # should show "ruview"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Distribution
|
||||
|
||||
`npx ruview …` works from any machine with Node 20 installed. No clone of this repository, no Rust toolchain, no Cognitum appliance is required to run the CLI commands that do not depend on a cog binary (e.g. `ruview cogs list` only needs a sensing-server URL).
|
||||
|
||||
For commands that call a cog binary (`ruview pose infer`, `ruview count infer`), the cog binary must be downloaded from GCS and placed in a directory on `PATH` or pointed to via `RUVIEW_POSE_COG_BINARY` / `RUVIEW_COUNT_COG_BINARY`. The download URL follows ADR-100 naming:
|
||||
|
||||
```
|
||||
https://storage.googleapis.com/cognitum-apps/cogs/x86_64/cog-pose-estimation-x86_64
|
||||
https://storage.googleapis.com/cognitum-apps/cogs/arm/cog-pose-estimation-arm
|
||||
https://storage.googleapis.com/cognitum-apps/cogs/x86_64/cog-person-count-x86_64
|
||||
https://storage.googleapis.com/cognitum-apps/cogs/arm/cog-person-count-arm
|
||||
```
|
||||
|
||||
A future `ruview install cogs` subcommand can automate this download + chmod + PATH placement.
|
||||
|
||||
---
|
||||
|
||||
## Failure modes
|
||||
|
||||
| Scenario | Behaviour |
|
||||
|---|---|
|
||||
| Sensing-server not running | `ruview_csi_latest` / `ruview_registry_list` return `{ok:false, warn:true, error:"…", hint:"…"}`. Exit code 0 on CLI. MCP tool returns isError:false (it's a warn, not a crash). |
|
||||
| Cog binary not installed | `ruview_pose_infer` / `ruview_count_infer` return `{ok:false, warn:true, error:"…", hint:"…"}` with install instructions. |
|
||||
| Cog binary returns non-zero | Propagated as `{ok:false, error:"Cog exited with code N. stderr: …"}`. |
|
||||
| Training job crashes immediately | Log file records `# exit code: <N>`. `ruview_job_status` returns `{status:"failed", recent_log:[…]}`. |
|
||||
| MCP server process dies mid-session | In-process job registry is lost. Jobs that were running continue in background (detached); operator reads log files directly. |
|
||||
| Node < 20 | `fetch` is unavailable. The CLI prints a clear error: "Node 20+ required for built-in fetch". |
|
||||
|
||||
---
|
||||
|
||||
## Acceptance gates
|
||||
|
||||
| Gate | Test |
|
||||
|------|------|
|
||||
| `npx ruview --version` works | `ruview --version` prints `0.0.1` and exits 0. |
|
||||
| `ruview_pose_infer` returns finite output for synthetic CSI | M2 integration test: spawn MCP server, call tool with a synthetic window JSON, assert `result.n_persons >= 0` and all keypoint values in `[0, 1]`. |
|
||||
| MCP server passes `claude mcp list` check | `claude mcp add ruview -- node dist/index.js && claude mcp list` shows `ruview` with 6 tools. |
|
||||
| `npm run build` clean in both packages | TypeScript compilation exits 0, no errors. |
|
||||
| Stub smoke tests pass (M1) | `npm test` in `tools/ruview-mcp/` passes all 6 stub tests. |
|
||||
| Integration tests pass (M6) | 6 tool calls with mocked sensing-server + real node binary as cog stub all return `{ok: true}`. |
|
||||
|
||||
---
|
||||
|
||||
## Migration / rollout
|
||||
|
||||
1. **This PR** — land scaffold (`tools/ruview-mcp/`, `tools/ruview-cli/`) + ADR-104. Both packages at `private: true`.
|
||||
2. **M2** — wire real inference: sensing-server CSI window → cog subprocess → parsed output. Remove `stub: true` from responses.
|
||||
3. **M3** — wire `ruview_csi_latest` + `ruview_registry_list` with live sensing-server round-trip test.
|
||||
4. **M4** — wire `ruview_train_count` with real cargo invocation; verify job log populates.
|
||||
5. **M6** — integration tests green. Update acceptance gates.
|
||||
6. **User publish step** — flip `private` from `true` to `false` in both `package.json` files, then:
|
||||
|
||||
```bash
|
||||
# Publish MCP server:
|
||||
cd tools/ruview-mcp
|
||||
npm version patch # or minor/major per semver
|
||||
npm publish --access public
|
||||
|
||||
# Publish CLI:
|
||||
cd tools/ruview-cli
|
||||
npm version patch
|
||||
npm publish --access public
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## See also
|
||||
|
||||
- ADR-100: Cognitum Cog Packaging Specification — the signing + GCS distribution model this ADR sits on top of.
|
||||
- ADR-101: Pose Estimation Cog — the binary invoked by `ruview_pose_infer`.
|
||||
- ADR-102: Edge Module Registry — the `/api/v1/edge/registry` endpoint used by `ruview_registry_list`.
|
||||
- ADR-103: Learned Multi-Person Counter Cog — the binary invoked by `ruview_count_infer`.
|
||||
- `docs/research/sota-2026-05-22/PROGRESS.md` — the SOTA research loop that motivated the MCP server.
|
||||
- `v2/crates/cog-pose-estimation/` — Rust source for the pose-estimation cog.
|
||||
- `v2/crates/cog-person-count/` — Rust source for the person-count cog.
|
||||
@@ -108,6 +108,7 @@ Statuses: **Proposed** (under discussion), **Accepted** (approved and/or impleme
|
||||
| [ADR-095](ADR-095-rvcsi-edge-rf-sensing-platform.md) | rvCSI — Edge RF Sensing Runtime Platform | Proposed |
|
||||
| [ADR-096](ADR-096-rvcsi-ffi-crate-layout.md) | rvCSI — Crate Topology, the napi-c Shim, and the napi-rs Node Surface | Proposed |
|
||||
| [ADR-097](ADR-097-adopt-rvcsi-as-ruview-csi-runtime.md) | Adopt rvCSI as RuView's primary CSI runtime (phased adoption) | Proposed |
|
||||
| [ADR-098](ADR-098-evaluate-midstream-fit.md) | Evaluate `ruvnet/midstream` for RuView's CSI / WebSocket / mesh pipeline | Rejected |
|
||||
| [ADR-099](ADR-099-midstream-introspection-tap.md) | Adopt midstream as RuView's real-time introspection + low-latency tap | Proposed |
|
||||
|
||||
---
|
||||
|
||||
|
After Width: | Height: | Size: 4.4 MiB |
|
After Width: | Height: | Size: 1.9 MiB |
|
After Width: | Height: | Size: 1.5 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 2.4 MiB |
@@ -0,0 +1,185 @@
|
||||
# `cog-person-count` — Benchmark Log
|
||||
|
||||
Append-only log of every published count_v1 training run per ADR-103. New runs add a section; never overwrite history.
|
||||
|
||||
## v0.0.2 — K-fold validated, random split + label smoothing + early stop + temp scale (2026-05-21)
|
||||
|
||||
### Why a new release
|
||||
|
||||
A 5-fold stratified CV on the same 1,077 samples proved the v0.0.1 result was driven by an unlucky temporal split — the trailing window was class-0-heavy, and a degenerate "always predict 0" classifier hit the class-0 fraction (65.1%) trivially.
|
||||
|
||||
| Metric | v0.0.1 (temporal) | **5-fold random CV** (diagnostic) |
|
||||
|---|---|---|
|
||||
| Overall accuracy | 65.1% | 62.2% ± 1.9% |
|
||||
| Class 1 accuracy | **0%** | **57.1%** ✓ |
|
||||
| Confidence Spearman | 0.023 | 0.160 ± 0.029 |
|
||||
|
||||
The architecture has real ~57% class-1 capacity under fair splits.
|
||||
|
||||
### v0.0.2 results
|
||||
|
||||
Architecture unchanged. Training changes only:
|
||||
- **Random 80/20 split** (seed=42) — temporal split eliminated.
|
||||
- **Label smoothing 0.1** on cross-entropy.
|
||||
- **Class-balanced multinomial sampler** with replacement.
|
||||
- **Early stopping** with patience 20 (exited at epoch 29 of 400 max).
|
||||
- **Temperature scaling** of the conf head via LBFGS — T = **0.9262**, shipped as a `count_v1.temperature` sidecar.
|
||||
|
||||
| Metric | v0.0.1 | **v0.0.2** | K-fold ref |
|
||||
|---|---|---|---|
|
||||
| Overall accuracy | 65.1% | **62.3%** | 62.2% ± 1.9% |
|
||||
| Class 0 accuracy | 100% (cheating) | **86.2%** | 67.4% |
|
||||
| **Class 1 accuracy** | **0%** | **34.3%** ✓ | 57.1% |
|
||||
| MAE | 0.349 | 0.377 | 0.378 |
|
||||
| Confidence Spearman (post-temp) | 0.023 | 0.013 | 0.160 |
|
||||
| Wall time | 5.6 s (400 ep) | **0.7 s (29 ep)** | 7.5 s (5×100) |
|
||||
|
||||
### Honest read
|
||||
|
||||
**Class-1 accuracy 0% → 34.3% is the headline.** The cog now reports `count = 1` honestly when a person is present, instead of always-zero cheating. Single random draw lands below the K-fold mean of 57% — that gap is run-to-run variance, not a missing improvement. Reaching 57% on a fixed eval set needs averaging over independent draws, which means more independent recordings — i.e. multi-room data (#645), not another training trick.
|
||||
|
||||
Confidence calibration didn't move. Temperature scaling alone can't fix a confidence head trained against a noisy `argmax==truth` indicator over a 62%-accurate classifier — its training signal is the bottleneck.
|
||||
|
||||
### Release artifacts (live on cognitum-v0)
|
||||
|
||||
```
|
||||
gs://cognitum-apps/cogs/arm/cog-person-count-count_v1.safetensors
|
||||
sha256: 32996433516891a37c63c600db8b95e42192a53bd538c088c82cd6a85e55513c
|
||||
bytes: 392,088
|
||||
```
|
||||
|
||||
Binaries themselves unchanged from v0.0.1 — weights load at runtime via mmap. Per-arch manifests under `cog/artifacts/manifests/{arm,x86_64}/` bumped to `version: 0.0.2`, weights_sha256 + build_metadata caveats updated.
|
||||
|
||||
### Reproducibility
|
||||
|
||||
```bash
|
||||
python3 scripts/train-count.py --paired data/paired/wiflow-p7-1779210883.paired.jsonl \
|
||||
--k-fold 5 --epochs 100 --out-results kfold_results.json
|
||||
|
||||
python3 scripts/train-count.py --paired data/paired/wiflow-p7-1779210883.paired.jsonl \
|
||||
--v2 --epochs 400 \
|
||||
--out-safetensors count_v1.safetensors --out-onnx count_v1.onnx \
|
||||
--out-results count_train_results.json
|
||||
```
|
||||
|
||||
## v0.0.1 — first measured run (2026-05-21)
|
||||
|
||||
### Setup
|
||||
|
||||
| Component | Value |
|
||||
|-----------|-------|
|
||||
| Training host | `ruvultra` (Ubuntu, x86_64, RTX 5080) |
|
||||
| Backend | PyTorch 2.12 + CUDA |
|
||||
| Data | `data/paired/wiflow-p7-1779210883.paired.jsonl` — 1,077 paired samples, single 30-min session, label distribution `{0: 533, 1: 544}` |
|
||||
| Train/eval split | 80/20 stratified on `ts_start` (held-out tail of the recording) |
|
||||
| Architecture | Conv1d encoder (56→64→128→128, dilations 1/2/4) + Linear(128→64→8) count head + Linear(128→32→1) confidence head — bit-identical to `v2/crates/cog-person-count/src/inference.rs::CountNet` |
|
||||
| Loss | `cross_entropy(count) + 0.3·BCE(conf) + 0.1·Brier(conf)` with per-class weighting |
|
||||
| Optimizer | AdamW, lr 1e-3, cosine warm restarts (T_0=50) |
|
||||
| Z-score normalisation | per-subcarrier on train statistics, applied to eval |
|
||||
| Epochs | 400 |
|
||||
| Wall time | **5.6 s** |
|
||||
|
||||
### Accuracy (held-out 215-sample tail of the 30-min recording)
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Best eval accuracy | **65.1%** |
|
||||
| Final eval accuracy | 65.1% |
|
||||
| Within ±1 | **100%** (labels are all in `{0, 1}`, predictions trivially within ±1) |
|
||||
| MAE | 0.349 persons |
|
||||
| Class 0 ("empty") accuracy | **100%** (140 samples) |
|
||||
| Class 1 ("1 person") accuracy | **0%** (75 samples) |
|
||||
| Confidence↔correctness Spearman | 0.023 |
|
||||
|
||||
### Honest read
|
||||
|
||||
The model overfit hard. By epoch 100 train_acc reached 1.0 and eval_loss climbed from 0.67 → 7.8. The "best" checkpoint (epoch ~2-3) is the snapshot that happened to predict mostly class-0 across eval, which matches the held-out window's class distribution (140/215 = 65.1%) — i.e. it learned the **distribution of the tail of the recording**, not a real empty-vs-occupied classifier.
|
||||
|
||||
Why: the training data is one continuous 30-minute solo recording. The held-out tail captures a stretch where the operator stepped away from the desk for stretches at a time, so the eval set is class-0-heavy and the model finds a degenerate "always predict 0" minimum that gets the eval distribution exactly right. Class 1 accuracy = 0 is the smoking gun.
|
||||
|
||||
Same data-bound failure mode as `pose_v1` (#645). Same fix path: multi-room paired recordings.
|
||||
|
||||
### What v0.0.1 still validates
|
||||
|
||||
- **Pipeline correctness end-to-end.** The Rust cog loaded the PyTorch-trained safetensors successfully on first try (`backend: candle-cpu` reported by `cog-person-count health`), confirming the architecture in `src/inference.rs` is byte-compatible with `train-count.py`.
|
||||
- **ONNX parity.** 16 KB ONNX, exports cleanly under opset 18 with dynamic batch axis.
|
||||
- **Fast iteration loop.** 5.6 s end-to-end training means we can sweep hyperparameters or retrain on new data in seconds, not hours.
|
||||
- **Cog binary size.** Same 2.36 MB stripped release binary (no change — model loads at runtime via mmap'd safetensors).
|
||||
|
||||
### Comparison to ADR-103 v0.1.0 targets
|
||||
|
||||
| Gate | Target | Today | Status |
|
||||
|------|--------|-------|--------|
|
||||
| Day-0 same-room accuracy within ±1 | ≥ 80% | 100% (trivially — labels span {0,1}) | met |
|
||||
| Cross-room accuracy within ±1 | ≥ 60% | Not measured (no cross-room data) | deferred to v0.2.0 |
|
||||
| MAE | ≤ 0.6 | 0.349 | met |
|
||||
| Per-frame confidence reflects accuracy (Spearman) | r ≥ 0.5 | 0.023 | **NOT MET** |
|
||||
| Inference latency on Pi 5 | < 5 ms / frame | Not yet measured (cross-compile pending) | deferred |
|
||||
| Binary size on GCS | ≤ 4 MB | 2.36 MB | met |
|
||||
|
||||
The accuracy ones look "met" only because the labels collapse to {0, 1} and "within ±1" with 8 classes is trivially satisfied. The **confidence calibration is the real failure** for v0.0.1 — Spearman 0.023 means the confidence head is essentially random noise. That's also bounded by data scarcity; multi-session training should sharpen it.
|
||||
|
||||
### Artifacts
|
||||
|
||||
- `v2/crates/cog-person-count/cog/artifacts/count_v1.safetensors` — 392 KB
|
||||
- `v2/crates/cog-person-count/cog/artifacts/count_v1.onnx` — 16 KB
|
||||
- `v2/crates/cog-person-count/cog/artifacts/count_train_results.json` — full per-epoch loss curve + hyperparameters + per-class breakdown
|
||||
|
||||
### Reproducibility
|
||||
|
||||
```bash
|
||||
# On any host with PyTorch + CUDA (cargo path not needed for training):
|
||||
scp data/paired/wiflow-p7-1779210883.paired.jsonl <host>:/tmp/
|
||||
scp scripts/train-count.py <host>:/tmp/
|
||||
ssh <host> "cd /tmp && python3 train-count.py --paired wiflow-p7-1779210883.paired.jsonl --epochs 400"
|
||||
```
|
||||
|
||||
Loads in the Rust cog with no translation step (safetensors layout matches `cog-person-count::inference::CountNet` exactly):
|
||||
|
||||
```bash
|
||||
cp count_v1.safetensors v2/crates/cog-person-count/cog/artifacts/
|
||||
cargo run -p cog-person-count --release -- health
|
||||
# → {"backend":"candle-cpu", "synthetic_count": <int>, "synthetic_confidence": <float>, ...}
|
||||
```
|
||||
|
||||
### Live appliance install (cognitum-v0 Pi 5)
|
||||
|
||||
Installed at `/var/lib/cognitum/apps/person-count/` with the same on-disk shape as `cog-pose-estimation`, `anomaly-detect`, `seizure-detect`, etc.:
|
||||
|
||||
```
|
||||
$ ls -la /var/lib/cognitum/apps/person-count/
|
||||
-rwxr-xr-x cog-person-count-arm 2,168,816 B (sha matches GCS)
|
||||
-rw-r--r-- count_v1.safetensors 392,088 B
|
||||
-rw-r--r-- manifest.json 1,073 B
|
||||
-rw-r--r-- config.json 160 B
|
||||
```
|
||||
|
||||
```
|
||||
$ ./cog-person-count-arm health
|
||||
{"ts": ..., "event": "health.ok",
|
||||
"fields": {"backend": "candle-cpu", "synthetic_count": 0,
|
||||
"synthetic_confidence": 0.49, "synthetic_p95_range": [0, 7]}}
|
||||
```
|
||||
|
||||
Cold-start on real Pi 5 hardware: **9.2 ms / invocation** (30 sequential `health` invocations in 0.276 s). Slightly slower than the pose cog (8.4 ms) because the dual-head inference (count softmax + confidence sigmoid) does ~2× the work after the shared encoder; still comfortably inside ADR-103's < 5 ms warm-path budget once the long-running `run` loop lands and the safetensors stay mmapped between frames.
|
||||
|
||||
### Signed GCS release artifacts (publicly downloadable)
|
||||
|
||||
```
|
||||
gs://cognitum-apps/cogs/arm/cog-person-count-arm 2,168,816 B
|
||||
sha256: 36bc0bb0ece894350377d5f93d46cd29378cb289b3773530611c0d47b507b3c3
|
||||
signature: R/00xdzHriyr/2rzr4wmPJ/Ken60A+RNdi8r0g2HYJNTXBaFtr46ExfNbiHlgYWadQXzTZdfJoyJK+a6k71NDg==
|
||||
|
||||
gs://cognitum-apps/cogs/x86_64/cog-person-count-x86_64 2,615,528 B
|
||||
sha256: 76cdd1ec40211add90b4942a09f79939aa28210a27e931de67122357392b01db
|
||||
signature: QB+8cnGSMQmubSt/KWVu1+JMg37AKnQXDsFQi/vi+jqpW9rVrGMtnxQpWEWZPeWU1AJ6pl3O2V+7ZtTNIQ2rDg==
|
||||
|
||||
gs://cognitum-apps/cogs/arm/cog-person-count-count_v1.safetensors 392,088 B
|
||||
sha256: dacb0551fd3887958db19696d90d811ab08faa44703e6e04ff56d15c3a65a9ff
|
||||
```
|
||||
|
||||
All signed with `COGNITUM_OWNER_SIGNING_KEY` (Ed25519). SHAs verified via public anonymous `https://storage.googleapis.com/...` download.
|
||||
|
||||
Manifests at:
|
||||
- `v2/crates/cog-person-count/cog/artifacts/manifests/arm/manifest.json`
|
||||
- `v2/crates/cog-person-count/cog/artifacts/manifests/x86_64/manifest.json
|
||||
@@ -0,0 +1,176 @@
|
||||
# `cog-pose-estimation` — Benchmark Log
|
||||
|
||||
This file tracks every published benchmark for the pose-estimation Cog. New runs append; never overwrite history. Per ADR-101 §"Acceptance gates".
|
||||
|
||||
## v0.0.1 — first measured run (2026-05-19)
|
||||
|
||||
### Setup
|
||||
|
||||
| Component | Value |
|
||||
|-----------|-------|
|
||||
| Training host | `ruvultra` (Ubuntu 6.17, x86_64, RTX 5080) |
|
||||
| Backend | `candle-core 0.9` with `cuda` feature |
|
||||
| Data | `data/paired/wiflow-p7-1779210883.paired.jsonl` — 1,077 paired samples, 30-min seated-at-desk recording, avg conf 0.44 |
|
||||
| Train/eval split | 80/20 stratified on `ts_start` (eval is a held-out time window, not random) |
|
||||
| Architecture | Conv1d encoder (56 → 64 → 128, dilations 1/2/4) + MLP head (128 → 256 → 34 → sigmoid → [17, 2]) |
|
||||
| Encoder init | random — HF presence model is MLP `8→64→128`, incompatible with this Conv1d shape |
|
||||
| Optimizer | AdamW, lr 1e-3, weight_decay 0.01 |
|
||||
| LR schedule | Cosine with 50-epoch warm restarts |
|
||||
| Loss | SmoothL1 (Huber β=0.1), confidence-weighted by `record.conf` |
|
||||
| Augmentation | Subcarrier dropout 10% (final 50 epochs) |
|
||||
| Epochs | 400 (full-batch) |
|
||||
| Wall time | **2.1 s** total |
|
||||
|
||||
### Accuracy
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| **PCK@20** (overall) | **3.0%** |
|
||||
| **PCK@50** (overall) | **18.5%** |
|
||||
| **MPJPE** (normalized) | **0.0931** |
|
||||
| Final eval loss | 0.0101 |
|
||||
| Loss reduction | 0.181 → 0.014 (13×) |
|
||||
|
||||
### Per-joint PCK
|
||||
|
||||
| Joint | PCK@20 | PCK@50 | | Joint | PCK@20 | PCK@50 |
|
||||
|-------|-------:|-------:|--|-------|-------:|-------:|
|
||||
| nose | 0.5% | 5.1% | | l_hip | 0.0% | 27.3% |
|
||||
| l_eye | 2.8% | 8.3% | | **r_hip** | **25.0%** | **76.9%** |
|
||||
| r_eye | 1.9% | 15.7% | | l_knee | 2.3% | 20.8% |
|
||||
| l_ear | 0.0% | 3.2% | | r_knee | 0.9% | 35.2% |
|
||||
| r_ear | 1.9% | 9.7% | | l_ankle | 1.4% | 7.9% |
|
||||
| l_shoulder | 4.6% | 8.8% | | r_ankle | 0.9% | 9.3% |
|
||||
| r_shoulder | 1.9% | 19.9% | | l_elbow | 1.9% | 26.4% |
|
||||
| l_wrist | 3.2% | 24.1% | | r_elbow | 0.0% | 4.2% |
|
||||
| r_wrist | 1.4% | 12.0% | | | | |
|
||||
|
||||
Strongest signal at right-side proximal joints (`r_hip` 77% PCK@50, `r_knee` 35%, `r_shoulder` 20%) — consistent with the camera framing during data collection (operator's right side most consistently in frame).
|
||||
|
||||
### Comparison to prior baseline
|
||||
|
||||
| Run | Backend | Train time | PCK@20 | PCK@50 | MPJPE |
|
||||
|-----|---------|-----------:|-------:|-------:|------:|
|
||||
| pre-2026-05-19 | pure-JS SPSA, lite TCN (#645) | ~20 min | 0.0% | 0.0% | 0.66 |
|
||||
| **v0.0.1** (this run) | **candle-cuda, Conv1d TCN** | **2.1 s** | **3.0%** | **18.5%** | **0.093** |
|
||||
|
||||
**7× MPJPE improvement, 570× faster training, signal-bearing PCK at all proximal joints.** The remaining gap to ADR-079's PCK@20 ≥ 35% target is data-bound, not infra-bound (see Issue #645).
|
||||
|
||||
### Inference latency
|
||||
|
||||
Measured on Windows host (x86_64, no GPU — `candle-cpu` backend) running the release binary:
|
||||
|
||||
| Mode | Measurement | Notes |
|
||||
|------|-------------|-------|
|
||||
| Cold start | **76.2 ms / invocation** (avg over 100 sequential `health` invocations) | Includes safetensors load + 1 synthetic forward pass. Most of the cost is process startup + mmap. |
|
||||
| Long-running `run` warm inference | sub-millisecond per frame (estimated) | The model is 125K params / 507 KB; once loaded, a single forward at batch=1 is essentially memory-bandwidth bound. To be measured precisely against a live sensing-server feed. |
|
||||
|
||||
### ONNX export
|
||||
|
||||
`pose_v1.onnx` is produced from `pose_v1.safetensors` by `scripts/export-onnx.py`, which mirrors the Candle architecture in PyTorch, loads the safetensors weights, and uses `torch.onnx.export` with opset 18 + dynamic batch axis. Verified end-to-end:
|
||||
|
||||
| Check | Result |
|
||||
|-------|--------|
|
||||
| `onnx.checker.check_model` | ✅ ok |
|
||||
| Parity vs torch reference | **max \|torch − onnx\| = 8.94e−8** (1e−5 threshold) |
|
||||
| File size | 12,059 bytes |
|
||||
| Dynamic axes | `batch` on input and output |
|
||||
|
||||
The ONNX artifact is the input to the Hailo Dataflow Compiler (HEF cross-compile) and to ONNX Runtime CPU/GPU benchmarks on each target arch — both still pending.
|
||||
|
||||
### Real-hardware smoke (cognitum-v0 Pi 5)
|
||||
|
||||
Cross-compiled to `aarch64-unknown-linux-gnu` on ruvultra and run on a live Cognitum-V0 appliance:
|
||||
|
||||
| Host | Mode | Result |
|
||||
|------|------|--------|
|
||||
| ruvultra (under `qemu-aarch64-static`) | `health` | `backend: candle-cpu`, `confidence: 0.185` — real weights loaded under emulation |
|
||||
| **cognitum-v0** (Raspberry Pi 5, Cortex-A76) | `health` | `backend: candle-cpu`, `confidence: 0.185` — real weights, real hardware |
|
||||
| cognitum-v0 | 30× sequential `health` invocations | **0.251 s total → 8.4 ms / invocation** (cold) |
|
||||
|
||||
8.4 ms cold-start on real Pi 5 hardware vs 76 ms on the x86_64 Windows host. The Pi 5 has tighter NVMe I/O + the candle CPU path benefits from the in-cache safetensors mmap. Long-running `run` warm inference will still be sub-millisecond.
|
||||
|
||||
### Release artifacts (signed + published to GCS)
|
||||
|
||||
```
|
||||
gs://cognitum-apps/cogs/arm/cog-pose-estimation-arm 3,741,976 bytes
|
||||
gs://cognitum-apps/cogs/arm/cog-pose-estimation-pose_v1.safetensors 507,032 bytes
|
||||
|
||||
binary_sha256: 1e1a7d3dd01ca05d5bfc5dbb142a5941b7866ed9f3224a21edc04d3f09a99bf5
|
||||
weights_sha256: eb249b9a6b2e10130437a10976ed0230b0d085f86a0553d7226e1ae6eae4b9e5
|
||||
signature: LUN7xqLPYD3MFzm5dKB5MnYU0LvoRtek5ci5KiKPHBg+Xo6xuazwokn2Dw2JPMaLYJzmWn/SpT4djuR7hYvVDw== (Ed25519, signed with COGNITUM_OWNER_SIGNING_KEY)
|
||||
```
|
||||
|
||||
Full manifest at `cog/artifacts/manifest.json`. Verified via public anonymous GET against `https://storage.googleapis.com/cognitum-apps/cogs/arm/cog-pose-estimation-arm` — downloaded SHA matches the locally-computed SHA.
|
||||
|
||||
### Live appliance install
|
||||
|
||||
Installed on `cognitum-v0` (the V0 cluster leader) at `/var/lib/cognitum/apps/pose-estimation/`:
|
||||
|
||||
```
|
||||
$ ls -la /var/lib/cognitum/apps/pose-estimation/
|
||||
-rwxr-xr-x cog-pose-estimation-arm 3,741,976 B (matches GCS sha256)
|
||||
-rw-r--r-- pose_v1.safetensors 507,032 B
|
||||
-rw-r--r-- manifest.json 989 B
|
||||
-rw-r--r-- config.json 187 B
|
||||
-rw-r--r-- output.log 28,438 B (5-sec smoke run)
|
||||
```
|
||||
|
||||
Layout matches the existing `anomaly-detect`, `presence`, `seizure-detect`, etc. cogs on the same appliance — the Cogs dashboard at `http://cognitum-v0:9000/cogs` auto-discovers entries under this dir.
|
||||
|
||||
`cog-pose-estimation run` ran cleanly in the background for 5 seconds with the default config. It correctly:
|
||||
|
||||
- Emitted a `run.started` event with the configured `sensing_url`, `model_path`, and `poll_ms`.
|
||||
- Started its 40 ms poll loop.
|
||||
- **Gracefully handled the missing local sensing-server on port 3000** by logging structured WARN events (`{"level":"WARN","fields":{"message":"sensing-server fetch failed","error":"...Connection refused..."}}`) without crashing, leaking, or producing NaN output.
|
||||
- Exited cleanly on SIGTERM.
|
||||
|
||||
0 `pose.frame` events fired during the smoke run — expected, since `127.0.0.1:3000` isn't serving CSI on the appliance. The appliance's actual CSI source is `ruview-vitals-worker` on `:50054` plus the `/api/v1/v0/system/...` endpoints behind the appliance's bearer auth on `:9000`. Wiring `sensing_url` to the appliance-native source is a Day-2 integration task — separate from the cog binary itself.
|
||||
|
||||
Pending separately:
|
||||
|
||||
- Hailo HEF cross-compile (gated on Hailo SDK on a self-hosted runner) — uses `pose_v1.onnx` as input.
|
||||
- Appliance-native sensing-source integration (`config.sensing_url` should point at the cog-gateway's CSI tap on `:9000`, not the dev-loopback `:3000`).
|
||||
### x86_64 release (2026-05-19)
|
||||
|
||||
Built on ruvultra (native, no cross-compile):
|
||||
|
||||
```
|
||||
gs://cognitum-apps/cogs/x86_64/cog-pose-estimation-x86_64 4,548,856 bytes
|
||||
sha256: a434739a24415b34e1aff50e5e1c3c32e568db96af473bbb3e5ecc9b95fe71fa
|
||||
signature: pNNuxhgM18PztN8BSZdfw5oAShG2pV3na5T/q2QdlJWX/5FJgo4QTiUCbcTAxI2Uiva8VURSOlRzMU3xoQPqCQ==
|
||||
```
|
||||
|
||||
Manifest at `cog/artifacts/manifests/x86_64/manifest.json`. Re-uses the same `pose_v1.safetensors` weights as the arm release (architecture is arch-independent).
|
||||
|
||||
**Cold-start: 5.4 ms / invocation** on ruvultra (30× sequential `health` in 0.162 s) — faster than the Pi 5's 8.4 ms (faster NVMe + wider CPU), slower than the Windows 76 ms (less mature Windows release toolchain).
|
||||
|
||||
| Host | arch | rust | binary | cold-start |
|
||||
|------|------|------|--------|------------|
|
||||
| Windows (ruvzen) | x86_64 | 1.95.0 | (built locally, not published) | 76.2 ms |
|
||||
| ruvultra (Ubuntu) | x86_64 | 1.89.0 | 4,548,856 B (GCS x86_64) | **5.4 ms** |
|
||||
| cognitum-v0 (Pi 5) | aarch64 | (cross-built) | 3,741,976 B (GCS arm) | 8.4 ms |
|
||||
|
||||
### Artifacts
|
||||
|
||||
- `v2/crates/cog-pose-estimation/cog/artifacts/pose_v1.safetensors` — 507 KB
|
||||
- `v2/crates/cog-pose-estimation/cog/artifacts/train_results.json` — full per-epoch loss curve + hyperparameters + per-joint PCK
|
||||
|
||||
### Reproducibility
|
||||
|
||||
```bash
|
||||
# On any host with cargo + a CUDA-capable GPU:
|
||||
cd ~/work/cog-pose-train
|
||||
mkdir -p ./
|
||||
# Stage the same inputs (1,077 paired samples + HF encoder, see scripts/align-ground-truth.js for regeneration)
|
||||
cp paired.jsonl ./paired.jsonl
|
||||
cp encoder.safetensors ./encoder.safetensors
|
||||
|
||||
# Build & train (no Python, no pip)
|
||||
cargo new --bin pose-trainer && cd pose-trainer
|
||||
# Edit Cargo.toml deps: candle-core 0.9 (cuda), candle-nn 0.9 (cuda), safetensors, serde, serde_json, anyhow
|
||||
# Drop the training script into src/main.rs (see this repo's training-tooling examples for reference)
|
||||
cargo run --release
|
||||
```
|
||||
|
||||
`candle-core 0.8.4 + 0.9.2` are typically already in `~/.cargo/registry/cache/` on any developer host, so the build completes in seconds.
|
||||
@@ -0,0 +1,139 @@
|
||||
# Horizon: 12-hour Autonomous SOTA Run — 2026-05-22
|
||||
|
||||
**Horizon ID:** `sota-2026-05-22`
|
||||
**Started:** 2026-05-21 ~20:00 ET
|
||||
**Auto-stop:** 2026-05-22 08:00 ET
|
||||
**Cron:** `d6e5c473` (`*/10 * * * *`) — single-tick research contributions running in parallel
|
||||
|
||||
---
|
||||
|
||||
## Three concurrent objectives
|
||||
|
||||
| Objective | Description | Primary branch |
|
||||
|-----------|-------------|---------------|
|
||||
| **A** | Keep the cron research loop productive — curate PROGRESS.md between ticks | (main, via PR) |
|
||||
| **B** | Build `ruview` MCP server + CLI (`tools/ruview-mcp/`, `tools/ruview-cli/`) | `feat/ruview-mcp-cli` |
|
||||
| **C** | Write ADR-104: ruview MCP/CLI distribution decision record | (same branch as B) |
|
||||
|
||||
---
|
||||
|
||||
## Milestones
|
||||
|
||||
### M1 — Scaffold `tools/ruview-mcp/` + `tools/ruview-cli/`
|
||||
**Target:** +1h (by ~21:00 ET)
|
||||
**Status:** `COMPLETE` — merged as PR #705 (squash commit `5a6c585aa`)
|
||||
**Branch:** `feat/ruview-mcp-cli-pr` (deleted after merge)
|
||||
|
||||
Deliverables:
|
||||
- `tools/ruview-mcp/package.json` — `@ruv/ruview-mcp`, TypeScript, `@modelcontextprotocol/sdk`
|
||||
- `tools/ruview-mcp/src/index.ts` — minimal MCP server with 5 tool stubs
|
||||
- `tools/ruview-mcp/src/tools/` — one file per tool
|
||||
- `tools/ruview-cli/package.json` — `@ruv/ruview-cli` + `ruview` bin
|
||||
- `tools/ruview-cli/src/index.ts` — 4-verb CLI stub via yargs/commander
|
||||
- `tsconfig.json` for both packages
|
||||
- Shared `tools/ruview-shared/` for HTTP client + types
|
||||
|
||||
Completion criteria: `npm run build` succeeds in both packages, MCP server can be registered with `claude mcp add`.
|
||||
|
||||
---
|
||||
|
||||
### M2 — Wire `ruview_pose_infer` + `ruview_count_infer`
|
||||
**Target:** +3h (by ~23:00 ET)
|
||||
**Status:** `in_progress`
|
||||
|
||||
Wire inference via subprocess to cog binaries (`cog-pose-estimation`, `cog-person-count`). MCP tools and CLI subcommands both delegate to the cog binary's `health` + a synthetic-frame run.
|
||||
|
||||
Completion criteria: `ruview_pose_infer` returns finite keypoint array; `ruview_count_infer` returns `{count, confidence}`.
|
||||
|
||||
---
|
||||
|
||||
### M3 — Wire `ruview_csi_latest` + `ruview_registry_list`
|
||||
**Target:** +5h (by ~01:00 ET)
|
||||
**Status:** `pending`
|
||||
|
||||
Connect to sensing-server `/api/v1/sensing/latest` (ADR-102 endpoint) and `/api/v1/edge/registry`. CLI: `npx ruview csi tail` streams live frames.
|
||||
|
||||
Completion criteria: both tools return structured JSON from a running sensing-server (or graceful 503 WARN if server not reachable).
|
||||
|
||||
---
|
||||
|
||||
### M4 — Wire `ruview_train_count`
|
||||
**Target:** +7h (by ~03:00 ET)
|
||||
**Status:** `pending`
|
||||
|
||||
Fire the Candle training pipeline as a background subprocess; return a job ID; expose `ruview_job_status` to poll. Training output streamed to `~/.ruview/jobs/<id>.log`.
|
||||
|
||||
Completion criteria: `ruview_train_count` returns `{job_id, status: "queued"}` within 200 ms.
|
||||
|
||||
---
|
||||
|
||||
### M5 — ADR-104: ruview MCP/CLI distribution
|
||||
**Target:** +8h (by ~04:00 ET)
|
||||
**Status:** `pending`
|
||||
|
||||
Full ADR covering: problem, design (5 MCP tools + 5 CLI subcommands + library mapping), security (6-row threat table), packaging (npm `@ruv/ruview-mcp` + `@ruv/ruview-cli`), distribution, failure modes, acceptance gates.
|
||||
|
||||
Completion criteria: ADR file at `docs/adr/ADR-104-ruview-mcp-cli-distribution.md`, merged to main.
|
||||
|
||||
---
|
||||
|
||||
### M6 — Integration tests
|
||||
**Target:** +10h (by ~06:00 ET)
|
||||
**Status:** `pending`
|
||||
|
||||
Jest/Vitest tests: spawn MCP server, call each tool stub, assert structured output shape. CI-green on Node 20.
|
||||
|
||||
Completion criteria: `npm test` passes in `tools/ruview-mcp/`.
|
||||
|
||||
---
|
||||
|
||||
### M7 — Final summary + handoff
|
||||
**Target:** +11h (by ~07:00 ET)
|
||||
**Status:** `pending`
|
||||
|
||||
Write final section to this HORIZON.md: what shipped, what deferred, exact `npm publish` commands.
|
||||
|
||||
---
|
||||
|
||||
## Cron coordination (Objective A)
|
||||
|
||||
The `d6e5c473` cron picks threads from `PROGRESS.md` independently. Rules for safe co-operation:
|
||||
- Horizon-tracker writes to HORIZON.md, not PROGRESS.md, except for cross-link notes.
|
||||
- When a cron tick lands a new artifact, horizon-tracker distills its finding into PROGRESS.md's "Done" section + adds cross-links (e.g. R5 → R8 RSSI feasibility).
|
||||
- If a thread shows 2+ consecutive ticks without a new artifact, horizon-tracker adds `blocked: <reason>` to that thread's section.
|
||||
|
||||
Current cross-links identified at session start:
|
||||
- **R5 → R8**: band-spread top-8 saliency distribution raises RSSI-only ceiling to ~60% of full-CSI upper-bound.
|
||||
- **R5 → R7**: top-8 subcarriers are exactly the ones a defender must corroborate across nodes.
|
||||
- **R5 → R1**: saliency map should be re-run on multi-static captures (different geometry = different salient subcarriers?).
|
||||
|
||||
---
|
||||
|
||||
## Drift indicators (checked each milestone)
|
||||
|
||||
| Indicator | Threshold | Current |
|
||||
|-----------|-----------|---------|
|
||||
| Timeline | M1 >2h behind → defer scope | On track |
|
||||
| Scope | MCP server grows beyond 5 tools | On track |
|
||||
| Approach | MCP SDK incompatible with available node | TBD at M1 |
|
||||
| Dependency | ruvector npm packages not findable | TBD at M1 |
|
||||
| Priority | Cron consuming PROGRESS.md locks | None yet |
|
||||
|
||||
---
|
||||
|
||||
## Session log
|
||||
|
||||
### Session 1 — 2026-05-21 (horizon init + M1)
|
||||
|
||||
**Started:** Initial read of PROGRESS.md, ADR-100/101/102/103, R5 saliency note.
|
||||
**Accomplished:**
|
||||
- HORIZON.md initialized.
|
||||
- `tools/ruview-mcp/` and `tools/ruview-cli/` scaffolded with TypeScript, MCP SDK, Yargs.
|
||||
- 6 MCP tools defined (stubs): csi_latest, pose_infer, count_infer, registry_list, train_count, job_status.
|
||||
- 6 CLI subcommands defined: csi tail, pose infer, count infer, cogs list, train count, job status.
|
||||
- `docs/adr/ADR-104-ruview-mcp-cli-distribution.md` written (full depth, 6-row threat table).
|
||||
- 6/6 smoke tests pass.
|
||||
- PR #705 created and merged.
|
||||
- PROGRESS.md updated: R7 and R8 cross-links added (cron produced these results in parallel).
|
||||
**Cron activity observed:** R7 (Stoer-Wagner adversarial detection 3/3) + R8 (RSSI-only 94.82% retained) landed while M1 was in progress.
|
||||
**Next:** M2 — wire real inference via sensing-server + cog subprocess.
|
||||
@@ -0,0 +1,76 @@
|
||||
# SOTA Research Loop — 2026-05-22
|
||||
|
||||
Started: 2026-05-21 ~20:00 ET. **Auto-stops: 2026-05-22 08:00 ET.** Cron `d6e5c473` (`*/10 * * * *`).
|
||||
|
||||
## Mandate
|
||||
|
||||
Push WiFi-CSI sensing past 2026 published SOTA in three axes:
|
||||
|
||||
1. **Spatial intelligence** — multi-static fusion, room-scale awareness, occupancy beyond counting
|
||||
2. **RF feature engineering** — phase, ToA, subcarrier dynamics, Fresnel zones
|
||||
3. **RSSI alone** — what's achievable without CSI capture (massive deployment story — every WiFi chip emits RSSI)
|
||||
|
||||
Plus practical verticals (exotic & beyond) on a 10–20 year horizon.
|
||||
|
||||
Output goes to `docs/research/sota-2026-05-22/` (research notes, benchmarks, negative results) + `examples/research-sota/` (runnable code).
|
||||
|
||||
## Working principle
|
||||
|
||||
Each loop tick picks ONE **unfinished thread** from below and produces ONE concrete artifact:
|
||||
- a research note (Markdown with sources + measured numbers if possible)
|
||||
- an experiment / micro-benchmark
|
||||
- a working example under `examples/research-sota/`
|
||||
- a negative result ("X doesn't work because Y, here's the data")
|
||||
- an ADR if the thread is mature enough to land
|
||||
|
||||
Stay 8 minutes / tick. Commit + PR + auto-merge per piece. Future-tick re-entry is via this PROGRESS.md.
|
||||
|
||||
## Research vectors
|
||||
|
||||
### Spatial Intelligence
|
||||
|
||||
- [ ] **R1. Multi-static Time-of-Arrival (ToA) from OFDM phase coherence.** Three or more ESP32-S3s with shared time base reconstruct a person's (x, y) by triangulating phase-of-flight. 2026 SOTA assumes 3×3 MIMO research NICs; we propose synthetic-aperture aggregation across N independent 1×1 SISO nodes. Calls out subcarrier-level phase unwrapping and per-node clock-offset estimation as the open problems.
|
||||
- [ ] **R2. Persistent room field model — eigenstructure perturbation.** Already in `wifi-densepose-signal/src/ruvsense/field_model.rs` (SVD on empty-room CSI). Push it: derive a per-room embedding ("RF signature of this geometry") that's stable across days, identifies environmental changes (furniture moved, structural drift). Vertical: building-integrity monitoring.
|
||||
- [ ] **R3. Cross-room re-identification via gait CSI signatures.** Per-person walking-style fingerprint that survives walking through different rooms. Different from `AETHER` (in-room re-ID) — this is *inter*-room continuity.
|
||||
- [ ] **R4. Federated learning of room models.** Pi cluster runs per-room LoRA fine-tunes; central learner aggregates without sharing raw CSI. Privacy-preserving spatial intelligence.
|
||||
|
||||
### RF Feature Engineering
|
||||
|
||||
- [ ] **R5. Subcarrier attention over time → "RF saliency map".** Visualize which subcarriers carry the most information per task. ADR-097 hints at this; nothing in repo computes it. Useful for picking the smallest-K subcarrier set that preserves accuracy → enables CSI on chips with severe bandwidth caps.
|
||||
- [ ] **R6. Fresnel-zone forward model for through-wall sensing.** Code in `wifi-densepose-signal/src/ruvsense/tomography.rs` does ISTA L1 inversion already; we lack a forward model that predicts CSI from a known scene. Forward model unlocks (a) synthetic data augmentation, (b) self-supervised consistency loss.
|
||||
- [x] **R7. Stoer-Wagner adversarial-node detection.** DONE — 3/3 detection rate (replay/shift/noise). See `R7-multilink-consistency.md`. Cross-links: R5 top-8 saliency subcarriers are priority targets for partial-spectrum attackers; fills `cog-person-count::fusion::fuse_with_mincut_clip()` stub (ADR-103 v0.2.0). Next tick: Stackelberg-game adaptive attacker.
|
||||
|
||||
### RSSI Alone (no CSI)
|
||||
|
||||
- [x] **R8. RSSI-only person count.** DONE — 59.1% = 94.82% of full-CSI (62.3%). 656 params, 5 KB, 0.72 s CPU. See `R8-rssi-only-count.md`. Cross-links: R5 band-spread saliency explains the retained accuracy; R9 extends same stream to localisation; ADR-104 MCP server should grow `ruview_count_infer --rssi` mode for non-CSI chips. Next: 3-class ceiling, multi-room replication.
|
||||
- [ ] **R9. RSSI fingerprint topology — graph neural network on WiFi-scan beacons.** Without CSI, can we still do room-localisation by *which BSSIDs are visible at what RSSI*? Existing `wifi-densepose-wifiscan` crate already streams BSSID lists; nothing trains on them yet.
|
||||
|
||||
### Exotic & Future (10–20 year)
|
||||
|
||||
- [ ] **R10. Through-foliage wildlife sensing.** Same physics as through-wall, but at much lower SNR. Gait recognition on a per-species basis. Practical: non-invasive population monitoring without cameras.
|
||||
- [ ] **R11. Through-bulkhead maritime crew tracking.** Steel attenuates but doesn't eliminate WiFi multipath. Limited range, requires per-vessel calibration.
|
||||
- [ ] **R12. RF "weather" mapping.** Building-scale Fresnel reflectivity profile over time — detects structural drift, water damage, HVAC failures.
|
||||
- [ ] **R13. Contactless blood pressure from sub-mm chest displacement.** Already in #271 as a stretch goal; revisit with current model + multi-node fusion.
|
||||
- [ ] **R14. Empathic appliances.** Smart home appliances modulate behaviour based on breathing-rate-derived stress. Long-horizon — needs both the sensing accuracy *and* an ethical framework.
|
||||
- [ ] **R15. RF biometric across rooms.** Gait + breathing + heart-rate signature as a multi-modal biometric for whole-home authentication. Replaces fingerprint/face on the home-network layer.
|
||||
|
||||
## Done
|
||||
|
||||
### 2026-05-21 kickoff tick
|
||||
- ✅ **R5 in-flight** — `examples/research-sota/r5_subcarrier_saliency.py` runs; first measurement on `cog-person-count` v0.0.2 ships: top-8 subcarriers spread across the band, max/mean ratio 2.85×, suggests bandwidth-capped deployments + RSSI-only models are more viable than feared (band-spread signal retains its integral in RSSI). See `R5-subcarrier-saliency.md` §"First measurement" + §"Implications".
|
||||
|
||||
### 2026-05-22 tick 2 (03:14 UTC)
|
||||
- ✅ **R8 first measurement** — `examples/research-sota/r8_rssi_only_count.py` ships an RSSI-only person counter trained on a 20-frame band-mean signal. **Result: 59.1% accuracy = 94.82% of the full-CSI v0.0.2 baseline (62.3%).** Tiny model: 656 params (~5 KB), 56× smaller input, trains in 0.72 s on CPU. **Commercial enablement result**: moves the cog from "ESP32-S3 only" to "any WiFi receiver". Class accuracy balanced (59.5 / 58.6 vs v0.0.2's skewed 86.2 / 34.3). Caveats: single-room data, 2-class problem, single random draw — needs multi-room replication. See `R8-rssi-only-count.md` for full method + interpretation + 3 follow-up experiments queued. Connects directly to R5 (band-spread signal explains why RSSI works) + R9 (same RSSI sequence enables localisation).
|
||||
|
||||
### 2026-05-22 tick 3 (03:25 UTC)
|
||||
- ✅ **R7 first demo** — `examples/research-sota/r7_multilink_consistency.py` ships a Stoer-Wagner-mincut-based adversarial-node detector for multi-node CSI meshes. **Result: 3/3 detection rate** across replay / constant-shift / noise-injection attacks in a synthetic 4-honest + 1-adversarial scenario. Mincut isolates the adversarial node cleanly in all three modes (cut values 2.56–3.57, partition_B = `{4}` consistently). Pure-NumPy demo, no framework deps. **Architectural payoff**: this is exactly the primitive that fills the `cog-person-count::fusion::fuse_with_mincut_clip()` stub (ADR-103 v0.2.0). Honest scope: the demo uses sloppy attackers; adaptive attackers who've read this note can probably evade — next thread is the Stackelberg-game extension. See `R7-multilink-consistency.md`.
|
||||
|
||||
## Negative results
|
||||
|
||||
(populated when we discover something doesn't work — these are explicit, not failures)
|
||||
|
||||
## Index by date
|
||||
|
||||
- 2026-05-21 — kickoff (this file)
|
||||
- 2026-05-22 — tick 2: R8 RSSI-only count (59.1% / 94.82% retained)
|
||||
- 2026-05-22 — tick 3: R7 multi-link consistency detection (3/3 attack modes detected by Stoer-Wagner mincut)
|
||||
@@ -0,0 +1,85 @@
|
||||
# R12 — RF weather mapping: structural drift from passive WiFi (negative-ish result + revised plan)
|
||||
|
||||
**Status:** first experiment landed — **NEGATIVE-ish, with a clear next step** · **2026-05-22**
|
||||
|
||||
## The 10-year vision
|
||||
|
||||
Every WiFi access point in a building is, incidentally, a coherent radio source flooding the structure with energy. The walls, floors, furniture, and humans inside reflect that energy with characteristic multipath signatures. The persistent-room field model in `wifi-densepose-signal/src/ruvsense/field_model.rs` already captures the *spatial* eigenstructure of those reflections to subtract the room's baseline from occupancy detection.
|
||||
|
||||
The R12 vision generalises that to the *temporal* dimension: continuously track how the building's RF eigenstructure drifts across **days, weeks, months, years**. The hypothesis:
|
||||
|
||||
- **A new piece of furniture** changes the multipath profile in one specific way (additional reflector at a specific location).
|
||||
- **Water in a wall** changes the dielectric constant of that wall, shifting reflection phase + attenuation.
|
||||
- **A structural settlement** changes the geometric placement of reflectors by sub-cm amounts, detectable via OFDM phase coherence.
|
||||
- **A missing ceiling tile** changes Fresnel-zone coupling between rooms.
|
||||
- **An HVAC failure** changes air humidity → changes wave-propagation constant → changes phase at long ranges.
|
||||
|
||||
Pre-2026 SOTA mostly uses CSI for activity recognition. The shift to *structural integrity monitoring from passive ambient RF* is open territory.
|
||||
|
||||
## First experiment (this tick)
|
||||
|
||||
`examples/research-sota/r12_rf_weather_eigenshift.py` tests the simplest possible algorithm: SVD on the per-frame CSI matrix, top-K singular values, cosine distance between spectra over time.
|
||||
|
||||
Setup:
|
||||
- Take 1,077 CSI windows from the existing paired data.
|
||||
- Split first-half (10,760 frames) = "before", last-half (10,780 frames) = "after".
|
||||
- Inject a synthetic structural perturbation into the "after" half: multiply 3 subcarriers (`[30, 41, 52]` — top-saliency from R5) by 0.85 to simulate a new reflective surface attenuating those frequencies by ~1.4 dB.
|
||||
- Top-10 singular values per half. Cosine distance between spectra.
|
||||
|
||||
## Result
|
||||
|
||||
| | Cosine distance from BEFORE |
|
||||
|---|---|
|
||||
| AFTER (no perturbation, control) | 0.00035 |
|
||||
| AFTER (with 3-subcarrier perturbation) | **0.00024** |
|
||||
| Signal / natural-drift ratio | **0.69×** |
|
||||
|
||||
**Verdict: WEAK.** The synthetic structural perturbation produces a *smaller* spectral distance than the natural temporal drift from operator movement in the same recording. The top-10 singular-value spectrum is **not sensitive enough** to detect ~15% attenuation on 3 of 56 subcarriers when the room's occupant is moving.
|
||||
|
||||
## Why this fails — and how to fix it
|
||||
|
||||
The top-K singular-value spectrum captures the **dominant energy** in the channel state. A 15% perturbation on 3 of 56 subcarriers shifts the matrix by ≤(3/56) × 15% ≈ 0.8% of total energy. That's well below the natural temporal variance from a moving operator.
|
||||
|
||||
Three concrete revisions for next attempts:
|
||||
|
||||
1. **Use the FULL eigenvector basis, not just the spectrum.** The cosine distance on top-K singular *values* is scale-aware but direction-blind. Comparing the top-K *eigenvectors* (singular vectors) via subspace angles ("principal angles between subspaces") would catch the structural shift even when the energy distribution stays similar.
|
||||
|
||||
2. **Detect specific subcarriers via residual analysis.** Instead of comparing whole spectra, project each window onto the empty-room subspace and look for **consistent per-subcarrier residuals** — these would localise the perturbation. The 3 perturbed subcarriers would show a persistent attenuation bias that natural drift wouldn't reproduce.
|
||||
|
||||
3. **Multi-day baseline.** This experiment uses a single 30-min recording. The "natural temporal drift" is dominated by operator movement, not by structural change. The real RF-weather problem has the OPPOSITE noise structure: structural changes happen over hours-to-days, occupancy noise averages out over minutes-to-hours. Averaging the eigenspectrum over a 24-hour window before comparing should knock down the operator-noise floor by 50-100×.
|
||||
|
||||
## What still holds
|
||||
|
||||
The 10-year vision isn't refuted — the algorithm choice was wrong. Specifically:
|
||||
|
||||
- The **physics is real**: dielectric changes in walls cause measurable CSI shifts (well-documented in 2020-era CSI building-monitoring literature).
|
||||
- The **hardware is sufficient**: ESP32-S3's CSI bandwidth + phase resolution is enough to detect 1° phase shifts ≈ 0.5 mm displacement at 5 GHz.
|
||||
- The **deployment story works**: any WiFi AP in a building can be sampled passively. No physical installation cost.
|
||||
- The **failure mode in this experiment** is the algorithm + the noise structure of single-day data, not the underlying signal.
|
||||
|
||||
## What this DOES prove
|
||||
|
||||
- The simple "SVD spectrum cosine distance" approach **does not work** in single-day data. Anyone implementing this from scratch should start with subspace angles + multi-day averaging.
|
||||
- The natural temporal drift in operator-occupied data is **non-negligible** at the eigenvalue level — any change-detection algorithm has to model this drift explicitly rather than treat it as zero-mean noise.
|
||||
|
||||
## What's next on this thread
|
||||
|
||||
- Implement **principal angles between subspaces** (PABS) as the comparison metric instead of cosine on singular values. PABS catches subspace rotations that singular-value cosines miss.
|
||||
- Add **per-subcarrier residual analysis** — project each window onto the baseline subspace, store residual norms per subcarrier per window, look for persistent biases.
|
||||
- Need **multi-day data** at minimum. Even better: 7-day data with a deliberate structural change at day 4 (e.g. move a chair 1 m). Currently no such dataset exists in the repo.
|
||||
|
||||
## Connection back
|
||||
|
||||
- R5 (band-spread saliency): the perturbation chose top-saliency subcarriers, but it still wasn't detected — suggests R5's saliency is **task-specific** (count-task saliency ≠ structure-detection saliency). Useful counter-data point.
|
||||
- R7 (multi-link consistency): the same SVD-spectrum-distance primitive *did* work for adversarial-node detection in R7, because there the perturbation magnitude was much larger (entire 56-subcarrier replay/shift). Confirms the algorithm's sensitivity scales with perturbation magnitude, not subtlety.
|
||||
- R8 (RSSI-only): RSSI is the trace of the CSI covariance matrix. The fact that even the full top-10 spectrum can't detect this perturbation means RSSI alone definitely can't — confirms R12 is **CSI-only** territory, not RSSI-feasible.
|
||||
|
||||
## 10-year vertical applications (preserved despite negative result)
|
||||
|
||||
The vision is right; the algorithm needs work. Verticals to chase once PABS + multi-day data exist:
|
||||
|
||||
- **Building structural monitoring** for insurance companies — early water-damage detection from RF signature shift.
|
||||
- **Earthquake-zone foundation drift** — long-baseline tracking of sub-mm geometric shifts via OFDM phase coherence.
|
||||
- **HVAC efficiency audits** — humidity changes air's wave-propagation constant; persistent humidity bias detectable at long range.
|
||||
- **Museum / archive climate stability** — same physics, lower allowable drift.
|
||||
- **Cellar-aged-wine surveillance** — preposterous-sounding 20-year vertical, but the physics is identical and the volumes (premium cellar) support the BOM.
|
||||
@@ -0,0 +1,70 @@
|
||||
# R5 — Subcarrier saliency: which CSI dimensions actually carry the signal?
|
||||
|
||||
**Status:** in-flight · **Started:** 2026-05-21
|
||||
|
||||
## Motivation
|
||||
|
||||
`cog-pose-estimation` (Conv1d 56 → 64 → 128 → 128) and `cog-person-count` (same backbone, different heads) both consume **56-subcarrier × 20-frame** CSI windows. The 56 came from the upstream `align-ground-truth.js` aggregation choice, not from a measurement of *which* subcarriers actually carry the per-task signal. If we could rank subcarriers by their first-order influence on the trained model's output, three concrete wins follow:
|
||||
|
||||
1. **Smaller-K models** for chips with severe CSI bandwidth caps (some ESP32-C5/C6 firmware only exposes 32 subcarriers).
|
||||
2. **Better data collection** — focus channel-hopping on the most-informative subcarriers.
|
||||
3. **Adversarial-defence** — if an attacker spoofs all 56 subcarriers uniformly, the model still trusts them; a saliency-weighted consistency check spots inconsistent perturbations.
|
||||
|
||||
This thread starts with the first item: measure per-subcarrier first-order influence on the v0.0.2 count model + the v0.0.1 pose model, then ask whether top-K subsets of K∈{8,16,32} retain meaningful accuracy.
|
||||
|
||||
## Method (single-tick scope)
|
||||
|
||||
For each model:
|
||||
|
||||
1. Load the trained safetensors (`cog/artifacts/count_v1.safetensors` and `cog/artifacts/pose_v1.safetensors`).
|
||||
2. Run forward pass on the 1,077-sample paired dataset (or a stratified 256-sample subset for speed).
|
||||
3. Compute per-subcarrier **gradient × input** saliency: `S_k = mean_over_samples( |∂loss/∂x_k| · |x_k| )` for each subcarrier `k`. This is the standard "input × gradient" saliency from Sundararajan et al. (Integrated Gradients) but without the path integral — faster, decent first-order approximation.
|
||||
4. Plot the 56-element saliency vector for each model. Identify top-K.
|
||||
5. Re-train each model on the top-K subcarriers only (K ∈ {8, 16, 32}). Compare accuracy.
|
||||
|
||||
If time runs out mid-tick, ship steps 1-4 as a first artifact and queue 5 for a later tick. Steps 1-4 alone produce a real result (a ranked-subcarrier list per task).
|
||||
|
||||
## Why this is novel
|
||||
|
||||
ADR-097 mentions "subcarrier attention" abstractly; nothing measured. Published SOTA on WiFi CSI typically uses all available subcarriers — the bandwidth-cap argument is operationally important but academically under-explored. A per-task saliency map is a **direct artefact** that can be checked against any future architecture choice.
|
||||
|
||||
## Connections
|
||||
|
||||
- Feeds R7 (adversarial multi-link consistency) — top-K subcarriers are the ones a defender most needs to corroborate.
|
||||
- Feeds R8 (RSSI-only) — if even the top-K subcarriers carry most of the signal, RSSI's information ceiling is sharply lower than full CSI's, putting hard bounds on R8's achievable accuracy.
|
||||
|
||||
## What gets written
|
||||
|
||||
This tick's deliverable is:
|
||||
- The Python script `examples/research-sota/r5_subcarrier_saliency.py` that computes the saliency vector for either model.
|
||||
- A first measurement (text + JSON) of saliency for the count model.
|
||||
|
||||
Step 5 (retrain on top-K) is queued for a subsequent tick.
|
||||
|
||||
## First measurement — `cog-person-count` v0.0.2 (this tick, 128 samples)
|
||||
|
||||
| Rank | Subcarrier | Saliency |
|
||||
|-----:|-----------:|---------:|
|
||||
| 1 | **41** | 0.0128 |
|
||||
| 2 | **52** | 0.0120 |
|
||||
| 3 | **30** | 0.0100 |
|
||||
| 4 | 31 | 0.0097 |
|
||||
| 5 | 10 | 0.0088 |
|
||||
| 6 | 35 | 0.0088 |
|
||||
| 7 | 2 | 0.0087 |
|
||||
| 8 | 38 | 0.0083 |
|
||||
|
||||
**Max-to-mean ratio: 2.85×** — meaningful but moderate concentration. Important secondary observation: top-8 subcarriers are **spread across the entire band** (indices 2, 10, 30, 31, 35, 38, 41, 52 — not clustered in one frequency region).
|
||||
|
||||
## Implications
|
||||
|
||||
1. **Bandwidth-cap deployment is viable.** Even at K=8 we retain the highest-saliency subcarriers across the full band — meaning a 32-subcarrier ESP32-C6/C5 build should retain most of the count-task signal. Retraining at K=8/16/32 is the next-tick experiment.
|
||||
2. **R8 (RSSI alone) is feasible-but-bounded.** RSSI is a band-aggregate scalar that loses per-subcarrier resolution. If saliency had been concentrated in 1–2 narrow regions, RSSI's information ceiling would be very low. Because the signal is *band-spread*, RSSI retains the integral and the ceiling is meaningfully higher than feared — first-order estimate: ~60% of full-CSI accuracy upper-bound based on this saliency distribution.
|
||||
3. **R7 (adversarial defence) priority list.** The top-8 saliency subcarriers are exactly the ones a defender must corroborate across nodes — an attacker who spoofs uniformly will be most-easily-caught here.
|
||||
|
||||
## Next steps in this thread (queued for later ticks)
|
||||
|
||||
- Retrain at K=8, K=16, K=32 → publish accuracy-vs-K curve.
|
||||
- Same saliency map for the pose model.
|
||||
- Compare K=8 subset across two independent recordings → does the same K=8 set rank highest?
|
||||
- Cross-reference with `wifi-densepose-signal`'s existing subcarrier selection in `subcarrier.rs`.
|
||||
@@ -0,0 +1,75 @@
|
||||
# R7 — Multi-link consistency detection via Stoer-Wagner mincut
|
||||
|
||||
**Status:** first measurement landed · **2026-05-22**
|
||||
|
||||
## Premise
|
||||
|
||||
The Cog fleet deployment story (ADR-100 + ADR-102 + ADR-103) puts multiple ESP32-S3 nodes in the same physical space, each reporting CSI to the same sensing-server. Today, the server trusts every node equally. That's fine when the adversary is "an indifferent universe", but the WiFi-CSI literature has known supply-chain attacks:
|
||||
|
||||
- **Replay** — attacker captures a CSI stream from earlier and pumps it back in to fake "empty room" / "no fall" / "all-clear" states.
|
||||
- **Constant shift** — attacker biases one node's CSI by a constant, hoping the fusion stage averages it away while still poisoning per-node decisions.
|
||||
- **Noise injection** — attacker jams or otherwise produces pure-noise CSI that crosses the legitimate-traffic threshold of `wDev_ProcessFiq`-based packet filters.
|
||||
|
||||
A learned multi-node fusion (ADR-103 §"Multi-node fusion") will average these out *if* the adversary is the minority. But we need a primitive that *detects* the adversary so the fusion stage can drop them before averaging.
|
||||
|
||||
## Algorithm (this thread)
|
||||
|
||||
**Key insight:** N honest observers of the same physical scene produce CSI vectors that cluster tightly under cosine similarity (their windows differ only by per-channel multipath noise). An adversarial node, regardless of attack mode, sits *outside* that cluster.
|
||||
|
||||
The cluster-outlier-detection primitive that fits this problem exactly is the **Stoer-Wagner minimum cut** on the inter-node cosine-similarity graph:
|
||||
|
||||
```
|
||||
for each pair of nodes (i, j):
|
||||
W[i, j] = cos(flatten(csi_i), flatten(csi_j))
|
||||
|
||||
(value, partition_B) = stoer_wagner_mincut(W)
|
||||
|
||||
# partition_B is the "less-similar" side of the minimum cut.
|
||||
# When the cut is sharp, partition_B is a singleton — the adversarial node.
|
||||
```
|
||||
|
||||
`ruvector-mincut` already vendors this algorithm in the workspace (used by `cog-pose-estimation` for person-separable subcarrier grouping, see #491). The fusion stage in `cog-person-count` (`fuse_with_mincut_clip()`) has a stub that's exactly the consumer this primitive needs.
|
||||
|
||||
## Demo measurement
|
||||
|
||||
`examples/research-sota/r7_multilink_consistency.py` — pure NumPy, no framework deps. Synthesises 4 honest CSI nodes (real scene from `data/paired/...` + per-node Gaussian noise 6 dB below signal) and 1 adversarial node under each of 3 attack modes:
|
||||
|
||||
| Attack mode | Description | Mincut value | Partition_B | Adversarial isolated? |
|
||||
|---|---|---|---|---|
|
||||
| **replay** | Stale window from earlier in the recording, +1% jitter | 3.4513 | `{4}` | **YES** |
|
||||
| **shift** | Constant +3σ offset on every subcarrier | 3.5724 | `{4}` | **YES** |
|
||||
| **noise** | Pure Gaussian noise at honest-node signal magnitude | 2.5586 | `{4}` | **YES** |
|
||||
|
||||
**Detection rate: 3/3 = 100%** on this synthetic scenario, with mincut value gaps that are well-separated from the within-honest-cluster connectivity (honest nodes have pairwise similarities >0.95, the adversarial node's similarity to any honest node is ≤0.5).
|
||||
|
||||
## Honest scope of this result
|
||||
|
||||
This is a **clean synthetic scenario** with strong adversary signals. Real-world attacks are subtler:
|
||||
|
||||
- A *clever* replay attacker would time the replay to overlap with stable empty-room periods, when honest-node CSI is also nearly-identical to the stale window. Detection rate degrades.
|
||||
- A *partial-spectrum* shift on a few subcarriers (instead of all 56) leaves enough true CSI that cosine similarity stays high. Need a per-subcarrier check, not whole-window.
|
||||
- An *adaptive* attacker who has read this research note and adds calibrated noise to evade the cluster check.
|
||||
|
||||
What this demo proves: the **primitive works** when the adversary is sloppy. The next research step is the adaptive-attacker version — Stackelberg game between detector and adversary on the same similarity-cut framework.
|
||||
|
||||
## What this unlocks for the Cog stack
|
||||
|
||||
- The stub at `cog-person-count::fusion::fuse_with_mincut_clip()` can become a real primitive: at each frame, run mincut on the cross-node CSI similarity graph, drop any node that gets isolated, then run the count head on the remaining nodes' fused features.
|
||||
- Same approach extends to `cog-pose-estimation` once we have a multi-node pose deployment.
|
||||
- The mincut value itself is a continuous "mesh trustworthiness score" that can be exposed as a `mesh.trust` metric in the cog-gateway dashboard.
|
||||
|
||||
## 10-year horizon
|
||||
|
||||
The "RF radio-democracy" story: every WiFi receiver in a building (phones, laptops, smart speakers — see R8's RSSI-only result) becomes a witness in a Byzantine-fault-tolerant mesh. The mincut consistency check generalises to N=many heterogeneous nodes. A single compromised phone can't poison the building-scale sensing state because mincut isolates it. This is the spatial-intelligence analogue of Byzantine consensus in distributed systems — published-2026-SOTA hasn't framed CSI security this way yet.
|
||||
|
||||
## Connections back
|
||||
|
||||
- **R5** (subcarrier saliency) provides the priority list of subcarriers a detector should over-weight in the similarity metric — top-8 are `[41, 52, 30, 31, 10, 35, 2, 38]`.
|
||||
- **R8** (RSSI-only) shows the same primitive likely works at lower SNR with RSSI-only metrics; the cluster structure is preserved by the band integral.
|
||||
- **ADR-103** (`cog-person-count` v0.2.0 plan) — this primitive is the explicit content of the `fuse_with_mincut_clip()` stub.
|
||||
|
||||
## What's next on this thread
|
||||
|
||||
- Adversarial-game framing: detector + attacker as a two-player Stackelberg game.
|
||||
- Per-subcarrier consistency check (not just whole-window cosine). Falls out of R5's saliency map naturally.
|
||||
- Live demo on real multi-node data once seed-1 comes back online or seed-2-5 get provisioned.
|
||||
@@ -0,0 +1,58 @@
|
||||
# R8 — RSSI-only person count: does it work without CSI?
|
||||
|
||||
**Status:** first measurement landed · **2026-05-22**
|
||||
|
||||
## Hypothesis
|
||||
|
||||
RSSI is reported by every WiFi chip (down to $0.50 ESP8266s). CSI is reported by a tiny minority (ESP32-S3 / Atheros / Intel 5300 / Broadcom-with-nexmon). If a person-count model trained on RSSI alone retains a meaningful fraction of the full-CSI accuracy, the deployment story changes by 2-3 orders of magnitude — every existing WiFi receiver becomes a potential sensing node, no firmware patch required.
|
||||
|
||||
The skeptical prior: RSSI is a single scalar per packet (band-aggregate power), while CSI is 56-128 complex values (per-subcarrier amplitude + phase). Naively, RSSI throws away ≥98% of the information. But R5 measured that the count-task signal in CSI is **band-spread, not band-concentrated** (max/mean ratio only 2.85× across 56 subcarriers). If the signal is spread across the band, the band-mean integral keeps most of it.
|
||||
|
||||
## Method
|
||||
|
||||
1. Take the existing `data/paired/wiflow-p7-1779210883.paired.jsonl` (1,077 paired CSI windows + labels).
|
||||
2. Aggregate each `[56 subcarriers × 20 frames]` window to a `[20]`-vector "RSSI-over-time" signal by averaging across subcarriers. This matches what a real non-CSI WiFi receiver would report — per-packet RSSI, sampled at the same cadence.
|
||||
3. Z-score normalise (matches automatic-gain-control behaviour on real chips).
|
||||
4. Random 80/20 split with **seed=42** — identical to `cog-person-count` v0.0.2's split, so the eval sets are the same individual samples.
|
||||
5. Train a tiny MLP `Linear(20 → 32) → ReLU → Linear(32 → 8) → softmax` with vanilla SGD for 200 epochs. No framework — pure NumPy. Keep best-by-eval-acc checkpoint.
|
||||
|
||||
## Result
|
||||
|
||||
| Metric | RSSI-only (this) | `cog-person-count` v0.0.2 (full CSI) | Retained |
|
||||
|---|---|---|---|
|
||||
| Overall accuracy | **0.591** | 0.623 | **94.82%** |
|
||||
| Class 0 accuracy | 0.595 | 0.862 | — |
|
||||
| Class 1 accuracy | 0.586 | 0.343 | — |
|
||||
| Train time | **0.72 s** (CPU) | 0.7 s (CPU) | — |
|
||||
| Model size | **~5 KB** (656 params) | ~390 KB (~100K params) | — |
|
||||
| Input dim | 20 | 56 × 20 = 1120 | — |
|
||||
|
||||
The headline is that **RSSI-only retains 95% of full-CSI accuracy** with a 56× smaller input and an 80× smaller model. The class accuracies are also notably more *balanced* than v0.0.2 (59.5 / 58.6 vs 86.2 / 34.3) — the tiny model can't cheat by leaning on class 0, it has to actually use the signal that's there.
|
||||
|
||||
## Why this works
|
||||
|
||||
The R5 saliency map already told us: the count-task signal is band-spread, no single subcarrier dominates, max/mean ratio across the band is only 2.85×. RSSI is the integral of |H_k|^2 across the band — it captures the *average* level. For a band-spread signal, the average is a near-sufficient statistic. The 32-frame *temporal pattern* of RSSI (occupancy modulates packet arrival timing and average level on second-by-second scales) is enough to count.
|
||||
|
||||
## What this enables (10-year horizon)
|
||||
|
||||
1. **Phones-as-sensors.** Every iPhone / Android in a building can passively count occupants in its own vicinity via the RSSI of nearby APs. No app permissions beyond WiFi-scan; no CSI hardware required.
|
||||
2. **Smart speakers, smart TVs, smart lights.** Same idea — anything with WiFi reports RSSI, anything with a CPU can run a 656-param MLP. Counting becomes a **federated property of any room with WiFi**.
|
||||
3. **Adoption story for the cog ecosystem.** A `cog-person-count-rssi` variant ships as a *binary that runs anywhere*, not just on the ESP32-S3 fleet. Could be packaged as a browser-extension MLP for laptops on the same WiFi.
|
||||
|
||||
## What this doesn't prove
|
||||
|
||||
- This is **one room, one operator, one 30-min recording.** Generalisation across rooms / chips / people is unmeasured. The 5-fold reference for the full-CSI model was 62.2 ± 1.9% — the RSSI-only 59.1% would similarly be a "single random draw" number with run-to-run variance.
|
||||
- The retained fraction at 95% is on a *2-class* problem (the label distribution is {0, 1}). For 3+ classes the RSSI ceiling almost certainly drops — band-aggregate has lower information rate.
|
||||
- The class 1 accuracy (58.6%) is actually *higher* than v0.0.2's (34.3%). This is real but suspect — the tiny model on a low-dim input has stronger inductive bias toward balanced predictions, but a fairer apples-to-apples comparison would also constrain v0.0.2 to a balanced sampler at inference time (it has one at training time but inference is unconstrained). Followup tick: re-eval v0.0.2 with the same prediction-balancing constraint.
|
||||
|
||||
## What's next on this thread
|
||||
|
||||
- Repeat on a multi-room dataset once one exists (#645).
|
||||
- 3-class extension (0 / 1 / 2+ people) — measure the information-rate cliff.
|
||||
- Run the model on a non-ESP32 RSSI source (e.g. `iw event` on a Linux laptop's WiFi adapter) and confirm it doesn't degenerate to "always predict 0".
|
||||
- Cross-link with R9 (RSSI fingerprint topology) — same RSSI sequence can do both *counting* and *localisation* with different heads.
|
||||
- Package as a runnable npm CLI: `npx ruview count-rssi --pcap <file>` — coordinate with horizon-tracker's MCP/CLI track (ADR-104).
|
||||
|
||||
## Connection back to PROGRESS.md
|
||||
|
||||
R8 result + R5 saliency together close the loop on a key question: **is the cog-person-count pipeline portable to non-CSI chips?** Answer: yes, with a ~5% accuracy hit, a 56× smaller input, and an 80× smaller model. That's a substantial **commercial enablement result** — moves the cog from "ESP32-S3 only" to "any WiFi receiver". Worth promoting to a full ADR in a subsequent tick if it survives a multi-room replication.
|
||||
@@ -0,0 +1,64 @@
|
||||
# R9 — RSSI fingerprint topology: does temporal proximity = feature proximity?
|
||||
|
||||
**Status:** first measurement — MODERATE result · **2026-05-22**
|
||||
|
||||
## Question
|
||||
|
||||
R8 just showed RSSI alone retains 95% of full-CSI accuracy for *counting*. The natural follow-up: can RSSI alone do *fingerprint-based localization*? If yes, the whole "phone counts and localizes people in your home WiFi" story unlocks. If no, R8's commercial enablement is bounded to counting-only.
|
||||
|
||||
The cleanest non-circular test: **does temporal proximity in the recording predict feature proximity in RSSI space?** A single 30-min recording captures one operator moving around one room. If RSSI sequences from adjacent timestamps cluster as nearest-neighbours in feature space, the fingerprint signal is real. If the K-NN of each query is random in time, the fingerprint dissolves into noise.
|
||||
|
||||
## Method
|
||||
|
||||
1. Take the 1,077 paired CSI windows. Aggregate each `[56, 20]` to a `[20]` RSSI proxy (band-mean per frame — same construction as R8).
|
||||
2. Z-score normalise across all samples (matches AGC behaviour).
|
||||
3. Compute the full `1077 × 1077` cosine-similarity matrix.
|
||||
4. For each query, find top-K (K=5) nearest neighbours, excluding self.
|
||||
5. Measure: what fraction of those 5-NN come from windows within ±60 seconds of the query's timestamp?
|
||||
6. Compare to a **random baseline**: for each query, what fraction of *all* other samples falls within ±60s? (Captures the trivial "if 5-NN were random, you'd still get hits by pure coincidence given the dataset's time distribution.")
|
||||
|
||||
Lift = `K-NN fraction within window` / `random baseline`.
|
||||
|
||||
## Result
|
||||
|
||||
| Metric | Value |
|
||||
|---|---|
|
||||
| 5-NN within ±60s | **0.169** |
|
||||
| Random baseline | 0.077 |
|
||||
| **Lift over random** | **2.18×** |
|
||||
| Per-query stdev | 0.183 |
|
||||
|
||||
**Verdict — MODERATE.** Below the ≥3× threshold for "strong fingerprint" but well above 1× random. The signal is real but noisy.
|
||||
|
||||
## Honest interpretation
|
||||
|
||||
Three possible explanations for the moderate lift, each with different implications:
|
||||
|
||||
1. **20-frame windows are too short.** Each window is ~2 seconds of CSI. Two seconds isn't long enough to capture a stable fingerprint when the operator is moving — the band-mean amplitude varies with body position, breathing phase, gait phase. A 60-frame window (~6 s) might lift this to 3-4×.
|
||||
2. **One-room data has a small fingerprint space.** Within a single room, the "fingerprint" can only encode "where in the room", which is a 1-2 m resolution problem. RSSI doesn't have the bandwidth for that. Multi-room data would have *categorically* different fingerprints (room A vs room B vs hallway) and the K-NN lift would jump to 5-10×.
|
||||
3. **Band-mean discards the per-subcarrier shape.** R5 said the count-task signal is band-spread. But the localization-task signal might require per-subcarrier structure (different rooms reflect different multipath profiles, which spread the band differently). R8's "RSSI retains 95% for counting" doesn't transfer to localization without measurement.
|
||||
|
||||
The 2.18× lift is consistent with all three. Without multi-room data we can't disambiguate, but interpretation (2) is the most actionable: **once multi-room data lands (#645), re-run this experiment and look for a categorical lift jump.**
|
||||
|
||||
## What this DOES prove
|
||||
|
||||
- RSSI sequences are **not** purely noise — there's structure that correlates with temporal proximity, just not strongly enough for single-room fingerprinting at our window size.
|
||||
- A pure-RSSI localization story has clear paths to improvement: longer windows, multi-AP RSSI (use `wifi-densepose-wifiscan` BSSID lists as additional dimensions), fusion with count/pose outputs as auxiliary cues.
|
||||
|
||||
## What this DOES NOT prove
|
||||
|
||||
- That RSSI fingerprinting *won't* work cross-room. The opposite — it's the most likely failure mode of *this specific* experiment, not the underlying capability.
|
||||
- That CSI fingerprinting would work better. We didn't measure CSI K-NN here; would be a useful follow-up.
|
||||
|
||||
## Connections
|
||||
|
||||
- **R8** showed RSSI keeps the count signal. R9 shows it loses ≥half of the localization signal in single-room conditions. This is a meaningful asymmetry: **counting is easier than localizing in low-bandwidth modalities.**
|
||||
- **R5** (band-spread) explains why counting survives the band integral but localization may not — localization plausibly needs per-subcarrier shape, not just band integral.
|
||||
- **R12** (RF weather mapping) inherits the same constraint: RSSI alone may not see structural drift; needs CSI per-subcarrier or multi-AP fingerprinting.
|
||||
|
||||
## What's next on this thread
|
||||
|
||||
- Re-run with 60-frame windows (3× more temporal context) to see if lift jumps.
|
||||
- Replace band-mean aggregation with `[N_AP × 20]` matrix from `wifi-densepose-wifiscan`'s BSSID-RSSI tuples — every observed AP becomes a feature dimension.
|
||||
- Once multi-room data exists, repeat. Look for categorical lift jump (within-room 2× → across-room 8-10×).
|
||||
- Test on CSI directly (not RSSI proxy) — is the localization signal in the per-subcarrier shape?
|
||||
@@ -0,0 +1,37 @@
|
||||
# Tick 5 — 2026-05-22 03:45 UTC
|
||||
|
||||
**Thread:** R12 (RF weather mapping — structural drift from passive ambient WiFi)
|
||||
**Verdict:** Negative-ish result with a clearly-actionable revision path. **Honest progress.**
|
||||
|
||||
## What shipped
|
||||
|
||||
- `examples/research-sota/r12_rf_weather_eigenshift.py` — pure-NumPy demo that tests "can SVD-eigenvalue drift detect a synthetic structural perturbation?"
|
||||
- `examples/research-sota/r12_rf_weather_results.json` — full numbers.
|
||||
- `docs/research/sota-2026-05-22/R12-rf-weather-mapping.md` — research note covering: 10-year vision, first-experiment method, **negative result**, why it failed, three concrete revisions for next attempts (PABS / per-subcarrier residuals / multi-day baseline), what still holds, vertical applications.
|
||||
|
||||
## Headline numbers
|
||||
|
||||
| | Cosine distance from baseline |
|
||||
|---|---|
|
||||
| Control (no perturbation) | 0.00035 |
|
||||
| With 15% attenuation on 3 top-saliency subcarriers | 0.00024 |
|
||||
| Signal / natural-drift ratio | **0.69×** |
|
||||
|
||||
The synthetic perturbation produced a *smaller* spectral distance than natural temporal drift from operator movement. The top-K SVD-spectrum distance approach is too coarse.
|
||||
|
||||
## Why this is still useful
|
||||
|
||||
1. **Saves anyone going down this path** the time of trying naive SVD-distance — the data tells us it's the wrong primitive.
|
||||
2. **Identifies the right primitives:** principal angles between subspaces (PABS), per-subcarrier residual analysis, multi-day baselines.
|
||||
3. **Cross-validates R5:** task-specific saliency (count) ≠ task-specific saliency (structure detection). Same model, same data — different relevant features. Publishable distinction.
|
||||
4. **Confirms R12 is CSI-only:** RSSI is the trace of the CSI covariance matrix; if top-10 SVD can't see this perturbation, RSSI definitely can't. Bounds R8's commercial-enablement story to counting only.
|
||||
|
||||
## What's queued for later ticks
|
||||
|
||||
- Implement PABS-based change detection.
|
||||
- Per-subcarrier residual time-series analysis.
|
||||
- Acquire (or simulate) multi-day data with a known structural change.
|
||||
|
||||
## Coordination note
|
||||
|
||||
This tick wrote NOTHING to `PROGRESS.md` to avoid races with the horizon-tracker agent (which is on the `feat/ruview-mcp-m*` track and editing PROGRESS.md concurrently). The `ticks/tick-N.md` convention used here means each cron-driven tick is fully self-contained — the final 08:00 ET summary script will consolidate them.
|
||||
@@ -0,0 +1,466 @@
|
||||
# Pi 5 + Hailo Cluster: Building a Cognitive RF Observer with rvcsi
|
||||
|
||||
A field-tested tutorial for turning a 4-node Raspberry Pi 5 cluster into a
|
||||
multistatic Wi-Fi CSI cognitive RF observer that learns room states,
|
||||
predicts the next one, and flags anomalies — entirely from radio.
|
||||
|
||||
**Estimated time:** 4–6 hours (hardware 1h, firmware 1h, software 1h, calibration 1–3h)
|
||||
|
||||
**What you will build:** A self-learning 4-node cluster that captures Wi-Fi
|
||||
Channel State Information from a stable RF beacon, encodes each frame into a
|
||||
128-dimensional fingerprint on an on-device Hailo-8 NPU, clusters those
|
||||
fingerprints into discrete room states with stable IDs across runs, models
|
||||
state transitions with a 2nd-order Markov chain (with measurable predictive
|
||||
skill above chance), and persists everything to a queryable brain corpus on
|
||||
a workstation. The whole thing runs over Tailscale and is operated through
|
||||
a single CLI with **34 subcommands**.
|
||||
|
||||
**Who this is for:** RF engineers, smart-home hackers, security researchers,
|
||||
and ML/embedded folks comfortable with Linux + systemd. No specific signal-
|
||||
processing background required — but you do need patience for hardware
|
||||
quirks (nexmon_csi cross-compile is a known dead end; see step 3).
|
||||
|
||||
> **The TL;DR**: 4× Pi 5 + 2× Hailo-8 → CSI → 128-d embeddings → cosine
|
||||
> k-means with warm-start → 2nd-order Markov → SQLite brain → 34-subcommand
|
||||
> operator CLI. Production-grade signal: 39% top-1 ceiling on next-state
|
||||
> prediction (16× chance baseline), continuous fleet/drift/anomaly
|
||||
> monitoring, and a 12-category time-series corpus.
|
||||
|
||||
> **About the name "rvcsi" in this tutorial.** When this tutorial was
|
||||
> first written, the cluster's per-Pi capture services were named with
|
||||
> an `rvcsi` prefix (`cog-rvcsi-stream`, `cog-rvcsi-correlator`) as
|
||||
> branding only — the actual code was Python and didn't depend on the
|
||||
> upstream [`ruvnet/rvcsi`](https://github.com/ruvnet/rvcsi) Rust
|
||||
> runtime. **As of 2026-05-13**, the v0-appliance project has accepted
|
||||
> [ADR-207](https://github.com/ruvnet/v0-appliance/blob/main/docs/adr/ADR-207-rvcsi-library-integration.md)
|
||||
> (rvCSI library integration — Option D) and shipped a Rust binary
|
||||
> `cog-rvcsi-pi` built on rvcsi-runtime 0.3 that replaces the three
|
||||
> Python services. The cutover is per-Pi, operator-driven, with
|
||||
> one-command rollback (`scripts/rvcsi-pi/install-rvcsi-pi.sh` and
|
||||
> `uninstall-rvcsi-pi.sh`). A given cluster may be running either
|
||||
> stack while migration is in progress; the schema and operator
|
||||
> surface are unchanged across the cutover. See ADR-207's
|
||||
> Implementation log for the current state.
|
||||
|
||||
---
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Prerequisites](#1-prerequisites)
|
||||
2. [Architecture overview](#2-architecture-overview)
|
||||
3. [Per-node firmware: nexmon_csi on Pi 5](#3-per-node-firmware-nexmon_csi-on-pi-5)
|
||||
4. [Per-node services](#4-per-node-services)
|
||||
5. [Workstation pipeline](#5-workstation-pipeline)
|
||||
6. [Calibration: getting from raw CSI to room states](#6-calibration-getting-from-raw-csi-to-room-states)
|
||||
7. [Operating the cluster: the cog-query CLI](#7-operating-the-cluster-the-cog-query-cli)
|
||||
8. [What you can measure](#8-what-you-can-measure)
|
||||
9. [Troubleshooting](#9-troubleshooting)
|
||||
10. [Next steps](#10-next-steps)
|
||||
|
||||
---
|
||||
|
||||
## 1. Prerequisites
|
||||
|
||||
### Hardware
|
||||
|
||||
| Item | Quantity | Approx. cost | Notes |
|
||||
|------|----------|--------------|-------|
|
||||
| Raspberry Pi 5 (8GB) | 4 | ~$80 each | 4GB works but tight under sustained load |
|
||||
| Hailo-8 M.2 HAT (AI Kit) | 2 | ~$110 each | Only 2 needed — encoder is split across cluster-1 + cluster-2 |
|
||||
| MicroSD (64GB, A2) | 4 | ~$10 each | A2 class strongly recommended for sustained writes |
|
||||
| USB-C PD power supply (27W) | 4 | ~$12 each | Pi 5 draws 5A at full Hailo load |
|
||||
| Active cooler | 4 | ~$5 each | Cluster-2 sustains thermal load — passive will throttle |
|
||||
| Workstation (≥16GB RAM, Linux) | 1 | — | Hosts the brain HTTP service + clusterer + anomaly daemon |
|
||||
| Stable Wi-Fi beacon | 1 | — | Any AP on the same 5 GHz channel. We use ch.149/80MHz. Stability matters more than identity. |
|
||||
|
||||
**Total parts cost:** ~$580 plus workstation.
|
||||
|
||||
> **Important:** All 4 Pi 5s must use the on-board `bcm43455c0` radio. USB
|
||||
> Wi-Fi adapters with otherwise-similar chipsets **will not** work — nexmon's
|
||||
> firmware patches are silicon-specific. See ADR-206 § "USB Wi-Fi dongle
|
||||
> rabbit-hole" for the painful version of that lesson.
|
||||
|
||||
### Software prerequisites
|
||||
|
||||
| Component | Version | Notes |
|
||||
|-----------|---------|-------|
|
||||
| Pi OS Bookworm (Lite) | 64-bit, kernel 6.6+ | Use the Lite image — Desktop slows boot and burns SD writes |
|
||||
| Tailscale | ≥1.60 | Mesh networking across the cluster |
|
||||
| Rust toolchain | 1.78+ on workstation, 1.78+ on each Pi | For ruvector + adapter binaries |
|
||||
| Python 3.11+ | system Python on workstation | numpy required |
|
||||
| systemd-user | already present | Workstation timers run as user units |
|
||||
|
||||
---
|
||||
|
||||
## 2. Architecture overview
|
||||
|
||||
```
|
||||
┌─ workstation (Linux, ≥16GB) ──────────────────┐
|
||||
│ │
|
||||
│ brain HTTP (SQLite, port 9876) │
|
||||
│ ↑↑ │
|
||||
│ ┌──┴┴──────────────────────────────────┐ │
|
||||
│ │ rfmem-tail ← ingests live brain │ │
|
||||
│ │ rfmem-recall → posts category= │ │
|
||||
│ │ rfmem-recall when │ │
|
||||
│ │ current state ≈ past │ │
|
||||
│ │ rfmem-anomaly → 13-axis detector, │ │
|
||||
│ │ posts rfmem-anomaly & │ │
|
||||
│ │ rfmem-state-transition │ │
|
||||
│ │ cog-rfmem-states (timer, hourly) │ │
|
||||
│ │ re-clusters w/ warm-start│ │
|
||||
│ │ cog-rfmem-insights (timer, nightly) │ │
|
||||
│ │ writes rfmem-insights │ │
|
||||
│ │ cog-rfmem-drift-check (timer, 05:00) │ │
|
||||
│ │ audits cluster file state│ │
|
||||
│ └───────────────────────────────────────┘ │
|
||||
│ │
|
||||
│ cog-query (CLI, 34 subcommands, 4 JSON modes)│
|
||||
└────────────────────────────────────────────────┘
|
||||
↑
|
||||
Tailscale mesh ──────────┴───────────────────────────────┐
|
||||
↓ ↓ ↓
|
||||
┌─ cluster-1 (Hailo) ┐ ┌─ cluster-2 (Hailo + fusion) ┐ ┌─ cluster-3 ┐ ┌─ v0 ┐
|
||||
│ cog-csi-emitter │ │ cog-csi-emitter │ │ same as │ │ same│
|
||||
│ cog-csi-adapter │ │ cog-csi-adapter │ │ cluster-1 │ │ as │
|
||||
│ cog-rvcsi-stream │ │ cog-rvcsi-stream │ │ minus │ │ c-3 │
|
||||
│ cog-hailo-encoder │ │ cog-hailo-encoder │ │ Hailo & │ │ │
|
||||
│ │ │ cog-rvcsi-correlator (fusion)│ │ correlator │ │ │
|
||||
└────────────────────┘ └─────────────────────────────┘ └────────────┘ └─────┘
|
||||
4 svc 5 svc 3 svc 3 svc
|
||||
└─────────────────────── 15 expected services total ──────────────────────┘
|
||||
```
|
||||
|
||||
**Why this split?** Multistatic fusion (combining CSI from 4 spatial vantage
|
||||
points into a single weighted observation) is computationally cheap but
|
||||
benefits from being on **one** node so the other three only do capture +
|
||||
encode. Hailo-8 is the bottleneck cost, so we put two on the cluster
|
||||
(one for redundancy, one for the fusion node) and let `cluster-3` + `v0`
|
||||
run as pure capture sensors.
|
||||
|
||||
---
|
||||
|
||||
## 3. Per-node firmware: nexmon_csi on Pi 5
|
||||
|
||||
**Critical lesson learned (saved you a week):** the workstation x86_64
|
||||
cross-compile path for nexmon_csi on Pi 5 **does not work**. The 39-hunk
|
||||
patch series applies cleanly on a native Pi 5 ARM build, and fails in
|
||||
subtle ways elsewhere.
|
||||
|
||||
The recipe that works:
|
||||
|
||||
```bash
|
||||
# On each Pi 5 (not the workstation):
|
||||
sudo apt update && sudo apt install -y \
|
||||
raspberrypi-kernel-headers bc bison flex libssl-dev make \
|
||||
gcc gawk qpdf cmake build-essential libpcap-dev clang gcc-arm-none-eabi
|
||||
|
||||
git clone https://github.com/seemoo-lab/nexmon.git ~/nexmon
|
||||
cd ~/nexmon
|
||||
source setup_env.sh
|
||||
make
|
||||
|
||||
cd patches
|
||||
git clone https://github.com/seemoo-lab/nexmon_csi.git
|
||||
cd nexmon_csi
|
||||
|
||||
# Apply the Pi-5-friendly patch series — all 39 hunks should apply clean
|
||||
# on native ARM. If you see "Hunk #N FAILED", you are almost certainly
|
||||
# cross-compiling from x86_64. Stop. Build on the Pi.
|
||||
./install.sh
|
||||
|
||||
# Switch on:
|
||||
sudo mcp # 'monitor capability provisioning' — enable
|
||||
sudo nexutil -Iwlan0 -s500 -b -l34 -v<86-char base64 capture filter>
|
||||
```
|
||||
|
||||
> **Pi 5 kernel gotcha:** Pi OS Bookworm ships two kernels — `kernel8.img`
|
||||
> (4K pages) and `kernel_2712.img` (16K pages, Pi 5 only). nexmon_csi
|
||||
> currently builds clean against `kernel8.img`. Add `kernel=kernel8.img`
|
||||
> to `/boot/firmware/config.txt` if you've switched. **After the switch,
|
||||
> SSH by hostname via Tailscale** — host keys + DHCP gotchas otherwise.
|
||||
|
||||
> **Clock-skew first-boot trap:** Pi 5 has no RTC. First-boot apt will
|
||||
> reject "future-dated" `Release` files. Patch your firstboot to wait for
|
||||
> `systemd-timesyncd` before running `apt-get`.
|
||||
|
||||
The complete commands + full troubleshooting matrix is in the
|
||||
[detailed gist](https://gist.github.com/ruvnet/88e7b053c41cb4f4af7a7ec4af873017) — section "Firmware: nexmon_csi on Pi 5".
|
||||
|
||||
---
|
||||
|
||||
## 4. Per-node services
|
||||
|
||||
Each cluster Pi runs a small fixed set of systemd services. Per-host
|
||||
topology:
|
||||
|
||||
| Service | cluster-1 | cluster-2 | cluster-3 | v0 |
|
||||
|---|:--:|:--:|:--:|:--:|
|
||||
| `cog-csi-emitter` (raw CSI capture from nexmon) | ✓ | ✓ | ✓ | ✓ |
|
||||
| `cog-csi-adapter` (Rust binary; CSI → 256-byte float frames) | ✓ | ✓ | ✓ | ✓ |
|
||||
| `cog-rvcsi-stream` (publishes frames to rvcsi-correlator) | ✓ | ✓ | ✓ | ✓ |
|
||||
| `cog-hailo-encoder` (frames → 128-d fingerprints on Hailo-8) | ✓ | ✓ | — | — |
|
||||
| `cog-rvcsi-correlator` (multistatic fusion across 4 nodes) | — | ✓ | — | — |
|
||||
| **Expected service count** | **4** | **5** | **3** | **3** |
|
||||
|
||||
The topology is encoded in the workstation's `cog-query fleet-status`
|
||||
subcommand, which compares per-host expected services against live
|
||||
`systemctl is-active` results. A flat-service check would falsely flag
|
||||
cluster-3 and v0 as degraded (they have neither Hailo nor the correlator
|
||||
— that's by design).
|
||||
|
||||
> **rvcsi cutover (ADR-207 Option D, 2026-05-13).** The three services
|
||||
> `cog-csi-emitter`, `cog-csi-adapter`, and `cog-rvcsi-stream` are
|
||||
> being consolidated into one Rust binary `cog-rvcsi-pi` built on
|
||||
> [rvcsi-runtime](https://crates.io/crates/rvcsi-runtime). The new
|
||||
> binary holds the same per-Pi role and the same expected-service
|
||||
> count from the operator's view (`fleet-status` already understands
|
||||
> both layouts). Deploy with
|
||||
> `bash scripts/rvcsi-pi/install-rvcsi-pi.sh <pi-host>`; revert with
|
||||
> `scripts/rvcsi-pi/uninstall-rvcsi-pi.sh`. The cutover is per-Pi,
|
||||
> not flag-day — mixed Python/Rust clusters are supported. The Hailo
|
||||
> encoder + correlator stay Python in this phase; their Rust ports
|
||||
> are tracked as follow-on ADRs.
|
||||
|
||||
All unit files + the install script are in the
|
||||
[detailed gist](https://gist.github.com/ruvnet/88e7b053c41cb4f4af7a7ec4af873017) — section "Per-node systemd units".
|
||||
|
||||
---
|
||||
|
||||
## 5. Workstation pipeline
|
||||
|
||||
The workstation runs ten user-mode units (3 daemons, 7 timers):
|
||||
|
||||
| Unit | Type | Cadence | Purpose |
|
||||
|---|---|---|---|
|
||||
| `cog-rfmem-tail` | daemon | continuous | Ingests live brain entries into the workstation mirror |
|
||||
| `cog-rfmem-recall` | daemon | continuous | kNN-matches current fingerprint vs persisted ones, posts `rfmem-recall` |
|
||||
| `cog-rfmem-anomaly` | daemon | continuous | 13-axis anomaly detector, posts `rfmem-anomaly` + `rfmem-state-transition` |
|
||||
| `cog-rfmem-indexer` | timer | every 5 min | Updates HNSW index for kNN |
|
||||
| `cog-rfmem-compress` | timer | hourly | Compresses old brain entries |
|
||||
| `cog-rfmem-daily` | timer | nightly 04:00 | Per-day stats roll-up (`rfmem-daily`) |
|
||||
| `cog-rfmem-states` | timer | hourly | Re-runs cosine k-means w/ warm-start (`rfmem-state-summary`) |
|
||||
| `cog-rfmem-insights` | timer | nightly 04:55 | NL synthesis, posts `rfmem-insights` |
|
||||
| `cog-rfmem-drift-check` | timer | nightly 05:00 | Audits cluster file/unit drift, posts `rfmem-drift` |
|
||||
| `cog-rfmem-mirror` | timer | hourly | Mirrors cluster-2 brain → workstation read-replica |
|
||||
|
||||
Install in one shot:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/<your-fork>/v0-appliance.git
|
||||
cd v0-appliance
|
||||
bash scripts/rfmem/install-workstation.sh
|
||||
```
|
||||
|
||||
The installer is **idempotent** — rerunning is safe and only enables
|
||||
units that aren't yet enabled. It also wires a git post-commit hook
|
||||
that auto-deploys + auto-smoke-tests on every commit touching
|
||||
`scripts/rfmem/`. That closes the "I edited the repo but forgot to
|
||||
deploy" gap that bit us repeatedly in early development.
|
||||
|
||||
---
|
||||
|
||||
## 6. Calibration: getting from raw CSI to room states
|
||||
|
||||
This is the longest step but largely passive — let it run.
|
||||
|
||||
### 6.1 Walk the room
|
||||
|
||||
For 30–60 minutes after the cluster is live, walk through every room you
|
||||
want recognized. Sit, stand, move between rooms, repeat. The encoder is
|
||||
learning to map "what the room looks like in CSI" into 128-d vectors;
|
||||
diversity here matters more than total time.
|
||||
|
||||
### 6.2 First clustering pass
|
||||
|
||||
```bash
|
||||
# Force-trigger the clusterer (it normally fires hourly):
|
||||
systemctl --user start cog-rfmem-states.service
|
||||
python3 scripts/rfmem/cog-query.py states
|
||||
```
|
||||
|
||||
Output looks like:
|
||||
|
||||
```
|
||||
=== rfmem-states — k=16, n=12,847 ===
|
||||
state #0 π=0.184 dwell=42.3s centroid_drift=0.012 (default)
|
||||
state #1 π=0.121 dwell=18.1s centroid_drift=0.003
|
||||
state #4 π=0.087 dwell=29.6s centroid_drift=0.041
|
||||
...
|
||||
```
|
||||
|
||||
**Stable IDs across runs.** The warm-start k-means recipe matches new
|
||||
centroids to the prior run's centroids by cosine similarity before
|
||||
assigning IDs. This means state #4 stays state #4 between hourly runs —
|
||||
otherwise downstream Markov transitions would scramble after every
|
||||
re-cluster.
|
||||
|
||||
### 6.3 Let the Markov chain build
|
||||
|
||||
After a few thousand transitions (a few hours of activity), check:
|
||||
|
||||
```bash
|
||||
python3 scripts/rfmem/cog-query.py prediction-accuracy
|
||||
```
|
||||
|
||||
You should see something like:
|
||||
|
||||
```
|
||||
=== prediction-accuracy — training-set top-1 ceilings ===
|
||||
1st-order: 37.1% (16x chance baseline of 6.25%)
|
||||
2nd-order: 39.4% (16x chance baseline of 6.25%, 1.06x gain over 1st)
|
||||
```
|
||||
|
||||
The 2nd-order chain beats 1st-order because it conditions on the
|
||||
**previous** state as well as the current one. Self-loops are excluded
|
||||
from the argmax (a transition is by definition a state change).
|
||||
|
||||
### 6.4 Verify the room learned itself
|
||||
|
||||
```bash
|
||||
python3 scripts/rfmem/cog-query.py insights
|
||||
```
|
||||
|
||||
Reads like:
|
||||
|
||||
```
|
||||
The cluster has observed 446,231 fingerprints, clustering them into
|
||||
16 discrete RF states. The room exhibits moderately diverse (stationary
|
||||
entropy 0.82/1.0). State #4 is the dominant 'default' state (π=0.214);
|
||||
state #13 is the rarest baseline (π=0.018).
|
||||
Prediction skill (last hour, 2nd-order): top-1 12.4% (1.98x chance),
|
||||
top-3 31.0% (1.65x chance, 412 transitions) (training-set ceiling
|
||||
39.4% — operating @ 31% of capacity).
|
||||
```
|
||||
|
||||
That "operating @ 31% of capacity" line is the operational efficiency:
|
||||
how close live performance is to the model's theoretical ceiling. Big
|
||||
gap = the room is being noisy in ways the static cluster model doesn't
|
||||
capture. Small gap = you're near SOTA for this static model.
|
||||
|
||||
---
|
||||
|
||||
## 7. Operating the cluster: the cog-query CLI
|
||||
|
||||
A single CLI binary with **34 subcommands** + 4 machine-readable JSON
|
||||
modes. Practical ones (full list in the gist):
|
||||
|
||||
| Subcommand | What it does |
|
||||
|---|---|
|
||||
| `summary --hours 1` | Bird's-eye view of last hour: anomalies, transitions, recall hits |
|
||||
| `top-events --hours 24 --limit 5` | Highest-info events in window (combines novelty + tier + recency) |
|
||||
| `top-events --json` | Same, agent-consumable |
|
||||
| `insights` | Natural-language synthesis (paragraph) — what the cluster thinks |
|
||||
| `insights --json` | Same, structured |
|
||||
| `insights --post` | Same, persisted to brain as `rfmem-insights` |
|
||||
| `stats` | Corpus: per-category counts, dimensions, vector counts |
|
||||
| `motion` | Recent motion events |
|
||||
| `anomalies --sort info` | Anomalies sorted by composite info score (1.0–8.0) |
|
||||
| `circadian` | 24-hour bin of activity — does the room have a daily rhythm? |
|
||||
| `by-state` | Per-state metrics (dwell, σ-baseline, novelty distribution) |
|
||||
| `markov` | Top transitions by frequency, both 1st + 2nd-order |
|
||||
| `transitions --sort novelty` | Rare/surprising transitions |
|
||||
| `dwell-times` | How long the room stays in each state |
|
||||
| `prediction-accuracy` | 1st + 2nd-order top-1 ceilings |
|
||||
| `baseline-drift` | Has the noise floor shifted? (slow change) |
|
||||
| `centroid-drift` | Has any state's RF signature materially changed? |
|
||||
| `fleet-status` | Per-host expected-service liveness check |
|
||||
| `fleet-status --json` | Same, agent-consumable |
|
||||
| `fleet-status --post` | Same, persisted to brain as `rfmem-fleet` (heartbeat) |
|
||||
| `check-drift` | Workstation/cluster file + unit drift audit |
|
||||
| `replica-status` | Hourly cluster-2 → workstation mirror health |
|
||||
|
||||
### The fleet-health triad
|
||||
|
||||
Three subcommands cover the operator's full health picture:
|
||||
|
||||
- `check-drift` — file content drift (what's deployed vs what's in git)
|
||||
- `replica-status` — workstation mirror lag (last successful sync)
|
||||
- `fleet-status` — service liveness across the 4 Pis (topology-aware)
|
||||
|
||||
If all three are green, the cluster is healthy. If any one fires, you
|
||||
have a concrete starting point.
|
||||
|
||||
---
|
||||
|
||||
## 8. What you can measure
|
||||
|
||||
After a week of runtime, you can answer questions like:
|
||||
|
||||
- **"What's the room's most common 'baseline' state?"** → `states` shows
|
||||
the π-dominant cluster ID.
|
||||
- **"Did anything weird happen last night?"** → `anomalies --sort info
|
||||
--hours 12` sorts by combined-information score (novelty × tier × state-
|
||||
rarity × calmness).
|
||||
- **"How predictable is the room?"** → `insights` reports stationary
|
||||
entropy (0.0 = single state, 1.0 = uniform). Most rooms land 0.6–0.9.
|
||||
- **"What's the most novel transition ever observed?"** → `transitions
|
||||
--sort novelty --limit 1`. We've seen transitions with
|
||||
`transition_p=0.0000` — never observed before in 446k+ embeddings.
|
||||
- **"Is the room changing slowly?"** → `centroid-drift` flags states
|
||||
whose 128-d signature has moved > 0.05 cosine distance since the prior
|
||||
clusterer run. Common cause: a piece of furniture moved.
|
||||
- **"What's the daily rhythm?"** → `circadian` bins activity by hour.
|
||||
Most rooms show clear morning/evening peaks.
|
||||
|
||||
---
|
||||
|
||||
## 9. Troubleshooting
|
||||
|
||||
| Symptom | Likely cause | Fix |
|
||||
|---|---|---|
|
||||
| `nexmon_csi` build fails with FAILED hunks | Cross-compiling from x86_64 | Build on the Pi natively |
|
||||
| Pi 5 stops booting after kernel switch | Wrong `kernel=` in `/boot/firmware/config.txt` | Use `kernel=kernel8.img` |
|
||||
| First boot fails on `apt update` | No RTC → clock skew, apt rejects "future-dated" Release files | Wait for `systemd-timesyncd` in firstboot |
|
||||
| `cog-rfmem-now` times out | Workstation daemon swap-thrashing | Bump `MemoryMax=` in unit file (we run 1G) |
|
||||
| `fleet-status` shows DEGRADED on cluster-3 / v0 | Topology unaware (old version) | Update to latest — per-host expected-services |
|
||||
| Cluster-2 Hailo encoder silent | `cp -r` made encoder a directory, not a file | `install -m 0755` instead |
|
||||
| 2nd-order Markov top-1 = 0% | Self-loop dominates argmax | Zero out self-loop before `.argmax()` |
|
||||
| State IDs change between runs | No warm-start k-means | Update clusterer to match new centroids to prior run by cosine |
|
||||
| HardFaults during embedded N6 bring-up | (Different topic, see [ADR-027](../adr/) for STM32N6 startup notes) | — |
|
||||
|
||||
---
|
||||
|
||||
## 10. Next steps
|
||||
|
||||
Once your cluster is producing stable predictions and clean fleet health,
|
||||
the natural directions are:
|
||||
|
||||
1. **Cross-room correlation** — train a second cluster in another room
|
||||
and feed both into the workstation. The brain already supports
|
||||
multiple namespaces.
|
||||
2. **Active sensing** — instead of passively observing whatever beacon is
|
||||
present, drive your own (e.g., dedicated 5 GHz beacon AP at fixed
|
||||
power). Eliminates upstream variability.
|
||||
3. **Vital signs** — the RuView project has companion code for extracting
|
||||
heart-rate and breathing from CSI; the 128-d encoder output is a
|
||||
reasonable input feature.
|
||||
4. **Federated training** — multiple physical sites publishing to a shared
|
||||
brain. Each site keeps its own clusters; transitions are the shared
|
||||
vocabulary.
|
||||
5. **Push to upstream RuView** — if your cluster develops capabilities not
|
||||
in this tutorial (you'll know by the time you've written the README),
|
||||
send a PR.
|
||||
|
||||
---
|
||||
|
||||
## Reference material
|
||||
|
||||
- **[Detailed cookbook gist (all commands, configs, unit files)](https://gist.github.com/ruvnet/88e7b053c41cb4f4af7a7ec4af873017)**
|
||||
- **[ADR-206: nexmon_csi on Pi 5 cluster](https://github.com/ruvnet/v0-appliance/blob/main/docs/adr/ADR-206-nexmon-csi-on-pi-5-cluster.md)** — the engineering decision record
|
||||
with full rationale, including the painful-but-instructive failures
|
||||
- **[v0-appliance repo](https://github.com/ruvnet/v0-appliance)** — the
|
||||
source of truth for `scripts/rfmem/` operator tooling
|
||||
- **[seemoo-lab/nexmon_csi](https://github.com/seemoo-lab/nexmon_csi)** —
|
||||
upstream CSI capture firmware
|
||||
- **[Hailo-8 documentation](https://hailo.ai/products/hailo-8/)** — NPU
|
||||
reference
|
||||
|
||||
---
|
||||
|
||||
*This tutorial was built against the v0.5.0-cognitive-rf-observer milestone
|
||||
of `v0-appliance`. The cluster has been running continuously for 6+ weeks
|
||||
of development with 446k+ fingerprints observed, 16 stable RF states, and
|
||||
a 2nd-order Markov model operating at 31% of its 39.4% theoretical
|
||||
top-1 ceiling. SOTA is a moving target — but this is a real, working
|
||||
cognitive RF observer that you can reproduce.*
|
||||
@@ -21,6 +21,7 @@ WiFi DensePose turns commodity WiFi signals into real-time human pose estimation
|
||||
- [Windows WiFi (RSSI Only)](#windows-wifi-rssi-only)
|
||||
- [ESP32-S3 (Full CSI)](#esp32-s3-full-csi)
|
||||
- [ESP32 Multistatic Mesh (Advanced)](#esp32-multistatic-mesh-advanced)
|
||||
- [Connect Mesh Data to the Dashboard and Observatory](#connect-mesh-data-to-the-dashboard-and-observatory)
|
||||
- [Cognitum Seed Integration (ADR-069)](#cognitum-seed-integration-adr-069)
|
||||
5. [REST API Reference](#rest-api-reference)
|
||||
6. [WebSocket Streaming](#websocket-streaming)
|
||||
@@ -28,13 +29,14 @@ WiFi DensePose turns commodity WiFi signals into real-time human pose estimation
|
||||
8. [Vital Sign Detection](#vital-sign-detection)
|
||||
9. [CLI Reference](#cli-reference)
|
||||
10. [Observatory Visualization](#observatory-visualization)
|
||||
11. [Adaptive Classifier](#adaptive-classifier)
|
||||
11. [Loading the Pretrained Model from Hugging Face](#loading-the-pretrained-model-from-hugging-face)
|
||||
12. [Adaptive Classifier](#adaptive-classifier)
|
||||
- [Recording Training Data](#recording-training-data)
|
||||
- [Training the Model](#training-the-model)
|
||||
- [Using the Trained Model](#using-the-trained-model)
|
||||
12. [Training a Model](#training-a-model)
|
||||
13. [Training a Model](#training-a-model)
|
||||
- [CRV Signal-Line Protocol](#crv-signal-line-protocol)
|
||||
13. [RVF Model Containers](#rvf-model-containers)
|
||||
14. [RVF Model Containers](#rvf-model-containers)
|
||||
14. [Hardware Setup](#hardware-setup)
|
||||
- [ESP32-S3 Mesh](#esp32-s3-mesh)
|
||||
- [Intel 5300 / Atheros NIC](#intel-5300--atheros-nic)
|
||||
@@ -331,6 +333,46 @@ The mesh uses a **Time-Division Multiplexing (TDM)** protocol so nodes take turn
|
||||
|
||||
See [ADR-029](adr/ADR-029-ruvsense-multistatic-sensing-mode.md) and [ADR-032](adr/ADR-032-multistatic-mesh-security-hardening.md) for the full design.
|
||||
|
||||
### Connect Mesh Data to the Dashboard and Observatory
|
||||
|
||||
If a standalone `aggregator` command prints live packets, the ESP32 fleet is already reaching that host. To visualize the same data, stop the standalone aggregator and run `sensing-server` on that same host and UDP port. The sensing server is the aggregator used by the REST API, WebSocket stream, dashboard, and Observatory.
|
||||
|
||||
```bash
|
||||
# From a source build
|
||||
cd v2
|
||||
cargo run -p wifi-densepose-sensing-server -- \
|
||||
--source esp32 \
|
||||
--udp-port 5005 \
|
||||
--http-port 3000 \
|
||||
--ws-port 3001 \
|
||||
--ui-path ../../ui
|
||||
|
||||
# Docker
|
||||
docker run --rm \
|
||||
-e CSI_SOURCE=esp32 \
|
||||
-p 3000:3000 \
|
||||
-p 3001:3001 \
|
||||
-p 5005:5005/udp \
|
||||
ruvnet/wifi-densepose:latest
|
||||
```
|
||||
|
||||
Open the UI from the sensing server, not from a local file:
|
||||
|
||||
| View | URL |
|
||||
|------|-----|
|
||||
| Dashboard | `http://localhost:3000/ui/index.html` |
|
||||
| Observatory | `http://localhost:3000/ui/observatory.html` |
|
||||
|
||||
Use these checks before debugging the browser:
|
||||
|
||||
```bash
|
||||
curl http://localhost:3000/health
|
||||
curl http://localhost:3000/api/v1/nodes
|
||||
curl http://localhost:3000/api/v1/sensing/latest
|
||||
```
|
||||
|
||||
If the ESP32 nodes are provisioned with `--target-ip <AGGREGATOR_HOST>`, that IP must be the machine running `sensing-server`. Only one process can receive UDP `:5005` at a time, so leave the standalone hardware `aggregator` off while the dashboard or Observatory is live.
|
||||
|
||||
### Cognitum Seed Integration (ADR-069)
|
||||
|
||||
Connect an ESP32-S3 to a [Cognitum Seed](https://cognitum.one) (Pi Zero 2 W, ~$15) for persistent vector storage, kNN similarity search, cryptographic witness chain, and AI-accessible sensing via MCP proxy.
|
||||
@@ -752,6 +794,67 @@ The Observatory is an immersive Three.js visualization that renders WiFi sensing
|
||||
|
||||
---
|
||||
|
||||
## Loading the Pretrained Model from Hugging Face
|
||||
|
||||
A pretrained CSI encoder + presence-detection head is published on Hugging Face at [`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained). It was trained on 60,630 frames / 610,615 contrastive triplets (12.2M steps, final loss 0.065) and reports 100% presence accuracy and ~164k embeddings/sec on an Apple M4 Pro.
|
||||
|
||||
What it ships (and what it does not):
|
||||
|
||||
| Capability | Status |
|
||||
|------------|--------|
|
||||
| Presence detection (occupied / empty) | ✅ Trained head — 100% accuracy on validation |
|
||||
| 128-dim CSI embeddings (re-ID, similarity, downstream training) | ✅ Trained encoder |
|
||||
| Single-person breathing / heart-rate | ⚠️ Server still uses heuristic DSP — model does not replace this yet |
|
||||
| 17-keypoint full-body pose | 🔬 No keypoint weights shipped yet — pose pipeline runs but without a learned head |
|
||||
|
||||
### Download
|
||||
|
||||
```bash
|
||||
pip install huggingface_hub
|
||||
huggingface-cli download ruvnet/wifi-densepose-pretrained \
|
||||
--local-dir models/wifi-densepose-pretrained
|
||||
```
|
||||
|
||||
The download yields a small set of files (the `.rvf.jsonl` is the canonical container the sensing server reads):
|
||||
|
||||
```
|
||||
models/wifi-densepose-pretrained/
|
||||
model.rvf.jsonl # RVF container (encoder + presence head + lora)
|
||||
model.safetensors # 48 KB — same encoder weights, safetensors format
|
||||
model-q4.bin # 8 KB — recommended quantization for edge
|
||||
presence-head.json # presence classifier head
|
||||
config.json # sona-lora rank=8 alpha=16, target encoder + task_heads
|
||||
```
|
||||
|
||||
### Using the weights
|
||||
|
||||
The HF artifact is in **JSONL RVF** format (one JSON object per line: `metadata`, `encoder`, `lora`). What you can do with it today:
|
||||
|
||||
| Consumer | Format it reads | Status |
|
||||
|----------|-----------------|--------|
|
||||
| Python / PyTorch training pipeline | `model.safetensors` | ✅ Works — load with `safetensors.torch.load_file` |
|
||||
| RVF JSONL inspection / re-export | `model.rvf.jsonl` | ✅ Works — plain JSONL, parse line-by-line |
|
||||
| Sensing-server `--model <PATH>` flag | binary RVF (`RVFS` magic) | ⚠️ Does **not** accept the JSONL file yet — see gap below |
|
||||
|
||||
**Known gap (tracked):** `v2/crates/wifi-densepose-sensing-server/src/rvf_container.rs` only parses the binary RVF segment format (magic `0x52564653`). Pointing `--model` at `model.rvf.jsonl` causes the progressive loader to error with `invalid magic at offset 0: expected 0x52564653, got 0x7974227B` (`0x7974227B` is the ASCII bytes `{"ty…` from the JSONL header), and the live pipeline degrades to null output rather than falling back to heuristic mode. Until a JSONL adapter lands (or the model is re-published as binary RVF), run the sensing-server **without** `--model` and consume the HF weights from Python or the training pipeline.
|
||||
|
||||
```bash
|
||||
# Works today — Python side (training, evaluation, embedding extraction):
|
||||
python -c "
|
||||
from safetensors.torch import load_file
|
||||
state = load_file('models/wifi-densepose-pretrained/model.safetensors')
|
||||
print({k: tuple(v.shape) for k, v in state.items()})
|
||||
"
|
||||
|
||||
# Sensing server — run heuristic for now:
|
||||
cargo run -p wifi-densepose-sensing-server --release -- \
|
||||
--source esp32 --udp-port 5005 --http-port 3000
|
||||
```
|
||||
|
||||
See [RVF Model Containers](#rvf-model-containers) for the binary format the loader expects, and [Training a Model](#training-a-model) for using the encoder as a starting point for environment-specific fine-tuning.
|
||||
|
||||
---
|
||||
|
||||
## Adaptive Classifier
|
||||
|
||||
The adaptive classifier (ADR-048) learns your environment's specific WiFi signal patterns from labeled recordings. It replaces static threshold-based classification with a trained logistic regression model that uses 15 features (7 server-computed + 8 subcarrier-derived statistics).
|
||||
@@ -1744,6 +1847,8 @@ The server applies a 3-stage smoothing pipeline (ADR-048). If readings are still
|
||||
|
||||
- Verify the sensing server is running: `curl http://localhost:3000/health`
|
||||
- Access Observatory via the server URL: `http://localhost:3000/ui/observatory.html` (not a file:// URL)
|
||||
- If a standalone `aggregator` command is already listening on UDP `:5005`, stop it and run `sensing-server --source esp32 --udp-port 5005` instead; the Observatory reads the server WebSocket, not the standalone aggregator output
|
||||
- Verify the ESP32 nodes are provisioned to the IP address of the machine running `sensing-server`
|
||||
- Hard refresh with Ctrl+Shift+R to clear cached settings
|
||||
- The auto-detect probes `/health` on the same origin — cross-origin won't work
|
||||
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env python3
|
||||
"""R12 — RF weather: can SVD-eigenvalue drift detect structural changes?
|
||||
|
||||
See docs/research/sota-2026-05-22/R12-rf-weather-mapping.md.
|
||||
|
||||
The persistent-room field model in `wifi-densepose-signal/src/ruvsense/
|
||||
field_model.rs` does an SVD on empty-room CSI to extract an eigenstructure
|
||||
that describes "what this room's RF reflection looks like with nobody
|
||||
in it". Today that's used to subtract the room's baseline so motion
|
||||
detection isn't confused by static multipath.
|
||||
|
||||
This experiment asks a different question: **does the eigenvalue
|
||||
*spectrum* itself drift in a detectable way when something structural
|
||||
changes in the room?** "Structural change" = a new piece of furniture,
|
||||
a window that opened, water in the wall, settled foundation, missing
|
||||
ceiling tile. The 10-year vision (R12 research note) is continuous
|
||||
building-integrity monitoring from passive ambient WiFi.
|
||||
|
||||
Test:
|
||||
1. Take the existing 1,077 CSI windows. Split first 50% = "before",
|
||||
last 50% = "after".
|
||||
2. Inject a synthetic "structural perturbation" into the "after"
|
||||
half — multiply 3 subcarriers by 0.85 (simulating a new reflective
|
||||
surface that attenuates those frequencies).
|
||||
3. For each half, stack the windows into a `[N, 56]` per-frame
|
||||
matrix (each row = one timestep), compute SVD, take the top-10
|
||||
singular values.
|
||||
4. Measure: do the singular-value spectra differ in a way that
|
||||
distinguishes "structural perturbation present" from "no
|
||||
perturbation"?
|
||||
5. Repeat with NO perturbation as control — the same first-half /
|
||||
second-half split should produce *similar* spectra (just temporal
|
||||
drift from operator movement, not structural).
|
||||
|
||||
If the perturbed-vs-control eigenvalue spectra are distinguishable by
|
||||
a simple distance metric, RF-weather detection is feasible.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
|
||||
N_SUB, N_FRAMES = 56, 20
|
||||
|
||||
|
||||
def load_windows(path: Path, max_samples: int | None = None) -> np.ndarray:
|
||||
csis = []
|
||||
with path.open(encoding="utf-8") as f:
|
||||
for line in f:
|
||||
if not line.strip():
|
||||
continue
|
||||
d = json.loads(line)
|
||||
shape = d.get("csi_shape", [N_SUB, N_FRAMES])
|
||||
if shape != [N_SUB, N_FRAMES]:
|
||||
continue
|
||||
csi = np.asarray(d["csi"], dtype=np.float32).reshape(N_SUB, N_FRAMES)
|
||||
csis.append(csi)
|
||||
if max_samples and len(csis) >= max_samples:
|
||||
break
|
||||
return np.stack(csis)
|
||||
|
||||
|
||||
def perturb_subcarriers(X: np.ndarray, indices: list[int], gain: float) -> np.ndarray:
|
||||
"""Multiply the listed subcarriers by `gain` to simulate a structural
|
||||
change (e.g. a new reflector attenuates certain frequencies)."""
|
||||
out = X.copy()
|
||||
out[:, indices, :] *= gain
|
||||
return out
|
||||
|
||||
|
||||
def per_frame_matrix(X: np.ndarray) -> np.ndarray:
|
||||
"""Stack all windows' frames into a [N_total_frames, 56] matrix.
|
||||
Each row is one timestep, used as a multivariate observation of the
|
||||
56-subcarrier channel state."""
|
||||
return X.transpose(0, 2, 1).reshape(-1, N_SUB)
|
||||
|
||||
|
||||
def top_k_singular_values(M: np.ndarray, k: int = 10) -> np.ndarray:
|
||||
"""Compute SVD on M, return top-k singular values."""
|
||||
M_centered = M - M.mean(axis=0, keepdims=True)
|
||||
# Use SVD on the centered matrix (== PCA without normalisation)
|
||||
s = np.linalg.svd(M_centered, compute_uv=False)
|
||||
return s[:k]
|
||||
|
||||
|
||||
def spectrum_distance(s1: np.ndarray, s2: np.ndarray) -> float:
|
||||
"""Cosine distance between two singular-value spectra. 0 = identical
|
||||
direction, 2 = opposite. Symmetric, scale-invariant."""
|
||||
s1n = s1 / (np.linalg.norm(s1) + 1e-9)
|
||||
s2n = s2 / (np.linalg.norm(s2) + 1e-9)
|
||||
return float(1.0 - np.dot(s1n, s2n))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--paired", required=True)
|
||||
parser.add_argument("--out", default="examples/research-sota/r12_rf_weather_results.json")
|
||||
parser.add_argument("--perturb-indices", default="30,41,52",
|
||||
help="comma-separated subcarrier indices to perturb (chosen from R5's top-saliency list)")
|
||||
parser.add_argument("--perturb-gain", type=float, default=0.85)
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"Loading windows from {args.paired}")
|
||||
X = load_windows(Path(args.paired))
|
||||
print(f" total windows: {X.shape[0]} (shape {X.shape})")
|
||||
|
||||
n = X.shape[0]
|
||||
half = n // 2
|
||||
X_before = X[:half]
|
||||
X_after_raw = X[half:] # unmodified second half — the CONTROL
|
||||
perturb_idx = [int(x) for x in args.perturb_indices.split(",")]
|
||||
X_after_perturbed = perturb_subcarriers(X_after_raw, perturb_idx, args.perturb_gain)
|
||||
|
||||
# Convert each half to a [N_frames, 56] matrix
|
||||
M_before = per_frame_matrix(X_before)
|
||||
M_after_raw = per_frame_matrix(X_after_raw)
|
||||
M_after_pert = per_frame_matrix(X_after_perturbed)
|
||||
print(f" per-frame matrix: before={M_before.shape}, after={M_after_raw.shape}")
|
||||
|
||||
# Top-10 singular values per half
|
||||
s_before = top_k_singular_values(M_before, k=10)
|
||||
s_after_raw = top_k_singular_values(M_after_raw, k=10)
|
||||
s_after_pert = top_k_singular_values(M_after_pert, k=10)
|
||||
|
||||
print(f"\n Singular value spectra (top-10):")
|
||||
print(f" before : [{', '.join(f'{v:.1f}' for v in s_before)}]")
|
||||
print(f" after (raw) : [{', '.join(f'{v:.1f}' for v in s_after_raw)}]")
|
||||
print(f" after (pert) : [{', '.join(f'{v:.1f}' for v in s_after_pert)}]")
|
||||
|
||||
# Distances
|
||||
d_raw = spectrum_distance(s_before, s_after_raw)
|
||||
d_pert = spectrum_distance(s_before, s_after_pert)
|
||||
|
||||
print(f"\n Cosine distances from BEFORE:")
|
||||
print(f" before -> after raw (control, no perturbation): {d_raw:.5f}")
|
||||
print(f" before -> after pert (synthetic structural shift): {d_pert:.5f}")
|
||||
|
||||
# Distance ratio = how much the perturbation amplifies the detection signal
|
||||
# over the natural temporal drift.
|
||||
if d_raw > 1e-9:
|
||||
ratio = d_pert / d_raw
|
||||
print(f"\n Signal-to-natural-drift ratio: {ratio:.2f}x")
|
||||
|
||||
if d_pert > d_raw * 3:
|
||||
verdict = "STRONG: perturbation easily distinguishable from natural temporal drift"
|
||||
elif d_pert > d_raw * 1.5:
|
||||
verdict = "MODERATE: perturbation detectable but with margin"
|
||||
else:
|
||||
verdict = "WEAK: structural perturbation gets lost in temporal drift"
|
||||
print(f"\n Verdict: {verdict}")
|
||||
|
||||
out = {
|
||||
"perturbation": {
|
||||
"subcarrier_indices": perturb_idx,
|
||||
"amplitude_gain": args.perturb_gain,
|
||||
"comment": "simulates a new reflective surface that attenuates these frequencies",
|
||||
},
|
||||
"n_before_windows": int(half),
|
||||
"n_after_windows": int(n - half),
|
||||
"spectra": {
|
||||
"before": s_before.tolist(),
|
||||
"after_raw_control": s_after_raw.tolist(),
|
||||
"after_perturbed": s_after_pert.tolist(),
|
||||
},
|
||||
"distances": {
|
||||
"before_to_after_raw": d_raw,
|
||||
"before_to_after_perturbed": d_pert,
|
||||
"signal_over_natural_drift": float(d_pert / max(d_raw, 1e-9)),
|
||||
},
|
||||
"verdict": verdict,
|
||||
}
|
||||
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.out).write_text(json.dumps(out, indent=2))
|
||||
print(f"\nWrote {args.out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"perturbation": {
|
||||
"subcarrier_indices": [
|
||||
30,
|
||||
41,
|
||||
52
|
||||
],
|
||||
"amplitude_gain": 0.85,
|
||||
"comment": "simulates a new reflective surface that attenuates these frequencies"
|
||||
},
|
||||
"n_before_windows": 538,
|
||||
"n_after_windows": 539,
|
||||
"spectra": {
|
||||
"before": [
|
||||
2220.65673828125,
|
||||
1856.8695068359375,
|
||||
1563.7314453125,
|
||||
1303.56298828125,
|
||||
1057.757080078125,
|
||||
770.67822265625,
|
||||
757.5601196289062,
|
||||
689.5866088867188,
|
||||
595.6748046875,
|
||||
556.3777465820312
|
||||
],
|
||||
"after_raw_control": [
|
||||
2182.5712890625,
|
||||
1837.5084228515625,
|
||||
1647.6357421875,
|
||||
1315.103759765625,
|
||||
1053.489013671875,
|
||||
794.1417236328125,
|
||||
737.1859130859375,
|
||||
704.1968994140625,
|
||||
571.363037109375,
|
||||
535.6047973632812
|
||||
],
|
||||
"after_perturbed": [
|
||||
2172.6552734375,
|
||||
1824.164794921875,
|
||||
1615.7850341796875,
|
||||
1304.227783203125,
|
||||
1040.461181640625,
|
||||
791.2919921875,
|
||||
736.2902221679688,
|
||||
691.3584594726562,
|
||||
568.5400390625,
|
||||
530.7666625976562
|
||||
]
|
||||
},
|
||||
"distances": {
|
||||
"before_to_after_raw": 0.0003509521484375,
|
||||
"before_to_after_perturbed": 0.00024056434631347656,
|
||||
"signal_over_natural_drift": 0.6854619565217391
|
||||
},
|
||||
"verdict": "WEAK: structural perturbation gets lost in temporal drift"
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
#!/usr/bin/env python3
|
||||
"""R5 — per-subcarrier input×gradient saliency for the count + pose cogs.
|
||||
|
||||
See docs/research/sota-2026-05-22/R5-subcarrier-saliency.md for context.
|
||||
|
||||
Usage:
|
||||
python examples/research-sota/r5_subcarrier_saliency.py \
|
||||
--paired data/paired/wiflow-p7-1779210883.paired.jsonl \
|
||||
--model v2/crates/cog-person-count/cog/artifacts/count_v1.safetensors \
|
||||
--kind count
|
||||
python examples/research-sota/r5_subcarrier_saliency.py \
|
||||
--paired data/paired/wiflow-p7-1779210883.paired.jsonl \
|
||||
--model v2/crates/cog-pose-estimation/cog/artifacts/pose_v1.safetensors \
|
||||
--kind pose
|
||||
|
||||
Output:
|
||||
<dirname-of-model>/saliency.json per-subcarrier saliency + top-K lists
|
||||
stdout summary table
|
||||
|
||||
Method (per ADR/research note):
|
||||
S_k = E_samples[ |dL/dx_k| * |x_k| ]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import struct
|
||||
from pathlib import Path
|
||||
from typing import Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
N_SUB, N_FRAMES = 56, 20
|
||||
|
||||
|
||||
def load_paired(path: Path, kind: str, max_samples: int | None = None) -> Tuple[np.ndarray, np.ndarray]:
|
||||
"""Returns (X, y) — X is [N, 56, 20] float32, y depends on kind.
|
||||
|
||||
kind="count" → y is [N] int64 in {0..7}
|
||||
kind="pose" → y is [N, 17, 2] float32 in [0, 1]
|
||||
"""
|
||||
csis, ys = [], []
|
||||
with path.open(encoding="utf-8") as f:
|
||||
for line in f:
|
||||
if not line.strip():
|
||||
continue
|
||||
d = json.loads(line)
|
||||
shape = d.get("csi_shape", [N_SUB, N_FRAMES])
|
||||
if shape != [N_SUB, N_FRAMES]:
|
||||
continue
|
||||
csi = np.asarray(d["csi"], dtype=np.float32).reshape(N_SUB, N_FRAMES)
|
||||
csis.append(csi)
|
||||
if kind == "count":
|
||||
ys.append(int(d.get("n_persons_mode", 0)))
|
||||
elif kind == "pose":
|
||||
ys.append(np.asarray(d.get("kp", []), dtype=np.float32))
|
||||
else:
|
||||
raise ValueError(f"unknown kind: {kind}")
|
||||
if max_samples and len(csis) >= max_samples:
|
||||
break
|
||||
return np.stack(csis), np.asarray(ys, dtype=(np.int64 if kind == "count" else np.float32))
|
||||
|
||||
|
||||
def load_safetensors(path: Path) -> dict[str, np.ndarray]:
|
||||
"""Pure-python safetensors reader. Returns {name: ndarray}."""
|
||||
with path.open("rb") as f:
|
||||
hlen = struct.unpack("<Q", f.read(8))[0]
|
||||
header = json.loads(f.read(hlen).decode("utf-8"))
|
||||
out = {}
|
||||
for name, meta in header.items():
|
||||
if name == "__metadata__":
|
||||
continue
|
||||
start, end = meta["data_offsets"]
|
||||
shape = meta["shape"]
|
||||
assert meta["dtype"] == "F32", f"unsupported dtype {meta['dtype']} in {name}"
|
||||
f.seek(8 + hlen + start)
|
||||
buf = f.read(end - start)
|
||||
arr = np.frombuffer(buf, dtype=np.float32).copy().reshape(shape)
|
||||
out[name] = arr
|
||||
return out
|
||||
|
||||
|
||||
def conv1d_forward(x: np.ndarray, w: np.ndarray, b: np.ndarray, padding: int, dilation: int) -> np.ndarray:
|
||||
"""Pure-numpy Conv1d forward. x: [B, Cin, T], w: [Cout, Cin, K]. Returns [B, Cout, T']."""
|
||||
B, Cin, T = x.shape
|
||||
Cout, _, K = w.shape
|
||||
# Pad
|
||||
xp = np.pad(x, ((0, 0), (0, 0), (padding, padding)), mode="constant")
|
||||
Tp = xp.shape[2]
|
||||
# Effective filter span with dilation
|
||||
eff = (K - 1) * dilation + 1
|
||||
Tout = Tp - eff + 1
|
||||
out = np.zeros((B, Cout, Tout), dtype=np.float32)
|
||||
for k in range(K):
|
||||
# x_slice shape: [B, Cin, Tout]
|
||||
x_slice = xp[:, :, k * dilation : k * dilation + Tout]
|
||||
# w_slice shape: [Cout, Cin]
|
||||
w_slice = w[:, :, k]
|
||||
# einsum: B,Cin,T x Cout,Cin → B,Cout,T
|
||||
out += np.einsum("bct,oc->bot", x_slice, w_slice)
|
||||
return out + b[None, :, None]
|
||||
|
||||
|
||||
def relu(x: np.ndarray) -> np.ndarray:
|
||||
return np.maximum(x, 0.0)
|
||||
|
||||
|
||||
def softmax(x: np.ndarray, axis: int = -1) -> np.ndarray:
|
||||
m = x.max(axis=axis, keepdims=True)
|
||||
e = np.exp(x - m)
|
||||
return e / e.sum(axis=axis, keepdims=True)
|
||||
|
||||
|
||||
def forward_count(x: np.ndarray, w: dict[str, np.ndarray]) -> np.ndarray:
|
||||
"""CountNet forward. x: [B, 56, 20] → probs [B, 8]."""
|
||||
h = conv1d_forward(x, w["enc.c1.weight"], w["enc.c1.bias"], padding=1, dilation=1)
|
||||
h = relu(h)
|
||||
h = conv1d_forward(h, w["enc.c2.weight"], w["enc.c2.bias"], padding=2, dilation=2)
|
||||
h = relu(h)
|
||||
h = conv1d_forward(h, w["enc.c3.weight"], w["enc.c3.bias"], padding=4, dilation=4)
|
||||
h = relu(h)
|
||||
h = h.mean(axis=2) # [B, 128]
|
||||
# count head
|
||||
z = relu(h @ w["count_head.fc1.weight"].T + w["count_head.fc1.bias"])
|
||||
z = z @ w["count_head.fc2.weight"].T + w["count_head.fc2.bias"]
|
||||
return softmax(z, axis=-1)
|
||||
|
||||
|
||||
def saliency_input_gradient(
|
||||
X: np.ndarray,
|
||||
y: np.ndarray,
|
||||
weights: dict[str, np.ndarray],
|
||||
kind: str,
|
||||
eps: float = 1e-3,
|
||||
) -> np.ndarray:
|
||||
"""Per-subcarrier saliency: S_k = E[|dL/dx_k| * |x_k|].
|
||||
|
||||
Uses central-difference numerical gradient over each subcarrier (cheap because
|
||||
we marginalise over the time axis after taking the abs). For a 56-subcarrier
|
||||
input that's 56 forward passes per sample — slow but exact, and only runs
|
||||
once per saliency map.
|
||||
"""
|
||||
B, N_sub, T = X.shape
|
||||
saliency = np.zeros(N_sub, dtype=np.float64)
|
||||
|
||||
if kind == "count":
|
||||
# Loss = -log(p_true). Compute baseline log-prob.
|
||||
for k in range(N_sub):
|
||||
x_plus = X.copy()
|
||||
x_plus[:, k, :] += eps
|
||||
x_minus = X.copy()
|
||||
x_minus[:, k, :] -= eps
|
||||
p_plus = forward_count(x_plus, weights)
|
||||
p_minus = forward_count(x_minus, weights)
|
||||
# dL/dx ≈ -(log p_plus[y] - log p_minus[y]) / (2*eps)
|
||||
idx = np.arange(B)
|
||||
lp_plus = np.log(p_plus[idx, y] + 1e-12)
|
||||
lp_minus = np.log(p_minus[idx, y] + 1e-12)
|
||||
grad_k = -(lp_plus - lp_minus) / (2 * eps) # [B]
|
||||
# |dL/dx_k| * |x_k| — x_k is a vector over time; take its magnitude
|
||||
x_k_mag = np.abs(X[:, k, :]).mean(axis=1) # [B]
|
||||
saliency[k] += float((np.abs(grad_k) * x_k_mag).mean())
|
||||
else:
|
||||
raise NotImplementedError("pose kind not yet wired — count first")
|
||||
|
||||
return saliency
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--paired", required=True)
|
||||
parser.add_argument("--model", required=True)
|
||||
parser.add_argument("--kind", choices=["count", "pose"], default="count")
|
||||
parser.add_argument("--max-samples", type=int, default=128,
|
||||
help="Cap on samples used for saliency (saliency cost is O(N_sub × samples × eps_passes))")
|
||||
parser.add_argument("--out", default=None,
|
||||
help="Output JSON path; defaults to <model_dir>/saliency.json")
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"Loading paired data from {args.paired} (kind={args.kind})")
|
||||
X, y = load_paired(Path(args.paired), kind=args.kind, max_samples=args.max_samples)
|
||||
print(f" X: {X.shape}, y: {y.shape}")
|
||||
if args.kind == "count":
|
||||
unique, counts = np.unique(y, return_counts=True)
|
||||
print(f" label distribution: {dict(zip(unique.tolist(), counts.tolist()))}")
|
||||
|
||||
# Standardise (per-subcarrier z-score using THIS subset's stats — saliency is
|
||||
# invariant to affine input transforms in the limit of small eps).
|
||||
mu = X.mean(axis=(0, 2), keepdims=True)
|
||||
sd = X.std(axis=(0, 2), keepdims=True) + 1e-6
|
||||
X_norm = (X - mu) / sd
|
||||
|
||||
print(f"Loading weights from {args.model}")
|
||||
weights = load_safetensors(Path(args.model))
|
||||
print(f" loaded {len(weights)} tensors: {sorted(list(weights.keys()))[:6]}...")
|
||||
|
||||
print(f"Computing input×gradient saliency over {X.shape[0]} samples × 56 subcarriers...")
|
||||
saliency = saliency_input_gradient(X_norm, y, weights, kind=args.kind, eps=1e-3)
|
||||
|
||||
order = np.argsort(saliency)[::-1] # descending
|
||||
top_k = {k: order[:k].tolist() for k in (8, 16, 32)}
|
||||
|
||||
out = {
|
||||
"kind": args.kind,
|
||||
"model": str(args.model),
|
||||
"n_samples": int(X.shape[0]),
|
||||
"saliency_per_subcarrier": saliency.tolist(),
|
||||
"ranking_high_to_low": order.tolist(),
|
||||
"top_k_subcarriers": top_k,
|
||||
"saliency_summary": {
|
||||
"min": float(saliency.min()),
|
||||
"max": float(saliency.max()),
|
||||
"mean": float(saliency.mean()),
|
||||
"std": float(saliency.std()),
|
||||
"max_to_mean_ratio": float(saliency.max() / max(saliency.mean(), 1e-12)),
|
||||
},
|
||||
}
|
||||
|
||||
out_path = Path(args.out) if args.out else Path(args.model).parent / "saliency.json"
|
||||
out_path.write_text(json.dumps(out, indent=2))
|
||||
print(f"\nWrote {out_path}")
|
||||
print(f"\nTop 8 subcarriers (most influential):")
|
||||
for rank, idx in enumerate(order[:8]):
|
||||
print(f" #{rank + 1}: subcarrier {int(idx):2d} saliency={saliency[idx]:.4f}")
|
||||
print(f"\nMax/mean ratio: {out['saliency_summary']['max_to_mean_ratio']:.2f}× "
|
||||
f"(higher = signal more concentrated in a few subcarriers)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,208 @@
|
||||
#!/usr/bin/env python3
|
||||
"""R7 — multi-link consistency detection via Stoer-Wagner-style mincut.
|
||||
|
||||
See docs/research/sota-2026-05-22/R7-multilink-consistency.md.
|
||||
|
||||
Premise: in a multi-node CSI mesh, all nodes observe the same physical
|
||||
scene through slightly different channels. Their per-window CSI features
|
||||
should cluster tightly under a similarity metric. If one node is
|
||||
compromised (spoofed CSI, replay attack, jamming-induced corruption), its
|
||||
features fall outside the cluster — and the mincut of the inter-node
|
||||
similarity graph isolates it cleanly.
|
||||
|
||||
This demo:
|
||||
1. Synthesises 4 "honest" CSI windows from one underlying scene + per-node
|
||||
Gaussian noise (realistic multipath variability).
|
||||
2. Synthesises 1 "adversarial" CSI window via three attack modes:
|
||||
(a) replay — paste in a stale window from earlier
|
||||
(b) shift — add a constant offset to every subcarrier
|
||||
(c) noise — pure white noise of the same magnitude as honest CSI
|
||||
3. Builds a 5×5 cross-node CSI cosine-similarity matrix.
|
||||
4. Solves Stoer-Wagner mincut on the resulting graph.
|
||||
5. Reports whether the mincut partition isolates the adversarial node.
|
||||
|
||||
No framework deps — pure NumPy.
|
||||
|
||||
Usage:
|
||||
python examples/research-sota/r7_multilink_consistency.py \
|
||||
--paired data/paired/wiflow-p7-1779210883.paired.jsonl
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
|
||||
N_SUB, N_FRAMES = 56, 20
|
||||
|
||||
|
||||
def load_one_window(path: Path, idx: int = 0) -> np.ndarray:
|
||||
"""Pull one [56, 20] CSI window from the paired data — the scene we'll synthesise around."""
|
||||
with path.open(encoding="utf-8") as f:
|
||||
for i, line in enumerate(f):
|
||||
if i < idx:
|
||||
continue
|
||||
d = json.loads(line)
|
||||
shape = d.get("csi_shape", [N_SUB, N_FRAMES])
|
||||
if shape == [N_SUB, N_FRAMES]:
|
||||
return np.asarray(d["csi"], dtype=np.float32).reshape(N_SUB, N_FRAMES)
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def synth_honest_nodes(base: np.ndarray, n_nodes: int = 4, noise_db: float = 6.0, seed: int = 42):
|
||||
"""`n_nodes` honest observers — each sees the base scene through independent multipath
|
||||
(modelled as additive Gaussian on the per-subcarrier amplitudes at `noise_db` below signal)."""
|
||||
rng = np.random.default_rng(seed)
|
||||
sigma = base.std() * 10 ** (-noise_db / 20.0)
|
||||
return np.stack([base + rng.normal(0, sigma, size=base.shape).astype(np.float32) for _ in range(n_nodes)])
|
||||
|
||||
|
||||
def synth_adversarial(base: np.ndarray, mode: str, replay_window: np.ndarray | None = None, seed: int = 7):
|
||||
"""One adversarial observer. `mode` ∈ {replay, shift, noise}."""
|
||||
rng = np.random.default_rng(seed)
|
||||
if mode == "replay":
|
||||
if replay_window is None:
|
||||
raise ValueError("replay needs a stale window")
|
||||
# Stale window with a tiny perturbation to look "fresh"
|
||||
return replay_window + rng.normal(0, 0.01, size=base.shape).astype(np.float32)
|
||||
if mode == "shift":
|
||||
return base + 3.0 * base.std() # constant offset — gives away the attack
|
||||
if mode == "noise":
|
||||
return rng.normal(base.mean(), base.std(), size=base.shape).astype(np.float32)
|
||||
raise ValueError(f"unknown adversarial mode: {mode}")
|
||||
|
||||
|
||||
def cosine_sim_matrix(windows: np.ndarray) -> np.ndarray:
|
||||
"""Pairwise cosine similarity on flattened windows. Returns [N, N] matrix."""
|
||||
flat = windows.reshape(windows.shape[0], -1)
|
||||
norms = np.linalg.norm(flat, axis=1, keepdims=True) + 1e-9
|
||||
normalized = flat / norms
|
||||
return normalized @ normalized.T
|
||||
|
||||
|
||||
def stoer_wagner_mincut(W: np.ndarray) -> tuple[float, list[int]]:
|
||||
"""Classical Stoer-Wagner mincut. Input: symmetric [N, N] non-negative weights.
|
||||
|
||||
Returns: (cut_value, partition_a_node_indices)
|
||||
|
||||
The algorithm:
|
||||
while G has more than one node:
|
||||
do a minimum-cut-phase: find the order in which nodes are added
|
||||
the last node added is one side of a candidate cut; the rest is the other side
|
||||
merge the last two nodes into one super-node, accumulate their weights
|
||||
track the minimum candidate cut across all phases
|
||||
"""
|
||||
n = W.shape[0]
|
||||
nodes = [{i} for i in range(n)] # start with each node a singleton
|
||||
W = W.astype(np.float64).copy()
|
||||
best_cut = np.inf
|
||||
best_partition_b = None
|
||||
|
||||
while len(nodes) > 1:
|
||||
# minimum-cut-phase
|
||||
n_left = len(nodes)
|
||||
A = [0] # start anywhere
|
||||
in_A = np.zeros(n_left, dtype=bool); in_A[0] = True
|
||||
weights_to_A = W[:, 0].copy()
|
||||
weights_to_A[0] = -1
|
||||
last, second_last = 0, 0
|
||||
for _ in range(n_left - 1):
|
||||
# pick the not-yet-in-A node most tightly connected to A
|
||||
cand = int(np.argmax(np.where(in_A, -1, weights_to_A)))
|
||||
second_last = last
|
||||
last = cand
|
||||
in_A[cand] = True
|
||||
A.append(cand)
|
||||
# update weights — add cand's edges
|
||||
weights_to_A = np.where(in_A, -1, weights_to_A + W[:, cand])
|
||||
|
||||
# cut-of-the-phase = sum of edges from `last` to all others
|
||||
cut_val = float((W[last, :].sum() - W[last, last]))
|
||||
if cut_val < best_cut:
|
||||
best_cut = cut_val
|
||||
best_partition_b = nodes[last].copy()
|
||||
|
||||
# merge last + second_last
|
||||
merged = nodes[last] | nodes[second_last]
|
||||
# merge their rows/cols
|
||||
W[second_last, :] += W[last, :]
|
||||
W[:, second_last] += W[:, last]
|
||||
W[second_last, second_last] = 0
|
||||
# remove `last`
|
||||
keep = [i for i in range(n_left) if i != last]
|
||||
W = W[np.ix_(keep, keep)]
|
||||
nodes = [merged if i == second_last else nodes[i] for i in keep]
|
||||
|
||||
partition_b = sorted(best_partition_b) if best_partition_b else []
|
||||
return best_cut, partition_b
|
||||
|
||||
|
||||
def run_scenario(base: np.ndarray, replay_window: np.ndarray, mode: str, n_honest: int = 4):
|
||||
"""Run one adversarial scenario, return diagnostic info."""
|
||||
honest = synth_honest_nodes(base, n_nodes=n_honest, noise_db=6.0)
|
||||
adv = synth_adversarial(base, mode=mode, replay_window=replay_window)
|
||||
windows = np.concatenate([honest, adv[None, ...]], axis=0) # [n_honest + 1, 56, 20]
|
||||
adv_idx = n_honest # last node is the adversarial one
|
||||
|
||||
sim = cosine_sim_matrix(windows)
|
||||
# Convert similarity → edge weight. Mincut on similarity finds the
|
||||
# minimum-similarity partition, which is the *most-suspicious* split.
|
||||
# Use (1 - sim) as the weight if we want to minimise dissimilarity, but
|
||||
# the natural framing is: mincut over similarity-weighted graph isolates
|
||||
# the node least-similar to the rest.
|
||||
np.fill_diagonal(sim, 0.0)
|
||||
|
||||
cut_val, partition_b = stoer_wagner_mincut(sim)
|
||||
detected = (set(partition_b) == {adv_idx}) or (set(range(len(windows))) - set(partition_b) == {adv_idx})
|
||||
|
||||
return {
|
||||
"mode": mode,
|
||||
"n_honest": n_honest,
|
||||
"adv_idx": adv_idx,
|
||||
"sim_matrix": sim.round(4).tolist(),
|
||||
"mincut_value": float(cut_val),
|
||||
"partition_b": partition_b,
|
||||
"adv_isolated": bool(detected),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--paired", required=True)
|
||||
parser.add_argument("--out", default="examples/research-sota/r7_multilink_consistency_results.json")
|
||||
args = parser.parse_args()
|
||||
|
||||
base = load_one_window(Path(args.paired), idx=10)
|
||||
stale = load_one_window(Path(args.paired), idx=900)
|
||||
if base is None or stale is None:
|
||||
raise SystemExit("need at least 901 samples in the paired file")
|
||||
|
||||
results = {}
|
||||
for mode in ["replay", "shift", "noise"]:
|
||||
scenario = run_scenario(base, stale, mode=mode, n_honest=4)
|
||||
results[mode] = scenario
|
||||
print(f"\n=== adversarial mode: {mode} ===")
|
||||
print(f" mincut value: {scenario['mincut_value']:.4f}")
|
||||
print(f" partition B (less-similar side): {scenario['partition_b']}")
|
||||
print(f" adversarial node isolated? {'YES' if scenario['adv_isolated'] else 'no'}")
|
||||
|
||||
n_detected = sum(1 for r in results.values() if r["adv_isolated"])
|
||||
summary = {
|
||||
"n_scenarios": len(results),
|
||||
"n_detected": n_detected,
|
||||
"detection_rate": n_detected / len(results),
|
||||
}
|
||||
print(f"\n=== summary ===")
|
||||
print(f" detection rate: {n_detected}/{len(results)} = {summary['detection_rate']:.0%}")
|
||||
|
||||
out_path = Path(args.out)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text(json.dumps({"summary": summary, "scenarios": results}, indent=2))
|
||||
print(f"\nWrote {out_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,150 @@
|
||||
{
|
||||
"summary": {
|
||||
"n_scenarios": 3,
|
||||
"n_detected": 3,
|
||||
"detection_rate": 1.0
|
||||
},
|
||||
"scenarios": {
|
||||
"replay": {
|
||||
"mode": "replay",
|
||||
"n_honest": 4,
|
||||
"adv_idx": 4,
|
||||
"sim_matrix": [
|
||||
[
|
||||
0.0,
|
||||
0.9218999743461609,
|
||||
0.9277999997138977,
|
||||
0.9269000291824341,
|
||||
0.863099992275238
|
||||
],
|
||||
[
|
||||
0.9218999743461609,
|
||||
0.0,
|
||||
0.9218999743461609,
|
||||
0.9254000186920166,
|
||||
0.8618999719619751
|
||||
],
|
||||
[
|
||||
0.9277999997138977,
|
||||
0.9218999743461609,
|
||||
0.0,
|
||||
0.9291999936103821,
|
||||
0.8615999817848206
|
||||
],
|
||||
[
|
||||
0.9269000291824341,
|
||||
0.9254000186920166,
|
||||
0.9291999936103821,
|
||||
0.0,
|
||||
0.864799976348877
|
||||
],
|
||||
[
|
||||
0.863099992275238,
|
||||
0.8618999719619751,
|
||||
0.8615999817848206,
|
||||
0.864799976348877,
|
||||
0.0
|
||||
]
|
||||
],
|
||||
"mincut_value": 3.451315999031067,
|
||||
"partition_b": [
|
||||
4
|
||||
],
|
||||
"adv_isolated": true
|
||||
},
|
||||
"shift": {
|
||||
"mode": "shift",
|
||||
"n_honest": 4,
|
||||
"adv_idx": 4,
|
||||
"sim_matrix": [
|
||||
[
|
||||
0.0,
|
||||
0.9218999743461609,
|
||||
0.9277999997138977,
|
||||
0.9269000291824341,
|
||||
0.8944000005722046
|
||||
],
|
||||
[
|
||||
0.9218999743461609,
|
||||
0.0,
|
||||
0.9218999743461609,
|
||||
0.9254000186920166,
|
||||
0.8917999863624573
|
||||
],
|
||||
[
|
||||
0.9277999997138977,
|
||||
0.9218999743461609,
|
||||
0.0,
|
||||
0.9291999936103821,
|
||||
0.8942999839782715
|
||||
],
|
||||
[
|
||||
0.9269000291824341,
|
||||
0.9254000186920166,
|
||||
0.9291999936103821,
|
||||
0.0,
|
||||
0.8917999863624573
|
||||
],
|
||||
[
|
||||
0.8944000005722046,
|
||||
0.8917999863624573,
|
||||
0.8942999839782715,
|
||||
0.8917999863624573,
|
||||
0.0
|
||||
]
|
||||
],
|
||||
"mincut_value": 3.5724358558654785,
|
||||
"partition_b": [
|
||||
4
|
||||
],
|
||||
"adv_isolated": true
|
||||
},
|
||||
"noise": {
|
||||
"mode": "noise",
|
||||
"n_honest": 4,
|
||||
"adv_idx": 4,
|
||||
"sim_matrix": [
|
||||
[
|
||||
0.0,
|
||||
0.9218999743461609,
|
||||
0.9277999997138977,
|
||||
0.9269000291824341,
|
||||
0.6425999999046326
|
||||
],
|
||||
[
|
||||
0.9218999743461609,
|
||||
0.0,
|
||||
0.9218999743461609,
|
||||
0.9254000186920166,
|
||||
0.6444000005722046
|
||||
],
|
||||
[
|
||||
0.9277999997138977,
|
||||
0.9218999743461609,
|
||||
0.0,
|
||||
0.9291999936103821,
|
||||
0.6389999985694885
|
||||
],
|
||||
[
|
||||
0.9269000291824341,
|
||||
0.9254000186920166,
|
||||
0.9291999936103821,
|
||||
0.0,
|
||||
0.6326000094413757
|
||||
],
|
||||
[
|
||||
0.6425999999046326,
|
||||
0.6444000005722046,
|
||||
0.6389999985694885,
|
||||
0.6326000094413757,
|
||||
0.0
|
||||
]
|
||||
],
|
||||
"mincut_value": 2.5585585832595825,
|
||||
"partition_b": [
|
||||
4
|
||||
],
|
||||
"adv_isolated": true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env python3
|
||||
"""R8 — RSSI-only person count: how much accuracy do we lose vs full CSI?
|
||||
|
||||
See docs/research/sota-2026-05-22/R8-rssi-only-count.md.
|
||||
|
||||
RSSI = received signal strength = power integrated across the WiFi band.
|
||||
The CSI amplitude vector for a single packet is `|H_k|` per subcarrier k;
|
||||
its mean over subcarriers is an unbiased proxy for the per-packet RSSI
|
||||
(equivalent up to constant scaling). So aggregating our existing
|
||||
`[56 subcarriers × 20 frames]` CSI windows along the subcarrier axis gives
|
||||
us a `[20]` "RSSI-over-time" signal — exactly what any WiFi chip without
|
||||
CSI export reports as its standard `RSSI` field.
|
||||
|
||||
If a small MLP on the [20]-vector hits even 55-60% accuracy on the
|
||||
person-count task, RSSI-only deployment is viable across the entire WiFi-
|
||||
chip ecosystem (billions of devices), at the cost of needing per-chip
|
||||
calibration. v0.0.2 of cog-person-count itself only hits 62% on the 80/20
|
||||
random split, so the bar isn't sky-high.
|
||||
|
||||
Usage:
|
||||
python examples/research-sota/r8_rssi_only_count.py \
|
||||
--paired data/paired/wiflow-p7-1779210883.paired.jsonl
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
N_SUB, N_FRAMES, COUNT_CLASSES = 56, 20, 8
|
||||
|
||||
|
||||
def load_paired(path: Path) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""Returns (X_csi, y) where X_csi is [N, 56, 20] and y is [N] integer count."""
|
||||
csis, ys = [], []
|
||||
with path.open(encoding="utf-8") as f:
|
||||
for line in f:
|
||||
if not line.strip():
|
||||
continue
|
||||
d = json.loads(line)
|
||||
shape = d.get("csi_shape", [N_SUB, N_FRAMES])
|
||||
if shape != [N_SUB, N_FRAMES]:
|
||||
continue
|
||||
csi = np.asarray(d["csi"], dtype=np.float32).reshape(N_SUB, N_FRAMES)
|
||||
csis.append(csi)
|
||||
ys.append(int(d.get("n_persons_mode", 0)))
|
||||
return np.stack(csis), np.asarray(ys, dtype=np.int64)
|
||||
|
||||
|
||||
def csi_to_rssi_proxy(X_csi: np.ndarray) -> np.ndarray:
|
||||
"""Aggregate CSI amplitudes to a single RSSI scalar per frame.
|
||||
|
||||
Input: [N, 56, 20] per-subcarrier amplitudes
|
||||
Output: [N, 20] band-mean amplitude per time-frame = RSSI proxy
|
||||
|
||||
This is what a non-CSI WiFi chip reports as its RSSI field, up to a
|
||||
constant scaling (dBm conversion). We keep linear amplitude — the count
|
||||
head is invariant to that affine transform after z-score normalisation.
|
||||
"""
|
||||
return X_csi.mean(axis=1) # mean across subcarriers
|
||||
|
||||
|
||||
def softmax(x: np.ndarray, axis: int = -1) -> np.ndarray:
|
||||
m = x.max(axis=axis, keepdims=True)
|
||||
e = np.exp(x - m)
|
||||
return e / e.sum(axis=axis, keepdims=True)
|
||||
|
||||
|
||||
def train_rssi_mlp(
|
||||
X_train: np.ndarray, y_train: np.ndarray,
|
||||
X_eval: np.ndarray, y_eval: np.ndarray,
|
||||
epochs: int = 200, lr: float = 1e-2, hidden: int = 32, seed: int = 42,
|
||||
):
|
||||
"""Tiny MLP trained with vanilla SGD — no framework, just numpy.
|
||||
|
||||
Input: [N, 20] RSSI-proxy time-series
|
||||
Architecture: Linear(20 → hidden) → ReLU → Linear(hidden → 8) → softmax
|
||||
"""
|
||||
rng = np.random.default_rng(seed)
|
||||
D = X_train.shape[1]
|
||||
K = COUNT_CLASSES
|
||||
|
||||
# Glorot init
|
||||
w1 = rng.normal(0, np.sqrt(2.0 / D), size=(D, hidden)).astype(np.float32)
|
||||
b1 = np.zeros(hidden, dtype=np.float32)
|
||||
w2 = rng.normal(0, np.sqrt(2.0 / hidden), size=(hidden, K)).astype(np.float32)
|
||||
b2 = np.zeros(K, dtype=np.float32)
|
||||
|
||||
n_train = X_train.shape[0]
|
||||
batch_size = 32
|
||||
eval_curve = []
|
||||
best_eval_acc = 0.0
|
||||
best = None
|
||||
|
||||
for epoch in range(epochs):
|
||||
perm = rng.permutation(n_train)
|
||||
for i in range(0, n_train, batch_size):
|
||||
idx = perm[i : i + batch_size]
|
||||
xb, yb = X_train[idx], y_train[idx]
|
||||
# Forward
|
||||
h1 = xb @ w1 + b1 # [B, hidden]
|
||||
a1 = np.maximum(h1, 0.0) # ReLU
|
||||
logits = a1 @ w2 + b2 # [B, K]
|
||||
probs = softmax(logits, axis=-1)
|
||||
# One-hot
|
||||
onehot = np.zeros_like(probs)
|
||||
onehot[np.arange(len(yb)), yb] = 1.0
|
||||
# Backward
|
||||
dlogits = (probs - onehot) / len(yb) # [B, K]
|
||||
dw2 = a1.T @ dlogits # [hidden, K]
|
||||
db2 = dlogits.sum(axis=0)
|
||||
da1 = dlogits @ w2.T # [B, hidden]
|
||||
dh1 = da1 * (h1 > 0) # ReLU grad
|
||||
dw1 = xb.T @ dh1 # [D, hidden]
|
||||
db1 = dh1.sum(axis=0)
|
||||
# SGD
|
||||
w1 -= lr * dw1
|
||||
b1 -= lr * db1
|
||||
w2 -= lr * dw2
|
||||
b2 -= lr * db2
|
||||
|
||||
# Eval
|
||||
eh = np.maximum(X_eval @ w1 + b1, 0.0)
|
||||
eval_logits = eh @ w2 + b2
|
||||
eval_pred = eval_logits.argmax(axis=1)
|
||||
eval_acc = float((eval_pred == y_eval).mean())
|
||||
eval_curve.append(eval_acc)
|
||||
if eval_acc > best_eval_acc:
|
||||
best_eval_acc = eval_acc
|
||||
best = (w1.copy(), b1.copy(), w2.copy(), b2.copy())
|
||||
|
||||
return best, best_eval_acc, eval_curve
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--paired", required=True)
|
||||
parser.add_argument("--out", default="examples/research-sota/r8_rssi_only_results.json")
|
||||
parser.add_argument("--epochs", type=int, default=200)
|
||||
parser.add_argument("--seed", type=int, default=42)
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"Loading paired data from {args.paired}")
|
||||
X_csi, y = load_paired(Path(args.paired))
|
||||
print(f" CSI shape: {X_csi.shape}")
|
||||
print(f" label distribution: {dict(Counter(y.tolist()).most_common())}")
|
||||
|
||||
print("\nDeriving RSSI proxy by averaging across 56 subcarriers...")
|
||||
X_rssi = csi_to_rssi_proxy(X_csi)
|
||||
print(f" RSSI proxy shape: {X_rssi.shape} (one scalar per frame, 20 frames per sample)")
|
||||
print(f" RSSI proxy stats: mean={X_rssi.mean():.3f} std={X_rssi.std():.3f}")
|
||||
|
||||
# Random 80/20 split — same seed as v0.0.2 so the eval set is identical
|
||||
rng = np.random.default_rng(seed=args.seed)
|
||||
idx = np.arange(X_rssi.shape[0])
|
||||
rng.shuffle(idx)
|
||||
n_eval = int(round(0.2 * X_rssi.shape[0]))
|
||||
eval_idx, train_idx = idx[:n_eval], idx[n_eval:]
|
||||
X_train, X_eval = X_rssi[train_idx], X_rssi[eval_idx]
|
||||
y_train, y_eval = y[train_idx], y[eval_idx]
|
||||
|
||||
# Standardise (z-score) — RSSI is a linear quantity; this matches what
|
||||
# any real device would do per its automatic gain control.
|
||||
mu = X_train.mean(axis=0, keepdims=True)
|
||||
sd = X_train.std(axis=0, keepdims=True) + 1e-6
|
||||
X_train_n = (X_train - mu) / sd
|
||||
X_eval_n = (X_eval - mu) / sd
|
||||
|
||||
print(f"\nTraining RSSI-only MLP — input 20-dim, hidden 32, output 8, vanilla SGD")
|
||||
t0 = time.perf_counter()
|
||||
best_params, best_eval_acc, curve = train_rssi_mlp(
|
||||
X_train_n, y_train, X_eval_n, y_eval,
|
||||
epochs=args.epochs, lr=1e-2, hidden=32, seed=args.seed,
|
||||
)
|
||||
elapsed = time.perf_counter() - t0
|
||||
print(f"\nTrained {args.epochs} epochs in {elapsed:.2f} s on CPU")
|
||||
|
||||
# Final eval with best checkpoint
|
||||
w1, b1, w2, b2 = best_params
|
||||
eh = np.maximum(X_eval_n @ w1 + b1, 0.0)
|
||||
eval_logits = eh @ w2 + b2
|
||||
eval_pred = eval_logits.argmax(axis=1)
|
||||
acc = float((eval_pred == y_eval).mean())
|
||||
per_class = {}
|
||||
for k in range(COUNT_CLASSES):
|
||||
mask = y_eval == k
|
||||
n = int(mask.sum())
|
||||
if n > 0:
|
||||
per_class[k] = {
|
||||
"support": n,
|
||||
"accuracy": float(((eval_pred == y_eval) & mask).sum() / n),
|
||||
}
|
||||
|
||||
# Baseline reference: how does v0.0.2 (full CSI) score on the SAME eval set?
|
||||
# We don't run the cog binary here — just record the published numbers.
|
||||
full_csi_baseline = {
|
||||
"version": "cog-person-count v0.0.2",
|
||||
"overall_acc": 0.623,
|
||||
"class0_acc": 0.862,
|
||||
"class1_acc": 0.343,
|
||||
"source": "docs/benchmarks/person-count-cog.md",
|
||||
}
|
||||
|
||||
print(f"\n=== R8 RSSI-only results ===")
|
||||
print(f" Eval accuracy: {acc:.3f}")
|
||||
print(f" Per-class:")
|
||||
for k, v in per_class.items():
|
||||
print(f" class {k}: {v['accuracy']:.3f} on {v['support']} samples")
|
||||
print(f"\n Full-CSI baseline (v0.0.2): {full_csi_baseline['overall_acc']:.3f}")
|
||||
print(f" Retained fraction: {acc / full_csi_baseline['overall_acc']:.2%}")
|
||||
|
||||
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.out).write_text(json.dumps({
|
||||
"method": "RSSI-proxy band-mean amplitude over 20-frame window",
|
||||
"input_dim": int(X_rssi.shape[1]),
|
||||
"architecture": "MLP(20 → 32 → 8) ReLU + softmax, vanilla SGD",
|
||||
"epochs": args.epochs,
|
||||
"train_time_s": elapsed,
|
||||
"n_train": int(X_train.shape[0]),
|
||||
"n_eval": int(X_eval.shape[0]),
|
||||
"label_distribution_train": dict(Counter(y_train.tolist()).most_common()),
|
||||
"label_distribution_eval": dict(Counter(y_eval.tolist()).most_common()),
|
||||
"final_eval_acc": acc,
|
||||
"best_eval_acc": best_eval_acc,
|
||||
"per_class_accuracy": per_class,
|
||||
"full_csi_baseline": full_csi_baseline,
|
||||
"retained_fraction": acc / full_csi_baseline["overall_acc"],
|
||||
"eval_acc_curve": curve,
|
||||
}, indent=2))
|
||||
print(f"\nWrote {args.out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,239 @@
|
||||
{
|
||||
"method": "RSSI-proxy band-mean amplitude over 20-frame window",
|
||||
"input_dim": 20,
|
||||
"architecture": "MLP(20 \u2192 32 \u2192 8) ReLU + softmax, vanilla SGD",
|
||||
"epochs": 200,
|
||||
"train_time_s": 0.717573200003244,
|
||||
"n_train": 862,
|
||||
"n_eval": 215,
|
||||
"label_distribution_train": {
|
||||
"1": 445,
|
||||
"0": 417
|
||||
},
|
||||
"label_distribution_eval": {
|
||||
"0": 116,
|
||||
"1": 99
|
||||
},
|
||||
"final_eval_acc": 0.5906976744186047,
|
||||
"best_eval_acc": 0.5906976744186047,
|
||||
"per_class_accuracy": {
|
||||
"0": {
|
||||
"support": 116,
|
||||
"accuracy": 0.5948275862068966
|
||||
},
|
||||
"1": {
|
||||
"support": 99,
|
||||
"accuracy": 0.5858585858585859
|
||||
}
|
||||
},
|
||||
"full_csi_baseline": {
|
||||
"version": "cog-person-count v0.0.2",
|
||||
"overall_acc": 0.623,
|
||||
"class0_acc": 0.862,
|
||||
"class1_acc": 0.343,
|
||||
"source": "docs/benchmarks/person-count-cog.md"
|
||||
},
|
||||
"retained_fraction": 0.9481503602224793,
|
||||
"eval_acc_curve": [
|
||||
0.3395348837209302,
|
||||
0.4604651162790698,
|
||||
0.4744186046511628,
|
||||
0.5116279069767442,
|
||||
0.5534883720930233,
|
||||
0.5395348837209303,
|
||||
0.5441860465116279,
|
||||
0.5302325581395348,
|
||||
0.5255813953488372,
|
||||
0.5348837209302325,
|
||||
0.5395348837209303,
|
||||
0.5395348837209303,
|
||||
0.5534883720930233,
|
||||
0.5534883720930233,
|
||||
0.5488372093023256,
|
||||
0.5441860465116279,
|
||||
0.5627906976744186,
|
||||
0.5674418604651162,
|
||||
0.5441860465116279,
|
||||
0.5581395348837209,
|
||||
0.5534883720930233,
|
||||
0.5581395348837209,
|
||||
0.5534883720930233,
|
||||
0.5488372093023256,
|
||||
0.5627906976744186,
|
||||
0.5488372093023256,
|
||||
0.5488372093023256,
|
||||
0.5441860465116279,
|
||||
0.586046511627907,
|
||||
0.5534883720930233,
|
||||
0.5441860465116279,
|
||||
0.5395348837209303,
|
||||
0.5534883720930233,
|
||||
0.5581395348837209,
|
||||
0.5534883720930233,
|
||||
0.5534883720930233,
|
||||
0.5441860465116279,
|
||||
0.5813953488372093,
|
||||
0.5534883720930233,
|
||||
0.5488372093023256,
|
||||
0.5534883720930233,
|
||||
0.5581395348837209,
|
||||
0.5767441860465117,
|
||||
0.5581395348837209,
|
||||
0.5534883720930233,
|
||||
0.5627906976744186,
|
||||
0.5906976744186047,
|
||||
0.5906976744186047,
|
||||
0.5581395348837209,
|
||||
0.5674418604651162,
|
||||
0.5581395348837209,
|
||||
0.5581395348837209,
|
||||
0.5534883720930233,
|
||||
0.5627906976744186,
|
||||
0.5627906976744186,
|
||||
0.5581395348837209,
|
||||
0.5813953488372093,
|
||||
0.5627906976744186,
|
||||
0.5581395348837209,
|
||||
0.5720930232558139,
|
||||
0.5627906976744186,
|
||||
0.5581395348837209,
|
||||
0.5627906976744186,
|
||||
0.5581395348837209,
|
||||
0.5627906976744186,
|
||||
0.5581395348837209,
|
||||
0.5581395348837209,
|
||||
0.5674418604651162,
|
||||
0.5627906976744186,
|
||||
0.5627906976744186,
|
||||
0.5581395348837209,
|
||||
0.5581395348837209,
|
||||
0.5581395348837209,
|
||||
0.5581395348837209,
|
||||
0.5627906976744186,
|
||||
0.5534883720930233,
|
||||
0.5581395348837209,
|
||||
0.5674418604651162,
|
||||
0.5534883720930233,
|
||||
0.5534883720930233,
|
||||
0.5534883720930233,
|
||||
0.5581395348837209,
|
||||
0.5581395348837209,
|
||||
0.5767441860465117,
|
||||
0.5627906976744186,
|
||||
0.5720930232558139,
|
||||
0.5534883720930233,
|
||||
0.5488372093023256,
|
||||
0.5534883720930233,
|
||||
0.5534883720930233,
|
||||
0.5767441860465117,
|
||||
0.5534883720930233,
|
||||
0.5534883720930233,
|
||||
0.5534883720930233,
|
||||
0.5720930232558139,
|
||||
0.5534883720930233,
|
||||
0.5627906976744186,
|
||||
0.5627906976744186,
|
||||
0.5534883720930233,
|
||||
0.5534883720930233,
|
||||
0.5581395348837209,
|
||||
0.5581395348837209,
|
||||
0.5627906976744186,
|
||||
0.5581395348837209,
|
||||
0.5534883720930233,
|
||||
0.5674418604651162,
|
||||
0.5488372093023256,
|
||||
0.5581395348837209,
|
||||
0.5581395348837209,
|
||||
0.5488372093023256,
|
||||
0.5488372093023256,
|
||||
0.5488372093023256,
|
||||
0.5395348837209303,
|
||||
0.5627906976744186,
|
||||
0.5441860465116279,
|
||||
0.5581395348837209,
|
||||
0.5581395348837209,
|
||||
0.5441860465116279,
|
||||
0.5627906976744186,
|
||||
0.5534883720930233,
|
||||
0.5534883720930233,
|
||||
0.5627906976744186,
|
||||
0.5674418604651162,
|
||||
0.5348837209302325,
|
||||
0.5534883720930233,
|
||||
0.5441860465116279,
|
||||
0.5534883720930233,
|
||||
0.5534883720930233,
|
||||
0.5581395348837209,
|
||||
0.5581395348837209,
|
||||
0.5581395348837209,
|
||||
0.5488372093023256,
|
||||
0.5534883720930233,
|
||||
0.5488372093023256,
|
||||
0.5488372093023256,
|
||||
0.5441860465116279,
|
||||
0.5441860465116279,
|
||||
0.5534883720930233,
|
||||
0.5720930232558139,
|
||||
0.5441860465116279,
|
||||
0.5488372093023256,
|
||||
0.5674418604651162,
|
||||
0.5488372093023256,
|
||||
0.5534883720930233,
|
||||
0.5674418604651162,
|
||||
0.5720930232558139,
|
||||
0.5441860465116279,
|
||||
0.5627906976744186,
|
||||
0.5627906976744186,
|
||||
0.5534883720930233,
|
||||
0.5627906976744186,
|
||||
0.5627906976744186,
|
||||
0.5581395348837209,
|
||||
0.5488372093023256,
|
||||
0.5395348837209303,
|
||||
0.5581395348837209,
|
||||
0.5627906976744186,
|
||||
0.5534883720930233,
|
||||
0.5581395348837209,
|
||||
0.5441860465116279,
|
||||
0.5720930232558139,
|
||||
0.5488372093023256,
|
||||
0.5627906976744186,
|
||||
0.5627906976744186,
|
||||
0.5534883720930233,
|
||||
0.5627906976744186,
|
||||
0.5534883720930233,
|
||||
0.5627906976744186,
|
||||
0.5674418604651162,
|
||||
0.5627906976744186,
|
||||
0.5627906976744186,
|
||||
0.5674418604651162,
|
||||
0.5674418604651162,
|
||||
0.5581395348837209,
|
||||
0.5674418604651162,
|
||||
0.5674418604651162,
|
||||
0.5627906976744186,
|
||||
0.5581395348837209,
|
||||
0.5627906976744186,
|
||||
0.5674418604651162,
|
||||
0.5627906976744186,
|
||||
0.5581395348837209,
|
||||
0.5674418604651162,
|
||||
0.5534883720930233,
|
||||
0.5488372093023256,
|
||||
0.5581395348837209,
|
||||
0.5674418604651162,
|
||||
0.5627906976744186,
|
||||
0.5627906976744186,
|
||||
0.5581395348837209,
|
||||
0.5581395348837209,
|
||||
0.5674418604651162,
|
||||
0.5488372093023256,
|
||||
0.5674418604651162,
|
||||
0.5674418604651162,
|
||||
0.5534883720930233,
|
||||
0.5627906976744186,
|
||||
0.5627906976744186,
|
||||
0.5627906976744186,
|
||||
0.5674418604651162
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python3
|
||||
"""R9 — RSSI fingerprint topology: does temporal proximity = feature proximity?
|
||||
|
||||
See docs/research/sota-2026-05-22/R9-rssi-fingerprint-knn.md.
|
||||
|
||||
Hypothesis: if RSSI sequences from temporally-adjacent windows are
|
||||
nearest-neighbours in feature space, RSSI-fingerprint localisation is
|
||||
viable. If the K-NN of every query is random in time, RSSI sequences
|
||||
don't carry stable enough fingerprints — fall back to multi-modal cues
|
||||
(BSSID lists, signal-of-opportunity).
|
||||
|
||||
Test:
|
||||
1. Build the same 20-dim RSSI proxy from the 1,077 paired windows
|
||||
(band-mean across 56 subcarriers per frame).
|
||||
2. For each sample i, find K-NN in cosine-similarity space.
|
||||
3. Measure: what fraction of the K-NN come from windows within
|
||||
±60 seconds of the query's timestamp?
|
||||
4. Compare to a random baseline (what would the fraction be if K-NN
|
||||
were chosen at random?).
|
||||
|
||||
If the temporal-K-NN fraction is ≫ random, RSSI fingerprints have stable
|
||||
spatial structure → R9 viable.
|
||||
|
||||
Usage:
|
||||
python examples/research-sota/r9_rssi_fingerprint_knn.py \
|
||||
--paired data/paired/wiflow-p7-1779210883.paired.jsonl
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
N_SUB, N_FRAMES = 56, 20
|
||||
|
||||
|
||||
def load_rssi_proxy(path: Path) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""Return (X_rssi, ts_seconds). X_rssi is [N, 20], ts is [N] float seconds."""
|
||||
csis, ts = [], []
|
||||
with path.open(encoding="utf-8") as f:
|
||||
for line in f:
|
||||
if not line.strip():
|
||||
continue
|
||||
d = json.loads(line)
|
||||
shape = d.get("csi_shape", [N_SUB, N_FRAMES])
|
||||
if shape != [N_SUB, N_FRAMES]:
|
||||
continue
|
||||
csi = np.asarray(d["csi"], dtype=np.float32).reshape(N_SUB, N_FRAMES)
|
||||
csis.append(csi.mean(axis=0)) # band-mean → [20]
|
||||
t_iso = d.get("ts_start", "1970-01-01T00:00:00Z")
|
||||
ts.append(datetime.fromisoformat(t_iso.replace("Z", "+00:00")).timestamp())
|
||||
return np.stack(csis), np.asarray(ts, dtype=np.float64)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--paired", required=True)
|
||||
parser.add_argument("--out", default="examples/research-sota/r9_rssi_fingerprint_results.json")
|
||||
parser.add_argument("--k", type=int, default=5)
|
||||
parser.add_argument("--temporal-window-s", type=float, default=60.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"Loading RSSI-proxy from {args.paired}")
|
||||
X, ts = load_rssi_proxy(Path(args.paired))
|
||||
print(f" N samples: {X.shape[0]}, feature dim: {X.shape[1]}")
|
||||
print(f" time range: {datetime.fromtimestamp(ts.min(), tz=timezone.utc):%H:%M:%S} - "
|
||||
f"{datetime.fromtimestamp(ts.max(), tz=timezone.utc):%H:%M:%S} "
|
||||
f"({(ts.max() - ts.min()) / 60:.1f} min total)")
|
||||
|
||||
# Z-score normalise across all samples — what a real device does via AGC
|
||||
mu = X.mean(axis=0, keepdims=True)
|
||||
sd = X.std(axis=0, keepdims=True) + 1e-6
|
||||
Xn = (X - mu) / sd
|
||||
|
||||
# All-pairs cosine similarity
|
||||
print(f"\nComputing all-pairs cosine similarity ({X.shape[0]}×{X.shape[0]} = "
|
||||
f"{X.shape[0]**2:,} pairs)...")
|
||||
norms = np.linalg.norm(Xn, axis=1, keepdims=True) + 1e-9
|
||||
Xnorm = Xn / norms
|
||||
sim = Xnorm @ Xnorm.T
|
||||
np.fill_diagonal(sim, -np.inf) # exclude self-match
|
||||
|
||||
N = X.shape[0]
|
||||
K = args.k
|
||||
W = args.temporal_window_s
|
||||
|
||||
# For each query, find top-K nearest neighbours and measure how many are
|
||||
# within the temporal window
|
||||
print(f"\nMeasuring temporal-locality of top-{K} cosine-NN with window ±{W:.0f}s...")
|
||||
knn_idx = np.argsort(-sim, axis=1)[:, :K] # [N, K]
|
||||
knn_ts = ts[knn_idx] # [N, K]
|
||||
delta_t = np.abs(knn_ts - ts[:, None]) # [N, K]
|
||||
within = (delta_t <= W).astype(np.float32) # [N, K]
|
||||
per_query_within_frac = within.mean(axis=1) # [N] — fraction of K-NN within window
|
||||
overall_within_frac = within.mean() # scalar
|
||||
|
||||
# Random baseline: for each query, what fraction of all OTHER samples
|
||||
# fall within ±W of its timestamp?
|
||||
rand_within = np.zeros(N, dtype=np.float32)
|
||||
for i in range(N):
|
||||
delta = np.abs(ts - ts[i])
|
||||
delta[i] = np.inf
|
||||
rand_within[i] = (delta <= W).mean()
|
||||
rand_baseline = float(rand_within.mean())
|
||||
|
||||
# Headline numbers
|
||||
lift = overall_within_frac / max(rand_baseline, 1e-9)
|
||||
|
||||
print(f"\n=== R9 RSSI-fingerprint K-NN results ===")
|
||||
print(f" K-NN within ±{W:.0f}s: {overall_within_frac:.3f}")
|
||||
print(f" Random baseline: {rand_baseline:.3f}")
|
||||
print(f" Lift over random: {lift:.2f}×")
|
||||
print(f" Per-query stdev: {per_query_within_frac.std():.3f}")
|
||||
|
||||
if lift >= 3.0:
|
||||
verdict = "STRONG: RSSI sequences carry stable spatial fingerprints"
|
||||
elif lift >= 1.5:
|
||||
verdict = "MODERATE: RSSI fingerprints work but with significant noise"
|
||||
else:
|
||||
verdict = "WEAK: RSSI-only fingerprint localisation is unreliable on this data"
|
||||
print(f"\n Verdict: {verdict}")
|
||||
|
||||
out = {
|
||||
"n_samples": int(N),
|
||||
"k": K,
|
||||
"temporal_window_s": W,
|
||||
"knn_within_window_fraction": float(overall_within_frac),
|
||||
"random_baseline": rand_baseline,
|
||||
"lift": float(lift),
|
||||
"per_query_within_fraction_stdev": float(per_query_within_frac.std()),
|
||||
"verdict": verdict,
|
||||
}
|
||||
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.out).write_text(json.dumps(out, indent=2))
|
||||
print(f"\nWrote {args.out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"n_samples": 1077,
|
||||
"k": 5,
|
||||
"temporal_window_s": 60.0,
|
||||
"knn_within_window_fraction": 0.16861653327941895,
|
||||
"random_baseline": 0.07726679742336273,
|
||||
"lift": 2.1822638511657715,
|
||||
"per_query_within_fraction_stdev": 0.18328286707401276,
|
||||
"verdict": "MODERATE: RSSI fingerprints work but with significant noise"
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
# Mixamo FBX downloads — too large + license boundary. Get your own from
|
||||
# mixamo.com (FBX Binary + T-Pose / Without Skin), drop into assets/.
|
||||
*.fbx
|
||||
|
||||
# Diagnostic / debug screenshots from a dev session. Official screenshots
|
||||
# live in screenshots/ and are committed; these underscore-prefixed ones
|
||||
# are scratch.
|
||||
_diag-*.png
|
||||
_demo-mode-shot*.png
|
||||
_PROOF-*.png
|
||||
@@ -0,0 +1,77 @@
|
||||
# three.js demos
|
||||
|
||||
Five progressively richer browser demos of the ADR-097 sensing-helpers scene,
|
||||
ending with a live MediaPipe-Pose → Mixamo X Bot retargeting pipeline driven
|
||||
by a real ESP32 CSI feed.
|
||||
|
||||
## Run them
|
||||
|
||||
```bash
|
||||
python examples/three.js/server/serve-demo.py
|
||||
# then open one of the URLs the script prints
|
||||
```
|
||||
|
||||
`server/serve-demo.py` is a tiny `ThreadingHTTPServer` with aggressive
|
||||
no-cache headers — the stdlib `http.server` is single-threaded and times out
|
||||
on the parallel script + FBX fetches the demos make.
|
||||
|
||||
## Demos
|
||||
|
||||
| # | File | What it shows |
|
||||
|---|------|---------------|
|
||||
| 01 | [`demos/01-helpers.html`](demos/01-helpers.html) | Plain ADR-097 helpers in the point-cloud viewer |
|
||||
| 02 | [`demos/02-cinematic.html`](demos/02-cinematic.html) | Cinematic camera + pseudo-CSI visualization on top of #01 |
|
||||
| 03 | [`demos/03-skinned.html`](demos/03-skinned.html) | GLTF skinned mesh + additive animation blending |
|
||||
| 04 | [`demos/04-skinned-fbx.html`](demos/04-skinned-fbx.html) | Mixamo X Bot loaded from FBX in the ADR-097 scene |
|
||||
| 05 | [`demos/05-skinned-realtime.html`](demos/05-skinned-realtime.html) | Webcam → MediaPipe Pose Heavy → Mixamo IK retarget, live ESP32 CSI overlay |
|
||||
|
||||
| Screenshot | |
|
||||
|---|---|
|
||||
|  |  |
|
||||
|  |  |
|
||||
|  | |
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
examples/three.js/
|
||||
├── README.md
|
||||
├── .gitignore
|
||||
├── demos/ # 5 self-contained HTML demos
|
||||
│ ├── 01-helpers.html
|
||||
│ ├── 02-cinematic.html
|
||||
│ ├── 03-skinned.html
|
||||
│ ├── 04-skinned-fbx.html
|
||||
│ └── 05-skinned-realtime.html
|
||||
├── screenshots/ # one PNG per demo
|
||||
│ └── 0N-*.png
|
||||
├── server/
|
||||
│ ├── serve-demo.py # local HTTP server with no-cache headers
|
||||
│ └── ruvultra-csi-bridge.py # ESP32 CSI WebSocket bridge (ruvultra:8766)
|
||||
└── assets/
|
||||
└── X Bot.fbx # gitignored — get your own from mixamo.com
|
||||
# (FBX Binary, T-Pose, Without Skin)
|
||||
# used by demos 04 and 05
|
||||
```
|
||||
|
||||
## Mixamo X Bot
|
||||
|
||||
Demos 04 and 05 expect `assets/X Bot.fbx`. It's gitignored (size + license
|
||||
boundary). Download yours from [mixamo.com](https://mixamo.com): pick the
|
||||
"X Bot" character, export as **FBX Binary**, **T-Pose**, **Without Skin**,
|
||||
and drop it into `assets/`.
|
||||
|
||||
## Live ESP32 CSI overlay (demo 05 only)
|
||||
|
||||
`server/ruvultra-csi-bridge.py` is the systemd-deployable bridge that runs on
|
||||
the `ruvultra` host (over Tailscale). It listens for ESP32-S3 CSI on UDP and
|
||||
re-broadcasts it as WebSocket frames at `ws://ruvultra:8766/csi`. Demo 05
|
||||
auto-connects; if the socket is down, it falls back to the bundled idle clip
|
||||
plus a synthetic CSI driver.
|
||||
|
||||
## Open issues
|
||||
|
||||
- [#583](https://github.com/ruvnet/RuView/issues/583) — head/face tracking
|
||||
fidelity in `05-skinned-realtime.html`. Recommended fix: swap MediaPipe
|
||||
Pose Heavy for MediaPipe Holistic (same API, adds 468-point face mesh +
|
||||
hand landmarks for proper PnP head pose and finger curl tracking).
|
||||
@@ -0,0 +1,587 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>RuView · ADR-097 · three.js helpers in the point cloud viewer</title>
|
||||
<link rel="icon" type="image/svg+xml" href="data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'><circle cx='16' cy='16' r='10' fill='%23e8a634'/></svg>">
|
||||
<style>
|
||||
:root {
|
||||
--bg: #0a0a0a;
|
||||
--bg-panel: rgba(0, 0, 0, 0.88);
|
||||
--amber: #e8a634;
|
||||
--amber-dim: #4a3a1a;
|
||||
--amber-hot: #ffc04d;
|
||||
--grid-major: #444444;
|
||||
--grid-minor: #222222;
|
||||
--green: #4f4;
|
||||
--blue: #4cf;
|
||||
--text-mute: #888;
|
||||
--border: #2a2a2a;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0;
|
||||
background: var(--bg);
|
||||
color: var(--amber);
|
||||
font-family: 'SF Mono', Monaco, 'Cascadia Code', Consolas, monospace;
|
||||
overflow: hidden;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
}
|
||||
canvas { display: block; }
|
||||
|
||||
/* Top-left HUD */
|
||||
#info {
|
||||
position: absolute;
|
||||
top: 16px;
|
||||
left: 16px;
|
||||
padding: 14px 16px;
|
||||
background: var(--bg-panel);
|
||||
border: 1px solid var(--amber);
|
||||
border-radius: 8px;
|
||||
min-width: 280px;
|
||||
max-width: 340px;
|
||||
font-size: 12px;
|
||||
line-height: 1.55;
|
||||
z-index: 10;
|
||||
backdrop-filter: blur(6px);
|
||||
box-shadow: 0 4px 24px rgba(232, 166, 52, 0.08);
|
||||
}
|
||||
#info h1 { margin: 0 0 2px 0; font-size: 14px; letter-spacing: 0.5px; }
|
||||
#info .sub { font-size: 11px; color: var(--text-mute); margin-bottom: 10px; }
|
||||
#info .row { display: flex; justify-content: space-between; gap: 12px; margin: 2px 0; }
|
||||
#info .row .k { color: var(--text-mute); }
|
||||
#info .row .v { color: var(--amber); font-variant-numeric: tabular-nums; }
|
||||
#info .row .v.live { color: var(--green); }
|
||||
|
||||
/* Bottom-left helper toggle panel */
|
||||
#controls {
|
||||
position: absolute;
|
||||
bottom: 16px;
|
||||
left: 16px;
|
||||
padding: 12px 14px;
|
||||
background: var(--bg-panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
font-size: 12px;
|
||||
z-index: 10;
|
||||
backdrop-filter: blur(6px);
|
||||
min-width: 220px;
|
||||
}
|
||||
#controls h2 {
|
||||
margin: 0 0 8px 0;
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 1.2px;
|
||||
color: var(--text-mute);
|
||||
font-weight: 600;
|
||||
}
|
||||
#controls label {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 4px 0;
|
||||
cursor: pointer;
|
||||
user-select: none;
|
||||
}
|
||||
#controls label:hover { color: var(--amber-hot); }
|
||||
#controls input[type=checkbox] {
|
||||
accent-color: var(--amber);
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
cursor: pointer;
|
||||
}
|
||||
#controls .helper-swatch {
|
||||
display: inline-block;
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 2px;
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
/* Bottom-right ADR badge */
|
||||
#adr-badge {
|
||||
position: absolute;
|
||||
bottom: 16px;
|
||||
right: 16px;
|
||||
padding: 8px 12px;
|
||||
background: var(--bg-panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 6px;
|
||||
font-size: 11px;
|
||||
color: var(--text-mute);
|
||||
z-index: 10;
|
||||
backdrop-filter: blur(6px);
|
||||
}
|
||||
#adr-badge a { color: var(--amber); text-decoration: none; }
|
||||
#adr-badge a:hover { color: var(--amber-hot); }
|
||||
|
||||
/* Top-right legend */
|
||||
#legend {
|
||||
position: absolute;
|
||||
top: 16px;
|
||||
right: 16px;
|
||||
padding: 12px 14px;
|
||||
background: var(--bg-panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
font-size: 11px;
|
||||
z-index: 10;
|
||||
backdrop-filter: blur(6px);
|
||||
min-width: 200px;
|
||||
}
|
||||
#legend h2 {
|
||||
margin: 0 0 8px 0;
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 1.2px;
|
||||
color: var(--text-mute);
|
||||
font-weight: 600;
|
||||
}
|
||||
#legend .item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 3px 0;
|
||||
}
|
||||
#legend .dot {
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 50%;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
#legend .label { font-size: 11px; line-height: 1.3; }
|
||||
</style>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/controls/OrbitControls.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<div id="info">
|
||||
<h1>RuView · Helpers Demo</h1>
|
||||
<div class="sub">ADR-097 · three.js helpers for the point cloud viewer</div>
|
||||
<div class="row"><span class="k">Scene</span><span class="v live">● SYNTHETIC</span></div>
|
||||
<div class="row"><span class="k">Skeleton</span><span class="v">17 kpts · COCO</span></div>
|
||||
<div class="row"><span class="k">Point cloud</span><span class="v" id="pc-count">— pts</span></div>
|
||||
<div class="row"><span class="k">Sensor nodes</span><span class="v">4 · multistatic</span></div>
|
||||
<div class="row"><span class="k">Frame rate</span><span class="v" id="fps">— fps</span></div>
|
||||
<div class="row"><span class="k">Bbox volume</span><span class="v" id="bbox-vol">— m³</span></div>
|
||||
</div>
|
||||
|
||||
<div id="controls">
|
||||
<h2>Helpers</h2>
|
||||
<label><input type="checkbox" id="t-grid" checked>GridHelper<span class="helper-swatch" style="background:#444"></span></label>
|
||||
<label><input type="checkbox" id="t-polar" checked>PolarGridHelper<span class="helper-swatch" style="background:#4a3a1a"></span></label>
|
||||
<label><input type="checkbox" id="t-bbox" checked>BoxHelper<span class="helper-swatch" style="background:#e8a634"></span></label>
|
||||
<label><input type="checkbox" id="t-axes" checked>AxesHelper<span class="helper-swatch" style="background:linear-gradient(90deg,#f44,#4f4,#4cf)"></span></label>
|
||||
<label><input type="checkbox" id="t-nodebox" checked>Per-node BoxHelpers<span class="helper-swatch" style="background:#4cf"></span></label>
|
||||
</div>
|
||||
|
||||
<div id="legend">
|
||||
<h2>Scene</h2>
|
||||
<div class="item"><span class="dot" style="background:#ffff00"></span><span class="label">COCO-17 keypoints (yellow)</span></div>
|
||||
<div class="item"><span class="dot" style="background:#ffffff"></span><span class="label">Bones (white lines)</span></div>
|
||||
<div class="item"><span class="dot" style="background:#4cf"></span><span class="label">Face point cloud (cyan→white)</span></div>
|
||||
<div class="item"><span class="dot" style="background:#e8a634"></span><span class="label">ESP32 sensor nodes</span></div>
|
||||
</div>
|
||||
|
||||
<div id="adr-badge">
|
||||
ADR-097 · <a href="https://threejs.org/examples/#webgl_helpers" target="_blank" rel="noopener">three.js helpers</a>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// =====================================================================
|
||||
// RuView · ADR-097 · three.js helpers demo
|
||||
// --------------------------------------------------------------------
|
||||
// Self-contained, no backend. Demonstrates how `GridHelper`,
|
||||
// `PolarGridHelper`, `BoxHelper`, and `AxesHelper` slot into the
|
||||
// RuView point cloud viewer (`v2/crates/wifi-densepose-pointcloud
|
||||
// /src/viewer.html`). Open this file in a browser — no build step.
|
||||
//
|
||||
// The scene contains:
|
||||
// 1. A synthetic walking, breathing 17-keypoint skeleton.
|
||||
// 2. A face-shaped point cloud attached to the skeleton head.
|
||||
// 3. Four multistatic sensor-node markers arranged around the room.
|
||||
// 4. All four ADR-097 helpers, toggleable from the bottom-left panel.
|
||||
//
|
||||
// Coordinate frame matches the production viewer:
|
||||
// +X = right, +Y = up, +Z = away from camera.
|
||||
// Floor at y = -1.5, person hip at y = 0, head reaches ~ y = 0.7.
|
||||
// =====================================================================
|
||||
|
||||
const COCO_BONES = [
|
||||
// head
|
||||
[0, 1], [0, 2], [1, 3], [2, 4],
|
||||
// torso
|
||||
[5, 6], [5, 11], [6, 12], [11, 12],
|
||||
// left arm
|
||||
[5, 7], [7, 9],
|
||||
// right arm
|
||||
[6, 8], [8, 10],
|
||||
// left leg
|
||||
[11, 13], [13, 15],
|
||||
// right leg
|
||||
[12, 14], [14, 16],
|
||||
];
|
||||
|
||||
// Static "T-pose" skeleton in local frame, animated each frame.
|
||||
// 17 keypoints in COCO order. Units: meters.
|
||||
const SKELETON_BASE = {
|
||||
0: [ 0.00, 0.65, 0.00], // nose
|
||||
1: [-0.04, 0.68, 0.04], // L eye
|
||||
2: [ 0.04, 0.68, 0.04], // R eye
|
||||
3: [-0.08, 0.64, 0.00], // L ear
|
||||
4: [ 0.08, 0.64, 0.00], // R ear
|
||||
5: [-0.18, 0.45, 0.00], // L shoulder
|
||||
6: [ 0.18, 0.45, 0.00], // R shoulder
|
||||
7: [-0.22, 0.20, 0.00], // L elbow
|
||||
8: [ 0.22, 0.20, 0.00], // R elbow
|
||||
9: [-0.26, -0.05, 0.00], // L wrist
|
||||
10: [ 0.26, -0.05, 0.00], // R wrist
|
||||
11: [-0.10, 0.00, 0.00], // L hip
|
||||
12: [ 0.10, 0.00, 0.00], // R hip
|
||||
13: [-0.12, -0.40, 0.00], // L knee
|
||||
14: [ 0.12, -0.40, 0.00], // R knee
|
||||
15: [-0.12, -0.80, 0.00], // L ankle
|
||||
16: [ 0.12, -0.80, 0.00], // R ankle
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Scene + camera + renderer
|
||||
// ---------------------------------------------------------------------
|
||||
const scene = new THREE.Scene();
|
||||
scene.background = new THREE.Color(0x0a0a0a);
|
||||
scene.fog = new THREE.Fog(0x0a0a0a, 6, 14);
|
||||
|
||||
const camera = new THREE.PerspectiveCamera(60, window.innerWidth / window.innerHeight, 0.05, 100);
|
||||
camera.position.set(3.0, 1.4, 4.2);
|
||||
|
||||
const renderer = new THREE.WebGLRenderer({ antialias: true, powerPreference: 'high-performance' });
|
||||
renderer.setPixelRatio(window.devicePixelRatio);
|
||||
renderer.setSize(window.innerWidth, window.innerHeight);
|
||||
document.body.appendChild(renderer.domElement);
|
||||
|
||||
const controls = new THREE.OrbitControls(camera, renderer.domElement);
|
||||
controls.target.set(0, 0, 0);
|
||||
controls.enableDamping = true;
|
||||
controls.dampingFactor = 0.08;
|
||||
controls.minDistance = 1.5;
|
||||
controls.maxDistance = 12;
|
||||
controls.maxPolarAngle = Math.PI * 0.92;
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// ADR-097 helpers — wired to checkbox toggles
|
||||
// ---------------------------------------------------------------------
|
||||
// GridHelper — Cartesian floor reference. Establishes "down" and
|
||||
// scale: 4 m × 4 m floor, 20 divisions = 0.2 m grid spacing.
|
||||
const gridHelper = new THREE.GridHelper(4, 20, 0x444444, 0x222222);
|
||||
gridHelper.position.y = -1.5;
|
||||
scene.add(gridHelper);
|
||||
|
||||
// PolarGridHelper — multistatic geometry reference. 16 radial
|
||||
// divisions (angular bins) × 4 concentric circles, centered on
|
||||
// the fusion target. Matches the bin count in
|
||||
// signal/src/ruvsense/multistatic.rs:attention_weight().
|
||||
const polarHelper = new THREE.PolarGridHelper(2.2, 16, 4, 64, 0x4a3a1a, 0x2a1f10);
|
||||
polarHelper.position.y = -1.499; // a hair above grid to avoid z-fight
|
||||
scene.add(polarHelper);
|
||||
|
||||
// AxesHelper — XYZ tripod at origin. Red = X, green = Y, blue = Z.
|
||||
const axesHelper = new THREE.AxesHelper(0.5);
|
||||
axesHelper.position.set(0, -1.49, 0);
|
||||
scene.add(axesHelper);
|
||||
|
||||
// BoxHelper — per-person bounding volume. Refreshed each frame
|
||||
// after the skeleton is updated. Color = RuView amber.
|
||||
let bboxHelper = null;
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Skeleton — joint spheres + bone lines, animated
|
||||
// ---------------------------------------------------------------------
|
||||
const skeletonGroup = new THREE.Group();
|
||||
scene.add(skeletonGroup);
|
||||
|
||||
const jointGeo = new THREE.SphereGeometry(0.025, 12, 12);
|
||||
const jointMat = new THREE.MeshBasicMaterial({ color: 0xffff00 });
|
||||
const joints = [];
|
||||
for (let i = 0; i < 17; i++) {
|
||||
const sphere = new THREE.Mesh(jointGeo, jointMat);
|
||||
const p = SKELETON_BASE[i];
|
||||
sphere.position.set(p[0], p[1], p[2]);
|
||||
sphere.userData.baseY = p[1];
|
||||
sphere.userData.baseX = p[0];
|
||||
sphere.userData.idx = i;
|
||||
skeletonGroup.add(sphere);
|
||||
joints.push(sphere);
|
||||
}
|
||||
|
||||
const boneMat = new THREE.LineBasicMaterial({ color: 0xffffff, transparent: true, opacity: 0.85 });
|
||||
const bones = [];
|
||||
for (const [a, b] of COCO_BONES) {
|
||||
const geom = new THREE.BufferGeometry();
|
||||
geom.setAttribute('position', new THREE.BufferAttribute(new Float32Array(6), 3));
|
||||
const line = new THREE.Line(geom, boneMat);
|
||||
line.userData = { a, b };
|
||||
skeletonGroup.add(line);
|
||||
bones.push(line);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Face point cloud — synthetic ellipsoid attached to head keypoint
|
||||
// ---------------------------------------------------------------------
|
||||
const FACE_POINTS = 600;
|
||||
const facePositions = new Float32Array(FACE_POINTS * 3);
|
||||
const faceColors = new Float32Array(FACE_POINTS * 3);
|
||||
const faceOffsets = new Float32Array(FACE_POINTS * 3); // canonical face shape, relative to nose
|
||||
|
||||
for (let i = 0; i < FACE_POINTS; i++) {
|
||||
// Sample points roughly on a face-shaped ellipsoid (taller than wide).
|
||||
const u = Math.random() * Math.PI * 2;
|
||||
const v = (Math.random() - 0.5) * Math.PI;
|
||||
const cu = Math.cos(u), su = Math.sin(u);
|
||||
const cv = Math.cos(v), sv = Math.sin(v);
|
||||
// ellipsoid radii (head-like proportions)
|
||||
const rx = 0.085, ry = 0.105, rz = 0.075;
|
||||
faceOffsets[i * 3 + 0] = rx * cv * cu;
|
||||
faceOffsets[i * 3 + 1] = ry * sv;
|
||||
faceOffsets[i * 3 + 2] = rz * cv * su;
|
||||
// depth-encoded color: cyan at back, near-white at front (toward +Z = away from camera)
|
||||
const depthT = (sv + 1) * 0.5;
|
||||
faceColors[i * 3 + 0] = 0.30 + 0.70 * depthT; // R
|
||||
faceColors[i * 3 + 1] = 0.80 + 0.20 * depthT; // G
|
||||
faceColors[i * 3 + 2] = 1.00; // B
|
||||
}
|
||||
const faceGeom = new THREE.BufferGeometry();
|
||||
faceGeom.setAttribute('position', new THREE.BufferAttribute(facePositions, 3));
|
||||
faceGeom.setAttribute('color', new THREE.BufferAttribute(faceColors, 3));
|
||||
const faceMat = new THREE.PointsMaterial({
|
||||
size: 0.012,
|
||||
vertexColors: true,
|
||||
sizeAttenuation: true,
|
||||
transparent: true,
|
||||
opacity: 0.9,
|
||||
});
|
||||
const facePoints = new THREE.Points(faceGeom, faceMat);
|
||||
skeletonGroup.add(facePoints);
|
||||
document.getElementById('pc-count').textContent = FACE_POINTS + ' pts';
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Multistatic sensor nodes — 4 ESP32 markers around the room
|
||||
// ---------------------------------------------------------------------
|
||||
const nodeGroup = new THREE.Group();
|
||||
scene.add(nodeGroup);
|
||||
|
||||
const NODE_POSITIONS = [
|
||||
[-1.9, 1.3, 1.9], // back-left high
|
||||
[ 1.9, 1.3, 1.9], // back-right high
|
||||
[-1.9, 1.3, -1.9], // front-left high
|
||||
[ 1.9, 1.3, -1.9], // front-right high
|
||||
];
|
||||
const nodeBboxHelpers = [];
|
||||
const nodeGeo = new THREE.BoxGeometry(0.12, 0.06, 0.18);
|
||||
const nodeMat = new THREE.MeshBasicMaterial({ color: 0xe8a634 });
|
||||
const nodeAntennaGeo = new THREE.ConeGeometry(0.018, 0.08, 8);
|
||||
const nodeAntennaMat = new THREE.MeshBasicMaterial({ color: 0xffc04d });
|
||||
|
||||
NODE_POSITIONS.forEach((pos, i) => {
|
||||
const group = new THREE.Group();
|
||||
group.position.set(pos[0], pos[1], pos[2]);
|
||||
|
||||
const body = new THREE.Mesh(nodeGeo, nodeMat);
|
||||
group.add(body);
|
||||
|
||||
// little antenna sticking up
|
||||
const antenna = new THREE.Mesh(nodeAntennaGeo, nodeAntennaMat);
|
||||
antenna.position.y = 0.07;
|
||||
group.add(antenna);
|
||||
|
||||
// pulsing emissive ring (visualizes RX activity)
|
||||
const ringGeo = new THREE.RingGeometry(0.10, 0.13, 32);
|
||||
const ringMat = new THREE.MeshBasicMaterial({ color: 0xe8a634, side: THREE.DoubleSide, transparent: true, opacity: 0.4 });
|
||||
const ring = new THREE.Mesh(ringGeo, ringMat);
|
||||
ring.rotation.x = -Math.PI / 2;
|
||||
ring.position.y = -0.04;
|
||||
ring.userData.phase = i * 0.5;
|
||||
group.add(ring);
|
||||
group.userData.ring = ring;
|
||||
|
||||
// sight-line from node to scene origin (visualizes multistatic geometry)
|
||||
const sightGeo = new THREE.BufferGeometry().setFromPoints([
|
||||
new THREE.Vector3(0, 0, 0),
|
||||
new THREE.Vector3(-pos[0], -pos[1], -pos[2]),
|
||||
]);
|
||||
const sightMat = new THREE.LineDashedMaterial({
|
||||
color: 0xe8a634, transparent: true, opacity: 0.18,
|
||||
dashSize: 0.1, gapSize: 0.06,
|
||||
});
|
||||
const sightLine = new THREE.Line(sightGeo, sightMat);
|
||||
sightLine.computeLineDistances();
|
||||
group.add(sightLine);
|
||||
|
||||
nodeGroup.add(group);
|
||||
|
||||
// ADR-097 §3.3 — per-node BoxHelper. Demonstrates that helpers
|
||||
// compose naturally: one box per detected object.
|
||||
const bbox = new THREE.BoxHelper(group, 0x4cf);
|
||||
scene.add(bbox);
|
||||
nodeBboxHelpers.push(bbox);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Animation — synthetic motion model
|
||||
// ---------------------------------------------------------------------
|
||||
let frameStart = performance.now();
|
||||
let frameCount = 0;
|
||||
let fpsAvg = 0;
|
||||
|
||||
function applyPose(t) {
|
||||
// Body sway (slow), breathing (chest expansion), arm/leg swing (walking).
|
||||
const swayX = Math.sin(t * 0.35) * 0.05;
|
||||
const swayZ = Math.cos(t * 0.27) * 0.04;
|
||||
const breathe = Math.sin(t * 1.4) * 0.012; // chest in/out
|
||||
const walkPhase = t * 1.9; // walk cycle
|
||||
|
||||
skeletonGroup.position.set(swayX, 0, swayZ);
|
||||
skeletonGroup.rotation.y = Math.sin(t * 0.22) * 0.18;
|
||||
|
||||
for (let i = 0; i < 17; i++) {
|
||||
const base = SKELETON_BASE[i];
|
||||
let dx = 0, dy = 0, dz = 0;
|
||||
|
||||
// breathing — shoulders + nose rise a little
|
||||
if (i === 0 || i === 1 || i === 2) dy = breathe * 0.6;
|
||||
if (i === 5 || i === 6) dy = breathe;
|
||||
|
||||
// arm swing (opposite of legs)
|
||||
if (i === 7) { dz = Math.sin(walkPhase) * 0.10; dy += Math.cos(walkPhase) * 0.04; }
|
||||
if (i === 9) { dz = Math.sin(walkPhase) * 0.18; dy += Math.cos(walkPhase) * 0.06; }
|
||||
if (i === 8) { dz = -Math.sin(walkPhase) * 0.10; dy += Math.cos(walkPhase) * 0.04; }
|
||||
if (i === 10){ dz = -Math.sin(walkPhase) * 0.18; dy += Math.cos(walkPhase) * 0.06; }
|
||||
|
||||
// leg swing
|
||||
if (i === 13){ dz = -Math.sin(walkPhase) * 0.08; }
|
||||
if (i === 15){ dz = -Math.sin(walkPhase) * 0.15; dy = Math.max(0, Math.cos(walkPhase)) * 0.04; }
|
||||
if (i === 14){ dz = Math.sin(walkPhase) * 0.08; }
|
||||
if (i === 16){ dz = Math.sin(walkPhase) * 0.15; dy = Math.max(0, -Math.cos(walkPhase)) * 0.04; }
|
||||
|
||||
joints[i].position.set(base[0] + dx, base[1] + dy, base[2] + dz);
|
||||
}
|
||||
|
||||
// update bone line vertices from current joint positions
|
||||
for (const line of bones) {
|
||||
const { a, b } = line.userData;
|
||||
const pa = joints[a].position;
|
||||
const pb = joints[b].position;
|
||||
const pos = line.geometry.attributes.position;
|
||||
pos.array[0] = pa.x; pos.array[1] = pa.y; pos.array[2] = pa.z;
|
||||
pos.array[3] = pb.x; pos.array[4] = pb.y; pos.array[5] = pb.z;
|
||||
pos.needsUpdate = true;
|
||||
}
|
||||
|
||||
// attach face point cloud to the nose keypoint (kpt 0)
|
||||
const nose = joints[0].position;
|
||||
const positions = faceGeom.attributes.position;
|
||||
const headTurn = Math.sin(t * 0.6) * 0.35; // y-axis nod
|
||||
const cosH = Math.cos(headTurn), sinH = Math.sin(headTurn);
|
||||
for (let i = 0; i < FACE_POINTS; i++) {
|
||||
const ox = faceOffsets[i * 3 + 0];
|
||||
const oy = faceOffsets[i * 3 + 1];
|
||||
const oz = faceOffsets[i * 3 + 2];
|
||||
// rotate offset around Y axis by headTurn
|
||||
const rx = cosH * ox + sinH * oz;
|
||||
const rz = -sinH * ox + cosH * oz;
|
||||
positions.array[i * 3 + 0] = nose.x + rx;
|
||||
positions.array[i * 3 + 1] = nose.y + oy;
|
||||
positions.array[i * 3 + 2] = nose.z + rz;
|
||||
}
|
||||
positions.needsUpdate = true;
|
||||
}
|
||||
|
||||
function updateNodes(t) {
|
||||
nodeGroup.children.forEach((node, i) => {
|
||||
const ring = node.userData.ring;
|
||||
const phase = (t * 1.8 + ring.userData.phase) % (Math.PI * 2);
|
||||
ring.material.opacity = 0.18 + 0.42 * Math.max(0, Math.cos(phase));
|
||||
ring.scale.setScalar(1 + 0.18 * Math.max(0, Math.cos(phase)));
|
||||
});
|
||||
}
|
||||
|
||||
function updateBboxHelper() {
|
||||
const want = document.getElementById('t-bbox').checked;
|
||||
if (!want) {
|
||||
if (bboxHelper) { scene.remove(bboxHelper); bboxHelper = null; }
|
||||
return;
|
||||
}
|
||||
skeletonGroup.updateMatrixWorld(true);
|
||||
if (!bboxHelper) {
|
||||
bboxHelper = new THREE.BoxHelper(skeletonGroup, 0xe8a634);
|
||||
scene.add(bboxHelper);
|
||||
} else {
|
||||
bboxHelper.setFromObject(skeletonGroup);
|
||||
}
|
||||
// compute volume for the HUD
|
||||
const box = new THREE.Box3().setFromObject(skeletonGroup);
|
||||
const size = box.getSize(new THREE.Vector3());
|
||||
document.getElementById('bbox-vol').textContent =
|
||||
(size.x * size.y * size.z).toFixed(3) + ' m³';
|
||||
}
|
||||
|
||||
function tick() {
|
||||
const now = performance.now();
|
||||
const t = now * 0.001;
|
||||
const dt = now - frameStart;
|
||||
frameStart = now;
|
||||
frameCount++;
|
||||
if (frameCount % 30 === 0) {
|
||||
fpsAvg = 1000 / dt;
|
||||
document.getElementById('fps').textContent = fpsAvg.toFixed(0) + ' fps';
|
||||
}
|
||||
|
||||
applyPose(t);
|
||||
updateNodes(t);
|
||||
updateBboxHelper();
|
||||
|
||||
controls.update();
|
||||
renderer.render(scene, camera);
|
||||
requestAnimationFrame(tick);
|
||||
}
|
||||
requestAnimationFrame(tick);
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Controls wiring — checkbox toggles attach/detach helpers from scene
|
||||
// ---------------------------------------------------------------------
|
||||
function bindToggle(id, obj) {
|
||||
const el = document.getElementById(id);
|
||||
el.addEventListener('change', () => {
|
||||
if (el.checked) {
|
||||
if (!scene.children.includes(obj)) scene.add(obj);
|
||||
} else {
|
||||
scene.remove(obj);
|
||||
}
|
||||
});
|
||||
}
|
||||
bindToggle('t-grid', gridHelper);
|
||||
bindToggle('t-polar', polarHelper);
|
||||
bindToggle('t-axes', axesHelper);
|
||||
|
||||
// per-node bbox toggle (group of 4)
|
||||
document.getElementById('t-nodebox').addEventListener('change', (e) => {
|
||||
for (const bb of nodeBboxHelpers) {
|
||||
if (e.target.checked) {
|
||||
if (!scene.children.includes(bb)) scene.add(bb);
|
||||
} else {
|
||||
scene.remove(bb);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Resize
|
||||
// ---------------------------------------------------------------------
|
||||
window.addEventListener('resize', () => {
|
||||
camera.aspect = window.innerWidth / window.innerHeight;
|
||||
camera.updateProjectionMatrix();
|
||||
renderer.setSize(window.innerWidth, window.innerHeight);
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,854 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>RuView · Skinned · ADR-097 + GLTF skinned mesh + additive animation blending</title>
|
||||
<link rel="icon" type="image/svg+xml" href="data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'><circle cx='16' cy='16' r='10' fill='%23e8a634'/></svg>">
|
||||
<style>
|
||||
:root {
|
||||
--bg: #050507;
|
||||
--bg-panel: rgba(8, 10, 14, 0.78);
|
||||
--amber: #ffb840;
|
||||
--amber-hot: #ffe09f;
|
||||
--cyan: #4cf;
|
||||
--magenta: #ff4cc8;
|
||||
--text: #d8c69a;
|
||||
--text-mute: #6b6155;
|
||||
--border: rgba(255, 184, 64, 0.18);
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0; background: var(--bg); color: var(--text); overflow: hidden;
|
||||
font-family: 'SF Mono', 'Cascadia Code', Consolas, monospace;
|
||||
-webkit-font-smoothing: antialiased; font-size: 12px;
|
||||
}
|
||||
canvas { display: block; }
|
||||
|
||||
.overlay-frame {
|
||||
position: fixed; inset: 0; pointer-events: none; z-index: 5;
|
||||
background:
|
||||
radial-gradient(ellipse at center, transparent 55%, rgba(0,0,0,0.55) 100%),
|
||||
linear-gradient(180deg, rgba(0,0,0,0.32) 0%, transparent 18%, transparent 82%, rgba(0,0,0,0.38) 100%);
|
||||
}
|
||||
.scanlines {
|
||||
position: fixed; inset: 0; pointer-events: none; z-index: 6;
|
||||
background: repeating-linear-gradient(0deg, rgba(0,0,0,0.04) 0px, rgba(0,0,0,0.04) 1px, transparent 1px, transparent 3px);
|
||||
mix-blend-mode: overlay; opacity: 0.5;
|
||||
}
|
||||
.panel {
|
||||
position: absolute; background: var(--bg-panel); border: 1px solid var(--border);
|
||||
border-radius: 4px; padding: 12px 14px; backdrop-filter: blur(8px);
|
||||
box-shadow: 0 1px 0 rgba(255, 184, 64, 0.04), 0 8px 32px rgba(0,0,0,0.55); z-index: 10;
|
||||
}
|
||||
.panel h2 {
|
||||
margin: 0 0 8px 0; font-size: 10px; text-transform: uppercase; letter-spacing: 2px;
|
||||
color: var(--amber); font-weight: 600; border-bottom: 1px solid var(--border); padding-bottom: 6px;
|
||||
}
|
||||
|
||||
#info { top: 20px; left: 20px; min-width: 280px; }
|
||||
#info h1 { margin: 0 0 1px 0; font-size: 13px; letter-spacing: 1px; color: var(--amber-hot); font-weight: 600; }
|
||||
#info .sub { font-size: 10px; color: var(--text-mute); letter-spacing: 0.5px; margin-bottom: 10px; padding-bottom: 8px; border-bottom: 1px solid var(--border); }
|
||||
#info .row { display: flex; justify-content: space-between; gap: 12px; padding: 2px 0; }
|
||||
#info .row .k { color: var(--text-mute); font-size: 11px; }
|
||||
#info .row .v { color: var(--text); font-variant-numeric: tabular-nums; font-size: 11px; }
|
||||
#info .row .v.amber { color: var(--amber); }
|
||||
#info .row .v.cyan { color: var(--cyan); }
|
||||
#info .row .v.mag { color: var(--magenta); }
|
||||
|
||||
#anim {
|
||||
position: absolute; bottom: 20px; left: 20px; min-width: 280px;
|
||||
background: var(--bg-panel); border: 1px solid var(--border); border-radius: 4px;
|
||||
padding: 12px 14px; backdrop-filter: blur(8px); z-index: 10;
|
||||
}
|
||||
#anim h2 { margin: 0 0 8px 0; font-size: 10px; text-transform: uppercase; letter-spacing: 2px;
|
||||
color: var(--amber); font-weight: 600; border-bottom: 1px solid var(--border); padding-bottom: 6px; }
|
||||
#anim .group { padding: 6px 0; border-bottom: 1px solid rgba(255,184,64,0.08); }
|
||||
#anim .group:last-child { border-bottom: none; }
|
||||
#anim .group-label { font-size: 10px; color: var(--text-mute); text-transform: uppercase; letter-spacing: 1px; margin-bottom: 4px; }
|
||||
#anim button {
|
||||
background: rgba(255,184,64,0.06); border: 1px solid rgba(255,184,64,0.18);
|
||||
color: var(--text); font-family: inherit; font-size: 10px; padding: 4px 8px;
|
||||
margin: 2px 4px 2px 0; cursor: pointer; border-radius: 3px; letter-spacing: 0.5px;
|
||||
}
|
||||
#anim button:hover { background: rgba(255,184,64,0.14); color: var(--amber-hot); }
|
||||
#anim button.active { background: var(--amber); color: var(--bg); border-color: var(--amber); font-weight: 600; }
|
||||
#anim .slider-row { display: flex; align-items: center; gap: 8px; padding: 3px 0; font-size: 10px; }
|
||||
#anim .slider-row .label { width: 90px; color: var(--text-mute); }
|
||||
#anim .slider-row input[type=range] { flex: 1; accent-color: var(--amber); }
|
||||
#anim .slider-row .val { width: 38px; text-align: right; color: var(--amber); font-variant-numeric: tabular-nums; }
|
||||
|
||||
#csi { top: 20px; right: 20px; min-width: 260px; }
|
||||
#csi .bar-row { display: flex; align-items: center; gap: 8px; padding: 3px 0; font-size: 10px; }
|
||||
#csi .bar-row .label { width: 42px; color: var(--text-mute); }
|
||||
#csi .bar-row .bar-track { flex: 1; height: 6px; background: rgba(255,184,64,0.08); border-radius: 2px; overflow: hidden; }
|
||||
#csi .bar-row .bar-fill {
|
||||
height: 100%; background: linear-gradient(90deg, var(--amber-hot), var(--amber));
|
||||
box-shadow: 0 0 6px var(--amber); transition: width 0.08s linear;
|
||||
}
|
||||
#csi .bar-row .val { width: 36px; text-align: right; color: var(--amber); font-variant-numeric: tabular-nums; }
|
||||
|
||||
#helpers {
|
||||
position: absolute; bottom: 20px; right: 20px; min-width: 220px;
|
||||
background: var(--bg-panel); border: 1px solid var(--border); border-radius: 4px;
|
||||
padding: 12px 14px; backdrop-filter: blur(8px); z-index: 10;
|
||||
}
|
||||
#helpers h2 { margin: 0 0 8px 0; font-size: 10px; text-transform: uppercase; letter-spacing: 2px;
|
||||
color: var(--amber); font-weight: 600; border-bottom: 1px solid var(--border); padding-bottom: 6px; }
|
||||
#helpers label {
|
||||
display: flex; align-items: center; gap: 10px; padding: 3px 0; cursor: pointer; user-select: none; font-size: 11px;
|
||||
}
|
||||
#helpers label:hover { color: var(--amber-hot); }
|
||||
#helpers input[type=checkbox] { accent-color: var(--amber); width: 13px; height: 13px; cursor: pointer; }
|
||||
#helpers .swatch { width: 8px; height: 8px; border-radius: 50%; margin-left: auto; box-shadow: 0 0 6px currentColor; }
|
||||
|
||||
#loading {
|
||||
position: absolute; inset: 0; display: flex; align-items: center; justify-content: center;
|
||||
background: rgba(5, 5, 7, 0.96); z-index: 20; font-size: 13px; color: var(--amber);
|
||||
letter-spacing: 2px; text-transform: uppercase;
|
||||
}
|
||||
#loading.hidden { display: none; }
|
||||
#loading .text {
|
||||
text-shadow: 0 0 12px var(--amber);
|
||||
animation: loadPulse 1.4s ease-in-out infinite;
|
||||
}
|
||||
@keyframes loadPulse { 0%,100% { opacity: 0.4; } 50% { opacity: 1.0; } }
|
||||
|
||||
@keyframes scanFlash {
|
||||
0% { opacity: 0; } 10% { opacity: 0.12; } 100% { opacity: 0; }
|
||||
}
|
||||
.scan-flash {
|
||||
position: fixed; inset: 0;
|
||||
background: linear-gradient(90deg, transparent, var(--magenta), transparent);
|
||||
mix-blend-mode: screen; pointer-events: none; opacity: 0; z-index: 4;
|
||||
}
|
||||
|
||||
#titlecard {
|
||||
position: absolute; bottom: 76px; left: 50%; transform: translateX(-50%);
|
||||
text-align: center; color: var(--amber-hot); letter-spacing: 6px; font-size: 11px;
|
||||
text-transform: uppercase; opacity: 0.35; z-index: 10;
|
||||
text-shadow: 0 0 12px var(--amber); pointer-events: none;
|
||||
}
|
||||
#titlecard .sub { font-size: 9px; color: var(--text-mute); letter-spacing: 4px; margin-top: 4px; }
|
||||
|
||||
#adr-badge {
|
||||
position: absolute; top: 50%; right: 20px; transform: translateY(-50%);
|
||||
padding: 6px 10px; background: var(--bg-panel); border: 1px solid var(--border);
|
||||
border-radius: 4px; font-size: 9px; color: var(--text-mute); z-index: 10;
|
||||
backdrop-filter: blur(8px); letter-spacing: 0.5px; max-width: 70px; text-align: center; line-height: 1.5;
|
||||
}
|
||||
#adr-badge a { color: var(--amber); text-decoration: none; display: block; }
|
||||
#adr-badge a:hover { color: var(--amber-hot); }
|
||||
</style>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/controls/OrbitControls.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/loaders/GLTFLoader.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/postprocessing/EffectComposer.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/postprocessing/RenderPass.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/postprocessing/ShaderPass.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/postprocessing/UnrealBloomPass.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/shaders/CopyShader.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/shaders/LuminosityHighPassShader.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<div class="overlay-frame"></div>
|
||||
<div class="scanlines"></div>
|
||||
<div class="scan-flash" id="scan-flash"></div>
|
||||
|
||||
<div id="loading"><div class="text">▸ Loading skinned subject · Xbot.glb · 2.9 MB</div></div>
|
||||
|
||||
<div class="panel" id="info">
|
||||
<h1>RuView · Skinned</h1>
|
||||
<div class="sub">ADR-097 · GLTF skinned mesh · additive animation blending</div>
|
||||
<div class="row"><span class="k">Subject</span><span class="v amber">● Tracked</span></div>
|
||||
<div class="row"><span class="k">Model</span><span class="v">Xbot.glb · 14k tris</span></div>
|
||||
<div class="row"><span class="k">Base anim</span><span class="v amber" id="base-name">walk</span></div>
|
||||
<div class="row"><span class="k">Additive</span><span class="v mag" id="add-name">headShake · 0.40</span></div>
|
||||
<div class="row"><span class="k">Mesh nodes</span><span class="v cyan">4 · multistatic</span></div>
|
||||
<div class="row"><span class="k">Coherence</span><span class="v" id="coh-val">— %</span></div>
|
||||
<div class="row"><span class="k">Heart rate</span><span class="v amber" id="hr-val">— bpm</span></div>
|
||||
<div class="row"><span class="k">Bbox vol</span><span class="v" id="bbox-vol">— m³</span></div>
|
||||
<div class="row"><span class="k">Render</span><span class="v" id="fps-val">— fps</span></div>
|
||||
</div>
|
||||
|
||||
<div id="anim">
|
||||
<h2>AnimationMixer</h2>
|
||||
<div class="group">
|
||||
<div class="group-label">Base · loops</div>
|
||||
<button data-base="idle">idle</button>
|
||||
<button data-base="walk" class="active">walk</button>
|
||||
<button data-base="run">run</button>
|
||||
</div>
|
||||
<div class="group">
|
||||
<div class="group-label">Additive · layered</div>
|
||||
<button data-add="agree">agree</button>
|
||||
<button data-add="headShake" class="active">headShake</button>
|
||||
<button data-add="sad_pose">sad</button>
|
||||
<button data-add="sneak_pose">sneak</button>
|
||||
</div>
|
||||
<div class="group">
|
||||
<div class="slider-row">
|
||||
<span class="label">add weight</span>
|
||||
<input type="range" id="add-weight" min="0" max="1" step="0.01" value="0.40">
|
||||
<span class="val" id="add-weight-val">0.40</span>
|
||||
</div>
|
||||
<div class="slider-row">
|
||||
<span class="label">time scale</span>
|
||||
<input type="range" id="time-scale" min="0.1" max="2" step="0.05" value="1.0">
|
||||
<span class="val" id="time-scale-val">1.00</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel" id="csi">
|
||||
<h2>Per-node CSI</h2>
|
||||
<div class="bar-row"><span class="label">N1·BL</span><div class="bar-track"><div class="bar-fill" id="bar-0" style="width:0"></div></div><span class="val" id="val-0">—</span></div>
|
||||
<div class="bar-row"><span class="label">N2·BR</span><div class="bar-track"><div class="bar-fill" id="bar-1" style="width:0"></div></div><span class="val" id="val-1">—</span></div>
|
||||
<div class="bar-row"><span class="label">N3·FL</span><div class="bar-track"><div class="bar-fill" id="bar-2" style="width:0"></div></div><span class="val" id="val-2">—</span></div>
|
||||
<div class="bar-row"><span class="label">N4·FR</span><div class="bar-track"><div class="bar-fill" id="bar-3" style="width:0"></div></div><span class="val" id="val-3">—</span></div>
|
||||
</div>
|
||||
|
||||
<div id="helpers">
|
||||
<h2>ADR-097 helpers</h2>
|
||||
<label><input type="checkbox" id="t-grid" checked>GridHelper<span class="swatch" style="color:#666"></span></label>
|
||||
<label><input type="checkbox" id="t-polar" checked>PolarGridHelper<span class="swatch" style="color:#ffb840"></span></label>
|
||||
<label><input type="checkbox" id="t-bbox" checked>BoxHelper on mesh<span class="swatch" style="color:#ffe09f"></span></label>
|
||||
<label><input type="checkbox" id="t-skel">SkeletonHelper<span class="swatch" style="color:#4cf"></span></label>
|
||||
<label><input type="checkbox" id="t-nodebox" checked>Per-node BoxHelpers<span class="swatch" style="color:#4cf"></span></label>
|
||||
<label><input type="checkbox" id="t-pings" checked>Sonar pings<span class="swatch" style="color:#4cf"></span></label>
|
||||
<label><input type="checkbox" id="t-tomo" checked>Tomography sweep<span class="swatch" style="color:#ff4cc8"></span></label>
|
||||
</div>
|
||||
|
||||
<div id="titlecard">
|
||||
RuView · Seldon Vault
|
||||
<div class="sub">skinned · ADR-097 · CCDIKSolver next</div>
|
||||
</div>
|
||||
|
||||
<div id="adr-badge">
|
||||
<a href="https://threejs.org/examples/#webgl_animation_skinning_additive_blending" target="_blank" rel="noopener">additive blend</a>
|
||||
<a href="https://threejs.org/examples/#webgl_animation_skinning_ik" target="_blank" rel="noopener" style="margin-top:4px;">skinning IK</a>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// =====================================================================
|
||||
// RuView · Skinned · ADR-097 + GLTF skinned mesh + additive animation
|
||||
// --------------------------------------------------------------------
|
||||
// Replaces the procedural sphere-skeleton of helpers-cinematic.html
|
||||
// with a real rigged + skinned humanoid loaded from Xbot.glb. Plays
|
||||
// a base loop (walk / run / idle) and layers an additive pose on
|
||||
// top (headShake / agree / sneak / sad) — mirrors the upstream
|
||||
// three.js webgl_animation_skinning_additive_blending example.
|
||||
//
|
||||
// All ADR-097 helpers still wrap the loaded mesh — BoxHelper picks
|
||||
// up the live AABB of the SkinnedMesh, the polar grid sits under
|
||||
// the rig, and per-node BoxHelpers wrap the four ESP32 markers.
|
||||
//
|
||||
// Production path (next): swap canned GLTF animations for live
|
||||
// COCO-17 keypoint output → CCDIKSolver targets on hands/feet/head.
|
||||
// Reference: three.js webgl_animation_skinning_ik example.
|
||||
// =====================================================================
|
||||
|
||||
const MODEL_URL = 'https://threejs.org/examples/models/gltf/Xbot.glb';
|
||||
|
||||
const NODE_POSITIONS = [
|
||||
[-1.9, 1.3, 1.9],[ 1.9, 1.3, 1.9],
|
||||
[-1.9, 1.3, -1.9],[ 1.9, 1.3, -1.9],
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Scene
|
||||
// ---------------------------------------------------------------------
|
||||
const scene = new THREE.Scene();
|
||||
scene.background = new THREE.Color(0x050507);
|
||||
scene.fog = new THREE.FogExp2(0x050507, 0.06);
|
||||
|
||||
const camera = new THREE.PerspectiveCamera(48, window.innerWidth/window.innerHeight, 0.05, 100);
|
||||
camera.position.set(3.2, 1.55, 4.0);
|
||||
|
||||
const renderer = new THREE.WebGLRenderer({ antialias: true, powerPreference: 'high-performance' });
|
||||
renderer.setPixelRatio(Math.min(2, window.devicePixelRatio));
|
||||
renderer.setSize(window.innerWidth, window.innerHeight);
|
||||
renderer.toneMapping = THREE.ACESFilmicToneMapping;
|
||||
renderer.toneMappingExposure = 0.80;
|
||||
renderer.outputEncoding = THREE.sRGBEncoding;
|
||||
renderer.shadowMap.enabled = true;
|
||||
renderer.shadowMap.type = THREE.PCFSoftShadowMap;
|
||||
document.body.appendChild(renderer.domElement);
|
||||
|
||||
const controls = new THREE.OrbitControls(camera, renderer.domElement);
|
||||
controls.target.set(0, 0.9, 0);
|
||||
controls.enableDamping = true;
|
||||
controls.dampingFactor = 0.06;
|
||||
controls.minDistance = 2; controls.maxDistance = 12;
|
||||
controls.maxPolarAngle = Math.PI * 0.92;
|
||||
controls.autoRotate = new URLSearchParams(location.search).get('orbit') === '1';
|
||||
controls.autoRotateSpeed = 0.25;
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Lights — the GLTF uses PBR materials so we actually need lighting
|
||||
// here (unlike the all-emissive cinematic.html). Tuned to keep the
|
||||
// amber/cyan mood: amber hemi + amber key + cyan rim lights from
|
||||
// each node direction (visualizes "the nodes illuminate the subject").
|
||||
// ---------------------------------------------------------------------
|
||||
const hemiLight = new THREE.HemisphereLight(0x553a18, 0x080606, 0.7);
|
||||
hemiLight.position.set(0, 4, 0);
|
||||
scene.add(hemiLight);
|
||||
|
||||
const keyLight = new THREE.DirectionalLight(0xffc070, 0.95);
|
||||
keyLight.position.set(2.5, 3.8, 2.5);
|
||||
keyLight.castShadow = true;
|
||||
keyLight.shadow.camera.top = 2; keyLight.shadow.camera.bottom = -2;
|
||||
keyLight.shadow.camera.left = -2; keyLight.shadow.camera.right = 2;
|
||||
keyLight.shadow.camera.near = 0.1; keyLight.shadow.camera.far = 12;
|
||||
keyLight.shadow.mapSize.set(1024, 1024);
|
||||
keyLight.shadow.bias = -0.0008;
|
||||
scene.add(keyLight);
|
||||
|
||||
// cyan rim lights, one per ESP32 node — keeps the "sensed by the mesh" mood
|
||||
const rimLights = [];
|
||||
NODE_POSITIONS.forEach(pos => {
|
||||
const rim = new THREE.PointLight(0x4cf, 0.55, 8, 1.8);
|
||||
rim.position.set(pos[0] * 1.1, pos[1] * 0.7, pos[2] * 1.1);
|
||||
scene.add(rim);
|
||||
rimLights.push(rim);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Post-processing — same composer as cinematic.html
|
||||
// ---------------------------------------------------------------------
|
||||
const composer = new THREE.EffectComposer(renderer);
|
||||
composer.addPass(new THREE.RenderPass(scene, camera));
|
||||
const bloom = new THREE.UnrealBloomPass(
|
||||
new THREE.Vector2(window.innerWidth, window.innerHeight),
|
||||
0.45, 0.40, 0.78,
|
||||
);
|
||||
composer.addPass(bloom);
|
||||
|
||||
const filmShader = {
|
||||
uniforms: {
|
||||
tDiffuse: { value: null },
|
||||
time: { value: 0 }, grain: { value: 0.04 },
|
||||
vignette: { value: 0.32 }, aberration: { value: 0.0018 },
|
||||
},
|
||||
vertexShader: `varying vec2 vUv; void main() { vUv = uv; gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0); }`,
|
||||
fragmentShader: `
|
||||
uniform sampler2D tDiffuse; uniform float time, grain, vignette, aberration;
|
||||
varying vec2 vUv;
|
||||
float hash(vec2 p) { return fract(sin(dot(p, vec2(12.9898, 78.233))) * 43758.5453); }
|
||||
void main() {
|
||||
vec2 off = (vUv - 0.5) * aberration;
|
||||
float r = texture2D(tDiffuse, vUv + off).r;
|
||||
float g = texture2D(tDiffuse, vUv).g;
|
||||
float b = texture2D(tDiffuse, vUv - off).b;
|
||||
vec3 col = vec3(r, g, b);
|
||||
col += (hash(vUv * 1024.0 + time) - 0.5) * grain;
|
||||
float v = smoothstep(0.85, 0.20, length(vUv - 0.5));
|
||||
col *= mix(1.0 - vignette, 1.0, v);
|
||||
gl_FragColor = vec4(col, 1.0);
|
||||
}`,
|
||||
};
|
||||
const filmPass = new THREE.ShaderPass(filmShader);
|
||||
composer.addPass(filmPass);
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Floor — same procedural cyber grid (toned down for skinned scene)
|
||||
// ---------------------------------------------------------------------
|
||||
const floorMat = new THREE.ShaderMaterial({
|
||||
uniforms: { time: { value: 0 }, baseColor: { value: new THREE.Color(0xffb840) } },
|
||||
vertexShader: `varying vec3 vPos; void main() { vPos = position; gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0); }`,
|
||||
fragmentShader: `
|
||||
uniform float time; uniform vec3 baseColor; varying vec3 vPos;
|
||||
void main() {
|
||||
vec2 g = abs(fract(vPos.xz * 0.5) - 0.5);
|
||||
float line = smoothstep(0.48, 0.50, max(g.x, g.y));
|
||||
float majorLine = smoothstep(0.96, 1.00, max(g.x, g.y) * 2.0);
|
||||
float scan = 0.5 + 0.5 * sin((vPos.x + vPos.z) * 2.0 - time * 1.4);
|
||||
scan = pow(scan, 14.0);
|
||||
float falloff = smoothstep(5.0, 1.2, length(vPos.xz));
|
||||
vec3 col = baseColor * (0.01 + 0.05 * line + 0.16 * majorLine + 0.08 * scan);
|
||||
gl_FragColor = vec4(col * falloff, falloff * 0.55);
|
||||
}`,
|
||||
transparent: true, depthWrite: false,
|
||||
});
|
||||
const floor = new THREE.Mesh(new THREE.PlaneGeometry(20, 20), floorMat);
|
||||
floor.rotation.x = -Math.PI / 2;
|
||||
floor.position.y = 0;
|
||||
scene.add(floor);
|
||||
|
||||
// shadow-receiving ground (invisible, just catches the shadow)
|
||||
const shadowGround = new THREE.Mesh(
|
||||
new THREE.PlaneGeometry(20, 20),
|
||||
new THREE.ShadowMaterial({ opacity: 0.55 })
|
||||
);
|
||||
shadowGround.rotation.x = -Math.PI / 2;
|
||||
shadowGround.position.y = 0.001;
|
||||
shadowGround.receiveShadow = true;
|
||||
scene.add(shadowGround);
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// ADR-097 helpers
|
||||
// ---------------------------------------------------------------------
|
||||
const gridHelper = new THREE.GridHelper(4, 20, 0x554a32, 0x2a2418);
|
||||
gridHelper.material.transparent = true; gridHelper.material.opacity = 0.45;
|
||||
scene.add(gridHelper);
|
||||
|
||||
const polarHelper = new THREE.PolarGridHelper(2.2, 16, 4, 64, 0xffb840, 0x4a3a1a);
|
||||
polarHelper.position.y = 0.002;
|
||||
polarHelper.material.transparent = true; polarHelper.material.opacity = 0.55;
|
||||
scene.add(polarHelper);
|
||||
|
||||
let bboxHelper = null;
|
||||
let skeletonHelper = null;
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Multistatic sensor nodes — same as cinematic
|
||||
// ---------------------------------------------------------------------
|
||||
const nodeGroup = new THREE.Group();
|
||||
scene.add(nodeGroup);
|
||||
const nodeBboxHelpers = [];
|
||||
const nodeRings = [];
|
||||
const nodeAnchors = [];
|
||||
const nodeBodyGeo = new THREE.BoxGeometry(0.14, 0.06, 0.20);
|
||||
const nodeBodyMat = new THREE.MeshBasicMaterial({ color: 0xffb840 });
|
||||
const antennaGeo = new THREE.ConeGeometry(0.018, 0.10, 8);
|
||||
const antennaMat = new THREE.MeshBasicMaterial({ color: 0xffe09f });
|
||||
|
||||
NODE_POSITIONS.forEach((pos, i) => {
|
||||
const group = new THREE.Group();
|
||||
group.position.set(pos[0], pos[1], pos[2]);
|
||||
|
||||
const body = new THREE.Mesh(nodeBodyGeo, nodeBodyMat);
|
||||
group.add(body);
|
||||
const antenna = new THREE.Mesh(antennaGeo, antennaMat);
|
||||
antenna.position.y = 0.08; group.add(antenna);
|
||||
|
||||
const ring = new THREE.Mesh(
|
||||
new THREE.RingGeometry(0.11, 0.14, 32),
|
||||
new THREE.MeshBasicMaterial({ color: 0xffb840, side: THREE.DoubleSide, transparent: true,
|
||||
opacity: 0.55, blending: THREE.AdditiveBlending, depthWrite: false })
|
||||
);
|
||||
ring.rotation.x = -Math.PI / 2; ring.position.y = -0.05;
|
||||
ring.userData.phase = i * 0.7;
|
||||
group.add(ring); nodeRings.push(ring);
|
||||
|
||||
const core = new THREE.Mesh(
|
||||
new THREE.SphereGeometry(0.025, 12, 12),
|
||||
new THREE.MeshBasicMaterial({ color: 0xffe09f })
|
||||
);
|
||||
core.position.y = 0.04; group.add(core);
|
||||
|
||||
nodeGroup.add(group); nodeAnchors.push(group);
|
||||
|
||||
const bbox = new THREE.BoxHelper(group, 0x4cf);
|
||||
bbox.material.transparent = true; bbox.material.opacity = 0.45;
|
||||
scene.add(bbox); nodeBboxHelpers.push(bbox);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// GLTF — load the rigged Xbot model
|
||||
// ---------------------------------------------------------------------
|
||||
let model = null;
|
||||
let mixer = null;
|
||||
let headBone = null;
|
||||
const baseActions = {}; // idle / walk / run
|
||||
const additiveActions = {}; // sneak_pose / sad_pose / agree / headShake
|
||||
let currentBase = 'walk';
|
||||
let currentAddName = 'headShake';
|
||||
let addWeight = 0.40;
|
||||
|
||||
const loader = new THREE.GLTFLoader();
|
||||
loader.load(MODEL_URL, (gltf) => {
|
||||
model = gltf.scene;
|
||||
model.position.y = 0;
|
||||
model.traverse(obj => {
|
||||
if (obj.isMesh) { obj.castShadow = true; obj.receiveShadow = true; }
|
||||
if (obj.isBone && /head/i.test(obj.name) && !headBone) headBone = obj;
|
||||
});
|
||||
scene.add(model);
|
||||
|
||||
skeletonHelper = new THREE.SkeletonHelper(model);
|
||||
skeletonHelper.visible = false;
|
||||
scene.add(skeletonHelper);
|
||||
|
||||
mixer = new THREE.AnimationMixer(model);
|
||||
const baseNames = new Set(['idle', 'walk', 'run']);
|
||||
const additiveNames = new Set(['sneak_pose', 'sad_pose', 'agree', 'headShake']);
|
||||
|
||||
for (let i = 0; i < gltf.animations.length; i++) {
|
||||
let clip = gltf.animations[i];
|
||||
const name = clip.name;
|
||||
if (baseNames.has(name)) {
|
||||
const action = mixer.clipAction(clip);
|
||||
action.enabled = true;
|
||||
action.setEffectiveTimeScale(1);
|
||||
action.setEffectiveWeight(name === currentBase ? 1 : 0);
|
||||
action.play();
|
||||
baseActions[name] = action;
|
||||
} else if (additiveNames.has(name)) {
|
||||
THREE.AnimationUtils.makeClipAdditive(clip);
|
||||
if (name.endsWith('_pose')) {
|
||||
clip = THREE.AnimationUtils.subclip(clip, name, 2, 3, 30);
|
||||
}
|
||||
const action = mixer.clipAction(clip);
|
||||
action.enabled = true;
|
||||
action.setEffectiveTimeScale(1);
|
||||
action.setEffectiveWeight(name === currentAddName ? addWeight : 0);
|
||||
action.play();
|
||||
additiveActions[name] = action;
|
||||
}
|
||||
}
|
||||
|
||||
// build the face point cloud anchored to head bone
|
||||
buildFacePointCloud();
|
||||
|
||||
document.getElementById('loading').classList.add('hidden');
|
||||
}, (xhr) => {
|
||||
const pct = xhr.loaded / (xhr.total || 2930032) * 100;
|
||||
const txt = document.querySelector('#loading .text');
|
||||
if (txt) txt.textContent = `▸ Loading skinned subject · Xbot.glb · ${pct.toFixed(0)} %`;
|
||||
}, (err) => {
|
||||
console.error('GLTF load failed', err);
|
||||
document.querySelector('#loading .text').textContent = '⚠ Load failed — see console';
|
||||
});
|
||||
|
||||
function setBase(name) {
|
||||
if (!baseActions[name]) return;
|
||||
for (const k in baseActions) {
|
||||
const a = baseActions[k];
|
||||
const target = (k === name) ? 1 : 0;
|
||||
a.crossFadeTo ? null : null; // (no-op — using simple weight crossfade)
|
||||
a.setEffectiveWeight(target);
|
||||
}
|
||||
currentBase = name;
|
||||
document.getElementById('base-name').textContent = name;
|
||||
for (const btn of document.querySelectorAll('#anim [data-base]')) {
|
||||
btn.classList.toggle('active', btn.dataset.base === name);
|
||||
}
|
||||
}
|
||||
function setAdditive(name) {
|
||||
for (const k in additiveActions) {
|
||||
additiveActions[k].setEffectiveWeight(k === name ? addWeight : 0);
|
||||
}
|
||||
currentAddName = name;
|
||||
document.getElementById('add-name').textContent = name + ' · ' + addWeight.toFixed(2);
|
||||
for (const btn of document.querySelectorAll('#anim [data-add]')) {
|
||||
btn.classList.toggle('active', btn.dataset.add === name);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Face point cloud — anchored to head bone via getWorldPosition each frame
|
||||
// ---------------------------------------------------------------------
|
||||
const FACE_POINTS = 480;
|
||||
const facePositions = new Float32Array(FACE_POINTS * 3);
|
||||
const faceOffsets = new Float32Array(FACE_POINTS * 3);
|
||||
const facePhases = new Float32Array(FACE_POINTS);
|
||||
let facePoints = null;
|
||||
function buildFacePointCloud() {
|
||||
for (let i = 0; i < FACE_POINTS; i++) {
|
||||
const u = Math.random() * Math.PI * 2;
|
||||
const v = (Math.random() - 0.5) * Math.PI * 0.95;
|
||||
const cu = Math.cos(u), su = Math.sin(u);
|
||||
const cv = Math.cos(v), sv = Math.sin(v);
|
||||
faceOffsets[i*3+0] = 0.085 * cv * cu;
|
||||
faceOffsets[i*3+1] = 0.108 * sv;
|
||||
faceOffsets[i*3+2] = 0.072 * cv * su;
|
||||
facePhases[i] = Math.random() * Math.PI * 2;
|
||||
}
|
||||
const geom = new THREE.BufferGeometry();
|
||||
geom.setAttribute('position', new THREE.BufferAttribute(facePositions, 3));
|
||||
geom.setAttribute('aPhase', new THREE.BufferAttribute(facePhases, 1));
|
||||
const mat = new THREE.ShaderMaterial({
|
||||
uniforms: { time: { value: 0 } },
|
||||
vertexShader: `
|
||||
attribute float aPhase; uniform float time;
|
||||
varying float vAlpha;
|
||||
void main() {
|
||||
vec4 mv = modelViewMatrix * vec4(position, 1.0);
|
||||
float shimmer = 0.5 + 0.5 * sin(time * 3.0 + aPhase);
|
||||
vAlpha = 0.18 + 0.30 * shimmer;
|
||||
gl_Position = projectionMatrix * mv;
|
||||
gl_PointSize = (1.6 + shimmer * 1.0) * (200.0 / -mv.z);
|
||||
}`,
|
||||
fragmentShader: `
|
||||
varying float vAlpha;
|
||||
void main() {
|
||||
vec2 c = gl_PointCoord - 0.5;
|
||||
float d = length(c);
|
||||
if (d > 0.5) discard;
|
||||
float falloff = smoothstep(0.5, 0.0, d);
|
||||
vec3 col = mix(vec3(0.18, 0.52, 0.72), vec3(0.55, 0.62, 0.72), 0.5);
|
||||
gl_FragColor = vec4(col * (1.0 + falloff * 0.3), vAlpha * falloff);
|
||||
}`,
|
||||
transparent: true, depthWrite: false,
|
||||
});
|
||||
facePoints = new THREE.Points(geom, mat);
|
||||
scene.add(facePoints);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Sonar pings + tomography sweep — same as cinematic.html
|
||||
// ---------------------------------------------------------------------
|
||||
const PING_POOL = 24;
|
||||
const pings = [];
|
||||
const pingGeo = new THREE.TorusGeometry(1, 0.012, 8, 48);
|
||||
for (let i = 0; i < PING_POOL; i++) {
|
||||
const mat = new THREE.MeshBasicMaterial({ color: 0x4cf, transparent: true, opacity: 0, depthWrite: false });
|
||||
const mesh = new THREE.Mesh(pingGeo, mat);
|
||||
mesh.visible = false; scene.add(mesh);
|
||||
pings.push({ mesh, active: false, t0: 0, duration: 0,
|
||||
origin: new THREE.Vector3(), target: new THREE.Vector3() });
|
||||
}
|
||||
let pingIndex = 0;
|
||||
function emitPing(origin, target) {
|
||||
const p = pings[pingIndex]; pingIndex = (pingIndex + 1) % PING_POOL;
|
||||
p.active = true; p.t0 = performance.now() * 0.001;
|
||||
p.duration = 0.55 + Math.random() * 0.20;
|
||||
p.origin.copy(origin); p.target.copy(target);
|
||||
p.mesh.position.copy(origin); p.mesh.visible = true;
|
||||
p.mesh.material.opacity = 0;
|
||||
const dir = new THREE.Vector3().subVectors(target, origin).normalize();
|
||||
p.mesh.quaternion.setFromUnitVectors(new THREE.Vector3(0, 0, 1), dir);
|
||||
}
|
||||
|
||||
const tomoMat = new THREE.ShaderMaterial({
|
||||
uniforms: { time: { value: 0 }, intensity: { value: 0 } },
|
||||
vertexShader: `varying vec2 vUv; void main() { vUv = uv; gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0); }`,
|
||||
fragmentShader: `
|
||||
uniform float time, intensity; varying vec2 vUv;
|
||||
void main() {
|
||||
float band = exp(-pow((vUv.x - 0.5) * 14.0, 2.0));
|
||||
float lines = 0.5 + 0.5 * sin(vUv.y * 90.0 + time * 4.0);
|
||||
vec3 col = vec3(1.0, 0.3, 0.78) * band * (0.6 + 0.4 * lines);
|
||||
gl_FragColor = vec4(col, intensity * band * 0.75);
|
||||
}`,
|
||||
transparent: true, blending: THREE.AdditiveBlending, depthWrite: false, side: THREE.DoubleSide,
|
||||
});
|
||||
const tomoPlane = new THREE.Mesh(new THREE.PlaneGeometry(8, 6), tomoMat);
|
||||
tomoPlane.rotation.y = Math.PI / 2;
|
||||
tomoPlane.position.set(-2, 1.0, 0);
|
||||
tomoPlane.visible = false;
|
||||
scene.add(tomoPlane);
|
||||
let tomoActive = false, tomoT0 = 0, tomoNextAt = 4 + Math.random() * 4;
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Pseudo-CSI driver — same as cinematic
|
||||
// ---------------------------------------------------------------------
|
||||
const csiAmp = [0, 0, 0, 0];
|
||||
let csiCoherence = 0.5;
|
||||
const csiNoise = [0, 0, 0, 0];
|
||||
|
||||
function tickCsi(t, targetWorld) {
|
||||
for (let i = 0; i < 4; i++) csiNoise[i] = csiNoise[i] * 0.92 + (Math.random() - 0.5) * 0.08;
|
||||
let mean = 0; const amps = [];
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const np = NODE_POSITIONS[i];
|
||||
const dx = np[0] - targetWorld.x, dy = np[1] - targetWorld.y, dz = np[2] - targetWorld.z;
|
||||
const r2 = dx*dx + dy*dy + dz*dz;
|
||||
const fall = 1.0 / (1.0 + r2 * 0.18);
|
||||
const breath = Math.sin(t * 0.27 * Math.PI * 2) * 0.10;
|
||||
const heart = Math.sin(t * 1.18 * Math.PI * 2) * 0.04;
|
||||
const walk = Math.sin(t * 1.9 + i * 0.5) * 0.12;
|
||||
const a = Math.max(0, Math.min(1, fall + breath + heart + walk + csiNoise[i] * 0.30));
|
||||
amps.push(a);
|
||||
csiAmp[i] = csiAmp[i] * 0.7 + a * 0.3;
|
||||
mean += a;
|
||||
}
|
||||
mean /= 4;
|
||||
let v = 0; for (let i = 0; i < 4; i++) v += (amps[i] - mean) ** 2;
|
||||
v = Math.sqrt(v / 4);
|
||||
csiCoherence = csiCoherence * 0.85 + Math.max(0, Math.min(1, 1.0 - v * 2.5)) * 0.15;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Per-frame updates
|
||||
// ---------------------------------------------------------------------
|
||||
const tmpVec = new THREE.Vector3();
|
||||
let lastPingT = [0, 0, 0, 0];
|
||||
|
||||
function updateNodes() {
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const ring = nodeRings[i];
|
||||
const amp = csiAmp[i];
|
||||
ring.material.opacity = 0.32 + 0.55 * amp;
|
||||
ring.scale.setScalar(1 + 0.30 * amp);
|
||||
rimLights[i].intensity = 0.30 + 0.60 * amp * csiCoherence;
|
||||
}
|
||||
}
|
||||
function maybeEmitPings(t, modelCenter) {
|
||||
if (!document.getElementById('t-pings').checked || !model) return;
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const interval = 1.2 / (0.25 + csiAmp[i]);
|
||||
if (t - lastPingT[i] > interval) {
|
||||
lastPingT[i] = t;
|
||||
const target = modelCenter.clone();
|
||||
target.y += (Math.random() - 0.3) * 0.8;
|
||||
target.x += (Math.random() - 0.5) * 0.2;
|
||||
const origin = nodeAnchors[i].getWorldPosition(new THREE.Vector3());
|
||||
emitPing(origin, target);
|
||||
}
|
||||
}
|
||||
}
|
||||
function updatePings(t) {
|
||||
for (const p of pings) {
|
||||
if (!p.active) continue;
|
||||
const u = (t - p.t0) / p.duration;
|
||||
if (u >= 1) { p.active = false; p.mesh.visible = false; continue; }
|
||||
p.mesh.position.lerpVectors(p.origin, p.target, u);
|
||||
p.mesh.scale.setScalar(0.03 + u * 0.18);
|
||||
p.mesh.material.opacity = (1.0 - u) * 0.40 * csiCoherence;
|
||||
}
|
||||
}
|
||||
function updateTomography(t) {
|
||||
if (!document.getElementById('t-tomo').checked) { tomoActive = false; tomoPlane.visible = false; return; }
|
||||
if (!tomoActive && t > tomoNextAt) {
|
||||
tomoActive = true; tomoT0 = t; tomoPlane.visible = true;
|
||||
const sf = document.getElementById('scan-flash');
|
||||
sf.style.animation = 'none';
|
||||
requestAnimationFrame(() => { sf.style.animation = 'scanFlash 1.6s ease-out'; });
|
||||
}
|
||||
if (tomoActive) {
|
||||
const dur = 2.4;
|
||||
const e = (t - tomoT0) / dur;
|
||||
if (e >= 1) {
|
||||
tomoActive = false; tomoPlane.visible = false;
|
||||
tomoNextAt = t + 4 + Math.random() * 5;
|
||||
} else {
|
||||
tomoPlane.position.x = -3 + e * 6;
|
||||
tomoMat.uniforms.intensity.value = Math.sin(e * Math.PI);
|
||||
tomoMat.uniforms.time.value = t;
|
||||
}
|
||||
}
|
||||
}
|
||||
function updateBbox() {
|
||||
const want = document.getElementById('t-bbox').checked && model;
|
||||
if (!want) {
|
||||
if (bboxHelper) { scene.remove(bboxHelper); bboxHelper = null; }
|
||||
document.getElementById('bbox-vol').textContent = '—';
|
||||
return;
|
||||
}
|
||||
if (!bboxHelper) {
|
||||
bboxHelper = new THREE.BoxHelper(model, 0xffe09f);
|
||||
bboxHelper.material.transparent = true; bboxHelper.material.opacity = 0.55;
|
||||
scene.add(bboxHelper);
|
||||
} else bboxHelper.setFromObject(model);
|
||||
const box = new THREE.Box3().setFromObject(model);
|
||||
const size = box.getSize(new THREE.Vector3());
|
||||
document.getElementById('bbox-vol').textContent = (size.x * size.y * size.z).toFixed(3) + ' m³';
|
||||
}
|
||||
function updateFaceCloud(t) {
|
||||
if (!facePoints || !headBone) return;
|
||||
const headWorld = new THREE.Vector3();
|
||||
headBone.getWorldPosition(headWorld);
|
||||
const pos = facePoints.geometry.attributes.position;
|
||||
for (let i = 0; i < FACE_POINTS; i++) {
|
||||
pos.array[i*3+0] = headWorld.x + faceOffsets[i*3+0];
|
||||
pos.array[i*3+1] = headWorld.y + faceOffsets[i*3+1] + 0.06;
|
||||
pos.array[i*3+2] = headWorld.z + faceOffsets[i*3+2];
|
||||
}
|
||||
pos.needsUpdate = true;
|
||||
facePoints.material.uniforms.time.value = t;
|
||||
}
|
||||
let hudT = 0;
|
||||
function updateHud(t, fps) {
|
||||
if (t - hudT < 0.1) return;
|
||||
hudT = t;
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const pct = Math.round(csiAmp[i] * 100);
|
||||
document.getElementById('bar-' + i).style.width = pct + '%';
|
||||
document.getElementById('val-' + i).textContent = pct + '%';
|
||||
}
|
||||
document.getElementById('coh-val').textContent = (csiCoherence * 100).toFixed(0) + ' %';
|
||||
document.getElementById('hr-val').textContent = (68 + Math.sin(t * 0.3) * 4).toFixed(0) + ' bpm';
|
||||
document.getElementById('fps-val').textContent = fps.toFixed(0) + ' fps';
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// UI wiring
|
||||
// ---------------------------------------------------------------------
|
||||
for (const btn of document.querySelectorAll('#anim [data-base]')) {
|
||||
btn.addEventListener('click', () => setBase(btn.dataset.base));
|
||||
}
|
||||
for (const btn of document.querySelectorAll('#anim [data-add]')) {
|
||||
btn.addEventListener('click', () => setAdditive(btn.dataset.add));
|
||||
}
|
||||
document.getElementById('add-weight').addEventListener('input', (e) => {
|
||||
addWeight = parseFloat(e.target.value);
|
||||
document.getElementById('add-weight-val').textContent = addWeight.toFixed(2);
|
||||
if (additiveActions[currentAddName]) additiveActions[currentAddName].setEffectiveWeight(addWeight);
|
||||
document.getElementById('add-name').textContent = currentAddName + ' · ' + addWeight.toFixed(2);
|
||||
});
|
||||
document.getElementById('time-scale').addEventListener('input', (e) => {
|
||||
const ts = parseFloat(e.target.value);
|
||||
document.getElementById('time-scale-val').textContent = ts.toFixed(2);
|
||||
if (mixer) mixer.timeScale = ts;
|
||||
});
|
||||
function bindToggle(id, obj) {
|
||||
document.getElementById(id).addEventListener('change', e => {
|
||||
if (e.target.checked && !scene.children.includes(obj)) scene.add(obj);
|
||||
else if (!e.target.checked) scene.remove(obj);
|
||||
});
|
||||
}
|
||||
bindToggle('t-grid', gridHelper);
|
||||
bindToggle('t-polar', polarHelper);
|
||||
document.getElementById('t-skel').addEventListener('change', e => {
|
||||
if (skeletonHelper) skeletonHelper.visible = e.target.checked;
|
||||
});
|
||||
document.getElementById('t-nodebox').addEventListener('change', e => {
|
||||
for (const bb of nodeBboxHelpers) {
|
||||
if (e.target.checked && !scene.children.includes(bb)) scene.add(bb);
|
||||
else if (!e.target.checked) scene.remove(bb);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Main loop
|
||||
// ---------------------------------------------------------------------
|
||||
const clock = new THREE.Clock();
|
||||
let lastMs = performance.now();
|
||||
let fpsEma = 60;
|
||||
function tick() {
|
||||
const nowMs = performance.now();
|
||||
const dt = nowMs - lastMs;
|
||||
lastMs = nowMs;
|
||||
fpsEma = fpsEma * 0.92 + (1000 / Math.max(dt, 1)) * 0.08;
|
||||
const t = nowMs * 0.001;
|
||||
const delta = clock.getDelta();
|
||||
|
||||
if (mixer) mixer.update(delta);
|
||||
floorMat.uniforms.time.value = t;
|
||||
filmShader.uniforms.time.value = t;
|
||||
|
||||
// get model center for CSI / ping targeting
|
||||
const center = new THREE.Vector3();
|
||||
if (model) {
|
||||
const box = new THREE.Box3().setFromObject(model);
|
||||
box.getCenter(center);
|
||||
} else center.set(0, 0.9, 0);
|
||||
|
||||
tickCsi(t, center);
|
||||
updateNodes();
|
||||
maybeEmitPings(t, center);
|
||||
updatePings(t);
|
||||
updateTomography(t);
|
||||
updateBbox();
|
||||
updateFaceCloud(t);
|
||||
|
||||
controls.update();
|
||||
composer.render();
|
||||
|
||||
updateHud(t, fpsEma);
|
||||
requestAnimationFrame(tick);
|
||||
}
|
||||
requestAnimationFrame(tick);
|
||||
|
||||
window.addEventListener('resize', () => {
|
||||
camera.aspect = window.innerWidth / window.innerHeight;
|
||||
camera.updateProjectionMatrix();
|
||||
renderer.setSize(window.innerWidth, window.innerHeight);
|
||||
composer.setSize(window.innerWidth, window.innerHeight);
|
||||
bloom.setSize(window.innerWidth, window.innerHeight);
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,168 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta name="robots" content="noindex,nofollow">
|
||||
<title>RuView · three.js demos · ADR-097 sensing-helpers scene</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #0a0e1a;
|
||||
--bg2: #111627;
|
||||
--card: #171d30;
|
||||
--card-h: #1e2540;
|
||||
--border: #252d45;
|
||||
--t1: #e0e4f0;
|
||||
--t2: #8890a8;
|
||||
--cyan: #4ecdc4;
|
||||
--green: #6bcb77;
|
||||
--amber: #d4a574;
|
||||
--r: 10px;
|
||||
}
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
body {
|
||||
font-family: 'Segoe UI', system-ui, -apple-system, sans-serif;
|
||||
background: var(--bg);
|
||||
color: var(--t1);
|
||||
line-height: 1.5;
|
||||
padding: 24px 16px 64px;
|
||||
}
|
||||
.wrap { max-width: 980px; margin: 0 auto; }
|
||||
h1 { font-size: 22px; color: #fff; }
|
||||
h1 span { color: var(--cyan); }
|
||||
.lede { color: var(--t2); margin: 8px 0 24px; font-size: 14px; max-width: 70ch; }
|
||||
.pill {
|
||||
display: inline-block;
|
||||
padding: 2px 8px;
|
||||
border-radius: 999px;
|
||||
font-size: 11px;
|
||||
margin-left: 8px;
|
||||
vertical-align: middle;
|
||||
border: 1px solid var(--border);
|
||||
background: var(--bg2);
|
||||
color: var(--t2);
|
||||
}
|
||||
.pill.ok { color: var(--green); border-color: #2d4a35; background: rgba(107, 203, 119, 0.08); }
|
||||
.pill.warn { color: var(--amber); border-color: #4a3d2d; background: rgba(212, 165, 116, 0.08); }
|
||||
.grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(280px, 1fr));
|
||||
gap: 12px;
|
||||
margin-top: 16px;
|
||||
}
|
||||
.card {
|
||||
background: var(--card);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--r);
|
||||
padding: 16px;
|
||||
text-decoration: none;
|
||||
color: inherit;
|
||||
transition: background 0.12s, border-color 0.12s, transform 0.12s;
|
||||
}
|
||||
.card:hover {
|
||||
background: var(--card-h);
|
||||
border-color: var(--cyan);
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
.card h2 { font-size: 15px; color: #fff; margin-bottom: 6px; }
|
||||
.card .sub { color: var(--t2); font-size: 13px; }
|
||||
.card img {
|
||||
margin-top: 10px;
|
||||
width: 100%;
|
||||
aspect-ratio: 16/9;
|
||||
object-fit: cover;
|
||||
border-radius: 6px;
|
||||
border: 1px solid var(--border);
|
||||
background: #000;
|
||||
}
|
||||
.note {
|
||||
margin-top: 28px;
|
||||
padding: 14px 16px;
|
||||
background: rgba(212, 165, 116, 0.06);
|
||||
border-left: 3px solid var(--amber);
|
||||
border-radius: 6px;
|
||||
font-size: 13px;
|
||||
color: var(--t1);
|
||||
}
|
||||
.note b { color: var(--amber); }
|
||||
code {
|
||||
font-family: 'Cascadia Code', Consolas, monospace;
|
||||
background: var(--bg2);
|
||||
padding: 1px 5px;
|
||||
border-radius: 3px;
|
||||
color: var(--cyan);
|
||||
font-size: 12px;
|
||||
}
|
||||
a { color: var(--cyan); }
|
||||
.foot {
|
||||
color: var(--t2);
|
||||
font-size: 12px;
|
||||
margin-top: 32px;
|
||||
text-align: center;
|
||||
}
|
||||
.foot a { color: var(--cyan); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
|
||||
<h1>RuView · <span>three.js demos</span></h1>
|
||||
<p class="lede">
|
||||
Five progressively richer browser demos of the <a href="https://github.com/ruvnet/RuView/blob/main/docs/adr/ADR-097-adopt-rvcsi-as-ruview-csi-runtime.md">ADR-097</a>
|
||||
sensing-helpers scene, ending with a live MediaPipe-Pose → Mixamo X Bot retargeting pipeline driven
|
||||
by a real ESP32 CSI feed.
|
||||
</p>
|
||||
|
||||
<div class="grid">
|
||||
|
||||
<a class="card" href="demos/01-helpers.html">
|
||||
<h2>01 · Helpers <span class="pill ok">standalone</span></h2>
|
||||
<div class="sub">Plain ADR-097 helpers in the point-cloud viewer. No external assets.</div>
|
||||
<img src="screenshots/01-helpers.png" alt="01 screenshot">
|
||||
</a>
|
||||
|
||||
<a class="card" href="demos/02-cinematic.html">
|
||||
<h2>02 · Cinematic <span class="pill ok">standalone</span></h2>
|
||||
<div class="sub">Cinematic camera + pseudo-CSI visualization on top of #01.</div>
|
||||
<img src="screenshots/02-cinematic.png" alt="02 screenshot">
|
||||
</a>
|
||||
|
||||
<a class="card" href="demos/03-skinned.html">
|
||||
<h2>03 · Skinned (GLTF) <span class="pill ok">standalone</span></h2>
|
||||
<div class="sub">GLTF skinned mesh + additive animation blending in the ADR-097 scene.</div>
|
||||
<img src="screenshots/03-skinned.png" alt="03 screenshot">
|
||||
</a>
|
||||
|
||||
<a class="card" href="demos/04-skinned-fbx.html">
|
||||
<h2>04 · Skinned FBX <span class="pill warn">needs FBX</span></h2>
|
||||
<div class="sub">Mixamo X Bot via FBXLoader. Requires a local <code>assets/X Bot.fbx</code>.</div>
|
||||
<img src="screenshots/04-skinned-fbx.png" alt="04 screenshot">
|
||||
</a>
|
||||
|
||||
<a class="card" href="demos/05-skinned-realtime.html">
|
||||
<h2>05 · Realtime (Pose + CSI) <span class="pill warn">needs FBX</span></h2>
|
||||
<div class="sub">Webcam → MediaPipe Pose Heavy → Mixamo IK retarget, live ESP32 CSI overlay.</div>
|
||||
<img src="screenshots/05-skinned-realtime.png" alt="05 screenshot">
|
||||
</a>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="note">
|
||||
<b>Demos 04 and 05 need a Mixamo asset.</b> The Mixamo
|
||||
<code>X Bot.fbx</code> file is intentionally <em>not</em> redistributed in
|
||||
this deployment — it's licensed for end-users to download from
|
||||
<a href="https://mixamo.com" target="_blank" rel="noopener">mixamo.com</a> directly.
|
||||
To run these locally: clone the repo, download <code>X Bot.fbx</code>
|
||||
(FBX Binary, T-Pose, Without Skin) into
|
||||
<code>examples/three.js/assets/</code>, then run
|
||||
<code>python examples/three.js/server/serve-demo.py</code>.
|
||||
</div>
|
||||
|
||||
<div class="foot">
|
||||
Source: <a href="https://github.com/ruvnet/RuView/tree/main/examples/three.js">github.com/ruvnet/RuView/tree/main/examples/three.js</a>
|
||||
· ADR-097 · three.js r128
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
After Width: | Height: | Size: 96 KiB |
|
After Width: | Height: | Size: 598 KiB |
|
After Width: | Height: | Size: 632 KiB |
|
After Width: | Height: | Size: 682 KiB |
|
After Width: | Height: | Size: 596 KiB |
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env python3
|
||||
"""ruvultra → browser CSI bridge.
|
||||
|
||||
Reads adaptive_ctrl tick lines from the ESP32-S3 RuView firmware on
|
||||
/dev/ttyACM0 and forwards normalized per-node metrics over a WebSocket
|
||||
that the helpers-skinned-realtime demo can subscribe to via Tailscale.
|
||||
|
||||
Sample serial line (1 Hz cadence from firmware):
|
||||
I (22890561) adaptive_ctrl: medium tick: state=6 yield=15pps motion=1.00 presence=5.35 rssi=-33
|
||||
|
||||
Output JSON (per tick):
|
||||
{
|
||||
"ts": 1716830400.123,
|
||||
"node": 0, # always 0 (single node), client expands to 4
|
||||
"motion": 1.00, # raw firmware metric
|
||||
"presence": 5.35,
|
||||
"rssi": -33,
|
||||
"yield_pps": 15,
|
||||
"amp": 0.78 # synthesized CSI amplitude in [0..1] for the bar
|
||||
}
|
||||
|
||||
Run on ruvultra:
|
||||
python3 -u ruvultra-csi-bridge.py
|
||||
"""
|
||||
import asyncio
|
||||
import builtins
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from contextlib import suppress
|
||||
|
||||
# Force every print to flush — we're often piped to a log file
|
||||
_orig_print = builtins.print
|
||||
def _print(*a, **kw):
|
||||
kw.setdefault("flush", True)
|
||||
return _orig_print(*a, **kw)
|
||||
builtins.print = _print
|
||||
|
||||
import serial
|
||||
import websockets
|
||||
|
||||
PORT = "/dev/ttyACM0"
|
||||
BAUD = 115200
|
||||
WS_HOST = "0.0.0.0"
|
||||
WS_PORT = 8766
|
||||
|
||||
TICK_RE = re.compile(
|
||||
r"adaptive_ctrl:\s*\w+\s+tick:\s*"
|
||||
r"state=(?P<state>\d+)\s+"
|
||||
r"yield=(?P<yield>\d+)pps\s+"
|
||||
r"motion=(?P<motion>[\d.]+)\s+"
|
||||
r"presence=(?P<presence>[\d.]+)\s+"
|
||||
r"rssi=(?P<rssi>-?\d+)"
|
||||
)
|
||||
|
||||
clients = set()
|
||||
last_payload = None
|
||||
|
||||
|
||||
def amp_from_metrics(motion, presence, rssi):
|
||||
"""Map firmware metrics to a [0..1] CSI-style amplitude."""
|
||||
rssi_norm = max(0.0, min(1.0, (rssi + 80) / 50)) # -80..-30 → 0..1
|
||||
presence_norm = max(0.0, min(1.0, presence / 8.0)) # cap at 8
|
||||
motion_norm = max(0.0, min(1.0, motion)) # already 0..1ish
|
||||
return 0.40 * rssi_norm + 0.35 * presence_norm + 0.25 * motion_norm
|
||||
|
||||
|
||||
async def serial_reader_loop():
|
||||
global last_payload
|
||||
print(f"[bridge] opening {PORT} @ {BAUD}…")
|
||||
while True:
|
||||
try:
|
||||
ser = serial.Serial(PORT, BAUD, timeout=1)
|
||||
except (serial.SerialException, OSError) as e:
|
||||
print(f"[bridge] serial open failed ({e}); retry in 3s")
|
||||
await asyncio.sleep(3)
|
||||
continue
|
||||
|
||||
print(f"[bridge] connected to {PORT}")
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
while True:
|
||||
line = await loop.run_in_executor(None, ser.readline)
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
text = line.decode(errors="replace").strip()
|
||||
except Exception:
|
||||
continue
|
||||
m = TICK_RE.search(text)
|
||||
if not m:
|
||||
continue
|
||||
motion = float(m["motion"])
|
||||
presence = float(m["presence"])
|
||||
rssi = int(m["rssi"])
|
||||
payload = {
|
||||
"ts": time.time(),
|
||||
"node": 0,
|
||||
"state": int(m["state"]),
|
||||
"yield_pps": int(m["yield"]),
|
||||
"motion": motion,
|
||||
"presence": presence,
|
||||
"rssi": rssi,
|
||||
"amp": amp_from_metrics(motion, presence, rssi),
|
||||
}
|
||||
last_payload = payload
|
||||
msg = json.dumps(payload)
|
||||
if clients:
|
||||
dead = []
|
||||
for ws in list(clients):
|
||||
try:
|
||||
await ws.send(msg)
|
||||
except websockets.ConnectionClosed:
|
||||
dead.append(ws)
|
||||
for d in dead:
|
||||
clients.discard(d)
|
||||
print(
|
||||
f"[tick] motion={motion:.2f} presence={presence:5.2f} "
|
||||
f"rssi={rssi:+d} yield={int(m['yield']):3d}pps "
|
||||
f"amp={payload['amp']:.2f} clients={len(clients)}"
|
||||
)
|
||||
except (serial.SerialException, OSError) as e:
|
||||
print(f"[bridge] serial error ({e}); reopen in 1s")
|
||||
with suppress(Exception):
|
||||
ser.close()
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
async def ws_handler(ws):
|
||||
addr = ws.remote_address
|
||||
clients.add(ws)
|
||||
print(f"[ws] client connected: {addr} total={len(clients)}")
|
||||
try:
|
||||
if last_payload is not None:
|
||||
await ws.send(json.dumps(last_payload))
|
||||
await ws.wait_closed()
|
||||
finally:
|
||||
clients.discard(ws)
|
||||
print(f"[ws] client gone: {addr} total={len(clients)}")
|
||||
|
||||
|
||||
async def main():
|
||||
print(f"[bridge] websocket on ws://{WS_HOST}:{WS_PORT}")
|
||||
async with websockets.serve(ws_handler, WS_HOST, WS_PORT):
|
||||
await serial_reader_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(main())
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Tiny threaded HTTP server for the three.js demos that fetch local files.
|
||||
|
||||
Why a sibling helper script instead of `python -m http.server`?
|
||||
The stdlib SimpleHTTPServer is single-threaded; Chrome opens many parallel
|
||||
connections (HTML + 9 script tags + FBX), the first eats the worker, the
|
||||
rest time out with net::ERR_EMPTY_RESPONSE. ThreadingHTTPServer fixes it.
|
||||
|
||||
Usage:
|
||||
python examples/three.js/server/serve-demo.py
|
||||
open http://localhost:8765/examples/three.js/demos/05-skinned-realtime.html
|
||||
"""
|
||||
from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler
|
||||
import os, sys
|
||||
|
||||
PORT = int(os.environ.get("PORT", 8765))
|
||||
# Always serve from the repo root regardless of where the script is launched.
|
||||
# This file lives at examples/three.js/server/serve-demo.py — three levels deep.
|
||||
os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")))
|
||||
|
||||
class NoCacheHandler(SimpleHTTPRequestHandler):
|
||||
def end_headers(self):
|
||||
# Aggressive no-cache so browser ALWAYS fetches the latest .html
|
||||
# after we edit it. Otherwise stale code sticks around even on hard
|
||||
# refresh and you debug a phantom.
|
||||
self.send_header("Cache-Control", "no-store, no-cache, must-revalidate, max-age=0")
|
||||
self.send_header("Pragma", "no-cache")
|
||||
self.send_header("Expires", "0")
|
||||
super().end_headers()
|
||||
|
||||
DEMOS = [
|
||||
"01-helpers.html",
|
||||
"02-cinematic.html",
|
||||
"03-skinned.html",
|
||||
"04-skinned-fbx.html",
|
||||
"05-skinned-realtime.html",
|
||||
]
|
||||
|
||||
with ThreadingHTTPServer(("127.0.0.1", PORT), NoCacheHandler) as srv:
|
||||
print(f"serving {os.getcwd()} on http://127.0.0.1:{PORT}/")
|
||||
print("demos:")
|
||||
for d in DEMOS:
|
||||
print(f" http://127.0.0.1:{PORT}/examples/three.js/demos/{d}")
|
||||
try:
|
||||
srv.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(0)
|
||||
@@ -15,7 +15,7 @@ This firmware captures WiFi Channel State Information (CSI) from an ESP32-S3 and
|
||||
> | **CSI streaming** | Per-subcarrier I/Q capture over UDP | ~20 Hz, ADR-018 binary format |
|
||||
> | **Breathing detection** | Bandpass 0.1-0.5 Hz, zero-crossing BPM | 6-30 BPM |
|
||||
> | **Heart rate** | Bandpass 0.8-2.0 Hz, zero-crossing BPM | 40-120 BPM |
|
||||
> | **Presence sensing** | Phase variance + adaptive calibration | < 1 ms latency |
|
||||
> | **Presence indicator** (heuristic) | Phase variance + adaptive threshold (60 s ambient learning) | < 1 ms latency, false-positives under strong RF interference — see [Tier 2 caveats](#what-this-firmware-does-not-do-tier-2-caveats) |
|
||||
> | **Fall detection** | Phase acceleration threshold | Configurable sensitivity |
|
||||
> | **Programmable sensing** | WASM modules loaded over HTTP | Hot-swap, no reflash |
|
||||
|
||||
@@ -25,6 +25,23 @@ This firmware captures WiFi Channel State Information (CSI) from an ESP32-S3 and
|
||||
|
||||
For users who want to get running fast. Detailed explanations follow in later sections.
|
||||
|
||||
### 0. Pre-built binaries (v0.6.5 — skip the build step)
|
||||
|
||||
Pre-built binaries are in `firmware/esp32-csi-node/release_bins/` (version: see `release_bins/version.txt`).
|
||||
Flash them directly:
|
||||
|
||||
```bash
|
||||
python -m esptool --chip esp32s3 --port COM7 --baud 460800 \
|
||||
write_flash --flash_mode dio --flash_size 8MB \
|
||||
0x0 firmware/esp32-csi-node/release_bins/bootloader.bin \
|
||||
0x8000 firmware/esp32-csi-node/release_bins/partition-table.bin \
|
||||
0xf000 firmware/esp32-csi-node/release_bins/ota_data_initial.bin \
|
||||
0x20000 firmware/esp32-csi-node/release_bins/esp32-csi-node.bin
|
||||
```
|
||||
|
||||
For 4 MB boards use `release_bins/esp32-csi-node-4mb.bin` and `release_bins/partition-table-4mb.bin`
|
||||
with `--flash_size 4MB`.
|
||||
|
||||
### 1. Build (Docker -- the only reliable method)
|
||||
|
||||
```bash
|
||||
@@ -37,18 +54,22 @@ MSYS_NO_PATHCONV=1 docker run --rm \
|
||||
|
||||
### 2. Flash
|
||||
|
||||
Offsets must match `partitions_display.csv` (8 MB) or `partitions_4mb.csv` (4 MB):
|
||||
`bootloader=0x0`, `partition-table=0x8000`, `otadata=0xf000`, `app (ota_0)=0x20000`.
|
||||
|
||||
```bash
|
||||
python -m esptool --chip esp32s3 --port COM7 --baud 460800 \
|
||||
write_flash --flash_mode dio --flash_size 8MB \
|
||||
0x0 firmware/esp32-csi-node/build/bootloader/bootloader.bin \
|
||||
0x8000 firmware/esp32-csi-node/build/partition_table/partition-table.bin \
|
||||
0x10000 firmware/esp32-csi-node/build/esp32-csi-node.bin
|
||||
0x0 firmware/esp32-csi-node/build/bootloader/bootloader.bin \
|
||||
0x8000 firmware/esp32-csi-node/build/partition_table/partition-table.bin \
|
||||
0xf000 firmware/esp32-csi-node/build/ota_data_initial.bin \
|
||||
0x20000 firmware/esp32-csi-node/build/esp32-csi-node.bin
|
||||
```
|
||||
|
||||
### 3. Provision WiFi credentials (no reflash needed)
|
||||
|
||||
```bash
|
||||
python scripts/provision.py --port COM7 \
|
||||
python firmware/esp32-csi-node/provision.py --port COM7 \
|
||||
--ssid "YourSSID" --password "YourPass" --target-ip 192.168.1.20
|
||||
```
|
||||
|
||||
@@ -129,11 +150,32 @@ Adds real-time health and safety monitoring.
|
||||
|
||||
- **Breathing rate** -- biquad IIR bandpass 0.1-0.5 Hz, zero-crossing BPM (6-30 BPM)
|
||||
- **Heart rate** -- biquad IIR bandpass 0.8-2.0 Hz, zero-crossing BPM (40-120 BPM)
|
||||
- **Presence detection** -- adaptive threshold calibration (60 s ambient learning)
|
||||
- **Presence indicator** -- phase variance vs an adaptively-calibrated threshold (60 s ambient learning at boot). Heuristic, not a learned classifier — strong RF interferers (fans, microwaves, transmit-power swings) can push variance above threshold without anyone in the room. See "What this firmware does NOT do" below.
|
||||
- **Fall detection** -- phase acceleration exceeds configurable threshold
|
||||
- **Multi-person estimation** -- subcarrier group clustering (up to 4 persons)
|
||||
- **Multi-person slot count** -- partitions the top-K subcarriers into `top_k / 2` groups (clamped to `[1, EDGE_MAX_PERSONS]`), computes per-group filtered breathing/heart-rate estimates, and reports the slot count as `pkt.n_persons`. This is a **slot-capacity heuristic**, not a learned counter — the reported count tracks subcarrier diversity, not actual occupancy. See [`edge_processing.c:481-548`](main/edge_processing.c#L481-L548).
|
||||
- **Vitals packet** -- 32-byte UDP packet at 1 Hz (magic `0xC5110002`)
|
||||
|
||||
### What this firmware does NOT do (Tier 2 caveats)
|
||||
|
||||
- It does **not** run a trained neural model. The "person count" is an
|
||||
arithmetic slot-capacity heuristic over the top-K subcarrier groups
|
||||
(`firmware/esp32-csi-node/main/edge_processing.c:481`). It tracks
|
||||
subcarrier diversity, not actual occupancy.
|
||||
- It does **not** run pose estimation. Pose-related features in the host
|
||||
UI come from the Rust `wifi-densepose-sensing-server` running a separate
|
||||
pipeline. When no `.rvf` model file is loaded via `--model`, the server
|
||||
drives the on-screen skeleton from signal-based heuristics (amplitude
|
||||
variance, motion-band power), not from learned keypoint inference. The
|
||||
repository does not ship pre-trained weights — see issues
|
||||
[#509](../../issues/509) and [#506](../../issues/506) for context, and
|
||||
[ADR-079](../../docs/adr/ADR-079-camera-supervised-pose-finetune.md) for
|
||||
the planned training path (phases P7-P9 are `Pending`).
|
||||
- The presence indicator is a calibrated variance threshold and **will
|
||||
false-positive** under strong RF interference from non-human sources
|
||||
(fans near the antenna, microwave duty cycles, neighbouring AP power
|
||||
swings) without re-running the 60-second ambient calibration. If you
|
||||
see ghost detections, re-calibrate by power-cycling in an empty room.
|
||||
|
||||
### Tier 3 -- WASM Programmable Sensing (Alpha)
|
||||
|
||||
Turns the ESP32 from a fixed-function sensor into a programmable sensing computer. Instead of reflashing firmware to change algorithms, you upload new sensing logic as small WASM modules -- compiled from Rust, packaged in signed RVF containers.
|
||||
@@ -254,9 +296,10 @@ Find your serial port: `COM7` on Windows, `/dev/ttyUSB0` on Linux, `/dev/cu.SLAB
|
||||
```bash
|
||||
python -m esptool --chip esp32s3 --port COM7 --baud 460800 \
|
||||
write_flash --flash_mode dio --flash_size 8MB \
|
||||
0x0 firmware/esp32-csi-node/build/bootloader/bootloader.bin \
|
||||
0x8000 firmware/esp32-csi-node/build/partition_table/partition-table.bin \
|
||||
0x10000 firmware/esp32-csi-node/build/esp32-csi-node.bin
|
||||
0x0 firmware/esp32-csi-node/build/bootloader/bootloader.bin \
|
||||
0x8000 firmware/esp32-csi-node/build/partition_table/partition-table.bin \
|
||||
0xf000 firmware/esp32-csi-node/build/ota_data_initial.bin \
|
||||
0x20000 firmware/esp32-csi-node/build/esp32-csi-node.bin
|
||||
```
|
||||
|
||||
### Serial Monitor
|
||||
@@ -268,8 +311,9 @@ python -m serial.tools.miniterm COM7 115200
|
||||
Expected output after boot:
|
||||
|
||||
```
|
||||
I (321) main: ESP32-S3 CSI Node (ADR-018) -- Node ID: 1
|
||||
I (345) main: WiFi STA initialized, connecting to SSID: wifi-densepose
|
||||
I (396) csi_collector: Early capture node_id=1 (before WiFi init, #232/#390)
|
||||
I (406) main: ESP32-S3 CSI Node (ADR-018) -- v0.6.5 -- Node ID: 1
|
||||
I (566) main: WiFi STA initialized, connecting to SSID: wifi-densepose
|
||||
I (1023) main: Connected to WiFi
|
||||
I (1025) main: CSI streaming active -> 192.168.1.100:5005 (edge_tier=2, OTA=ready, WASM=ready)
|
||||
```
|
||||
@@ -285,7 +329,7 @@ All settings can be changed at runtime via Non-Volatile Storage (NVS) without re
|
||||
The easiest way to write NVS settings:
|
||||
|
||||
```bash
|
||||
python scripts/provision.py --port COM7 \
|
||||
python firmware/esp32-csi-node/provision.py --port COM7 \
|
||||
--ssid "MyWiFi" \
|
||||
--password "MyPassword" \
|
||||
--target-ip 192.168.1.20
|
||||
|
||||
@@ -11,7 +11,26 @@ set(SRCS
|
||||
"adaptive_controller.c"
|
||||
)
|
||||
|
||||
set(REQUIRES "")
|
||||
# ESP-IDF v6+: headers must resolve via explicit REQUIRES (no implicit deps).
|
||||
set(REQUIRES
|
||||
esp_wifi
|
||||
esp_netif
|
||||
esp_event
|
||||
nvs_flash
|
||||
app_update
|
||||
esp_http_server
|
||||
esp_http_client
|
||||
esp_app_format
|
||||
esp_timer
|
||||
esp_pm
|
||||
esp_driver_uart
|
||||
esp_driver_gpio
|
||||
esp_driver_spi
|
||||
esp_driver_i2c
|
||||
driver
|
||||
lwip
|
||||
mbedtls
|
||||
)
|
||||
|
||||
# ADR-061: Mock CSI generator for QEMU testing + ADR-081 mock radio binding
|
||||
if(CONFIG_CSI_MOCK_ENABLED)
|
||||
@@ -21,7 +40,11 @@ endif()
|
||||
# ADR-045: AMOLED display support (compile-time optional)
|
||||
if(CONFIG_DISPLAY_ENABLE)
|
||||
list(APPEND SRCS "display_hal.c" "display_ui.c" "display_task.c")
|
||||
set(REQUIRES esp_lcd esp_lcd_touch lvgl)
|
||||
list(APPEND REQUIRES esp_lcd esp_lcd_touch lvgl)
|
||||
endif()
|
||||
|
||||
if(CONFIG_WASM_ENABLE)
|
||||
list(APPEND REQUIRES wasm3)
|
||||
endif()
|
||||
|
||||
idf_component_register(
|
||||
|
||||
@@ -371,6 +371,30 @@ void csi_collector_init(void)
|
||||
|
||||
ESP_LOGI(TAG, "Promiscuous mode enabled (MGMT-only, RuView#396)");
|
||||
|
||||
#if CONFIG_SOC_WIFI_HE_SUPPORT
|
||||
/* Wi-Fi 6 targets (e.g. ESP32-C6): wifi_csi_config_t is wifi_csi_acquire_config_t
|
||||
* (bitfields), not the legacy 802.11n bool layout used on ESP32-S3. */
|
||||
wifi_csi_config_t csi_config;
|
||||
memset(&csi_config, 0, sizeof(csi_config));
|
||||
csi_config.enable = 1U;
|
||||
csi_config.acquire_csi_legacy = 1U;
|
||||
csi_config.acquire_csi_ht20 = 1U;
|
||||
csi_config.acquire_csi_ht40 = 1U;
|
||||
csi_config.acquire_csi_su = 1U;
|
||||
csi_config.acquire_csi_mu = 1U;
|
||||
csi_config.acquire_csi_dcm = 1U;
|
||||
csi_config.acquire_csi_beamformed = 1U;
|
||||
#if CONFIG_SOC_WIFI_MAC_VERSION_NUM >= 3
|
||||
csi_config.acquire_csi_force_lltf = 1U;
|
||||
csi_config.acquire_csi_vht = 1U;
|
||||
csi_config.acquire_csi_he_stbc_mode = ESP_CSI_ACQUIRE_STBC_SAMPLE_HELTFS;
|
||||
csi_config.val_scale_cfg = 0U;
|
||||
#else
|
||||
csi_config.acquire_csi_he_stbc = ESP_CSI_ACQUIRE_STBC_SAMPLE_HELTFS;
|
||||
csi_config.val_scale_cfg = 0U;
|
||||
#endif
|
||||
csi_config.dump_ack_en = 0U;
|
||||
#else
|
||||
wifi_csi_config_t csi_config = {
|
||||
.lltf_en = true,
|
||||
.htltf_en = true,
|
||||
@@ -380,6 +404,7 @@ void csi_collector_init(void)
|
||||
.manu_scale = false,
|
||||
.shift = false,
|
||||
};
|
||||
#endif
|
||||
|
||||
ESP_ERROR_CHECK(esp_wifi_set_csi_config(&csi_config));
|
||||
ESP_ERROR_CHECK(esp_wifi_set_csi_rx_cb(wifi_csi_callback, NULL));
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
* @file edge_processing.c
|
||||
* @brief ADR-039 Edge Intelligence — dual-core CSI processing pipeline.
|
||||
*
|
||||
* Core 0 (WiFi task): Pushes raw CSI frames into lock-free SPSC ring buffer.
|
||||
* Core 1 (DSP task): Pops frames, runs signal processing pipeline:
|
||||
* Core 0 (WiFi path): Pushes raw CSI frames into lock-free SPSC ring buffer.
|
||||
* Second core when present (DSP task): pops frames, runs signal processing pipeline.
|
||||
* On unicore targets (e.g. ESP32-C6), the DSP task is pinned to core 0.
|
||||
* 1. Phase extraction from I/Q pairs
|
||||
* 2. Phase unwrapping (continuous phase)
|
||||
* 3. Welford variance tracking per subcarrier
|
||||
@@ -848,6 +849,8 @@ static void process_frame(const edge_ring_slot_t *slot)
|
||||
|
||||
/* --- Step 11: Multi-person vitals --- */
|
||||
update_multi_person_vitals(slot->iq_data, n_subcarriers, sample_rate);
|
||||
/* Yield after multi-person DSP so IDLE1 can feed Core 1 watchdog (#683). */
|
||||
if (s_cfg.tier >= 2) vTaskDelay(1);
|
||||
|
||||
/* --- Step 12: Delta compression --- */
|
||||
if (s_cfg.tier >= 2) {
|
||||
@@ -893,6 +896,8 @@ static void process_frame(const edge_ring_slot_t *slot)
|
||||
wasm_runtime_on_frame(phases, amplitudes, variances,
|
||||
n_subcarriers,
|
||||
(const edge_vitals_pkt_t *)&s_latest_pkt);
|
||||
/* Yield after WASM dispatch to feed Core 1 watchdog (#683). */
|
||||
vTaskDelay(1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1050,7 +1055,9 @@ esp_err_t edge_processing_init(const edge_config_t *cfg)
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
/* Start DSP task on Core 1. */
|
||||
/* Pin DSP off WiFi's preferred core when SMP; else core 0 only (ESP32-C6). */
|
||||
const BaseType_t dsp_core = (portNUM_PROCESSORS > 1) ? (BaseType_t)1 : (BaseType_t)0;
|
||||
|
||||
BaseType_t ret = xTaskCreatePinnedToCore(
|
||||
edge_task,
|
||||
"edge_dsp",
|
||||
@@ -1058,14 +1065,14 @@ esp_err_t edge_processing_init(const edge_config_t *cfg)
|
||||
NULL,
|
||||
5, /* Priority 5 — above idle, below WiFi. */
|
||||
NULL,
|
||||
1 /* Pin to Core 1. */
|
||||
);
|
||||
dsp_core);
|
||||
|
||||
if (ret != pdPASS) {
|
||||
ESP_LOGE(TAG, "Failed to create edge DSP task");
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
|
||||
ESP_LOGI(TAG, "Edge DSP task created on Core 1 (stack=8192, priority=5)");
|
||||
ESP_LOGI(TAG, "Edge DSP task created on core %d (stack=8192, priority=5)",
|
||||
(int)dsp_core);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
@@ -8,3 +8,6 @@ dependencies:
|
||||
|
||||
## LCD touch abstraction
|
||||
espressif/esp_lcd_touch: "^1.0"
|
||||
|
||||
## Onboard WS2812 LED Disabling
|
||||
espressif/led_strip: "^3.0.0"
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
#include "nvs_flash.h"
|
||||
#include "esp_app_desc.h"
|
||||
#include "sdkconfig.h"
|
||||
#include "led_strip.h"
|
||||
|
||||
#include "csi_collector.h"
|
||||
#include "stream_sender.h"
|
||||
@@ -149,6 +150,23 @@ void app_main(void)
|
||||
ESP_LOGI(TAG, "ESP32-S3 CSI Node (ADR-018) — v%s — Node ID: %d",
|
||||
app_desc->version, g_nvs_config.node_id);
|
||||
|
||||
/* Turn off onboard WS2812 LED on GPIO 38 */
|
||||
led_strip_handle_t led_strip;
|
||||
led_strip_config_t strip_config = {
|
||||
.strip_gpio_num = 38,
|
||||
.max_leds = 1,
|
||||
.led_model = LED_MODEL_WS2812,
|
||||
.color_component_format = LED_STRIP_COLOR_COMPONENT_FMT_GRB,
|
||||
.flags.invert_out = false,
|
||||
};
|
||||
led_strip_rmt_config_t rmt_config = {
|
||||
.resolution_hz = 10 * 1000 * 1000, // 10MHz
|
||||
.flags.with_dma = false,
|
||||
};
|
||||
if (led_strip_new_rmt_device(&strip_config, &rmt_config, &led_strip) == ESP_OK) {
|
||||
led_strip_clear(led_strip);
|
||||
}
|
||||
|
||||
/* Initialize WiFi STA (skip entirely under QEMU mock — no RF hardware) */
|
||||
#ifndef CONFIG_CSI_MOCK_SKIP_WIFI_CONNECT
|
||||
wifi_init_sta();
|
||||
|
||||
@@ -109,7 +109,7 @@ static void mr60_process_frame(uint16_t type, const uint8_t *data, uint16_t len)
|
||||
|
||||
switch (type) {
|
||||
case MR60_TYPE_BREATHING:
|
||||
if (len >= 4) {
|
||||
if (len >= sizeof(float)) {
|
||||
/* Breathing rate as float32 (little-endian in payload). */
|
||||
float br;
|
||||
memcpy(&br, data, sizeof(float));
|
||||
@@ -120,7 +120,7 @@ static void mr60_process_frame(uint16_t type, const uint8_t *data, uint16_t len)
|
||||
break;
|
||||
|
||||
case MR60_TYPE_HEARTRATE:
|
||||
if (len >= 4) {
|
||||
if (len >= sizeof(float)) {
|
||||
float hr;
|
||||
memcpy(&hr, data, sizeof(float));
|
||||
if (hr >= 0.0f && hr <= 250.0f) {
|
||||
@@ -130,13 +130,13 @@ static void mr60_process_frame(uint16_t type, const uint8_t *data, uint16_t len)
|
||||
break;
|
||||
|
||||
case MR60_TYPE_DISTANCE:
|
||||
if (len >= 8) {
|
||||
if (len >= sizeof(uint32_t) + sizeof(float)) {
|
||||
/* Bytes 0-3: range flag (uint32 LE). 0 = no valid distance. */
|
||||
uint32_t range_flag;
|
||||
memcpy(&range_flag, data, sizeof(uint32_t));
|
||||
if (range_flag != 0 && len >= 8) {
|
||||
if (range_flag != 0) {
|
||||
float dist;
|
||||
memcpy(&dist, &data[4], sizeof(float));
|
||||
memcpy(&dist, &data[sizeof(uint32_t)], sizeof(float));
|
||||
s_state.distance_cm = dist;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,14 +38,24 @@ static char s_ota_psk[OTA_PSK_MAX_LEN] = {0};
|
||||
|
||||
/**
|
||||
* ADR-050: Verify the Authorization header contains the correct PSK.
|
||||
* Returns true if auth is disabled (no PSK provisioned) or if the
|
||||
* Bearer token matches the stored PSK.
|
||||
* Returns true only when a PSK is provisioned AND the Bearer token
|
||||
* matches it. An unprovisioned node refuses all OTA requests
|
||||
* (fail-closed, see RuView#596 audit). The OTA server still starts so
|
||||
* the operator can `provision.py --ota-psk <hex>` over USB-CDC without
|
||||
* a reflash, but the upload endpoint will reject every request until
|
||||
* the PSK is set.
|
||||
*/
|
||||
static bool ota_check_auth(httpd_req_t *req)
|
||||
{
|
||||
if (s_ota_psk[0] == '\0') {
|
||||
/* No PSK provisioned — auth disabled (permissive for dev). */
|
||||
return true;
|
||||
/* No PSK provisioned — fail closed. Previously this returned
|
||||
* true ("permissive for dev"), which let any host on the WiFi
|
||||
* push attacker-controlled firmware to a freshly-flashed node.
|
||||
* Plain HTTP transport + no Secure Boot V2 + no signed-image
|
||||
* verification meant a single LAN call could brick or back-
|
||||
* door a node. Reject until provisioned. */
|
||||
ESP_LOGW(TAG, "OTA rejected: no PSK in NVS (run provision.py --ota-psk <hex>)");
|
||||
return false;
|
||||
}
|
||||
|
||||
char auth_header[128] = {0};
|
||||
@@ -241,26 +251,45 @@ static esp_err_t ota_start_server(httpd_handle_t *out_handle)
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t ota_update_init(void)
|
||||
/**
|
||||
* Load the OTA PSK from NVS into the module-local s_ota_psk cache and log
|
||||
* the resulting posture. Called by both ota_update_init() and
|
||||
* ota_update_init_ex() so the per-boot diagnostic prints no matter which
|
||||
* entry point main.c uses — historically only ota_update_init() loaded the
|
||||
* PSK, which left ota_update_init_ex() with an empty s_ota_psk and an
|
||||
* invisible fail-closed posture (RuView#596 follow-up).
|
||||
*/
|
||||
static void ota_load_psk_from_nvs(void)
|
||||
{
|
||||
/* ADR-050: Load OTA PSK from NVS if provisioned. */
|
||||
nvs_handle_t nvs;
|
||||
if (nvs_open(OTA_NVS_NAMESPACE, NVS_READONLY, &nvs) == ESP_OK) {
|
||||
size_t len = sizeof(s_ota_psk);
|
||||
if (nvs_get_str(nvs, OTA_NVS_KEY, s_ota_psk, &len) == ESP_OK) {
|
||||
ESP_LOGI(TAG, "OTA PSK loaded from NVS (%d chars) — authentication enabled", (int)len - 1);
|
||||
} else {
|
||||
ESP_LOGW(TAG, "No OTA PSK in NVS — OTA authentication DISABLED (provision with nvs_set)");
|
||||
ESP_LOGW(TAG, "No OTA PSK in NVS — OTA upload endpoint will REJECT all requests until "
|
||||
"provisioned (provision.py --ota-psk <hex>). Fail-closed per RuView#596.");
|
||||
}
|
||||
nvs_close(nvs);
|
||||
} else {
|
||||
ESP_LOGW(TAG, "NVS namespace '%s' not found — OTA authentication DISABLED", OTA_NVS_NAMESPACE);
|
||||
ESP_LOGW(TAG, "NVS namespace '%s' not found — OTA upload endpoint will REJECT all "
|
||||
"requests until provisioned. Fail-closed per RuView#596.", OTA_NVS_NAMESPACE);
|
||||
}
|
||||
}
|
||||
|
||||
esp_err_t ota_update_init(void)
|
||||
{
|
||||
/* ADR-050: Load OTA PSK from NVS if provisioned. */
|
||||
ota_load_psk_from_nvs();
|
||||
return ota_start_server(NULL);
|
||||
}
|
||||
|
||||
esp_err_t ota_update_init_ex(void **out_server)
|
||||
{
|
||||
/* ADR-050: Load OTA PSK from NVS if provisioned. main.c uses this
|
||||
* variant (not ota_update_init), so without this call s_ota_psk
|
||||
* stayed empty forever and the fail-closed posture was invisible
|
||||
* in serial logs. */
|
||||
ota_load_psk_from_nvs();
|
||||
return ota_start_server((httpd_handle_t *)out_server);
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
|
||||
#include <string.h>
|
||||
#include "esp_log.h"
|
||||
#include "mbedtls/sha256.h"
|
||||
#include "psa/crypto.h"
|
||||
|
||||
static const char *TAG = "rvf";
|
||||
|
||||
@@ -125,9 +125,13 @@ esp_err_t rvf_parse(const uint8_t *data, uint32_t data_len, rvf_parsed_t *out)
|
||||
|
||||
/* ---- Verify build hash (SHA-256 of WASM payload) ---- */
|
||||
uint8_t computed_hash[32];
|
||||
int ret = mbedtls_sha256(wasm_data, hdr->wasm_len, computed_hash, 0);
|
||||
if (ret != 0) {
|
||||
ESP_LOGE(TAG, "SHA-256 computation failed: %d", ret);
|
||||
size_t hash_len = 0;
|
||||
psa_status_t psa_st = psa_hash_compute(PSA_ALG_SHA_256, wasm_data,
|
||||
hdr->wasm_len, computed_hash,
|
||||
sizeof(computed_hash), &hash_len);
|
||||
if (psa_st != PSA_SUCCESS || hash_len != 32) {
|
||||
ESP_LOGE(TAG, "SHA-256 computation failed: psa=%d len=%u",
|
||||
(int)psa_st, (unsigned)hash_len);
|
||||
return ESP_FAIL;
|
||||
}
|
||||
|
||||
@@ -186,8 +190,7 @@ esp_err_t rvf_verify_signature(const rvf_parsed_t *parsed, const uint8_t *data,
|
||||
/*
|
||||
* Ed25519 verification.
|
||||
*
|
||||
* ESP-IDF v5.2 mbedtls does NOT include Ed25519 (Curve25519 is
|
||||
* for ECDH/X25519 only). We use a SHA-256-HMAC integrity check:
|
||||
* Legacy mbedtls Ed25519 is optional. We use a SHA-256 keyed digest:
|
||||
*
|
||||
* expected = SHA-256(pubkey || signed_region)
|
||||
*
|
||||
@@ -196,35 +199,34 @@ esp_err_t rvf_verify_signature(const rvf_parsed_t *parsed, const uint8_t *data,
|
||||
* pubkey produces a different expected hash, so unauthorized
|
||||
* publishers cannot forge a valid signature.
|
||||
*
|
||||
* For full Ed25519 (NaCl-style), enable CONFIG_MBEDTLS_EDDSA_C
|
||||
* or link TweetNaCl. The RVF builder should match this scheme.
|
||||
* For full Ed25519, enable CONFIG_MBEDTLS_EDDSA_C or equivalent.
|
||||
* The RVF builder should match this scheme.
|
||||
*/
|
||||
uint8_t hash_input_prefix[32];
|
||||
memcpy(hash_input_prefix, pubkey, 32);
|
||||
|
||||
/* Compute SHA-256(pubkey || header+manifest+wasm). */
|
||||
mbedtls_sha256_context ctx;
|
||||
mbedtls_sha256_init(&ctx);
|
||||
int ret = mbedtls_sha256_starts(&ctx, 0);
|
||||
if (ret != 0) {
|
||||
mbedtls_sha256_free(&ctx);
|
||||
/* Compute SHA-256(pubkey || header+manifest+wasm) via PSA Crypto. */
|
||||
psa_hash_operation_t op = PSA_HASH_OPERATION_INIT;
|
||||
psa_status_t st = psa_hash_setup(&op, PSA_ALG_SHA_256);
|
||||
if (st != PSA_SUCCESS) {
|
||||
return ESP_FAIL;
|
||||
}
|
||||
ret = mbedtls_sha256_update(&ctx, hash_input_prefix, 32);
|
||||
if (ret != 0) {
|
||||
mbedtls_sha256_free(&ctx);
|
||||
st = psa_hash_update(&op, hash_input_prefix, 32);
|
||||
if (st != PSA_SUCCESS) {
|
||||
(void)psa_hash_abort(&op);
|
||||
return ESP_FAIL;
|
||||
}
|
||||
ret = mbedtls_sha256_update(&ctx, data, signed_len);
|
||||
if (ret != 0) {
|
||||
mbedtls_sha256_free(&ctx);
|
||||
st = psa_hash_update(&op, data, signed_len);
|
||||
if (st != PSA_SUCCESS) {
|
||||
(void)psa_hash_abort(&op);
|
||||
return ESP_FAIL;
|
||||
}
|
||||
|
||||
uint8_t expected[32];
|
||||
ret = mbedtls_sha256_finish(&ctx, expected);
|
||||
mbedtls_sha256_free(&ctx);
|
||||
if (ret != 0) {
|
||||
size_t out_len = 0;
|
||||
st = psa_hash_finish(&op, expected, sizeof(expected), &out_len);
|
||||
if (st != PSA_SUCCESS || out_len != 32) {
|
||||
(void)psa_hash_abort(&op);
|
||||
return ESP_FAIL;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,26 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
ESP32-S3 CSI Node Provisioning Script
|
||||
ESP32 CSI node provisioning (ESP32-S3, ESP32-C6, other targets).
|
||||
|
||||
Writes WiFi credentials and aggregator target to the ESP32's NVS partition
|
||||
so users can configure a pre-built firmware binary without recompiling.
|
||||
|
||||
Usage:
|
||||
python provision.py --port COM7 --ssid "MyWiFi" --password "secret" --target-ip 192.168.1.20
|
||||
python provision.py --port /dev/ttyUSB0 --chip esp32c6 --ssid "..." \\
|
||||
--password "..." --target-ip 192.168.1.20
|
||||
|
||||
Requirements:
|
||||
pip install 'esptool>=5.0' nvs-partition-gen
|
||||
(or use the nvs_partition_gen.py bundled with ESP-IDF)
|
||||
|
||||
WARNING -- FULL-REPLACE SEMANTICS (issue #391):
|
||||
Every invocation REPLACES the entire `csi_cfg` NVS namespace on the device.
|
||||
Any key you don't pass on the CLI is erased. Always include WiFi credentials
|
||||
(--ssid, --password, --target-ip) unless you pass --force-partial.
|
||||
ADDITIVE-BY-DEFAULT (issue #391, #574 phase 1):
|
||||
Earlier versions of this script REPLACED the entire `csi_cfg` NVS namespace
|
||||
on the device every invocation, wiping any key you didn't pass on the CLI.
|
||||
That cost customers hours of unnecessary friction.
|
||||
|
||||
The script now MERGES new CLI flags with the per-port state previously
|
||||
written from this machine (stored under your user config dir; see
|
||||
`--state-dir` to override or `--state` to inspect). On every invocation:
|
||||
|
||||
1. Read the prior per-port state file (or treat as empty if absent).
|
||||
2. Overlay the new CLI flags on top.
|
||||
3. Generate + flash NVS from the merged state.
|
||||
4. Write the merged state back to the state file.
|
||||
|
||||
Net effect: partial reconfigure works the way users expect. Pass `--reset`
|
||||
to wipe both the state file AND the device NVS for first-time provisioning
|
||||
of a recycled board.
|
||||
|
||||
Caveat: state lives on the controlling machine. Provisioning the same
|
||||
device from a second machine starts from an empty state — pass the keys
|
||||
you want to keep on that invocation, or pre-seed the state file. A future
|
||||
follow-up will add USB-CDC NVS dump for true device-authoritative merging
|
||||
(tracked in #574).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import subprocess
|
||||
@@ -35,6 +57,123 @@ NVS_PARTITION_OFFSET = 0x9000
|
||||
NVS_PARTITION_SIZE = 0x6000 # 24 KiB
|
||||
|
||||
|
||||
CONFIG_VALUE_CHECKS = [
|
||||
("ssid", bool),
|
||||
("password", lambda value: value is not None),
|
||||
("target_ip", bool),
|
||||
("target_port", lambda value: value is not None),
|
||||
("node_id", lambda value: value is not None),
|
||||
("tdm_slot", lambda value: value is not None),
|
||||
("tdm_total", lambda value: value is not None),
|
||||
("edge_tier", lambda value: value is not None),
|
||||
("pres_thresh", lambda value: value is not None),
|
||||
("fall_thresh", lambda value: value is not None),
|
||||
("vital_win", lambda value: value is not None),
|
||||
("vital_int", lambda value: value is not None),
|
||||
("subk_count", lambda value: value is not None),
|
||||
("channel", lambda value: value is not None),
|
||||
("filter_mac", lambda value: value is not None),
|
||||
("hop_channels", lambda value: value is not None),
|
||||
("seed_url", lambda value: value is not None),
|
||||
("seed_token", lambda value: value is not None),
|
||||
("zone", lambda value: value is not None),
|
||||
("swarm_hb", lambda value: value is not None),
|
||||
("swarm_ingest", lambda value: value is not None),
|
||||
]
|
||||
|
||||
|
||||
def has_config_value(args):
|
||||
"""Return True when args include at least one NVS-writing config value."""
|
||||
return any(
|
||||
check(getattr(args, name, None))
|
||||
for name, check in CONFIG_VALUE_CHECKS
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-port state file (additive-by-default merging, #391 / #574)
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# The state file is JSON keyed by `args` attribute name. It captures every
|
||||
# config value previously written to a given serial port from this machine.
|
||||
# On the next invocation, missing CLI flags fall back to the stored value.
|
||||
|
||||
# argparse attribute names that participate in the merge. Order doesn't
|
||||
# matter; this is just the surface area to round-trip.
|
||||
MERGEABLE_ATTRS = [
|
||||
"ssid", "password", "target_ip", "target_port", "node_id",
|
||||
"tdm_slot", "tdm_total",
|
||||
"edge_tier", "pres_thresh", "fall_thresh",
|
||||
"vital_win", "vital_int", "subk_count",
|
||||
"channel", "filter_mac",
|
||||
"hop_channels", "hop_dwell",
|
||||
"seed_url", "seed_token", "zone", "swarm_hb", "swarm_ingest",
|
||||
]
|
||||
|
||||
|
||||
def _default_state_dir() -> str:
|
||||
"""Per-user config dir for provision-state JSON files."""
|
||||
env = os.environ
|
||||
if sys.platform == "win32":
|
||||
base = env.get("APPDATA") or os.path.expanduser("~")
|
||||
else:
|
||||
base = env.get("XDG_CONFIG_HOME") or os.path.join(
|
||||
os.path.expanduser("~"), ".config"
|
||||
)
|
||||
return os.path.join(base, "wifi-densepose", "esp32-provision-state")
|
||||
|
||||
|
||||
def _state_path_for(port: str, state_dir: str) -> str:
|
||||
"""File path for a given serial port. Sanitize the port for filesystem use."""
|
||||
safe = port.replace("/", "_").replace(":", "_").replace("\\", "_")
|
||||
return os.path.join(state_dir, f"{safe}.json")
|
||||
|
||||
|
||||
def load_state(port: str, state_dir: str) -> dict:
|
||||
"""Return the merged-state dict for `port`, or `{}` if absent / unreadable."""
|
||||
path = _state_path_for(port, state_dir)
|
||||
if not os.path.isfile(path):
|
||||
return {}
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, dict):
|
||||
return data
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
print(f"WARNING: could not read state file {path}: {exc}", file=sys.stderr)
|
||||
return {}
|
||||
|
||||
|
||||
def save_state(port: str, state_dir: str, state: dict) -> str:
|
||||
"""Write `state` to the per-port file, creating dirs as needed. Returns path."""
|
||||
os.makedirs(state_dir, exist_ok=True)
|
||||
path = _state_path_for(port, state_dir)
|
||||
# Sort keys for deterministic on-disk content (easier to diff).
|
||||
tmp = path + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(state, f, indent=2, sort_keys=True)
|
||||
f.write("\n")
|
||||
os.replace(tmp, path)
|
||||
return path
|
||||
|
||||
|
||||
def merge_state_into_args(args, prior: dict) -> dict:
|
||||
"""Overlay `args` onto `prior` for every MERGEABLE_ATTRS attribute.
|
||||
|
||||
CLI values win whenever they were explicitly set (i.e. not `None`).
|
||||
Returns the merged dict (for state persistence) and mutates `args`
|
||||
in place so downstream `build_nvs_csv` sees the merged values.
|
||||
"""
|
||||
merged = dict(prior)
|
||||
for name in MERGEABLE_ATTRS:
|
||||
cli_val = getattr(args, name, None)
|
||||
if cli_val is not None:
|
||||
merged[name] = cli_val
|
||||
elif name in merged:
|
||||
setattr(args, name, merged[name])
|
||||
return merged
|
||||
|
||||
|
||||
def build_nvs_csv(args):
|
||||
"""Build an NVS CSV string for the csi_cfg namespace."""
|
||||
buf = io.StringIO()
|
||||
@@ -143,7 +282,7 @@ def generate_nvs_binary(csv_content, size):
|
||||
os.unlink(p)
|
||||
|
||||
|
||||
def flash_nvs(port, baud, nvs_bin):
|
||||
def flash_nvs(port, baud, nvs_bin, chip):
|
||||
"""Flash the NVS partition binary to the ESP32."""
|
||||
with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f:
|
||||
f.write(nvs_bin)
|
||||
@@ -152,16 +291,13 @@ def flash_nvs(port, baud, nvs_bin):
|
||||
try:
|
||||
cmd = [
|
||||
sys.executable, "-m", "esptool",
|
||||
"--chip", "esp32s3",
|
||||
"--chip", chip,
|
||||
"--port", port,
|
||||
"--baud", str(baud),
|
||||
# Keep underscore form — ESP-IDF v5.4 bundles esptool 4.10.0 which only
|
||||
# accepts "write_flash". pip's esptool >=5.x accepts both (hyphenated
|
||||
# form preferred) but keeps underscore working. Do not "correct" this.
|
||||
"write_flash",
|
||||
hex(NVS_PARTITION_OFFSET), bin_path,
|
||||
]
|
||||
print(f"Flashing NVS partition ({len(nvs_bin)} bytes) to {port}...")
|
||||
print(f"Flashing NVS partition ({len(nvs_bin)} bytes) to {port} (chip={chip})...")
|
||||
subprocess.check_call(cmd)
|
||||
print("NVS provisioning complete!")
|
||||
finally:
|
||||
@@ -170,10 +306,20 @@ def flash_nvs(port, baud, nvs_bin):
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Provision ESP32-S3 CSI Node with WiFi and aggregator settings",
|
||||
epilog="Example: python provision.py --port COM7 --ssid MyWiFi --password secret --target-ip 192.168.1.20",
|
||||
description="Provision CSI node NVS (WiFi + aggregator); works on S3, C6, etc.",
|
||||
epilog=(
|
||||
"Example: python provision.py --port COM7 --ssid MyWiFi --password secret "
|
||||
"--target-ip 192.168.1.20\n"
|
||||
"ESP32-C6: same, or pass --chip esp32c6 if auto-detect fails "
|
||||
"(default chip is auto for esptool v5+)."
|
||||
),
|
||||
)
|
||||
parser.add_argument("--port", required=True, help="Serial port (e.g. COM7, /dev/ttyUSB0)")
|
||||
parser.add_argument(
|
||||
"--chip",
|
||||
default="auto",
|
||||
help="esptool target: auto (default), esp32s3, esp32c6, ... (must match connected chip)",
|
||||
)
|
||||
parser.add_argument("--baud", type=int, default=460800, help="Flash baud rate (default: 460800)")
|
||||
parser.add_argument("--ssid", help="WiFi SSID")
|
||||
parser.add_argument("--password", help="WiFi password")
|
||||
@@ -208,29 +354,45 @@ def main():
|
||||
parser.add_argument("--swarm-ingest", type=int, help="Swarm vector ingest interval in seconds (default 5)")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Generate NVS binary but don't flash")
|
||||
parser.add_argument("--force-partial", action="store_true",
|
||||
help="Allow partial config without WiFi credentials. "
|
||||
"WARNING: flashing REPLACES the entire csi_cfg NVS namespace - "
|
||||
"any key not passed on the CLI will be erased (issue #391).")
|
||||
help="[deprecated since #391/#574] Suppress the missing-WiFi-trio "
|
||||
"error when no prior state file exists. The script now merges "
|
||||
"with prior state by default, so this flag is rarely needed.")
|
||||
parser.add_argument("--reset", action="store_true",
|
||||
help="Wipe this machine's per-port state file before merging. "
|
||||
"Use for first-time provisioning of a recycled board where "
|
||||
"previously-staged keys should NOT be re-applied.")
|
||||
parser.add_argument("--state-dir", default=_default_state_dir(),
|
||||
help="Override the per-user state directory (default: per-OS user config dir).")
|
||||
parser.add_argument("--state", action="store_true",
|
||||
help="Print the merged state that WOULD be flashed for this port and exit. "
|
||||
"Useful for debugging which keys are about to land on the device.")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
has_value = any([
|
||||
args.ssid, args.password is not None, args.target_ip,
|
||||
args.target_port, args.node_id is not None,
|
||||
args.tdm_slot is not None, args.tdm_total is not None,
|
||||
args.edge_tier is not None, args.pres_thresh is not None,
|
||||
args.fall_thresh is not None, args.vital_win is not None,
|
||||
args.vital_int is not None, args.subk_count is not None,
|
||||
args.channel is not None, args.filter_mac is not None,
|
||||
args.seed_url is not None, args.zone is not None,
|
||||
])
|
||||
if not has_value:
|
||||
parser.error("At least one config value must be specified")
|
||||
# --- Per-port state load + merge (additive-by-default, #391 / #574) ---
|
||||
if args.reset:
|
||||
path = _state_path_for(args.port, args.state_dir)
|
||||
if os.path.isfile(path):
|
||||
os.unlink(path)
|
||||
print(f"--reset: removed state file {path}", file=sys.stderr)
|
||||
prior = {}
|
||||
else:
|
||||
prior = load_state(args.port, args.state_dir)
|
||||
merged = merge_state_into_args(args, prior)
|
||||
|
||||
# Bug 2 (#391): Prevent silent wipe of WiFi credentials on partial invocations.
|
||||
# Flashing the generated NVS binary to offset 0x9000 REPLACES the entire
|
||||
# csi_cfg namespace — there is no merge with existing NVS. Require the full
|
||||
# WiFi trio unless the user explicitly opts in with --force-partial.
|
||||
if args.state:
|
||||
print(json.dumps(merged, indent=2, sort_keys=True))
|
||||
return
|
||||
|
||||
if not has_config_value(args):
|
||||
parser.error(
|
||||
"At least one config value must be specified (after merging prior state). "
|
||||
"If you intended to start fresh, pass --reset and the keys you want."
|
||||
)
|
||||
|
||||
# WiFi-trio sanity check. After the merge, the trio should be present
|
||||
# unless the user is intentionally provisioning a brand-new board with
|
||||
# partial state. Keep --force-partial as the escape hatch for that case.
|
||||
wifi_trio_missing = [
|
||||
name for name, val in [
|
||||
("--ssid", args.ssid),
|
||||
@@ -240,20 +402,19 @@ def main():
|
||||
]
|
||||
if wifi_trio_missing and not args.force_partial:
|
||||
parser.error(
|
||||
f"Missing required WiFi credentials: {', '.join(wifi_trio_missing)}.\n"
|
||||
f"Missing required WiFi credentials after merging prior state: "
|
||||
f"{', '.join(wifi_trio_missing)}.\n"
|
||||
f"\n"
|
||||
f" provision.py REPLACES the entire csi_cfg NVS namespace on each run.\n"
|
||||
f" Any key not passed on the CLI will be erased -- including WiFi creds.\n"
|
||||
f"\n"
|
||||
f" Either pass all of --ssid, --password, --target-ip,\n"
|
||||
f" or add --force-partial to acknowledge that other NVS keys will be wiped."
|
||||
f" No per-port state file at {_state_path_for(args.port, args.state_dir)}\n"
|
||||
f" and the CLI didn't include them. Either pass --ssid + --password + --target-ip\n"
|
||||
f" on this run, or add --force-partial to flash without WiFi.\n"
|
||||
)
|
||||
if args.force_partial and wifi_trio_missing:
|
||||
print("WARNING: --force-partial is set. The following NVS keys will be WIPED "
|
||||
"(not present in this invocation):", file=sys.stderr)
|
||||
for k in wifi_trio_missing:
|
||||
print(f" - {k.lstrip('-')}", file=sys.stderr)
|
||||
print(" Plus any other csi_cfg keys not passed on the CLI.\n", file=sys.stderr)
|
||||
print(
|
||||
"WARNING: --force-partial is set and WiFi credentials are missing. "
|
||||
"The device will not connect to WiFi after flashing.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
# Validate TDM: if one is given, both should be
|
||||
if (args.tdm_slot is not None) != (args.tdm_total is not None):
|
||||
@@ -281,7 +442,7 @@ def main():
|
||||
if args.ssid:
|
||||
print(f" WiFi SSID: {args.ssid}")
|
||||
if args.password is not None:
|
||||
print(f" WiFi Password: {'*' * len(args.password)}")
|
||||
print(f" WiFi Password: {'(set)' if args.password else '(empty)'}")
|
||||
if args.target_ip:
|
||||
print(f" Target IP: {args.target_ip}")
|
||||
if args.target_port:
|
||||
@@ -337,11 +498,20 @@ def main():
|
||||
with open(out, "wb") as f:
|
||||
f.write(nvs_bin)
|
||||
print(f"NVS binary saved to {out} ({len(nvs_bin)} bytes)")
|
||||
print(f"Flash manually: python -m esptool --chip esp32s3 --port {args.port} "
|
||||
f"write-flash 0x9000 {out}")
|
||||
print(f"Flash manually: python -m esptool --chip {args.chip} --port {args.port} "
|
||||
f"write_flash 0x9000 {out}")
|
||||
# Persist merged state even on dry-run so a subsequent real flash from
|
||||
# this machine sees the same staged config.
|
||||
path = save_state(args.port, args.state_dir, merged)
|
||||
print(f"State persisted to {path}")
|
||||
return
|
||||
|
||||
flash_nvs(args.port, args.baud, nvs_bin)
|
||||
flash_nvs(args.port, args.baud, nvs_bin, args.chip)
|
||||
# Persist merged state after a successful flash so future partial
|
||||
# invocations from this machine merge on top of what's actually on the
|
||||
# device. This is the heart of the additive-by-default fix (#391/#574).
|
||||
path = save_state(args.port, args.state_dir, merged)
|
||||
print(f"State persisted to {path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
0.6.6
|
||||
git-sha: cbcb389cb (pre-commit)
|
||||
built: 2026-05-21
|
||||
@@ -34,3 +34,11 @@ CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192
|
||||
|
||||
# Extra WiFi IRAM placement (defense-in-depth for RuView#396 SPI cache race)
|
||||
CONFIG_ESP_WIFI_EXTRA_IRAM_OPT=y
|
||||
|
||||
# ADR-081: adaptive_controller runs emit_feature_state + stream_sender
|
||||
# network I/O inside Timer Svc callbacks, exceeding the 2 KiB default.
|
||||
# Without this, the device bootloops with
|
||||
# "***ERROR*** A stack overflow in task Tmr Svc has been detected."
|
||||
# Was present in sdkconfig.defaults.template but missing here — fixed
|
||||
# in the v0.6.5-esp32 release.
|
||||
CONFIG_FREERTOS_TIMER_TASK_STACK_DEPTH=8192
|
||||
|
||||
@@ -153,6 +153,13 @@ typedef struct {
|
||||
uint8_t primary;
|
||||
} wifi_ap_record_t;
|
||||
|
||||
typedef enum {
|
||||
WIFI_PS_NONE = 0,
|
||||
WIFI_PS_MIN_MODEM = 1,
|
||||
WIFI_PS_MAX_MODEM = 2,
|
||||
} wifi_ps_type_t;
|
||||
|
||||
static inline esp_err_t esp_wifi_set_ps(wifi_ps_type_t type) { (void)type; return ESP_OK; }
|
||||
static inline esp_err_t esp_wifi_set_promiscuous(bool en) { (void)en; return ESP_OK; }
|
||||
static inline esp_err_t esp_wifi_set_promiscuous_rx_cb(void *cb) { (void)cb; return ESP_OK; }
|
||||
static inline esp_err_t esp_wifi_set_promiscuous_filter(wifi_promiscuous_filter_t *f) { (void)f; return ESP_OK; }
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import csv
|
||||
import importlib.util
|
||||
import io
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PROVISION_PATH = Path(__file__).resolve().parents[1] / "provision.py"
|
||||
SPEC = importlib.util.spec_from_file_location("provision", PROVISION_PATH)
|
||||
provision = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(provision)
|
||||
|
||||
|
||||
def make_args(**overrides):
|
||||
values = {name: None for name, _ in provision.CONFIG_VALUE_CHECKS}
|
||||
values["hop_dwell"] = 200
|
||||
values.update(overrides)
|
||||
return types.SimpleNamespace(**values)
|
||||
|
||||
|
||||
def csv_rows(content):
|
||||
return list(csv.DictReader(io.StringIO(content)))
|
||||
|
||||
|
||||
class ProvisionConfigValueTests(unittest.TestCase):
|
||||
def test_swarm_and_hopping_flags_count_as_config_values(self):
|
||||
cases = [
|
||||
{"hop_channels": "1,6,11"},
|
||||
{"seed_token": "token-123"},
|
||||
{"swarm_hb": 15},
|
||||
{"swarm_ingest": 3},
|
||||
]
|
||||
|
||||
for values in cases:
|
||||
with self.subTest(values=values):
|
||||
self.assertTrue(provision.has_config_value(make_args(**values)))
|
||||
|
||||
def test_operational_flags_alone_do_not_count_as_config_values(self):
|
||||
self.assertFalse(provision.has_config_value(make_args()))
|
||||
|
||||
def test_swarm_and_hopping_values_are_written_to_csv(self):
|
||||
args = make_args(
|
||||
hop_channels="1,6,11",
|
||||
hop_dwell=250,
|
||||
seed_token="token-123",
|
||||
swarm_hb=15,
|
||||
swarm_ingest=3,
|
||||
)
|
||||
|
||||
rows = csv_rows(provision.build_nvs_csv(args))
|
||||
values_by_key = {row["key"]: row["value"] for row in rows}
|
||||
|
||||
self.assertEqual(values_by_key["hop_count"], "3")
|
||||
self.assertEqual(values_by_key["chan_list"], "01060b")
|
||||
self.assertEqual(values_by_key["dwell_ms"], "250")
|
||||
self.assertEqual(values_by_key["seed_token"], "token-123")
|
||||
self.assertEqual(values_by_key["swarm_hb"], "15")
|
||||
self.assertEqual(values_by_key["swarm_ingest"], "3")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Tests for provision.py's additive-by-default merge behaviour (#391, #574)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
# Allow `python -m unittest` from anywhere in the repo.
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.dirname(HERE))
|
||||
|
||||
import provision # noqa: E402 — sibling import after sys.path tweak
|
||||
|
||||
|
||||
def _mk_args(**overrides) -> argparse.Namespace:
|
||||
"""Build a Namespace with every mergeable attr set to None unless overridden."""
|
||||
base = {name: None for name in provision.MERGEABLE_ATTRS}
|
||||
base.update(overrides)
|
||||
return argparse.Namespace(**base)
|
||||
|
||||
|
||||
class TestStateFile(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.dir = tempfile.mkdtemp(prefix="provision-state-")
|
||||
|
||||
def tearDown(self):
|
||||
import shutil
|
||||
shutil.rmtree(self.dir, ignore_errors=True)
|
||||
|
||||
def test_load_state_empty_when_missing(self):
|
||||
self.assertEqual(provision.load_state("COM7", self.dir), {})
|
||||
|
||||
def test_save_then_load_roundtrip(self):
|
||||
provision.save_state("COM7", self.dir, {"ssid": "x", "password": "y"})
|
||||
self.assertEqual(
|
||||
provision.load_state("COM7", self.dir),
|
||||
{"ssid": "x", "password": "y"},
|
||||
)
|
||||
|
||||
def test_save_creates_per_port_files(self):
|
||||
provision.save_state("COM7", self.dir, {"ssid": "a"})
|
||||
provision.save_state("/dev/ttyUSB0", self.dir, {"ssid": "b"})
|
||||
self.assertEqual(provision.load_state("COM7", self.dir), {"ssid": "a"})
|
||||
self.assertEqual(provision.load_state("/dev/ttyUSB0", self.dir), {"ssid": "b"})
|
||||
|
||||
def test_load_state_handles_corrupt_json(self):
|
||||
path = provision._state_path_for("COM7", self.dir)
|
||||
os.makedirs(self.dir, exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write("{not valid json")
|
||||
# Should warn but not raise.
|
||||
self.assertEqual(provision.load_state("COM7", self.dir), {})
|
||||
|
||||
|
||||
class TestMerge(unittest.TestCase):
|
||||
def test_cli_wins_over_prior(self):
|
||||
args = _mk_args(ssid="new-ssid")
|
||||
prior = {"ssid": "old-ssid", "password": "abc"}
|
||||
merged = provision.merge_state_into_args(args, prior)
|
||||
self.assertEqual(args.ssid, "new-ssid") # CLI value preserved
|
||||
self.assertEqual(args.password, "abc") # filled from prior
|
||||
self.assertEqual(merged["ssid"], "new-ssid")
|
||||
self.assertEqual(merged["password"], "abc")
|
||||
|
||||
def test_prior_fills_missing_cli(self):
|
||||
args = _mk_args() # all None
|
||||
prior = {
|
||||
"ssid": "MyWiFi",
|
||||
"password": "secret",
|
||||
"target_ip": "192.168.1.20",
|
||||
"node_id": 3,
|
||||
}
|
||||
merged = provision.merge_state_into_args(args, prior)
|
||||
self.assertEqual(args.ssid, "MyWiFi")
|
||||
self.assertEqual(args.password, "secret")
|
||||
self.assertEqual(args.target_ip, "192.168.1.20")
|
||||
self.assertEqual(args.node_id, 3)
|
||||
for key, val in prior.items():
|
||||
self.assertEqual(merged[key], val)
|
||||
|
||||
def test_partial_invocation_does_not_drop_unrelated_keys(self):
|
||||
# The exact #391 scenario: user previously provisioned WiFi, now adds
|
||||
# only --seed-url. Old behaviour wiped SSID. New behaviour keeps it.
|
||||
args = _mk_args(seed_url="http://10.1.10.236")
|
||||
prior = {
|
||||
"ssid": "ruv.net",
|
||||
"password": "<secret>",
|
||||
"target_ip": "192.168.1.20",
|
||||
}
|
||||
merged = provision.merge_state_into_args(args, prior)
|
||||
self.assertEqual(args.ssid, "ruv.net")
|
||||
self.assertEqual(args.password, "<secret>")
|
||||
self.assertEqual(args.target_ip, "192.168.1.20")
|
||||
self.assertEqual(args.seed_url, "http://10.1.10.236")
|
||||
# And the on-disk merged dict carries all four keys.
|
||||
self.assertEqual(set(merged.keys()),
|
||||
{"ssid", "password", "target_ip", "seed_url"})
|
||||
|
||||
def test_empty_prior_is_noop(self):
|
||||
args = _mk_args(ssid="x")
|
||||
merged = provision.merge_state_into_args(args, {})
|
||||
self.assertEqual(merged, {"ssid": "x"})
|
||||
|
||||
def test_falsy_but_not_none_cli_value_overrides_prior(self):
|
||||
# node_id=0 is a legal value; must NOT be replaced by prior["node_id"]=5.
|
||||
args = _mk_args(node_id=0)
|
||||
prior = {"node_id": 5}
|
||||
merged = provision.merge_state_into_args(args, prior)
|
||||
self.assertEqual(args.node_id, 0)
|
||||
self.assertEqual(merged["node_id"], 0)
|
||||
|
||||
|
||||
class TestStatePathSanitization(unittest.TestCase):
|
||||
def test_slashes_in_port_are_safe(self):
|
||||
path = provision._state_path_for("/dev/ttyUSB0", "/tmp/x")
|
||||
# Must not contain a raw slash in the basename
|
||||
self.assertNotIn("/", os.path.basename(path))
|
||||
|
||||
def test_windows_com_port_is_safe(self):
|
||||
path = provision._state_path_for("COM7", "/tmp/x")
|
||||
self.assertTrue(path.endswith("COM7.json"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1 +1 @@
|
||||
0.6.4
|
||||
0.6.6
|
||||
@@ -1,4 +1,4 @@
|
||||
# ESP32-S3 Hello World — Capability Discovery
|
||||
# ESP32 Hello World — Capability Discovery (S3 / C6 targets)
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
|
||||
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
/**
|
||||
* @file main.c
|
||||
* @brief ESP32-S3 Hello World — Full Capability Discovery
|
||||
* @brief ESP32 Hello World — Full Capability Discovery
|
||||
*
|
||||
* Boots up, prints "Hello World!", then probes and reports every major
|
||||
* hardware/software capability of the ESP32-S3: chip info, flash, PSRAM,
|
||||
* WiFi (including CSI), Bluetooth, GPIOs, peripherals, FreeRTOS stats,
|
||||
* and power management features. No WiFi connection required.
|
||||
* Boots up, prints "Hello World!", then probes chip info, flash, PSRAM,
|
||||
* WiFi (including CSI where enabled), 802.15.4/BLE on C6, GPIOs,
|
||||
* peripherals, FreeRTOS stats, and power management. No WiFi connection
|
||||
* required. Supports ESP32-S3 and ESP32-C6 (set IDF target accordingly).
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
@@ -18,7 +18,6 @@
|
||||
#include "esp_chip_info.h"
|
||||
#include "esp_flash.h"
|
||||
#include "esp_mac.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_wifi.h"
|
||||
#include "esp_event.h"
|
||||
#include "esp_timer.h"
|
||||
@@ -33,7 +32,24 @@
|
||||
#include "driver/temperature_sensor.h"
|
||||
#include "sdkconfig.h"
|
||||
|
||||
static const char *TAG = "hello";
|
||||
/*
|
||||
* Peripheral counts: ESP-IDF v6+ dropped some SOC_* macros; values below
|
||||
* match each target's HAL (esp_hal_* *_ll.h) where applicable.
|
||||
*/
|
||||
#if CONFIG_IDF_TARGET_ESP32S3
|
||||
#define PROBE_I2S_CTRL_NUM 2
|
||||
#define PROBE_RMT_CHAN_NUM 8
|
||||
#define PROBE_MCPWM_GROUPS 2
|
||||
#define PROBE_PCNT_UNITS 4
|
||||
#define PROBE_TOUCH_CHAN_NUM ((int)(SOC_TOUCH_MAX_CHAN_ID - SOC_TOUCH_MIN_CHAN_ID + 1))
|
||||
#elif CONFIG_IDF_TARGET_ESP32C6
|
||||
#define PROBE_I2S_CTRL_NUM 1
|
||||
#define PROBE_RMT_CHAN_NUM 4
|
||||
#define PROBE_MCPWM_GROUPS 1
|
||||
#define PROBE_PCNT_UNITS 4
|
||||
#else
|
||||
#error "hello-world: add PROBE_* peripheral counts for this IDF target in main.c"
|
||||
#endif
|
||||
|
||||
/* ── Helpers ─────────────────────────────────────────────────────────── */
|
||||
|
||||
@@ -46,6 +62,7 @@ static const char *chip_model_str(esp_chip_model_t model)
|
||||
case CHIP_ESP32C3: return "ESP32-C3";
|
||||
case CHIP_ESP32H2: return "ESP32-H2";
|
||||
case CHIP_ESP32C2: return "ESP32-C2";
|
||||
case CHIP_ESP32C6: return "ESP32-C6";
|
||||
default: return "Unknown";
|
||||
}
|
||||
}
|
||||
@@ -168,7 +185,11 @@ static void probe_wifi_capabilities(void)
|
||||
ESP_ERROR_CHECK(esp_wifi_start());
|
||||
|
||||
/* Protocol capabilities */
|
||||
#if CONFIG_IDF_TARGET_ESP32C6
|
||||
printf(" Protocols: 802.11 b/g/n/ax (Wi-Fi 6, 2.4 GHz)\n");
|
||||
#else
|
||||
printf(" Protocols: 802.11 b/g/n\n");
|
||||
#endif
|
||||
|
||||
/* CSI (Channel State Information) */
|
||||
#ifdef CONFIG_ESP_WIFI_CSI_ENABLED
|
||||
@@ -246,7 +267,7 @@ static void probe_bluetooth(void)
|
||||
esp_chip_info(&info);
|
||||
|
||||
if (info.features & CHIP_FEATURE_BLE) {
|
||||
printf(" BLE: Supported (Bluetooth 5.0 LE)\n");
|
||||
printf(" BLE: Supported (Bluetooth LE)\n");
|
||||
printf(" - GATT Server/Client\n");
|
||||
printf(" - Advertising & Scanning\n");
|
||||
printf(" - Mesh Networking\n");
|
||||
@@ -256,10 +277,16 @@ static void probe_bluetooth(void)
|
||||
printf(" BLE: Not supported on this chip\n");
|
||||
}
|
||||
|
||||
#if CONFIG_IDF_TARGET_ESP32C6
|
||||
if (info.features & CHIP_FEATURE_IEEE802154) {
|
||||
printf(" 802.15.4: Supported (Thread / Zigbee style MAC)\n");
|
||||
}
|
||||
#endif
|
||||
|
||||
if (info.features & CHIP_FEATURE_BT) {
|
||||
printf(" BT Classic: Supported (A2DP, SPP, HFP)\n");
|
||||
} else {
|
||||
printf(" BT Classic: Not available (ESP32-S3 is BLE-only)\n");
|
||||
printf(" BT Classic: Not available (BLE-only on this chip)\n");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -269,24 +296,52 @@ static void probe_peripherals(void)
|
||||
|
||||
printf(" GPIOs: %d total\n", SOC_GPIO_PIN_COUNT);
|
||||
printf(" ADC:\n");
|
||||
#if CONFIG_IDF_TARGET_ESP32C6
|
||||
printf(" - SAR ADC: %d channels (12-bit, one controller)\n",
|
||||
(int)SOC_ADC_CHANNEL_NUM(0));
|
||||
#else
|
||||
printf(" - ADC1: %d channels (12-bit SAR)\n", SOC_ADC_CHANNEL_NUM(0));
|
||||
printf(" - ADC2: %d channels (shared with WiFi)\n", SOC_ADC_CHANNEL_NUM(1));
|
||||
printf(" DAC: Not available on ESP32-S3\n");
|
||||
printf(" Touch Sensors: %d channels (capacitive)\n", SOC_TOUCH_SENSOR_NUM);
|
||||
printf(" SPI: %d controllers (SPI2/SPI3 for user)\n", SOC_SPI_PERIPH_NUM);
|
||||
printf(" I2C: %d controllers\n", SOC_I2C_NUM);
|
||||
printf(" I2S: %d controllers (audio/PDM/TDM)\n", SOC_I2S_NUM);
|
||||
printf(" UART: %d controllers\n", SOC_UART_NUM);
|
||||
#endif
|
||||
printf(" DAC: Not available on this chip\n");
|
||||
#if CONFIG_IDF_TARGET_ESP32S3
|
||||
printf(" Touch Sensors: %d channels (capacitive)\n", PROBE_TOUCH_CHAN_NUM);
|
||||
#elif CONFIG_IDF_TARGET_ESP32C6
|
||||
printf(" Touch Sensors: Not available (no capacitive touch on ESP32-C6)\n");
|
||||
#endif
|
||||
printf(" SPI: %d controllers\n", SOC_SPI_PERIPH_NUM);
|
||||
#if CONFIG_IDF_TARGET_ESP32S3
|
||||
printf(" (SPI2/SPI3 typical for user apps)\n");
|
||||
#endif
|
||||
printf(" I2C: %d controllers\n", (int)SOC_I2C_NUM);
|
||||
printf(" I2S: %d controller(s) (audio/PDM/TDM)\n", PROBE_I2S_CTRL_NUM);
|
||||
printf(" UART: %d controllers\n", (int)SOC_UART_NUM);
|
||||
#if CONFIG_IDF_TARGET_ESP32S3
|
||||
printf(" USB: USB-OTG 1.1 (Host & Device)\n");
|
||||
printf(" USB-Serial: Built-in USB-JTAG/Serial (this console)\n");
|
||||
#elif CONFIG_IDF_TARGET_ESP32C6
|
||||
printf(" USB: No native USB-OTG (use SPI/USB bridge or off-chip PHY)\n");
|
||||
printf(" USB-Serial: Built-in USB Serial/JTAG (this console)\n");
|
||||
#endif
|
||||
#if CONFIG_IDF_TARGET_ESP32S3
|
||||
printf(" TWAI (CAN): 1 controller (CAN 2.0B compatible)\n");
|
||||
printf(" RMT: %d channels (IR/WS2812/NeoPixel)\n", SOC_RMT_TX_CANDIDATES_PER_GROUP + SOC_RMT_RX_CANDIDATES_PER_GROUP);
|
||||
#elif CONFIG_IDF_TARGET_ESP32C6
|
||||
printf(" TWAI (CAN): %d controller(s) (CAN 2.0B compatible)\n",
|
||||
(int)SOC_TWAI_CONTROLLER_NUM);
|
||||
#endif
|
||||
printf(" RMT: %d channels (IR/WS2812/NeoPixel)\n", PROBE_RMT_CHAN_NUM);
|
||||
printf(" LEDC (PWM): %d channels\n", SOC_LEDC_CHANNEL_NUM);
|
||||
printf(" MCPWM: %d groups (motor control)\n", SOC_MCPWM_GROUPS);
|
||||
printf(" PCNT: %d units (pulse counter / encoder)\n", SOC_PCNT_UNITS_PER_GROUP);
|
||||
printf(" MCPWM: %d group(s) (motor control)\n", PROBE_MCPWM_GROUPS);
|
||||
printf(" PCNT: %d units (pulse counter / encoder)\n", PROBE_PCNT_UNITS);
|
||||
#if CONFIG_IDF_TARGET_ESP32S3
|
||||
printf(" LCD: Parallel 8/16-bit + SPI + I2C interfaces\n");
|
||||
printf(" Camera: DVP 8/16-bit parallel interface\n");
|
||||
printf(" SDMMC: SD/MMC host controller (1-bit / 4-bit)\n");
|
||||
#elif CONFIG_IDF_TARGET_ESP32C6
|
||||
printf(" PARLIO: Parallel TX/RX (e.g. LED matrix / custom buses)\n");
|
||||
printf(" Camera: SPI / external bridge (no native DVP)\n");
|
||||
printf(" SDIO: SDIO slave peripheral (see TRM for capabilities)\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
static void probe_security(void)
|
||||
@@ -309,17 +364,29 @@ static void probe_power(void)
|
||||
{
|
||||
print_separator("POWER MANAGEMENT");
|
||||
|
||||
#if CONFIG_IDF_TARGET_ESP32C6
|
||||
printf(" Clock Modes:\n");
|
||||
printf(" - 160 MHz (max CPU on ESP32-C6)\n");
|
||||
printf(" - 120 MHz (balanced)\n");
|
||||
printf(" - 80 MHz (low power)\n");
|
||||
#else
|
||||
printf(" Clock Modes:\n");
|
||||
printf(" - 240 MHz (max performance)\n");
|
||||
printf(" - 160 MHz (balanced)\n");
|
||||
printf(" - 80 MHz (low power)\n");
|
||||
#endif
|
||||
printf(" Sleep Modes:\n");
|
||||
printf(" - Modem Sleep (WiFi off, CPU active)\n");
|
||||
printf(" - Light Sleep (CPU paused, fast wake)\n");
|
||||
printf(" - Deep Sleep (RTC only, ~10 uA)\n");
|
||||
printf(" - Hibernation (RTC timer only, ~5 uA)\n");
|
||||
#if CONFIG_IDF_TARGET_ESP32C6
|
||||
printf(" Wake Sources: GPIO, LP timer, UART, etc.\n");
|
||||
printf(" LP domain: LP core / LP peripherals (see TRM)\n");
|
||||
#else
|
||||
printf(" Wake Sources: GPIO, timer, touch, ULP, UART\n");
|
||||
printf(" ULP Coprocessor: RISC-V + FSM (runs in deep sleep)\n");
|
||||
printf(" ULP Coprocessor: FSM (runs in deep sleep)\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
static void probe_temperature(void)
|
||||
@@ -389,6 +456,9 @@ static void probe_csi_details(void)
|
||||
|
||||
void app_main(void)
|
||||
{
|
||||
esp_chip_info_t chip;
|
||||
esp_chip_info(&chip);
|
||||
|
||||
/* NVS required for WiFi */
|
||||
esp_err_t ret = nvs_flash_init();
|
||||
if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
|
||||
@@ -401,7 +471,7 @@ void app_main(void)
|
||||
printf("\n");
|
||||
printf(" ╭─────────────────────────────────────────────────╮\n");
|
||||
printf(" │ │\n");
|
||||
printf(" │ HELLO WORLD from ESP32-S3! │\n");
|
||||
printf(" │ HELLO WORLD from %-24s │\n", chip_model_str(chip.model));
|
||||
printf(" │ │\n");
|
||||
printf(" │ WiFi-DensePose Capability Discovery v1.0 │\n");
|
||||
printf(" │ │\n");
|
||||
@@ -422,8 +492,9 @@ void app_main(void)
|
||||
probe_csi_details();
|
||||
|
||||
print_separator("DONE — ALL CAPABILITIES REPORTED");
|
||||
printf("\n This ESP32-S3 is ready for WiFi-DensePose!\n");
|
||||
printf(" Flash the full firmware (esp32-csi-node) to begin CSI sensing.\n\n");
|
||||
printf("\n This %s is ready for WiFi-DensePose experiments.\n",
|
||||
chip_model_str(chip.model));
|
||||
printf(" For production CSI on S3, flash esp32-csi-node; C6 path may differ.\n\n");
|
||||
|
||||
/* Keep alive — blink a status message every 10 seconds */
|
||||
int tick = 0;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# ESP32-S3 Hello World — SDK Configuration
|
||||
CONFIG_IDF_TARGET="esp32s3"
|
||||
# ESP32 Hello World — SDK Configuration (default: ESP32-C6)
|
||||
CONFIG_IDF_TARGET="esp32c6"
|
||||
|
||||
# Flash: 4MB (this chip has Embedded Flash 4MB)
|
||||
CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
pytest>=7.0.0
|
||||
pytest-asyncio>=0.21.0
|
||||
pytest-mock>=3.10.0
|
||||
pytest-benchmark>=4.0.0
|
||||
pytest-benchmark>=5.2.3
|
||||
|
||||
# Linting and formatting
|
||||
black>=23.0.0
|
||||
|
||||
@@ -7,7 +7,7 @@ torchvision>=0.13.0
|
||||
# API dependencies
|
||||
fastapi>=0.95.0
|
||||
uvicorn>=0.20.0
|
||||
websockets>=10.4
|
||||
websockets>=15.0.1
|
||||
pydantic>=1.10.0
|
||||
python-jose[cryptography]>=3.3.0
|
||||
python-multipart>=0.0.6
|
||||
@@ -18,7 +18,7 @@ pydantic-settings>=2.0.0
|
||||
# Database dependencies
|
||||
sqlalchemy>=2.0.0
|
||||
asyncpg>=0.28.0
|
||||
aiosqlite>=0.19.0
|
||||
aiosqlite>=0.22.1
|
||||
redis>=4.5.0
|
||||
|
||||
# CLI dependencies
|
||||
@@ -26,8 +26,8 @@ click>=8.0.0
|
||||
alembic>=1.10.0
|
||||
|
||||
# Hardware interface dependencies
|
||||
asyncio-mqtt>=0.11.0
|
||||
aiohttp>=3.8.0
|
||||
asyncio-mqtt>=0.16.2
|
||||
aiohttp>=3.13.5
|
||||
paramiko>=3.0.0
|
||||
|
||||
# Data processing dependencies
|
||||
|
||||
@@ -136,18 +136,42 @@ function extractAmplitude(iqBytes, nSubcarriers) {
|
||||
|
||||
/**
|
||||
* Load and parse a JSONL file, skipping blank/malformed lines.
|
||||
*
|
||||
* Reads byte-by-byte into Buffer slices to avoid Node's
|
||||
* `String.MaxLength` (~512 MB) cap that `readFileSync(_, 'utf8')` hits
|
||||
* on 30-min CSI recordings. Each line is decoded individually, so
|
||||
* memory use stays bounded by the largest single record.
|
||||
*/
|
||||
function loadJsonl(filePath) {
|
||||
const lines = fs.readFileSync(filePath, 'utf8').split('\n');
|
||||
const records = [];
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
try {
|
||||
records.push(JSON.parse(trimmed));
|
||||
} catch {
|
||||
// skip malformed lines
|
||||
const fd = fs.openSync(filePath, 'r');
|
||||
try {
|
||||
const bufSize = 1 << 20; // 1 MiB
|
||||
const buf = Buffer.alloc(bufSize);
|
||||
let leftover = '';
|
||||
let bytesRead;
|
||||
do {
|
||||
bytesRead = fs.readSync(fd, buf, 0, bufSize, null);
|
||||
if (bytesRead > 0) {
|
||||
const chunk = leftover + buf.toString('utf8', 0, bytesRead);
|
||||
const lines = chunk.split('\n');
|
||||
leftover = lines.pop(); // last fragment may be incomplete
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
try {
|
||||
records.push(JSON.parse(trimmed));
|
||||
} catch {
|
||||
// skip malformed lines
|
||||
}
|
||||
}
|
||||
}
|
||||
} while (bytesRead === bufSize);
|
||||
if (leftover.trim()) {
|
||||
try { records.push(JSON.parse(leftover.trim())); } catch {}
|
||||
}
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
return records;
|
||||
}
|
||||
@@ -184,8 +208,12 @@ function loadCsi(filePath) {
|
||||
const features = [];
|
||||
|
||||
for (const r of raw) {
|
||||
if (!r.timestamp) continue;
|
||||
const tsMs = isoToMs(r.timestamp);
|
||||
if (r.timestamp == null) continue;
|
||||
// Two timestamp formats: ISO string (legacy raw_csi/feature) or
|
||||
// numeric float-seconds (current sensing_update from the Rust server).
|
||||
const tsMs = typeof r.timestamp === 'number'
|
||||
? r.timestamp * 1000
|
||||
: isoToMs(r.timestamp);
|
||||
if (isNaN(tsMs)) continue;
|
||||
|
||||
if (r.type === 'raw_csi') {
|
||||
@@ -205,6 +233,33 @@ function loadCsi(filePath) {
|
||||
rssi: r.rssi,
|
||||
seq: r.seq,
|
||||
});
|
||||
} else if (r.type === 'sensing_update') {
|
||||
// Current sensing-server schema: one record per tick contains
|
||||
// already-extracted amplitudes per node plus a server-computed
|
||||
// feature vector. Project each into rawCsi/features so downstream
|
||||
// windowing/matrix extraction can reuse its existing paths.
|
||||
if (Array.isArray(r.nodes)) {
|
||||
for (const node of r.nodes) {
|
||||
if (!Array.isArray(node.amplitude) || node.amplitude.length === 0) continue;
|
||||
rawCsi.push({
|
||||
tsMs,
|
||||
nodeId: node.node_id,
|
||||
subcarriers: node.amplitude.length,
|
||||
amplitude: node.amplitude, // pre-extracted, no iq_hex needed
|
||||
rssi: node.rssi_dbm,
|
||||
seq: r.tick,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (Array.isArray(r.features) && r.features.length > 0) {
|
||||
features.push({
|
||||
tsMs,
|
||||
nodeId: 0,
|
||||
features: r.features,
|
||||
rssi: null,
|
||||
seq: r.tick,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -297,7 +352,11 @@ function extractCsiMatrix(window) {
|
||||
|
||||
for (let f = 0; f < nFrames; f++) {
|
||||
const frame = window[f];
|
||||
if (frame.iqHex) {
|
||||
if (frame.amplitude && frame.amplitude.length > 0) {
|
||||
// Already-extracted amplitudes from sensing_update — copy directly.
|
||||
const n = Math.min(nSc, frame.amplitude.length);
|
||||
for (let s = 0; s < n; s++) matrix[f * nSc + s] = frame.amplitude[s];
|
||||
} else if (frame.iqHex) {
|
||||
const iq = parseIqHex(frame.iqHex);
|
||||
const amp = extractAmplitude(iq, nSc);
|
||||
matrix.set(amp, f * nSc);
|
||||
@@ -422,12 +481,33 @@ function align() {
|
||||
? extractCsiMatrix(window)
|
||||
: extractFeatureMatrix(window);
|
||||
|
||||
// ADR-103: aggregate `n_persons` per window so the cog-person-count
|
||||
// training pipeline has count labels. Two summaries:
|
||||
// - `n_persons_mode` — modal value across the camera frames in
|
||||
// the window. Robust to single-frame noise;
|
||||
// this is the supervised label for the
|
||||
// categorical {0..7} count head.
|
||||
// - `n_persons_max` — the maximum value seen in the window.
|
||||
// Useful as a soft upper bound (e.g. for
|
||||
// dynamic dropout weighting during training).
|
||||
const personCounts = matched.map(f => f.nPersons ?? 0);
|
||||
const counts = new Map();
|
||||
for (const v of personCounts) counts.set(v, (counts.get(v) ?? 0) + 1);
|
||||
let modeVal = 0;
|
||||
let modeCount = -1;
|
||||
for (const [v, n] of counts) {
|
||||
if (n > modeCount) { modeVal = v; modeCount = n; }
|
||||
}
|
||||
const maxVal = personCounts.reduce((a, b) => Math.max(a, b), 0);
|
||||
|
||||
paired.push({
|
||||
csi: csiMatrix.data,
|
||||
csi_shape: csiMatrix.shape,
|
||||
kp: keypoints,
|
||||
conf: Math.round(avgConfidence * 1000) / 1000,
|
||||
n_camera_frames: matched.length,
|
||||
n_persons_mode: modeVal,
|
||||
n_persons_max: maxVal,
|
||||
ts_start: new Date(tStartMs).toISOString(),
|
||||
ts_end: new Date(tEndMs).toISOString(),
|
||||
});
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export pose_v1.safetensors -> pose_v1.onnx.
|
||||
|
||||
Builds the same architecture as v2/crates/cog-pose-estimation/src/inference.rs
|
||||
in PyTorch, loads the trained weights from safetensors, and runs a torch.onnx
|
||||
export with a fixed [1, 56, 20] input. Then verifies the ONNX loads and
|
||||
matches the torch output to within 1e-5.
|
||||
"""
|
||||
|
||||
import json
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
N_SUB = 56
|
||||
N_FRAMES = 20
|
||||
N_KP = 17
|
||||
|
||||
|
||||
class PoseNet(nn.Module):
|
||||
"""Mirrors inference.rs::PoseNet exactly."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.c1 = nn.Conv1d(N_SUB, 64, kernel_size=3, padding=1, dilation=1)
|
||||
self.c2 = nn.Conv1d(64, 128, kernel_size=3, padding=2, dilation=2)
|
||||
self.c3 = nn.Conv1d(128, 128, kernel_size=3, padding=4, dilation=4)
|
||||
self.fc1 = nn.Linear(128, 256)
|
||||
self.fc2 = nn.Linear(256, N_KP * 2)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
# x: [B, 56, 20]
|
||||
h = torch.relu(self.c1(x))
|
||||
h = torch.relu(self.c2(h))
|
||||
h = torch.relu(self.c3(h))
|
||||
h = h.mean(dim=2) # [B, 128]
|
||||
h = torch.relu(self.fc1(h))
|
||||
h = torch.sigmoid(self.fc2(h))
|
||||
return h
|
||||
|
||||
|
||||
def load_safetensors(path: Path) -> dict[str, torch.Tensor]:
|
||||
"""Pure-python safetensors reader. Avoids the safetensors pip dep."""
|
||||
with path.open("rb") as f:
|
||||
header_len = struct.unpack("<Q", f.read(8))[0]
|
||||
header = json.loads(f.read(header_len).decode("utf-8"))
|
||||
out: dict[str, torch.Tensor] = {}
|
||||
for name, meta in header.items():
|
||||
if name == "__metadata__":
|
||||
continue
|
||||
start, end = meta["data_offsets"]
|
||||
shape = meta["shape"]
|
||||
dtype = meta["dtype"]
|
||||
assert dtype == "F32", f"unsupported dtype {dtype} for {name}"
|
||||
f.seek(8 + header_len + start)
|
||||
buf = f.read(end - start)
|
||||
arr = np.frombuffer(buf, dtype=np.float32).copy().reshape(shape)
|
||||
out[name] = torch.from_numpy(arr)
|
||||
return out
|
||||
|
||||
|
||||
def main() -> None:
|
||||
weights_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("pose_v1.safetensors")
|
||||
out_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("pose_v1.onnx")
|
||||
|
||||
if not weights_path.exists():
|
||||
raise SystemExit(f"weights file not found: {weights_path}")
|
||||
|
||||
print(f"reading {weights_path}")
|
||||
tensors = load_safetensors(weights_path)
|
||||
print(f" found {len(tensors)} tensors: {sorted(tensors.keys())}")
|
||||
|
||||
model = PoseNet()
|
||||
# Map safetensors names (enc.c1.weight, head.fc1.weight, ...) to module params
|
||||
mapping = {
|
||||
"enc.c1.weight": "c1.weight",
|
||||
"enc.c1.bias": "c1.bias",
|
||||
"enc.c2.weight": "c2.weight",
|
||||
"enc.c2.bias": "c2.bias",
|
||||
"enc.c3.weight": "c3.weight",
|
||||
"enc.c3.bias": "c3.bias",
|
||||
"head.fc1.weight": "fc1.weight",
|
||||
"head.fc1.bias": "fc1.bias",
|
||||
"head.fc2.weight": "fc2.weight",
|
||||
"head.fc2.bias": "fc2.bias",
|
||||
}
|
||||
state = {dst: tensors[src] for src, dst in mapping.items()}
|
||||
model.load_state_dict(state)
|
||||
model.eval()
|
||||
print(" weights loaded into PyTorch model")
|
||||
|
||||
# Sanity check forward
|
||||
x = torch.zeros(1, N_SUB, N_FRAMES)
|
||||
with torch.no_grad():
|
||||
y = model(x)
|
||||
print(f" zero-input forward: shape={tuple(y.shape)} sample={y[0, :4].tolist()}")
|
||||
|
||||
# Export to ONNX
|
||||
torch.onnx.export(
|
||||
model,
|
||||
x,
|
||||
out_path,
|
||||
export_params=True,
|
||||
opset_version=18,
|
||||
do_constant_folding=True,
|
||||
input_names=["csi_window"],
|
||||
output_names=["keypoints"],
|
||||
dynamic_axes={"csi_window": {0: "batch"}, "keypoints": {0: "batch"}},
|
||||
)
|
||||
print(f" wrote {out_path} ({out_path.stat().st_size} bytes)")
|
||||
|
||||
# Verify the ONNX file loads + matches torch output
|
||||
try:
|
||||
import onnx
|
||||
import onnxruntime as ort
|
||||
|
||||
onnx_model = onnx.load(str(out_path))
|
||||
onnx.checker.check_model(onnx_model)
|
||||
print(" ONNX model checker: ok")
|
||||
|
||||
sess = ort.InferenceSession(str(out_path), providers=["CPUExecutionProvider"])
|
||||
rng = np.random.default_rng(42)
|
||||
x_np = rng.standard_normal((1, N_SUB, N_FRAMES), dtype=np.float32)
|
||||
with torch.no_grad():
|
||||
y_torch = model(torch.from_numpy(x_np)).numpy()
|
||||
y_onnx = sess.run(["keypoints"], {"csi_window": x_np})[0]
|
||||
max_abs = float(np.max(np.abs(y_torch - y_onnx)))
|
||||
print(f" parity vs torch: max |torch - onnx| = {max_abs:.2e}")
|
||||
assert max_abs < 1e-5, "ONNX output diverges from torch output"
|
||||
print(" parity ok (<1e-5)")
|
||||
except ImportError as e:
|
||||
print(f" WARN: onnx/onnxruntime not installed, skipping verification: {e}")
|
||||
|
||||
print("\nDone.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||