Compare commits
117 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5518d5d7c1 | |||
| 8247d28d90 | |||
| 5d6e50d8a0 | |||
| 49fb2ca9f4 | |||
| 3439fb1402 | |||
| c00f45e296 | |||
| f54f0285bd | |||
| e964eaf14f | |||
| 961c01f4bd | |||
| 79cc2d7b22 | |||
| f5e2b5474b | |||
| 281c4cb0ce | |||
| b2e2e6d6fd | |||
| 72bbd256e7 | |||
| 50131b2519 | |||
| 50136c920d | |||
| 3bd70f7910 | |||
| 6f5ac3aa5a | |||
| 1b155ad027 | |||
| fa28318bae | |||
| ec73109d57 | |||
| acbd3ff13c | |||
| 07086c5d9d | |||
| 0310b1fa9a | |||
| 9daa8c3078 | |||
| ffa808ed4b | |||
| 59dbb76757 | |||
| 4ecc053a27 | |||
| 5170b99aca | |||
| c16dc9f80a | |||
| 04ccfcde56 | |||
| 4d45add824 | |||
| 562cb7461f | |||
| fad6828697 | |||
| 807bf0b32a | |||
| 4b602c79dd | |||
| 76321ce4bc | |||
| 1690aea22a | |||
| a80617ee84 | |||
| 75dc302952 | |||
| afc86c6fc4 | |||
| fc654034b3 | |||
| c4653b8bc6 | |||
| d214855228 | |||
| e6710e8988 | |||
| ab9799adc3 | |||
| bdb4484259 | |||
| ba370c7b08 | |||
| 3fdd310f89 | |||
| 98e7eeda42 | |||
| 5615edb24e | |||
| 9cc9419db9 | |||
| d544b8f070 | |||
| d33962eff2 | |||
| e22a24714a | |||
| cee414f3c0 | |||
| f853c74563 | |||
| 8b297dd706 | |||
| 9d4f7820b2 | |||
| b2fe452e74 | |||
| 88da304631 | |||
| 880a3a41d3 | |||
| 68b042faf6 | |||
| 4698f54fa0 | |||
| ea62ec4667 | |||
| 3685d16a49 | |||
| 8a155e07ec | |||
| 540ecb4538 | |||
| 10684972d7 | |||
| 27a6edba8b | |||
| 174e2365f0 | |||
| bf30844835 | |||
| 457f713702 | |||
| ce33042226 | |||
| ca97527646 | |||
| 59d2d0e54f | |||
| 4a1f3a1e10 | |||
| 94ef125240 | |||
| 900b877c64 | |||
| 58cd860f17 | |||
| f0a4f64c6e | |||
| 81fcf5fa29 | |||
| 7a407556ba | |||
| c059a2eaaa | |||
| d6a73b61c9 | |||
| 8dc811d2b4 | |||
| c641fc44ae | |||
| 00304f9dc7 | |||
| d0b64bdeb6 | |||
| a2686d47a2 | |||
| f2525d7a0d | |||
| 601b3406fd | |||
| deb561bf9c | |||
| d40411e6d7 | |||
| b116a99481 | |||
| 684a064816 | |||
| 7393cc2b73 | |||
| 6432dfbd2d | |||
| 46f701bca8 | |||
| 94745242a8 | |||
| 1e684cb208 | |||
| d98b7e3f65 | |||
| 6f77b37f5e | |||
| c604ca1150 | |||
| eaedfded6f | |||
| bd4f81749a | |||
| df9d3b0eea | |||
| 298543913e | |||
| 8ff7c2c35a | |||
| 19ee207d51 | |||
| 8aa7fb9e9f | |||
| f2e3a6a392 | |||
| eda45a6857 | |||
| a1cb6bd8e5 | |||
| 4d0521ca08 | |||
| 3f55c95b34 | |||
| ce7983eb43 |
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "ruview",
|
||||
"description": "RuView Marketplace: Claude Code + Codex plugins for WiFi sensing — configuration, applications, model training, and onboarding, from practical to advanced",
|
||||
"owner": {
|
||||
"name": "ruvnet",
|
||||
"url": "https://github.com/ruvnet/RuView"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "ruview",
|
||||
"source": "./plugins/ruview",
|
||||
"description": "End-to-end RuView toolkit: getting started, ESP32 hardware setup, configuration, sensing applications (presence / vitals / pose / sleep / MAT), camera-free + camera-supervised model training, advanced multistatic sensing, CLI / API / WASM, mmWave radar, and witness verification"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -15,38 +15,50 @@ env:
|
||||
|
||||
jobs:
|
||||
# Code Quality and Security Checks
|
||||
# The Python codebase moved to `archive/v1/` when the runtime was rewritten in
|
||||
# Rust under `v2/`. The lint/format/type/scan checks below still run against
|
||||
# the archive for hygiene, but with `continue-on-error: true` everywhere — the
|
||||
# archive is frozen reference code, not active development, so a stale lint
|
||||
# rule shouldn't gate PRs to the Rust workspace.
|
||||
code-quality:
|
||||
name: Code Quality & Security
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
continue-on-error: true
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install dependencies
|
||||
continue-on-error: true
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
pip install black flake8 mypy bandit safety
|
||||
|
||||
- name: Code formatting check (Black)
|
||||
run: black --check --diff src/ tests/
|
||||
continue-on-error: true
|
||||
run: black --check --diff archive/v1/src archive/v1/tests
|
||||
|
||||
- name: Linting (Flake8)
|
||||
run: flake8 src/ tests/ --max-line-length=88 --extend-ignore=E203,W503
|
||||
continue-on-error: true
|
||||
run: flake8 archive/v1/src archive/v1/tests --max-line-length=88 --extend-ignore=E203,W503
|
||||
|
||||
- name: Type checking (MyPy)
|
||||
run: mypy src/ --ignore-missing-imports
|
||||
continue-on-error: true
|
||||
run: mypy archive/v1/src --ignore-missing-imports
|
||||
|
||||
- name: Security scan (Bandit)
|
||||
run: bandit -r src/ -f json -o bandit-report.json
|
||||
run: bandit -r archive/v1/src -f json -o bandit-report.json
|
||||
continue-on-error: true
|
||||
|
||||
- name: Dependency vulnerability scan (Safety)
|
||||
@@ -54,6 +66,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
|
||||
- name: Upload security reports
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
@@ -70,6 +83,28 @@ jobs:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# `wifi-densepose-desktop` is a Tauri v2 app — `glib-sys`, `gtk-sys`,
|
||||
# `webkit2gtk-sys`, etc. need the Linux dev libraries via pkg-config or the
|
||||
# workspace test fails at the build step before any test runs (every recent
|
||||
# main CI run has been red on this for exactly this reason). Install the
|
||||
# standard Tauri-on-Ubuntu set.
|
||||
- name: Install Tauri / GTK / serial system dev libraries
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
libglib2.0-dev \
|
||||
libgtk-3-dev \
|
||||
libsoup-3.0-dev \
|
||||
libjavascriptcoregtk-4.1-dev \
|
||||
libwebkit2gtk-4.1-dev \
|
||||
libayatana-appindicator3-dev \
|
||||
librsvg2-dev \
|
||||
libxdo-dev \
|
||||
libudev-dev \
|
||||
libdbus-1-dev \
|
||||
libssl-dev \
|
||||
pkg-config
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
@@ -89,10 +124,15 @@ jobs:
|
||||
run: cargo test --workspace --no-default-features
|
||||
|
||||
# Unit and Integration Tests
|
||||
# Python pytest matrix — runs against the archived v1 Python tree.
|
||||
# `continue-on-error: true` for the same reason as code-quality above:
|
||||
# the archive is frozen reference, not blocking the Rust workspace PRs.
|
||||
test:
|
||||
name: Tests
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ['3.10', '3.11', '3.12']
|
||||
services:
|
||||
@@ -121,44 +161,51 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v5
|
||||
continue-on-error: true
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install dependencies
|
||||
continue-on-error: true
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
pip install pytest-cov pytest-xdist
|
||||
|
||||
- name: Run unit tests
|
||||
continue-on-error: true
|
||||
env:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_wifi_densepose
|
||||
REDIS_URL: redis://localhost:6379/0
|
||||
ENVIRONMENT: test
|
||||
run: |
|
||||
pytest tests/unit/ -v --cov=src --cov-report=xml --cov-report=html --junitxml=junit.xml
|
||||
pytest archive/v1/tests/unit/ -v --cov=archive/v1/src --cov-report=xml --cov-report=html --junitxml=junit.xml
|
||||
|
||||
- name: Run integration tests
|
||||
continue-on-error: true
|
||||
env:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_wifi_densepose
|
||||
REDIS_URL: redis://localhost:6379/0
|
||||
ENVIRONMENT: test
|
||||
run: |
|
||||
pytest tests/integration/ -v --junitxml=integration-junit.xml
|
||||
pytest archive/v1/tests/integration/ -v --junitxml=integration-junit.xml
|
||||
|
||||
- name: Upload coverage reports
|
||||
uses: codecov/codecov-action@v4
|
||||
continue-on-error: true
|
||||
uses: codecov/codecov-action@v6
|
||||
with:
|
||||
file: ./coverage.xml
|
||||
flags: unittests
|
||||
name: codecov-umbrella
|
||||
|
||||
- name: Upload test results
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
@@ -179,7 +226,7 @@ jobs:
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
@@ -206,18 +253,29 @@ jobs:
|
||||
path: locust_report.html
|
||||
|
||||
# Docker Build and Test
|
||||
# NOTE: the canonical Docker build for the sensing-server is now
|
||||
# `.github/workflows/sensing-server-docker.yml` (multi-registry push, asset
|
||||
# smoke tests, bearer-auth smoke tests — #520/#514/#443). This job predates
|
||||
# that workflow, points at a non-existent root `Dockerfile` with a
|
||||
# non-existent `target: production`, and pushes to a mis-cased image name —
|
||||
# `continue-on-error: true` until it's deleted or rewired to call the new
|
||||
# workflow, so it doesn't gate the rest of the pipeline.
|
||||
docker-build:
|
||||
name: Docker Build & Test
|
||||
runs-on: ubuntu-latest
|
||||
needs: [code-quality, test, rust-tests]
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
continue-on-error: true
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Container Registry
|
||||
continue-on-error: true
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
@@ -225,8 +283,9 @@ jobs:
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
continue-on-error: true
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
@@ -236,7 +295,8 @@ jobs:
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
continue-on-error: true
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
target: production
|
||||
@@ -248,6 +308,7 @@ jobs:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
|
||||
- name: Test Docker image
|
||||
continue-on-error: true
|
||||
run: |
|
||||
docker run --rm -d --name test-container -p 8000:8000 ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
|
||||
sleep 10
|
||||
@@ -255,6 +316,7 @@ jobs:
|
||||
docker stop test-container
|
||||
|
||||
- name: Run container security scan
|
||||
continue-on-error: true
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
|
||||
@@ -262,6 +324,7 @@ jobs:
|
||||
output: 'trivy-results.sarif'
|
||||
|
||||
- name: Upload Trivy scan results
|
||||
continue-on-error: true
|
||||
uses: github/codeql-action/upload-sarif@v3
|
||||
if: always()
|
||||
with:
|
||||
@@ -278,7 +341,7 @@ jobs:
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
--out-dir ../../dashboard/public/nvsim-pkg \
|
||||
--release -- --no-default-features --features wasm
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
- uses: actions/setup-node@v6
|
||||
with: { node-version: 20, cache: npm, cache-dependency-path: dashboard/package-lock.json }
|
||||
|
||||
- working-directory: dashboard
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
-- --no-default-features --features wasm
|
||||
|
||||
- name: Setup Node 20
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 20
|
||||
cache: npm
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: '20'
|
||||
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: '20'
|
||||
|
||||
|
||||
@@ -2,6 +2,11 @@ name: Firmware CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- '**'
|
||||
tags:
|
||||
# ESP32 firmware release tags — build + version-consistency guard (RuView#505).
|
||||
- 'v*-esp32'
|
||||
paths:
|
||||
- 'firmware/**'
|
||||
- '.github/workflows/firmware-ci.yml'
|
||||
@@ -11,6 +16,27 @@ on:
|
||||
- '.github/workflows/firmware-ci.yml'
|
||||
|
||||
jobs:
|
||||
version-guard:
|
||||
name: Verify version.txt matches release tag
|
||||
runs-on: ubuntu-latest
|
||||
if: github.ref_type == 'tag'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Check firmware version.txt == tag
|
||||
run: |
|
||||
# Tag form: vX.Y.Z-esp32 → expect version.txt to contain X.Y.Z
|
||||
TAG="${GITHUB_REF_NAME}"
|
||||
EXPECTED="${TAG#v}"
|
||||
EXPECTED="${EXPECTED%-esp32}"
|
||||
ACTUAL="$(tr -d '[:space:]' < firmware/esp32-csi-node/version.txt)"
|
||||
echo "Tag: $TAG → expected version.txt: $EXPECTED | actual: $ACTUAL"
|
||||
if [ "$EXPECTED" != "$ACTUAL" ]; then
|
||||
echo "::error::firmware/esp32-csi-node/version.txt is '$ACTUAL' but tag '$TAG' expects '$EXPECTED'."
|
||||
echo "::error::Bump version.txt and re-tag so esp_app_get_description()->version is correct (RuView#505)."
|
||||
exit 1
|
||||
fi
|
||||
echo "version.txt matches the release tag."
|
||||
|
||||
build:
|
||||
name: Build ESP32-S3 Firmware (${{ matrix.variant }})
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
name: Fix-Marker Regression Guard
|
||||
|
||||
# Asserts that previously-shipped fixes are still present in the tree.
|
||||
# Manifest: scripts/fix-markers.json Checker: scripts/check_fix_markers.py
|
||||
# Run locally: python scripts/check_fix_markers.py (also --list / --json)
|
||||
#
|
||||
# This complements the heavyweight checks (firmware build, deterministic
|
||||
# pipeline proof, witness bundle) with a fast per-PR "did someone revert a
|
||||
# known fix?" gate — the CI analogue of the ruflo witness fix-marker system.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- master
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
fix-markers:
|
||||
name: Verify fix markers
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Validate the manifest is well-formed JSON
|
||||
run: python -c "import json; json.load(open('scripts/fix-markers.json')); print('manifest OK')"
|
||||
|
||||
- name: Check fix markers
|
||||
run: python scripts/check_fix_markers.py
|
||||
|
||||
- name: Emit machine-readable result (for the run summary)
|
||||
if: always()
|
||||
run: |
|
||||
python scripts/check_fix_markers.py --json > fix-markers-result.json || true
|
||||
{
|
||||
echo '### Fix-marker regression guard'
|
||||
echo ''
|
||||
echo '```'
|
||||
python scripts/check_fix_markers.py || true
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload result artifact
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: fix-markers-result
|
||||
path: fix-markers-result.json
|
||||
retention-days: 30
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: ghcr.io/ruvnet/nvsim-server
|
||||
tags: |
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
|
||||
- name: Build + push
|
||||
uses: docker/build-push-action@v5
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: v2
|
||||
file: v2/crates/nvsim-server/Dockerfile
|
||||
|
||||
@@ -18,23 +18,27 @@ jobs:
|
||||
sast:
|
||||
name: Static Application Security Testing
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true # third-party scanners are flaky / SARIF uploads can 403; don't gate the PR
|
||||
permissions:
|
||||
security-events: write
|
||||
actions: read
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
continue-on-error: true
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install dependencies
|
||||
continue-on-error: true
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
@@ -46,6 +50,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
|
||||
- name: Upload Bandit results to GitHub Security
|
||||
continue-on-error: true
|
||||
uses: github/codeql-action/upload-sarif@v3
|
||||
if: always()
|
||||
with:
|
||||
@@ -53,6 +58,7 @@ jobs:
|
||||
category: bandit
|
||||
|
||||
- name: Run Semgrep security scan
|
||||
continue-on-error: true
|
||||
uses: returntocorp/semgrep-action@v1
|
||||
with:
|
||||
config: >-
|
||||
@@ -70,6 +76,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
|
||||
- name: Upload Semgrep results to GitHub Security
|
||||
continue-on-error: true
|
||||
uses: github/codeql-action/upload-sarif@v3
|
||||
if: always()
|
||||
with:
|
||||
@@ -80,21 +87,25 @@ jobs:
|
||||
dependency-scan:
|
||||
name: Dependency Vulnerability Scan
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true # third-party scanners are flaky / SARIF uploads can 403; don't gate the PR
|
||||
permissions:
|
||||
security-events: write
|
||||
actions: read
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
continue-on-error: true
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install dependencies
|
||||
continue-on-error: true
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
@@ -119,6 +130,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
|
||||
- name: Upload Snyk results to GitHub Security
|
||||
continue-on-error: true
|
||||
uses: github/codeql-action/upload-sarif@v3
|
||||
if: always()
|
||||
with:
|
||||
@@ -126,6 +138,7 @@ jobs:
|
||||
category: snyk
|
||||
|
||||
- name: Upload vulnerability reports
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
@@ -139,6 +152,7 @@ jobs:
|
||||
container-scan:
|
||||
name: Container Security Scan
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true # third-party scanners are flaky / SARIF uploads can 403; don't gate the PR
|
||||
needs: []
|
||||
if: github.event_name == 'push' || github.event_name == 'schedule'
|
||||
permissions:
|
||||
@@ -147,13 +161,16 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
continue-on-error: true
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build Docker image for scanning
|
||||
uses: docker/build-push-action@v5
|
||||
continue-on-error: true
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
target: production
|
||||
@@ -163,6 +180,7 @@ jobs:
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Run Trivy vulnerability scanner
|
||||
continue-on-error: true
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
image-ref: 'wifi-densepose:scan'
|
||||
@@ -170,6 +188,7 @@ jobs:
|
||||
output: 'trivy-results.sarif'
|
||||
|
||||
- name: Upload Trivy results to GitHub Security
|
||||
continue-on-error: true
|
||||
uses: github/codeql-action/upload-sarif@v3
|
||||
if: always()
|
||||
with:
|
||||
@@ -177,7 +196,8 @@ jobs:
|
||||
category: trivy
|
||||
|
||||
- name: Run Grype vulnerability scanner
|
||||
uses: anchore/scan-action@v3
|
||||
continue-on-error: true
|
||||
uses: anchore/scan-action@v7
|
||||
id: grype-scan
|
||||
with:
|
||||
image: 'wifi-densepose:scan'
|
||||
@@ -186,6 +206,7 @@ jobs:
|
||||
output-format: sarif
|
||||
|
||||
- name: Upload Grype results to GitHub Security
|
||||
continue-on-error: true
|
||||
uses: github/codeql-action/upload-sarif@v3
|
||||
if: always()
|
||||
with:
|
||||
@@ -193,6 +214,7 @@ jobs:
|
||||
category: grype
|
||||
|
||||
- name: Run Docker Scout
|
||||
continue-on-error: true
|
||||
uses: docker/scout-action@v1
|
||||
if: always()
|
||||
with:
|
||||
@@ -202,6 +224,7 @@ jobs:
|
||||
summary: true
|
||||
|
||||
- name: Upload Docker Scout results
|
||||
continue-on-error: true
|
||||
uses: github/codeql-action/upload-sarif@v3
|
||||
if: always()
|
||||
with:
|
||||
@@ -212,15 +235,18 @@ jobs:
|
||||
iac-scan:
|
||||
name: Infrastructure Security Scan
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true # third-party scanners are flaky / SARIF uploads can 403; don't gate the PR
|
||||
permissions:
|
||||
security-events: write
|
||||
actions: read
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Run Checkov IaC scan
|
||||
continue-on-error: true
|
||||
uses: bridgecrewio/checkov-action@99bb2caf247dfd9f03cf984373bc6043d4e32ebf # v12.1347.0
|
||||
with:
|
||||
directory: .
|
||||
@@ -231,6 +257,7 @@ jobs:
|
||||
soft_fail: true
|
||||
|
||||
- name: Upload Checkov results to GitHub Security
|
||||
continue-on-error: true
|
||||
uses: github/codeql-action/upload-sarif@v3
|
||||
if: always()
|
||||
with:
|
||||
@@ -238,6 +265,7 @@ jobs:
|
||||
category: checkov
|
||||
|
||||
- name: Run Terrascan IaC scan
|
||||
continue-on-error: true
|
||||
uses: tenable/terrascan-action@3a6e87da8e244513bd77b631e624552643f794c6 # v1.4.1
|
||||
with:
|
||||
iac_type: 'k8s'
|
||||
@@ -247,6 +275,7 @@ jobs:
|
||||
sarif_upload: true
|
||||
|
||||
- name: Run KICS IaC scan
|
||||
continue-on-error: true
|
||||
uses: checkmarx/kics-github-action@05aa5eb70eede1355220f4ca5238d96b397e30a6 # v2.1.20
|
||||
with:
|
||||
path: '.'
|
||||
@@ -256,6 +285,7 @@ jobs:
|
||||
exclude_queries: 'a7ef1e8c-fbf8-4ac1-b8c7-2c3b0e6c6c6c'
|
||||
|
||||
- name: Upload KICS results to GitHub Security
|
||||
continue-on-error: true
|
||||
uses: github/codeql-action/upload-sarif@v3
|
||||
if: always()
|
||||
with:
|
||||
@@ -266,17 +296,20 @@ jobs:
|
||||
secret-scan:
|
||||
name: Secret Scanning
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true # third-party scanners are flaky / SARIF uploads can 403; don't gate the PR
|
||||
permissions:
|
||||
security-events: write
|
||||
actions: read
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Run TruffleHog secret scan
|
||||
continue-on-error: true
|
||||
uses: trufflesecurity/trufflehog@17456f8c7d042d8c82c9a8ca9e937231f9f42e26 # v3.95.2
|
||||
with:
|
||||
path: ./
|
||||
@@ -285,6 +318,7 @@ jobs:
|
||||
extra_args: --debug --only-verified
|
||||
|
||||
- name: Run GitLeaks secret scan
|
||||
continue-on-error: true
|
||||
uses: gitleaks/gitleaks-action@v2
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -301,28 +335,34 @@ jobs:
|
||||
license-scan:
|
||||
name: License Compliance Scan
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true # third-party scanners are flaky / SARIF uploads can 403; don't gate the PR
|
||||
steps:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
continue-on-error: true
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install dependencies
|
||||
continue-on-error: true
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
pip install pip-licenses licensecheck
|
||||
|
||||
- name: Run license check
|
||||
continue-on-error: true
|
||||
run: |
|
||||
pip-licenses --format=json --output-file=licenses.json
|
||||
licensecheck --zero
|
||||
|
||||
- name: Upload license report
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: license-report
|
||||
@@ -332,11 +372,14 @@ jobs:
|
||||
compliance-check:
|
||||
name: Security Policy Compliance
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true # third-party scanners are flaky / SARIF uploads can 403; don't gate the PR
|
||||
steps:
|
||||
- name: Checkout code
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Check security policy files
|
||||
continue-on-error: true
|
||||
run: |
|
||||
# Check for required security files
|
||||
files=("SECURITY.md" ".github/SECURITY.md" "docs/SECURITY.md")
|
||||
@@ -354,11 +397,13 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Check for security headers in code
|
||||
continue-on-error: true
|
||||
run: |
|
||||
# Check for security-related configurations
|
||||
grep -r "X-Frame-Options\|X-Content-Type-Options\|X-XSS-Protection\|Content-Security-Policy" src/ || echo "⚠️ Consider adding security headers"
|
||||
|
||||
- name: Validate Kubernetes security contexts
|
||||
continue-on-error: true
|
||||
run: |
|
||||
# Check for security contexts in Kubernetes manifests
|
||||
if [[ -d "k8s" ]]; then
|
||||
@@ -375,6 +420,7 @@ jobs:
|
||||
security-report:
|
||||
name: Security Report
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true # third-party scanners are flaky / SARIF uploads can 403; don't gate the PR
|
||||
needs: [sast, dependency-scan, container-scan, iac-scan, secret-scan, license-scan, compliance-check]
|
||||
if: always()
|
||||
# Promote secret to env-scope so the gating `if:` on the Slack-notify
|
||||
@@ -384,9 +430,11 @@ jobs:
|
||||
SECURITY_SLACK_WEBHOOK_URL: ${{ secrets.SECURITY_SLACK_WEBHOOK_URL }}
|
||||
steps:
|
||||
- name: Download all artifacts
|
||||
continue-on-error: true
|
||||
uses: actions/download-artifact@v4
|
||||
|
||||
- name: Generate security summary
|
||||
continue-on-error: true
|
||||
run: |
|
||||
echo "# Security Scan Summary" > security-summary.md
|
||||
echo "" >> security-summary.md
|
||||
@@ -402,6 +450,7 @@ jobs:
|
||||
echo "Generated on: $(date)" >> security-summary.md
|
||||
|
||||
- name: Upload security summary
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: security-summary
|
||||
@@ -411,6 +460,7 @@ jobs:
|
||||
# use env.X instead. Inherits SECURITY_SLACK_WEBHOOK_URL from the
|
||||
# job-level env block (added below).
|
||||
- name: Notify security team on critical findings
|
||||
continue-on-error: true
|
||||
if: ${{ env.SECURITY_SLACK_WEBHOOK_URL != '' && (needs.sast.result == 'failure' || needs.dependency-scan.result == 'failure' || needs.container-scan.result == 'failure') }}
|
||||
uses: 8398a7/action-slack@v3
|
||||
with:
|
||||
@@ -426,6 +476,7 @@ jobs:
|
||||
SLACK_WEBHOOK_URL: ${{ env.SECURITY_SLACK_WEBHOOK_URL }}
|
||||
|
||||
- name: Create security issue on critical findings
|
||||
continue-on-error: true
|
||||
if: needs.sast.result == 'failure' || needs.dependency-scan.result == 'failure'
|
||||
uses: actions/github-script@v6
|
||||
with:
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
name: wifi-densepose sensing-server → Docker Hub + ghcr.io
|
||||
|
||||
# Build + publish the `wifi-densepose` sensing-server image to both Docker Hub
|
||||
# (`ruvnet/wifi-densepose`) and ghcr.io (`ghcr.io/ruvnet/wifi-densepose`) on:
|
||||
# - push to main affecting the Dockerfile, the server crate, the UI assets,
|
||||
# or this workflow itself,
|
||||
# - tag push matching v* (release builds),
|
||||
# - manual workflow_dispatch.
|
||||
#
|
||||
# Closes #520 and #514: the stale `:latest` is rebuilt and pushed automatically
|
||||
# whenever the surface that produces it changes, and the Dockerfile fails the
|
||||
# build if the observatory/pose-fusion UI assets ever go missing again.
|
||||
#
|
||||
# Secrets:
|
||||
# DOCKERHUB_USERNAME — `ruvnet` (Docker Hub login name)
|
||||
# DOCKERHUB_TOKEN — Docker Hub access token with read/write/delete scope
|
||||
# (ghcr.io uses the workflow's GITHUB_TOKEN — no secret needed.)
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'docker/Dockerfile.rust'
|
||||
- 'docker/docker-entrypoint.sh'
|
||||
- 'v2/crates/wifi-densepose-sensing-server/**'
|
||||
- 'v2/crates/wifi-densepose-signal/**'
|
||||
- 'v2/crates/wifi-densepose-vitals/**'
|
||||
- 'v2/crates/wifi-densepose-wifiscan/**'
|
||||
- 'v2/Cargo.toml'
|
||||
- 'v2/Cargo.lock'
|
||||
- 'ui/**'
|
||||
- '.github/workflows/sensing-server-docker.yml'
|
||||
tags: ['v*']
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
concurrency:
|
||||
group: sensing-server-docker-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-and-publish:
|
||||
name: build · push · smoke-test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
# QEMU is required so the amd64 GitHub runner can cross-build the
|
||||
# linux/arm64 layer below (Dockerfile.rust is arch-agnostic — no `--target`
|
||||
# flag — so buildx + QEMU is all that's needed; arm64 builds are emulated
|
||||
# by the runner, not built on a separate arm64 host).
|
||||
- uses: docker/setup-qemu-action@v3
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: docker.io
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Log in to ghcr.io
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Compute tags
|
||||
id: meta
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: |
|
||||
docker.io/ruvnet/wifi-densepose
|
||||
ghcr.io/ruvnet/wifi-densepose
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=tag
|
||||
type=sha,format=short
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
|
||||
- name: Build + push
|
||||
id: build
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile.rust
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
# README badge advertises `amd64 + arm64`, and #547 promised multi-arch
|
||||
# as part of the docker publish refresh; arm64 was never actually wired
|
||||
# in, so Apple Silicon Macs hit `no matching manifest for linux/arm64/v8`
|
||||
# on `docker pull ruvnet/wifi-densepose:latest` (#136, #625). Build both.
|
||||
platforms: linux/amd64,linux/arm64
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Smoke-test the freshly-pushed image:
|
||||
# 1. UI assets that closed #520 are inside `/app/ui` (the Dockerfile's
|
||||
# RUN guard catches missing ones at build time, this re-checks the
|
||||
# pushed artifact post-hoc as belt-and-braces).
|
||||
# 2. /health is up.
|
||||
# 3. /api/v1/info returns 200 with no auth (LAN-mode default).
|
||||
# 4. With RUVIEW_API_TOKEN set, /api/v1/info returns 401 without a
|
||||
# Bearer header, 200 with the correct one (the #443 auth middleware).
|
||||
# ---------------------------------------------------------------------
|
||||
- name: Smoke-test image assets + LAN-mode HTTP
|
||||
run: |
|
||||
set -euo pipefail
|
||||
IMAGE="ghcr.io/ruvnet/wifi-densepose:sha-${GITHUB_SHA::7}"
|
||||
docker pull "$IMAGE"
|
||||
docker run --rm "$IMAGE" sh -c \
|
||||
'ls /app/ui/observatory.html /app/ui/pose-fusion.html /app/ui/index.html /app/ui/viz.html >/dev/null'
|
||||
docker run --rm "$IMAGE" sh -c 'ls -d /app/ui/observatory /app/ui/pose-fusion >/dev/null'
|
||||
|
||||
docker run -d --name sm -p 3000:3000 -e CSI_SOURCE=simulated "$IMAGE"
|
||||
# Wait up to 30 s for /health.
|
||||
for _ in $(seq 1 30); do
|
||||
if curl -fsS http://127.0.0.1:3000/health >/dev/null 2>&1; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
curl -fsS http://127.0.0.1:3000/health
|
||||
curl -fsS http://127.0.0.1:3000/api/v1/info >/dev/null
|
||||
curl -fsS http://127.0.0.1:3000/ui/observatory.html >/dev/null
|
||||
curl -fsS http://127.0.0.1:3000/ui/pose-fusion.html >/dev/null
|
||||
docker stop sm
|
||||
|
||||
- name: Smoke-test the bearer-token auth path
|
||||
run: |
|
||||
set -euo pipefail
|
||||
IMAGE="ghcr.io/ruvnet/wifi-densepose:sha-${GITHUB_SHA::7}"
|
||||
docker run -d --name auth \
|
||||
-p 3000:3000 \
|
||||
-e CSI_SOURCE=simulated \
|
||||
-e RUVIEW_API_TOKEN=smoke-test-token-do-not-use \
|
||||
"$IMAGE"
|
||||
for _ in $(seq 1 30); do
|
||||
if curl -fsS http://127.0.0.1:3000/health >/dev/null 2>&1; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
# /health stays unauthenticated.
|
||||
curl -fsS http://127.0.0.1:3000/health >/dev/null
|
||||
# /api/v1/info without a bearer → 401.
|
||||
code=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3000/api/v1/info)
|
||||
test "$code" = "401" || { echo "expected 401, got $code"; exit 1; }
|
||||
# Wrong bearer → 401.
|
||||
code=$(curl -s -o /dev/null -w '%{http_code}' -H 'Authorization: Bearer wrong' http://127.0.0.1:3000/api/v1/info)
|
||||
test "$code" = "401" || { echo "expected 401 (wrong token), got $code"; exit 1; }
|
||||
# Correct bearer → 200.
|
||||
curl -fsS -H 'Authorization: Bearer smoke-test-token-do-not-use' http://127.0.0.1:3000/api/v1/info >/dev/null
|
||||
docker stop auth
|
||||
|
||||
- name: Summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## sensing-server image published"
|
||||
echo
|
||||
echo "Tags:"
|
||||
echo '```'
|
||||
echo "${{ steps.meta.outputs.tags }}"
|
||||
echo '```'
|
||||
echo
|
||||
echo "Closes #520 (missing observatory/pose-fusion UI assets) and #514 (stale `:latest` for the v0.6+ packet format)."
|
||||
echo "The Dockerfile fails the build if those UI assets ever disappear again, and this workflow rebuilds + pushes automatically on every change to the surface."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -19,8 +19,24 @@ jobs:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Update submodules to latest main
|
||||
run: git submodule update --remote --merge
|
||||
# Identity must be set BEFORE any operation that can create a commit.
|
||||
# `git submodule update --remote --merge` used to fail here with
|
||||
# "Committer identity unknown" because the merge inside vendor/ruvector
|
||||
# needs an author when the pinned commit isn't a fast-forward of upstream.
|
||||
- name: Configure git identity
|
||||
run: |
|
||||
git config --global user.name "github-actions[bot]"
|
||||
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
# Use a plain `--remote` checkout (detached HEAD at each submodule's
|
||||
# configured `branch` tip from .gitmodules) rather than `--merge`. We only
|
||||
# want to bump the superproject's gitlink to the latest upstream commit;
|
||||
# there's no reason to create merge commits inside the vendored repos, and
|
||||
# `--merge` breaks whenever the current pin has diverged from that branch.
|
||||
- name: Update submodules to latest tracked branch
|
||||
run: |
|
||||
git submodule sync --recursive
|
||||
git submodule update --remote --recursive
|
||||
|
||||
- name: Check for changes
|
||||
id: check
|
||||
@@ -29,21 +45,22 @@ jobs:
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
echo "--- submodule pointer changes ---"
|
||||
git submodule status --recursive || true
|
||||
git diff --submodule=log -- vendor/ || true
|
||||
fi
|
||||
|
||||
- name: Create PR with updates
|
||||
if: steps.check.outputs.changed == 'true'
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
BRANCH="chore/update-submodules-$(date +%Y%m%d-%H%M%S)"
|
||||
git checkout -b "$BRANCH"
|
||||
git add vendor/
|
||||
git commit -m "chore: update vendor submodules to latest main"
|
||||
git commit -m "chore: update vendor submodules to latest upstream"
|
||||
git push origin "$BRANCH"
|
||||
gh pr create \
|
||||
--title "chore: update vendor submodules" \
|
||||
--body "Automated submodule update to latest upstream main." \
|
||||
--body "Automated submodule update to the latest upstream commit on each submodule's tracked branch (see \`.gitmodules\`). Review the pointer diff before merging." \
|
||||
--base main \
|
||||
--head "$BRANCH"
|
||||
env:
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
@@ -57,7 +57,18 @@ jobs:
|
||||
"
|
||||
|
||||
- name: Run pipeline verification
|
||||
working-directory: v1
|
||||
working-directory: archive/v1
|
||||
env:
|
||||
# Pin thread count for scipy.fft / BLAS — multi-threaded reduction
|
||||
# order is otherwise non-deterministic across CI runs (issue #560
|
||||
# follow-up: 9- and 6-decimal quantization were not enough because
|
||||
# the divergence is from threading order, not SIMD reordering).
|
||||
# Single-threaded keeps the proof reproducible at a ~2-3x slowdown.
|
||||
OMP_NUM_THREADS: "1"
|
||||
OPENBLAS_NUM_THREADS: "1"
|
||||
MKL_NUM_THREADS: "1"
|
||||
VECLIB_MAXIMUM_THREADS: "1"
|
||||
NUMEXPR_NUM_THREADS: "1"
|
||||
run: |
|
||||
echo "=== Running pipeline verification ==="
|
||||
python data/proof/verify.py
|
||||
@@ -65,7 +76,13 @@ jobs:
|
||||
echo "Pipeline verification PASSED."
|
||||
|
||||
- name: Run verification twice to confirm determinism
|
||||
working-directory: v1
|
||||
working-directory: archive/v1
|
||||
env:
|
||||
OMP_NUM_THREADS: "1"
|
||||
OPENBLAS_NUM_THREADS: "1"
|
||||
MKL_NUM_THREADS: "1"
|
||||
VECLIB_MAXIMUM_THREADS: "1"
|
||||
NUMEXPR_NUM_THREADS: "1"
|
||||
run: |
|
||||
echo "=== Second run for determinism confirmation ==="
|
||||
python data/proof/verify.py
|
||||
|
||||
@@ -13,6 +13,9 @@ firmware/esp32-csi-node/managed_components/
|
||||
firmware/esp32-csi-node/dependencies.lock
|
||||
firmware/esp32-csi-node/sdkconfig.defaults.bak
|
||||
|
||||
# ESP-IDF set-target backup (local only)
|
||||
firmware/esp32-hello-world/sdkconfig.old
|
||||
|
||||
# Claude Flow swarm runtime state
|
||||
.swarm/
|
||||
|
||||
@@ -252,3 +255,9 @@ firmware/esp32-csi-node/build_firmware.batdata/
|
||||
models/
|
||||
demo_pointcloud.ply
|
||||
demo_splats.json
|
||||
|
||||
# rvCSI napi-rs addon — generated by `napi build` (do not commit)
|
||||
v2/crates/rvcsi-node/*.node
|
||||
v2/crates/rvcsi-node/binding.js
|
||||
v2/crates/rvcsi-node/binding.d.ts
|
||||
v2/crates/rvcsi-node/npm/
|
||||
|
||||
@@ -10,3 +10,7 @@
|
||||
path = vendor/sublinear-time-solver
|
||||
url = https://github.com/ruvnet/sublinear-time-solver
|
||||
branch = main
|
||||
[submodule "vendor/rvcsi"]
|
||||
path = vendor/rvcsi
|
||||
url = https://github.com/ruvnet/rvcsi
|
||||
branch = main
|
||||
|
||||
@@ -7,7 +7,177 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Security
|
||||
- **ESP32 OTA upload now fails closed when no PSK is provisioned** (#596 audit finding — critical, **breaking change for unprovisioned nodes**). `ota_check_auth()` previously returned `true` when `s_ota_psk[0] == '\0'`, so a freshly-flashed node would accept attacker-controlled firmware over plain HTTP on port 8032 from any host on the WiFi. No Secure Boot V2, no signed-image verification — a single LAN call could brick or backdoor a node. The fix rejects every OTA upload until a PSK is written to NVS (the OTA HTTP server still starts so operators can run `provision.py --ota-psk <hex>` over USB-CDC without reflashing). **Operators affected**: any deployment that relied on the unauthenticated OTA endpoint working out of the box now needs to provision a PSK before subsequent OTA pushes will succeed. Boot-time `ESP_LOGW` makes the new posture visible.
|
||||
- **Path-traversal vulnerabilities patched in five sensing-server endpoints** (closes #615 — critical). New `wifi_densepose_sensing_server::path_safety::safe_id()` enforces `[A-Za-z0-9._-]` only (no leading `.`, max 64 chars) before any user-controlled identifier reaches a `format!()` building a filesystem path. Applied at:
|
||||
- `POST /api/v1/recording/start` (`recording.rs` — `session_name`)
|
||||
- `GET /api/v1/recording/download/:id` (`recording.rs` — `id`)
|
||||
- `DELETE /api/v1/recording/delete/:id` (`recording.rs` — `id`)
|
||||
- `POST /api/v1/models/load` (`model_manager.rs` — `model_id`)
|
||||
- `training_api.rs` `load_recording_frames` (`dataset_id`s)
|
||||
|
||||
Pre-fix, unauthenticated callers could read `../../etc/passwd`-style paths, write arbitrary JSONL files, load attacker-controlled `.rvf` model files, or delete arbitrary files the server process could touch. 9 unit tests in `path_safety::tests` exercise the rejection envelope (empty, too-long, path separators, parent-dir traversal, null byte, whitespace/specials, non-ASCII).
|
||||
|
||||
### Fixed
|
||||
- **WebSocket `/ws/sensing` now reports `esp32:offline` when ESP32 hardware goes stale** (closes #618). `broadcast_tick_task` was re-emitting the cached `latest_update` with a frozen `source: "esp32"` field forever after the hardware lost power or network. The REST `/health` endpoint already called `effective_source()` (which returns `"esp32:offline"` after `ESP32_OFFLINE_TIMEOUT` = 5 s with no UDP frames), but the WS broadcast path was the one consumer that didn't. Result: the UI's "LIVE — ESP32 HARDWARE Connected" banner stayed green long after the hardware went away, and `vital_signs`/`features`/`classification` re-broadcasted the last-seen values indefinitely. Fix: clone the cached `latest_update` per tick, overwrite `source` with `s.effective_source()`, then serialize and broadcast. UI can now switch to an offline state on the same 5-second budget the REST surface uses.
|
||||
- **Proof replay (`archive/v1/data/proof/verify.py`) is now cross-platform deterministic** (closes #560). Three changes together: (1) `features_to_bytes()` now `np.round(.., HASH_QUANTIZATION_DECIMALS=6)`s each feature array before packing as little-endian f64, collapsing ULP-level drift from scipy.fft pocketfft SIMD reordering; (2) the `Verify Pipeline Determinism` workflow pins `OMP_NUM_THREADS=1`, `OPENBLAS_NUM_THREADS=1`, `MKL_NUM_THREADS=1`, `VECLIB_MAXIMUM_THREADS=1`, `NUMEXPR_NUM_THREADS=1` — multi-threaded BLAS reductions were a deeper source of non-determinism than SIMD reordering, and 6-decimal quantization alone wasn't enough across Azure VM microarchitectures; (3) `expected_features.sha256` regenerated under the new conditions. CI now passes the determinism check (same hash across consecutive runs on canonical Linux x86_64 CI runner: `667eb054c44ac510342665bf9c93d608868a8ead948ae8774b2796ebce6f8fe7`). `scripts/probe-fft-platform.py` updated to mirror `HASH_QUANTIZATION_DECIMALS=6` for cross-machine spot-checks.
|
||||
- **`archive/v1/src/services/pose_service.py:223` calls the right method on `PhaseSanitizer`** (closes #612). The call was `self.phase_sanitizer.sanitize(phase_data)`, but `PhaseSanitizer`'s full-pipeline entry point is named `sanitize_phase()` (`unwrap_phase` + `remove_outliers` + `smooth_phase` chained, see `archive/v1/src/core/phase_sanitizer.py:266`). The shorter `sanitize` name doesn't exist on the class, so any path that reached this branch raised `AttributeError` and crashed the pose service mid-frame.
|
||||
- **`adaptive_classifier.rs:94` no longer panics on NaN feature values** (closes #611).
|
||||
`sorted.sort_by(|a, b| a.partial_cmp(b).unwrap())` returned `None` and panicked
|
||||
whenever a single `NaN` reached the classifier from real ESP32 hardware (silent
|
||||
DSP div-by-zero, empty buffer). One bad frame killed the entire sensing-server
|
||||
process. Swapped for `unwrap_or(Ordering::Equal)`, matching the pattern the
|
||||
same file already used at lines 149-150 and 155. Per-frame hot path; this was
|
||||
a real production crash vector.
|
||||
- **Completed the #611 NaN-panic audit across the sensing-server crate** (follow-up
|
||||
to #613). The original audit grepped for the literal `partial_cmp(b).unwrap()`
|
||||
and missed seven additional production sites that use comparator variants
|
||||
(`partial_cmp(b.1).unwrap()`, `partial_cmp(&variances[b]).unwrap()`). All share
|
||||
the same crash class — a single `NaN` in CSI-derived state panics the whole
|
||||
sensing-server. Fixed:
|
||||
- `adaptive_classifier.rs:205` — `AdaptiveModel::classify()` argmax over softmax
|
||||
probs. **Same per-frame hot path as #611**; NaN flows through normalise →
|
||||
logits → softmax and still reaches this site even after the #613 IQR fix.
|
||||
- `adaptive_classifier.rs:480, 500` — training-loop argmax in `train()`
|
||||
(training/per-class accuracy reporting).
|
||||
- `main.rs:2446, 2449` and `csi.rs:602, 605` — variance-based source/sink
|
||||
selection in `count_persons_mincut`. The outer `unwrap_or((0, &0))` only
|
||||
catches an empty iterator; it cannot rescue a comparator panic.
|
||||
|
||||
Remaining `partial_cmp(...).unwrap()` sites in the workspace are all inside
|
||||
`#[cfg(test)]` / `#[test]` blocks (`spectrogram.rs:269`, `depth.rs:234`,
|
||||
`connectivity.rs:477`, `vital_signs.rs:737`) where inputs are controlled.
|
||||
- **`ui/utils/pose-renderer.js` no longer divides by zero** when two render frames land in the same `performance.now()` tick (issue #519 Bug 2). `deltaTime` is now `Math.max(currentTime - lastFrameTime, 1)` before the `1000 / deltaTime` division, capping displayed FPS at 1000 — far above any real render rate, but finite so the EMA `averageFps = averageFps * 0.9 + fps * 0.1` no longer poisons itself to `Infinity` on a single zero-dt tick.
|
||||
|
||||
### Removed
|
||||
- **Stub crates `wifi-densepose-api`, `wifi-densepose-db`, `wifi-densepose-config`** (closes #578).
|
||||
Each was a single-line doc-comment placeholder with an empty `[dependencies]`
|
||||
section and zero references from any source file or `Cargo.toml`. The names
|
||||
were reserved early for an envisioned REST/database/config split that never
|
||||
materialised; the functionality they would provide is covered today by
|
||||
`wifi-densepose-sensing-server` (Axum REST/WS), per-crate config + CLI args,
|
||||
and the project's real-time-only (no-persistent-state) posture. Removing them
|
||||
from the workspace prevents `cargo` from listing dead crates and shipping
|
||||
empty published artifacts. If any of these names is needed in the future,
|
||||
they can be reintroduced with a real implementation.
|
||||
|
||||
### Added
|
||||
- **Real-time CSI introspection / low-latency tap on `wifi-densepose-sensing-server` (ADR-099).**
|
||||
New `wifi_densepose_sensing_server::introspection` module wires
|
||||
[midstream](https://github.com/ruvnet/midstream)'s `temporal-attractor` (Lyapunov +
|
||||
regime classification) and `temporal-compare` (DTW pattern matching) as a
|
||||
**parallel tap** alongside RuView's existing event pipeline — no replacement,
|
||||
no behaviour change to the existing `/ws/sensing` fan-out or `wifi-densepose-signal`
|
||||
DSP. Two new endpoints (off by default, enabled via `--introspection`):
|
||||
- `GET /ws/introspection` — newline-delimited JSON snapshots streamed at the CSI
|
||||
frame rate. Each snapshot carries `frame_count`, `regime` (Idle / Periodic /
|
||||
Transient / Chaotic / Unknown), `lyapunov_exponent`, `attractor_dim`,
|
||||
`attractor_confidence`, `regime_changed` (boolean — flips on the first frame
|
||||
after a regime transition), and `top_k_similarity[]` (highest-scoring
|
||||
signature matches against a per-deployment library).
|
||||
- `GET /api/v1/introspection/snapshot` — single-shot JSON snapshot, auth-gated
|
||||
when `RUVIEW_API_TOKEN` is set.
|
||||
Per-frame `update()` budget measured at **0.041 ms p99** on the I5 bench
|
||||
(~24× under ADR-099 D4's 1 ms target). Shape-match latency on a 1-D
|
||||
mean-amplitude L1 stand-in: **5 frames** (3.20× ratio vs the 16-frame event-path
|
||||
floor). ADR-099 D8 honestly amended — the aspirational 10× bar is contingent on
|
||||
ADR-208 Phase 2 multi-dim NPU embeddings; this release ships the tap off-by-default
|
||||
while the foundation lands. 8 lib tests + 5 latency/regression tests (`tests/introspection_latency.rs`,
|
||||
including a 200-frame noise warm-up → 10-frame motion-ramp signature benchmark).
|
||||
- **Opt-in bearer-token auth on `wifi-densepose-sensing-server`'s `/api/v1/*` HTTP surface (closes #443).**
|
||||
New `wifi_densepose_sensing_server::bearer_auth` module: when the
|
||||
`RUVIEW_API_TOKEN` env var is set, every request whose path begins with
|
||||
`/api/v1/` must carry an `Authorization: Bearer <token>` header (constant-time
|
||||
compared) or the server responds `401 Unauthorized`. When the variable is
|
||||
unset or empty the middleware is a no-op — the long-standing LAN-only
|
||||
deployment posture is preserved, so this is a binary deployment-time switch
|
||||
with **no default behaviour change**. `/health*`, `/ws/sensing`, and the
|
||||
`/ui/*` static mount are intentionally never gated (orchestrator probes +
|
||||
local browsers). Startup logs which mode is active and warns when auth is on
|
||||
with a `0.0.0.0` bind. 8 unit tests on the middleware (lib test count 191 → 199).
|
||||
Resolves the security audit raised in #443.
|
||||
|
||||
### Changed
|
||||
- **Docker image: build-time guard for the UI assets, plus a CI workflow that
|
||||
rebuilds and pushes on every change (closes #520, #514).** `docker/Dockerfile.rust`
|
||||
now `RUN`s a guard after `COPY ui/` that fails the build if any of
|
||||
`index.html` / `observatory.html` / `pose-fusion.html` / `viz.html` / the
|
||||
`observatory/` / `pose-fusion/` / `components/` / `services/` directories are
|
||||
missing, so a stale image can never be silently produced again. New
|
||||
`.github/workflows/sensing-server-docker.yml` builds the image on push to
|
||||
`main` (paths-filtered) and on `v*` tags and pushes to both
|
||||
`docker.io/ruvnet/wifi-densepose` and `ghcr.io/ruvnet/wifi-densepose` with
|
||||
`latest` + `vX.Y.Z` + `sha-<short>` tags, then smoke-tests the published
|
||||
artifact: `/health`, `/api/v1/info`, the observatory + pose-fusion UI assets,
|
||||
and the `RUVIEW_API_TOKEN` auth path (no token → 401, wrong → 401, correct
|
||||
→ 200). Uses `DOCKERHUB_USERNAME` / `DOCKERHUB_TOKEN` repo secrets for the
|
||||
Docker Hub push; ghcr.io uses the workflow's `GITHUB_TOKEN`.
|
||||
- **rvCSI moved to its own repo and is now vendored as a submodule.** The 9 `rvcsi-*`
|
||||
crates (`rvcsi-core`/`-dsp`/`-events`/`-adapter-file`/`-adapter-nexmon`/`-ruvector`/
|
||||
`-runtime`/`-node`/`-cli` — added inline in #542) now live in
|
||||
[`github.com/ruvnet/rvcsi`](https://github.com/ruvnet/rvcsi): published to crates.io
|
||||
as `rvcsi-* 0.3.x`, to npm as `@ruv/rvcsi`, with a Claude Code plugin marketplace and
|
||||
a RuView-style README. RuView vendors it under `vendor/rvcsi` (alongside
|
||||
`vendor/ruvector` / `vendor/midstream` / `vendor/sublinear-time-solver`) and no longer
|
||||
carries inline copies in `v2/crates/`; consumers depend on the published crates (or the
|
||||
submodule's `crates/rvcsi-*` paths). `v2/Cargo.toml`, `CLAUDE.md`, and the README docs
|
||||
table updated accordingly. The ADRs (ADR-095, ADR-096), PRD, and DDD model stay in
|
||||
`docs/` here as the design record of the incubation.
|
||||
|
||||
### Fixed
|
||||
- **README: corrected the camera-supervised pose-accuracy claim.** The README stated
|
||||
"92.9% PCK@20" for camera-supervised training; that figure does not appear in
|
||||
ADR-079 and is ~2.6× the ADR's own success target (>35% PCK@20). ADR-079 phases
|
||||
P7 (data collection), P8 (training + evaluation on real paired data) and P9
|
||||
(cross-room LoRA) are still `Pending`, so no measured camera-supervised PCK@20 has
|
||||
been published. README now states the proxy-supervised baseline (≈2.5%) and the
|
||||
ADR-079 target (35%+), and notes the eval phases are pending. Surfaced by the
|
||||
PowerPlatePulse training-pipeline audit (2026-05-11); 6 remaining audit findings
|
||||
tracked in the PR.
|
||||
- **rvCSI `BaselineDriftDetector`: drift thresholds are now scale-relative, not absolute.**
|
||||
The detector compared `mean_amplitude` against its EWMA baseline with absolute
|
||||
thresholds (`anomaly_threshold = 1.0`, `drift_threshold = 0.15`) — fine for the
|
||||
synthetic unit tests (amplitudes ≈ 1.0), but raw ESP32 CSI is `int8` I/Q with
|
||||
amplitudes up to ~128, so the window-to-window RMS distance is routinely 5–50 ≫ 1.0
|
||||
and `AnomalyDetected` fired on ~96 % of windows (319/331 on a real node-1 capture).
|
||||
Drift is now `‖current − baseline‖₂ / ‖baseline‖₂` (a fraction, with an `eps` floor
|
||||
for a degenerate near-zero baseline), so one tuning works across raw-`int8` ESP32,
|
||||
`int16`-scaled Nexmon, and baseline-subtracted streams alike — `AnomalyDetected`
|
||||
drops to 40/331 on the same data, the existing detector tests still pass, and a
|
||||
`baseline_drift_is_scale_invariant_no_anomaly_storm` regression test was added.
|
||||
ADR-095 D13 / ADR-096 §2.1, §5 updated. Surfaced by an end-to-end test against
|
||||
real ESP32 CSI (a 7,000-frame node-1 capture; transcoder at
|
||||
`scripts/esp32_jsonl_to_rvcsi.py`).
|
||||
|
||||
### Added
|
||||
- **rvCSI — edge RF sensing runtime (design + first implementation).** New subsystem **rvCSI**: a Rust-first / TypeScript-accessible / hardware-abstracted edge RF sensing runtime that normalizes WiFi CSI from Nexmon, ESP32, Intel, Atheros, file and replay sources into one validated `CsiFrame` schema, runs reusable DSP, emits typed confidence-scored events, and bridges to RuVector RF memory, an MCP tool server and a TS SDK.
|
||||
- **Design docs:** `docs/prd/rvcsi-platform-prd.md` (purpose, users, success criteria, FR1–FR10, NFRs, system architecture, data model); `docs/adr/ADR-095-rvcsi-edge-rf-sensing-platform.md` (the 15 architectural decisions: Rust core, C-at-the-boundary, TS SDK via napi-rs, normalized schema, validate-before-FFI, CSI-as-temporal-delta, RuVector as RF memory, replayability, detection≠decision, local-first, read-first/write-gated MCP, mandatory quality scoring, versioned calibration, plugin adapters); `docs/adr/ADR-096-rvcsi-ffi-crate-layout.md` (crate topology, the napi-c shim record format & contract, the napi-rs Node surface, build/test invariants); `docs/ddd/rvcsi-domain-model.md` (7 bounded contexts: Capture, Validation, Signal, Calibration, Event, Memory, Agent — with aggregates, invariants, context map and domain services). Indexed in `docs/adr/README.md` and `docs/ddd/README.md`.
|
||||
- **Crates** (9 new `v2/crates/rvcsi-*` workspace members): `rvcsi-core` (normalized `CsiFrame`/`CsiWindow`/`CsiEvent` schema, `AdapterProfile`, `CsiSource` plugin trait, id newtypes + `IdGenerator`, `RvcsiError`, the `validate_frame` pipeline + quality scoring; `forbid(unsafe_code)`); `rvcsi-adapter-nexmon` — the **napi-c** seam: `native/rvcsi_nexmon_shim.{c,h}` (the only C in the runtime — allocation-free, bounds-checked, ABI `1.1`), compiled via `build.rs`+`cc`, handling **two byte formats** — the compact self-describing "rvCSI Nexmon record", and the **real nexmon_csi UDP payload** (the 18-byte `magic 0x1111 · rssi · fctl · src_mac · seq · core/stream · chanspec · chip_ver` header + `nsub` int16 I/Q samples, the modern BCM43455c0/4358/4366c0 export read by CSIKit/`csireader.py`), with a Broadcom d11ac **chanspec decoder** (channel/bandwidth/band) — plus a pure-Rust **libpcap reader** (classic `.pcap`, all byte-order/timestamp-resolution magics, Ethernet/raw-IPv4/Linux-SLL link types) and a **Nexmon-chip / Raspberry-Pi-model registry** (`NexmonChip` / `RaspberryPiModel` — including the **Raspberry Pi 5** (CYW43455/BCM43455c0, same wireless as the Pi 4 — 20/40/80 MHz, 2.4+5 GHz, 64/128/256 subcarriers), the Pi 3B+/4/400, and the Pi Zero 2 W (BCM43436b0); `nexmon_adapter_profile` / `raspberry_pi_profile` build the per-chip `AdapterProfile`; `chip_ver` words auto-resolve to a chip). Wrapped by a documented `ffi` module and two `CsiSource`s: `NexmonAdapter` (record buffers) and `NexmonPcapAdapter` (real nexmon_csi UDP inside a `tcpdump -i wlan0 dst port 5500 -w csi.pcap` capture — the pcap timestamp stamps each frame; the chip is auto-detected from `chip_ver`, overridable via `.with_pi_model(Pi5)` / `.with_chip(...)`). `rvcsi-dsp` (DC removal, phase unwrap, smoothing, Hampel/MAD filter, sliding variance, baseline subtraction, motion-energy/presence/confidence features, heuristic breathing-band estimate, non-destructive `SignalPipeline`); `rvcsi-events` (`WindowBuffer`, the `EventDetector` trait + presence/motion/quality/baseline-drift state machines, `EventPipeline`; the baseline-drift detector uses **scale-relative** thresholds — drift as a fraction of the baseline's RMS magnitude — so one tuning works across raw-`int8` ESP32, `int16`-scaled Nexmon, and baseline-subtracted streams alike); `rvcsi-adapter-file` (the `.rvcsi` JSONL capture format, `FileRecorder`, `FileReplayAdapter` deterministic replay); `rvcsi-ruvector` (deterministic window/event embeddings, `cosine_similarity`, the `RfMemoryStore` trait, `InMemoryRfMemory` + `JsonlRfMemory` — a standin until the production RuVector binding); `rvcsi-runtime` (the no-FFI composition layer: `CaptureRuntime` = `CsiSource` + `validate_frame` + `SignalPipeline` + `EventPipeline`, plus one-shot helpers `summarize_capture`/`decode_nexmon_records`/`decode_nexmon_pcap`/`summarize_nexmon_pcap`/`events_from_capture`/`export_capture_to_rf_memory`); `rvcsi-node` — the **napi-rs** seam (a `["cdylib","rlib"]` Node addon, `build.rs` runs `napi_build::setup()`; thin `#[napi]` wrappers over `rvcsi-runtime` — `nexmonDecodeRecords`/`nexmonDecodePcap` (with optional `chip`)/`inspectNexmonPcap`/`decodeChanspec`/`nexmonChipName`/`nexmonProfile`/`nexmonChips`/`inspectCaptureFile`/`eventsFromCaptureFile`/`exportCaptureToRfMemory` + an `RvcsiRuntime` streaming class; everything that crosses to JS is a validated/normalized struct serialized to JSON); `rvcsi-cli` (the `rvcsi` binary: `record` (Nexmon-dump *or* `--source nexmon-pcap [--chip pi5]` → `.rvcsi`), `inspect`, `inspect-nexmon`, `nexmon-chips`, `decode-chanspec`, `replay`, `stream`, `events`, `health`, `calibrate` v0-baseline, `export ruvector`). Plus the `@ruv/rvcsi` npm package (`package.json`/`index.js`/`index.d.ts`/`README`/`__test__`) alongside `rvcsi-node` — a curated JS surface that parses the addon's JSON into plain `CsiFrame`/`CsiWindow`/`CsiEvent`/`SourceHealth`/`CaptureSummary`/`NexmonPcapSummary`/`DecodedChanspec` objects, with a lazy native-addon load.
|
||||
- **Tests:** 169 across the rvcsi crates (core 29, dsp 28, events 19 — incl. a baseline-drift scale-invariance regression, adapter-file 20 + 1 doctest, adapter-nexmon 28 — round-tripping through the C shim and synthetic libpcap files, incl. Pi 5 / chip-detection, ruvector 20 + 1 doctest, runtime 13, cli 10), 0 failures; all rvcsi crates build together and are clippy-clean (`rvcsi-node` under `deny(clippy::all)`); `forbid(unsafe_code)` everywhere except `rvcsi-adapter-nexmon` (FFI, every `unsafe` block documented). Also exercised end-to-end against a real 7,000-frame ESP32 node-1 capture (transcoded with `scripts/esp32_jsonl_to_rvcsi.py` — the stand-in for the not-yet-shipped `record --source esp32-jsonl`): `rvcsi inspect`/`replay`/`calibrate`/`events` all run on real hardware data. Not yet wired in: live radio capture, `rvcsi-adapter-esp32` (live serial/UDP ESP32 source), the WebSocket daemon (`rvcsi-daemon`), the MCP tool server (`rvcsi-mcp`), and the legacy nexmon *packed-float* CSI export — follow-ups on top of these crates.
|
||||
- **`wifi-densepose-train`: `signal_features` module — wires `wifi-densepose-signal` into the training pipeline.** `wifi-densepose-signal` was previously a phantom dependency of `wifi-densepose-train` (listed in `Cargo.toml`, never imported). New `wifi_densepose_train::signal_features::extract_signal_features` (and `CsiSample::signal_features()`) run a windowed CSI observation's centre frame through `wifi_densepose_signal::features::FeatureExtractor`, producing a fixed-length (`FEATURE_LEN = 12`) amplitude/phase/PSD feature vector — the hook for a future vitals / multi-task supervision head (breathing- and heart-rate-band power are read off the PSD summary). The vector is produced on demand and not yet fed back into the loss. Surfaced by the 2026-05-11 training-pipeline audit (findings #1 "vitals features absent from training" and #2 "`wifi-densepose-signal` ghost dep").
|
||||
- **`wifi-densepose-train`: `TrainingConfig` subcarrier-layout presets + a real-loader integration test.** New `TrainingConfig::for_subcarriers(native, target)` plus named presets `ht40_192()` (≈192-sc ESP32 HT40 → 56) and `multiband_168()` (168-sc ADR-078 multi-band mesh → 56), so non-MM-Fi CSI shapes are first-class instead of requiring manual `native_subcarriers`/`num_subcarriers` overrides; field docs now list the supported source counts and the multi-NIC mapping. New `tests/test_real_loader.rs` round-trips synthetic CSI through `.npy` files → `MmFiDataset::discover`/`get` (including the subcarrier-interpolation branch and the empty-root case) — exercising the on-disk loader path the deterministic `verify-training` proof intentionally bypasses. Addresses training-pipeline audit findings #6 (56-sc/1-NIC config default) and #7 (multi-band mesh not in config); the #4 concern ("proof uses synthetic data") is reframed — the proof *should* use a reproducible source, and this test covers the real loader it skips.
|
||||
|
||||
### Fixed
|
||||
- **HuggingFace `MODEL_CARD.md`: marked the PIR/BME280 environmental-sensor ground-truth path as planned, not implemented** (training-pipeline audit finding #3) — the card presented PIR/BME280 weak-label fine-tuning as a current capability; there is no env-sensor ingestion in the training pipeline today.
|
||||
- **README: corrected the camera-supervised pose-accuracy claim** (audit finding #5; see PR #535) — "92.9% PCK@20" → the ADR-079 target (35%+; proxy baseline 35.3%), noting P7/P8/P9 are pending.
|
||||
|
||||
### Added
|
||||
- **`RollingP95` adaptive feature normalizer** (`v2/crates/wifi-densepose-sensing-server`) —
|
||||
Streaming P95 estimator (600-sample / ~30 s sliding window) that self-calibrates
|
||||
feature normalization to whatever distribution the deployment produces. Replaces
|
||||
fixed-scale denominators (`variance/300`, `motion/250`, `spectral/500`) which saturated
|
||||
when live ESP32 values exceeded those limits, collapsing dynamic range to zero.
|
||||
Cold-start (<60 samples) falls back to the legacy denominators so day-0 behaviour
|
||||
is preserved. Deployment-neutral: no hardcoded values. (ADR-044 §5.2)
|
||||
|
||||
- **`dedup_factor` runtime configuration API** (`v2/crates/wifi-densepose-sensing-server`) —
|
||||
Exposes the multi-node person-count deduplication divisor at runtime via REST:
|
||||
- `GET /api/v1/config/dedup-factor` — read current value.
|
||||
- `POST /api/v1/config/dedup-factor` — set value (clamped 1.0–10.0, persisted).
|
||||
- `POST /api/v1/config/ground-truth` — auto-tunes `dedup_factor` from a known
|
||||
person count (`{"count": N}`); derives optimal divisor from current node-sum.
|
||||
Config is persisted to `data/config.json` and reloaded on restart. (ADR-044 §5.3)
|
||||
|
||||
- **`nvsim` crate — deterministic NV-diamond magnetometer pipeline simulator** (ADR-089) —
|
||||
New standalone leaf crate at `v2/crates/nvsim` modeling a forward-only
|
||||
magnetic sensing path: scene → source synthesis (Biot–Savart, dipole,
|
||||
@@ -27,6 +197,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
saturation, hyperfine spectroscopy, or pulsed protocols become required.
|
||||
|
||||
### Fixed
|
||||
- **WebSocket broadcast handler now handles Lagged events gracefully and sends periodic ping keepalives to prevent dashboard disconnects** —
|
||||
`handle_ws_client` and `handle_ws_pose_client` in `wifi-densepose-sensing-server`
|
||||
were treating `RecvError::Lagged` as a fatal error, causing instant disconnect
|
||||
when clients fell behind the 256-frame broadcast buffer at 10 Hz ingest.
|
||||
Clients would reconnect, immediately lag again, and rapid-cycle every 2–4 s.
|
||||
`Lagged` now continues (drops missed frames, logs debug) rather than breaking.
|
||||
Added 30 s ping keepalive on the sensing handler to prevent proxy idle timeouts.
|
||||
- **Ghost skeletons in live UI with multi-node ESP32 setups** (#420, ADR-082) —
|
||||
`tracker_bridge::tracker_to_person_detections` documented itself as filtering
|
||||
to `is_alive()` tracks but in fact passed every non-Terminated track to the
|
||||
|
||||
@@ -14,15 +14,13 @@ Dual codebase: Python v1 (`v1/`) and Rust port (`v2/`).
|
||||
| `wifi-densepose-mat` | Mass Casualty Assessment Tool — disaster survivor detection |
|
||||
| `wifi-densepose-hardware` | ESP32 aggregator, TDM protocol, channel hopping firmware |
|
||||
| `wifi-densepose-ruvector` | RuVector v2.0.4 integration + cross-viewpoint fusion (5 modules) |
|
||||
| `wifi-densepose-api` | REST API (Axum) |
|
||||
| `wifi-densepose-db` | Database layer (Postgres, SQLite, Redis) |
|
||||
| `wifi-densepose-config` | Configuration management |
|
||||
| `wifi-densepose-wasm` | WebAssembly bindings for browser deployment |
|
||||
| `wifi-densepose-cli` | CLI tool (`wifi-densepose` binary) |
|
||||
| `wifi-densepose-sensing-server` | Lightweight Axum server for WiFi sensing UI |
|
||||
| `wifi-densepose-wifiscan` | Multi-BSSID WiFi scanning (ADR-022) |
|
||||
| `wifi-densepose-vitals` | ESP32 CSI-grade vital sign extraction (ADR-021) |
|
||||
| `nvsim` | Deterministic NV-diamond magnetometer pipeline simulator (ADR-089) — standalone leaf, WASM-ready |
|
||||
| `vendor/rvcsi` (submodule) | **rvCSI** — edge RF sensing runtime (ADR-095/096): 9 crates (`rvcsi-core`/`-dsp`/`-events`/`-adapter-file`/`-adapter-nexmon`/`-ruvector`/`-runtime`/`-node`/`-cli`). Lives in its own repo ([github.com/ruvnet/rvcsi](https://github.com/ruvnet/rvcsi)), vendored here under `vendor/rvcsi`, published to crates.io as `rvcsi-* 0.3.x` and to npm as `@ruv/rvcsi`. Not a `v2/` workspace member — depend on the published crates (or the submodule's `crates/rvcsi-*` paths). Normalized `CsiFrame`/`CsiWindow`/`CsiEvent` schema, validate-before-FFI, reusable DSP, typed confidence-scored events, the napi-c Nexmon shim (real nexmon_csi `.pcap` from a Raspberry Pi 5 / 4 / 3B+ — BCM43455c0), the napi-rs SDK, the `rvcsi` CLI, a Claude Code plugin. |
|
||||
|
||||
### RuvSense Modules (`signal/src/ruvsense/`)
|
||||
| Module | Purpose |
|
||||
@@ -134,17 +132,14 @@ Crates must be published in dependency order:
|
||||
2. `wifi-densepose-vitals` (no internal deps)
|
||||
3. `wifi-densepose-wifiscan` (no internal deps)
|
||||
4. `wifi-densepose-hardware` (no internal deps)
|
||||
5. `wifi-densepose-config` (no internal deps)
|
||||
6. `wifi-densepose-db` (no internal deps)
|
||||
7. `wifi-densepose-signal` (depends on core)
|
||||
8. `wifi-densepose-nn` (no internal deps, workspace only)
|
||||
9. `wifi-densepose-ruvector` (no internal deps, workspace only)
|
||||
10. `wifi-densepose-train` (depends on signal, nn)
|
||||
11. `wifi-densepose-mat` (depends on core, signal, nn)
|
||||
12. `wifi-densepose-api` (no internal deps)
|
||||
13. `wifi-densepose-wasm` (depends on mat)
|
||||
14. `wifi-densepose-sensing-server` (depends on wifiscan)
|
||||
15. `wifi-densepose-cli` (depends on mat)
|
||||
5. `wifi-densepose-signal` (depends on core)
|
||||
6. `wifi-densepose-nn` (no internal deps, workspace only)
|
||||
7. `wifi-densepose-ruvector` (no internal deps, workspace only)
|
||||
8. `wifi-densepose-train` (depends on signal, nn)
|
||||
9. `wifi-densepose-mat` (depends on core, signal, nn)
|
||||
10. `wifi-densepose-wasm` (depends on mat)
|
||||
11. `wifi-densepose-sensing-server` (depends on wifiscan)
|
||||
12. `wifi-densepose-cli` (depends on mat)
|
||||
|
||||
### Validation & Witness Verification (ADR-028)
|
||||
|
||||
|
||||
@@ -9,13 +9,13 @@
|
||||
> **Beta Software** — Under active development. APIs and firmware may change. Known limitations:
|
||||
> - ESP32-C3 and original ESP32 are not supported (single-core, insufficient for CSI DSP)
|
||||
> - Single ESP32 deployments have limited spatial resolution — use 2+ nodes or add a [Cognitum Seed](https://cognitum.one) for best results
|
||||
> - Camera-free pose accuracy is limited — use [camera ground-truth training](docs/adr/ADR-079-camera-ground-truth-training.md) for 92.9% PCK@20
|
||||
> - Camera-free pose accuracy is limited (PCK@20 ≈ 2.5% with proxy labels) — [camera ground-truth training](docs/adr/ADR-079-camera-ground-truth-training.md) targets **35%+ PCK@20**; the pipeline is implemented, but the data-collection and evaluation phases (ADR-079 P7–P9) are still pending, so no measured camera-supervised PCK@20 has been published yet
|
||||
>
|
||||
> Contributions and bug reports welcome at [Issues](https://github.com/ruvnet/RuView/issues).
|
||||
|
||||
## **See through walls with WiFi** ##
|
||||
|
||||
**Turn ordinary WiFi into a spacial intelligence / sensing system.** Detect people, measure breathing and heart rate, track movement, and monitor rooms — through walls, in the dark, with no cameras or wearables. Just physics.
|
||||
**Turn ordinary WiFi into a spatial intelligence / sensing system.** Detect people, measure breathing and heart rate, track movement, and monitor rooms — through walls, in the dark, with no cameras or wearables. Just physics.
|
||||
|
||||
### π RuView is a WiFi sensing platform that turns radio signals into spatial intelligence.
|
||||
|
||||
@@ -32,7 +32,7 @@ Built on [RuVector](https://github.com/ruvnet/ruvector/) and [Cognitum Seed](htt
|
||||
|
||||
The system learns each environment locally using spiking neural networks that adapt in under 30 seconds, with multi-frequency mesh scanning across 6 WiFi channels that uses your neighbors' routers as free radar illuminators. Every measurement is cryptographically attested via an Ed25519 witness chain.
|
||||
|
||||
RuView also supports pose estimation (17 COCO keypoints via the WiFlow architecture), trained entirely without cameras using 10 sensor signals — a technique pioneered from the original *DensePose From WiFi* research at Carnegie Mellon University.
|
||||
RuView **ships pretrained CSI weights on Hugging Face** at [`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained) — a self-supervised contrastive CSI encoder (128-dim embeddings, 12.2M training steps, 60K frames) + a presence-detection head reporting 100% accuracy on the validation set + per-node LoRA adapters. Models are released as `.safetensors`, 4-bit/8-bit/2-bit quantized `.bin` (4 KB–16 KB), and a JSONL RVF container. The Python training and evaluation tooling consumes these today via `safetensors`. **Pending wiring**: the sensing-server's `--model` flag still expects binary RVF, so live-server consumption of the JSONL bundle is gated on a JSONL adapter (or a re-publish in binary RVF) — see [Pretrained model on Hugging Face](#-pretrained-model-on-hugging-face) below for the workaround. **Not yet released**: a 17-keypoint pose-estimation model — training pipeline is implemented (WiFlow + AETHER + MERIDIAN heads) but camera-supervised fine-tune phases P7–P9 of [ADR-079](docs/adr/ADR-079-camera-supervised-pose-finetune.md) are `Pending`, tracked in [#509](https://github.com/ruvnet/RuView/issues/509). The live sensing server therefore drives the on-screen output from signal-based DSP heuristics today.
|
||||
|
||||
### Built for low-power edge applications
|
||||
|
||||
@@ -47,18 +47,26 @@ RuView also supports pose estimation (17 COCO keypoints via the WiFlow architect
|
||||
[](https://crates.io/crates/wifi-densepose-ruvector)
|
||||
|
||||
|
||||
> | What | How | Speed |
|
||||
> |------|-----|-------|
|
||||
> | 🦴 **Pose estimation** | CSI subcarrier amplitude/phase → 17 COCO keypoints | 171K emb/s (M4 Pro) |
|
||||
> | 🫁 **Breathing detection** | Bandpass 0.1-0.5 Hz → zero-crossing BPM | 6-30 BPM |
|
||||
> | 💓 **Heart rate** | Bandpass 0.8-2.0 Hz → zero-crossing BPM | 40-120 BPM |
|
||||
> | 👤 **Presence sensing** | Trained model + PIR fusion — 100% accuracy | 0.012 ms latency |
|
||||
> | 🧱 **Through-wall** | Fresnel zone geometry + multipath modeling | Up to 5m depth |
|
||||
> | 🧠 **Edge intelligence** | 8-dim feature vectors + RVF store on Cognitum Seed | $140 total BOM |
|
||||
> | 🎯 **Camera-free training** | 10 sensor signals, no labels needed | 84s on M4 Pro |
|
||||
> | 📷 **Camera-supervised training** | MediaPipe + ESP32 CSI → 92.9% PCK@20 | 19 min on laptop |
|
||||
> | 📡 **Multi-frequency mesh** | Channel hopping across 6 bands, neighbor APs as illuminators | 3x sensing bandwidth |
|
||||
> | 🌐 **3D point cloud** *(optional fusion)* | Camera depth (MiDaS) + WiFi CSI + mmWave radar → unified spatial model | 22 ms pipeline · 19K+ points/frame |
|
||||
> | What | Status | How | Speed |
|
||||
> |------|--------|-----|-------|
|
||||
> | 🫁 **Breathing rate** | ✅ Works today | Bandpass 0.1-0.5 Hz → zero-crossing BPM, circular variance on wrapped phase ([#593](https://github.com/ruvnet/RuView/issues/593)) | 6-30 BPM |
|
||||
> | 💓 **Heart rate** | ✅ Works today | Bandpass 0.8-2.0 Hz → zero-crossing BPM | 40-120 BPM (needs good SNR) |
|
||||
> | 👤 **Presence detection** | ✅ Heuristic in server · 🤗 Trained head on HF (loader wiring pending) | Live server uses phase-variance vs adaptive threshold (60 s ambient calibration). A trained `presence-head.json` reporting 100% validation accuracy is published in [`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained) but the sensing-server's `--model` loader only accepts binary RVF today — JSONL adapter pending. | <1 ms heuristic |
|
||||
> | 🧬 **CSI embeddings** | 🤗 Trained encoder on HF | 128-dim contrastive encoder, **164,183 emb/s** on M4 Pro. Usable today from Python / training via `model.safetensors`; sensing-server consumption pending the same JSONL loader gap as above. | 8 KB q4 fits ESP32 SRAM |
|
||||
> | 🚶 **Motion / activity** | ✅ Works today | Motion-band power + phase acceleration | Real-time |
|
||||
> | 🤸 **Fall detection** | ✅ Works today | Phase acceleration > threshold + 3-frame debounce + 5 s cooldown ([#263](https://github.com/ruvnet/RuView/issues/263)) | < 200 ms |
|
||||
> | 🧮 **Multi-person slot count** | ⚠️ Heuristic, not learned | Subcarrier diversity divided by 2 (capped). **Not** a learned counter — see [firmware README](firmware/esp32-csi-node/README.md#tier-2--full-pipeline-stable) "Tier 2 caveats". Adaptive normalisation in [#491](https://github.com/ruvnet/RuView/pull/491). | Real-time |
|
||||
> | 🦴 **17-keypoint pose estimation** | 🔬 Pipeline only, no shipped weights | Training infrastructure complete (WiFlow + AETHER + MERIDIAN heads); the published HF model is presence + embeddings, not keypoints. Tracked in [#509](https://github.com/ruvnet/RuView/issues/509). | Pending data collection |
|
||||
> | 🧱 **Through-wall sensing** | ✅ Works today | Fresnel zone geometry + multipath modeling | Up to ~5m signal-dependent |
|
||||
> | 🧠 **Edge intelligence** | ✅ Works today | Optional Cognitum Seed for persistent vector store + kNN + witness chain | $140 total BOM |
|
||||
> | 🎯 **Camera-free pre-training** | ✅ Shipped weights on HF | Self-supervised contrastive encoder, 12.2M training steps on 60K frames. See [`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained). | 84 s/epoch retrain on M4 Pro |
|
||||
> | 📷 **Camera-supervised fine-tune** | 🔬 Pipeline only | MediaPipe + ESP32 CSI paired training, [ADR-079](docs/adr/ADR-079-camera-supervised-pose-finetune.md). Target **35%+ PCK@20**. P7–P9 (data + train + eval) `Pending`. | ~19 min/epoch on laptop |
|
||||
> | 📡 **Multi-frequency mesh** | ✅ Works today | Channel hopping across 6 bands, TDM slot scheduling (ADR-029) | 3x sensing bandwidth |
|
||||
> | 🌐 **3D point cloud fusion** | 🔬 Reference impl | Camera depth (MiDaS) + WiFi CSI + mmWave radar → unified spatial model. Requires camera. | 22 ms pipeline · 19K+ points/frame |
|
||||
>
|
||||
> Legend: ✅ shipped + tested on hardware (some have learned weights on [HF](https://huggingface.co/ruvnet/wifi-densepose-pretrained), others are deterministic DSP) · ⚠️ ships and runs, but is a heuristic/threshold (not a learned classifier) — accuracy depends on calibration · 🔬 implementation + tests in repo, weights/data/eval pending
|
||||
>
|
||||
> 🤗 **Pretrained weights**: download from [`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained) — see [Loading the pretrained model](#loading-the-pretrained-model) below for one-command setup.
|
||||
|
||||
```bash
|
||||
# Option 1: Docker (simulated data, no hardware needed)
|
||||
@@ -88,10 +96,10 @@ node scripts/mincut-person-counter.js --port 5006 # Correct person counting
|
||||
>
|
||||
> | Option | Hardware | Cost | Full CSI | Capabilities |
|
||||
> |--------|----------|------|----------|-------------|
|
||||
> | **ESP32 + Cognitum Seed** (recommended) | ESP32-S3 + [Cognitum Seed](https://cognitum.one) | ~$140 | Yes | Pose, breathing, heartbeat, motion, presence + persistent vector store, kNN search, witness chain, MCP proxy |
|
||||
> | **ESP32 Mesh** | 3-6x ESP32-S3 + WiFi router | ~$54 | Yes | Pose, breathing, heartbeat, motion, presence |
|
||||
> | **ESP32 + Cognitum Seed** (recommended) | ESP32-S3 + [Cognitum Seed](https://cognitum.one) | ~$140 | Yes | Presence indicator, motion, breathing rate, heart rate, fall detection, slot-count multi-person heuristic + persistent vector store, kNN search, witness chain, MCP proxy. (Pose pending weights — see [#509](https://github.com/ruvnet/RuView/issues/509).) |
|
||||
> | **ESP32 Mesh** | 3-6x ESP32-S3 + WiFi router | ~$54 | Yes | Same capabilities as above without the persistent-memory features |
|
||||
> | **Research NIC** | Intel 5300 / Atheros AR9580 | ~$50-100 | Yes | Full CSI with 3x3 MIMO |
|
||||
> | **Any WiFi** | Windows, macOS, or Linux laptop | $0 | No | RSSI-only: coarse presence and motion |
|
||||
> | **Any WiFi** | Windows, macOS, or Linux laptop | $0 | No | RSSI-only: coarse presence and motion (see [tutorial #36](https://github.com/ruvnet/RuView/issues/36)) |
|
||||
>
|
||||
> No hardware? Verify the signal processing pipeline with the deterministic reference signal: `python archive/v1/data/proof/verify.py`
|
||||
>
|
||||
@@ -115,6 +123,31 @@ node scripts/mincut-person-counter.js --port 5006 # Correct person counting
|
||||
> **Live ESP32 pipeline**: Connect an ESP32-S3 node → run the [sensing server](#sensing-server) → open the [pose fusion demo](https://ruvnet.github.io/RuView/pose-fusion.html) for real-time dual-modal pose estimation (webcam + WiFi CSI). See [ADR-059](docs/adr/ADR-059-live-esp32-csi-pipeline.md).
|
||||
|
||||
|
||||
## 🤗 Pretrained model on Hugging Face
|
||||
|
||||
Pretrained CSI weights live at [`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained) — 12.2M training steps on 60K frames / 610K contrastive triplets, **100% presence accuracy** on the validation set, 4-bit quantized variant fits in 8 KB. The release includes a contrastive **CSI encoder** producing 128-dim embeddings (164,183 emb/s on M4 Pro) and a **presence-detection head**. Per-node LoRA adapters are included for environment-specific fine-tuning.
|
||||
|
||||
```bash
|
||||
# Download the model bundle
|
||||
pip install huggingface_hub
|
||||
huggingface-cli download ruvnet/wifi-densepose-pretrained --local-dir models/wifi-densepose-pretrained
|
||||
```
|
||||
|
||||
**What works today vs. what's pending wiring:**
|
||||
|
||||
| Consumer | Format used | Status |
|
||||
|----------|-------------|--------|
|
||||
| Python training / evaluation / embedding extraction | `model.safetensors` | ✅ Works — load with `safetensors.torch.load_file` |
|
||||
| Inspect / re-export the bundle | `model.rvf.jsonl` (line-by-line JSON) | ✅ Works — plain JSONL |
|
||||
| Sensing-server `--model <PATH>` flag | binary RVF (`RVFS` magic) | ⚠️ Loader does not yet accept the JSONL container |
|
||||
|
||||
**Known gap:** the HF model ships in JSONL RVF format, but `v2/crates/wifi-densepose-sensing-server/src/rvf_container.rs` only parses the binary RVF segment format. Pointing `--model` at `model.rvf.jsonl` currently errors with `invalid magic at offset 0: expected 0x52564653, got 0x7974227B` and the live pipeline degrades to null output rather than falling back to heuristic mode — so for the live sensing-server, run **without** `--model` until a JSONL adapter lands (or the model is re-published as binary RVF). Use the weights from Python / training in the meantime.
|
||||
|
||||
**Quantization choices** (all in the HF repo): `model-q2.bin` (4 KB) · `model-q4.bin` ⭐ recommended (8 KB) · `model-q8.bin` (16 KB) · `model.safetensors` full (48 KB)
|
||||
|
||||
The separate **17-keypoint pose-estimation model** is not in this release — pipeline is implemented but keypoint weights are still pending. Tracked in [#509](https://github.com/ruvnet/RuView/issues/509); see [ADR-079](docs/adr/ADR-079-camera-supervised-pose-finetune.md) phases P7–P9.
|
||||
|
||||
|
||||
## 🔬 How It Works
|
||||
|
||||
WiFi routers flood every room with radio waves. When a person moves — or even breathes — those waves scatter differently. WiFi DensePose reads that scattering pattern and reconstructs what happened:
|
||||
@@ -485,14 +518,44 @@ See [`docs/adr/ADR-024-contrastive-csi-embedding-model.md`](docs/adr/ADR-024-con
|
||||
|
||||
---
|
||||
|
||||
## 🧩 Claude Code & Codex Plugin
|
||||
|
||||
RuView ships a [Claude Code](https://docs.anthropic.com/en/docs/claude-code) plugin (and Codex prompt mirror) that wraps the whole workflow — onboarding, ESP32 setup, configuration, sensing apps, model training, advanced multistatic sensing, CLI/API/WASM, mmWave radar, and witness verification — as 9 skills, 7 `/ruview-*` commands, and 3 agents. It lives in [`plugins/ruview/`](plugins/ruview/README.md); the marketplace manifest is [`.claude-plugin/marketplace.json`](.claude-plugin/marketplace.json) at the repo root.
|
||||
|
||||
```bash
|
||||
# In Claude Code — add this repo as a plugin marketplace, then install:
|
||||
/plugin marketplace add ruvnet/RuView
|
||||
/plugin install ruview@ruview
|
||||
|
||||
# Or try it for one session without installing (from a local clone of the repo):
|
||||
claude --plugin-dir ./plugins/ruview
|
||||
|
||||
# Then, in Claude Code:
|
||||
# /ruview-start → onboarding (Docker demo / repo build / live ESP32)
|
||||
# /ruview-flash → build + flash ESP32 firmware
|
||||
# /ruview-provision → provision WiFi creds, sink IP, channel/MAC, mesh slots
|
||||
# /ruview-app → run a sensing application (presence / vitals / pose / sleep / MAT / point cloud)
|
||||
# /ruview-train → train / evaluate / publish a model (incl. GPU on GCloud)
|
||||
# /ruview-advanced → multistatic / tomography / cross-viewpoint / mesh-security
|
||||
# /ruview-verify → tests + deterministic proof + witness bundle
|
||||
```
|
||||
|
||||
**Codex (OpenAI CLI):** `cp plugins/ruview/codex/prompts/*.md ~/.codex/prompts/` — the seven `/ruview-*` commands are mirrored as Codex prompts; [`plugins/ruview/codex/AGENTS.md`](plugins/ruview/codex/AGENTS.md) carries the project rules. See [`plugins/ruview/codex/README.md`](plugins/ruview/codex/README.md).
|
||||
|
||||
Verify the plugin structure: `bash plugins/ruview/scripts/smoke.sh`. Full details: [`plugins/ruview/README.md`](plugins/ruview/README.md).
|
||||
|
||||
---
|
||||
|
||||
## 📖 Documentation
|
||||
|
||||
| Document | Description |
|
||||
|----------|-------------|
|
||||
| [User Guide](docs/user-guide.md) | Step-by-step guide: installation, first run, API usage, hardware setup, training |
|
||||
| [Build Guide](docs/build-guide.md) | Building from source (Rust and Python) |
|
||||
| [Architecture Decisions](docs/adr/README.md) | 79 ADRs — why each technical choice was made, organized by domain (hardware, signal processing, ML, platform, infrastructure) |
|
||||
| [Domain Models](docs/ddd/README.md) | 7 DDD models (RuvSense, Signal Processing, Training Pipeline, Hardware Platform, Sensing Server, WiFi-Mat, CHCI) — bounded contexts, aggregates, domain events, and ubiquitous language |
|
||||
| [Claude Code / Codex Plugin](plugins/ruview/README.md) | The `ruview` plugin + marketplace — skills, `/ruview-*` commands, agents, and the Codex prompt mirror |
|
||||
| [Architecture Decisions](docs/adr/README.md) | 96 ADRs — why each technical choice was made, organized by domain (hardware, signal processing, ML, platform, infrastructure) |
|
||||
| [Domain Models](docs/ddd/README.md) | 8 DDD models (RuvSense, Signal Processing, Training Pipeline, Hardware Platform, Sensing Server, WiFi-Mat, CHCI, rvCSI) — bounded contexts, aggregates, domain events, and ubiquitous language |
|
||||
| [rvCSI — edge RF sensing runtime](https://github.com/ruvnet/rvcsi) | Rust-first / TypeScript-accessible / hardware-abstracted CSI runtime: multi-source ingestion (incl. real nexmon_csi `.pcap` from a **Raspberry Pi 5** / Pi 4 / Pi 3B+ — CYW43455 / BCM43455c0) → validation → DSP → typed events → RuVector RF memory ([ADR-095](docs/adr/ADR-095-rvcsi-edge-rf-sensing-platform.md), [ADR-096](docs/adr/ADR-096-rvcsi-ffi-crate-layout.md), [domain model](docs/ddd/rvcsi-domain-model.md)). Now its own repo — [`ruvnet/rvcsi`](https://github.com/ruvnet/rvcsi) — vendored here under `vendor/rvcsi`; 9 `rvcsi-*` crates on crates.io, `@ruv/rvcsi` on npm, plus a Claude Code plugin. |
|
||||
| [Desktop App](v2/crates/wifi-densepose-desktop/README.md) | **WIP** — Tauri v2 desktop app for node management, OTA updates, WASM deployment, and mesh visualization |
|
||||
| [Medical Examples](examples/medical/README.md) | Contactless blood pressure, heart rate, breathing rate via 60 GHz mmWave radar — $15 hardware, no wearable |
|
||||
| [Extended Documentation](docs/readme-details.md) | Latest additions, key features, installation, quick start, signal processing, training, CLI, testing, deployment, and changelog |
|
||||
|
||||
@@ -1 +1 @@
|
||||
8c0680d7d285739ea9597715e84959d9c356c87ee3ad35b5f1e69a4ca41151c6
|
||||
667eb054c44ac510342665bf9c93d608868a8ead948ae8774b2796ebce6f8fe7
|
||||
@@ -164,18 +164,44 @@ def frame_to_csi_data(frame, signal_meta):
|
||||
)
|
||||
|
||||
|
||||
# Quantization precision for cross-platform hash stability (issue #560).
|
||||
#
|
||||
# The bytes packed below feed SHA-256. Without quantization, the hash diverges
|
||||
# across SIMD backends (Intel AVX2/AVX-512 vs ARM NEON vs different x86 micro-
|
||||
# architectures in the same CI pool) because scipy.fft's pocketfft kernels
|
||||
# reorder vectorized FP operations differently per build. IEEE 754 guarantees
|
||||
# per-operation determinism, not associativity under reordering.
|
||||
#
|
||||
# Empirically: 9 decimals was NOT enough to collapse the divergence — two
|
||||
# back-to-back Ubuntu 24.04 / Python 3.11 / scipy 1.17 CI runs landed on
|
||||
# different Azure VM microarchitectures (likely Skylake vs Cascade Lake)
|
||||
# and produced two different SHA-256s even after np.round(.., 9). The DSP
|
||||
# pipeline (preprocess → biquad bandpass → FFT → PSD → variance accumulation)
|
||||
# amplifies the ~1e-14 raw FFT divergence by several orders of magnitude
|
||||
# downstream — the actual drift at features_to_bytes() input can reach 1e-7
|
||||
# or worse.
|
||||
#
|
||||
# 6 decimals (parts per million) gives ~6 orders of magnitude headroom over
|
||||
# observed pipeline-amplified ULP drift and is still far below any meaningful
|
||||
# signal change (CSI phase precision is ~1e-3 rad; PSD bins differ by orders
|
||||
# of magnitude). Round to this precision, then hash.
|
||||
HASH_QUANTIZATION_DECIMALS = 6
|
||||
|
||||
|
||||
def features_to_bytes(features):
|
||||
"""Convert CSIFeatures to a deterministic byte representation.
|
||||
|
||||
We serialize each numpy array to bytes in a canonical order
|
||||
using little-endian float64 representation. This ensures the
|
||||
hash is platform-independent for IEEE 754 compliant systems.
|
||||
Each feature array is quantized to ``HASH_QUANTIZATION_DECIMALS`` decimal
|
||||
places before being packed as little-endian float64. The quantization is
|
||||
what makes the resulting SHA-256 hash actually platform-independent — the
|
||||
raw float values diverge at ULP precision across scipy.fft SIMD backends
|
||||
(issue #560), even though all platforms compute the "correct" answer.
|
||||
|
||||
Args:
|
||||
features: CSIFeatures instance.
|
||||
|
||||
Returns:
|
||||
bytes: Canonical byte representation.
|
||||
bytes: Canonical, quantized byte representation.
|
||||
"""
|
||||
parts = []
|
||||
|
||||
@@ -189,6 +215,10 @@ def features_to_bytes(features):
|
||||
features.power_spectral_density,
|
||||
]:
|
||||
flat = np.asarray(array, dtype=np.float64).ravel()
|
||||
# Quantize before packing so SIMD-level FP reordering across
|
||||
# Intel AVX vs Apple Silicon NEON pocketfft kernels does not
|
||||
# leak into the SHA-256 input.
|
||||
flat = np.round(flat, HASH_QUANTIZATION_DECIMALS)
|
||||
# Pack as little-endian double (8 bytes each)
|
||||
parts.append(struct.pack(f"<{len(flat)}d", *flat))
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ from datetime import datetime, timedelta
|
||||
|
||||
from fastapi import Request, Response, HTTPException, status
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from jose import JWTError, jwt
|
||||
from passlib.context import CryptContext
|
||||
|
||||
@@ -155,16 +156,17 @@ class UserManager:
|
||||
return False
|
||||
|
||||
|
||||
class AuthenticationMiddleware:
|
||||
class AuthenticationMiddleware(BaseHTTPMiddleware):
|
||||
"""Authentication middleware for FastAPI."""
|
||||
|
||||
def __init__(self, settings: Settings):
|
||||
|
||||
def __init__(self, app, settings: Settings):
|
||||
super().__init__(app)
|
||||
self.settings = settings
|
||||
self.token_manager = TokenManager(settings)
|
||||
self.user_manager = UserManager()
|
||||
self.enabled = settings.enable_authentication
|
||||
|
||||
async def __call__(self, request: Request, call_next: Callable) -> Response:
|
||||
|
||||
async def dispatch(self, request: Request, call_next: Callable) -> Response:
|
||||
"""Process request through authentication middleware."""
|
||||
start_time = time.time()
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ from collections import defaultdict, deque
|
||||
from dataclasses import dataclass
|
||||
|
||||
from fastapi import Request, Response, HTTPException, status
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.types import ASGIApp
|
||||
|
||||
from src.config.settings import Settings
|
||||
@@ -299,15 +300,16 @@ class RateLimiter:
|
||||
}
|
||||
|
||||
|
||||
class RateLimitMiddleware:
|
||||
class RateLimitMiddleware(BaseHTTPMiddleware):
|
||||
"""Rate limiting middleware for FastAPI."""
|
||||
|
||||
def __init__(self, settings: Settings):
|
||||
|
||||
def __init__(self, app, settings: Settings):
|
||||
super().__init__(app)
|
||||
self.settings = settings
|
||||
self.rate_limiter = RateLimiter(settings)
|
||||
self.enabled = settings.enable_rate_limiting
|
||||
|
||||
async def __call__(self, request: Request, call_next: Callable) -> Response:
|
||||
|
||||
async def dispatch(self, request: Request, call_next: Callable) -> Response:
|
||||
"""Process request through rate limiting middleware."""
|
||||
if not self.enabled:
|
||||
return await call_next(request)
|
||||
|
||||
@@ -220,7 +220,11 @@ class PoseService:
|
||||
# Apply phase sanitization if we have phase data
|
||||
if hasattr(detection_result.features, 'phase_difference'):
|
||||
phase_data = detection_result.features.phase_difference
|
||||
sanitized_phase = self.phase_sanitizer.sanitize(phase_data)
|
||||
# PhaseSanitizer's full-pipeline method is sanitize_phase,
|
||||
# not sanitize (issue #612). The shorter name was an
|
||||
# AttributeError waiting to fire on any code path that
|
||||
# reaches this branch.
|
||||
sanitized_phase = self.phase_sanitizer.sanitize_phase(phase_data)
|
||||
# Combine amplitude and phase data
|
||||
return np.concatenate([amplitude_data, sanitized_phase])
|
||||
|
||||
|
||||
@@ -33,6 +33,25 @@ COPY --from=builder /build/target/release/sensing-server /app/sensing-server
|
||||
# Copy UI assets
|
||||
COPY ui/ /app/ui/
|
||||
|
||||
# Sanity-check the assets the runtime actually serves (regression guard for
|
||||
# #520/#514 — the published image must include the observatory and pose-fusion
|
||||
# dashboards, not just the legacy `index.html` set). Build fails if any of
|
||||
# these are missing, so a stale image can't be silently pushed.
|
||||
RUN set -e; \
|
||||
for f in /app/ui/index.html /app/ui/observatory.html /app/ui/pose-fusion.html /app/ui/viz.html; do \
|
||||
test -f "$f" || { echo "FATAL: missing UI asset $f"; exit 1; }; \
|
||||
done; \
|
||||
for d in /app/ui/observatory /app/ui/pose-fusion /app/ui/components /app/ui/services; do \
|
||||
test -d "$d" || { echo "FATAL: missing UI directory $d"; exit 1; }; \
|
||||
done; \
|
||||
test -x /app/sensing-server || { echo "FATAL: /app/sensing-server is not executable"; exit 1; }; \
|
||||
echo "image assets OK"
|
||||
|
||||
# Optional bearer-token auth on /api/v1/*: leave unset for LAN-mode (default),
|
||||
# set to enforce `Authorization: Bearer <token>` (see bearer_auth module, #443).
|
||||
# docker run -e RUVIEW_API_TOKEN=$(openssl rand -hex 32) ...
|
||||
ENV RUVIEW_API_TOKEN=
|
||||
|
||||
# HTTP API
|
||||
EXPOSE 3000
|
||||
# WebSocket
|
||||
|
||||
@@ -9,7 +9,18 @@ services:
|
||||
ports:
|
||||
- "3000:3000" # REST API
|
||||
- "3001:3001" # WebSocket
|
||||
- "5005:5005/udp" # ESP32 UDP
|
||||
# ESP32 UDP. On Linux/macOS this works with multiple ESP32 nodes out of
|
||||
# the box. On Docker Desktop for Windows, multi-source UDP is collapsed
|
||||
# to one source IP at the WSL/Hyper-V boundary, so all-but-one node's
|
||||
# frames are silently dropped (issue #374, #386).
|
||||
#
|
||||
# Windows workaround: change this to "5006:5005/udp" and run the host
|
||||
# relay so every datagram arrives from the same loopback source:
|
||||
#
|
||||
# python scripts/udp-relay.py --listen-port 5005 --forward-port 5006
|
||||
#
|
||||
# See docs/TROUBLESHOOTING.md §9 for details.
|
||||
- "5005:5005/udp"
|
||||
environment:
|
||||
- RUST_LOG=info
|
||||
# CSI_SOURCE controls the data source for the sensing server.
|
||||
|
||||
@@ -109,3 +109,75 @@ ssh thyhack@100.90.238.87
|
||||
**Symptom:** Plugging into the right USB-C port (when facing the board with USB-C toward you) shows no serial device on the host.
|
||||
|
||||
**Fix:** Use the left USB-C port. On most ESP32-S3-DevKitC boards, the left port is the USB-to-UART bridge (CP2102/CH340) used for flashing and serial monitor. The right port is the native USB (USB-JTAG) which requires different drivers and isn't used by the RuView firmware.
|
||||
|
||||
---
|
||||
|
||||
## 9. Docker Desktop on Windows drops UDP from multiple ESP32 nodes
|
||||
|
||||
**Symptom:** Two or more ESP32 nodes are flashed, provisioned, and visibly transmit on the network — `tcpdump`/Wireshark on the Windows host shows datagrams from every node — but inside the Docker container only one source IP arrives. `/api/v1/sensing/latest` shows a single node and the live UI freezes or only tracks one body. Reported in #374 (4-node bench) and reproduced in #386 (6-node demo, RuView v0.7.0).
|
||||
|
||||
**Root cause:** Docker Desktop on Windows runs the engine inside a WSL2 / Hyper-V VM. Inbound UDP from the host LAN is forwarded through `vpnkit` / `vEthernet` and the multi-source-IP datagrams are demultiplexed onto a single virtual socket. The first source-IP "wins"; subsequent unique sources are silently dropped at the VM boundary. This is a Docker Desktop limitation, not a sensing-server bug — `host.docker.internal` and `--network host` do not help (host networking is not implemented for the Linux engine on Windows).
|
||||
|
||||
**Fix:** Run the bundled UDP relay on the host so every forwarded datagram arrives from the same loopback source IP, which Docker passes through unchanged.
|
||||
|
||||
```powershell
|
||||
# 1. Start the relay (PowerShell or any terminal)
|
||||
python scripts/udp-relay.py --listen-port 5005 --forward-port 5006
|
||||
|
||||
# 2. Edit docker/docker-compose.yml — change the ESP32 UDP mapping from
|
||||
# - "5005:5005/udp"
|
||||
# to
|
||||
# - "5006:5005/udp"
|
||||
|
||||
# 3. Bring the stack up
|
||||
docker compose -f docker/docker-compose.yml up
|
||||
```
|
||||
|
||||
ESP32 nodes still target the host on `--target-ip <host>:5005` — no firmware re-provisioning is needed. The relay is `scripts/udp-relay.py` (stdlib only, no extra deps). Verify with `--verbose` that each node's source IP appears at least once before forwarding stabilises on a single ephemeral relay port.
|
||||
|
||||
**Prevention:** Linux and macOS hosts are unaffected; the relay only needs to run on Docker Desktop for Windows. If Docker Desktop ships per-source UDP forwarding (tracked at [docker/for-win#1144](https://github.com/docker/for-win/issues/1144) and related), this workaround can be retired.
|
||||
|
||||
**Prior art:** PR #413 (`txhno`) proposed a docs-only writeup of the same workaround; this entry supersedes it.
|
||||
|
||||
---
|
||||
|
||||
## 10. `404` on the visualization page when running sensing-server
|
||||
|
||||
**Symptom:** `sensing-server` starts cleanly, logs `HTTP server listening on http://localhost:3000`, but loading `http://localhost:3000/` (or `/ui/index.html`) returns `404 Not Found`. Reported in #188.
|
||||
|
||||
**Root cause:** The default `--ui-path ../../ui` is resolved relative to the binary's *current working directory*, not the binary location. When the binary is launched from anywhere other than `crates/wifi-densepose-sensing-server/`, the relative path doesn't reach the UI assets and Axum's static file handler returns 404.
|
||||
|
||||
**Fix:** Pass an absolute UI path, run the binary from the crate directory, or use the Docker image (which bundles the UI under `/app/ui`).
|
||||
|
||||
```bash
|
||||
# Option A — absolute path (recommended for production)
|
||||
sensing-server --source esp32 --udp-port 5005 --http-port 3000 \
|
||||
--ws-port 3001 --ui-path /absolute/path/to/ui
|
||||
|
||||
# Option B — run from the crate dir (works for local dev / cargo run)
|
||||
cd v2/crates/wifi-densepose-sensing-server
|
||||
cargo run -- --source esp32
|
||||
|
||||
# Option C — Docker (no path config needed)
|
||||
docker compose -f docker/docker-compose.yml up sensing-server
|
||||
```
|
||||
|
||||
**Prevention:** Track future work in #188 to fall back to a path resolved relative to the executable when the cwd-relative path doesn't exist, so the binary works regardless of where it's launched.
|
||||
|
||||
---
|
||||
|
||||
## 11. Boot loop on `--edge-tier 1` or `--edge-tier 2`
|
||||
|
||||
**Symptom:** ESP32-S3 boots normally with `--edge-tier 0`, but flashing the same firmware with `--edge-tier 1` or `2` produces a boot loop. Serial output reaches `cpu_start` and `heap_init`, then resets repeatedly. Reported in #438 against firmware `v0.4.3.1-esp32-3-g66e2fa083-dir`.
|
||||
|
||||
**Root cause:** Edge tiers 1 and 2 enable the on-device DSP pipeline on Core 1. In the affected build, the `edge_dsp` task ran a tight per-frame loop without yielding, so the FreeRTOS task watchdog tripped on Core 1 and panicked. Tier 0 is passthrough only and doesn't activate the pipeline, so the watchdog never fires there.
|
||||
|
||||
**Fix:** Flash the [v0.4.3.1-esp32](https://github.com/ruvnet/RuView/releases/tag/v0.4.3.1-esp32) release or later — the DSP task yield fixes have shipped on `main` since the build in the report.
|
||||
|
||||
```bash
|
||||
# Verify what version you're on (look for "App version" in serial output on boot)
|
||||
python -m serial.tools.miniterm COM7 115200
|
||||
# Expect: "App version: v0.4.3.1-esp32" or higher
|
||||
```
|
||||
|
||||
If the boot loop persists on a release build, capture a full serial trace including the watchdog backtrace and reopen #438 with the new build hash.
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
# ADR-095: rvCSI — Edge RF Sensing Runtime Platform
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Proposed |
|
||||
| **Date** | 2026-05-12 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codename** | **rvCSI** — RuVector Channel State Information runtime |
|
||||
| **Relates to** | ADR-012 (ESP32 CSI mesh), ADR-013 (feature-level sensing on commodity gear), ADR-014 (SOTA signal processing), ADR-016 (RuVector integration), ADR-024 (AETHER contrastive embeddings), ADR-031 (RuView sensing-first RF mode), ADR-040 (WASM programmable sensing), ADR-049 (cross-platform WiFi interface detection) |
|
||||
| **PRD** | [rvCSI Platform PRD](../prd/rvcsi-platform-prd.md) |
|
||||
| **Domain model** | [rvCSI Domain Model](../ddd/rvcsi-domain-model.md) |
|
||||
|
||||
---
|
||||
|
||||
## 1. Context
|
||||
|
||||
WiFi Channel State Information (CSI) is a powerful camera-free sensing primitive — but in practice it is hard to operationalize. Most CSI pipelines today are Linux shell scripts, patched firmware, kernel modules, Python notebooks, PCAP dumps, and ad-hoc signal processing. Packet formats are inconsistent across chips; drivers are unstable; malformed packets are common; and device-specific assumptions leak everywhere. CSI works in the lab and falls over in the field.
|
||||
|
||||
RuView already contains substantial CSI infrastructure (`wifi-densepose-signal`, `wifi-densepose-ruvector`, the ESP32 mesh of ADR-012, the RuView multistatic work of ADR-031). What is missing is a **stable, hardware-abstracted runtime layer** that:
|
||||
|
||||
- ingests CSI from many sources behind one interface,
|
||||
- validates every packet before it can touch application code,
|
||||
- normalizes everything into one schema,
|
||||
- runs reusable signal processing,
|
||||
- emits typed, confidence-scored events,
|
||||
- exposes a safe TypeScript SDK, a CLI, MCP tools, and a RuVector bridge,
|
||||
- and runs unattended on Raspberry Pi-class hardware.
|
||||
|
||||
This ADR establishes that runtime — **rvCSI** — and the architectural decisions that constrain it. Detailed requirements are in the [PRD](../prd/rvcsi-platform-prd.md); the bounded contexts, aggregates, and ubiquitous language are in the [domain model](../ddd/rvcsi-domain-model.md).
|
||||
|
||||
### 1.1 What rvCSI is not (day one)
|
||||
|
||||
rvCSI is *not* a pure-Rust replacement for vendor firmware patches, *not* a universal driver for all WiFi chips, and *not* an identity/pose/medical/legal-grade claim. It is a **structural sensing** runtime: excellent at detecting change, presence, motion, drift, and learned patterns; deliberately silent on exact identity, exact pose, and certainty guarantees. The product surface stays inside that boundary (see Decision D7).
|
||||
|
||||
### 1.2 Existing assets rvCSI builds on
|
||||
|
||||
| Asset | Source | Reuse in rvCSI |
|
||||
|-------|--------|----------------|
|
||||
| SOTA DSP (Hampel, phase unwrap, Fresnel, BVP, spectrograms) | `wifi-densepose-signal` (ADR-014) | `rvcsi-dsp` wraps/extends rather than re-implements |
|
||||
| RuVector integration (5 crates) | `wifi-densepose-ruvector` (ADR-016) | `rvcsi-ruvector` exporter rides on the existing integration |
|
||||
| ESP32 CSI firmware + aggregator | `wifi-densepose-hardware` / firmware (ADR-012) | `rvcsi-adapter-esp32` consumes the existing serial/UDP stream |
|
||||
| AETHER contrastive embeddings | ADR-024 | optional embedding backend for window/event vectors |
|
||||
| Cross-platform interface detection | ADR-049 | adapter discovery / health checks |
|
||||
|
||||
---
|
||||
|
||||
## 2. Decision
|
||||
|
||||
**Adopt rvCSI as a layered edge RF sensing runtime** with the boundary discipline `C → Rust → TypeScript`, a single normalized `CsiFrame` schema, mandatory validation before any language boundary crossing, and RuVector as RF memory. The fifteen decisions below are the architectural contract.
|
||||
|
||||
### D1 — Rust is the core runtime
|
||||
|
||||
CSI parsing and DSP require memory safety, predictable latency, and high throughput; C/Python research stacks are fragile for unattended edge deployment. **rvCSI uses Rust** for parsing, validation, signal processing, event extraction, and daemon execution.
|
||||
*Consequences:* safer packet handling; better long-running stability; stronger portability to edge devices; more complex build system than pure TypeScript.
|
||||
|
||||
### D2 — C only at the hardware-compatibility boundary
|
||||
|
||||
Nexmon and similar CSI sources often require C shims, legacy drivers, or firmware-patch hooks. **C is isolated to thin shims** for existing capture and firmware compatibility — never in the data path beyond decode.
|
||||
*Consequences:* existing Nexmon capability reused; unsafe surface stays small; full firmware rewrite avoided; some device support stays dependent on upstream tools.
|
||||
|
||||
### D3 — TypeScript for SDK, CLI, and developer orchestration
|
||||
|
||||
Developers need an approachable SDK, agent integrations, dashboards, and scripts. **rvCSI exposes a first-class TypeScript SDK** (`@ruv/rvcsi`) and CLI; native performance stays in Rust.
|
||||
*Consequences:* easy adoption by app/agent developers; native perf preserved; requires a native build + prebuild release pipeline.
|
||||
|
||||
### D4 — napi-rs for Node bindings
|
||||
|
||||
Native Node modules need a stable ABI and ergonomic Rust integration. **rvCSI uses napi-rs** for the `rvcsi-node` bindings.
|
||||
*Consequences:* Rust exposes typed APIs to TypeScript; prebuilt binaries distributable; careful memory-ownership rules required.
|
||||
|
||||
### D5 — Normalize all sources into one `CsiFrame` / `CsiWindow` schema
|
||||
|
||||
Different CSI sources expose incompatible formats; application code must not know device-specific details. **Every source is normalized into `CsiFrame` and `CsiWindow`** (schema in the domain model).
|
||||
*Consequences:* hardware-agnostic application code; easier RuVector integration; some source-specific metadata needs extension fields.
|
||||
|
||||
### D6 — Validate before crossing language boundaries
|
||||
|
||||
Malformed packets and unsafe pointers are the dominant stability risk. **All raw data is validated in Rust before it crosses into TypeScript or RuVector**; rejected frames are quarantined (when enabled); parser failures return structured errors; TypeScript never receives raw unchecked pointers.
|
||||
*Consequences:* safer SDK; cleaner error model; small validation overhead.
|
||||
|
||||
### D7 — Treat CSI as a temporal delta, not absolute truth
|
||||
|
||||
CSI is noisy and environment-specific. **rvCSI frames CSI as a temporal delta stream against learned baselines**, not as exact vision.
|
||||
*Consequences:* honest product claims; good fit for presence/motion/drift/anomaly; identity and exact pose excluded from core claims.
|
||||
|
||||
### D8 — RuVector is RF memory
|
||||
|
||||
CSI becomes far more valuable stored as temporal embeddings and room signatures. **rvCSI integrates with RuVector** for vector storage, similarity search, drift detection, and sensor-graph relationships.
|
||||
*Consequences:* rvCSI joins the broader ruvnet cognitive stack; RF field history becomes queryable; requires embedding design and retention policy.
|
||||
|
||||
### D9 — Design for replayability
|
||||
|
||||
Signal algorithms need repeatable benchmarks and debugging. **rvCSI supports deterministic replay** of captured sessions (timestamps, ordering, validation decisions, event output, calibration version, runtime config all preserved).
|
||||
*Consequences:* easier testing; better audit trail; enables benchmark datasets.
|
||||
|
||||
### D10 — Separate detection from decision
|
||||
|
||||
rvCSI detects RF events; agents/applications decide what to do. **rvCSI emits events with confidence and evidence and performs no high-consequence actions by default.**
|
||||
*Consequences:* cleaner safety model; clean integration with Cognitum proof-gated execution; applications implement policy.
|
||||
|
||||
### D11 — Local-first operation
|
||||
|
||||
RF sensing is privacy-sensitive and often valuable offline. **rvCSI runs locally by default and requires no cloud service**; remote observability is opt-in.
|
||||
*Consequences:* better privacy posture; usable in industrial/care/sovereign deployments; remote observability must be explicitly enabled.
|
||||
|
||||
### D12 — MCP tools are read-first, write-gated
|
||||
|
||||
Agents should observe RF state safely; device mutation and calibration change system behavior. **MCP tools default to read actions**; capture start/stop, calibration, and export are gated.
|
||||
*Consequences:* safer agent integration; lower accidental device disruption; more explicit operational control.
|
||||
|
||||
### D13 — Quality scoring is mandatory
|
||||
|
||||
CSI quality varies widely by chip, antenna, environment, channel, and interference. **Every frame, window, and event carries quality or confidence scoring.**
|
||||
*Consequences:* downstream systems can suppress weak evidence; easier debugging; requires calibration and thresholds. Where a detector compares against a learned baseline (e.g. baseline-drift / anomaly), thresholds are expressed **relative to the baseline's magnitude**, not as absolute amplitude units, so a single tuning is valid across sources whose raw CSI scales differ by orders of magnitude (raw `int8` ESP32 vs. `int16`-scaled Nexmon vs. baseline-subtracted streams).
|
||||
|
||||
### D14 — Versioned calibration profiles
|
||||
|
||||
Room baselines change over time. **Calibration profiles are versioned**, and event outputs reference the calibration version used.
|
||||
*Consequences:* more auditable detection; replay can reproduce prior outputs; slight storage overhead.
|
||||
|
||||
### D15 — Hardware adapters are plugins
|
||||
|
||||
Device support will evolve and vary by platform. **Source adapters are plugins behind a common Rust trait** (`CsiSource`).
|
||||
*Consequences:* easier support for Nexmon/ESP32/Intel/Atheros/SDR/future sources; cleaner testability; adapter certification becomes important.
|
||||
|
||||
---
|
||||
|
||||
## 3. Architecture
|
||||
|
||||
```
|
||||
CSI Source
|
||||
↓ ┌─ Capture context ──────────────┐
|
||||
Adapter Layer (C shims here) │ Source · CaptureSession · │
|
||||
↓ │ AdapterProfile │
|
||||
Rust Validation Pipeline ─────┤ Validation context │
|
||||
↓ │ ValidationPolicy · Quarantine │
|
||||
Normalized CsiFrame ──────────┘ ← FFI-safe boundary object
|
||||
↓ ┌─ Signal context ───────────────┐
|
||||
Signal Processing │ SignalPipeline · WindowBuffer │
|
||||
↓ ├─ Calibration context ──────────┤
|
||||
Window Aggregator ───────────┤ CalibrationProfile · │
|
||||
↓ │ RoomSignature · BaselineModel │
|
||||
Event Extractor ─────────────┤ Event context │
|
||||
↓ │ EventDetector · StateMachine │
|
||||
TS SDK · CLI · MCP · RuVector └─ Memory + Agent contexts ──────┘
|
||||
```
|
||||
|
||||
**Crates (within RuView's `v2/crates/`, or a standalone `rvcsi/crates/`):**
|
||||
`rvcsi-core` · `rvcsi-adapter-file` · `rvcsi-adapter-nexmon` · `rvcsi-adapter-esp32` · `rvcsi-dsp` · `rvcsi-events` · `rvcsi-ruvector` · `rvcsi-daemon` · `rvcsi-node` · `rvcsi-mcp` — plus TypeScript packages `sdk`, `cli`, `dashboard`, and `native/nexmon-shim-c`.
|
||||
|
||||
See the [PRD §9](../prd/rvcsi-platform-prd.md#9-system-architecture) for the full component table and reference layout, and the [domain model](../ddd/rvcsi-domain-model.md) for bounded contexts, aggregates, invariants, and domain services.
|
||||
|
||||
---
|
||||
|
||||
## 4. Consequences
|
||||
|
||||
**Positive**
|
||||
|
||||
- CSI becomes reusable infrastructure: npm-installable, reproducible, typed, safe-parsed, embeddable, WebSocket-streamable, WASM-portable, MCP-exposed, agent-integrable.
|
||||
- One application codebase works across Nexmon, ESP32, Intel, and Atheros sources.
|
||||
- Bad packets cannot crash the daemon; unattended operation becomes realistic.
|
||||
- RuView/RuVector/Cognitum/agents gain a validated live source of RF observations.
|
||||
- Honest product framing ("structural sensing") avoids over-claiming.
|
||||
|
||||
**Negative / costs**
|
||||
|
||||
- Larger build surface: Rust core + napi-rs native module + C shims + TypeScript packages + prebuild pipeline.
|
||||
- Adapter certification and a supported-hardware matrix become ongoing maintenance.
|
||||
- Embedding design, calibration thresholds, and retention policy are non-trivial open questions (tracked in the PRD).
|
||||
- Risk of duplicating `wifi-densepose-signal` / `wifi-densepose-ruvector`; mitigated by wrapping, not re-implementing.
|
||||
|
||||
**Risks**
|
||||
|
||||
- Nexmon coupling: some device support remains dependent on upstream firmware/driver projects.
|
||||
- CSI quality variance: weak-signal environments may yield low-confidence events; mitigated by mandatory quality scoring (D13) and versioned calibration (D14).
|
||||
|
||||
---
|
||||
|
||||
## 5. Alternatives considered
|
||||
|
||||
| Alternative | Why not |
|
||||
|-------------|---------|
|
||||
| Pure-Python runtime (extend the v1 stack) | Fragile under malformed packets; GC pauses break the < 50 ms latency target; poor unattended stability. |
|
||||
| Pure-Rust including firmware (replace Nexmon) | Enormous scope; vendor-specific; would block v0 indefinitely. D2 keeps C at the boundary instead. |
|
||||
| Per-source SDKs (no normalized schema) | Pushes device specifics into application code; defeats the "same app code across adapters" success criterion. |
|
||||
| WASM-only core | No raw socket / serial / monitor-mode access for live capture; fine for offline parsing (a later target) but not v0 live capture. |
|
||||
| Cloud-first ingestion | Violates the privacy posture and the local-first requirement; unacceptable for care/industrial/sovereign deployments. |
|
||||
|
||||
---
|
||||
|
||||
## 6. Implementation phases (proposed)
|
||||
|
||||
1. **v0** — `rvcsi-core` + file/replay/ESP32 adapters + validation + `rvcsi-dsp` (presence/motion) + `rvcsi-node` SDK + `rvcsi-cli` + WebSocket output + `rvcsi-ruvector` export + basic calibration + health checks. Targets all eight PRD success criteria.
|
||||
2. **v1** — multi-node sync, RF room signatures, breathing-rate where signal permits, temporal embeddings, drift detection, room-topology graph, `rvcsi-mcp` tool server, replayable benchmark datasets, RuView sensor fusion, Cognitum deployment profile.
|
||||
3. **v2** — hardware-agnostic RF sensor fabric, multi-room RF memory, streaming anomaly detection, RF-SLAM research mode, on-device embedding model, federated room-signature learning, signed sensor-evidence records, proof-gated event publication, dynamic cut-based coherence over RF graphs, agent-driven calibration and self-repair.
|
||||
|
||||
---
|
||||
|
||||
## 7. References
|
||||
|
||||
- [rvCSI Platform PRD](../prd/rvcsi-platform-prd.md)
|
||||
- [rvCSI Domain Model](../ddd/rvcsi-domain-model.md)
|
||||
- ADR-012 — ESP32 CSI Sensor Mesh
|
||||
- ADR-013 — Feature-Level Sensing on Commodity Gear
|
||||
- ADR-014 — SOTA Signal Processing
|
||||
- ADR-016 — RuVector Integration
|
||||
- ADR-024 — Project AETHER: Contrastive CSI Embeddings
|
||||
- ADR-031 — RuView Sensing-First RF Mode
|
||||
- ADR-040 — WASM Programmable Sensing
|
||||
- ADR-049 — Cross-Platform WiFi Interface Detection
|
||||
@@ -0,0 +1,144 @@
|
||||
# ADR-096: rvCSI — Crate Topology, the napi-c Shim, and the napi-rs Node Surface
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Proposed |
|
||||
| **Date** | 2026-05-12 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codename** | **rvCSI** — RuVector Channel State Information runtime |
|
||||
| **Relates to** | ADR-095 (rvCSI platform — D1 Rust core, D2 C-at-the-boundary, D3 TS SDK, D4 napi-rs, D5 normalized schema, D6 validate-before-FFI, D15 plugin adapters), ADR-009/ADR-040 (WASM runtimes), ADR-049 (cross-platform WiFi interface detection) |
|
||||
| **PRD** | [rvCSI Platform PRD](../prd/rvcsi-platform-prd.md) |
|
||||
| **Domain model** | [rvCSI Domain Model](../ddd/rvcsi-domain-model.md) |
|
||||
| **Implements** | `v2/crates/rvcsi-core`, `rvcsi-dsp`, `rvcsi-events`, `rvcsi-adapter-file`, `rvcsi-adapter-nexmon`, `rvcsi-ruvector`, `rvcsi-node`, `rvcsi-cli` |
|
||||
|
||||
---
|
||||
|
||||
## 1. Context
|
||||
|
||||
ADR-095 set the platform-level invariant `C → Rust → TypeScript` and the fifteen decisions that constrain rvCSI. This ADR makes the *implementation* concrete: which crates exist, what each owns, where the two FFI seams are (the **napi-c** C shim below Rust, and the **napi-rs** Node addon above it), and the rules that keep `unsafe` confined and the boundary objects validated.
|
||||
|
||||
The two seams:
|
||||
|
||||
- **napi-c** — the *downward* seam to fragile vendor/firmware/driver code. Per ADR-095 D2, C is the only language allowed here, and only as a thin, allocation-free, bounds-checked shim. The Nexmon family is the first consumer.
|
||||
- **napi-rs** — the *upward* seam to Node.js/TypeScript. Per ADR-095 D3/D4, the Rust runtime is exposed to JS via [napi-rs](https://napi.rs/); nothing crosses this seam that hasn't been validated (D6) and normalized (D5).
|
||||
|
||||
Both seams are *narrow on purpose*: everything in between — parsing, validation, DSP, windowing, event extraction, RuVector export — is safe Rust (`#![forbid(unsafe_code)]` in every crate except `rvcsi-adapter-nexmon`, which needs `extern "C"`).
|
||||
|
||||
---
|
||||
|
||||
## 2. Decision
|
||||
|
||||
### 2.1 Crate topology
|
||||
|
||||
Eight new workspace members under `v2/crates/`:
|
||||
|
||||
| Crate | `unsafe`? | Depends on | Owns |
|
||||
|-------|-----------|------------|------|
|
||||
| `rvcsi-core` | no (`forbid`) | — (serde, thiserror) | The normalized schema (`CsiFrame`/`CsiWindow`/`CsiEvent`), `AdapterProfile`, the `CsiSource` plugin trait, id newtypes + `IdGenerator`, `RvcsiError`, and the `validate_frame` pipeline + quality scoring. The shared kernel. |
|
||||
| `rvcsi-dsp` | no (`forbid`) | `rvcsi-core` | Reusable DSP stages (DC removal, phase unwrap, smoothing, Hampel/MAD outlier filter, sliding variance, baseline subtraction) and scalar features (motion energy, presence score, confidence, heuristic breathing-band estimate), plus a non-destructive `SignalPipeline::process_frame`. |
|
||||
| `rvcsi-events` | no (`forbid`) | `rvcsi-core` | `WindowBuffer` (frames → `CsiWindow`), the `EventDetector` trait + presence/motion/quality/baseline-drift state machines, and `EventPipeline` (windows → `CsiEvent`s). The baseline-drift detector measures drift **relative to the running baseline's RMS magnitude** (a fraction, not absolute amplitude units), so the same thresholds work for raw `int8` ESP32 CSI, `int16`-scaled Nexmon CSI, and baseline-subtracted streams alike — see ADR-095 D13. |
|
||||
| `rvcsi-adapter-file` | no (`forbid`) | `rvcsi-core` | The `.rvcsi` capture format (JSONL: a header line + one `CsiFrame` per line), `FileRecorder`, and `FileReplayAdapter` (a `CsiSource`) — deterministic replay (D9). |
|
||||
| `rvcsi-adapter-nexmon` | **yes** (FFI only) | `rvcsi-core` + the C shim | The **napi-c** seam: `native/rvcsi_nexmon_shim.{c,h}` compiled via `build.rs`+`cc`, a documented `ffi` module wrapping it, a pure-Rust libpcap reader (`pcap.rs`), the Nexmon-chip / Raspberry-Pi-model registry (`chips.rs` — `NexmonChip`, `RaspberryPiModel` incl. **Pi 5**, profile builders), and two `CsiSource`s — `NexmonAdapter` (rvCSI-record buffers) and `NexmonPcapAdapter` (real nexmon_csi UDP payloads inside a `.pcap`, with chip auto-detection). |
|
||||
| `rvcsi-ruvector` | no (`forbid`) | `rvcsi-core` | The RuVector RF-memory bridge: deterministic `window_embedding`/`event_embedding`, `cosine_similarity`, the `RfMemoryStore` trait, and `InMemoryRfMemory` + `JsonlRfMemory` (a standin until the production RuVector binding lands). |
|
||||
| `rvcsi-runtime` | no (`forbid`) | core, dsp, events, adapter-file, adapter-nexmon, ruvector | The composition layer (no FFI): `CaptureRuntime` (a `CsiSource` + `validate_frame` + `SignalPipeline` + `EventPipeline`) plus one-shot helpers (`summarize_capture`, `decode_nexmon_records`, `decode_nexmon_pcap`, `summarize_nexmon_pcap`, `events_from_capture`, `export_capture_to_rf_memory`). The shared layer under `rvcsi-node` and `rvcsi-cli`. |
|
||||
| `rvcsi-node` | no (`deny(clippy::all)`) | `rvcsi-core`, `rvcsi-runtime`, `rvcsi-adapter-nexmon` | The **napi-rs** seam: the `.node` addon (cdylib + rlib) exposing a safe TS-facing surface (thin `#[napi]` wrappers over `rvcsi-runtime`); `build.rs` runs `napi_build::setup()`. |
|
||||
| `rvcsi-cli` | no | core, adapter-file, adapter-nexmon, runtime | The `rvcsi` binary: `record` (Nexmon-dump or nexmon-pcap → `.rvcsi`), `inspect`, `inspect-nexmon`, `decode-chanspec`, `replay`, `stream`, `events`, `health`, `calibrate`, `export ruvector` (ADR-095 FR7). |
|
||||
|
||||
`rvcsi-events` does **not** call into `rvcsi-dsp`: window statistics are simple enough to compute in `WindowBuffer` itself, and keeping the two leaves independent removes a coordination point. `rvcsi-cli` does **not** depend on `rvcsi-node` (a binary can't link a napi cdylib's undefined Node symbols) — the shared logic lives in `rvcsi-runtime`, which both build on. Higher layers wire `SignalPipeline::process_frame` → `WindowBuffer::push` when they want cleaned frames.
|
||||
|
||||
The MCP tool server (`rvcsi-mcp`) and the long-running daemon (`rvcsi-daemon`) — and live radio capture — are *not* in this ADR's scope; they sit on top of `rvcsi-runtime` / the crates above and are tracked as follow-ups. The `@ruv/rvcsi` npm package ships alongside `rvcsi-node`.
|
||||
|
||||
### 2.2 The napi-c shim — record formats and contract
|
||||
|
||||
`native/rvcsi_nexmon_shim.{c,h}` is the only C in the runtime. It handles **two byte formats** (ABI `1.1`):
|
||||
|
||||
**(1) The "rvCSI Nexmon record"** — a compact, self-describing record (`'RVNX'` magic, version, flags, RSSI/noise, channel, bandwidth, timestamp, then interleaved `int16` I/Q in Q8.8 fixed point; total `24 + 4*N`). Used by the `rvcsi capture`/`record` recorder, the file replay path, and tests. Functions: `rvcsi_nx_record_len`, `rvcsi_nx_parse_record`, `rvcsi_nx_write_record`.
|
||||
|
||||
**(2) The *real* nexmon_csi UDP payload** — what the patched Broadcom firmware actually sends to the host (port 5500 by default): the 18-byte header `magic=0x1111 (2) · rssi int8 (1) · fctl (1) · src_mac (6) · seq_cnt (2) · core/stream (2) · chanspec (2) · chip_ver (2)`, followed by `nsub` complex CSI samples. The shim implements the **modern int16 I/Q export** (`nsub` pairs of little-endian `int16` `(real, imag)`, raw counts — what CSIKit / `csireader.py` read for the BCM43455c0 / 4358 / 4366c0); `nsub` is derived from the payload length, `(len − 18) / 4`. Functions: `rvcsi_nx_csi_udp_header` (just the 18-byte header), `rvcsi_nx_csi_udp_decode` (header + CSI body, `csi_format` selector), `rvcsi_nx_csi_udp_write` (synthesize a payload — tests/examples), and `rvcsi_nx_decode_chanspec` (decode a Broadcom d11ac chanspec word → `channel` = `chanspec & 0xff`, bandwidth from bits `[13:11]` cross-checked against the FFT size, band from bits `[15:14]` cross-checked against the channel number). The legacy nexmon *packed-float* export used by some 4339/4358 firmwares is a documented follow-up (it sits behind the same `csi_format` selector).
|
||||
|
||||
The `timestamp_ns` of a frame from format (2) comes from the **pcap packet timestamp**, not the wire (nexmon_csi doesn't carry one). The pcap file itself is parsed in **pure Rust** (`rvcsi-adapter-nexmon::pcap` — classic libpcap, all four byte-order/timestamp-resolution magics, Ethernet / raw-IPv4 / Linux-SLL link types; pcapng is a follow-up): peeling the Ethernet/IPv4/UDP headers down to the payload is not a vendor-fragility concern, so it doesn't belong in C.
|
||||
|
||||
Contract (both formats):
|
||||
|
||||
- **Allocation-free, global-free.** Every read is bounds-checked against the caller-supplied length; nothing can scribble outside caller buffers; no `malloc`, no statics.
|
||||
- **Structured errors, never panics.** Functions return one of a small set of `RvcsiNxError` codes (`TOO_SHORT`, `BAD_MAGIC`, `BAD_VERSION`, `CAPACITY`, `TRUNCATED`, `ZERO_SUBCARRIERS`, `TOO_MANY_SUBCARRIERS`, `NULL_ARG`, `BAD_NEXMON_MAGIC`, `BAD_CSI_LEN`, `UNKNOWN_FORMAT`); `rvcsi_nx_strerror` maps each to a static string.
|
||||
- **ABI versioned.** `rvcsi_nx_abi_version()` returns `major << 16 | minor` (`0x0001_0001`); the Rust side `debug_assert`s the major matches the header it was compiled against. The minor was bumped from `1.0` → `1.1` when the format-(2) entry points landed (additive — format (1) is unchanged).
|
||||
- The Rust `ffi` module wraps these in safe functions (`record_len`, `decode_record`, `encode_record`, `decode_chanspec`, `parse_nexmon_udp_header`, `decode_nexmon_udp`, `encode_nexmon_udp`, `shim_abi_version`); every `unsafe` block is limited to the FFI call (and reading back C-initialised structs) and carries a `// SAFETY:` comment, per the project rule.
|
||||
|
||||
**Chip registry (`rvcsi-adapter-nexmon::chips`).** nexmon_csi runs on a handful of patched Broadcom/Cypress chips; `NexmonChip` names them, `RaspberryPiModel` maps Pi boards to their chip, and `nexmon_adapter_profile` / `raspberry_pi_profile` build the [`AdapterProfile`] (supported channels / bandwidths / expected subcarrier counts — 20→64, 40→128, 80→256, 160→512) `validate_frame` bounds CSI frames against. The **Raspberry Pi 5** carries the same **CYW43455 / BCM43455c0** 802.11ac wireless as the Pi 3B+ / Pi 4 / Pi 400 (20/40/80 MHz, 2.4 + 5 GHz) — the chip with the most mature nexmon_csi support — so `RaspberryPiModel::Pi5 → NexmonChip::Bcm43455c0`; the Pi Zero 2 W is `Bcm43436b0` (2.4 GHz, ≤40 MHz). `NexmonPcapAdapter` **auto-detects** the chip from each packet's `chip_ver` word (`0x4345` → `Bcm43455c0`, etc.) and uses the matching profile; `.with_chip(...)` / `.with_pi_model(...)` override it. `NexmonChip::from_chip_ver` and the `chip_ver` field are best-effort/preserved respectively — the c0/b0 revision suffix isn't carried by that word, and the int16-vs-packed-float export distinction is handled by the `csi_format` selector, not by chip-ver parsing.
|
||||
|
||||
A real deployment captures with `tcpdump -i wlan0 dst port 5500 -w csi.pcap` on the Pi and feeds the `.pcap` to `NexmonPcapAdapter::open` (or `rvcsi record --source nexmon-pcap --in csi.pcap --out cap.rvcsi --chip pi5`, then the rest of the toolchain works on the `.rvcsi`; `rvcsi inspect-nexmon` reports the resolved chip, `rvcsi nexmon-chips` lists the matrix). Production *live* capture (binding the UDP socket, monitor mode, firmware patch hooks) is a later increment that reuses the same shim parse path — the shim's job is the *parse*, not the *socket*.
|
||||
|
||||
### 2.3 The napi-rs surface — what crosses the seam
|
||||
|
||||
`rvcsi-node` is a `["cdylib", "rlib"]` crate (cdylib = the `.node` addon; rlib so `cargo test --workspace` can link and test the Rust side without Node). Rules:
|
||||
|
||||
- **Only normalized/validated data crosses.** The boundary types are JS-friendly mirrors of `CsiFrame`/`CsiWindow`/`CsiEvent`/`AdapterProfile`/`SourceHealth`, or plain JSON strings — never raw pointers, never `Pending` frames. A frame is run through `rvcsi_core::validate_frame` before it is handed to JS.
|
||||
- **Errors map to JS exceptions** via napi-rs's `Result` integration; `RvcsiError`'s `Display` is the message.
|
||||
- **The build emits link args + `binding.js`/`binding.d.ts`** via `napi_build::setup()` in `build.rs`; the `@ruv/rvcsi` npm package's hand-written `index.js`/`index.d.ts` wrap that loader and `JSON.parse` the addon's returns into plain `CsiFrame`/`CsiWindow`/`CsiEvent`/`SourceHealth`/`CaptureSummary`/`NexmonPcapSummary`/`DecodedChanspec` objects.
|
||||
- The free functions exposed are: `rvcsiVersion`, `nexmonShimAbiVersion` (the linked shim's ABI), `nexmonDecodeRecords`, `nexmonDecodePcap`, `inspectNexmonPcap`, `decodeChanspec`, `inspectCaptureFile`, `eventsFromCaptureFile`, `exportCaptureToRfMemory`; plus the `RvcsiRuntime` streaming class (`openCaptureFile` / `openNexmonFile` / `openNexmonPcap` factories + `nextFrameJson` / `nextCleanFrameJson` / `drainEventsJson` / `healthJson`).
|
||||
|
||||
### 2.4 Build & test invariants
|
||||
|
||||
- `cargo build --workspace` and `cargo test --workspace --no-default-features` (the repo's pre-merge gate) must stay green; the new crates add tests and don't regress the existing 1,031+.
|
||||
- `rvcsi-node` stays a workspace *member* (not `exclude`d like `wifi-densepose-wasm-edge`): on Linux/macOS a napi cdylib links fine with Node symbols left undefined (resolved at addon-load time), so `cargo build`/`cargo test` work without a Node toolchain. Only `napi build` (npm packaging) needs Node.
|
||||
- No new heavy dependencies in the rvCSI crates: `serde`, `serde_json`, `thiserror`, `cc` (build only), `napi`/`napi-derive`/`napi-build`, `clap` (CLI only), `tempfile` (dev only). DSP math is hand-rolled — no `ndarray`/`rustfft`.
|
||||
|
||||
---
|
||||
|
||||
## 3. Consequences
|
||||
|
||||
**Positive**
|
||||
|
||||
- The two FFI seams are small, audited, and independently testable: the C shim round-trips through Rust tests; the napi surface tests run under `cargo test` without Node.
|
||||
- `unsafe` is confined to one crate (`rvcsi-adapter-nexmon`) and within it to one module (`ffi`), every block documented.
|
||||
- Each leaf crate (`rvcsi-dsp`, `rvcsi-events`, `rvcsi-adapter-file`, `rvcsi-ruvector`) depends only on `rvcsi-core`, so they can evolve (and be reviewed, and be swarm-implemented) independently.
|
||||
- The `.rvcsi` JSONL capture format and the `JsonlRfMemory` standin make the whole pipeline runnable and testable end-to-end before any hardware or the real RuVector binding exists.
|
||||
|
||||
**Negative / costs**
|
||||
|
||||
- A `cc`-built C library means a C toolchain is required to build `rvcsi-adapter-nexmon` (already true for many workspace crates via transitive `cc` deps; acceptable).
|
||||
- The "rvCSI Nexmon record" is a *normalized* format, not byte-identical to any upstream nexmon_csi build — a thin demux/transcode step is needed when wiring real Nexmon output. This is intentional (we control the contract the shim parses) and documented.
|
||||
- JSONL captures are larger than a packed binary format; fine for v0 (and the PRD already standardizes on JSON/WebSocket on the wire), revisit if capture size becomes a problem.
|
||||
- `rvcsi-node` as a workspace member adds the `napi` dependency tree to `cargo build --workspace`; mitigated by it being a small, well-maintained crate.
|
||||
|
||||
**Risks**
|
||||
|
||||
- napi-rs major-version churn could change the macro/`build.rs` surface; pinned to `napi = "2.16"` in workspace deps, bumped deliberately.
|
||||
- If a future platform can't link a napi cdylib under plain `cargo build`, `rvcsi-node` moves to the workspace `exclude` list (like `wifi-densepose-wasm-edge`) with a separate build command — same pattern, already established.
|
||||
|
||||
---
|
||||
|
||||
## 4. Alternatives considered
|
||||
|
||||
| Alternative | Why not |
|
||||
|-------------|---------|
|
||||
| One mega-crate `rvcsi` instead of eight | Couples DSP/events/adapters/FFI; can't review or implement them independently; bloats compile units for downstream users who only want `rvcsi-core`. |
|
||||
| `bindgen` for the C shim | Pulls in `libclang`; the shim's C API is six functions — hand-written `extern "C"` decls are clearer and dependency-free. |
|
||||
| Binary `.rvcsi` capture format (bincode/custom) | Smaller, but not human-inspectable; JSONL is debuggable, append-friendly, and matches the PRD's on-the-wire JSON. Revisit if size matters. |
|
||||
| Expose raw `CsiFrame` pointers / typed arrays across napi for zero-copy | Violates ADR-095 D6 (validate-before-FFI) and the "no raw pointers to TS" safety NFR; the per-frame copy cost is negligible at the target rates. |
|
||||
| `wasm-bindgen` instead of napi-rs for the JS surface | WASM can't do live capture (no raw sockets/serial); great for offline parsing (a later target) but not the primary Node runtime. |
|
||||
| `rvcsi-events` depending on `rvcsi-dsp` for window stats | Adds a coordination point for two leaf crates; the stats are a few lines — keep the leaves independent and let higher layers compose them. |
|
||||
|
||||
---
|
||||
|
||||
## 5. Status of the implementation
|
||||
|
||||
- `rvcsi-core` — implemented, `forbid(unsafe_code)`, 29 unit tests.
|
||||
- `rvcsi-adapter-nexmon` + the napi-c shim — implemented; C (ABI `1.1`) compiled via `build.rs`+`cc`; the `ffi` module wraps both record formats (rvCSI record **and** the real nexmon_csi UDP payload + chanspec decode); a pure-Rust `pcap` reader; the Nexmon-chip / Raspberry-Pi-model registry (`chips.rs` — incl. **Pi 5 → BCM43455c0** + chip auto-detection from `chip_ver`); `NexmonAdapter` + `NexmonPcapAdapter` `CsiSource`s; 28 tests, several round-tripping through the C shim and through synthetic libpcap files.
|
||||
- `rvcsi-dsp` (28 tests), `rvcsi-events` (19 tests — incl. a scale-invariance regression for the baseline-drift detector), `rvcsi-adapter-file` (20 + 1 doctest), `rvcsi-ruvector` (20 + 1 doctest) — implemented.
|
||||
- `rvcsi-runtime` (13 tests) — composition layer + the one-shot helpers, including `decode_nexmon_pcap` / `decode_nexmon_pcap_for` (per-chip) / `summarize_nexmon_pcap` / `nexmon_profile_for`.
|
||||
- `rvcsi-node` (napi-rs surface — incl. `nexmonDecodePcap` (with `chip`) / `inspectNexmonPcap` / `decodeChanspec` / `nexmonChipName` / `nexmonProfile` / `nexmonChips` / `RvcsiRuntime.openNexmonPcap`) and `rvcsi-cli` (10 tests — incl. `record --source nexmon-pcap [--chip pi5]`, `inspect-nexmon`, `nexmon-chips`, `decode-chanspec`) — implemented; the `@ruv/rvcsi` npm package + a Node smoke test ship alongside.
|
||||
- Totals: 169 rvcsi unit/integration tests + 2 doctests, 0 failures; all rvcsi crates build together and are clippy-clean.
|
||||
- **Validated against real ESP32 CSI** (a 7,000-frame node-1 capture, transcoded to `.rvcsi` via `scripts/esp32_jsonl_to_rvcsi.py` — the stand-in for the not-yet-shipped `record --source esp32-jsonl`): `rvcsi inspect` / `replay` / `calibrate` / `events` all run end-to-end. This surfaced and fixed the baseline-drift over-trigger (absolute → relative thresholds, above).
|
||||
- `rvcsi-adapter-esp32` (live serial/UDP ESP32 source — ADR-095 §1.2 / D15), `rvcsi-mcp` (MCP tool server), `rvcsi-daemon` (live capture + WebSocket), and the legacy nexmon *packed-float* CSI export — not in this PR; tracked as follow-ups.
|
||||
|
||||
---
|
||||
|
||||
## 6. References
|
||||
|
||||
- [ADR-095 — rvCSI Edge RF Sensing Platform](ADR-095-rvcsi-edge-rf-sensing-platform.md)
|
||||
- [rvCSI Platform PRD](../prd/rvcsi-platform-prd.md)
|
||||
- [rvCSI Domain Model](../ddd/rvcsi-domain-model.md)
|
||||
- napi-rs — https://napi.rs/
|
||||
- nexmon_csi — the upstream Broadcom CSI extractor the record format normalizes
|
||||
@@ -0,0 +1,157 @@
|
||||
# ADR-097: Adopt rvCSI as RuView's primary CSI runtime
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Proposed |
|
||||
| **Date** | 2026-05-13 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codename** | **rvCSI-in-RuView** |
|
||||
| **Relates to** | ADR-095 (rvCSI platform), ADR-096 (rvCSI crate topology / FFI), ADR-014 (SOTA signal processing in `wifi-densepose-signal`), ADR-016 (RuVector training pipeline integration), ADR-024 (AETHER contrastive embeddings), ADR-031 (RuView sensing-first RF mode), ADR-049 (cross-platform WiFi interface detection) |
|
||||
| **rvCSI repo** | [github.com/ruvnet/rvcsi](https://github.com/ruvnet/rvcsi) (vendored at `vendor/rvcsi`) |
|
||||
|
||||
---
|
||||
|
||||
## 1. Context
|
||||
|
||||
rvCSI — the **edge RF sensing runtime** — was incubated inside RuView under ADR-095 and ADR-096 (PR #542), extracted into its own repo (`ruvnet/rvcsi`, PR #543), and the inline `v2/crates/rvcsi-*` copies were removed in favour of the `vendor/rvcsi` submodule (PR #544). All nine crates are published on crates.io at `0.3.1`; `@ruv/rvcsi 0.3.1` is on npm; a Claude Code plugin marketplace ships with the repo.
|
||||
|
||||
> rvCSI normalizes WiFi CSI from many sources (Nexmon, ESP32, Intel, Atheros, file, replay) into one validated `CsiFrame` / `CsiWindow` / `CsiEvent` schema, runs reusable DSP, emits typed confidence-scored events, and bridges to RuVector RF memory. The crate topology — `rvcsi-core` (kernel) → `rvcsi-dsp` / `rvcsi-events` / `rvcsi-adapter-{file,nexmon}` / `rvcsi-ruvector` (leaves) → `rvcsi-runtime` (composition) → `rvcsi-node` (napi-rs) + `rvcsi-cli` — is fixed by ADR-096.
|
||||
|
||||
**Today, RuView vendors rvCSI but does not consume it.** No Cargo `Cargo.toml` in `v2/crates/*` depends on any `rvcsi-*` crate; no Rust source `use rvcsi_…`; no `@ruv/rvcsi` import in `ui/`, `dashboard/`, or anywhere else. The submodule (`vendor/rvcsi`) is a pinned reference-only — currently at the initial `0.3.0` commit (not even tracking the latest `0.3.1`).
|
||||
|
||||
Meanwhile, RuView's `v2/` workspace carries its own substantial CSI infrastructure that overlaps directly with rvCSI:
|
||||
|
||||
| RuView crate (today) | Overlapping rvCSI crate |
|
||||
|---|---|
|
||||
| `wifi-densepose-signal` (DSP stages, RuvSense modules) — ADR-014 | `rvcsi-dsp` (DC removal, phase unwrap, Hampel/MAD, smoothing, baseline subtraction, motion-energy/presence) |
|
||||
| `wifi-densepose-signal::ruvsense::pose_tracker` etc. (per-window aggregates, presence/motion) | `rvcsi-events` (`WindowBuffer`, presence / motion / quality / baseline-drift detectors) |
|
||||
| `wifi-densepose-hardware` (ESP32 aggregator, TDM, channel hopping) | `rvcsi-adapter-esp32` *(not yet shipped — ADR-095 §1.2 / D15 follow-up)* |
|
||||
| `wifi-densepose-ruvector` (cross-viewpoint fusion + RuVector v2.0.4 integration) — ADR-016 | `rvcsi-ruvector` (deterministic window/event embeddings, `RfMemoryStore`) |
|
||||
| `wifi-densepose-sensing-server` (Axum REST + WS) | `rvcsi-node` (napi-rs SDK) + `rvcsi-cli` |
|
||||
|
||||
Carrying both indefinitely is a maintenance liability: two diverging code paths for the same concepts, two test surfaces, two bug-fix queues, two API contracts. The extraction of rvCSI was explicitly motivated by giving these primitives a stable, hardware-abstracted home; the natural next step is for RuView to *consume* that home rather than carry parallel implementations.
|
||||
|
||||
This ADR decides **how RuView starts depending on rvCSI, where the seams are, and what survives in `v2/crates/wifi-densepose-*`.**
|
||||
|
||||
### 1.1 What this ADR is *not*
|
||||
|
||||
- Not a rewrite of `wifi-densepose-signal`'s SOTA / RuvSense modules. Those modules go beyond rvCSI's scope (cross-viewpoint fusion, AETHER re-ID, RF tomography, longitudinal biomechanics, adversarial detection) and *stay* in RuView — they consume rvCSI's normalized `CsiFrame` rather than reimplementing the parsing/validation/DSP plumbing below them.
|
||||
- Not a forced migration of every consumer simultaneously. Adoption is phased.
|
||||
- Not a decision on whether to delete `archive/v1/` (the Python reference) — that's its own discussion.
|
||||
|
||||
---
|
||||
|
||||
## 2. Decision
|
||||
|
||||
**Adopt rvCSI as the primary CSI ingestion / validation / DSP / event-extraction runtime for RuView, consumed via the published crates.** The decisions below are the architectural contract for that adoption.
|
||||
|
||||
### D1 — Depend on the published `rvcsi-*` crates, not the submodule path
|
||||
|
||||
Each consuming RuView crate adds `rvcsi-runtime = "0.3"` (or whichever rvCSI crate(s) it needs) to its `Cargo.toml`. Cargo resolves these from crates.io. `vendor/rvcsi` remains a **pinned source-of-truth for local dev / patches / offline builds**, not the build path.
|
||||
*Consequences:* normal `cargo build` works without `git submodule update --init`; version pinning is explicit in `Cargo.toml`; coordinated upgrades are a single SemVer bump per crate; the submodule pin can lag and that's fine.
|
||||
|
||||
### D2 — `wifi-densepose-sensing-server` is the pilot consumer
|
||||
|
||||
The sensing-server (Axum REST + WebSocket) is the smallest, best-bounded touchpoint: its UDP CSI receiver and `latest`/`vital-signs`/`edge-vitals` endpoints map cleanly onto `rvcsi-runtime::CaptureRuntime` + the `rvcsi_events` pipeline. The pilot replaces only the **ingestion / validation / DSP / event** path; the existing handlers, the WebSocket fan-out, the RVF model loader, the adaptive classifier and the vital-sign extractor stay.
|
||||
*Consequences:* one PR-sized adoption to learn from before touching the heavier crates; integration tests in `wifi-densepose-sensing-server` exercise the rvCSI surface against synthetic + real ESP32 captures (the `scripts/esp32_jsonl_to_rvcsi.py` bridge in the standalone repo is the de-facto fixture path).
|
||||
|
||||
### D3 — `wifi-densepose-signal` is *layered on top of* rvCSI, not replaced
|
||||
|
||||
The RuvSense modules (`multistatic`, `phase_align`, `tomography`, `pose_tracker`, `field_model`, `longitudinal`, `intention`, `cross_room`, `gesture`, `adversarial`, `coherence_gate`) go strictly beyond `rvcsi-dsp` and stay in RuView. They consume `rvcsi_core::CsiFrame` / `CsiWindow` instead of the current `wifi_densepose_core::CsiFrame`-like types.
|
||||
The genuinely-overlapping primitives in `wifi-densepose-signal` (basic DSP — DC removal, phase unwrap, Hampel, smoothing, baseline subtraction, motion-energy / presence) are either replaced with `rvcsi-dsp::stages::*` calls or kept as thin shims that delegate. A single `From<wifi_densepose_core::CsiFrame> for rvcsi_core::CsiFrame` (and the reverse) lives in `wifi-densepose-signal` during the transition.
|
||||
*Consequences:* the SOTA work stays in RuView (where it belongs); the parsing/validation/baseline plumbing centralizes in rvCSI; the public API of `wifi-densepose-signal` shifts gradually toward "modules built on top of `rvcsi-*`".
|
||||
|
||||
### D4 — `wifi-densepose-hardware` stops carrying ESP32 wire-format parsing
|
||||
|
||||
The ESP32 ADR-018 binary frame parsing (magic 0xC5110001, 20-byte header, int8 I/Q — see the `scripts/esp32_jsonl_to_rvcsi.py` bridge in the rvCSI repo) becomes part of a new `rvcsi-adapter-esp32` crate (ADR-095 §1.2 / D15 follow-up, owned in the rvCSI repo). `wifi-densepose-hardware` keeps the firmware/aggregator side (UDP listener, mesh, TDM, channel hopping, NVS provisioning) — i.e. the parts above the wire — and emits parsed `CsiFrame`s via the new adapter trait.
|
||||
*Consequences:* the firmware-side and host-side concerns split cleanly; the parser lives once (in rvCSI) and is testable in isolation; the wire format is documented once.
|
||||
|
||||
### D5 — Embeddings & RF memory: the two `ruvector` paths stay separate (for now)
|
||||
|
||||
`wifi-densepose-ruvector` (ADR-016) is the **training** pipeline integration — feeding RuvSense outputs into RuVector for cross-viewpoint fusion, AETHER contrastive embeddings, domain generalization (MERIDIAN). `rvcsi-ruvector` is the **runtime RF-memory** bridge — deterministic per-window/per-event embeddings + `RfMemoryStore`. They serve different jobs; both stay. A follow-up ADR can unify them once `rvcsi-ruvector`'s production backend (currently the `JsonlRfMemory` standin) lands the real RuVector binding.
|
||||
*Consequences:* no churn in the training pipeline today; the runtime memory and the training-time fusion remain distinct contexts in the DDD sense.
|
||||
|
||||
### D6 — Schema: `rvcsi_core::CsiFrame` becomes the boundary type at the runtime edge
|
||||
|
||||
At the *runtime* edge (sensing-server, future daemon, any new adapter), `rvcsi_core::CsiFrame` is the validated normalized object. RuView's internal types (`wifi_densepose_core::CsiFrame` and friends) continue to exist for training and SOTA pipelines, but a single explicit conversion happens at the boundary and is the only allowed translation point.
|
||||
*Consequences:* one validation gate at one edge; downstream code stops re-deriving amplitude/phase / re-checking finiteness; the `validate_frame` quality scoring is the only source of truth for "is this frame usable".
|
||||
|
||||
### D7 — Versioning: track rvCSI via SemVer-compatible ranges + pin the submodule
|
||||
|
||||
`Cargo.toml` deps use `rvcsi-runtime = "0.3"` etc. (`^0.3`, so 0.3.x picks up automatically). The `vendor/rvcsi` submodule pin is **bumped per RuView release** to whatever rvCSI commit RuView was tested against — providing reproducible offline builds and a source-level reference, even though the actual build resolves from crates.io.
|
||||
*Consequences:* RuView keeps moving; rvCSI patch releases roll in automatically; minor-version bumps require a deliberate `^0.3` → `^0.4` change (and a re-test of the consumers); the submodule pin advances with each release tag so it never silently drifts.
|
||||
|
||||
### D8 — Replace `vendor/rvcsi` with crates.io once D1–D7 are merged
|
||||
|
||||
If, after the pilot, every consumer depends on crates.io (no consumer touches `vendor/rvcsi/crates/*`), `vendor/rvcsi` is *redundant*. A future ADR can decide to drop the submodule entirely. Until then it stays.
|
||||
*Consequences:* the migration path has a clear terminal state; no decision on submodule removal made today.
|
||||
|
||||
---
|
||||
|
||||
## 3. Adoption phases
|
||||
|
||||
| Phase | Scope | Closes |
|
||||
|---|---|---|
|
||||
| **P1 (pilot)** — `wifi-densepose-sensing-server` ingestion | UDP receiver + simulated source go through `rvcsi-runtime::CaptureRuntime` + `rvcsi_events::EventPipeline`; sensing-server emits rvCSI events on `/api/v1/events` and the WebSocket. | D1, D2, D6 partly |
|
||||
| **P2 (signal shim)** — `wifi-densepose-signal` thin-shim adoption | Overlapping DSP primitives delegate to `rvcsi-dsp`; SOTA modules stay; `From`/`Into` bridge added. | D3, D6 |
|
||||
| **P3 (ESP32 adapter)** — `rvcsi-adapter-esp32` lands in the rvCSI repo; `wifi-densepose-hardware` switches over | New crate in `ruvnet/rvcsi`; RuView consumes it as `rvcsi-adapter-esp32 = "0.3"`. | D4 |
|
||||
| **P4 (clean-up)** — duplicates removed | Inline DSP primitives in `wifi-densepose-signal` deleted (only shims left for back-compat or fully removed). | D3 fully |
|
||||
| **P5 (post-pilot)** — `vendor/rvcsi` review | Decide whether to keep the submodule. | D8 |
|
||||
|
||||
Each phase is one PR, each PR has unit + integration tests against the rvCSI surface, the workspace test stays green (1,031+ tests).
|
||||
|
||||
---
|
||||
|
||||
## 4. Consequences
|
||||
|
||||
**Positive**
|
||||
|
||||
- Single normalized schema (`CsiFrame` / `CsiWindow` / `CsiEvent`) across RuView's runtime surface — fewer bespoke types, less duplication.
|
||||
- Bad packets quarantined at one place (rvCSI's `validate_frame`), not at every consumer.
|
||||
- New CSI sources (Intel `iwlwifi`, Atheros, SDR) plug in once at the rvCSI layer, work for every RuView consumer immediately.
|
||||
- rvCSI's structured `RvcsiError` + the C shim's panic-free contract replace ad-hoc parser error handling in RuView's hardware-side code.
|
||||
- The sensing-server inherits the FFI-boundary hardening from rvCSI (e.g. the NaN-safe `napi-c` encode fix in `rvcsi-adapter-nexmon 0.3.1` flows in automatically).
|
||||
|
||||
**Negative / costs**
|
||||
|
||||
- Two repos to keep in lockstep during the adoption (`ruvnet/RuView` + `ruvnet/rvcsi`). Mitigated by SemVer + the per-release submodule bump.
|
||||
- Per-frame conversion at the boundary in P1/P2 (one `From<rvcsi_core::CsiFrame> for wifi_densepose_core::CsiFrame`-style hop). Cost is a single `Vec` clone of the I/Q + amplitude/phase arrays per frame; at the project's target rates this is well under the 50 ms latency budget.
|
||||
- The training pipeline (`wifi-densepose-ruvector`) and the runtime RF memory (`rvcsi-ruvector`) coexist until D5's follow-up.
|
||||
- The Nexmon ESP32 adapter (D4 / P3) is real work in the rvCSI repo before P3 can land.
|
||||
|
||||
**Risks**
|
||||
|
||||
- API drift between `wifi_densepose_core::CsiFrame` and `rvcsi_core::CsiFrame` if both keep evolving; mitigated by D6 (one explicit conversion point, every other consumer reads only `rvcsi_core::CsiFrame`).
|
||||
- crates.io as a hard dependency — if crates.io is unreachable in an air-gapped build, `vendor/rvcsi` + `[patch.crates-io]` is the documented escape hatch.
|
||||
|
||||
---
|
||||
|
||||
## 5. Alternatives considered
|
||||
|
||||
| Alternative | Why not |
|
||||
|---|---|
|
||||
| Keep both in parallel indefinitely | Two diverging implementations of the same concepts → twice the bug-fix surface, twice the docs, twice the tests; defeats the reason rvCSI was extracted in the first place. |
|
||||
| Big-bang adoption — replace `wifi-densepose-signal` end-to-end in one PR | Too much surface to land safely; the SOTA modules go *beyond* rvCSI's scope and don't lift cleanly. D3's "layered on top" preserves what matters. |
|
||||
| Consume `vendor/rvcsi/crates/*` via path deps instead of crates.io | Couples RuView to the submodule's HEAD; loses the SemVer ratchet; makes `cargo build` fail when the submodule isn't initialized. D1 (published crates) is the standard pattern. |
|
||||
| Move RuView itself into `ruvnet/rvcsi` (monorepo) | Defeats the reason rvCSI was extracted — rvCSI is a runtime usable beyond RuView (other agents, other apps, the standalone CLI + npm SDK). The repo split is intentional. |
|
||||
| Stay on `wifi-densepose-signal` and treat rvCSI as a sibling library only | Means RuView reimplements every adapter, every validation rule, every event detector forever. D2's pilot validates whether the seams are right before committing to D3. |
|
||||
|
||||
---
|
||||
|
||||
## 6. Open questions
|
||||
|
||||
- **Per-subcarrier calibration baseline.** rvCSI's `events` pipeline benefits from a learned baseline (`SignalPipeline::baseline_amplitude`) — RuView's existing per-node calibration logic (in `wifi-densepose-sensing-server`'s field-model endpoints) should feed that baseline in. The plumbing is straightforward; documenting the format is a P1 sub-task.
|
||||
- **Single-frame schema overhead.** `rvcsi_core::CsiFrame` carries `i_values + q_values + amplitude + phase + quality_reasons` (four `Vec<f32>` plus a `Vec<String>`). RuView's training pipeline (which sometimes processes 100k+ frames in batch) may want a "lean frame" view to avoid the extra allocations. Track as a separate optimization once P1 is in.
|
||||
- **Cross-viewpoint fusion outputs as `CsiEvent` metadata.** The `metadata_json: String` field on `CsiEvent` is the natural carrier for RuvSense-derived multistatic fusion outputs; a small `serde` helper in `wifi-densepose-signal` standardizes the JSON shape.
|
||||
|
||||
---
|
||||
|
||||
## 7. References
|
||||
|
||||
- [ADR-095 — rvCSI Edge RF Sensing Platform](ADR-095-rvcsi-edge-rf-sensing-platform.md)
|
||||
- [ADR-096 — rvCSI Crate Topology, the napi-c Shim, the napi-rs Surface](ADR-096-rvcsi-ffi-crate-layout.md)
|
||||
- [ADR-014 — SOTA Signal Processing in `wifi-densepose-signal`](ADR-014-sota-signal-processing.md)
|
||||
- [ADR-016 — RuVector Training Pipeline Integration](ADR-016-ruvector-training-pipeline.md)
|
||||
- [ADR-031 — RuView Sensing-First RF Mode](ADR-031-ruview-sensing-first-rf-mode.md)
|
||||
- [`github.com/ruvnet/rvcsi`](https://github.com/ruvnet/rvcsi) — 9 crates on crates.io @ 0.3.1, `@ruv/rvcsi 0.3.1` on npm, Claude Code plugin marketplace
|
||||
- `vendor/rvcsi` (submodule) — currently pinned at `acd5689d` (0.3.0 commit); bumps to `0.3.1` HEAD as part of P1
|
||||
@@ -0,0 +1,191 @@
|
||||
# ADR-098: Evaluate `ruvnet/midstream` for RuView's CSI / WebSocket / mesh pipeline
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Rejected (with crate-level carve-outs for future evaluation) |
|
||||
| **Date** | 2026-05-13 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codename** | **midstream-in-RuView** |
|
||||
| **Relates to** | ADR-095 (rvCSI platform), ADR-096 (rvCSI crate topology), ADR-097 (adopt rvCSI as RuView's CSI runtime), ADR-012 (ESP32 CSI mesh), ADR-029 (RuvSense multistatic / TDM), ADR-031 (RuView sensing-first RF mode), ADR-043 (sensing-server UI API completion) |
|
||||
| **midstream repo** | [github.com/ruvnet/midstream](https://github.com/ruvnet/midstream) — vendored at `vendor/midstream`, currently pinned at [`30fe5eb`](https://github.com/ruvnet/midstream/commit/30fe5eb7a1f1494aa1ad00d54160088a565ec766) |
|
||||
| **Outcome** | Do **not** adopt as a system component. Two of midstream's six workspace crates (`temporal-compare`, `nanosecond-scheduler`) are plausible future-use building blocks; the rest do not fit. `vendor/midstream` is retained as a reference-only submodule. |
|
||||
|
||||
---
|
||||
|
||||
## 1. Context
|
||||
|
||||
`vendor/midstream` is a git submodule of RuView (`.gitmodules:1-4`) but, like `vendor/rvcsi` was before ADR-097, it is **vendored but not consumed**: no `v2/crates/*/Cargo.toml` depends on a `midstreamer-*` crate, no Rust source contains `use midstreamer_…`, and the ESP32 firmware and TypeScript dashboard have no midstream imports.
|
||||
|
||||
This ADR settles the standing question of *whether RuView should consume midstream at all*, and if so, where. The user-facing prompt enumerated four candidate seams to evaluate:
|
||||
|
||||
1. Streaming / pub-sub for the WebSocket fan-out (today: `tokio::sync::broadcast::channel::<String>(256)` at `v2/crates/wifi-densepose-sensing-server/src/main.rs:4769`).
|
||||
2. Stream processing for the CSI → DSP → event pipeline (today: synchronous `EventPipeline` at `vendor/rvcsi/crates/rvcsi-events/src/pipeline.rs`, freshly adopted via ADR-097).
|
||||
3. Multi-source merging / TDM coordination for the ESP32 mesh (ADR-029, ADR-073).
|
||||
4. Backpressure / flow control between the UDP receiver and downstream consumers (`v2/crates/wifi-densepose-sensing-server/src/main.rs:3638` `udp_receiver_task`; firmware-side `stream_sender` ENOMEM backoff at `firmware/esp32-csi-node/main/csi_collector.c:223-228`).
|
||||
|
||||
To evaluate each, we read midstream's workspace `Cargo.toml` (`vendor/midstream/Cargo.toml:1-99`), the `README.md` and `BENCHMARKS_SUMMARY.md`, and every crate's `lib.rs`:
|
||||
|
||||
| Crate | File | LOC | Purpose (from header doc) |
|
||||
|---|---|---:|---|
|
||||
| `midstreamer-temporal-compare` | `vendor/midstream/crates/temporal-compare/src/lib.rs:1-697` | 697 | DTW, LCS, Levenshtein, generic pattern matching on `Sequence<T>` of `TemporalElement<T>` |
|
||||
| `midstreamer-scheduler` | `vendor/midstream/crates/nanosecond-scheduler/src/lib.rs:1-406` | 406 | Priority + deadline-aware task scheduler (RM, EDF, LLF) for low-latency real-time tasks |
|
||||
| `midstreamer-attractor` | `vendor/midstream/crates/temporal-attractor-studio/src/lib.rs:1-482` | 482 | Phase-space reconstruction, Lyapunov exponents, attractor classification |
|
||||
| `midstreamer-neural-solver` | `vendor/midstream/crates/temporal-neural-solver/src/lib.rs:1-509` | 509 | LTL / CTL / MTL temporal-logic verification with neural reasoning |
|
||||
| `midstreamer-strange-loop` | `vendor/midstream/crates/strange-loop/src/lib.rs:1-496` | 496 | Multi-level meta-learning, self-referential systems |
|
||||
| `midstreamer-quic` | `vendor/midstream/crates/quic-multistream/src/lib.rs:1-255`, `native.rs:1-303`, `wasm.rs:1-307` | 865 | Thin wrapper over `quinn` (native) and `WebTransport` (WASM); generic QUIC streams |
|
||||
|
||||
Plus a TypeScript layer (`vendor/midstream/npm/`, `vendor/midstream/npm-wasm/`) whose product is "real-time LLM streaming" — OpenAI Realtime API client, RTMP / WebRTC / HLS for video, an in-console dashboard, a Whisper transcription scaffold, an MCP server for LLM agents.
|
||||
|
||||
The top-level identity is unambiguous: `Cargo.toml:16` describes the package as **`"Real-time LLM streaming with inflight analysis"`**, and the README (`vendor/midstream/README.md:45-80`) frames midstream as a platform that "analyzes [LLM] responses **as they stream in real-time** — enabling instant insights, pattern detection, and intelligent decision-making" — i.e. the streaming domain is **LLM tokens and dashboard telemetry**, not RF signals. A search for any of `csi`, `wifi`, `sensing`, or `sensor` across `vendor/midstream/crates/*/src/*.rs` returns zero hits.
|
||||
|
||||
This shapes the conclusion: midstream's *abstractions* (DTW pattern matching, attractor analysis, LTL verification, meta-learning) were chosen for a fundamentally different problem domain than CSI, and its *transport* (QUIC) is a thin `quinn` wrapper rather than a sensing-aware backplane. The candidate seams enumerated above are either already filled by simpler primitives in RuView, or filled better by rvCSI under ADR-097.
|
||||
|
||||
### 1.1 What this ADR is *not*
|
||||
|
||||
- Not a judgment on midstream's quality. It has 139 passing tests and clean Rust; it is well-engineered for its target domain.
|
||||
- Not a decision to drop `vendor/midstream`. The submodule pin is cheap to keep, and the carve-outs in §3 may justify revisiting it.
|
||||
- Not a position on the *standalone* midstream product (LLM streaming, OpenAI Realtime, dashboards). That product is unaffected by this ADR.
|
||||
|
||||
---
|
||||
|
||||
## 2. Decision
|
||||
|
||||
**Reject midstream as a system component of RuView.** The four candidate seams are either filled (well) by existing RuView primitives, or are filled by rvCSI's freshly-adopted `EventPipeline` and `RfMemoryStore`. The eight decisions below are the architectural contract.
|
||||
|
||||
### D1 — Streaming / pub-sub for the WebSocket fan-out: no change
|
||||
|
||||
RuView's sensing-server currently fans out updates to WebSocket clients via `tokio::sync::broadcast::channel::<String>(256)` (`v2/crates/wifi-densepose-sensing-server/src/main.rs:4769`). midstream offers no equivalent in-process broadcast primitive — its TypeScript dashboard fan-out is HTTP-server based (`vendor/midstream/npm/src/dashboard.ts`), and its Rust `midstreamer-quic` crate is a generic point-to-point QUIC wrapper (`vendor/midstream/crates/quic-multistream/src/native.rs:31-69`), not a pub-sub bus.
|
||||
|
||||
Tokio's `broadcast` channel is the standard Rust idiom for this pattern, costs effectively nothing per subscriber, integrates with the rest of the Axum + Tokio stack already in use (`v2/crates/wifi-densepose-sensing-server/src/main.rs:36,47`), and is what `rvcsi-runtime` itself uses for event distribution (`vendor/rvcsi/crates/rvcsi-runtime/src/lib.rs`). **Keep `tokio::sync::broadcast`.**
|
||||
*Consequences:* zero migration; zero new dependency surface; the WebSocket handlers at `main.rs:1989,2030` continue to work unchanged.
|
||||
|
||||
### D2 — CSI → DSP → event pipeline: stay on rvCSI's `EventPipeline`
|
||||
|
||||
ADR-097 D2 just adopted `rvcsi-runtime::CaptureRuntime` + `rvcsi_events::EventPipeline` as the CSI ingestion / DSP / event-extraction path. `EventPipeline` is **deterministic, synchronous, single-frame-at-a-time** (`vendor/rvcsi/crates/rvcsi-events/src/pipeline.rs:1-5`: *"Feed it frames with `EventPipeline::process_frame` and drain the tail with `EventPipeline::flush`"*) — and that determinism is load-bearing for ADR-095 D9 (replayability) and ADR-095 D13 (quality scoring against learned baselines).
|
||||
|
||||
midstream's stream-processing primitives are designed for the opposite shape: `temporal-attractor-studio` (phase-space reconstruction, Lyapunov exponents) and `temporal-neural-solver` (LTL formula verification) operate on **trajectories** of multi-dimensional states over hundreds-to-thousands of samples (`vendor/midstream/README.md:528-531`: *"Attractor detection: <5ms for 1000-point series"*) — that is closer to RuView's existing RuvSense modules (`v2/crates/wifi-densepose-signal/src/ruvsense/longitudinal.rs`, `intention.rs`) than to anything the runtime DSP layer needs.
|
||||
|
||||
Replacing rvCSI's event detectors with midstream constructs would (a) break determinism, (b) re-introduce a parallel CSI-processing implementation — exactly the duplication ADR-097 was opened to remove — and (c) force RuView to invent a `Sequence<T: temporal-compare::TemporalElement>` shim around `CsiFrame` for marginal benefit. **Stay on `rvcsi-events::EventPipeline`.**
|
||||
*Consequences:* the determinism / replay guarantees of ADR-095 D9 and ADR-097 D6 remain intact; the work to land `rvcsi-adapter-esp32` (ADR-097 D4, P3) is not duplicated.
|
||||
|
||||
### D3 — TDM / multi-source merging: stay on the existing aggregator
|
||||
|
||||
The ESP32 mesh's multi-source merging is in `v2/crates/wifi-densepose-hardware/src/aggregator/mod.rs:74-220` — a `UdpSocket`-backed aggregator (`mod.rs:74,85`) that receives parsed `CsiFrame`s from N nodes and forwards them on a `SyncSender<CsiFrame>` to the consumer. The TDM coordination (slot assignment, channel hopping, dwell time) lives in firmware (`firmware/esp32-csi-node/main/`) and is governed by ADR-029 and ADR-073. midstream offers nothing for either side: it has no UDP merger, no slot scheduler, and no firmware-side primitives.
|
||||
|
||||
`midstreamer-scheduler` is conceptually adjacent — it does priority + deadline-aware scheduling (`vendor/midstream/crates/nanosecond-scheduler/src/lib.rs:53-63`: `RateMonotonic`, `EarliestDeadlineFirst`, `LeastLaxityFirst`, `FixedPriority`) — but its target is **in-process tokio tasks on a 4-thread executor** (`vendor/midstream/README.md:466-477`: *"4 worker threads"*, *"<50 ns scheduling latency"*), not the cross-device, wall-clock-anchored TDM that RuvSense needs. **Keep the existing `wifi-densepose-hardware` aggregator and firmware-side TDM.**
|
||||
*Consequences:* ADR-029 stays as-is; the work to migrate the parser to `rvcsi-adapter-esp32` (ADR-097 D4) is unaffected.
|
||||
|
||||
### D4 — UDP receiver backpressure / flow control: existing solutions are correct at each end
|
||||
|
||||
There are two distinct backpressure problems in RuView, and neither benefits from midstream:
|
||||
|
||||
- **Firmware side (`firmware/esp32-csi-node/main/csi_collector.c:64,223-228`):** lwIP pbuf exhaustion produces `ENOMEM` when the ESP32 tries to UDP-send faster than the network drains. The fix in code is a rate-limit on `stream_sender_send` *inside the CSI callback*. This is a C-level firmware concern with no Rust analogue — midstream cannot run on the ESP32.
|
||||
- **Host side (`v2/crates/wifi-densepose-sensing-server/src/main.rs:3638-3640`, `4769`):** `udp_receiver_task` reads from `UdpSocket` and pushes onto `broadcast::channel::<String>(256)`. The bounded channel is itself the backpressure mechanism: lagged subscribers see `RecvError::Lagged`, the buffer wraps, no producer ever blocks. The 256-slot capacity is sized to one second of frame envelopes at the target rate; the per-second packet-yield collapse symptom (`adaptive_controller_decide.c:26-28`) is detected and surfaced by ADR-039 / ADR-081's `pkt_yield_per_sec` accessor, not by transport-layer flow control.
|
||||
|
||||
midstream's `quic-multistream` provides per-stream prioritization (`vendor/midstream/crates/quic-multistream/src/native.rs:1-303`), which is a useful flow-control primitive *for QUIC* but not for the UDP-CSI / WS-fan-out topology RuView actually uses. Adopting QUIC end-to-end would mean (a) replacing the ESP32's UDP sender — which would need a QUIC stack on a memory-constrained Xtensa MCU and is out of scope for this project — or (b) terminating QUIC at the aggregator only, which provides no benefit the current bounded `broadcast` channel doesn't. **Keep the existing two-tier backpressure.**
|
||||
*Consequences:* the ENOMEM rate-limit at `csi_collector.c:223-228` and the bounded `broadcast::channel::<String>(256)` at `main.rs:4769` continue to be the load-bearing primitives.
|
||||
|
||||
### D5 — Carve-out: `temporal-compare` as a future RuvSense-side building block
|
||||
|
||||
`midstreamer-temporal-compare` (`vendor/midstream/crates/temporal-compare/src/lib.rs:1-697`) is a clean DTW / LCS / Levenshtein implementation with an LRU cache. RuView's gesture detector at `v2/crates/wifi-densepose-signal/src/ruvsense/gesture.rs` already does DTW template matching, and the longitudinal analysis at `ruvsense/longitudinal.rs` could plausibly benefit from cached pattern matching. If we ever need a *separate* DTW implementation that is decoupled from RuvSense's internal types, `temporal-compare` is a reasonable starting point — but only if and when that need arises.
|
||||
|
||||
We **do not adopt it today** because RuvSense's gesture matcher already exists, works, and uses RuView-native types, and pulling in `dashmap`, `lru`, and a generic `TemporalElement<T>` abstraction would be net-negative right now. **Tracked as a future evaluation, not a decision.**
|
||||
*Consequences:* zero today; one named option for a future ADR if a "second" DTW pattern appears.
|
||||
|
||||
### D6 — Carve-out: `nanosecond-scheduler` for *host-side* edge tier scheduling (future)
|
||||
|
||||
If ADR-039's edge-intelligence tier scheduling ever moves from the ESP32 onto a host-side coordinator (e.g. a Raspberry Pi running the cluster aggregator), `nanosecond-scheduler`'s deadline-aware policies (`vendor/midstream/crates/nanosecond-scheduler/src/lib.rs:53-63`) could plausibly host that scheduler. Today the scheduling is firmware-side and the C-level RTOS handles it; there is nothing to schedule in Rust at the granularity midstream offers.
|
||||
|
||||
Again: **not a current decision, just an option kept open.**
|
||||
*Consequences:* zero today.
|
||||
|
||||
### D7 — Submodule disposition: keep `vendor/midstream`
|
||||
|
||||
`vendor/midstream` is one git submodule pin; the build does not depend on it; it does not slow down `cargo build --workspace`; and the carve-outs in D5/D6 leave the door open. Removing the submodule would also remove the reference material that justified the carve-outs.
|
||||
|
||||
**Keep the submodule, no per-release pin advancement.** Unlike `vendor/rvcsi` (whose pin is bumped per RuView release under ADR-097 D7), `vendor/midstream` has no in-build consumer to validate against. If D5 or D6 ever activates, *that* ADR will start the per-release pin process. Until then the pin can drift freely.
|
||||
*Consequences:* one line of `.gitmodules` (`.gitmodules:1-4`) stays; `git submodule update --init` remains a no-op for normal RuView development.
|
||||
|
||||
### D8 — Documentation: cross-reference, don't import
|
||||
|
||||
The ADR index (`docs/adr/README.md`) gets ADR-098 added under "Architecture and infrastructure". No other docs are updated. The README on the RuView side is untouched; midstream is not part of the RuView platform story.
|
||||
*Consequences:* one row added to the ADR index; no churn elsewhere.
|
||||
|
||||
---
|
||||
|
||||
## 3. Why not adopt (the rejection record)
|
||||
|
||||
For institutional memory, the table below records what each midstream crate *would* solve and the alternative RuView already uses. This is the answer to "but we vendored midstream — what is it for?"
|
||||
|
||||
| midstream crate | Plausible RuView seam | Already filled by | Verdict |
|
||||
|---|---|---|---|
|
||||
| `midstreamer-temporal-compare` (DTW, LCS, Levenshtein) | Gesture template matching (`ruvsense/gesture.rs`); longitudinal biomechanics drift | RuvSense's existing DTW gesture matcher | Carve-out only (D5) — not adopted today |
|
||||
| `midstreamer-scheduler` (nanosecond priority + deadline) | ESP32 edge-tier scheduling (ADR-039); RuvSense TDM (ADR-029) | Firmware-side RTOS (ESP32); ADR-029's wall-clock-anchored TDM | Carve-out only (D6) — wrong scope today |
|
||||
| `midstreamer-attractor` (Lyapunov, phase-space) | RF-field stability detection in `ruvsense/field_model.rs`, `longitudinal.rs` | Welford stats + biomechanics drift (longitudinal.rs); SVD eigenstructure (field_model.rs) | Not adopted — RuvSense's approach is calibrated to RF signal scale and the project's existing dataset, not generic dynamical-systems theory |
|
||||
| `midstreamer-neural-solver` (LTL / CTL / MTL verification) | Adversarial signal detection (`ruvsense/adversarial.rs`); coherence-gate decisions | Multi-link consistency checks (adversarial.rs); `coherence_gate.rs` state machine | Not adopted — RuView's adversarial detector is not a formal-verification problem; it's a multi-link physical-consistency check |
|
||||
| `midstreamer-strange-loop` (meta-learning, self-modification) | None in RuView's scope | RuView is not a self-modifying learner; AETHER (ADR-024) is contrastive embedding, not meta-learning | Not adopted — out of scope |
|
||||
| `midstreamer-quic` (QUIC native + WASM) | Sensing-server → external client transport (alternative to WS) | `tokio::sync::broadcast` + Axum WebSocket + UDP (`main.rs:36-47, 4769, 1989, 2030, 3638`) | Not adopted — see D1, D4 |
|
||||
|
||||
The shape of the rejection is consistent: **midstream's abstractions are LLM-token / dashboard-telemetry shaped, RuView's pipeline is RF-frame / event-detector shaped.** Where the two share vocabulary ("streaming", "temporal", "real-time"), the implementations diverge sharply — and the case-by-case analysis above shows that the closer one looks at each seam, the worse the fit gets.
|
||||
|
||||
---
|
||||
|
||||
## 4. Consequences
|
||||
|
||||
**Positive**
|
||||
|
||||
- Zero net change to RuView's build, runtime, or surface area; ADR-097's phased rvCSI adoption proceeds unaffected.
|
||||
- The decision space around midstream is now bounded and documented; future contributors and AI agents see "ADR-098 already evaluated this; here is why not" before re-opening the question.
|
||||
- The two crate-level carve-outs (D5, D6) are explicit, so if the relevant seams appear later, the evaluation can pick up from this ADR rather than start over.
|
||||
- `vendor/midstream` (the submodule) remains as reference material, but is correctly marked as not part of the build path.
|
||||
|
||||
**Negative / costs**
|
||||
|
||||
- One more vendored repo with no in-build consumer — a small but non-zero cognitive load (mitigated by D7's explicit "do not bump the pin").
|
||||
- If midstream's published crates evolve materially (e.g. a CSI-aware feature lands), the reasoning in §3 needs revisiting; this is the standard "rejected ADRs go stale" risk and applies to every Rejected ADR in the index.
|
||||
|
||||
**Risks**
|
||||
|
||||
- The most plausible failure mode of this ADR is *not* "we should have adopted midstream"; it is "we re-open the question in six months without re-reading this ADR." Mitigated by indexing ADR-098 in `docs/adr/README.md` and by the per-crate table in §3 being precise enough to short-circuit the next evaluator.
|
||||
|
||||
---
|
||||
|
||||
## 5. Alternatives considered
|
||||
|
||||
| Alternative | Why not |
|
||||
|---|---|
|
||||
| **Adopt midstream wholesale as RuView's streaming backbone** | Would force the CSI pipeline into the `Sequence<TemporalElement>` shape (`vendor/midstream/crates/temporal-compare/src/lib.rs:42-70`) and the `quic-multistream` transport (`vendor/midstream/crates/quic-multistream/src/native.rs:1-303`) — both are designed for LLM tokens / arbitrary streams, not validated RF frames with quality scoring. Conflicts directly with ADR-095 D5 (one `CsiFrame` schema), D6 (validate before crossing boundaries), and D9 (deterministic replay). |
|
||||
| **Replace `tokio::sync::broadcast` with midstream's QUIC fan-out** | Solves no observed problem. `broadcast::channel::<String>(256)` at `v2/crates/wifi-densepose-sensing-server/src/main.rs:4769` handles N WebSocket subscribers at zero per-subscriber cost; the lagged-subscriber semantics (`RecvError::Lagged`) are exactly what an event-feed wants. QUIC adds TLS + congestion control + per-stream priority — useful for *external* clients across a network, but the sensing-server's clients connect over WS on the same host or LAN. |
|
||||
| **Replace `EventPipeline` with `temporal-attractor-studio` / `temporal-neural-solver`** | `EventPipeline` is deterministic by contract (`vendor/rvcsi/crates/rvcsi-events/src/lib.rs:20`) and ADR-097 just made it RuView's event source of truth. Attractor analysis and LTL verification operate on entirely different abstractions; using them as event detectors would re-invent rvCSI's pipeline in a less-determined way. |
|
||||
| **Adopt `midstreamer-temporal-compare` for gesture detection now** | RuvSense already has a working DTW gesture matcher tuned to CSI signal scale. Swapping it for a generic `TemporalElement<T>` matcher buys cleanliness but costs a re-tune and a new dep tree (`dashmap`, `lru`). Tracked as D5 for if/when a *second* DTW use case shows up. |
|
||||
| **Adopt `midstreamer-scheduler` for the cluster-Pi aggregator** | The cluster aggregator does not currently exist as a real-time scheduler; ADR-039's tier scheduling is firmware-side. Until the host-side schedule appears, importing a deadline-aware scheduler is solution-looking-for-a-problem. Tracked as D6. |
|
||||
| **Drop the `vendor/midstream` submodule entirely** | Cheap to keep, useful as the reference material this ADR cites. D7 keeps it on the explicit understanding that the pin is not advanced. |
|
||||
|
||||
---
|
||||
|
||||
## 6. Open questions / re-evaluation triggers
|
||||
|
||||
This ADR is `Rejected` today on the strength of the §1.1 / §3 analysis. The following events would justify re-opening it:
|
||||
|
||||
1. **A second DTW / LCS / Levenshtein use case appears in RuView** (e.g. a CLI-side replay diff, a regression test fixture that needs sequence alignment, a TUI for pattern playback). Then re-evaluate `midstreamer-temporal-compare` per D5.
|
||||
2. **A host-side real-time scheduler enters RuView's scope** (e.g. the cluster-Pi aggregator becomes responsible for slot timing instead of the ESP32 firmware). Then re-evaluate `midstreamer-scheduler` per D6.
|
||||
3. **midstream ships a CSI-aware adapter or RF-scale `Sequence<T>` extension** — i.e. midstream's own scope grows to include sensing primitives. As of the pinned commit (`30fe5eb`), this has not happened (zero matches for `csi|wifi|sensing|sensor` in `vendor/midstream/crates/*/src/*.rs`).
|
||||
4. **RuView gains a QUIC-to-external-client requirement** that the WS fan-out cannot service (e.g. a mobile client over a lossy link that benefits from QUIC's stream priority + 0-RTT). Then re-evaluate `midstreamer-quic` per D1 / D4.
|
||||
|
||||
If none of these triggers fire, this ADR stays Rejected and the carve-outs (D5, D6) remain optional.
|
||||
|
||||
---
|
||||
|
||||
## 7. References
|
||||
|
||||
- [ADR-095 — rvCSI Edge RF Sensing Platform](ADR-095-rvcsi-edge-rf-sensing-platform.md) — sets the single-`CsiFrame` schema, deterministic replay, and quality-scoring constraints that midstream's abstractions conflict with.
|
||||
- [ADR-096 — rvCSI Crate Topology, the napi-c Shim, the napi-rs Surface](ADR-096-rvcsi-ffi-crate-layout.md) — the crate topology that rvCSI fills the candidate seams with.
|
||||
- [ADR-097 — Adopt rvCSI as RuView's primary CSI runtime](ADR-097-adopt-rvcsi-as-ruview-csi-runtime.md) — phased adoption (P1-P5) that this ADR explicitly does not duplicate.
|
||||
- [ADR-012 — ESP32 CSI Sensor Mesh](ADR-012-esp32-csi-sensor-mesh.md) — the multi-source TDM context for D3.
|
||||
- [ADR-029 — RuvSense Multistatic Sensing Mode](ADR-029-ruvsense-multistatic-sensing-mode.md) — the wall-clock-anchored TDM that `midstreamer-scheduler` is the wrong shape for.
|
||||
- [ADR-039 — ESP32 Edge Intelligence Pipeline](ADR-039-esp32-edge-intelligence.md) — the firmware-side tier scheduling that would need to move host-side before D6 activates.
|
||||
- [`github.com/ruvnet/midstream`](https://github.com/ruvnet/midstream) — 5 published crates on crates.io (`temporal-compare`, `nanosecond-scheduler`, `temporal-attractor-studio`, `temporal-neural-solver`, `strange-loop`) + 1 local crate (`quic-multistream`); 139 passing tests.
|
||||
- `vendor/midstream` (submodule) — pinned at `30fe5eb` (`vendor/midstream/Cargo.toml:16` describes the package as *"Real-time LLM streaming with inflight analysis"*).
|
||||
- RuView code paths cited in §1: `v2/crates/wifi-densepose-sensing-server/src/main.rs:36,47,1989,2030,3638-3640,4769`; `v2/crates/wifi-densepose-hardware/src/aggregator/mod.rs:74-220`; `firmware/esp32-csi-node/main/csi_collector.c:64,223-228`; `firmware/esp32-csi-node/main/adaptive_controller_decide.c:26-28`.
|
||||
- RuvSense code paths cited in §3: `v2/crates/wifi-densepose-signal/src/ruvsense/gesture.rs`, `longitudinal.rs`, `field_model.rs`, `adversarial.rs`, `coherence_gate.rs`.
|
||||
- rvCSI code paths cited in §2: `vendor/rvcsi/crates/rvcsi-events/src/lib.rs:1-37`, `vendor/rvcsi/crates/rvcsi-events/src/pipeline.rs:1-5`.
|
||||
@@ -0,0 +1,242 @@
|
||||
# ADR-099: Adopt midstream as RuView's real-time introspection + low-latency tap
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Status** | Proposed |
|
||||
| **Date** | 2026-05-13 |
|
||||
| **Deciders** | ruv |
|
||||
| **Codename** | **midstream-introspection** |
|
||||
| **Relates to** | ADR-097 (rvCSI adoption — provides the validated `CsiFrame` stream this ADR taps), ADR-098 (Rejected midstream as a *replacement* for RuView's existing seams — this ADR is the *parallel-addition* answer that complements it), ADR-095/096 (rvCSI platform + FFI), ADR-014 (SOTA signal processing in `wifi-densepose-signal`) |
|
||||
| **midstream repo** | [github.com/ruvnet/midstream](https://github.com/ruvnet/midstream) (vendored at `vendor/midstream`); 5 crates on crates.io at `0.2.1` |
|
||||
|
||||
---
|
||||
|
||||
## 1. Context
|
||||
|
||||
[ADR-098](ADR-098-evaluate-midstream-fit.md) rejected midstream as a **replacement** for RuView's existing seams — the four candidate substitutions (WS fan-out, the `wifi-densepose-signal` DSP pipeline, ESP32 mesh TDM coordination, `tokio::sync::broadcast` backpressure) all checked out as "current solution fits, midstream is the wrong tool". That verdict stands.
|
||||
|
||||
This ADR is the **other half** of that conversation. Two of midstream's primitives — `temporal-compare` (DTW) and `temporal-attractor-studio` (Lyapunov + regime classification) — were carved out under ADR-098 D5 as "re-evaluate if a second use case appears". The use case is now named: **real-time introspection of the CSI stream + low-latency detection of motion-shape events**, running as a parallel tap *alongside* RuView's existing event pipeline rather than replacing it.
|
||||
|
||||
### 1.1 The latency floor today, by construction
|
||||
|
||||
[`vendor/rvcsi/crates/rvcsi-events/src/window_buffer.rs:20`](../../vendor/rvcsi/crates/rvcsi-events/src/window_buffer.rs#L20) defines `WindowBuffer::new(max_frames: usize, max_duration_ns: u64)`. The events pipeline emits *only at window close*. At RuView's ~30 Hz CSI rate with the default 16-frame / 1-second windows, the soonest `MotionDetected` or `PresenceStarted` can fire is roughly **500–1000 ms after the actual RF perturbation**. That's an architectural floor, not an implementation accident — `WindowBuffer` is the integration tier, and integration takes time.
|
||||
|
||||
For high-touch UI (the live dashboard) and for downstream consumers that need to react to motion *as it starts*, that floor matters. The `wifi-densepose-sensing-server` already maintains continuous per-frame state (`AppStateInner::{frame_history, rssi_history, smoothed_motion, baseline_motion, last_novelty_score}` at [`main.rs:307–423`](../../v2/crates/wifi-densepose-sensing-server/src/main.rs#L307)), but exposes them only as endpoint-poll scalars — there's no streaming-tap surface for "what's happening *inside* the pipeline right now". A consumer that wants reflex-level reaction has to invent it.
|
||||
|
||||
### 1.2 What midstream's primitives actually map onto
|
||||
|
||||
Ground-truth grep across `vendor/midstream/crates/`:
|
||||
|
||||
| Term | Hits | Where |
|
||||
|---|---|---|
|
||||
| `Lyapunov` | 284 | `temporal-attractor-studio` |
|
||||
| `LTL` | 230 | `temporal-neural-solver` |
|
||||
| `Attractor` | 1252 | `temporal-attractor-studio` |
|
||||
| `DTW` | 540 | `temporal-compare` |
|
||||
| `phase-space` | 23 | `temporal-attractor-studio` |
|
||||
|
||||
`temporal-compare/src/lib.rs:5` advertises *"Dynamic Time Warping (DTW), Longest Common Subsequence (LCS), Edit Distance (Levenshtein), Pattern matching and detection, Efficient caching"* — and the bench prose (in midstream's `README.md`) puts a cached pattern match at **~12 µs**. `temporal-attractor-studio/src/lib.rs:6` advertises *"Attractor classification (point, limit cycle, strange), Lyapunov exponent calculation, Phase space analysis, Stability detection"*. At RuView's ~30 Hz tick budget (33 ms), the per-frame cost of either is well under 1 % of the budget.
|
||||
|
||||
### 1.3 Why this isn't ADR-214
|
||||
|
||||
ADR-214 (the V0 / Cognitum cluster correlator decision, owned in a separate repo) takes a much larger commitment: all five midstream crates, a full new `cognitum-rvcsi-correlator` crate, a `WireRecord` adapter layer, multi-Pi cadence alignment via `nanosecond-scheduler`. That's the right shape for V0 because V0 is filling a "no Rust correlator binary exists yet" gap (ADR-209 §C.1) — *replacing* a Python prototype.
|
||||
|
||||
RuView's case is different and smaller. The Rust pipeline already exists and works. This ADR adds two midstream crates and one tap — same primitives, much narrower scope, no replacement.
|
||||
|
||||
---
|
||||
|
||||
## 2. Decision
|
||||
|
||||
**Adopt `midstreamer-temporal-compare` and `midstreamer-attractor` as a parallel real-time introspection tap inside `wifi-densepose-sensing-server`.** All eight decisions below are the architectural contract.
|
||||
|
||||
### D1 — Only two midstream crates, no more
|
||||
|
||||
`midstreamer-temporal-compare = "0.2"` and `midstreamer-attractor = "0.2"` enter as dependencies of `wifi-densepose-sensing-server`. The other three midstream crates are explicitly **not** in scope:
|
||||
|
||||
* `midstreamer-scheduler` — sub-µs host-side scheduling has no fit in RuView; the per-Pi / per-ESP32 timing-sensitive work happens in firmware (ADR-073 channel hopping, the ESP32 TDM) where it belongs.
|
||||
* `midstreamer-neural-solver` (LTL) — relevant for the MAT (Mass Casualty Assessment Tool) audit-trail use case, *not* for real-time introspection. Tracked as a follow-up ADR.
|
||||
* `midstreamer-strange-loop` — long-horizon meta-learning for `adaptive_classifier` confidence; out of scope of "real-time".
|
||||
|
||||
*Consequences:* the dependency footprint is two A+-security `unsafe_code = "deny"` crates, not the full midstream workspace.
|
||||
|
||||
### D2 — The tap point is post-validate, parallel to `WindowBuffer::push`
|
||||
|
||||
Each `CsiFrame` that survives `rvcsi_core::validate_frame` and `SignalPipeline::process_frame` (the same gate ADR-097 D6 establishes as the boundary) is fanned out to **two consumers**:
|
||||
|
||||
1. The existing `WindowBuffer::push` → `EventPipeline` → `broadcast::<String>` → `/ws/sensing` path. Unchanged.
|
||||
2. The new `IntrospectionState::update_per_frame` → `broadcast::<IntrospectionSnapshot>` → `/ws/introspection` path. Per-frame, never window-blocked.
|
||||
|
||||
*Consequences:* zero behavioural change to the existing `/ws/sensing` / `/api/v1/sensing/latest` / vital-sign / pose / model-management endpoints; the bearer-auth middleware from #547 (PR-merged) wraps the new endpoint exactly like every other `/api/v1/*` and `/ws/*`.
|
||||
|
||||
### D3 — One new WS topic + one new REST endpoint
|
||||
|
||||
* `WS /ws/introspection` — continuous stream of `IntrospectionSnapshot` JSON frames (one per CSI frame received, modulo a small coalesce window if the client is slow).
|
||||
* `GET /api/v1/introspection/snapshot` — one-shot poll for the latest snapshot (mirrors the existing `/api/v1/sensing/latest` shape).
|
||||
|
||||
`IntrospectionSnapshot` carries: `timestamp_ns`, `regime` (one of `Idle`/`Periodic`/`Transient`/`Chaotic`), `lyapunov_exponent: f32`, `attractor_dim: f32`, `top_k_similarity: Vec<(signature_id: String, score: f32)>` (k = 5 by default).
|
||||
|
||||
*Consequences:* dashboard widgets can subscribe directly; the existing `/ws/sensing` stays the canonical "events" topic; the new topic is the "continuous state" topic.
|
||||
|
||||
### D4 — Per-frame update only, never window-blocked
|
||||
|
||||
The new introspection path **must not** block on window close. The DTW path operates over a sliding tail buffer (default 64 frames) of derived feature vectors; the attractor path operates over a sliding tail of `mean_amplitude` scalars. Both update on every accepted frame.
|
||||
|
||||
*Consequences:* the soonest "shape-matches signature" emission is bounded by the per-frame update cost (target ≤1 ms p99 on a Pi-5-class host), not by the 16-frame window — a **~16× collapse** of the latency floor on this specific class of event.
|
||||
|
||||
### D5 — `temporal-neural-solver` (LTL) is out of scope of this ADR
|
||||
|
||||
The MAT audit-trail use case (provable triggers with proof artefacts, ADR-style "this `SurvivorTrack` activation was provably (LTL formula) satisfied") is a separate concern. Tracked as a follow-up ADR; the same crate that lives in `vendor/midstream/crates/temporal-neural-solver` will be revisited there.
|
||||
|
||||
*Consequences:* this ADR does not deliver audit-grade proof artefacts; if you need them, wait for the MAT ADR.
|
||||
|
||||
### D6 — ESP32 firmware is unchanged
|
||||
|
||||
Introspection runs entirely on the host side (`wifi-densepose-sensing-server`). The ESP32 ADR-018 wire format, the firmware's CSI collector, the TDM protocol, the NVS provisioning — none change. No firmware re-flash required to consume this feature.
|
||||
|
||||
*Consequences:* deployment is "update the host-side binary / Docker image"; existing ESP32-S3 / ESP32-C6 / mmWave node fleets work as-is.
|
||||
|
||||
### D7 — Signature library is JSON, on-disk, customer-owned
|
||||
|
||||
A "signature" is a short labelled sequence of derived feature vectors. Schema (one file per signature under `--signatures-dir /etc/cognitum/signatures/`):
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"id": "walking_slow_v1",
|
||||
"label": "Walking — slow pace",
|
||||
"captured_at": "2026-05-13T20:00:00Z",
|
||||
"feature_kind": "amplitude_l2_per_subcarrier", // or "vec128" once an embedding source exists
|
||||
"length": 64,
|
||||
"dtw": { "window": 8, "step_pattern": "symmetric2" },
|
||||
"vectors": [ [ ... ], [ ... ], /* length-64 of feature vectors */ ],
|
||||
"promotion_threshold": 0.78
|
||||
}
|
||||
```
|
||||
|
||||
Three reference signatures ship under `signatures/` in the crate as developer fixtures (`idle_room.sig.json`, `walking_slow.sig.json`, `door_open.sig.json`). Customer-trained signatures are not committed.
|
||||
|
||||
*Consequences:* the library is a deployment-time concern, not a build-time one; customers can tune the threshold per environment.
|
||||
|
||||
### D8 — Measurement-first adoption — promotion bar is empirical
|
||||
|
||||
Phase 0 spike measures the latency win against the existing `/ws/sensing` path on a recorded session. **Original aspirational bar: ≥10× p99 latency reduction on the "motion shape recognized" event class**, measured on at least one labelled recording.
|
||||
|
||||
**Empirical baseline from `tests/introspection_latency.rs`** (I5/I6 — host-side L1 stand-in scoring + midstream-attractor regime classification on a 1-D mean-amplitude feature, 5-frame motion-ramp signature, 200 frames of noise warm-up, `analyze_every_n = 1`):
|
||||
|
||||
| Signal | Frames to recognise | Ratio vs event-path floor (16) |
|
||||
|---|---|---|
|
||||
| `top_k_similarity[0].above_threshold` | 5 | **3.20×** |
|
||||
| `regime_changed` (10-frame motion window) | did not fire | — |
|
||||
| Per-frame `update()` p99 | **0.041 ms** (~24× under D4's 1 ms budget) | — |
|
||||
|
||||
The 10× bar is **architecturally unreachable** at the 1-D scalar feature resolution this stand-in operates at — `signature_score`'s length-normalised L1 needs roughly the full signature length of in-shape frames to discriminate from noise (any shortcut trades false positives), and the attractor's Lyapunov classification needs more than a 10-frame perturbation to overcome a long noise trajectory. The 3.2× ratio is the structural ceiling for this feature class.
|
||||
|
||||
**Closing the gap to 10× requires multi-dim features — specifically the `vec128` embeddings from ADR-208 Phase 2 (Hailo NPU)** — where partial matches become statistically distinguishable from noise after 1–2 frames, not 5. Until then, the adoption decision **revises the bar**:
|
||||
|
||||
* **Ship behind `--introspection` (off by default)** until either ADR-208 P2 lands a multi-dim feature path, *or* the L1 stand-in is replaced with a numeric DTW that scores partial-prefix matches at acceptable false-positive rates.
|
||||
* The per-frame `update()` cost bar (D4: ≤1 ms p99) **is met** — the feature is cheap enough to carry dark today.
|
||||
* **Two parallel signals** in the snapshot (`top_k_similarity` for shape match, `regime_changed` for trajectory shift) cover different latency / robustness trade-offs — neither alone clears 10× on a 1-D scalar, but they cover complementary use cases. Downstream consumers pick.
|
||||
|
||||
> **Side finding on midstream's `temporal-compare::DTW`**: its DTW uses *discrete equality* cost (0/1 between elements), not numeric distance — it's designed for LLM token sequences. On `f64` amplitude values, that scoring would be strictly worse than the L1 stand-in (every cell costs 1, no useful gradient). "Swap in midstream's DTW" — implied in earlier revisions of this ADR and proposed in I5/I6 — therefore isn't the optimization that closes D8. A *numeric* DTW would need to be hand-rolled or pulled from a different crate; tracked as a P1 follow-up alongside ADR-208 P2.
|
||||
|
||||
*Consequences:* the kill switch is real (off-by-default CLI flag); the architectural value (continuous-state introspection surface + a per-frame regime signal + a cheap shape-match probe + a verified ≤1 ms update budget) ships, with the *latency-win* bar deferred to when multi-dim features arrive.
|
||||
|
||||
---
|
||||
|
||||
## 3. Architecture
|
||||
|
||||
```
|
||||
┌── (existing) ──┐
|
||||
│ WindowBuffer │── EventPipeline ─┐
|
||||
UDP / CSI source ─→ validate ─→│ │ ↓
|
||||
+ DSP ───→│ │ broadcast<String>
|
||||
│ (16 frames / │ ↓
|
||||
│ 1 s window) │ /ws/sensing
|
||||
└────────────────┘
|
||||
───→──────┐
|
||||
↓
|
||||
(NEW — this ADR)
|
||||
IntrospectionState::update_per_frame
|
||||
├─ DTW vs signature library (temporal-compare)
|
||||
├─ Attractor / Lyapunov sliding (attractor-studio)
|
||||
└─ Coalesce client-slow → snapshot
|
||||
↓
|
||||
broadcast<IntrospectionSnapshot>
|
||||
↓
|
||||
/ws/introspection (NEW)
|
||||
/api/v1/introspection/snapshot (NEW)
|
||||
```
|
||||
|
||||
The tap is added once, in `csi.rs`'s frame loop, right after the line that currently feeds the `WindowBuffer`. Implementation lives in one new module: `v2/crates/wifi-densepose-sensing-server/src/introspection.rs`.
|
||||
|
||||
The new path **never reads or writes** the existing `AppStateInner` introspection scalars (`smoothed_motion`, `baseline_motion`, etc.) — those stay as the dashboard's continuous-summary backing. The new path produces *additional* signal, not replacement signal.
|
||||
|
||||
---
|
||||
|
||||
## 4. Implementation phases
|
||||
|
||||
| Phase | Scope | Bar |
|
||||
|---|---|---|
|
||||
| **P0 — Spike + benchmark** | Add deps, scaffold `introspection.rs`, wire the tap, add `/ws/introspection`, measure p50/p99 latency on a recorded session. | ≥ 10× p99 latency reduction on the "shape recognized" path vs. `/ws/sensing` event path. If miss, the feature stays behind a CLI flag. |
|
||||
| **P1 — First real signature library** | Capture 3 labelled segments (`idle_room`, `walking_slow`, `door_open`) on the ESP32-S3 on COM7, build the developer fixture under `signatures/`. | A live person walking in front of the node produces a `walking_slow` match in /ws/introspection ≥1 frame before `MotionDetected` fires on /ws/sensing. |
|
||||
| **P2 — Dashboard widget** | Add an "Introspection" panel to the live dashboard subscribing to `/ws/introspection`: regime indicator, Lyapunov gauge, top-k matches with confidence. | Visual confirmation of D4 ("never window-blocked") — the panel responds to a perturbation before the `MotionDetected` toast appears. |
|
||||
| **P3 — Signature capture workflow** | CLI sub-command `rvcsi capture-signature --label <name> --duration 2s --out signatures/<id>.json` (or its sensing-server equivalent) that records and labels a segment in one step. | A non-developer can extend the library without writing JSON by hand. |
|
||||
| **P4 — Adaptive classifier hook (optional)** | Feed introspection's continuous regime scalar + top-k similarities into the existing `adaptive_classifier` as auxiliary features. | Measurable classifier accuracy improvement on a held-out test set; if no improvement, abandon and document. |
|
||||
|
||||
P0 is the commitment. P1–P3 are sequential per-PR follow-ups. P4 is research-shaped and explicitly failure-tolerant.
|
||||
|
||||
---
|
||||
|
||||
## 5. Consequences
|
||||
|
||||
**Positive**
|
||||
|
||||
* Soonest-event latency on the "shape recognized" path drops from ~533 ms (16-frame window @ 30 Hz) to ~33 ms (one frame at 30 Hz) — a 16× collapse, dwarfed only by network RTT and the DTW math itself (~12 µs / cached pattern).
|
||||
* Dashboards and downstream consumers get a streaming-tap surface for *what the pipeline is seeing right now*, not just summary scalars at endpoint-poll time.
|
||||
* `adaptive_classifier` and the novelty bank gain a richer per-frame feature input (regime, Lyapunov, top-k similarity) — augmenting, not replacing, their current inputs.
|
||||
* Zero behavioural change to existing endpoints, no firmware change, no schema migration. Pure addition.
|
||||
* Two A+-security `unsafe_code = "deny"` crates — bounded, audited dependency footprint.
|
||||
|
||||
**Negative**
|
||||
|
||||
* Dependency surface grows by two crates. Mitigation: both pinned `^0.2`, both ours (user owns midstream), both `unsafe_code = "deny"`.
|
||||
* The DTW path is only as good as its signature library — a poor library means false matches. D7's per-deployment library + D8's `promotion_threshold` per signature mitigate; P3's capture workflow makes the library tractable to grow.
|
||||
* Adding a second broadcast topic adds memory pressure under fan-out (each subscriber holds a ring slot). The default ring size (32 snapshots) caps it.
|
||||
|
||||
**Neutral**
|
||||
|
||||
* Existing `/ws/sensing` consumers continue to see the same events at the same cadence.
|
||||
* ADR-097's rvCSI adoption is unaffected — this tap *consumes* rvCSI's validated `CsiFrame` output, doesn't replace any rvCSI seam.
|
||||
* The `vendor/rvcsi` submodule and the `vendor/midstream` submodule both stay; this ADR uses crates.io versions of both for the build, with the submodules as reference / patch escape hatches (ADR-097 D7 and ADR-098 D7 patterns respectively).
|
||||
|
||||
---
|
||||
|
||||
## 6. Alternatives considered
|
||||
|
||||
| Alternative | Why not |
|
||||
|---|---|
|
||||
| **Tighten the rvCSI `WindowBuffer` to 1-frame / 0 ms windows.** | Defeats the purpose — `EventPipeline`'s state machines (`PresenceDetector::enter_windows = 2`, `MotionDetector::debounce_windows = 2`) need stable window-aggregated input to debounce noise. Single-frame windows produce per-frame events with no hysteresis, which is *worse* than today, not better. |
|
||||
| **Write the DTW + attractor math from scratch in `wifi-densepose-signal`.** | This is what midstream's crates *are*. ~640 hits for DTW and 1252 for Attractor across midstream's existing source — re-implementing would be 1–2k LOC of math we'd own and maintain forever. Not free. |
|
||||
| **Use the heuristic `smoothed_motion` / `baseline_motion` as the introspection signal.** | They already exist (`main.rs:310,377`), they're already broadcast on the dashboard's continuous-summary path. But they're a single scalar derived from EWMA — they don't classify regime, don't match shapes, don't give phase-space stability. Worth keeping as the "always-on lite indicator"; not a substitute for D3's snapshot. |
|
||||
| **All five midstream crates at once.** | The other three (`scheduler`, `neural-solver`, `strange-loop`) don't fit the "real-time introspection" framing — they fit "host-side hard scheduling", "audit-grade proofs", "long-horizon meta-learning". Mixing them in would balloon the surface and dilute the latency-win measurement. D1 keeps it to two. |
|
||||
| **Defer until ADR-214's V0 correlator ships and copy its design.** | V0's correlator is the *replacement* shape (Python prototype → Rust). RuView's case is the *addition* shape. The designs share crates but not topologies; deferring would leave RuView's latency floor in place for months while V0 lands. |
|
||||
|
||||
---
|
||||
|
||||
## 7. Open questions
|
||||
|
||||
* **Feature vector for `vec128`-class DTW.** Until ADR-208 Phase 2 ships real Hailo NPU embeddings, the per-frame feature vector is a derived scalar tuple (RSSI + per-subcarrier amplitude L2 norm). When the encoder lands, the DTW path consumes `vec128` directly — what version-skew strategy do signature libraries use?
|
||||
* **Coalesce window for slow WS clients.** A subscriber falling behind shouldn't make the broadcast ring grow unboundedly. Default proposal: drop oldest, log a `warn!` after N consecutive drops. The exact N is tunable.
|
||||
* **Cross-node introspection.** Today the snapshot is per-node. For multi-node deployments, do we want a fused cluster-level snapshot too? Likely yes — but as a separate ADR; this one keeps to per-node.
|
||||
|
||||
---
|
||||
|
||||
## 8. References
|
||||
|
||||
* [ADR-097 — Adopt rvCSI as RuView's primary CSI runtime](ADR-097-adopt-rvcsi-as-ruview-csi-runtime.md) — provides the validated `CsiFrame` stream this tap reads.
|
||||
* [ADR-098 — Evaluate `ruvnet/midstream` for RuView's CSI / WebSocket / mesh pipeline (Rejected)](ADR-098-evaluate-midstream-fit.md) — Rejected midstream as a *replacement* for existing seams. This ADR is the *addition* answer; D5/D6 of ADR-098 explicitly carved out `temporal-compare` and the attractor crate for this case.
|
||||
* [ADR-095 — rvCSI Edge RF Sensing Platform](ADR-095-rvcsi-edge-rf-sensing-platform.md), [ADR-096 — rvCSI Crate Topology](ADR-096-rvcsi-ffi-crate-layout.md) — the upstream platform.
|
||||
* [`midstreamer-temporal-compare` 0.2.1](https://crates.io/crates/midstreamer-temporal-compare), [`midstreamer-attractor` 0.2.1](https://crates.io/crates/midstreamer-attractor) — the two crates this ADR adopts.
|
||||
* [`vendor/midstream/crates/temporal-compare/src/lib.rs:5`](../../vendor/midstream/crates/temporal-compare/src/lib.rs#L5) — DTW / LCS / edit-distance pattern matching, public API.
|
||||
* [`vendor/midstream/crates/temporal-attractor-studio/src/lib.rs:6`](../../vendor/midstream/crates/temporal-attractor-studio/src/lib.rs#L6) — attractor classification + Lyapunov exponent, public API.
|
||||
* [`vendor/rvcsi/crates/rvcsi-events/src/window_buffer.rs:20`](../../vendor/rvcsi/crates/rvcsi-events/src/window_buffer.rs#L20) — the window-aggregation step whose latency floor this tap bypasses.
|
||||
* [`v2/crates/wifi-densepose-sensing-server/src/main.rs:307-423`](../../v2/crates/wifi-densepose-sensing-server/src/main.rs#L307) — the existing per-frame state surface this tap augments.
|
||||
@@ -105,6 +105,11 @@ Statuses: **Proposed** (under discussion), **Accepted** (approved and/or impleme
|
||||
| [ADR-011](ADR-011-python-proof-of-reality-mock-elimination.md) | Proof-of-Reality and Mock Elimination | Proposed |
|
||||
| [ADR-026](ADR-026-survivor-track-lifecycle.md) | Survivor Track Lifecycle (MAT crate) | Accepted |
|
||||
| [ADR-038](ADR-038-sublinear-goal-oriented-action-planning.md) | Sublinear GOAP for Roadmap Optimization | Proposed |
|
||||
| [ADR-095](ADR-095-rvcsi-edge-rf-sensing-platform.md) | rvCSI — Edge RF Sensing Runtime Platform | Proposed |
|
||||
| [ADR-096](ADR-096-rvcsi-ffi-crate-layout.md) | rvCSI — Crate Topology, the napi-c Shim, and the napi-rs Node Surface | Proposed |
|
||||
| [ADR-097](ADR-097-adopt-rvcsi-as-ruview-csi-runtime.md) | Adopt rvCSI as RuView's primary CSI runtime (phased adoption) | Proposed |
|
||||
| [ADR-098](ADR-098-evaluate-midstream-fit.md) | Evaluate `ruvnet/midstream` for RuView's CSI / WebSocket / mesh pipeline | Rejected |
|
||||
| [ADR-099](ADR-099-midstream-introspection-tap.md) | Adopt midstream as RuView's real-time introspection + low-latency tap | Proposed |
|
||||
|
||||
---
|
||||
|
||||
|
||||
|
After Width: | Height: | Size: 4.4 MiB |
|
After Width: | Height: | Size: 1.9 MiB |
|
After Width: | Height: | Size: 1.5 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 2.4 MiB |
@@ -15,6 +15,7 @@ DDD organizes the codebase around the problem being solved — not around techni
|
||||
| [Sensing Server](sensing-server-domain-model.md) | Single-binary Axum server: CSI ingestion, model management, recording, training, visualization | 5 contexts: CSI Ingestion, Model Management, CSI Recording, Training Pipeline, Visualization |
|
||||
| [WiFi-Mat](wifi-mat-domain-model.md) | Disaster response: survivor detection, START triage, mass casualty assessment | 3 contexts: Detection, Localization, Alerting |
|
||||
| [CHCI](chci-domain-model.md) | Coherent Human Channel Imaging: sub-millimeter body surface reconstruction | 3 contexts: Sounding, Channel Estimation, Imaging |
|
||||
| [rvCSI](rvcsi-domain-model.md) | Edge RF sensing runtime: multi-source CSI ingestion, validation, normalization, event extraction, RuVector RF memory, agent/MCP integration | 7 contexts: Capture, Validation, Signal, Calibration, Event, Memory, Agent |
|
||||
|
||||
## How to read these
|
||||
|
||||
|
||||
@@ -0,0 +1,395 @@
|
||||
# rvCSI — Edge RF Sensing Runtime Domain Model
|
||||
|
||||
## Domain-Driven Design Specification
|
||||
|
||||
> Companion documents: [rvCSI Platform PRD](../prd/rvcsi-platform-prd.md) · [ADR-095 — rvCSI Edge RF Sensing Platform](../adr/ADR-095-rvcsi-edge-rf-sensing-platform.md)
|
||||
|
||||
### Domain
|
||||
|
||||
Camera-free RF spatial sensing from WiFi Channel State Information (CSI).
|
||||
|
||||
### Core domain
|
||||
|
||||
**RF field interpretation.** rvCSI converts noisy radio channel measurements into validated events and temporal embeddings that represent changes in physical space. CSI is treated as a *temporal delta stream* against learned baselines — not as exact vision.
|
||||
|
||||
### Supporting subdomains
|
||||
|
||||
Hardware adapter management · packet parsing · signal processing · calibration · event extraction · temporal memory · agent integration · replay and audit.
|
||||
|
||||
### Generic subdomains
|
||||
|
||||
Logging · configuration · CLI parsing · WebSocket streaming · package publishing · dashboard visualization.
|
||||
|
||||
---
|
||||
|
||||
## Ubiquitous Language
|
||||
|
||||
| Term | Definition |
|
||||
|------|------------|
|
||||
| **CSI** | Channel State Information — per-subcarrier complex channel response measured by a WiFi receiver |
|
||||
| **Source** | A physical or replayed producer of CSI frames (a NIC, an ESP32 node, a PCAP file, a recorded capture) |
|
||||
| **Adapter** | A software module that knows how to receive and decode source-specific CSI and normalize it into a `CsiFrame` |
|
||||
| **Frame** | One CSI observation at a timestamp — the unit of ingestion |
|
||||
| **Window** | A bounded sequence of frames from one source/session, used for analysis |
|
||||
| **Baseline** | The learned normal RF-field state for a space |
|
||||
| **Delta** | The measured difference of the current field from baseline |
|
||||
| **Event** | A semantic interpretation of one or more windows (presence started, motion detected, anomaly, …) |
|
||||
| **Quality score** | Confidence, in [0, 1], that a signal/frame/window is usable |
|
||||
| **Calibration** | The process of learning a stable baseline for a space |
|
||||
| **Room signature** | A vector representation of a space under normal conditions |
|
||||
| **Drift** | Slow movement of the field away from baseline |
|
||||
| **Anomaly** | A significant, unexplained deviation from baseline |
|
||||
| **RF memory** | Persisted temporal vectors and events for a physical space (stored in RuVector) |
|
||||
| **Coherence** | Consistency among sources, windows, and learned baselines |
|
||||
| **Quarantine** | A holding store for rejected/corrupt frames, kept for audit rather than discarded |
|
||||
| **Adapter profile** | A capability descriptor for a source (chip, firmware/driver versions, supported channels/bandwidths, expected subcarrier counts, capture/injection/monitor-mode support) |
|
||||
| **Calibration version** | An immutable identifier for a particular learned baseline; every event references the calibration version it was detected against |
|
||||
| **Evidence window set** | The set of `WindowId`s an event references as its justification — an event with no evidence is invalid |
|
||||
|
||||
---
|
||||
|
||||
## Bounded Contexts
|
||||
|
||||
```
|
||||
┌─────────────┐ ┌──────────────┐ ┌────────────┐ ┌──────────────┐
|
||||
│ Capture │──▶│ Validation │──▶│ Signal │──▶│ Calibration │
|
||||
│ context │ │ context │ │ context │ │ context │
|
||||
└─────────────┘ └──────────────┘ └─────┬──────┘ └──────┬───────┘
|
||||
│ │
|
||||
▼ │
|
||||
┌────────────┐ │
|
||||
│ Event │◀──────────┘
|
||||
│ context │
|
||||
└─────┬──────┘
|
||||
│
|
||||
┌─────────────┴─────────────┐
|
||||
▼ ▼
|
||||
┌────────────┐ ┌────────────┐
|
||||
│ Memory │ │ Agent │
|
||||
│ context │ │ context │
|
||||
└────────────┘ └────────────┘
|
||||
```
|
||||
|
||||
- **Capture** upstreams raw input from sources.
|
||||
- **Validation** protects every downstream context — nothing crosses into SDK/DSP/memory/agents unvalidated.
|
||||
- **Signal** turns frames into windows.
|
||||
- **Calibration** gives windows a room-specific baseline.
|
||||
- **Event** converts deltas into meaning.
|
||||
- **Memory** stores time, similarity, drift, and coherence (RuVector).
|
||||
- **Agent** exposes safe actions and queries (MCP / TypeScript).
|
||||
|
||||
---
|
||||
|
||||
### 1. Capture context
|
||||
|
||||
**Responsibility:** connect to CSI sources and produce raw frames.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────┐
|
||||
│ Capture Context │
|
||||
├──────────────────────────────────────────────────────────────┤
|
||||
│ ┌────────────┐ ┌─────────────────┐ ┌─────────────────┐ │
|
||||
│ │ Source │ │ CaptureSession │ │ AdapterProfile │ │
|
||||
│ │ (adapter │ │ (aggregate root)│ │ (capability │ │
|
||||
│ │ plugin) │ │ │ │ descriptor) │ │
|
||||
│ └────────────┘ └─────────────────┘ └─────────────────┘ │
|
||||
│ │
|
||||
│ CsiSource trait: open · start · next_frame · stop · health │
|
||||
└──────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
| Element | Kind | Notes |
|
||||
|---------|------|-------|
|
||||
| `Source` | Entity | A configured adapter instance bound to a device or file |
|
||||
| `CaptureSession` | Entity / **aggregate root** | Owns exactly one `AdapterProfile` and one runtime configuration |
|
||||
| `AdapterProfile` | Entity | Chip, firmware/driver versions, supported channels/bandwidths, expected subcarrier counts, capability flags |
|
||||
| `Channel`, `Bandwidth`, `FirmwareVersion`, `DriverVersion` | Value objects | Immutable |
|
||||
|
||||
**Commands:** `StartCapture` · `StopCapture` · `RestartCapture` · `InspectSource`
|
||||
**Domain events:** `CaptureStarted` · `CaptureStopped` · `SourceDisconnected` · `AdapterUnsupported`
|
||||
|
||||
---
|
||||
|
||||
### 2. Validation context
|
||||
|
||||
**Responsibility:** make frames safe and trustworthy before any language-boundary crossing.
|
||||
|
||||
| Element | Kind | Notes |
|
||||
|---------|------|-------|
|
||||
| `ValidationPolicy` | Entity | Bounds, monotonicity rules, finiteness checks, quarantine on/off |
|
||||
| `QuarantineStore` | Entity | Holds rejected/corrupt frames for audit |
|
||||
| `ValidatedFrame` | **Aggregate root** | The frame once it has passed (or been degraded by) validation |
|
||||
| `ValidationError`, `QualityScore`, `FrameBounds` | Value objects | `QualityScore` ∈ [0, 1] |
|
||||
|
||||
**Commands:** `ValidateFrame` · `QuarantineFrame`
|
||||
**Domain events:** `FrameAccepted` · `FrameRejected` · `QualityDropped`
|
||||
|
||||
---
|
||||
|
||||
### 3. Signal context
|
||||
|
||||
**Responsibility:** DSP and window features.
|
||||
|
||||
```
|
||||
Frame stream ─▶ SignalPipeline ─▶ WindowBuffer ─▶ CsiWindow
|
||||
(DC removal, phase unwrap, (mean amplitude,
|
||||
smoothing, Hampel filter, phase variance,
|
||||
variance, baseline subtraction, motion energy,
|
||||
motion energy, presence score) presence/quality scores)
|
||||
```
|
||||
|
||||
| Element | Kind | Notes |
|
||||
|---------|------|-------|
|
||||
| `SignalPipeline` | Entity | Ordered DSP stages; reuses `wifi-densepose-signal` primitives |
|
||||
| `WindowBuffer` | Entity | Accumulates frames into bounded windows |
|
||||
| `CsiWindow` | **Aggregate root** | Frames from exactly one source/session |
|
||||
| `AmplitudeVector`, `PhaseVector`, `MotionEnergy`, `PresenceScore` | Value objects | |
|
||||
|
||||
**Commands:** `ProcessFrame` · `BuildWindow` · `EstimateBaselineDelta`
|
||||
**Domain events:** `WindowReady` · `BaselineDeltaMeasured`
|
||||
|
||||
---
|
||||
|
||||
### 4. Calibration context
|
||||
|
||||
**Responsibility:** learn and version the normal RF state and room signatures.
|
||||
|
||||
| Element | Kind | Notes |
|
||||
|---------|------|-------|
|
||||
| `CalibrationProfile` | **Aggregate root** | Linked to source, room, adapter profile, configuration |
|
||||
| `RoomSignature` | Entity | Vector representation of a space under normal conditions |
|
||||
| `BaselineModel` | Entity | Statistical model of the baseline field; carries version history |
|
||||
| `CalibrationVersion`, `StabilityScore`, `RoomId` | Value objects | Calibration cannot complete if `StabilityScore` < threshold |
|
||||
|
||||
**Commands:** `StartCalibration` · `CompleteCalibration` · `UpdateBaseline` · `RejectUnstableCalibration`
|
||||
**Domain events:** `CalibrationStarted` · `CalibrationCompleted` · `CalibrationFailed` · `BaselineUpdated`
|
||||
|
||||
---
|
||||
|
||||
### 5. Event context
|
||||
|
||||
**Responsibility:** semantic event extraction with confidence and evidence.
|
||||
|
||||
| Element | Kind | Notes |
|
||||
|---------|------|-------|
|
||||
| `EventDetector` | Entity | One per event family (presence, motion, breathing, anomaly, …) |
|
||||
| `EventStateMachine` | Entity | Holds the per-source detection state; emits transitions |
|
||||
| `CsiEvent` | **Aggregate root** | Must reference ≥ 1 evidence window; confidence ∈ [0, 1]; references calibration version |
|
||||
| `Confidence`, `EvidenceWindowSet`, `EventKind` | Value objects | |
|
||||
|
||||
**Commands:** `DetectEvents` · `PublishEvent` · `SuppressEvent`
|
||||
**Domain events (the `CsiEventKind` enum):** `PresenceStarted` · `PresenceEnded` · `MotionDetected` · `MotionSettled` · `BaselineChanged` · `SignalQualityDropped` · `DeviceDisconnected` · `BreathingCandidate` · `AnomalyDetected` · `CalibrationRequired`
|
||||
|
||||
---
|
||||
|
||||
### 6. Memory context
|
||||
|
||||
**Responsibility:** RuVector storage and retrieval — RF memory.
|
||||
|
||||
| Element | Kind | Notes |
|
||||
|---------|------|-------|
|
||||
| `RfMemoryCollection` | Entity | A RuVector collection scoped to a deployment |
|
||||
| `TemporalEmbedding` | Entity | Frame / window / event embedding with timestamp |
|
||||
| `SensorGraph` | Entity | Graph of sources and their topological relationships |
|
||||
| `RoomMemory` | **Aggregate root** | Stored embeddings must be traceable to frame windows or event windows |
|
||||
| `EmbeddingVector`, `DriftScore`, `CoherenceScore` | Value objects | `DriftScore` must include the baseline version |
|
||||
|
||||
**Commands:** `StoreWindowEmbedding` · `StoreEventEmbedding` · `QuerySimilarWindows` · `ComputeDrift`
|
||||
**Domain events:** `EmbeddingStored` · `DriftDetected` · `SimilarPatternFound`
|
||||
|
||||
Data stored: frame embeddings · window embeddings · room baseline vectors · event vectors · drift snapshots · sensor-topology graph edges · source health records. Retention policy applies at collection level. No orphan embeddings.
|
||||
|
||||
---
|
||||
|
||||
### 7. Agent context
|
||||
|
||||
**Responsibility:** MCP and TypeScript agent interaction — safe actions and queries.
|
||||
|
||||
| Element | Kind | Notes |
|
||||
|---------|------|-------|
|
||||
| `AgentSubscription` | Entity | An agent's filtered stream of events |
|
||||
| `McpToolSession` | Entity | A tool invocation context with permissions |
|
||||
| `AgentSession` | **Aggregate root** | |
|
||||
| `ToolPermission`, `EventFilter`, `AgentIntent` | Value objects | `ToolPermission` distinguishes read vs. write-gated |
|
||||
|
||||
**Commands:** `SubscribeToEvents` · `RequestStatus` · `RequestCalibration` · `QueryMemory`
|
||||
**Domain events:** `AgentSubscribed` · `ToolExecuted` · `PermissionDenied`
|
||||
|
||||
**MCP tools** (read by default; write-gated marked `*`): `rvcsi_status` · `rvcsi_list_sources` · `rvcsi_start_capture *` · `rvcsi_stop_capture *` · `rvcsi_get_presence` · `rvcsi_get_recent_events` · `rvcsi_calibrate_room *` · `rvcsi_export_window *` · `rvcsi_query_ruvector` · `rvcsi_health_report`.
|
||||
|
||||
---
|
||||
|
||||
## Context Map
|
||||
|
||||
| Upstream → Downstream | Relationship | ACL / contract |
|
||||
|-----------------------|--------------|----------------|
|
||||
| Capture → Validation | Customer/Supplier | Raw frames pass through `ValidationPolicy`; only `Accepted`/`Degraded` continue |
|
||||
| Validation → Signal | Conformist (Signal accepts `ValidatedFrame` as-is) | `CsiFrame` schema is the published language |
|
||||
| Signal → Calibration | Customer/Supplier | Windows + baseline-delta measurements feed baseline modeling |
|
||||
| Calibration → Event | Customer/Supplier | Detectors declare which `CalibrationVersion` they used |
|
||||
| Signal/Event → Memory | Published Language (`EmbeddingVector`, event metadata) | `rvcsi-ruvector` ACL translates to RuVector's API |
|
||||
| Event → Agent | Open Host Service (event stream + MCP tools) | `EventFilter` + `ToolPermission` enforced at the boundary |
|
||||
| Capture → Agent | Conformist (health/status only, via MCP read tools) | No raw frames cross to agents |
|
||||
|
||||
The **`CsiFrame` schema is the shared kernel** between Capture, Validation, Signal, and the language-boundary (napi-rs) layer. It is the FFI-safe object; nothing device-specific leaks past it.
|
||||
|
||||
---
|
||||
|
||||
## Aggregates and Invariants
|
||||
|
||||
### `CaptureSession` aggregate
|
||||
|
||||
**Invariant:** a capture session has exactly one source profile and one runtime configuration.
|
||||
|
||||
1. A session cannot emit frames before it is started.
|
||||
2. A session cannot change channel without restart unless the adapter supports dynamic retune.
|
||||
3. A session must emit `SourceDisconnected` before stopping due to device loss.
|
||||
|
||||
### `ValidatedFrame` aggregate
|
||||
|
||||
**Invariant:** no frame crosses into SDK, DSP, memory, or agents unless its validation status is `Accepted` or `Degraded`.
|
||||
|
||||
1. Rejected frames go to quarantine when quarantine is enabled.
|
||||
2. Degraded frames must carry quality-reason metadata.
|
||||
3. Missing *optional* hardware metadata must not invalidate a frame.
|
||||
|
||||
### `CsiWindow` aggregate
|
||||
|
||||
**Invariant:** a window contains frames from exactly one source and one session.
|
||||
|
||||
1. Mixed-source windows are not allowed.
|
||||
2. Window start time must be strictly less than end time.
|
||||
3. Window quality is bounded in [0, 1].
|
||||
|
||||
### `CalibrationProfile` aggregate
|
||||
|
||||
**Invariant:** a calibration profile is linked to source, room, adapter profile, and configuration.
|
||||
|
||||
1. Calibration cannot complete if `StabilityScore` is below threshold.
|
||||
2. Baseline updates must preserve version history.
|
||||
3. Event detectors must declare which calibration version they used.
|
||||
|
||||
### `CsiEvent` aggregate
|
||||
|
||||
**Invariant:** an event must have evidence.
|
||||
|
||||
1. Every event references at least one evidence window.
|
||||
2. Confidence is bounded in [0, 1].
|
||||
3. Event suppression must be explainable by policy.
|
||||
|
||||
### `RoomMemory` aggregate
|
||||
|
||||
**Invariant:** stored embeddings are traceable to frame windows or event windows.
|
||||
|
||||
1. No orphan embeddings.
|
||||
2. Retention policy applies at collection level.
|
||||
3. Drift scores must include the baseline version.
|
||||
|
||||
---
|
||||
|
||||
## Data Model
|
||||
|
||||
```rust
|
||||
pub struct CsiFrame {
|
||||
pub frame_id: FrameId,
|
||||
pub session_id: SessionId,
|
||||
pub source_id: SourceId,
|
||||
pub adapter_kind: AdapterKind,
|
||||
pub timestamp_ns: u64,
|
||||
pub channel: u16,
|
||||
pub bandwidth_mhz: u16,
|
||||
pub rssi_dbm: Option<i16>,
|
||||
pub noise_floor_dbm: Option<i16>,
|
||||
pub antenna_index: Option<u8>,
|
||||
pub tx_chain: Option<u8>,
|
||||
pub rx_chain: Option<u8>,
|
||||
pub subcarrier_count: u16,
|
||||
pub i_values: Vec<f32>,
|
||||
pub q_values: Vec<f32>,
|
||||
pub amplitude: Vec<f32>,
|
||||
pub phase: Vec<f32>,
|
||||
pub validation: ValidationStatus,
|
||||
pub quality_score: f32,
|
||||
pub calibration_version: Option<String>,
|
||||
}
|
||||
|
||||
pub struct CsiWindow {
|
||||
pub window_id: WindowId,
|
||||
pub session_id: SessionId,
|
||||
pub source_id: SourceId,
|
||||
pub start_ns: u64,
|
||||
pub end_ns: u64,
|
||||
pub frame_count: u32,
|
||||
pub mean_amplitude: Vec<f32>,
|
||||
pub phase_variance: Vec<f32>,
|
||||
pub motion_energy: f32,
|
||||
pub presence_score: f32,
|
||||
pub quality_score: f32,
|
||||
}
|
||||
|
||||
pub enum CsiEventKind {
|
||||
PresenceStarted,
|
||||
PresenceEnded,
|
||||
MotionDetected,
|
||||
MotionSettled,
|
||||
BaselineChanged,
|
||||
SignalQualityDropped,
|
||||
DeviceDisconnected,
|
||||
BreathingCandidate,
|
||||
AnomalyDetected,
|
||||
CalibrationRequired,
|
||||
}
|
||||
|
||||
pub struct CsiEvent {
|
||||
pub event_id: EventId,
|
||||
pub kind: CsiEventKind,
|
||||
pub session_id: SessionId,
|
||||
pub source_id: SourceId,
|
||||
pub timestamp_ns: u64,
|
||||
pub confidence: f32,
|
||||
pub evidence_window_ids: Vec<WindowId>,
|
||||
pub metadata_json: String,
|
||||
}
|
||||
|
||||
pub struct AdapterProfile {
|
||||
pub adapter_kind: AdapterKind,
|
||||
pub chip: Option<String>,
|
||||
pub firmware_version: Option<String>,
|
||||
pub driver_version: Option<String>,
|
||||
pub supported_channels: Vec<u16>,
|
||||
pub supported_bandwidths_mhz: Vec<u16>,
|
||||
pub expected_subcarrier_counts: Vec<u16>,
|
||||
pub supports_live_capture: bool,
|
||||
pub supports_injection: bool,
|
||||
pub supports_monitor_mode: bool,
|
||||
}
|
||||
|
||||
pub enum ValidationStatus { Accepted, Degraded, Rejected, Recovered }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Domain Services
|
||||
|
||||
| Service | Input | Output | Responsibility |
|
||||
|---------|-------|--------|----------------|
|
||||
| `FrameValidationService` | `RawFrame`, `AdapterProfile`, `ValidationPolicy` | `ValidatedFrame` or `RejectedFrame` | Enforce bounds, finiteness, monotonicity; assign initial `QualityScore`; route rejects to quarantine; emit structured errors |
|
||||
| `SignalProcessingService` | `ValidatedFrame` stream | `CsiWindow` stream | Run the DSP pipeline; build bounded windows; compute motion energy, presence score, window quality |
|
||||
| `BaselineDeltaService` | `CsiWindow`, `BaselineModel` | `BaselineDelta` | Subtract the calibrated baseline; measure deviation magnitude |
|
||||
| `CalibrationService` | `CsiWindow` stream over a calibration window | `CalibrationProfile` (new version) or `CalibrationFailed` | Learn a stable baseline; compute `StabilityScore`; reject unstable calibrations; preserve version history |
|
||||
| `EventDetectionService` | `CsiWindow` + `BaselineDelta` + `CalibrationVersion` | `CsiEvent` stream | Drive per-source state machines; attach confidence + evidence windows + calibration version; apply suppression policy |
|
||||
| `EmbeddingService` | `CsiWindow` / `CsiEvent` | `TemporalEmbedding` | Produce frame/window/event vectors (v0: deterministic DSP feature vector; later: AETHER / on-device model) |
|
||||
| `RfMemoryService` | `TemporalEmbedding`, query | `EmbeddingStored` / similar windows / `DriftScore` | Store to RuVector; similarity search; drift computation against a baseline version |
|
||||
| `ReplayService` | A captured session bundle | A deterministic frame/window/event stream | Replay preserving timestamps, ordering, validation decisions, event output, calibration version, runtime config |
|
||||
| `AdapterRegistryService` | — | List of available adapters + `AdapterProfile`s | Discover sources (reuses ADR-049 interface detection); report health; flag unsupported firmware/driver state |
|
||||
| `AgentGatewayService` | MCP tool call / SDK subscription | Tool result / filtered event stream | Enforce `ToolPermission` (read vs. write-gated), apply `EventFilter`, audit `ToolExecuted` / `PermissionDenied` |
|
||||
|
||||
---
|
||||
|
||||
## Related
|
||||
|
||||
- [rvCSI Platform PRD](../prd/rvcsi-platform-prd.md) — requirements, success criteria, scope
|
||||
- [ADR-095 — rvCSI Edge RF Sensing Platform](../adr/ADR-095-rvcsi-edge-rf-sensing-platform.md) — the fifteen architectural decisions
|
||||
- [RuvSense Domain Model](ruvsense-domain-model.md) — adjacent multistatic sensing context
|
||||
- [Signal Processing Domain Model](signal-processing-domain-model.md) — the DSP primitives `rvcsi-dsp` reuses
|
||||
- [ADR Index](../adr/README.md)
|
||||
@@ -168,14 +168,14 @@ The training process works like this:
|
||||
1. **Collect** raw CSI frames from ESP32-S3 nodes placed in a room
|
||||
2. **Extract** 8-dimensional feature vectors from sliding windows of CSI data
|
||||
3. **Contrast** -- the model learns that features from nearby time windows should produce similar embeddings, while features from different scenarios should produce different embeddings
|
||||
4. **Fine-tune** task heads using weak labels from environmental sensors (PIR motion, temperature, pressure) on the Cognitum Seed companion device
|
||||
4. **Fine-tune** task heads — *planned:* weak labels from environmental sensors (PIR motion, temperature, pressure) on the Cognitum Seed companion device. **This environmental-sensor ground-truth path is not yet implemented** (no PIR/BME280 ingestion in the training pipeline today); current task-head supervision uses the proxy/camera labels described elsewhere.
|
||||
|
||||
### Data provenance
|
||||
|
||||
- **Source:** Live CSI from 2x ESP32-S3 nodes (802.11n, HT40, 114 subcarriers)
|
||||
- **Volume:** ~360,000 CSI frames (~3,600 feature vectors) per collection run
|
||||
- **Environment:** Residential room, ~4x5 meters
|
||||
- **Ground truth:** Environmental sensors on Cognitum Seed (PIR, BME280, light)
|
||||
- **Ground truth:** *Planned* — environmental sensors on the Cognitum Seed (PIR, BME280, light). Not yet wired into training; treat the PIR/BME280 references in this card as the intended design, not a current capability.
|
||||
- **Attestation:** Every collection run produces a cryptographic witness chain (`collection-witness.json`) that proves data provenance and integrity
|
||||
|
||||
### Witness chain
|
||||
@@ -208,7 +208,7 @@ Add a second ESP32-S3 to enable cross-node signal fusion for better accuracy and
|
||||
| USB-C cables (x3) | Power + data | ~$9 |
|
||||
| **Total** | | **~$27** |
|
||||
|
||||
The Cognitum Seed runs the ONNX models on-device, orchestrates the ESP32 nodes over USB serial, and provides environmental ground truth via its onboard PIR and BME280 sensors.
|
||||
The Cognitum Seed runs the ONNX models on-device and orchestrates the ESP32 nodes over USB serial. (Using its onboard PIR/BME280 sensors as training ground truth is planned but not yet implemented — see "Data provenance" above.)
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
# rvCSI — Edge RF Sensing Runtime
|
||||
|
||||
## Product Design Requirements (PRD)
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Product name** | rvCSI |
|
||||
| **Category** | Edge RF sensing runtime and developer platform |
|
||||
| **Status** | Proposed (v0 design) |
|
||||
| **Date** | 2026-05-12 |
|
||||
| **Owner** | ruv |
|
||||
| **Relates to** | [ADR-095](../adr/ADR-095-rvcsi-edge-rf-sensing-platform.md) (rvCSI platform), [ADR-012](../adr/ADR-012-esp32-csi-sensor-mesh.md) (ESP32 mesh), [ADR-013](../adr/ADR-013-feature-level-sensing-commodity-gear.md) (feature-level sensing), [ADR-014](../adr/ADR-014-sota-signal-processing.md) (SOTA signal processing), [ADR-016](../adr/ADR-016-ruvector-integration.md) (RuVector integration), [ADR-024](../adr/ADR-024-contrastive-csi-embedding-model.md) (AETHER embeddings), [ADR-031](../adr/ADR-031-ruview-sensing-first-rf-mode.md) (RuView sensing-first RF mode), [ADR-040](../adr/ADR-040-wasm-programmable-sensing.md) (WASM programmable sensing) |
|
||||
| **Domain model** | [rvCSI Domain Model](../ddd/rvcsi-domain-model.md) |
|
||||
|
||||
---
|
||||
|
||||
## 1. Purpose
|
||||
|
||||
rvCSI is a **Rust-first, TypeScript-accessible, hardware-abstracted Channel State Information (CSI) platform** for WiFi-based spatial sensing.
|
||||
|
||||
The goal is to convert CSI from fragile research data into a durable edge sensing runtime that can feed RuView, RuVector, Cognitum, and agentic systems with validated live radio-field observations.
|
||||
|
||||
rvCSI does **not** try to replace Nexmon on day one. It wraps, validates, normalizes, streams, embeds, and learns from CSI produced by Nexmon, ESP32 CSI, Intel CSI, Atheros CSI, SDR pipelines, and future RF sensor sources.
|
||||
|
||||
### 1.1 System framing
|
||||
|
||||
CSI is treated as a **physical-world delta stream**.
|
||||
|
||||
A room, hallway, vehicle, warehouse, machine bay, or care facility has a radio-field baseline. Human motion, breathing, door movement, equipment vibration, device movement, and environmental change perturb that baseline. rvCSI captures those perturbations, normalizes them into tensors, converts them into events, stores them as temporal memory, and exposes them to agents.
|
||||
|
||||
The core invariant:
|
||||
|
||||
| Layer | Owns |
|
||||
|-------|------|
|
||||
| **C** | Fragile vendor and firmware compatibility |
|
||||
| **Rust** | Safety, validation, signal processing, memory discipline, deterministic runtime behavior |
|
||||
| **TypeScript** | Developer experience, orchestration, dashboards, SDKs, agent integration |
|
||||
| **RuVector** | Memory, similarity, drift, graph relationships, coherence over time |
|
||||
| **Cognitum** | Low-power event-driven deployment, local decision loops |
|
||||
|
||||
### 1.2 Strategic framing
|
||||
|
||||
Most CSI projects today are Linux shell scripts, kernel patching, Python notebooks, PCAP dumps, and ad-hoc signal processing. A Rust + TypeScript + napi-rs architecture turns CSI into **real-time sensor infrastructure**: npm-installable, reproducible, typed, safe-parsed, embeddable, WebSocket-streamable, WASM-portable, MCP-exposed, agent-integrable, and edge/cloud-federated.
|
||||
|
||||
The right framing is **structural sensing**, not "magic X-ray vision". CSI is excellent for detecting change, presence, and learned patterns; it is weak for exact identity, exact pose, legal/security certainty, and highly dynamic RF spaces. rvCSI's product claims stay inside that boundary (see Non-goals, §6).
|
||||
|
||||
---
|
||||
|
||||
## 2. Users
|
||||
|
||||
| User | Need |
|
||||
|------|------|
|
||||
| AI engineers building physical-world agents | A stable sensing primitive that emits typed events agents can react to |
|
||||
| Researchers working with WiFi CSI and RF sensing | Reproducible ingestion, replay, and benchmark datasets |
|
||||
| Smart-building and elder-care solution builders | Privacy-preserving presence/motion/breathing without cameras |
|
||||
| Industrial monitoring teams | Camera-free movement/anomaly detection that runs unattended |
|
||||
| Developers using RuView / RuVector / Cognitum | A drop-in source of RF observations for the broader ruvnet stack |
|
||||
|
||||
---
|
||||
|
||||
## 3. Problem & Hypothesis
|
||||
|
||||
**Problem.** WiFi CSI is useful but hard to operationalize. Most CSI pipelines are built from fragile scripts, patched firmware, lab notebooks, inconsistent packet formats, unstable drivers, and device-specific assumptions. This makes CSI difficult to deploy outside research settings. The system needs a production-grade runtime that can ingest CSI from multiple sources, validate packets, normalize formats, stream typed events, support signal processing, and feed vector-based learning systems.
|
||||
|
||||
**Hypothesis.** If rvCSI provides a stable Rust core with TypeScript APIs and hardware adapters, then CSI can become a reusable sensing primitive for camera-free spatial intelligence.
|
||||
|
||||
---
|
||||
|
||||
## 4. Success criteria
|
||||
|
||||
1. A developer can install rvCSI and parse recorded CSI files in **under five minutes**.
|
||||
2. A supported live device can stream **validated** CSI frames into TypeScript.
|
||||
3. Bad packets **cannot crash** the process.
|
||||
4. The same application code consumes CSI from Nexmon, ESP32, Intel, or Atheros adapters.
|
||||
5. Presence and motion detection work from **normalized tensors**, not device-specific raw packets.
|
||||
6. rvCSI can publish embeddings and event summaries into **RuVector**.
|
||||
7. rvCSI can run as a **local daemon on Raspberry Pi-class hardware**.
|
||||
8. rvCSI can expose events to **MCP tools and local agents**.
|
||||
|
||||
---
|
||||
|
||||
## 5. Scope
|
||||
|
||||
### 5.1 Version zero — safe ingestion, normalized data, live streaming, SDK usability, RuVector integration
|
||||
|
||||
1. Recorded CSI file parser
|
||||
2. Live capture adapter for existing Nexmon CSI output where supported
|
||||
3. ESP32 CSI adapter
|
||||
4. Unified CSI frame schema
|
||||
5. Rust validation pipeline
|
||||
6. TypeScript SDK through napi-rs
|
||||
7. CLI for capture, inspect, replay, stream
|
||||
8. WebSocket output
|
||||
9. Presence and motion baseline detectors
|
||||
10. RuVector export interface
|
||||
11. Basic calibration model
|
||||
12. Hardware and driver health checks
|
||||
|
||||
### 5.2 Version one
|
||||
|
||||
1. Multi-node synchronization
|
||||
2. RF room signatures
|
||||
3. Breathing-rate estimation where signal quality permits
|
||||
4. Temporal embeddings
|
||||
5. Drift detection
|
||||
6. Graph-based room topology
|
||||
7. Local MCP tool server
|
||||
8. Replayable benchmark datasets
|
||||
9. Sensor fusion with RuView
|
||||
10. Deployment profile for Cognitum Seed and Appliance
|
||||
|
||||
### 5.3 Version two
|
||||
|
||||
1. Hardware-agnostic RF sensor fabric
|
||||
2. Multi-room RF memory
|
||||
3. Streaming anomaly detection
|
||||
4. RF SLAM research mode
|
||||
5. On-device embedding model
|
||||
6. Federated learning of room signatures
|
||||
7. Secure signed sensor-evidence records
|
||||
8. Proof-gated event publication
|
||||
9. Dynamic cut-based coherence over RF graphs
|
||||
10. Agent-driven calibration and self-repair
|
||||
|
||||
---
|
||||
|
||||
## 6. Non-goals (version zero)
|
||||
|
||||
1. Pure-Rust replacement for Broadcom firmware patches
|
||||
2. Universal support for all WiFi chips
|
||||
3. Identity recognition from RF signals
|
||||
4. Medical-grade vital-sign diagnosis
|
||||
5. Legal-grade occupancy proof
|
||||
6. Guaranteed through-wall pose detection
|
||||
7. Cloud dependency
|
||||
8. Camera-replacement claims
|
||||
|
||||
---
|
||||
|
||||
## 7. Functional requirements
|
||||
|
||||
### FR1 — CSI ingestion
|
||||
|
||||
rvCSI shall ingest CSI from multiple sources. Initial source types: recorded binary dump, PCAP file, Nexmon CSI live stream, ESP32 CSI serial/UDP stream, Intel CSI logs (where supported), Atheros CSI logs (where supported). **Output:** a normalized `CsiFrame` object.
|
||||
|
||||
### FR2 — Packet validation
|
||||
|
||||
rvCSI shall validate every frame before exposing it to TypeScript or RuVector:
|
||||
|
||||
1. Frame length must match declared schema.
|
||||
2. Subcarrier count must be inside adapter-profile limits.
|
||||
3. Timestamp must be monotonic within a capture session unless marked as recovered.
|
||||
4. RSSI must be within plausible device bounds.
|
||||
5. Complex values must be finite.
|
||||
6. Corrupt frames must be rejected or quarantined.
|
||||
7. Parser failures must return structured errors.
|
||||
|
||||
### FR3 — Normalized frame schema
|
||||
|
||||
rvCSI shall normalize all hardware output into a common schema. Required fields: `frame_id`, `session_id`, `source_id`, `adapter_kind`, `timestamp_ns`, `channel`, `bandwidth_mhz`, `rssi_dbm`, `noise_floor_dbm` (when available), `antenna_index` (when available), `tx_chain` (when available), `rx_chain` (when available), `subcarrier_count`, `i_values`, `q_values`, `amplitude`, `phase`, `validation_status`, `quality_score`, `calibration_version`.
|
||||
|
||||
### FR4 — Signal processing
|
||||
|
||||
rvCSI shall provide reusable Rust signal-processing stages: DC offset removal, phase unwrap, amplitude smoothing, Hampel/median outlier filter, short-window variance, baseline subtraction, motion energy, presence score, breathing-band estimator (where supported), confidence scoring.
|
||||
|
||||
### FR5 — Event extraction
|
||||
|
||||
rvCSI shall convert frame streams into typed events: `PresenceStarted`, `PresenceEnded`, `MotionDetected`, `MotionSettled`, `BaselineChanged`, `SignalQualityDropped`, `DeviceDisconnected`, `BreathingCandidate`, `AnomalyDetected`, `CalibrationRequired`.
|
||||
|
||||
### FR6 — TypeScript SDK
|
||||
|
||||
rvCSI shall expose a TypeScript SDK:
|
||||
|
||||
```ts
|
||||
import { RvCsi } from "@ruv/rvcsi";
|
||||
|
||||
const sensor = await RvCsi.open({
|
||||
source: "nexmon",
|
||||
iface: "wlan0",
|
||||
channel: 6,
|
||||
bandwidthMHz: 20,
|
||||
});
|
||||
|
||||
sensor.on("frame", (frame) => {
|
||||
console.log(frame.qualityScore);
|
||||
});
|
||||
|
||||
sensor.on("presence", (event) => {
|
||||
console.log(event.confidence);
|
||||
});
|
||||
|
||||
await sensor.start();
|
||||
```
|
||||
|
||||
### FR7 — CLI
|
||||
|
||||
```bash
|
||||
rvcsi inspect file sample.csi
|
||||
rvcsi capture start --source nexmon --iface wlan0 --channel 6
|
||||
rvcsi replay sample.csi --speed 1x
|
||||
rvcsi stream --format json --port 8787
|
||||
rvcsi calibrate --room livingroom --duration 60
|
||||
rvcsi health --source nexmon
|
||||
rvcsi export ruvector --collection room_rf
|
||||
```
|
||||
|
||||
### FR8 — RuVector integration
|
||||
|
||||
rvCSI shall export temporal RF embeddings and event metadata to RuVector. Data stored: frame embeddings, window embeddings, room baseline vectors, event vectors, drift snapshots, sensor-topology graph edges, source health records.
|
||||
|
||||
### FR9 — MCP integration
|
||||
|
||||
rvCSI shall expose MCP tools for local agents: `rvcsi_status`, `rvcsi_list_sources`, `rvcsi_start_capture`, `rvcsi_stop_capture`, `rvcsi_get_presence`, `rvcsi_get_recent_events`, `rvcsi_calibrate_room`, `rvcsi_export_window`, `rvcsi_query_ruvector`, `rvcsi_health_report`. Tools default to read actions; capture start/stop, calibration, and export are write-gated.
|
||||
|
||||
### FR10 — Replay and audit
|
||||
|
||||
rvCSI shall support deterministic replay of captured sessions, preserving: original timestamps, frame ordering, validation decisions, event-extraction output, calibration version, runtime configuration.
|
||||
|
||||
---
|
||||
|
||||
## 8. Non-functional requirements
|
||||
|
||||
### 8.1 Safety
|
||||
|
||||
1. TypeScript shall never receive raw unchecked pointers.
|
||||
2. Rust shall validate all frames before the FFI boundary export.
|
||||
3. C shims shall be minimal and isolated.
|
||||
4. All `unsafe` blocks shall be documented.
|
||||
5. Fuzz tests shall cover parsers.
|
||||
|
||||
### 8.2 Performance (v0 targets)
|
||||
|
||||
1. Parse one CSI frame in **< 1 ms** on Raspberry Pi 5.
|
||||
2. Sustain **≥ 1000 frames/s** on Pi 5 for normalized parsing.
|
||||
3. Keep memory **< 256 MB** for one active source.
|
||||
4. Keep event latency **< 50 ms** for presence and motion.
|
||||
5. Avoid heap growth during steady capture.
|
||||
|
||||
### 8.3 Reliability
|
||||
|
||||
1. Bad packets shall not crash the daemon.
|
||||
2. Device disconnect shall produce a typed event.
|
||||
3. Capture sessions shall be restartable.
|
||||
4. Logs shall include source, adapter, session, and validation details.
|
||||
5. Health checks shall identify unsupported firmware or driver state.
|
||||
|
||||
### 8.4 Privacy
|
||||
|
||||
1. rvCSI shall operate locally by default.
|
||||
2. No cloud endpoint shall be required.
|
||||
3. Raw CSI export shall be disableable by policy.
|
||||
4. Event-level export shall be supported for privacy-preserving deployments.
|
||||
5. Retention policies shall be configurable.
|
||||
|
||||
### 8.5 Security
|
||||
|
||||
1. Device-control operations shall require explicit permission.
|
||||
2. Firmware-installation operations shall be separated from capture operations.
|
||||
3. Signed capture profiles shall be supported in later versions.
|
||||
4. MCP tools shall mark write actions as gated.
|
||||
5. File parsing shall be fuzzed and sandbox-friendly.
|
||||
|
||||
### 8.6 Portability
|
||||
|
||||
1. Linux first.
|
||||
2. Raspberry Pi first among edge devices.
|
||||
3. macOS and Windows support for file replay and SDK development.
|
||||
4. Live-capture support depends on adapter and driver capability.
|
||||
5. WASM support for offline parsing and visualization is a later target.
|
||||
|
||||
---
|
||||
|
||||
## 9. System architecture
|
||||
|
||||
### 9.1 High-level pipeline
|
||||
|
||||
```
|
||||
CSI Source
|
||||
↓
|
||||
Adapter Layer (vendor-specific decode, C shims isolated here)
|
||||
↓
|
||||
Rust Validation Pipeline (bounds, finiteness, monotonicity, quarantine)
|
||||
↓
|
||||
Normalized CSI Frame (CsiFrame schema — the FFI-safe boundary object)
|
||||
↓
|
||||
Signal Processing (DC removal, phase unwrap, smoothing, motion energy …)
|
||||
↓
|
||||
Window Aggregator (bounded frame sequences → CsiWindow)
|
||||
↓
|
||||
Event Extractor (state machines → CsiEvent with confidence + evidence)
|
||||
↓
|
||||
TypeScript SDK · CLI · MCP · RuVector
|
||||
```
|
||||
|
||||
### 9.2 Runtime components
|
||||
|
||||
| # | Component | Role |
|
||||
|---|-----------|------|
|
||||
| 1 | `rvcsi-core` | Frame types, parser traits, validation, quality scoring, shared abstractions |
|
||||
| 2 | `rvcsi-adapter-*` | Rust/C-backed adapters: Nexmon, ESP32, Intel, Atheros, files, replay |
|
||||
| 3 | `rvcsi-dsp` | Rust signal-processing primitives |
|
||||
| 4 | `rvcsi-events` | Windowing, baseline modeling, event extraction, state machines |
|
||||
| 5 | `rvcsi-node` | napi-rs bindings exposing safe APIs to Node.js |
|
||||
| 6 | `rvcsi-sdk` | TypeScript SDK |
|
||||
| 7 | `rvcsi-cli` | Command-line interface |
|
||||
| 8 | `rvcsi-daemon` | Long-running capture and event service |
|
||||
| 9 | `rvcsi-mcp` | MCP tool server |
|
||||
| 10 | `rvcsi-ruvector` | Exporter and query bridge |
|
||||
|
||||
### 9.3 Reference repository layout
|
||||
|
||||
```
|
||||
rvcsi/
|
||||
crates/
|
||||
rvcsi-core/
|
||||
rvcsi-adapter-file/
|
||||
rvcsi-adapter-nexmon/
|
||||
rvcsi-adapter-esp32/
|
||||
rvcsi-dsp/
|
||||
rvcsi-events/
|
||||
rvcsi-ruvector/
|
||||
rvcsi-daemon/
|
||||
rvcsi-node/
|
||||
rvcsi-mcp/
|
||||
packages/
|
||||
sdk/
|
||||
cli/
|
||||
dashboard/
|
||||
native/
|
||||
nexmon-shim-c/
|
||||
docs/
|
||||
adr/
|
||||
ddd/
|
||||
prd/
|
||||
benchmarks/
|
||||
testdata/
|
||||
captures/
|
||||
malformed/
|
||||
replay/
|
||||
```
|
||||
|
||||
> Within the RuView monorepo, rvCSI would be introduced as a new bounded context (see the [domain model](../ddd/rvcsi-domain-model.md)) and a small set of `v2/crates/rvcsi-*` crates, reusing existing `wifi-densepose-signal` DSP and `wifi-densepose-ruvector` integration where they overlap rather than duplicating them.
|
||||
|
||||
---
|
||||
|
||||
## 10. Data model (summary)
|
||||
|
||||
The authoritative definitions live in the [rvCSI domain model](../ddd/rvcsi-domain-model.md). Summary:
|
||||
|
||||
- **`CsiFrame`** — one validated CSI observation at a timestamp (the FFI-safe object). Carries I/Q, amplitude, phase, RSSI, channel/bandwidth, optional antenna/chain metadata, validation status, quality score, calibration version.
|
||||
- **`CsiWindow`** — a bounded sequence of frames from one source/session, with mean amplitude, phase variance, motion energy, presence score, quality score.
|
||||
- **`CsiEvent`** — a semantic interpretation of one or more windows, with `kind`, confidence, evidence window IDs, and metadata.
|
||||
- **`AdapterProfile`** — capability descriptor for a source: chip, firmware/driver versions, supported channels/bandwidths, expected subcarrier counts, capture/injection/monitor-mode support.
|
||||
|
||||
---
|
||||
|
||||
## 11. Open questions
|
||||
|
||||
1. **Embedding model.** What produces frame/window embeddings in v0 — a fixed DSP feature vector, the existing AETHER contrastive model (ADR-024), or a lightweight on-device model? v0 leans on a deterministic DSP feature vector; v2 targets an on-device model.
|
||||
2. **Calibration UX.** How long must a calibration window be before `StabilityScore` is trustworthy, and how is that surfaced in the SDK/CLI?
|
||||
3. **Nexmon coupling.** Which Nexmon-supported chips/firmwares are in the v0 "supported" matrix vs. "best effort"?
|
||||
4. **Monorepo vs. standalone.** Does rvCSI ship as `v2/crates/rvcsi-*` inside RuView or as a separate `rvcsi/` repo? This PRD assumes monorepo crates that reuse `wifi-densepose-signal` and `wifi-densepose-ruvector`.
|
||||
5. **MCP transport.** stdio-only for v1, or also a local socket for multi-agent fan-out?
|
||||
|
||||
---
|
||||
|
||||
## 12. References
|
||||
|
||||
- [ADR-095 — rvCSI Edge RF Sensing Platform](../adr/ADR-095-rvcsi-edge-rf-sensing-platform.md)
|
||||
- [rvCSI Domain Model](../ddd/rvcsi-domain-model.md)
|
||||
- [ADR-013 — Feature-Level Sensing on Commodity Gear](../adr/ADR-013-feature-level-sensing-commodity-gear.md)
|
||||
- [ADR-014 — SOTA Signal Processing](../adr/ADR-014-sota-signal-processing.md)
|
||||
- [ADR-016 — RuVector Integration](../adr/ADR-016-ruvector-integration.md)
|
||||
- [ADR-024 — Project AETHER: Contrastive CSI Embeddings](../adr/ADR-024-contrastive-csi-embedding-model.md)
|
||||
- [ADR-031 — RuView Sensing-First RF Mode](../adr/ADR-031-ruview-sensing-first-rf-mode.md)
|
||||
- [ADR-040 — WASM Programmable Sensing](../adr/ADR-040-wasm-programmable-sensing.md)
|
||||
@@ -0,0 +1,466 @@
|
||||
# Pi 5 + Hailo Cluster: Building a Cognitive RF Observer with rvcsi
|
||||
|
||||
A field-tested tutorial for turning a 4-node Raspberry Pi 5 cluster into a
|
||||
multistatic Wi-Fi CSI cognitive RF observer that learns room states,
|
||||
predicts the next one, and flags anomalies — entirely from radio.
|
||||
|
||||
**Estimated time:** 4–6 hours (hardware 1h, firmware 1h, software 1h, calibration 1–3h)
|
||||
|
||||
**What you will build:** A self-learning 4-node cluster that captures Wi-Fi
|
||||
Channel State Information from a stable RF beacon, encodes each frame into a
|
||||
128-dimensional fingerprint on an on-device Hailo-8 NPU, clusters those
|
||||
fingerprints into discrete room states with stable IDs across runs, models
|
||||
state transitions with a 2nd-order Markov chain (with measurable predictive
|
||||
skill above chance), and persists everything to a queryable brain corpus on
|
||||
a workstation. The whole thing runs over Tailscale and is operated through
|
||||
a single CLI with **34 subcommands**.
|
||||
|
||||
**Who this is for:** RF engineers, smart-home hackers, security researchers,
|
||||
and ML/embedded folks comfortable with Linux + systemd. No specific signal-
|
||||
processing background required — but you do need patience for hardware
|
||||
quirks (nexmon_csi cross-compile is a known dead end; see step 3).
|
||||
|
||||
> **The TL;DR**: 4× Pi 5 + 2× Hailo-8 → CSI → 128-d embeddings → cosine
|
||||
> k-means with warm-start → 2nd-order Markov → SQLite brain → 34-subcommand
|
||||
> operator CLI. Production-grade signal: 39% top-1 ceiling on next-state
|
||||
> prediction (16× chance baseline), continuous fleet/drift/anomaly
|
||||
> monitoring, and a 12-category time-series corpus.
|
||||
|
||||
> **About the name "rvcsi" in this tutorial.** When this tutorial was
|
||||
> first written, the cluster's per-Pi capture services were named with
|
||||
> an `rvcsi` prefix (`cog-rvcsi-stream`, `cog-rvcsi-correlator`) as
|
||||
> branding only — the actual code was Python and didn't depend on the
|
||||
> upstream [`ruvnet/rvcsi`](https://github.com/ruvnet/rvcsi) Rust
|
||||
> runtime. **As of 2026-05-13**, the v0-appliance project has accepted
|
||||
> [ADR-207](https://github.com/ruvnet/v0-appliance/blob/main/docs/adr/ADR-207-rvcsi-library-integration.md)
|
||||
> (rvCSI library integration — Option D) and shipped a Rust binary
|
||||
> `cog-rvcsi-pi` built on rvcsi-runtime 0.3 that replaces the three
|
||||
> Python services. The cutover is per-Pi, operator-driven, with
|
||||
> one-command rollback (`scripts/rvcsi-pi/install-rvcsi-pi.sh` and
|
||||
> `uninstall-rvcsi-pi.sh`). A given cluster may be running either
|
||||
> stack while migration is in progress; the schema and operator
|
||||
> surface are unchanged across the cutover. See ADR-207's
|
||||
> Implementation log for the current state.
|
||||
|
||||
---
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Prerequisites](#1-prerequisites)
|
||||
2. [Architecture overview](#2-architecture-overview)
|
||||
3. [Per-node firmware: nexmon_csi on Pi 5](#3-per-node-firmware-nexmon_csi-on-pi-5)
|
||||
4. [Per-node services](#4-per-node-services)
|
||||
5. [Workstation pipeline](#5-workstation-pipeline)
|
||||
6. [Calibration: getting from raw CSI to room states](#6-calibration-getting-from-raw-csi-to-room-states)
|
||||
7. [Operating the cluster: the cog-query CLI](#7-operating-the-cluster-the-cog-query-cli)
|
||||
8. [What you can measure](#8-what-you-can-measure)
|
||||
9. [Troubleshooting](#9-troubleshooting)
|
||||
10. [Next steps](#10-next-steps)
|
||||
|
||||
---
|
||||
|
||||
## 1. Prerequisites
|
||||
|
||||
### Hardware
|
||||
|
||||
| Item | Quantity | Approx. cost | Notes |
|
||||
|------|----------|--------------|-------|
|
||||
| Raspberry Pi 5 (8GB) | 4 | ~$80 each | 4GB works but tight under sustained load |
|
||||
| Hailo-8 M.2 HAT (AI Kit) | 2 | ~$110 each | Only 2 needed — encoder is split across cluster-1 + cluster-2 |
|
||||
| MicroSD (64GB, A2) | 4 | ~$10 each | A2 class strongly recommended for sustained writes |
|
||||
| USB-C PD power supply (27W) | 4 | ~$12 each | Pi 5 draws 5A at full Hailo load |
|
||||
| Active cooler | 4 | ~$5 each | Cluster-2 sustains thermal load — passive will throttle |
|
||||
| Workstation (≥16GB RAM, Linux) | 1 | — | Hosts the brain HTTP service + clusterer + anomaly daemon |
|
||||
| Stable Wi-Fi beacon | 1 | — | Any AP on the same 5 GHz channel. We use ch.149/80MHz. Stability matters more than identity. |
|
||||
|
||||
**Total parts cost:** ~$580 plus workstation.
|
||||
|
||||
> **Important:** All 4 Pi 5s must use the on-board `bcm43455c0` radio. USB
|
||||
> Wi-Fi adapters with otherwise-similar chipsets **will not** work — nexmon's
|
||||
> firmware patches are silicon-specific. See ADR-206 § "USB Wi-Fi dongle
|
||||
> rabbit-hole" for the painful version of that lesson.
|
||||
|
||||
### Software prerequisites
|
||||
|
||||
| Component | Version | Notes |
|
||||
|-----------|---------|-------|
|
||||
| Pi OS Bookworm (Lite) | 64-bit, kernel 6.6+ | Use the Lite image — Desktop slows boot and burns SD writes |
|
||||
| Tailscale | ≥1.60 | Mesh networking across the cluster |
|
||||
| Rust toolchain | 1.78+ on workstation, 1.78+ on each Pi | For ruvector + adapter binaries |
|
||||
| Python 3.11+ | system Python on workstation | numpy required |
|
||||
| systemd-user | already present | Workstation timers run as user units |
|
||||
|
||||
---
|
||||
|
||||
## 2. Architecture overview
|
||||
|
||||
```
|
||||
┌─ workstation (Linux, ≥16GB) ──────────────────┐
|
||||
│ │
|
||||
│ brain HTTP (SQLite, port 9876) │
|
||||
│ ↑↑ │
|
||||
│ ┌──┴┴──────────────────────────────────┐ │
|
||||
│ │ rfmem-tail ← ingests live brain │ │
|
||||
│ │ rfmem-recall → posts category= │ │
|
||||
│ │ rfmem-recall when │ │
|
||||
│ │ current state ≈ past │ │
|
||||
│ │ rfmem-anomaly → 13-axis detector, │ │
|
||||
│ │ posts rfmem-anomaly & │ │
|
||||
│ │ rfmem-state-transition │ │
|
||||
│ │ cog-rfmem-states (timer, hourly) │ │
|
||||
│ │ re-clusters w/ warm-start│ │
|
||||
│ │ cog-rfmem-insights (timer, nightly) │ │
|
||||
│ │ writes rfmem-insights │ │
|
||||
│ │ cog-rfmem-drift-check (timer, 05:00) │ │
|
||||
│ │ audits cluster file state│ │
|
||||
│ └───────────────────────────────────────┘ │
|
||||
│ │
|
||||
│ cog-query (CLI, 34 subcommands, 4 JSON modes)│
|
||||
└────────────────────────────────────────────────┘
|
||||
↑
|
||||
Tailscale mesh ──────────┴───────────────────────────────┐
|
||||
↓ ↓ ↓
|
||||
┌─ cluster-1 (Hailo) ┐ ┌─ cluster-2 (Hailo + fusion) ┐ ┌─ cluster-3 ┐ ┌─ v0 ┐
|
||||
│ cog-csi-emitter │ │ cog-csi-emitter │ │ same as │ │ same│
|
||||
│ cog-csi-adapter │ │ cog-csi-adapter │ │ cluster-1 │ │ as │
|
||||
│ cog-rvcsi-stream │ │ cog-rvcsi-stream │ │ minus │ │ c-3 │
|
||||
│ cog-hailo-encoder │ │ cog-hailo-encoder │ │ Hailo & │ │ │
|
||||
│ │ │ cog-rvcsi-correlator (fusion)│ │ correlator │ │ │
|
||||
└────────────────────┘ └─────────────────────────────┘ └────────────┘ └─────┘
|
||||
4 svc 5 svc 3 svc 3 svc
|
||||
└─────────────────────── 15 expected services total ──────────────────────┘
|
||||
```
|
||||
|
||||
**Why this split?** Multistatic fusion (combining CSI from 4 spatial vantage
|
||||
points into a single weighted observation) is computationally cheap but
|
||||
benefits from being on **one** node so the other three only do capture +
|
||||
encode. Hailo-8 is the bottleneck cost, so we put two on the cluster
|
||||
(one for redundancy, one for the fusion node) and let `cluster-3` + `v0`
|
||||
run as pure capture sensors.
|
||||
|
||||
---
|
||||
|
||||
## 3. Per-node firmware: nexmon_csi on Pi 5
|
||||
|
||||
**Critical lesson learned (saved you a week):** the workstation x86_64
|
||||
cross-compile path for nexmon_csi on Pi 5 **does not work**. The 39-hunk
|
||||
patch series applies cleanly on a native Pi 5 ARM build, and fails in
|
||||
subtle ways elsewhere.
|
||||
|
||||
The recipe that works:
|
||||
|
||||
```bash
|
||||
# On each Pi 5 (not the workstation):
|
||||
sudo apt update && sudo apt install -y \
|
||||
raspberrypi-kernel-headers bc bison flex libssl-dev make \
|
||||
gcc gawk qpdf cmake build-essential libpcap-dev clang gcc-arm-none-eabi
|
||||
|
||||
git clone https://github.com/seemoo-lab/nexmon.git ~/nexmon
|
||||
cd ~/nexmon
|
||||
source setup_env.sh
|
||||
make
|
||||
|
||||
cd patches
|
||||
git clone https://github.com/seemoo-lab/nexmon_csi.git
|
||||
cd nexmon_csi
|
||||
|
||||
# Apply the Pi-5-friendly patch series — all 39 hunks should apply clean
|
||||
# on native ARM. If you see "Hunk #N FAILED", you are almost certainly
|
||||
# cross-compiling from x86_64. Stop. Build on the Pi.
|
||||
./install.sh
|
||||
|
||||
# Switch on:
|
||||
sudo mcp # 'monitor capability provisioning' — enable
|
||||
sudo nexutil -Iwlan0 -s500 -b -l34 -v<86-char base64 capture filter>
|
||||
```
|
||||
|
||||
> **Pi 5 kernel gotcha:** Pi OS Bookworm ships two kernels — `kernel8.img`
|
||||
> (4K pages) and `kernel_2712.img` (16K pages, Pi 5 only). nexmon_csi
|
||||
> currently builds clean against `kernel8.img`. Add `kernel=kernel8.img`
|
||||
> to `/boot/firmware/config.txt` if you've switched. **After the switch,
|
||||
> SSH by hostname via Tailscale** — host keys + DHCP gotchas otherwise.
|
||||
|
||||
> **Clock-skew first-boot trap:** Pi 5 has no RTC. First-boot apt will
|
||||
> reject "future-dated" `Release` files. Patch your firstboot to wait for
|
||||
> `systemd-timesyncd` before running `apt-get`.
|
||||
|
||||
The complete commands + full troubleshooting matrix is in the
|
||||
[detailed gist](https://gist.github.com/ruvnet/88e7b053c41cb4f4af7a7ec4af873017) — section "Firmware: nexmon_csi on Pi 5".
|
||||
|
||||
---
|
||||
|
||||
## 4. Per-node services
|
||||
|
||||
Each cluster Pi runs a small fixed set of systemd services. Per-host
|
||||
topology:
|
||||
|
||||
| Service | cluster-1 | cluster-2 | cluster-3 | v0 |
|
||||
|---|:--:|:--:|:--:|:--:|
|
||||
| `cog-csi-emitter` (raw CSI capture from nexmon) | ✓ | ✓ | ✓ | ✓ |
|
||||
| `cog-csi-adapter` (Rust binary; CSI → 256-byte float frames) | ✓ | ✓ | ✓ | ✓ |
|
||||
| `cog-rvcsi-stream` (publishes frames to rvcsi-correlator) | ✓ | ✓ | ✓ | ✓ |
|
||||
| `cog-hailo-encoder` (frames → 128-d fingerprints on Hailo-8) | ✓ | ✓ | — | — |
|
||||
| `cog-rvcsi-correlator` (multistatic fusion across 4 nodes) | — | ✓ | — | — |
|
||||
| **Expected service count** | **4** | **5** | **3** | **3** |
|
||||
|
||||
The topology is encoded in the workstation's `cog-query fleet-status`
|
||||
subcommand, which compares per-host expected services against live
|
||||
`systemctl is-active` results. A flat-service check would falsely flag
|
||||
cluster-3 and v0 as degraded (they have neither Hailo nor the correlator
|
||||
— that's by design).
|
||||
|
||||
> **rvcsi cutover (ADR-207 Option D, 2026-05-13).** The three services
|
||||
> `cog-csi-emitter`, `cog-csi-adapter`, and `cog-rvcsi-stream` are
|
||||
> being consolidated into one Rust binary `cog-rvcsi-pi` built on
|
||||
> [rvcsi-runtime](https://crates.io/crates/rvcsi-runtime). The new
|
||||
> binary holds the same per-Pi role and the same expected-service
|
||||
> count from the operator's view (`fleet-status` already understands
|
||||
> both layouts). Deploy with
|
||||
> `bash scripts/rvcsi-pi/install-rvcsi-pi.sh <pi-host>`; revert with
|
||||
> `scripts/rvcsi-pi/uninstall-rvcsi-pi.sh`. The cutover is per-Pi,
|
||||
> not flag-day — mixed Python/Rust clusters are supported. The Hailo
|
||||
> encoder + correlator stay Python in this phase; their Rust ports
|
||||
> are tracked as follow-on ADRs.
|
||||
|
||||
All unit files + the install script are in the
|
||||
[detailed gist](https://gist.github.com/ruvnet/88e7b053c41cb4f4af7a7ec4af873017) — section "Per-node systemd units".
|
||||
|
||||
---
|
||||
|
||||
## 5. Workstation pipeline
|
||||
|
||||
The workstation runs ten user-mode units (3 daemons, 7 timers):
|
||||
|
||||
| Unit | Type | Cadence | Purpose |
|
||||
|---|---|---|---|
|
||||
| `cog-rfmem-tail` | daemon | continuous | Ingests live brain entries into the workstation mirror |
|
||||
| `cog-rfmem-recall` | daemon | continuous | kNN-matches current fingerprint vs persisted ones, posts `rfmem-recall` |
|
||||
| `cog-rfmem-anomaly` | daemon | continuous | 13-axis anomaly detector, posts `rfmem-anomaly` + `rfmem-state-transition` |
|
||||
| `cog-rfmem-indexer` | timer | every 5 min | Updates HNSW index for kNN |
|
||||
| `cog-rfmem-compress` | timer | hourly | Compresses old brain entries |
|
||||
| `cog-rfmem-daily` | timer | nightly 04:00 | Per-day stats roll-up (`rfmem-daily`) |
|
||||
| `cog-rfmem-states` | timer | hourly | Re-runs cosine k-means w/ warm-start (`rfmem-state-summary`) |
|
||||
| `cog-rfmem-insights` | timer | nightly 04:55 | NL synthesis, posts `rfmem-insights` |
|
||||
| `cog-rfmem-drift-check` | timer | nightly 05:00 | Audits cluster file/unit drift, posts `rfmem-drift` |
|
||||
| `cog-rfmem-mirror` | timer | hourly | Mirrors cluster-2 brain → workstation read-replica |
|
||||
|
||||
Install in one shot:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/<your-fork>/v0-appliance.git
|
||||
cd v0-appliance
|
||||
bash scripts/rfmem/install-workstation.sh
|
||||
```
|
||||
|
||||
The installer is **idempotent** — rerunning is safe and only enables
|
||||
units that aren't yet enabled. It also wires a git post-commit hook
|
||||
that auto-deploys + auto-smoke-tests on every commit touching
|
||||
`scripts/rfmem/`. That closes the "I edited the repo but forgot to
|
||||
deploy" gap that bit us repeatedly in early development.
|
||||
|
||||
---
|
||||
|
||||
## 6. Calibration: getting from raw CSI to room states
|
||||
|
||||
This is the longest step but largely passive — let it run.
|
||||
|
||||
### 6.1 Walk the room
|
||||
|
||||
For 30–60 minutes after the cluster is live, walk through every room you
|
||||
want recognized. Sit, stand, move between rooms, repeat. The encoder is
|
||||
learning to map "what the room looks like in CSI" into 128-d vectors;
|
||||
diversity here matters more than total time.
|
||||
|
||||
### 6.2 First clustering pass
|
||||
|
||||
```bash
|
||||
# Force-trigger the clusterer (it normally fires hourly):
|
||||
systemctl --user start cog-rfmem-states.service
|
||||
python3 scripts/rfmem/cog-query.py states
|
||||
```
|
||||
|
||||
Output looks like:
|
||||
|
||||
```
|
||||
=== rfmem-states — k=16, n=12,847 ===
|
||||
state #0 π=0.184 dwell=42.3s centroid_drift=0.012 (default)
|
||||
state #1 π=0.121 dwell=18.1s centroid_drift=0.003
|
||||
state #4 π=0.087 dwell=29.6s centroid_drift=0.041
|
||||
...
|
||||
```
|
||||
|
||||
**Stable IDs across runs.** The warm-start k-means recipe matches new
|
||||
centroids to the prior run's centroids by cosine similarity before
|
||||
assigning IDs. This means state #4 stays state #4 between hourly runs —
|
||||
otherwise downstream Markov transitions would scramble after every
|
||||
re-cluster.
|
||||
|
||||
### 6.3 Let the Markov chain build
|
||||
|
||||
After a few thousand transitions (a few hours of activity), check:
|
||||
|
||||
```bash
|
||||
python3 scripts/rfmem/cog-query.py prediction-accuracy
|
||||
```
|
||||
|
||||
You should see something like:
|
||||
|
||||
```
|
||||
=== prediction-accuracy — training-set top-1 ceilings ===
|
||||
1st-order: 37.1% (16x chance baseline of 6.25%)
|
||||
2nd-order: 39.4% (16x chance baseline of 6.25%, 1.06x gain over 1st)
|
||||
```
|
||||
|
||||
The 2nd-order chain beats 1st-order because it conditions on the
|
||||
**previous** state as well as the current one. Self-loops are excluded
|
||||
from the argmax (a transition is by definition a state change).
|
||||
|
||||
### 6.4 Verify the room learned itself
|
||||
|
||||
```bash
|
||||
python3 scripts/rfmem/cog-query.py insights
|
||||
```
|
||||
|
||||
Reads like:
|
||||
|
||||
```
|
||||
The cluster has observed 446,231 fingerprints, clustering them into
|
||||
16 discrete RF states. The room exhibits moderately diverse (stationary
|
||||
entropy 0.82/1.0). State #4 is the dominant 'default' state (π=0.214);
|
||||
state #13 is the rarest baseline (π=0.018).
|
||||
Prediction skill (last hour, 2nd-order): top-1 12.4% (1.98x chance),
|
||||
top-3 31.0% (1.65x chance, 412 transitions) (training-set ceiling
|
||||
39.4% — operating @ 31% of capacity).
|
||||
```
|
||||
|
||||
That "operating @ 31% of capacity" line is the operational efficiency:
|
||||
how close live performance is to the model's theoretical ceiling. Big
|
||||
gap = the room is being noisy in ways the static cluster model doesn't
|
||||
capture. Small gap = you're near SOTA for this static model.
|
||||
|
||||
---
|
||||
|
||||
## 7. Operating the cluster: the cog-query CLI
|
||||
|
||||
A single CLI binary with **34 subcommands** + 4 machine-readable JSON
|
||||
modes. Practical ones (full list in the gist):
|
||||
|
||||
| Subcommand | What it does |
|
||||
|---|---|
|
||||
| `summary --hours 1` | Bird's-eye view of last hour: anomalies, transitions, recall hits |
|
||||
| `top-events --hours 24 --limit 5` | Highest-info events in window (combines novelty + tier + recency) |
|
||||
| `top-events --json` | Same, agent-consumable |
|
||||
| `insights` | Natural-language synthesis (paragraph) — what the cluster thinks |
|
||||
| `insights --json` | Same, structured |
|
||||
| `insights --post` | Same, persisted to brain as `rfmem-insights` |
|
||||
| `stats` | Corpus: per-category counts, dimensions, vector counts |
|
||||
| `motion` | Recent motion events |
|
||||
| `anomalies --sort info` | Anomalies sorted by composite info score (1.0–8.0) |
|
||||
| `circadian` | 24-hour bin of activity — does the room have a daily rhythm? |
|
||||
| `by-state` | Per-state metrics (dwell, σ-baseline, novelty distribution) |
|
||||
| `markov` | Top transitions by frequency, both 1st + 2nd-order |
|
||||
| `transitions --sort novelty` | Rare/surprising transitions |
|
||||
| `dwell-times` | How long the room stays in each state |
|
||||
| `prediction-accuracy` | 1st + 2nd-order top-1 ceilings |
|
||||
| `baseline-drift` | Has the noise floor shifted? (slow change) |
|
||||
| `centroid-drift` | Has any state's RF signature materially changed? |
|
||||
| `fleet-status` | Per-host expected-service liveness check |
|
||||
| `fleet-status --json` | Same, agent-consumable |
|
||||
| `fleet-status --post` | Same, persisted to brain as `rfmem-fleet` (heartbeat) |
|
||||
| `check-drift` | Workstation/cluster file + unit drift audit |
|
||||
| `replica-status` | Hourly cluster-2 → workstation mirror health |
|
||||
|
||||
### The fleet-health triad
|
||||
|
||||
Three subcommands cover the operator's full health picture:
|
||||
|
||||
- `check-drift` — file content drift (what's deployed vs what's in git)
|
||||
- `replica-status` — workstation mirror lag (last successful sync)
|
||||
- `fleet-status` — service liveness across the 4 Pis (topology-aware)
|
||||
|
||||
If all three are green, the cluster is healthy. If any one fires, you
|
||||
have a concrete starting point.
|
||||
|
||||
---
|
||||
|
||||
## 8. What you can measure
|
||||
|
||||
After a week of runtime, you can answer questions like:
|
||||
|
||||
- **"What's the room's most common 'baseline' state?"** → `states` shows
|
||||
the π-dominant cluster ID.
|
||||
- **"Did anything weird happen last night?"** → `anomalies --sort info
|
||||
--hours 12` sorts by combined-information score (novelty × tier × state-
|
||||
rarity × calmness).
|
||||
- **"How predictable is the room?"** → `insights` reports stationary
|
||||
entropy (0.0 = single state, 1.0 = uniform). Most rooms land 0.6–0.9.
|
||||
- **"What's the most novel transition ever observed?"** → `transitions
|
||||
--sort novelty --limit 1`. We've seen transitions with
|
||||
`transition_p=0.0000` — never observed before in 446k+ embeddings.
|
||||
- **"Is the room changing slowly?"** → `centroid-drift` flags states
|
||||
whose 128-d signature has moved > 0.05 cosine distance since the prior
|
||||
clusterer run. Common cause: a piece of furniture moved.
|
||||
- **"What's the daily rhythm?"** → `circadian` bins activity by hour.
|
||||
Most rooms show clear morning/evening peaks.
|
||||
|
||||
---
|
||||
|
||||
## 9. Troubleshooting
|
||||
|
||||
| Symptom | Likely cause | Fix |
|
||||
|---|---|---|
|
||||
| `nexmon_csi` build fails with FAILED hunks | Cross-compiling from x86_64 | Build on the Pi natively |
|
||||
| Pi 5 stops booting after kernel switch | Wrong `kernel=` in `/boot/firmware/config.txt` | Use `kernel=kernel8.img` |
|
||||
| First boot fails on `apt update` | No RTC → clock skew, apt rejects "future-dated" Release files | Wait for `systemd-timesyncd` in firstboot |
|
||||
| `cog-rfmem-now` times out | Workstation daemon swap-thrashing | Bump `MemoryMax=` in unit file (we run 1G) |
|
||||
| `fleet-status` shows DEGRADED on cluster-3 / v0 | Topology unaware (old version) | Update to latest — per-host expected-services |
|
||||
| Cluster-2 Hailo encoder silent | `cp -r` made encoder a directory, not a file | `install -m 0755` instead |
|
||||
| 2nd-order Markov top-1 = 0% | Self-loop dominates argmax | Zero out self-loop before `.argmax()` |
|
||||
| State IDs change between runs | No warm-start k-means | Update clusterer to match new centroids to prior run by cosine |
|
||||
| HardFaults during embedded N6 bring-up | (Different topic, see [ADR-027](../adr/) for STM32N6 startup notes) | — |
|
||||
|
||||
---
|
||||
|
||||
## 10. Next steps
|
||||
|
||||
Once your cluster is producing stable predictions and clean fleet health,
|
||||
the natural directions are:
|
||||
|
||||
1. **Cross-room correlation** — train a second cluster in another room
|
||||
and feed both into the workstation. The brain already supports
|
||||
multiple namespaces.
|
||||
2. **Active sensing** — instead of passively observing whatever beacon is
|
||||
present, drive your own (e.g., dedicated 5 GHz beacon AP at fixed
|
||||
power). Eliminates upstream variability.
|
||||
3. **Vital signs** — the RuView project has companion code for extracting
|
||||
heart-rate and breathing from CSI; the 128-d encoder output is a
|
||||
reasonable input feature.
|
||||
4. **Federated training** — multiple physical sites publishing to a shared
|
||||
brain. Each site keeps its own clusters; transitions are the shared
|
||||
vocabulary.
|
||||
5. **Push to upstream RuView** — if your cluster develops capabilities not
|
||||
in this tutorial (you'll know by the time you've written the README),
|
||||
send a PR.
|
||||
|
||||
---
|
||||
|
||||
## Reference material
|
||||
|
||||
- **[Detailed cookbook gist (all commands, configs, unit files)](https://gist.github.com/ruvnet/88e7b053c41cb4f4af7a7ec4af873017)**
|
||||
- **[ADR-206: nexmon_csi on Pi 5 cluster](https://github.com/ruvnet/v0-appliance/blob/main/docs/adr/ADR-206-nexmon-csi-on-pi-5-cluster.md)** — the engineering decision record
|
||||
with full rationale, including the painful-but-instructive failures
|
||||
- **[v0-appliance repo](https://github.com/ruvnet/v0-appliance)** — the
|
||||
source of truth for `scripts/rfmem/` operator tooling
|
||||
- **[seemoo-lab/nexmon_csi](https://github.com/seemoo-lab/nexmon_csi)** —
|
||||
upstream CSI capture firmware
|
||||
- **[Hailo-8 documentation](https://hailo.ai/products/hailo-8/)** — NPU
|
||||
reference
|
||||
|
||||
---
|
||||
|
||||
*This tutorial was built against the v0.5.0-cognitive-rf-observer milestone
|
||||
of `v0-appliance`. The cluster has been running continuously for 6+ weeks
|
||||
of development with 446k+ fingerprints observed, 16 stable RF states, and
|
||||
a 2nd-order Markov model operating at 31% of its 39.4% theoretical
|
||||
top-1 ceiling. SOTA is a moving target — but this is a real, working
|
||||
cognitive RF observer that you can reproduce.*
|
||||
@@ -21,6 +21,7 @@ WiFi DensePose turns commodity WiFi signals into real-time human pose estimation
|
||||
- [Windows WiFi (RSSI Only)](#windows-wifi-rssi-only)
|
||||
- [ESP32-S3 (Full CSI)](#esp32-s3-full-csi)
|
||||
- [ESP32 Multistatic Mesh (Advanced)](#esp32-multistatic-mesh-advanced)
|
||||
- [Connect Mesh Data to the Dashboard and Observatory](#connect-mesh-data-to-the-dashboard-and-observatory)
|
||||
- [Cognitum Seed Integration (ADR-069)](#cognitum-seed-integration-adr-069)
|
||||
5. [REST API Reference](#rest-api-reference)
|
||||
6. [WebSocket Streaming](#websocket-streaming)
|
||||
@@ -28,13 +29,14 @@ WiFi DensePose turns commodity WiFi signals into real-time human pose estimation
|
||||
8. [Vital Sign Detection](#vital-sign-detection)
|
||||
9. [CLI Reference](#cli-reference)
|
||||
10. [Observatory Visualization](#observatory-visualization)
|
||||
11. [Adaptive Classifier](#adaptive-classifier)
|
||||
11. [Loading the Pretrained Model from Hugging Face](#loading-the-pretrained-model-from-hugging-face)
|
||||
12. [Adaptive Classifier](#adaptive-classifier)
|
||||
- [Recording Training Data](#recording-training-data)
|
||||
- [Training the Model](#training-the-model)
|
||||
- [Using the Trained Model](#using-the-trained-model)
|
||||
12. [Training a Model](#training-a-model)
|
||||
13. [Training a Model](#training-a-model)
|
||||
- [CRV Signal-Line Protocol](#crv-signal-line-protocol)
|
||||
13. [RVF Model Containers](#rvf-model-containers)
|
||||
14. [RVF Model Containers](#rvf-model-containers)
|
||||
14. [Hardware Setup](#hardware-setup)
|
||||
- [ESP32-S3 Mesh](#esp32-s3-mesh)
|
||||
- [Intel 5300 / Atheros NIC](#intel-5300--atheros-nic)
|
||||
@@ -331,6 +333,46 @@ The mesh uses a **Time-Division Multiplexing (TDM)** protocol so nodes take turn
|
||||
|
||||
See [ADR-029](adr/ADR-029-ruvsense-multistatic-sensing-mode.md) and [ADR-032](adr/ADR-032-multistatic-mesh-security-hardening.md) for the full design.
|
||||
|
||||
### Connect Mesh Data to the Dashboard and Observatory
|
||||
|
||||
If a standalone `aggregator` command prints live packets, the ESP32 fleet is already reaching that host. To visualize the same data, stop the standalone aggregator and run `sensing-server` on that same host and UDP port. The sensing server is the aggregator used by the REST API, WebSocket stream, dashboard, and Observatory.
|
||||
|
||||
```bash
|
||||
# From a source build
|
||||
cd v2
|
||||
cargo run -p wifi-densepose-sensing-server -- \
|
||||
--source esp32 \
|
||||
--udp-port 5005 \
|
||||
--http-port 3000 \
|
||||
--ws-port 3001 \
|
||||
--ui-path ../../ui
|
||||
|
||||
# Docker
|
||||
docker run --rm \
|
||||
-e CSI_SOURCE=esp32 \
|
||||
-p 3000:3000 \
|
||||
-p 3001:3001 \
|
||||
-p 5005:5005/udp \
|
||||
ruvnet/wifi-densepose:latest
|
||||
```
|
||||
|
||||
Open the UI from the sensing server, not from a local file:
|
||||
|
||||
| View | URL |
|
||||
|------|-----|
|
||||
| Dashboard | `http://localhost:3000/ui/index.html` |
|
||||
| Observatory | `http://localhost:3000/ui/observatory.html` |
|
||||
|
||||
Use these checks before debugging the browser:
|
||||
|
||||
```bash
|
||||
curl http://localhost:3000/health
|
||||
curl http://localhost:3000/api/v1/nodes
|
||||
curl http://localhost:3000/api/v1/sensing/latest
|
||||
```
|
||||
|
||||
If the ESP32 nodes are provisioned with `--target-ip <AGGREGATOR_HOST>`, that IP must be the machine running `sensing-server`. Only one process can receive UDP `:5005` at a time, so leave the standalone hardware `aggregator` off while the dashboard or Observatory is live.
|
||||
|
||||
### Cognitum Seed Integration (ADR-069)
|
||||
|
||||
Connect an ESP32-S3 to a [Cognitum Seed](https://cognitum.one) (Pi Zero 2 W, ~$15) for persistent vector storage, kNN similarity search, cryptographic witness chain, and AI-accessible sensing via MCP proxy.
|
||||
@@ -752,6 +794,67 @@ The Observatory is an immersive Three.js visualization that renders WiFi sensing
|
||||
|
||||
---
|
||||
|
||||
## Loading the Pretrained Model from Hugging Face
|
||||
|
||||
A pretrained CSI encoder + presence-detection head is published on Hugging Face at [`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained). It was trained on 60,630 frames / 610,615 contrastive triplets (12.2M steps, final loss 0.065) and reports 100% presence accuracy and ~164k embeddings/sec on an Apple M4 Pro.
|
||||
|
||||
What it ships (and what it does not):
|
||||
|
||||
| Capability | Status |
|
||||
|------------|--------|
|
||||
| Presence detection (occupied / empty) | ✅ Trained head — 100% accuracy on validation |
|
||||
| 128-dim CSI embeddings (re-ID, similarity, downstream training) | ✅ Trained encoder |
|
||||
| Single-person breathing / heart-rate | ⚠️ Server still uses heuristic DSP — model does not replace this yet |
|
||||
| 17-keypoint full-body pose | 🔬 No keypoint weights shipped yet — pose pipeline runs but without a learned head |
|
||||
|
||||
### Download
|
||||
|
||||
```bash
|
||||
pip install huggingface_hub
|
||||
huggingface-cli download ruvnet/wifi-densepose-pretrained \
|
||||
--local-dir models/wifi-densepose-pretrained
|
||||
```
|
||||
|
||||
The download yields a small set of files (the `.rvf.jsonl` is the canonical container the sensing server reads):
|
||||
|
||||
```
|
||||
models/wifi-densepose-pretrained/
|
||||
model.rvf.jsonl # RVF container (encoder + presence head + lora)
|
||||
model.safetensors # 48 KB — same encoder weights, safetensors format
|
||||
model-q4.bin # 8 KB — recommended quantization for edge
|
||||
presence-head.json # presence classifier head
|
||||
config.json # sona-lora rank=8 alpha=16, target encoder + task_heads
|
||||
```
|
||||
|
||||
### Using the weights
|
||||
|
||||
The HF artifact is in **JSONL RVF** format (one JSON object per line: `metadata`, `encoder`, `lora`). What you can do with it today:
|
||||
|
||||
| Consumer | Format it reads | Status |
|
||||
|----------|-----------------|--------|
|
||||
| Python / PyTorch training pipeline | `model.safetensors` | ✅ Works — load with `safetensors.torch.load_file` |
|
||||
| RVF JSONL inspection / re-export | `model.rvf.jsonl` | ✅ Works — plain JSONL, parse line-by-line |
|
||||
| Sensing-server `--model <PATH>` flag | binary RVF (`RVFS` magic) | ⚠️ Does **not** accept the JSONL file yet — see gap below |
|
||||
|
||||
**Known gap (tracked):** `v2/crates/wifi-densepose-sensing-server/src/rvf_container.rs` only parses the binary RVF segment format (magic `0x52564653`). Pointing `--model` at `model.rvf.jsonl` causes the progressive loader to error with `invalid magic at offset 0: expected 0x52564653, got 0x7974227B` (`0x7974227B` is the ASCII bytes `{"ty…` from the JSONL header), and the live pipeline degrades to null output rather than falling back to heuristic mode. Until a JSONL adapter lands (or the model is re-published as binary RVF), run the sensing-server **without** `--model` and consume the HF weights from Python or the training pipeline.
|
||||
|
||||
```bash
|
||||
# Works today — Python side (training, evaluation, embedding extraction):
|
||||
python -c "
|
||||
from safetensors.torch import load_file
|
||||
state = load_file('models/wifi-densepose-pretrained/model.safetensors')
|
||||
print({k: tuple(v.shape) for k, v in state.items()})
|
||||
"
|
||||
|
||||
# Sensing server — run heuristic for now:
|
||||
cargo run -p wifi-densepose-sensing-server --release -- \
|
||||
--source esp32 --udp-port 5005 --http-port 3000
|
||||
```
|
||||
|
||||
See [RVF Model Containers](#rvf-model-containers) for the binary format the loader expects, and [Training a Model](#training-a-model) for using the encoder as a starting point for environment-specific fine-tuning.
|
||||
|
||||
---
|
||||
|
||||
## Adaptive Classifier
|
||||
|
||||
The adaptive classifier (ADR-048) learns your environment's specific WiFi signal patterns from labeled recordings. It replaces static threshold-based classification with a trained logistic regression model that uses 15 features (7 server-computed + 8 subcarrier-derived statistics).
|
||||
@@ -1744,6 +1847,8 @@ The server applies a 3-stage smoothing pipeline (ADR-048). If readings are still
|
||||
|
||||
- Verify the sensing server is running: `curl http://localhost:3000/health`
|
||||
- Access Observatory via the server URL: `http://localhost:3000/ui/observatory.html` (not a file:// URL)
|
||||
- If a standalone `aggregator` command is already listening on UDP `:5005`, stop it and run `sensing-server --source esp32 --udp-port 5005` instead; the Observatory reads the server WebSocket, not the standalone aggregator output
|
||||
- Verify the ESP32 nodes are provisioned to the IP address of the machine running `sensing-server`
|
||||
- Hard refresh with Ctrl+Shift+R to clear cached settings
|
||||
- The auto-detect probes `/health` on the same origin — cross-origin won't work
|
||||
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
# Mixamo FBX downloads — too large + license boundary. Get your own from
|
||||
# mixamo.com (FBX Binary + T-Pose / Without Skin), drop into assets/.
|
||||
*.fbx
|
||||
|
||||
# Diagnostic / debug screenshots from a dev session. Official screenshots
|
||||
# live in screenshots/ and are committed; these underscore-prefixed ones
|
||||
# are scratch.
|
||||
_diag-*.png
|
||||
_demo-mode-shot*.png
|
||||
_PROOF-*.png
|
||||
@@ -0,0 +1,77 @@
|
||||
# three.js demos
|
||||
|
||||
Five progressively richer browser demos of the ADR-097 sensing-helpers scene,
|
||||
ending with a live MediaPipe-Pose → Mixamo X Bot retargeting pipeline driven
|
||||
by a real ESP32 CSI feed.
|
||||
|
||||
## Run them
|
||||
|
||||
```bash
|
||||
python examples/three.js/server/serve-demo.py
|
||||
# then open one of the URLs the script prints
|
||||
```
|
||||
|
||||
`server/serve-demo.py` is a tiny `ThreadingHTTPServer` with aggressive
|
||||
no-cache headers — the stdlib `http.server` is single-threaded and times out
|
||||
on the parallel script + FBX fetches the demos make.
|
||||
|
||||
## Demos
|
||||
|
||||
| # | File | What it shows |
|
||||
|---|------|---------------|
|
||||
| 01 | [`demos/01-helpers.html`](demos/01-helpers.html) | Plain ADR-097 helpers in the point-cloud viewer |
|
||||
| 02 | [`demos/02-cinematic.html`](demos/02-cinematic.html) | Cinematic camera + pseudo-CSI visualization on top of #01 |
|
||||
| 03 | [`demos/03-skinned.html`](demos/03-skinned.html) | GLTF skinned mesh + additive animation blending |
|
||||
| 04 | [`demos/04-skinned-fbx.html`](demos/04-skinned-fbx.html) | Mixamo X Bot loaded from FBX in the ADR-097 scene |
|
||||
| 05 | [`demos/05-skinned-realtime.html`](demos/05-skinned-realtime.html) | Webcam → MediaPipe Pose Heavy → Mixamo IK retarget, live ESP32 CSI overlay |
|
||||
|
||||
| Screenshot | |
|
||||
|---|---|
|
||||
|  |  |
|
||||
|  |  |
|
||||
|  | |
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
examples/three.js/
|
||||
├── README.md
|
||||
├── .gitignore
|
||||
├── demos/ # 5 self-contained HTML demos
|
||||
│ ├── 01-helpers.html
|
||||
│ ├── 02-cinematic.html
|
||||
│ ├── 03-skinned.html
|
||||
│ ├── 04-skinned-fbx.html
|
||||
│ └── 05-skinned-realtime.html
|
||||
├── screenshots/ # one PNG per demo
|
||||
│ └── 0N-*.png
|
||||
├── server/
|
||||
│ ├── serve-demo.py # local HTTP server with no-cache headers
|
||||
│ └── ruvultra-csi-bridge.py # ESP32 CSI WebSocket bridge (ruvultra:8766)
|
||||
└── assets/
|
||||
└── X Bot.fbx # gitignored — get your own from mixamo.com
|
||||
# (FBX Binary, T-Pose, Without Skin)
|
||||
# used by demos 04 and 05
|
||||
```
|
||||
|
||||
## Mixamo X Bot
|
||||
|
||||
Demos 04 and 05 expect `assets/X Bot.fbx`. It's gitignored (size + license
|
||||
boundary). Download yours from [mixamo.com](https://mixamo.com): pick the
|
||||
"X Bot" character, export as **FBX Binary**, **T-Pose**, **Without Skin**,
|
||||
and drop it into `assets/`.
|
||||
|
||||
## Live ESP32 CSI overlay (demo 05 only)
|
||||
|
||||
`server/ruvultra-csi-bridge.py` is the systemd-deployable bridge that runs on
|
||||
the `ruvultra` host (over Tailscale). It listens for ESP32-S3 CSI on UDP and
|
||||
re-broadcasts it as WebSocket frames at `ws://ruvultra:8766/csi`. Demo 05
|
||||
auto-connects; if the socket is down, it falls back to the bundled idle clip
|
||||
plus a synthetic CSI driver.
|
||||
|
||||
## Open issues
|
||||
|
||||
- [#583](https://github.com/ruvnet/RuView/issues/583) — head/face tracking
|
||||
fidelity in `05-skinned-realtime.html`. Recommended fix: swap MediaPipe
|
||||
Pose Heavy for MediaPipe Holistic (same API, adds 468-point face mesh +
|
||||
hand landmarks for proper PnP head pose and finger curl tracking).
|
||||
@@ -0,0 +1,587 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>RuView · ADR-097 · three.js helpers in the point cloud viewer</title>
|
||||
<link rel="icon" type="image/svg+xml" href="data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'><circle cx='16' cy='16' r='10' fill='%23e8a634'/></svg>">
|
||||
<style>
|
||||
:root {
|
||||
--bg: #0a0a0a;
|
||||
--bg-panel: rgba(0, 0, 0, 0.88);
|
||||
--amber: #e8a634;
|
||||
--amber-dim: #4a3a1a;
|
||||
--amber-hot: #ffc04d;
|
||||
--grid-major: #444444;
|
||||
--grid-minor: #222222;
|
||||
--green: #4f4;
|
||||
--blue: #4cf;
|
||||
--text-mute: #888;
|
||||
--border: #2a2a2a;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0;
|
||||
background: var(--bg);
|
||||
color: var(--amber);
|
||||
font-family: 'SF Mono', Monaco, 'Cascadia Code', Consolas, monospace;
|
||||
overflow: hidden;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
}
|
||||
canvas { display: block; }
|
||||
|
||||
/* Top-left HUD */
|
||||
#info {
|
||||
position: absolute;
|
||||
top: 16px;
|
||||
left: 16px;
|
||||
padding: 14px 16px;
|
||||
background: var(--bg-panel);
|
||||
border: 1px solid var(--amber);
|
||||
border-radius: 8px;
|
||||
min-width: 280px;
|
||||
max-width: 340px;
|
||||
font-size: 12px;
|
||||
line-height: 1.55;
|
||||
z-index: 10;
|
||||
backdrop-filter: blur(6px);
|
||||
box-shadow: 0 4px 24px rgba(232, 166, 52, 0.08);
|
||||
}
|
||||
#info h1 { margin: 0 0 2px 0; font-size: 14px; letter-spacing: 0.5px; }
|
||||
#info .sub { font-size: 11px; color: var(--text-mute); margin-bottom: 10px; }
|
||||
#info .row { display: flex; justify-content: space-between; gap: 12px; margin: 2px 0; }
|
||||
#info .row .k { color: var(--text-mute); }
|
||||
#info .row .v { color: var(--amber); font-variant-numeric: tabular-nums; }
|
||||
#info .row .v.live { color: var(--green); }
|
||||
|
||||
/* Bottom-left helper toggle panel */
|
||||
#controls {
|
||||
position: absolute;
|
||||
bottom: 16px;
|
||||
left: 16px;
|
||||
padding: 12px 14px;
|
||||
background: var(--bg-panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
font-size: 12px;
|
||||
z-index: 10;
|
||||
backdrop-filter: blur(6px);
|
||||
min-width: 220px;
|
||||
}
|
||||
#controls h2 {
|
||||
margin: 0 0 8px 0;
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 1.2px;
|
||||
color: var(--text-mute);
|
||||
font-weight: 600;
|
||||
}
|
||||
#controls label {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 4px 0;
|
||||
cursor: pointer;
|
||||
user-select: none;
|
||||
}
|
||||
#controls label:hover { color: var(--amber-hot); }
|
||||
#controls input[type=checkbox] {
|
||||
accent-color: var(--amber);
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
cursor: pointer;
|
||||
}
|
||||
#controls .helper-swatch {
|
||||
display: inline-block;
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 2px;
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
/* Bottom-right ADR badge */
|
||||
#adr-badge {
|
||||
position: absolute;
|
||||
bottom: 16px;
|
||||
right: 16px;
|
||||
padding: 8px 12px;
|
||||
background: var(--bg-panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 6px;
|
||||
font-size: 11px;
|
||||
color: var(--text-mute);
|
||||
z-index: 10;
|
||||
backdrop-filter: blur(6px);
|
||||
}
|
||||
#adr-badge a { color: var(--amber); text-decoration: none; }
|
||||
#adr-badge a:hover { color: var(--amber-hot); }
|
||||
|
||||
/* Top-right legend */
|
||||
#legend {
|
||||
position: absolute;
|
||||
top: 16px;
|
||||
right: 16px;
|
||||
padding: 12px 14px;
|
||||
background: var(--bg-panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
font-size: 11px;
|
||||
z-index: 10;
|
||||
backdrop-filter: blur(6px);
|
||||
min-width: 200px;
|
||||
}
|
||||
#legend h2 {
|
||||
margin: 0 0 8px 0;
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 1.2px;
|
||||
color: var(--text-mute);
|
||||
font-weight: 600;
|
||||
}
|
||||
#legend .item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 3px 0;
|
||||
}
|
||||
#legend .dot {
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 50%;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
#legend .label { font-size: 11px; line-height: 1.3; }
|
||||
</style>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/controls/OrbitControls.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<div id="info">
|
||||
<h1>RuView · Helpers Demo</h1>
|
||||
<div class="sub">ADR-097 · three.js helpers for the point cloud viewer</div>
|
||||
<div class="row"><span class="k">Scene</span><span class="v live">● SYNTHETIC</span></div>
|
||||
<div class="row"><span class="k">Skeleton</span><span class="v">17 kpts · COCO</span></div>
|
||||
<div class="row"><span class="k">Point cloud</span><span class="v" id="pc-count">— pts</span></div>
|
||||
<div class="row"><span class="k">Sensor nodes</span><span class="v">4 · multistatic</span></div>
|
||||
<div class="row"><span class="k">Frame rate</span><span class="v" id="fps">— fps</span></div>
|
||||
<div class="row"><span class="k">Bbox volume</span><span class="v" id="bbox-vol">— m³</span></div>
|
||||
</div>
|
||||
|
||||
<div id="controls">
|
||||
<h2>Helpers</h2>
|
||||
<label><input type="checkbox" id="t-grid" checked>GridHelper<span class="helper-swatch" style="background:#444"></span></label>
|
||||
<label><input type="checkbox" id="t-polar" checked>PolarGridHelper<span class="helper-swatch" style="background:#4a3a1a"></span></label>
|
||||
<label><input type="checkbox" id="t-bbox" checked>BoxHelper<span class="helper-swatch" style="background:#e8a634"></span></label>
|
||||
<label><input type="checkbox" id="t-axes" checked>AxesHelper<span class="helper-swatch" style="background:linear-gradient(90deg,#f44,#4f4,#4cf)"></span></label>
|
||||
<label><input type="checkbox" id="t-nodebox" checked>Per-node BoxHelpers<span class="helper-swatch" style="background:#4cf"></span></label>
|
||||
</div>
|
||||
|
||||
<div id="legend">
|
||||
<h2>Scene</h2>
|
||||
<div class="item"><span class="dot" style="background:#ffff00"></span><span class="label">COCO-17 keypoints (yellow)</span></div>
|
||||
<div class="item"><span class="dot" style="background:#ffffff"></span><span class="label">Bones (white lines)</span></div>
|
||||
<div class="item"><span class="dot" style="background:#4cf"></span><span class="label">Face point cloud (cyan→white)</span></div>
|
||||
<div class="item"><span class="dot" style="background:#e8a634"></span><span class="label">ESP32 sensor nodes</span></div>
|
||||
</div>
|
||||
|
||||
<div id="adr-badge">
|
||||
ADR-097 · <a href="https://threejs.org/examples/#webgl_helpers" target="_blank" rel="noopener">three.js helpers</a>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// =====================================================================
|
||||
// RuView · ADR-097 · three.js helpers demo
|
||||
// --------------------------------------------------------------------
|
||||
// Self-contained, no backend. Demonstrates how `GridHelper`,
|
||||
// `PolarGridHelper`, `BoxHelper`, and `AxesHelper` slot into the
|
||||
// RuView point cloud viewer (`v2/crates/wifi-densepose-pointcloud
|
||||
// /src/viewer.html`). Open this file in a browser — no build step.
|
||||
//
|
||||
// The scene contains:
|
||||
// 1. A synthetic walking, breathing 17-keypoint skeleton.
|
||||
// 2. A face-shaped point cloud attached to the skeleton head.
|
||||
// 3. Four multistatic sensor-node markers arranged around the room.
|
||||
// 4. All four ADR-097 helpers, toggleable from the bottom-left panel.
|
||||
//
|
||||
// Coordinate frame matches the production viewer:
|
||||
// +X = right, +Y = up, +Z = away from camera.
|
||||
// Floor at y = -1.5, person hip at y = 0, head reaches ~ y = 0.7.
|
||||
// =====================================================================
|
||||
|
||||
const COCO_BONES = [
|
||||
// head
|
||||
[0, 1], [0, 2], [1, 3], [2, 4],
|
||||
// torso
|
||||
[5, 6], [5, 11], [6, 12], [11, 12],
|
||||
// left arm
|
||||
[5, 7], [7, 9],
|
||||
// right arm
|
||||
[6, 8], [8, 10],
|
||||
// left leg
|
||||
[11, 13], [13, 15],
|
||||
// right leg
|
||||
[12, 14], [14, 16],
|
||||
];
|
||||
|
||||
// Static "T-pose" skeleton in local frame, animated each frame.
|
||||
// 17 keypoints in COCO order. Units: meters.
|
||||
const SKELETON_BASE = {
|
||||
0: [ 0.00, 0.65, 0.00], // nose
|
||||
1: [-0.04, 0.68, 0.04], // L eye
|
||||
2: [ 0.04, 0.68, 0.04], // R eye
|
||||
3: [-0.08, 0.64, 0.00], // L ear
|
||||
4: [ 0.08, 0.64, 0.00], // R ear
|
||||
5: [-0.18, 0.45, 0.00], // L shoulder
|
||||
6: [ 0.18, 0.45, 0.00], // R shoulder
|
||||
7: [-0.22, 0.20, 0.00], // L elbow
|
||||
8: [ 0.22, 0.20, 0.00], // R elbow
|
||||
9: [-0.26, -0.05, 0.00], // L wrist
|
||||
10: [ 0.26, -0.05, 0.00], // R wrist
|
||||
11: [-0.10, 0.00, 0.00], // L hip
|
||||
12: [ 0.10, 0.00, 0.00], // R hip
|
||||
13: [-0.12, -0.40, 0.00], // L knee
|
||||
14: [ 0.12, -0.40, 0.00], // R knee
|
||||
15: [-0.12, -0.80, 0.00], // L ankle
|
||||
16: [ 0.12, -0.80, 0.00], // R ankle
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Scene + camera + renderer
|
||||
// ---------------------------------------------------------------------
|
||||
const scene = new THREE.Scene();
|
||||
scene.background = new THREE.Color(0x0a0a0a);
|
||||
scene.fog = new THREE.Fog(0x0a0a0a, 6, 14);
|
||||
|
||||
const camera = new THREE.PerspectiveCamera(60, window.innerWidth / window.innerHeight, 0.05, 100);
|
||||
camera.position.set(3.0, 1.4, 4.2);
|
||||
|
||||
const renderer = new THREE.WebGLRenderer({ antialias: true, powerPreference: 'high-performance' });
|
||||
renderer.setPixelRatio(window.devicePixelRatio);
|
||||
renderer.setSize(window.innerWidth, window.innerHeight);
|
||||
document.body.appendChild(renderer.domElement);
|
||||
|
||||
const controls = new THREE.OrbitControls(camera, renderer.domElement);
|
||||
controls.target.set(0, 0, 0);
|
||||
controls.enableDamping = true;
|
||||
controls.dampingFactor = 0.08;
|
||||
controls.minDistance = 1.5;
|
||||
controls.maxDistance = 12;
|
||||
controls.maxPolarAngle = Math.PI * 0.92;
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// ADR-097 helpers — wired to checkbox toggles
|
||||
// ---------------------------------------------------------------------
|
||||
// GridHelper — Cartesian floor reference. Establishes "down" and
|
||||
// scale: 4 m × 4 m floor, 20 divisions = 0.2 m grid spacing.
|
||||
const gridHelper = new THREE.GridHelper(4, 20, 0x444444, 0x222222);
|
||||
gridHelper.position.y = -1.5;
|
||||
scene.add(gridHelper);
|
||||
|
||||
// PolarGridHelper — multistatic geometry reference. 16 radial
|
||||
// divisions (angular bins) × 4 concentric circles, centered on
|
||||
// the fusion target. Matches the bin count in
|
||||
// signal/src/ruvsense/multistatic.rs:attention_weight().
|
||||
const polarHelper = new THREE.PolarGridHelper(2.2, 16, 4, 64, 0x4a3a1a, 0x2a1f10);
|
||||
polarHelper.position.y = -1.499; // a hair above grid to avoid z-fight
|
||||
scene.add(polarHelper);
|
||||
|
||||
// AxesHelper — XYZ tripod at origin. Red = X, green = Y, blue = Z.
|
||||
const axesHelper = new THREE.AxesHelper(0.5);
|
||||
axesHelper.position.set(0, -1.49, 0);
|
||||
scene.add(axesHelper);
|
||||
|
||||
// BoxHelper — per-person bounding volume. Refreshed each frame
|
||||
// after the skeleton is updated. Color = RuView amber.
|
||||
let bboxHelper = null;
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Skeleton — joint spheres + bone lines, animated
|
||||
// ---------------------------------------------------------------------
|
||||
const skeletonGroup = new THREE.Group();
|
||||
scene.add(skeletonGroup);
|
||||
|
||||
const jointGeo = new THREE.SphereGeometry(0.025, 12, 12);
|
||||
const jointMat = new THREE.MeshBasicMaterial({ color: 0xffff00 });
|
||||
const joints = [];
|
||||
for (let i = 0; i < 17; i++) {
|
||||
const sphere = new THREE.Mesh(jointGeo, jointMat);
|
||||
const p = SKELETON_BASE[i];
|
||||
sphere.position.set(p[0], p[1], p[2]);
|
||||
sphere.userData.baseY = p[1];
|
||||
sphere.userData.baseX = p[0];
|
||||
sphere.userData.idx = i;
|
||||
skeletonGroup.add(sphere);
|
||||
joints.push(sphere);
|
||||
}
|
||||
|
||||
const boneMat = new THREE.LineBasicMaterial({ color: 0xffffff, transparent: true, opacity: 0.85 });
|
||||
const bones = [];
|
||||
for (const [a, b] of COCO_BONES) {
|
||||
const geom = new THREE.BufferGeometry();
|
||||
geom.setAttribute('position', new THREE.BufferAttribute(new Float32Array(6), 3));
|
||||
const line = new THREE.Line(geom, boneMat);
|
||||
line.userData = { a, b };
|
||||
skeletonGroup.add(line);
|
||||
bones.push(line);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Face point cloud — synthetic ellipsoid attached to head keypoint
|
||||
// ---------------------------------------------------------------------
|
||||
const FACE_POINTS = 600;
|
||||
const facePositions = new Float32Array(FACE_POINTS * 3);
|
||||
const faceColors = new Float32Array(FACE_POINTS * 3);
|
||||
const faceOffsets = new Float32Array(FACE_POINTS * 3); // canonical face shape, relative to nose
|
||||
|
||||
for (let i = 0; i < FACE_POINTS; i++) {
|
||||
// Sample points roughly on a face-shaped ellipsoid (taller than wide).
|
||||
const u = Math.random() * Math.PI * 2;
|
||||
const v = (Math.random() - 0.5) * Math.PI;
|
||||
const cu = Math.cos(u), su = Math.sin(u);
|
||||
const cv = Math.cos(v), sv = Math.sin(v);
|
||||
// ellipsoid radii (head-like proportions)
|
||||
const rx = 0.085, ry = 0.105, rz = 0.075;
|
||||
faceOffsets[i * 3 + 0] = rx * cv * cu;
|
||||
faceOffsets[i * 3 + 1] = ry * sv;
|
||||
faceOffsets[i * 3 + 2] = rz * cv * su;
|
||||
// depth-encoded color: cyan at back, near-white at front (toward +Z = away from camera)
|
||||
const depthT = (sv + 1) * 0.5;
|
||||
faceColors[i * 3 + 0] = 0.30 + 0.70 * depthT; // R
|
||||
faceColors[i * 3 + 1] = 0.80 + 0.20 * depthT; // G
|
||||
faceColors[i * 3 + 2] = 1.00; // B
|
||||
}
|
||||
const faceGeom = new THREE.BufferGeometry();
|
||||
faceGeom.setAttribute('position', new THREE.BufferAttribute(facePositions, 3));
|
||||
faceGeom.setAttribute('color', new THREE.BufferAttribute(faceColors, 3));
|
||||
const faceMat = new THREE.PointsMaterial({
|
||||
size: 0.012,
|
||||
vertexColors: true,
|
||||
sizeAttenuation: true,
|
||||
transparent: true,
|
||||
opacity: 0.9,
|
||||
});
|
||||
const facePoints = new THREE.Points(faceGeom, faceMat);
|
||||
skeletonGroup.add(facePoints);
|
||||
document.getElementById('pc-count').textContent = FACE_POINTS + ' pts';
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Multistatic sensor nodes — 4 ESP32 markers around the room
|
||||
// ---------------------------------------------------------------------
|
||||
const nodeGroup = new THREE.Group();
|
||||
scene.add(nodeGroup);
|
||||
|
||||
const NODE_POSITIONS = [
|
||||
[-1.9, 1.3, 1.9], // back-left high
|
||||
[ 1.9, 1.3, 1.9], // back-right high
|
||||
[-1.9, 1.3, -1.9], // front-left high
|
||||
[ 1.9, 1.3, -1.9], // front-right high
|
||||
];
|
||||
const nodeBboxHelpers = [];
|
||||
const nodeGeo = new THREE.BoxGeometry(0.12, 0.06, 0.18);
|
||||
const nodeMat = new THREE.MeshBasicMaterial({ color: 0xe8a634 });
|
||||
const nodeAntennaGeo = new THREE.ConeGeometry(0.018, 0.08, 8);
|
||||
const nodeAntennaMat = new THREE.MeshBasicMaterial({ color: 0xffc04d });
|
||||
|
||||
NODE_POSITIONS.forEach((pos, i) => {
|
||||
const group = new THREE.Group();
|
||||
group.position.set(pos[0], pos[1], pos[2]);
|
||||
|
||||
const body = new THREE.Mesh(nodeGeo, nodeMat);
|
||||
group.add(body);
|
||||
|
||||
// little antenna sticking up
|
||||
const antenna = new THREE.Mesh(nodeAntennaGeo, nodeAntennaMat);
|
||||
antenna.position.y = 0.07;
|
||||
group.add(antenna);
|
||||
|
||||
// pulsing emissive ring (visualizes RX activity)
|
||||
const ringGeo = new THREE.RingGeometry(0.10, 0.13, 32);
|
||||
const ringMat = new THREE.MeshBasicMaterial({ color: 0xe8a634, side: THREE.DoubleSide, transparent: true, opacity: 0.4 });
|
||||
const ring = new THREE.Mesh(ringGeo, ringMat);
|
||||
ring.rotation.x = -Math.PI / 2;
|
||||
ring.position.y = -0.04;
|
||||
ring.userData.phase = i * 0.5;
|
||||
group.add(ring);
|
||||
group.userData.ring = ring;
|
||||
|
||||
// sight-line from node to scene origin (visualizes multistatic geometry)
|
||||
const sightGeo = new THREE.BufferGeometry().setFromPoints([
|
||||
new THREE.Vector3(0, 0, 0),
|
||||
new THREE.Vector3(-pos[0], -pos[1], -pos[2]),
|
||||
]);
|
||||
const sightMat = new THREE.LineDashedMaterial({
|
||||
color: 0xe8a634, transparent: true, opacity: 0.18,
|
||||
dashSize: 0.1, gapSize: 0.06,
|
||||
});
|
||||
const sightLine = new THREE.Line(sightGeo, sightMat);
|
||||
sightLine.computeLineDistances();
|
||||
group.add(sightLine);
|
||||
|
||||
nodeGroup.add(group);
|
||||
|
||||
// ADR-097 §3.3 — per-node BoxHelper. Demonstrates that helpers
|
||||
// compose naturally: one box per detected object.
|
||||
const bbox = new THREE.BoxHelper(group, 0x4cf);
|
||||
scene.add(bbox);
|
||||
nodeBboxHelpers.push(bbox);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Animation — synthetic motion model
|
||||
// ---------------------------------------------------------------------
|
||||
let frameStart = performance.now();
|
||||
let frameCount = 0;
|
||||
let fpsAvg = 0;
|
||||
|
||||
function applyPose(t) {
|
||||
// Body sway (slow), breathing (chest expansion), arm/leg swing (walking).
|
||||
const swayX = Math.sin(t * 0.35) * 0.05;
|
||||
const swayZ = Math.cos(t * 0.27) * 0.04;
|
||||
const breathe = Math.sin(t * 1.4) * 0.012; // chest in/out
|
||||
const walkPhase = t * 1.9; // walk cycle
|
||||
|
||||
skeletonGroup.position.set(swayX, 0, swayZ);
|
||||
skeletonGroup.rotation.y = Math.sin(t * 0.22) * 0.18;
|
||||
|
||||
for (let i = 0; i < 17; i++) {
|
||||
const base = SKELETON_BASE[i];
|
||||
let dx = 0, dy = 0, dz = 0;
|
||||
|
||||
// breathing — shoulders + nose rise a little
|
||||
if (i === 0 || i === 1 || i === 2) dy = breathe * 0.6;
|
||||
if (i === 5 || i === 6) dy = breathe;
|
||||
|
||||
// arm swing (opposite of legs)
|
||||
if (i === 7) { dz = Math.sin(walkPhase) * 0.10; dy += Math.cos(walkPhase) * 0.04; }
|
||||
if (i === 9) { dz = Math.sin(walkPhase) * 0.18; dy += Math.cos(walkPhase) * 0.06; }
|
||||
if (i === 8) { dz = -Math.sin(walkPhase) * 0.10; dy += Math.cos(walkPhase) * 0.04; }
|
||||
if (i === 10){ dz = -Math.sin(walkPhase) * 0.18; dy += Math.cos(walkPhase) * 0.06; }
|
||||
|
||||
// leg swing
|
||||
if (i === 13){ dz = -Math.sin(walkPhase) * 0.08; }
|
||||
if (i === 15){ dz = -Math.sin(walkPhase) * 0.15; dy = Math.max(0, Math.cos(walkPhase)) * 0.04; }
|
||||
if (i === 14){ dz = Math.sin(walkPhase) * 0.08; }
|
||||
if (i === 16){ dz = Math.sin(walkPhase) * 0.15; dy = Math.max(0, -Math.cos(walkPhase)) * 0.04; }
|
||||
|
||||
joints[i].position.set(base[0] + dx, base[1] + dy, base[2] + dz);
|
||||
}
|
||||
|
||||
// update bone line vertices from current joint positions
|
||||
for (const line of bones) {
|
||||
const { a, b } = line.userData;
|
||||
const pa = joints[a].position;
|
||||
const pb = joints[b].position;
|
||||
const pos = line.geometry.attributes.position;
|
||||
pos.array[0] = pa.x; pos.array[1] = pa.y; pos.array[2] = pa.z;
|
||||
pos.array[3] = pb.x; pos.array[4] = pb.y; pos.array[5] = pb.z;
|
||||
pos.needsUpdate = true;
|
||||
}
|
||||
|
||||
// attach face point cloud to the nose keypoint (kpt 0)
|
||||
const nose = joints[0].position;
|
||||
const positions = faceGeom.attributes.position;
|
||||
const headTurn = Math.sin(t * 0.6) * 0.35; // y-axis nod
|
||||
const cosH = Math.cos(headTurn), sinH = Math.sin(headTurn);
|
||||
for (let i = 0; i < FACE_POINTS; i++) {
|
||||
const ox = faceOffsets[i * 3 + 0];
|
||||
const oy = faceOffsets[i * 3 + 1];
|
||||
const oz = faceOffsets[i * 3 + 2];
|
||||
// rotate offset around Y axis by headTurn
|
||||
const rx = cosH * ox + sinH * oz;
|
||||
const rz = -sinH * ox + cosH * oz;
|
||||
positions.array[i * 3 + 0] = nose.x + rx;
|
||||
positions.array[i * 3 + 1] = nose.y + oy;
|
||||
positions.array[i * 3 + 2] = nose.z + rz;
|
||||
}
|
||||
positions.needsUpdate = true;
|
||||
}
|
||||
|
||||
function updateNodes(t) {
|
||||
nodeGroup.children.forEach((node, i) => {
|
||||
const ring = node.userData.ring;
|
||||
const phase = (t * 1.8 + ring.userData.phase) % (Math.PI * 2);
|
||||
ring.material.opacity = 0.18 + 0.42 * Math.max(0, Math.cos(phase));
|
||||
ring.scale.setScalar(1 + 0.18 * Math.max(0, Math.cos(phase)));
|
||||
});
|
||||
}
|
||||
|
||||
function updateBboxHelper() {
|
||||
const want = document.getElementById('t-bbox').checked;
|
||||
if (!want) {
|
||||
if (bboxHelper) { scene.remove(bboxHelper); bboxHelper = null; }
|
||||
return;
|
||||
}
|
||||
skeletonGroup.updateMatrixWorld(true);
|
||||
if (!bboxHelper) {
|
||||
bboxHelper = new THREE.BoxHelper(skeletonGroup, 0xe8a634);
|
||||
scene.add(bboxHelper);
|
||||
} else {
|
||||
bboxHelper.setFromObject(skeletonGroup);
|
||||
}
|
||||
// compute volume for the HUD
|
||||
const box = new THREE.Box3().setFromObject(skeletonGroup);
|
||||
const size = box.getSize(new THREE.Vector3());
|
||||
document.getElementById('bbox-vol').textContent =
|
||||
(size.x * size.y * size.z).toFixed(3) + ' m³';
|
||||
}
|
||||
|
||||
function tick() {
|
||||
const now = performance.now();
|
||||
const t = now * 0.001;
|
||||
const dt = now - frameStart;
|
||||
frameStart = now;
|
||||
frameCount++;
|
||||
if (frameCount % 30 === 0) {
|
||||
fpsAvg = 1000 / dt;
|
||||
document.getElementById('fps').textContent = fpsAvg.toFixed(0) + ' fps';
|
||||
}
|
||||
|
||||
applyPose(t);
|
||||
updateNodes(t);
|
||||
updateBboxHelper();
|
||||
|
||||
controls.update();
|
||||
renderer.render(scene, camera);
|
||||
requestAnimationFrame(tick);
|
||||
}
|
||||
requestAnimationFrame(tick);
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Controls wiring — checkbox toggles attach/detach helpers from scene
|
||||
// ---------------------------------------------------------------------
|
||||
function bindToggle(id, obj) {
|
||||
const el = document.getElementById(id);
|
||||
el.addEventListener('change', () => {
|
||||
if (el.checked) {
|
||||
if (!scene.children.includes(obj)) scene.add(obj);
|
||||
} else {
|
||||
scene.remove(obj);
|
||||
}
|
||||
});
|
||||
}
|
||||
bindToggle('t-grid', gridHelper);
|
||||
bindToggle('t-polar', polarHelper);
|
||||
bindToggle('t-axes', axesHelper);
|
||||
|
||||
// per-node bbox toggle (group of 4)
|
||||
document.getElementById('t-nodebox').addEventListener('change', (e) => {
|
||||
for (const bb of nodeBboxHelpers) {
|
||||
if (e.target.checked) {
|
||||
if (!scene.children.includes(bb)) scene.add(bb);
|
||||
} else {
|
||||
scene.remove(bb);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Resize
|
||||
// ---------------------------------------------------------------------
|
||||
window.addEventListener('resize', () => {
|
||||
camera.aspect = window.innerWidth / window.innerHeight;
|
||||
camera.updateProjectionMatrix();
|
||||
renderer.setSize(window.innerWidth, window.innerHeight);
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,854 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>RuView · Skinned · ADR-097 + GLTF skinned mesh + additive animation blending</title>
|
||||
<link rel="icon" type="image/svg+xml" href="data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'><circle cx='16' cy='16' r='10' fill='%23e8a634'/></svg>">
|
||||
<style>
|
||||
:root {
|
||||
--bg: #050507;
|
||||
--bg-panel: rgba(8, 10, 14, 0.78);
|
||||
--amber: #ffb840;
|
||||
--amber-hot: #ffe09f;
|
||||
--cyan: #4cf;
|
||||
--magenta: #ff4cc8;
|
||||
--text: #d8c69a;
|
||||
--text-mute: #6b6155;
|
||||
--border: rgba(255, 184, 64, 0.18);
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0; background: var(--bg); color: var(--text); overflow: hidden;
|
||||
font-family: 'SF Mono', 'Cascadia Code', Consolas, monospace;
|
||||
-webkit-font-smoothing: antialiased; font-size: 12px;
|
||||
}
|
||||
canvas { display: block; }
|
||||
|
||||
.overlay-frame {
|
||||
position: fixed; inset: 0; pointer-events: none; z-index: 5;
|
||||
background:
|
||||
radial-gradient(ellipse at center, transparent 55%, rgba(0,0,0,0.55) 100%),
|
||||
linear-gradient(180deg, rgba(0,0,0,0.32) 0%, transparent 18%, transparent 82%, rgba(0,0,0,0.38) 100%);
|
||||
}
|
||||
.scanlines {
|
||||
position: fixed; inset: 0; pointer-events: none; z-index: 6;
|
||||
background: repeating-linear-gradient(0deg, rgba(0,0,0,0.04) 0px, rgba(0,0,0,0.04) 1px, transparent 1px, transparent 3px);
|
||||
mix-blend-mode: overlay; opacity: 0.5;
|
||||
}
|
||||
.panel {
|
||||
position: absolute; background: var(--bg-panel); border: 1px solid var(--border);
|
||||
border-radius: 4px; padding: 12px 14px; backdrop-filter: blur(8px);
|
||||
box-shadow: 0 1px 0 rgba(255, 184, 64, 0.04), 0 8px 32px rgba(0,0,0,0.55); z-index: 10;
|
||||
}
|
||||
.panel h2 {
|
||||
margin: 0 0 8px 0; font-size: 10px; text-transform: uppercase; letter-spacing: 2px;
|
||||
color: var(--amber); font-weight: 600; border-bottom: 1px solid var(--border); padding-bottom: 6px;
|
||||
}
|
||||
|
||||
#info { top: 20px; left: 20px; min-width: 280px; }
|
||||
#info h1 { margin: 0 0 1px 0; font-size: 13px; letter-spacing: 1px; color: var(--amber-hot); font-weight: 600; }
|
||||
#info .sub { font-size: 10px; color: var(--text-mute); letter-spacing: 0.5px; margin-bottom: 10px; padding-bottom: 8px; border-bottom: 1px solid var(--border); }
|
||||
#info .row { display: flex; justify-content: space-between; gap: 12px; padding: 2px 0; }
|
||||
#info .row .k { color: var(--text-mute); font-size: 11px; }
|
||||
#info .row .v { color: var(--text); font-variant-numeric: tabular-nums; font-size: 11px; }
|
||||
#info .row .v.amber { color: var(--amber); }
|
||||
#info .row .v.cyan { color: var(--cyan); }
|
||||
#info .row .v.mag { color: var(--magenta); }
|
||||
|
||||
#anim {
|
||||
position: absolute; bottom: 20px; left: 20px; min-width: 280px;
|
||||
background: var(--bg-panel); border: 1px solid var(--border); border-radius: 4px;
|
||||
padding: 12px 14px; backdrop-filter: blur(8px); z-index: 10;
|
||||
}
|
||||
#anim h2 { margin: 0 0 8px 0; font-size: 10px; text-transform: uppercase; letter-spacing: 2px;
|
||||
color: var(--amber); font-weight: 600; border-bottom: 1px solid var(--border); padding-bottom: 6px; }
|
||||
#anim .group { padding: 6px 0; border-bottom: 1px solid rgba(255,184,64,0.08); }
|
||||
#anim .group:last-child { border-bottom: none; }
|
||||
#anim .group-label { font-size: 10px; color: var(--text-mute); text-transform: uppercase; letter-spacing: 1px; margin-bottom: 4px; }
|
||||
#anim button {
|
||||
background: rgba(255,184,64,0.06); border: 1px solid rgba(255,184,64,0.18);
|
||||
color: var(--text); font-family: inherit; font-size: 10px; padding: 4px 8px;
|
||||
margin: 2px 4px 2px 0; cursor: pointer; border-radius: 3px; letter-spacing: 0.5px;
|
||||
}
|
||||
#anim button:hover { background: rgba(255,184,64,0.14); color: var(--amber-hot); }
|
||||
#anim button.active { background: var(--amber); color: var(--bg); border-color: var(--amber); font-weight: 600; }
|
||||
#anim .slider-row { display: flex; align-items: center; gap: 8px; padding: 3px 0; font-size: 10px; }
|
||||
#anim .slider-row .label { width: 90px; color: var(--text-mute); }
|
||||
#anim .slider-row input[type=range] { flex: 1; accent-color: var(--amber); }
|
||||
#anim .slider-row .val { width: 38px; text-align: right; color: var(--amber); font-variant-numeric: tabular-nums; }
|
||||
|
||||
#csi { top: 20px; right: 20px; min-width: 260px; }
|
||||
#csi .bar-row { display: flex; align-items: center; gap: 8px; padding: 3px 0; font-size: 10px; }
|
||||
#csi .bar-row .label { width: 42px; color: var(--text-mute); }
|
||||
#csi .bar-row .bar-track { flex: 1; height: 6px; background: rgba(255,184,64,0.08); border-radius: 2px; overflow: hidden; }
|
||||
#csi .bar-row .bar-fill {
|
||||
height: 100%; background: linear-gradient(90deg, var(--amber-hot), var(--amber));
|
||||
box-shadow: 0 0 6px var(--amber); transition: width 0.08s linear;
|
||||
}
|
||||
#csi .bar-row .val { width: 36px; text-align: right; color: var(--amber); font-variant-numeric: tabular-nums; }
|
||||
|
||||
#helpers {
|
||||
position: absolute; bottom: 20px; right: 20px; min-width: 220px;
|
||||
background: var(--bg-panel); border: 1px solid var(--border); border-radius: 4px;
|
||||
padding: 12px 14px; backdrop-filter: blur(8px); z-index: 10;
|
||||
}
|
||||
#helpers h2 { margin: 0 0 8px 0; font-size: 10px; text-transform: uppercase; letter-spacing: 2px;
|
||||
color: var(--amber); font-weight: 600; border-bottom: 1px solid var(--border); padding-bottom: 6px; }
|
||||
#helpers label {
|
||||
display: flex; align-items: center; gap: 10px; padding: 3px 0; cursor: pointer; user-select: none; font-size: 11px;
|
||||
}
|
||||
#helpers label:hover { color: var(--amber-hot); }
|
||||
#helpers input[type=checkbox] { accent-color: var(--amber); width: 13px; height: 13px; cursor: pointer; }
|
||||
#helpers .swatch { width: 8px; height: 8px; border-radius: 50%; margin-left: auto; box-shadow: 0 0 6px currentColor; }
|
||||
|
||||
#loading {
|
||||
position: absolute; inset: 0; display: flex; align-items: center; justify-content: center;
|
||||
background: rgba(5, 5, 7, 0.96); z-index: 20; font-size: 13px; color: var(--amber);
|
||||
letter-spacing: 2px; text-transform: uppercase;
|
||||
}
|
||||
#loading.hidden { display: none; }
|
||||
#loading .text {
|
||||
text-shadow: 0 0 12px var(--amber);
|
||||
animation: loadPulse 1.4s ease-in-out infinite;
|
||||
}
|
||||
@keyframes loadPulse { 0%,100% { opacity: 0.4; } 50% { opacity: 1.0; } }
|
||||
|
||||
@keyframes scanFlash {
|
||||
0% { opacity: 0; } 10% { opacity: 0.12; } 100% { opacity: 0; }
|
||||
}
|
||||
.scan-flash {
|
||||
position: fixed; inset: 0;
|
||||
background: linear-gradient(90deg, transparent, var(--magenta), transparent);
|
||||
mix-blend-mode: screen; pointer-events: none; opacity: 0; z-index: 4;
|
||||
}
|
||||
|
||||
#titlecard {
|
||||
position: absolute; bottom: 76px; left: 50%; transform: translateX(-50%);
|
||||
text-align: center; color: var(--amber-hot); letter-spacing: 6px; font-size: 11px;
|
||||
text-transform: uppercase; opacity: 0.35; z-index: 10;
|
||||
text-shadow: 0 0 12px var(--amber); pointer-events: none;
|
||||
}
|
||||
#titlecard .sub { font-size: 9px; color: var(--text-mute); letter-spacing: 4px; margin-top: 4px; }
|
||||
|
||||
#adr-badge {
|
||||
position: absolute; top: 50%; right: 20px; transform: translateY(-50%);
|
||||
padding: 6px 10px; background: var(--bg-panel); border: 1px solid var(--border);
|
||||
border-radius: 4px; font-size: 9px; color: var(--text-mute); z-index: 10;
|
||||
backdrop-filter: blur(8px); letter-spacing: 0.5px; max-width: 70px; text-align: center; line-height: 1.5;
|
||||
}
|
||||
#adr-badge a { color: var(--amber); text-decoration: none; display: block; }
|
||||
#adr-badge a:hover { color: var(--amber-hot); }
|
||||
</style>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/controls/OrbitControls.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/loaders/GLTFLoader.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/postprocessing/EffectComposer.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/postprocessing/RenderPass.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/postprocessing/ShaderPass.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/postprocessing/UnrealBloomPass.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/shaders/CopyShader.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/shaders/LuminosityHighPassShader.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<div class="overlay-frame"></div>
|
||||
<div class="scanlines"></div>
|
||||
<div class="scan-flash" id="scan-flash"></div>
|
||||
|
||||
<div id="loading"><div class="text">▸ Loading skinned subject · Xbot.glb · 2.9 MB</div></div>
|
||||
|
||||
<div class="panel" id="info">
|
||||
<h1>RuView · Skinned</h1>
|
||||
<div class="sub">ADR-097 · GLTF skinned mesh · additive animation blending</div>
|
||||
<div class="row"><span class="k">Subject</span><span class="v amber">● Tracked</span></div>
|
||||
<div class="row"><span class="k">Model</span><span class="v">Xbot.glb · 14k tris</span></div>
|
||||
<div class="row"><span class="k">Base anim</span><span class="v amber" id="base-name">walk</span></div>
|
||||
<div class="row"><span class="k">Additive</span><span class="v mag" id="add-name">headShake · 0.40</span></div>
|
||||
<div class="row"><span class="k">Mesh nodes</span><span class="v cyan">4 · multistatic</span></div>
|
||||
<div class="row"><span class="k">Coherence</span><span class="v" id="coh-val">— %</span></div>
|
||||
<div class="row"><span class="k">Heart rate</span><span class="v amber" id="hr-val">— bpm</span></div>
|
||||
<div class="row"><span class="k">Bbox vol</span><span class="v" id="bbox-vol">— m³</span></div>
|
||||
<div class="row"><span class="k">Render</span><span class="v" id="fps-val">— fps</span></div>
|
||||
</div>
|
||||
|
||||
<div id="anim">
|
||||
<h2>AnimationMixer</h2>
|
||||
<div class="group">
|
||||
<div class="group-label">Base · loops</div>
|
||||
<button data-base="idle">idle</button>
|
||||
<button data-base="walk" class="active">walk</button>
|
||||
<button data-base="run">run</button>
|
||||
</div>
|
||||
<div class="group">
|
||||
<div class="group-label">Additive · layered</div>
|
||||
<button data-add="agree">agree</button>
|
||||
<button data-add="headShake" class="active">headShake</button>
|
||||
<button data-add="sad_pose">sad</button>
|
||||
<button data-add="sneak_pose">sneak</button>
|
||||
</div>
|
||||
<div class="group">
|
||||
<div class="slider-row">
|
||||
<span class="label">add weight</span>
|
||||
<input type="range" id="add-weight" min="0" max="1" step="0.01" value="0.40">
|
||||
<span class="val" id="add-weight-val">0.40</span>
|
||||
</div>
|
||||
<div class="slider-row">
|
||||
<span class="label">time scale</span>
|
||||
<input type="range" id="time-scale" min="0.1" max="2" step="0.05" value="1.0">
|
||||
<span class="val" id="time-scale-val">1.00</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel" id="csi">
|
||||
<h2>Per-node CSI</h2>
|
||||
<div class="bar-row"><span class="label">N1·BL</span><div class="bar-track"><div class="bar-fill" id="bar-0" style="width:0"></div></div><span class="val" id="val-0">—</span></div>
|
||||
<div class="bar-row"><span class="label">N2·BR</span><div class="bar-track"><div class="bar-fill" id="bar-1" style="width:0"></div></div><span class="val" id="val-1">—</span></div>
|
||||
<div class="bar-row"><span class="label">N3·FL</span><div class="bar-track"><div class="bar-fill" id="bar-2" style="width:0"></div></div><span class="val" id="val-2">—</span></div>
|
||||
<div class="bar-row"><span class="label">N4·FR</span><div class="bar-track"><div class="bar-fill" id="bar-3" style="width:0"></div></div><span class="val" id="val-3">—</span></div>
|
||||
</div>
|
||||
|
||||
<div id="helpers">
|
||||
<h2>ADR-097 helpers</h2>
|
||||
<label><input type="checkbox" id="t-grid" checked>GridHelper<span class="swatch" style="color:#666"></span></label>
|
||||
<label><input type="checkbox" id="t-polar" checked>PolarGridHelper<span class="swatch" style="color:#ffb840"></span></label>
|
||||
<label><input type="checkbox" id="t-bbox" checked>BoxHelper on mesh<span class="swatch" style="color:#ffe09f"></span></label>
|
||||
<label><input type="checkbox" id="t-skel">SkeletonHelper<span class="swatch" style="color:#4cf"></span></label>
|
||||
<label><input type="checkbox" id="t-nodebox" checked>Per-node BoxHelpers<span class="swatch" style="color:#4cf"></span></label>
|
||||
<label><input type="checkbox" id="t-pings" checked>Sonar pings<span class="swatch" style="color:#4cf"></span></label>
|
||||
<label><input type="checkbox" id="t-tomo" checked>Tomography sweep<span class="swatch" style="color:#ff4cc8"></span></label>
|
||||
</div>
|
||||
|
||||
<div id="titlecard">
|
||||
RuView · Seldon Vault
|
||||
<div class="sub">skinned · ADR-097 · CCDIKSolver next</div>
|
||||
</div>
|
||||
|
||||
<div id="adr-badge">
|
||||
<a href="https://threejs.org/examples/#webgl_animation_skinning_additive_blending" target="_blank" rel="noopener">additive blend</a>
|
||||
<a href="https://threejs.org/examples/#webgl_animation_skinning_ik" target="_blank" rel="noopener" style="margin-top:4px;">skinning IK</a>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// =====================================================================
|
||||
// RuView · Skinned · ADR-097 + GLTF skinned mesh + additive animation
|
||||
// --------------------------------------------------------------------
|
||||
// Replaces the procedural sphere-skeleton of helpers-cinematic.html
|
||||
// with a real rigged + skinned humanoid loaded from Xbot.glb. Plays
|
||||
// a base loop (walk / run / idle) and layers an additive pose on
|
||||
// top (headShake / agree / sneak / sad) — mirrors the upstream
|
||||
// three.js webgl_animation_skinning_additive_blending example.
|
||||
//
|
||||
// All ADR-097 helpers still wrap the loaded mesh — BoxHelper picks
|
||||
// up the live AABB of the SkinnedMesh, the polar grid sits under
|
||||
// the rig, and per-node BoxHelpers wrap the four ESP32 markers.
|
||||
//
|
||||
// Production path (next): swap canned GLTF animations for live
|
||||
// COCO-17 keypoint output → CCDIKSolver targets on hands/feet/head.
|
||||
// Reference: three.js webgl_animation_skinning_ik example.
|
||||
// =====================================================================
|
||||
|
||||
const MODEL_URL = 'https://threejs.org/examples/models/gltf/Xbot.glb';
|
||||
|
||||
const NODE_POSITIONS = [
|
||||
[-1.9, 1.3, 1.9],[ 1.9, 1.3, 1.9],
|
||||
[-1.9, 1.3, -1.9],[ 1.9, 1.3, -1.9],
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Scene
|
||||
// ---------------------------------------------------------------------
|
||||
const scene = new THREE.Scene();
|
||||
scene.background = new THREE.Color(0x050507);
|
||||
scene.fog = new THREE.FogExp2(0x050507, 0.06);
|
||||
|
||||
const camera = new THREE.PerspectiveCamera(48, window.innerWidth/window.innerHeight, 0.05, 100);
|
||||
camera.position.set(3.2, 1.55, 4.0);
|
||||
|
||||
const renderer = new THREE.WebGLRenderer({ antialias: true, powerPreference: 'high-performance' });
|
||||
renderer.setPixelRatio(Math.min(2, window.devicePixelRatio));
|
||||
renderer.setSize(window.innerWidth, window.innerHeight);
|
||||
renderer.toneMapping = THREE.ACESFilmicToneMapping;
|
||||
renderer.toneMappingExposure = 0.80;
|
||||
renderer.outputEncoding = THREE.sRGBEncoding;
|
||||
renderer.shadowMap.enabled = true;
|
||||
renderer.shadowMap.type = THREE.PCFSoftShadowMap;
|
||||
document.body.appendChild(renderer.domElement);
|
||||
|
||||
const controls = new THREE.OrbitControls(camera, renderer.domElement);
|
||||
controls.target.set(0, 0.9, 0);
|
||||
controls.enableDamping = true;
|
||||
controls.dampingFactor = 0.06;
|
||||
controls.minDistance = 2; controls.maxDistance = 12;
|
||||
controls.maxPolarAngle = Math.PI * 0.92;
|
||||
controls.autoRotate = new URLSearchParams(location.search).get('orbit') === '1';
|
||||
controls.autoRotateSpeed = 0.25;
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Lights — the GLTF uses PBR materials so we actually need lighting
|
||||
// here (unlike the all-emissive cinematic.html). Tuned to keep the
|
||||
// amber/cyan mood: amber hemi + amber key + cyan rim lights from
|
||||
// each node direction (visualizes "the nodes illuminate the subject").
|
||||
// ---------------------------------------------------------------------
|
||||
const hemiLight = new THREE.HemisphereLight(0x553a18, 0x080606, 0.7);
|
||||
hemiLight.position.set(0, 4, 0);
|
||||
scene.add(hemiLight);
|
||||
|
||||
const keyLight = new THREE.DirectionalLight(0xffc070, 0.95);
|
||||
keyLight.position.set(2.5, 3.8, 2.5);
|
||||
keyLight.castShadow = true;
|
||||
keyLight.shadow.camera.top = 2; keyLight.shadow.camera.bottom = -2;
|
||||
keyLight.shadow.camera.left = -2; keyLight.shadow.camera.right = 2;
|
||||
keyLight.shadow.camera.near = 0.1; keyLight.shadow.camera.far = 12;
|
||||
keyLight.shadow.mapSize.set(1024, 1024);
|
||||
keyLight.shadow.bias = -0.0008;
|
||||
scene.add(keyLight);
|
||||
|
||||
// cyan rim lights, one per ESP32 node — keeps the "sensed by the mesh" mood
|
||||
const rimLights = [];
|
||||
NODE_POSITIONS.forEach(pos => {
|
||||
const rim = new THREE.PointLight(0x4cf, 0.55, 8, 1.8);
|
||||
rim.position.set(pos[0] * 1.1, pos[1] * 0.7, pos[2] * 1.1);
|
||||
scene.add(rim);
|
||||
rimLights.push(rim);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Post-processing — same composer as cinematic.html
|
||||
// ---------------------------------------------------------------------
|
||||
const composer = new THREE.EffectComposer(renderer);
|
||||
composer.addPass(new THREE.RenderPass(scene, camera));
|
||||
const bloom = new THREE.UnrealBloomPass(
|
||||
new THREE.Vector2(window.innerWidth, window.innerHeight),
|
||||
0.45, 0.40, 0.78,
|
||||
);
|
||||
composer.addPass(bloom);
|
||||
|
||||
const filmShader = {
|
||||
uniforms: {
|
||||
tDiffuse: { value: null },
|
||||
time: { value: 0 }, grain: { value: 0.04 },
|
||||
vignette: { value: 0.32 }, aberration: { value: 0.0018 },
|
||||
},
|
||||
vertexShader: `varying vec2 vUv; void main() { vUv = uv; gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0); }`,
|
||||
fragmentShader: `
|
||||
uniform sampler2D tDiffuse; uniform float time, grain, vignette, aberration;
|
||||
varying vec2 vUv;
|
||||
float hash(vec2 p) { return fract(sin(dot(p, vec2(12.9898, 78.233))) * 43758.5453); }
|
||||
void main() {
|
||||
vec2 off = (vUv - 0.5) * aberration;
|
||||
float r = texture2D(tDiffuse, vUv + off).r;
|
||||
float g = texture2D(tDiffuse, vUv).g;
|
||||
float b = texture2D(tDiffuse, vUv - off).b;
|
||||
vec3 col = vec3(r, g, b);
|
||||
col += (hash(vUv * 1024.0 + time) - 0.5) * grain;
|
||||
float v = smoothstep(0.85, 0.20, length(vUv - 0.5));
|
||||
col *= mix(1.0 - vignette, 1.0, v);
|
||||
gl_FragColor = vec4(col, 1.0);
|
||||
}`,
|
||||
};
|
||||
const filmPass = new THREE.ShaderPass(filmShader);
|
||||
composer.addPass(filmPass);
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Floor — same procedural cyber grid (toned down for skinned scene)
|
||||
// ---------------------------------------------------------------------
|
||||
const floorMat = new THREE.ShaderMaterial({
|
||||
uniforms: { time: { value: 0 }, baseColor: { value: new THREE.Color(0xffb840) } },
|
||||
vertexShader: `varying vec3 vPos; void main() { vPos = position; gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0); }`,
|
||||
fragmentShader: `
|
||||
uniform float time; uniform vec3 baseColor; varying vec3 vPos;
|
||||
void main() {
|
||||
vec2 g = abs(fract(vPos.xz * 0.5) - 0.5);
|
||||
float line = smoothstep(0.48, 0.50, max(g.x, g.y));
|
||||
float majorLine = smoothstep(0.96, 1.00, max(g.x, g.y) * 2.0);
|
||||
float scan = 0.5 + 0.5 * sin((vPos.x + vPos.z) * 2.0 - time * 1.4);
|
||||
scan = pow(scan, 14.0);
|
||||
float falloff = smoothstep(5.0, 1.2, length(vPos.xz));
|
||||
vec3 col = baseColor * (0.01 + 0.05 * line + 0.16 * majorLine + 0.08 * scan);
|
||||
gl_FragColor = vec4(col * falloff, falloff * 0.55);
|
||||
}`,
|
||||
transparent: true, depthWrite: false,
|
||||
});
|
||||
const floor = new THREE.Mesh(new THREE.PlaneGeometry(20, 20), floorMat);
|
||||
floor.rotation.x = -Math.PI / 2;
|
||||
floor.position.y = 0;
|
||||
scene.add(floor);
|
||||
|
||||
// shadow-receiving ground (invisible, just catches the shadow)
|
||||
const shadowGround = new THREE.Mesh(
|
||||
new THREE.PlaneGeometry(20, 20),
|
||||
new THREE.ShadowMaterial({ opacity: 0.55 })
|
||||
);
|
||||
shadowGround.rotation.x = -Math.PI / 2;
|
||||
shadowGround.position.y = 0.001;
|
||||
shadowGround.receiveShadow = true;
|
||||
scene.add(shadowGround);
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// ADR-097 helpers
|
||||
// ---------------------------------------------------------------------
|
||||
const gridHelper = new THREE.GridHelper(4, 20, 0x554a32, 0x2a2418);
|
||||
gridHelper.material.transparent = true; gridHelper.material.opacity = 0.45;
|
||||
scene.add(gridHelper);
|
||||
|
||||
const polarHelper = new THREE.PolarGridHelper(2.2, 16, 4, 64, 0xffb840, 0x4a3a1a);
|
||||
polarHelper.position.y = 0.002;
|
||||
polarHelper.material.transparent = true; polarHelper.material.opacity = 0.55;
|
||||
scene.add(polarHelper);
|
||||
|
||||
let bboxHelper = null;
|
||||
let skeletonHelper = null;
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Multistatic sensor nodes — same as cinematic
|
||||
// ---------------------------------------------------------------------
|
||||
const nodeGroup = new THREE.Group();
|
||||
scene.add(nodeGroup);
|
||||
const nodeBboxHelpers = [];
|
||||
const nodeRings = [];
|
||||
const nodeAnchors = [];
|
||||
const nodeBodyGeo = new THREE.BoxGeometry(0.14, 0.06, 0.20);
|
||||
const nodeBodyMat = new THREE.MeshBasicMaterial({ color: 0xffb840 });
|
||||
const antennaGeo = new THREE.ConeGeometry(0.018, 0.10, 8);
|
||||
const antennaMat = new THREE.MeshBasicMaterial({ color: 0xffe09f });
|
||||
|
||||
NODE_POSITIONS.forEach((pos, i) => {
|
||||
const group = new THREE.Group();
|
||||
group.position.set(pos[0], pos[1], pos[2]);
|
||||
|
||||
const body = new THREE.Mesh(nodeBodyGeo, nodeBodyMat);
|
||||
group.add(body);
|
||||
const antenna = new THREE.Mesh(antennaGeo, antennaMat);
|
||||
antenna.position.y = 0.08; group.add(antenna);
|
||||
|
||||
const ring = new THREE.Mesh(
|
||||
new THREE.RingGeometry(0.11, 0.14, 32),
|
||||
new THREE.MeshBasicMaterial({ color: 0xffb840, side: THREE.DoubleSide, transparent: true,
|
||||
opacity: 0.55, blending: THREE.AdditiveBlending, depthWrite: false })
|
||||
);
|
||||
ring.rotation.x = -Math.PI / 2; ring.position.y = -0.05;
|
||||
ring.userData.phase = i * 0.7;
|
||||
group.add(ring); nodeRings.push(ring);
|
||||
|
||||
const core = new THREE.Mesh(
|
||||
new THREE.SphereGeometry(0.025, 12, 12),
|
||||
new THREE.MeshBasicMaterial({ color: 0xffe09f })
|
||||
);
|
||||
core.position.y = 0.04; group.add(core);
|
||||
|
||||
nodeGroup.add(group); nodeAnchors.push(group);
|
||||
|
||||
const bbox = new THREE.BoxHelper(group, 0x4cf);
|
||||
bbox.material.transparent = true; bbox.material.opacity = 0.45;
|
||||
scene.add(bbox); nodeBboxHelpers.push(bbox);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// GLTF — load the rigged Xbot model
|
||||
// ---------------------------------------------------------------------
|
||||
let model = null;
|
||||
let mixer = null;
|
||||
let headBone = null;
|
||||
const baseActions = {}; // idle / walk / run
|
||||
const additiveActions = {}; // sneak_pose / sad_pose / agree / headShake
|
||||
let currentBase = 'walk';
|
||||
let currentAddName = 'headShake';
|
||||
let addWeight = 0.40;
|
||||
|
||||
const loader = new THREE.GLTFLoader();
|
||||
loader.load(MODEL_URL, (gltf) => {
|
||||
model = gltf.scene;
|
||||
model.position.y = 0;
|
||||
model.traverse(obj => {
|
||||
if (obj.isMesh) { obj.castShadow = true; obj.receiveShadow = true; }
|
||||
if (obj.isBone && /head/i.test(obj.name) && !headBone) headBone = obj;
|
||||
});
|
||||
scene.add(model);
|
||||
|
||||
skeletonHelper = new THREE.SkeletonHelper(model);
|
||||
skeletonHelper.visible = false;
|
||||
scene.add(skeletonHelper);
|
||||
|
||||
mixer = new THREE.AnimationMixer(model);
|
||||
const baseNames = new Set(['idle', 'walk', 'run']);
|
||||
const additiveNames = new Set(['sneak_pose', 'sad_pose', 'agree', 'headShake']);
|
||||
|
||||
for (let i = 0; i < gltf.animations.length; i++) {
|
||||
let clip = gltf.animations[i];
|
||||
const name = clip.name;
|
||||
if (baseNames.has(name)) {
|
||||
const action = mixer.clipAction(clip);
|
||||
action.enabled = true;
|
||||
action.setEffectiveTimeScale(1);
|
||||
action.setEffectiveWeight(name === currentBase ? 1 : 0);
|
||||
action.play();
|
||||
baseActions[name] = action;
|
||||
} else if (additiveNames.has(name)) {
|
||||
THREE.AnimationUtils.makeClipAdditive(clip);
|
||||
if (name.endsWith('_pose')) {
|
||||
clip = THREE.AnimationUtils.subclip(clip, name, 2, 3, 30);
|
||||
}
|
||||
const action = mixer.clipAction(clip);
|
||||
action.enabled = true;
|
||||
action.setEffectiveTimeScale(1);
|
||||
action.setEffectiveWeight(name === currentAddName ? addWeight : 0);
|
||||
action.play();
|
||||
additiveActions[name] = action;
|
||||
}
|
||||
}
|
||||
|
||||
// build the face point cloud anchored to head bone
|
||||
buildFacePointCloud();
|
||||
|
||||
document.getElementById('loading').classList.add('hidden');
|
||||
}, (xhr) => {
|
||||
const pct = xhr.loaded / (xhr.total || 2930032) * 100;
|
||||
const txt = document.querySelector('#loading .text');
|
||||
if (txt) txt.textContent = `▸ Loading skinned subject · Xbot.glb · ${pct.toFixed(0)} %`;
|
||||
}, (err) => {
|
||||
console.error('GLTF load failed', err);
|
||||
document.querySelector('#loading .text').textContent = '⚠ Load failed — see console';
|
||||
});
|
||||
|
||||
function setBase(name) {
|
||||
if (!baseActions[name]) return;
|
||||
for (const k in baseActions) {
|
||||
const a = baseActions[k];
|
||||
const target = (k === name) ? 1 : 0;
|
||||
a.crossFadeTo ? null : null; // (no-op — using simple weight crossfade)
|
||||
a.setEffectiveWeight(target);
|
||||
}
|
||||
currentBase = name;
|
||||
document.getElementById('base-name').textContent = name;
|
||||
for (const btn of document.querySelectorAll('#anim [data-base]')) {
|
||||
btn.classList.toggle('active', btn.dataset.base === name);
|
||||
}
|
||||
}
|
||||
function setAdditive(name) {
|
||||
for (const k in additiveActions) {
|
||||
additiveActions[k].setEffectiveWeight(k === name ? addWeight : 0);
|
||||
}
|
||||
currentAddName = name;
|
||||
document.getElementById('add-name').textContent = name + ' · ' + addWeight.toFixed(2);
|
||||
for (const btn of document.querySelectorAll('#anim [data-add]')) {
|
||||
btn.classList.toggle('active', btn.dataset.add === name);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Face point cloud — anchored to head bone via getWorldPosition each frame
|
||||
// ---------------------------------------------------------------------
|
||||
const FACE_POINTS = 480;
|
||||
const facePositions = new Float32Array(FACE_POINTS * 3);
|
||||
const faceOffsets = new Float32Array(FACE_POINTS * 3);
|
||||
const facePhases = new Float32Array(FACE_POINTS);
|
||||
let facePoints = null;
|
||||
function buildFacePointCloud() {
|
||||
for (let i = 0; i < FACE_POINTS; i++) {
|
||||
const u = Math.random() * Math.PI * 2;
|
||||
const v = (Math.random() - 0.5) * Math.PI * 0.95;
|
||||
const cu = Math.cos(u), su = Math.sin(u);
|
||||
const cv = Math.cos(v), sv = Math.sin(v);
|
||||
faceOffsets[i*3+0] = 0.085 * cv * cu;
|
||||
faceOffsets[i*3+1] = 0.108 * sv;
|
||||
faceOffsets[i*3+2] = 0.072 * cv * su;
|
||||
facePhases[i] = Math.random() * Math.PI * 2;
|
||||
}
|
||||
const geom = new THREE.BufferGeometry();
|
||||
geom.setAttribute('position', new THREE.BufferAttribute(facePositions, 3));
|
||||
geom.setAttribute('aPhase', new THREE.BufferAttribute(facePhases, 1));
|
||||
const mat = new THREE.ShaderMaterial({
|
||||
uniforms: { time: { value: 0 } },
|
||||
vertexShader: `
|
||||
attribute float aPhase; uniform float time;
|
||||
varying float vAlpha;
|
||||
void main() {
|
||||
vec4 mv = modelViewMatrix * vec4(position, 1.0);
|
||||
float shimmer = 0.5 + 0.5 * sin(time * 3.0 + aPhase);
|
||||
vAlpha = 0.18 + 0.30 * shimmer;
|
||||
gl_Position = projectionMatrix * mv;
|
||||
gl_PointSize = (1.6 + shimmer * 1.0) * (200.0 / -mv.z);
|
||||
}`,
|
||||
fragmentShader: `
|
||||
varying float vAlpha;
|
||||
void main() {
|
||||
vec2 c = gl_PointCoord - 0.5;
|
||||
float d = length(c);
|
||||
if (d > 0.5) discard;
|
||||
float falloff = smoothstep(0.5, 0.0, d);
|
||||
vec3 col = mix(vec3(0.18, 0.52, 0.72), vec3(0.55, 0.62, 0.72), 0.5);
|
||||
gl_FragColor = vec4(col * (1.0 + falloff * 0.3), vAlpha * falloff);
|
||||
}`,
|
||||
transparent: true, depthWrite: false,
|
||||
});
|
||||
facePoints = new THREE.Points(geom, mat);
|
||||
scene.add(facePoints);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Sonar pings + tomography sweep — same as cinematic.html
|
||||
// ---------------------------------------------------------------------
|
||||
const PING_POOL = 24;
|
||||
const pings = [];
|
||||
const pingGeo = new THREE.TorusGeometry(1, 0.012, 8, 48);
|
||||
for (let i = 0; i < PING_POOL; i++) {
|
||||
const mat = new THREE.MeshBasicMaterial({ color: 0x4cf, transparent: true, opacity: 0, depthWrite: false });
|
||||
const mesh = new THREE.Mesh(pingGeo, mat);
|
||||
mesh.visible = false; scene.add(mesh);
|
||||
pings.push({ mesh, active: false, t0: 0, duration: 0,
|
||||
origin: new THREE.Vector3(), target: new THREE.Vector3() });
|
||||
}
|
||||
let pingIndex = 0;
|
||||
function emitPing(origin, target) {
|
||||
const p = pings[pingIndex]; pingIndex = (pingIndex + 1) % PING_POOL;
|
||||
p.active = true; p.t0 = performance.now() * 0.001;
|
||||
p.duration = 0.55 + Math.random() * 0.20;
|
||||
p.origin.copy(origin); p.target.copy(target);
|
||||
p.mesh.position.copy(origin); p.mesh.visible = true;
|
||||
p.mesh.material.opacity = 0;
|
||||
const dir = new THREE.Vector3().subVectors(target, origin).normalize();
|
||||
p.mesh.quaternion.setFromUnitVectors(new THREE.Vector3(0, 0, 1), dir);
|
||||
}
|
||||
|
||||
const tomoMat = new THREE.ShaderMaterial({
|
||||
uniforms: { time: { value: 0 }, intensity: { value: 0 } },
|
||||
vertexShader: `varying vec2 vUv; void main() { vUv = uv; gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0); }`,
|
||||
fragmentShader: `
|
||||
uniform float time, intensity; varying vec2 vUv;
|
||||
void main() {
|
||||
float band = exp(-pow((vUv.x - 0.5) * 14.0, 2.0));
|
||||
float lines = 0.5 + 0.5 * sin(vUv.y * 90.0 + time * 4.0);
|
||||
vec3 col = vec3(1.0, 0.3, 0.78) * band * (0.6 + 0.4 * lines);
|
||||
gl_FragColor = vec4(col, intensity * band * 0.75);
|
||||
}`,
|
||||
transparent: true, blending: THREE.AdditiveBlending, depthWrite: false, side: THREE.DoubleSide,
|
||||
});
|
||||
const tomoPlane = new THREE.Mesh(new THREE.PlaneGeometry(8, 6), tomoMat);
|
||||
tomoPlane.rotation.y = Math.PI / 2;
|
||||
tomoPlane.position.set(-2, 1.0, 0);
|
||||
tomoPlane.visible = false;
|
||||
scene.add(tomoPlane);
|
||||
let tomoActive = false, tomoT0 = 0, tomoNextAt = 4 + Math.random() * 4;
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Pseudo-CSI driver — same as cinematic
|
||||
// ---------------------------------------------------------------------
|
||||
const csiAmp = [0, 0, 0, 0];
|
||||
let csiCoherence = 0.5;
|
||||
const csiNoise = [0, 0, 0, 0];
|
||||
|
||||
function tickCsi(t, targetWorld) {
|
||||
for (let i = 0; i < 4; i++) csiNoise[i] = csiNoise[i] * 0.92 + (Math.random() - 0.5) * 0.08;
|
||||
let mean = 0; const amps = [];
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const np = NODE_POSITIONS[i];
|
||||
const dx = np[0] - targetWorld.x, dy = np[1] - targetWorld.y, dz = np[2] - targetWorld.z;
|
||||
const r2 = dx*dx + dy*dy + dz*dz;
|
||||
const fall = 1.0 / (1.0 + r2 * 0.18);
|
||||
const breath = Math.sin(t * 0.27 * Math.PI * 2) * 0.10;
|
||||
const heart = Math.sin(t * 1.18 * Math.PI * 2) * 0.04;
|
||||
const walk = Math.sin(t * 1.9 + i * 0.5) * 0.12;
|
||||
const a = Math.max(0, Math.min(1, fall + breath + heart + walk + csiNoise[i] * 0.30));
|
||||
amps.push(a);
|
||||
csiAmp[i] = csiAmp[i] * 0.7 + a * 0.3;
|
||||
mean += a;
|
||||
}
|
||||
mean /= 4;
|
||||
let v = 0; for (let i = 0; i < 4; i++) v += (amps[i] - mean) ** 2;
|
||||
v = Math.sqrt(v / 4);
|
||||
csiCoherence = csiCoherence * 0.85 + Math.max(0, Math.min(1, 1.0 - v * 2.5)) * 0.15;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Per-frame updates
|
||||
// ---------------------------------------------------------------------
|
||||
const tmpVec = new THREE.Vector3();
|
||||
let lastPingT = [0, 0, 0, 0];
|
||||
|
||||
function updateNodes() {
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const ring = nodeRings[i];
|
||||
const amp = csiAmp[i];
|
||||
ring.material.opacity = 0.32 + 0.55 * amp;
|
||||
ring.scale.setScalar(1 + 0.30 * amp);
|
||||
rimLights[i].intensity = 0.30 + 0.60 * amp * csiCoherence;
|
||||
}
|
||||
}
|
||||
function maybeEmitPings(t, modelCenter) {
|
||||
if (!document.getElementById('t-pings').checked || !model) return;
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const interval = 1.2 / (0.25 + csiAmp[i]);
|
||||
if (t - lastPingT[i] > interval) {
|
||||
lastPingT[i] = t;
|
||||
const target = modelCenter.clone();
|
||||
target.y += (Math.random() - 0.3) * 0.8;
|
||||
target.x += (Math.random() - 0.5) * 0.2;
|
||||
const origin = nodeAnchors[i].getWorldPosition(new THREE.Vector3());
|
||||
emitPing(origin, target);
|
||||
}
|
||||
}
|
||||
}
|
||||
function updatePings(t) {
|
||||
for (const p of pings) {
|
||||
if (!p.active) continue;
|
||||
const u = (t - p.t0) / p.duration;
|
||||
if (u >= 1) { p.active = false; p.mesh.visible = false; continue; }
|
||||
p.mesh.position.lerpVectors(p.origin, p.target, u);
|
||||
p.mesh.scale.setScalar(0.03 + u * 0.18);
|
||||
p.mesh.material.opacity = (1.0 - u) * 0.40 * csiCoherence;
|
||||
}
|
||||
}
|
||||
function updateTomography(t) {
|
||||
if (!document.getElementById('t-tomo').checked) { tomoActive = false; tomoPlane.visible = false; return; }
|
||||
if (!tomoActive && t > tomoNextAt) {
|
||||
tomoActive = true; tomoT0 = t; tomoPlane.visible = true;
|
||||
const sf = document.getElementById('scan-flash');
|
||||
sf.style.animation = 'none';
|
||||
requestAnimationFrame(() => { sf.style.animation = 'scanFlash 1.6s ease-out'; });
|
||||
}
|
||||
if (tomoActive) {
|
||||
const dur = 2.4;
|
||||
const e = (t - tomoT0) / dur;
|
||||
if (e >= 1) {
|
||||
tomoActive = false; tomoPlane.visible = false;
|
||||
tomoNextAt = t + 4 + Math.random() * 5;
|
||||
} else {
|
||||
tomoPlane.position.x = -3 + e * 6;
|
||||
tomoMat.uniforms.intensity.value = Math.sin(e * Math.PI);
|
||||
tomoMat.uniforms.time.value = t;
|
||||
}
|
||||
}
|
||||
}
|
||||
function updateBbox() {
|
||||
const want = document.getElementById('t-bbox').checked && model;
|
||||
if (!want) {
|
||||
if (bboxHelper) { scene.remove(bboxHelper); bboxHelper = null; }
|
||||
document.getElementById('bbox-vol').textContent = '—';
|
||||
return;
|
||||
}
|
||||
if (!bboxHelper) {
|
||||
bboxHelper = new THREE.BoxHelper(model, 0xffe09f);
|
||||
bboxHelper.material.transparent = true; bboxHelper.material.opacity = 0.55;
|
||||
scene.add(bboxHelper);
|
||||
} else bboxHelper.setFromObject(model);
|
||||
const box = new THREE.Box3().setFromObject(model);
|
||||
const size = box.getSize(new THREE.Vector3());
|
||||
document.getElementById('bbox-vol').textContent = (size.x * size.y * size.z).toFixed(3) + ' m³';
|
||||
}
|
||||
function updateFaceCloud(t) {
|
||||
if (!facePoints || !headBone) return;
|
||||
const headWorld = new THREE.Vector3();
|
||||
headBone.getWorldPosition(headWorld);
|
||||
const pos = facePoints.geometry.attributes.position;
|
||||
for (let i = 0; i < FACE_POINTS; i++) {
|
||||
pos.array[i*3+0] = headWorld.x + faceOffsets[i*3+0];
|
||||
pos.array[i*3+1] = headWorld.y + faceOffsets[i*3+1] + 0.06;
|
||||
pos.array[i*3+2] = headWorld.z + faceOffsets[i*3+2];
|
||||
}
|
||||
pos.needsUpdate = true;
|
||||
facePoints.material.uniforms.time.value = t;
|
||||
}
|
||||
let hudT = 0;
|
||||
function updateHud(t, fps) {
|
||||
if (t - hudT < 0.1) return;
|
||||
hudT = t;
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const pct = Math.round(csiAmp[i] * 100);
|
||||
document.getElementById('bar-' + i).style.width = pct + '%';
|
||||
document.getElementById('val-' + i).textContent = pct + '%';
|
||||
}
|
||||
document.getElementById('coh-val').textContent = (csiCoherence * 100).toFixed(0) + ' %';
|
||||
document.getElementById('hr-val').textContent = (68 + Math.sin(t * 0.3) * 4).toFixed(0) + ' bpm';
|
||||
document.getElementById('fps-val').textContent = fps.toFixed(0) + ' fps';
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// UI wiring
|
||||
// ---------------------------------------------------------------------
|
||||
for (const btn of document.querySelectorAll('#anim [data-base]')) {
|
||||
btn.addEventListener('click', () => setBase(btn.dataset.base));
|
||||
}
|
||||
for (const btn of document.querySelectorAll('#anim [data-add]')) {
|
||||
btn.addEventListener('click', () => setAdditive(btn.dataset.add));
|
||||
}
|
||||
document.getElementById('add-weight').addEventListener('input', (e) => {
|
||||
addWeight = parseFloat(e.target.value);
|
||||
document.getElementById('add-weight-val').textContent = addWeight.toFixed(2);
|
||||
if (additiveActions[currentAddName]) additiveActions[currentAddName].setEffectiveWeight(addWeight);
|
||||
document.getElementById('add-name').textContent = currentAddName + ' · ' + addWeight.toFixed(2);
|
||||
});
|
||||
document.getElementById('time-scale').addEventListener('input', (e) => {
|
||||
const ts = parseFloat(e.target.value);
|
||||
document.getElementById('time-scale-val').textContent = ts.toFixed(2);
|
||||
if (mixer) mixer.timeScale = ts;
|
||||
});
|
||||
function bindToggle(id, obj) {
|
||||
document.getElementById(id).addEventListener('change', e => {
|
||||
if (e.target.checked && !scene.children.includes(obj)) scene.add(obj);
|
||||
else if (!e.target.checked) scene.remove(obj);
|
||||
});
|
||||
}
|
||||
bindToggle('t-grid', gridHelper);
|
||||
bindToggle('t-polar', polarHelper);
|
||||
document.getElementById('t-skel').addEventListener('change', e => {
|
||||
if (skeletonHelper) skeletonHelper.visible = e.target.checked;
|
||||
});
|
||||
document.getElementById('t-nodebox').addEventListener('change', e => {
|
||||
for (const bb of nodeBboxHelpers) {
|
||||
if (e.target.checked && !scene.children.includes(bb)) scene.add(bb);
|
||||
else if (!e.target.checked) scene.remove(bb);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Main loop
|
||||
// ---------------------------------------------------------------------
|
||||
const clock = new THREE.Clock();
|
||||
let lastMs = performance.now();
|
||||
let fpsEma = 60;
|
||||
function tick() {
|
||||
const nowMs = performance.now();
|
||||
const dt = nowMs - lastMs;
|
||||
lastMs = nowMs;
|
||||
fpsEma = fpsEma * 0.92 + (1000 / Math.max(dt, 1)) * 0.08;
|
||||
const t = nowMs * 0.001;
|
||||
const delta = clock.getDelta();
|
||||
|
||||
if (mixer) mixer.update(delta);
|
||||
floorMat.uniforms.time.value = t;
|
||||
filmShader.uniforms.time.value = t;
|
||||
|
||||
// get model center for CSI / ping targeting
|
||||
const center = new THREE.Vector3();
|
||||
if (model) {
|
||||
const box = new THREE.Box3().setFromObject(model);
|
||||
box.getCenter(center);
|
||||
} else center.set(0, 0.9, 0);
|
||||
|
||||
tickCsi(t, center);
|
||||
updateNodes();
|
||||
maybeEmitPings(t, center);
|
||||
updatePings(t);
|
||||
updateTomography(t);
|
||||
updateBbox();
|
||||
updateFaceCloud(t);
|
||||
|
||||
controls.update();
|
||||
composer.render();
|
||||
|
||||
updateHud(t, fpsEma);
|
||||
requestAnimationFrame(tick);
|
||||
}
|
||||
requestAnimationFrame(tick);
|
||||
|
||||
window.addEventListener('resize', () => {
|
||||
camera.aspect = window.innerWidth / window.innerHeight;
|
||||
camera.updateProjectionMatrix();
|
||||
renderer.setSize(window.innerWidth, window.innerHeight);
|
||||
composer.setSize(window.innerWidth, window.innerHeight);
|
||||
bloom.setSize(window.innerWidth, window.innerHeight);
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,961 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>RuView · Skinned (FBX) · Mixamo X Bot in the ADR-097 helpers scene</title>
|
||||
<link rel="icon" type="image/svg+xml" href="data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'><circle cx='16' cy='16' r='10' fill='%23e8a634'/></svg>">
|
||||
<style>
|
||||
:root {
|
||||
--bg: #050507; --bg-panel: rgba(8,10,14,0.78);
|
||||
--amber: #ffb840; --amber-hot: #ffe09f;
|
||||
--cyan: #4cf; --magenta: #ff4cc8;
|
||||
--text: #d8c69a; --text-mute: #6b6155;
|
||||
--border: rgba(255,184,64,0.18);
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0; background: var(--bg); color: var(--text); overflow: hidden;
|
||||
font-family: 'SF Mono', 'Cascadia Code', Consolas, monospace;
|
||||
-webkit-font-smoothing: antialiased; font-size: 12px;
|
||||
}
|
||||
canvas { display: block; }
|
||||
.overlay-frame {
|
||||
position: fixed; inset: 0; pointer-events: none; z-index: 5;
|
||||
background:
|
||||
radial-gradient(ellipse at center, transparent 55%, rgba(0,0,0,0.55) 100%),
|
||||
linear-gradient(180deg, rgba(0,0,0,0.32) 0%, transparent 18%, transparent 82%, rgba(0,0,0,0.38) 100%);
|
||||
}
|
||||
.scanlines {
|
||||
position: fixed; inset: 0; pointer-events: none; z-index: 6;
|
||||
background: repeating-linear-gradient(0deg, rgba(0,0,0,0.04) 0px, rgba(0,0,0,0.04) 1px, transparent 1px, transparent 3px);
|
||||
mix-blend-mode: overlay; opacity: 0.5;
|
||||
}
|
||||
.panel {
|
||||
position: absolute; background: var(--bg-panel); border: 1px solid var(--border);
|
||||
border-radius: 4px; padding: 12px 14px; backdrop-filter: blur(8px);
|
||||
box-shadow: 0 1px 0 rgba(255,184,64,0.04), 0 8px 32px rgba(0,0,0,0.55); z-index: 10;
|
||||
}
|
||||
.panel h2 {
|
||||
margin: 0 0 8px 0; font-size: 10px; text-transform: uppercase; letter-spacing: 2px;
|
||||
color: var(--amber); font-weight: 600; border-bottom: 1px solid var(--border); padding-bottom: 6px;
|
||||
}
|
||||
#info { top: 20px; left: 20px; min-width: 280px; }
|
||||
#info h1 { margin: 0 0 1px 0; font-size: 13px; letter-spacing: 1px; color: var(--amber-hot); font-weight: 600; }
|
||||
#info .sub { font-size: 10px; color: var(--text-mute); letter-spacing: 0.5px; margin-bottom: 10px; padding-bottom: 8px; border-bottom: 1px solid var(--border); }
|
||||
#info .row { display: flex; justify-content: space-between; gap: 12px; padding: 2px 0; }
|
||||
#info .row .k { color: var(--text-mute); font-size: 11px; }
|
||||
#info .row .v { color: var(--text); font-variant-numeric: tabular-nums; font-size: 11px; }
|
||||
#info .row .v.amber { color: var(--amber); }
|
||||
#info .row .v.cyan { color: var(--cyan); }
|
||||
#info .row .v.mag { color: var(--magenta); }
|
||||
|
||||
#anim {
|
||||
position: absolute; bottom: 20px; left: 20px; min-width: 280px;
|
||||
background: var(--bg-panel); border: 1px solid var(--border); border-radius: 4px;
|
||||
padding: 12px 14px; backdrop-filter: blur(8px); z-index: 10;
|
||||
}
|
||||
#anim h2 { margin: 0 0 8px 0; font-size: 10px; text-transform: uppercase; letter-spacing: 2px;
|
||||
color: var(--amber); font-weight: 600; border-bottom: 1px solid var(--border); padding-bottom: 6px; }
|
||||
#anim .row { padding: 6px 0; font-size: 10px; }
|
||||
#anim .row .label { color: var(--text-mute); margin-right: 8px; }
|
||||
#anim button {
|
||||
background: rgba(255,184,64,0.06); border: 1px solid rgba(255,184,64,0.18);
|
||||
color: var(--text); font-family: inherit; font-size: 10px; padding: 4px 8px;
|
||||
margin: 2px 4px 2px 0; cursor: pointer; border-radius: 3px; letter-spacing: 0.5px;
|
||||
}
|
||||
#anim button:hover { background: rgba(255,184,64,0.14); color: var(--amber-hot); }
|
||||
#anim button.active { background: var(--amber); color: var(--bg); border-color: var(--amber); font-weight: 600; }
|
||||
#anim .slider-row { display: flex; align-items: center; gap: 8px; padding: 3px 0; font-size: 10px; margin-top: 6px; border-top: 1px solid rgba(255,184,64,0.08); padding-top: 8px; }
|
||||
#anim .slider-row .label { width: 90px; }
|
||||
#anim .slider-row input[type=range] { flex: 1; accent-color: var(--amber); }
|
||||
#anim .slider-row .val { width: 38px; text-align: right; color: var(--amber); font-variant-numeric: tabular-nums; }
|
||||
#anim .empty-hint {
|
||||
font-size: 10px; color: var(--text-mute); line-height: 1.5; margin-top: 4px;
|
||||
padding: 8px; background: rgba(255,184,64,0.04); border-radius: 3px;
|
||||
border-left: 2px solid var(--amber);
|
||||
}
|
||||
#anim .empty-hint a { color: var(--amber); text-decoration: none; }
|
||||
#anim .empty-hint a:hover { color: var(--amber-hot); text-decoration: underline; }
|
||||
|
||||
#helpers {
|
||||
position: absolute; bottom: 20px; right: 20px; min-width: 220px;
|
||||
background: var(--bg-panel); border: 1px solid var(--border); border-radius: 4px;
|
||||
padding: 12px 14px; backdrop-filter: blur(8px); z-index: 10;
|
||||
}
|
||||
#helpers h2 { margin: 0 0 8px 0; font-size: 10px; text-transform: uppercase; letter-spacing: 2px;
|
||||
color: var(--amber); font-weight: 600; border-bottom: 1px solid var(--border); padding-bottom: 6px; }
|
||||
#helpers label {
|
||||
display: flex; align-items: center; gap: 10px; padding: 3px 0; cursor: pointer; user-select: none; font-size: 11px;
|
||||
}
|
||||
#helpers label:hover { color: var(--amber-hot); }
|
||||
#helpers input[type=checkbox] { accent-color: var(--amber); width: 13px; height: 13px; cursor: pointer; }
|
||||
#helpers .swatch { width: 8px; height: 8px; border-radius: 50%; margin-left: auto; box-shadow: 0 0 6px currentColor; }
|
||||
|
||||
#csi { top: 20px; right: 20px; min-width: 260px; }
|
||||
#csi .bar-row { display: flex; align-items: center; gap: 8px; padding: 3px 0; font-size: 10px; }
|
||||
#csi .bar-row .label { width: 42px; color: var(--text-mute); }
|
||||
#csi .bar-row .bar-track { flex: 1; height: 6px; background: rgba(255,184,64,0.08); border-radius: 2px; overflow: hidden; }
|
||||
#csi .bar-row .bar-fill {
|
||||
height: 100%; background: linear-gradient(90deg, var(--amber-hot), var(--amber));
|
||||
box-shadow: 0 0 6px var(--amber); transition: width 0.08s linear;
|
||||
}
|
||||
#csi .bar-row .val { width: 36px; text-align: right; color: var(--amber); font-variant-numeric: tabular-nums; }
|
||||
|
||||
#loading {
|
||||
position: absolute; inset: 0; display: flex; align-items: center; justify-content: center;
|
||||
background: rgba(5,5,7,0.96); z-index: 20; font-size: 13px; color: var(--amber);
|
||||
letter-spacing: 2px; text-transform: uppercase;
|
||||
}
|
||||
#loading.hidden { display: none; }
|
||||
#loading .text { text-shadow: 0 0 12px var(--amber); animation: loadPulse 1.4s ease-in-out infinite; }
|
||||
@keyframes loadPulse { 0%,100% { opacity: 0.4; } 50% { opacity: 1.0; } }
|
||||
|
||||
@keyframes scanFlash { 0% { opacity: 0; } 10% { opacity: 0.12; } 100% { opacity: 0; } }
|
||||
.scan-flash {
|
||||
position: fixed; inset: 0;
|
||||
background: linear-gradient(90deg, transparent, var(--magenta), transparent);
|
||||
mix-blend-mode: screen; pointer-events: none; opacity: 0; z-index: 4;
|
||||
}
|
||||
|
||||
#titlecard {
|
||||
position: absolute; bottom: 76px; left: 50%; transform: translateX(-50%);
|
||||
text-align: center; color: var(--amber-hot); letter-spacing: 6px; font-size: 11px;
|
||||
text-transform: uppercase; opacity: 0.35; z-index: 10;
|
||||
text-shadow: 0 0 12px var(--amber); pointer-events: none;
|
||||
}
|
||||
#titlecard .sub { font-size: 9px; color: var(--text-mute); letter-spacing: 4px; margin-top: 4px; }
|
||||
</style>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/controls/OrbitControls.js"></script>
|
||||
<script src="https://unpkg.com/fflate@0.7.4/umd/index.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/curves/NURBSCurve.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/loaders/FBXLoader.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/postprocessing/EffectComposer.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/postprocessing/RenderPass.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/postprocessing/ShaderPass.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/postprocessing/UnrealBloomPass.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/shaders/CopyShader.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/shaders/LuminosityHighPassShader.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<div class="overlay-frame"></div>
|
||||
<div class="scanlines"></div>
|
||||
<div class="scan-flash" id="scan-flash"></div>
|
||||
<div id="loading"><div class="text">▸ Loading skinned subject · X Bot.fbx</div></div>
|
||||
|
||||
<div class="panel" id="info">
|
||||
<h1>RuView · Skinned (FBX)</h1>
|
||||
<div class="sub">ADR-097 · Mixamo X Bot · loaded via FBXLoader</div>
|
||||
<div class="row"><span class="k">Subject</span><span class="v amber">● Tracked</span></div>
|
||||
<div class="row"><span class="k">Source</span><span class="v" id="src-name">X Bot.fbx</span></div>
|
||||
<div class="row"><span class="k">Format</span><span class="v">FBX 7700 · 1.75 MB</span></div>
|
||||
<div class="row"><span class="k">Bones</span><span class="v" id="bone-count">—</span></div>
|
||||
<div class="row"><span class="k">Animation</span><span class="v amber" id="anim-name">—</span></div>
|
||||
<div class="row"><span class="k">Mesh nodes</span><span class="v cyan">4 · multistatic</span></div>
|
||||
<div class="row"><span class="k">Coherence</span><span class="v" id="coh-val">— %</span></div>
|
||||
<div class="row"><span class="k">Heart rate</span><span class="v amber" id="hr-val">— bpm</span></div>
|
||||
<div class="row"><span class="k">Bbox vol</span><span class="v" id="bbox-vol">— m³</span></div>
|
||||
<div class="row"><span class="k">Render</span><span class="v" id="fps-val">— fps</span></div>
|
||||
</div>
|
||||
|
||||
<div id="anim">
|
||||
<h2>AnimationMixer</h2>
|
||||
<div class="row">
|
||||
<span class="label">clips</span>
|
||||
<span id="clip-buttons"></span>
|
||||
</div>
|
||||
<div class="slider-row">
|
||||
<span class="label">time scale</span>
|
||||
<input type="range" id="time-scale" min="0.1" max="2" step="0.05" value="1.0">
|
||||
<span class="val" id="time-scale-val">1.00</span>
|
||||
</div>
|
||||
<div class="empty-hint" id="empty-hint" style="display:none;">
|
||||
<strong>No animations in this FBX.</strong><br>
|
||||
Mixamo's "T-Pose / Without Skin" export rigs the model but has no clips.
|
||||
Re-download with <em>"Original Pose"</em> + an animation selected
|
||||
(e.g. <a href="https://www.mixamo.com/#/?page=1&query=walking&type=Motion%2CMotionPack" target="_blank" rel="noopener">Walking</a>) to get a clip, or drop another FBX with anim and reload.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel" id="csi">
|
||||
<h2>Per-node CSI</h2>
|
||||
<div class="bar-row"><span class="label">N1·BL</span><div class="bar-track"><div class="bar-fill" id="bar-0"></div></div><span class="val" id="val-0">—</span></div>
|
||||
<div class="bar-row"><span class="label">N2·BR</span><div class="bar-track"><div class="bar-fill" id="bar-1"></div></div><span class="val" id="val-1">—</span></div>
|
||||
<div class="bar-row"><span class="label">N3·FL</span><div class="bar-track"><div class="bar-fill" id="bar-2"></div></div><span class="val" id="val-2">—</span></div>
|
||||
<div class="bar-row"><span class="label">N4·FR</span><div class="bar-track"><div class="bar-fill" id="bar-3"></div></div><span class="val" id="val-3">—</span></div>
|
||||
</div>
|
||||
|
||||
<div id="helpers">
|
||||
<h2>ADR-097 helpers</h2>
|
||||
<label><input type="checkbox" id="t-grid" checked>GridHelper<span class="swatch" style="color:#666"></span></label>
|
||||
<label><input type="checkbox" id="t-polar" checked>PolarGridHelper<span class="swatch" style="color:#ffb840"></span></label>
|
||||
<label><input type="checkbox" id="t-bbox" checked>BoxHelper on mesh<span class="swatch" style="color:#ffe09f"></span></label>
|
||||
<label><input type="checkbox" id="t-skel">SkeletonHelper<span class="swatch" style="color:#4cf"></span></label>
|
||||
<label><input type="checkbox" id="t-nodebox" checked>Per-node BoxHelpers<span class="swatch" style="color:#4cf"></span></label>
|
||||
<label><input type="checkbox" id="t-pings" checked>Sonar pings<span class="swatch" style="color:#4cf"></span></label>
|
||||
<label><input type="checkbox" id="t-tomo" checked>Tomography sweep<span class="swatch" style="color:#ff4cc8"></span></label>
|
||||
<label><input type="checkbox" id="t-rays" checked>RF illumination cones<span class="swatch" style="color:#ffb840"></span></label>
|
||||
</div>
|
||||
|
||||
<div id="titlecard">
|
||||
RuView · Seldon Vault
|
||||
<div class="sub">FBXLoader · Mixamo · ADR-097</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// =====================================================================
|
||||
// RuView · Skinned (FBX) · Mixamo X Bot loaded via FBXLoader
|
||||
// --------------------------------------------------------------------
|
||||
// Sibling of helpers-skinned.html that loads a local .fbx file
|
||||
// rather than the canonical GLB. Same cinematic atmosphere
|
||||
// (UnrealBloomPass, sonar pings, tomography sweep, pseudo-CSI),
|
||||
// same ADR-097 helpers wrapping the rigged mesh.
|
||||
//
|
||||
// Mixamo FBX caveats handled here:
|
||||
// 1. Mixamo exports in cm (100 = 1 m). We auto-detect by the
|
||||
// loaded model's bbox height and rescale to ~1.7 m human size.
|
||||
// 2. PhongMaterial → StandardMaterial swap for cleaner shading
|
||||
// under our amber key + cyan rim lights.
|
||||
// 3. Bone name probing for the head (Mixamo: "mixamorigHead",
|
||||
// legacy: "Bip01_Head", or any bone with /head/i match).
|
||||
// 4. Graceful no-animations case — many Mixamo exports are
|
||||
// rig-only.
|
||||
// =====================================================================
|
||||
|
||||
const MODEL_URL = '../assets/X%20Bot.fbx';
|
||||
|
||||
const NODE_POSITIONS = [
|
||||
[-1.9, 1.3, 1.9],[ 1.9, 1.3, 1.9],
|
||||
[-1.9, 1.3, -1.9],[ 1.9, 1.3, -1.9],
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Scene / camera / renderer
|
||||
// ---------------------------------------------------------------------
|
||||
const scene = new THREE.Scene();
|
||||
scene.background = new THREE.Color(0x050507);
|
||||
scene.fog = new THREE.FogExp2(0x050507, 0.06);
|
||||
|
||||
const camera = new THREE.PerspectiveCamera(48, window.innerWidth/window.innerHeight, 0.05, 100);
|
||||
camera.position.set(3.2, 1.55, 4.0);
|
||||
|
||||
const renderer = new THREE.WebGLRenderer({ antialias: true, powerPreference: 'high-performance' });
|
||||
renderer.setPixelRatio(Math.min(2, window.devicePixelRatio));
|
||||
renderer.setSize(window.innerWidth, window.innerHeight);
|
||||
renderer.toneMapping = THREE.ACESFilmicToneMapping;
|
||||
renderer.toneMappingExposure = 0.80;
|
||||
renderer.outputEncoding = THREE.sRGBEncoding;
|
||||
renderer.shadowMap.enabled = true;
|
||||
renderer.shadowMap.type = THREE.PCFSoftShadowMap;
|
||||
document.body.appendChild(renderer.domElement);
|
||||
|
||||
const controls = new THREE.OrbitControls(camera, renderer.domElement);
|
||||
controls.target.set(0, 0.9, 0);
|
||||
controls.enableDamping = true; controls.dampingFactor = 0.06;
|
||||
controls.minDistance = 2; controls.maxDistance = 12;
|
||||
controls.maxPolarAngle = Math.PI * 0.92;
|
||||
controls.autoRotate = new URLSearchParams(location.search).get('orbit') === '1';
|
||||
controls.autoRotateSpeed = 0.25;
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Lights — amber key + cyan rim from each ESP32 direction
|
||||
// ---------------------------------------------------------------------
|
||||
scene.add(new THREE.HemisphereLight(0x553a18, 0x080606, 0.7));
|
||||
|
||||
const keyLight = new THREE.DirectionalLight(0xffc070, 1.05);
|
||||
keyLight.position.set(2.5, 3.8, 2.5);
|
||||
keyLight.castShadow = true;
|
||||
keyLight.shadow.camera.top = 2; keyLight.shadow.camera.bottom = -2;
|
||||
keyLight.shadow.camera.left = -2; keyLight.shadow.camera.right = 2;
|
||||
keyLight.shadow.camera.near = 0.1; keyLight.shadow.camera.far = 12;
|
||||
keyLight.shadow.mapSize.set(1024, 1024);
|
||||
keyLight.shadow.bias = -0.0008;
|
||||
scene.add(keyLight);
|
||||
|
||||
const rimLights = [];
|
||||
NODE_POSITIONS.forEach(pos => {
|
||||
const rim = new THREE.PointLight(0x4cf, 0.55, 8, 1.8);
|
||||
rim.position.set(pos[0] * 1.1, pos[1] * 0.7, pos[2] * 1.1);
|
||||
scene.add(rim); rimLights.push(rim);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Post-processing
|
||||
// ---------------------------------------------------------------------
|
||||
const composer = new THREE.EffectComposer(renderer);
|
||||
composer.addPass(new THREE.RenderPass(scene, camera));
|
||||
const bloom = new THREE.UnrealBloomPass(
|
||||
new THREE.Vector2(window.innerWidth, window.innerHeight),
|
||||
0.45, 0.40, 0.78,
|
||||
);
|
||||
composer.addPass(bloom);
|
||||
const filmShader = {
|
||||
uniforms: { tDiffuse: { value: null }, time: { value: 0 }, grain: { value: 0.04 },
|
||||
vignette: { value: 0.32 }, aberration: { value: 0.0018 } },
|
||||
vertexShader: `varying vec2 vUv; void main() { vUv = uv; gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0); }`,
|
||||
fragmentShader: `
|
||||
uniform sampler2D tDiffuse; uniform float time, grain, vignette, aberration;
|
||||
varying vec2 vUv;
|
||||
float hash(vec2 p) { return fract(sin(dot(p, vec2(12.9898, 78.233))) * 43758.5453); }
|
||||
void main() {
|
||||
vec2 off = (vUv - 0.5) * aberration;
|
||||
float r = texture2D(tDiffuse, vUv + off).r;
|
||||
float g = texture2D(tDiffuse, vUv).g;
|
||||
float b = texture2D(tDiffuse, vUv - off).b;
|
||||
vec3 col = vec3(r, g, b);
|
||||
col += (hash(vUv * 1024.0 + time) - 0.5) * grain;
|
||||
float v = smoothstep(0.85, 0.20, length(vUv - 0.5));
|
||||
col *= mix(1.0 - vignette, 1.0, v);
|
||||
gl_FragColor = vec4(col, 1.0);
|
||||
}`,
|
||||
};
|
||||
const filmPass = new THREE.ShaderPass(filmShader);
|
||||
composer.addPass(filmPass);
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Floor (same procedural shader as cinematic / skinned-glb)
|
||||
// ---------------------------------------------------------------------
|
||||
const floorMat = new THREE.ShaderMaterial({
|
||||
uniforms: { time: { value: 0 }, baseColor: { value: new THREE.Color(0xffb840) } },
|
||||
vertexShader: `varying vec3 vPos; void main() { vPos = position; gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0); }`,
|
||||
fragmentShader: `
|
||||
uniform float time; uniform vec3 baseColor; varying vec3 vPos;
|
||||
void main() {
|
||||
vec2 g = abs(fract(vPos.xz * 0.5) - 0.5);
|
||||
float line = smoothstep(0.48, 0.50, max(g.x, g.y));
|
||||
float majorLine = smoothstep(0.96, 1.00, max(g.x, g.y) * 2.0);
|
||||
float scan = 0.5 + 0.5 * sin((vPos.x + vPos.z) * 2.0 - time * 1.4);
|
||||
scan = pow(scan, 14.0);
|
||||
float falloff = smoothstep(5.0, 1.2, length(vPos.xz));
|
||||
vec3 col = baseColor * (0.01 + 0.05 * line + 0.16 * majorLine + 0.08 * scan);
|
||||
gl_FragColor = vec4(col * falloff, falloff * 0.55);
|
||||
}`,
|
||||
transparent: true, depthWrite: false,
|
||||
});
|
||||
const floor = new THREE.Mesh(new THREE.PlaneGeometry(20, 20), floorMat);
|
||||
floor.rotation.x = -Math.PI / 2;
|
||||
scene.add(floor);
|
||||
|
||||
const shadowGround = new THREE.Mesh(
|
||||
new THREE.PlaneGeometry(20, 20),
|
||||
new THREE.ShadowMaterial({ opacity: 0.55 })
|
||||
);
|
||||
shadowGround.rotation.x = -Math.PI / 2;
|
||||
shadowGround.position.y = 0.001;
|
||||
shadowGround.receiveShadow = true;
|
||||
scene.add(shadowGround);
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// ADR-097 helpers + sensor nodes (same as helpers-skinned.html)
|
||||
// ---------------------------------------------------------------------
|
||||
const gridHelper = new THREE.GridHelper(4, 20, 0x554a32, 0x2a2418);
|
||||
gridHelper.material.transparent = true; gridHelper.material.opacity = 0.45;
|
||||
scene.add(gridHelper);
|
||||
|
||||
const polarHelper = new THREE.PolarGridHelper(2.2, 16, 4, 64, 0xffb840, 0x4a3a1a);
|
||||
polarHelper.position.y = 0.002;
|
||||
polarHelper.material.transparent = true; polarHelper.material.opacity = 0.55;
|
||||
scene.add(polarHelper);
|
||||
|
||||
let bboxHelper = null;
|
||||
let skeletonHelper = null;
|
||||
|
||||
const nodeBboxHelpers = [];
|
||||
const nodeRings = [];
|
||||
const nodeAnchors = [];
|
||||
NODE_POSITIONS.forEach((pos, i) => {
|
||||
const group = new THREE.Group();
|
||||
group.position.set(pos[0], pos[1], pos[2]);
|
||||
|
||||
group.add(new THREE.Mesh(
|
||||
new THREE.BoxGeometry(0.14, 0.06, 0.20),
|
||||
new THREE.MeshBasicMaterial({ color: 0xffb840 })
|
||||
));
|
||||
const antenna = new THREE.Mesh(
|
||||
new THREE.ConeGeometry(0.018, 0.10, 8),
|
||||
new THREE.MeshBasicMaterial({ color: 0xffe09f })
|
||||
);
|
||||
antenna.position.y = 0.08; group.add(antenna);
|
||||
|
||||
const ring = new THREE.Mesh(
|
||||
new THREE.RingGeometry(0.11, 0.14, 32),
|
||||
new THREE.MeshBasicMaterial({ color: 0xffb840, side: THREE.DoubleSide,
|
||||
transparent: true, opacity: 0.55, blending: THREE.AdditiveBlending, depthWrite: false })
|
||||
);
|
||||
ring.rotation.x = -Math.PI / 2; ring.position.y = -0.05;
|
||||
group.add(ring); nodeRings.push(ring);
|
||||
|
||||
const core = new THREE.Mesh(
|
||||
new THREE.SphereGeometry(0.025, 12, 12),
|
||||
new THREE.MeshBasicMaterial({ color: 0xffe09f })
|
||||
);
|
||||
core.position.y = 0.04; group.add(core);
|
||||
|
||||
scene.add(group); nodeAnchors.push(group);
|
||||
|
||||
const bbox = new THREE.BoxHelper(group, 0x4cf);
|
||||
bbox.material.transparent = true; bbox.material.opacity = 0.45;
|
||||
scene.add(bbox); nodeBboxHelpers.push(bbox);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// God-ray cones — one per node, pointed at the subject. Visualizes
|
||||
// "the four ESP32s are jointly illuminating the body with RF". Each
|
||||
// cone has a volumetric-feeling gradient shader and is opacity-
|
||||
// modulated by that node's csiAmp × csiCoherence (so when a node's
|
||||
// signal degrades, its ray dims).
|
||||
// ---------------------------------------------------------------------
|
||||
const godRayMat = (color, idx) => new THREE.ShaderMaterial({
|
||||
uniforms: {
|
||||
time: { value: 0 },
|
||||
intensity: { value: 0.0 },
|
||||
color: { value: new THREE.Color(color) },
|
||||
seed: { value: idx * 17.3 },
|
||||
},
|
||||
vertexShader: `
|
||||
varying vec2 vUv;
|
||||
varying float vY;
|
||||
void main() {
|
||||
vUv = uv;
|
||||
vY = position.y;
|
||||
gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
|
||||
}`,
|
||||
fragmentShader: `
|
||||
uniform float time, intensity, seed;
|
||||
uniform vec3 color;
|
||||
varying vec2 vUv;
|
||||
varying float vY;
|
||||
float hash(vec2 p) { return fract(sin(dot(p, vec2(12.9898, 78.233))) * 43758.5453); }
|
||||
void main() {
|
||||
// along the cone (uv.y goes 0=tip → 1=base), fade out at the base
|
||||
float edgeFade = smoothstep(0.0, 0.18, vUv.y) * smoothstep(1.0, 0.65, vUv.y);
|
||||
// soft radial falloff (cone-edge transparency)
|
||||
float radial = sin(vUv.y * 3.14159);
|
||||
radial = pow(radial, 2.0);
|
||||
// volumetric noise (slow scrolling)
|
||||
float n = hash(floor(vUv * vec2(40.0, 60.0)) + vec2(seed, time * 0.4));
|
||||
float scroll = 0.85 + 0.30 * sin(vUv.y * 32.0 - time * 1.4 + seed);
|
||||
float a = edgeFade * radial * scroll * (0.55 + 0.45 * n);
|
||||
gl_FragColor = vec4(color, a * intensity * 0.25);
|
||||
}`,
|
||||
transparent: true,
|
||||
blending: THREE.AdditiveBlending,
|
||||
depthWrite: false,
|
||||
side: THREE.DoubleSide,
|
||||
});
|
||||
const godRays = [];
|
||||
for (let i = 0; i < 4; i++) {
|
||||
// cone with apex at node, expanding toward the subject
|
||||
// height 4 m (more than enough to reach subject), radius 0.45 m at base
|
||||
const geom = new THREE.ConeGeometry(0.45, 4.0, 28, 1, true);
|
||||
// ConeGeometry tip is at +Y, base at -Y — rotate so tip is along -Y
|
||||
// (we'll later orient each cone so its tip touches the node).
|
||||
geom.translate(0, -2.0, 0); // shift so apex is at origin
|
||||
const mat = godRayMat(0xffb840, i);
|
||||
const cone = new THREE.Mesh(geom, mat);
|
||||
scene.add(cone);
|
||||
godRays.push({ mesh: cone, mat });
|
||||
}
|
||||
function updateGodRays(t) {
|
||||
if (!model) return;
|
||||
const want = document.getElementById('t-rays').checked;
|
||||
const center = new THREE.Vector3();
|
||||
const box = new THREE.Box3().setFromObject(model);
|
||||
box.getCenter(center);
|
||||
for (let i = 0; i < 4; i++) {
|
||||
godRays[i].mesh.visible = want;
|
||||
if (!want) continue;
|
||||
const node = nodeAnchors[i];
|
||||
const np = node.getWorldPosition(new THREE.Vector3());
|
||||
const dir = new THREE.Vector3().subVectors(center, np);
|
||||
const len = dir.length();
|
||||
dir.normalize();
|
||||
const ray = godRays[i];
|
||||
ray.mesh.position.copy(np);
|
||||
// align cone's -Y axis (apex direction after the geometry shift)
|
||||
// to point along `dir`
|
||||
ray.mesh.quaternion.setFromUnitVectors(new THREE.Vector3(0, -1, 0), dir);
|
||||
// stretch cone length to actual distance, keep base width reasonable
|
||||
ray.mesh.scale.set(1, len / 4.0, 1);
|
||||
ray.mat.uniforms.time.value = t;
|
||||
// intensity follows that node's CSI amplitude * global coherence
|
||||
const target = csiAmp[i] * csiCoherence * 1.4;
|
||||
ray.mat.uniforms.intensity.value =
|
||||
ray.mat.uniforms.intensity.value * 0.85 + target * 0.15;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// FBX load with Mixamo-aware fixups
|
||||
// ---------------------------------------------------------------------
|
||||
let model = null;
|
||||
let mixer = null;
|
||||
let headBone = null;
|
||||
let boneCount = 0;
|
||||
const clipActions = {}; // by clip name
|
||||
let currentClip = null;
|
||||
|
||||
const loader = new THREE.FBXLoader();
|
||||
loader.load(MODEL_URL, (object) => {
|
||||
model = object;
|
||||
|
||||
// 1. Scale fix — Mixamo defaults to cm; detect by bbox height and
|
||||
// rescale so the rig reads as ~1.7 m human size.
|
||||
const raw = new THREE.Box3().setFromObject(model);
|
||||
const height = raw.max.y - raw.min.y;
|
||||
if (height > 10) {
|
||||
model.scale.setScalar(1 / 100); // cm → m
|
||||
} else if (height > 5) {
|
||||
model.scale.setScalar(1 / 50); // catch in-between rigs
|
||||
}
|
||||
// recenter on origin at floor
|
||||
const b2 = new THREE.Box3().setFromObject(model);
|
||||
model.position.y -= b2.min.y;
|
||||
|
||||
// 2. Material upgrade + shadow casting + head/bone scan
|
||||
model.traverse((obj) => {
|
||||
if (obj.isMesh) {
|
||||
obj.castShadow = true;
|
||||
obj.receiveShadow = true;
|
||||
// Phong → Standard for cleaner shading under our PBR lights.
|
||||
// Keep diffuse map + skinning intact.
|
||||
if (obj.material && obj.material.isMeshPhongMaterial) {
|
||||
const m = obj.material;
|
||||
const upgraded = new THREE.MeshStandardMaterial({
|
||||
map: m.map, normalMap: m.normalMap, color: m.color,
|
||||
skinning: !!obj.isSkinnedMesh,
|
||||
metalness: 0.0, roughness: 0.85,
|
||||
});
|
||||
obj.material = upgraded;
|
||||
}
|
||||
}
|
||||
if (obj.isBone) {
|
||||
boneCount++;
|
||||
if (!headBone && /head/i.test(obj.name)) headBone = obj;
|
||||
}
|
||||
});
|
||||
document.getElementById('bone-count').textContent = boneCount;
|
||||
|
||||
scene.add(model);
|
||||
|
||||
skeletonHelper = new THREE.SkeletonHelper(model);
|
||||
skeletonHelper.visible = false;
|
||||
scene.add(skeletonHelper);
|
||||
|
||||
// 3. Animations — Mixamo exports one clip per FBX (sometimes none)
|
||||
const clips = object.animations || [];
|
||||
if (clips.length === 0) {
|
||||
document.getElementById('anim-name').textContent = 'none (rig-only)';
|
||||
document.getElementById('empty-hint').style.display = 'block';
|
||||
} else {
|
||||
mixer = new THREE.AnimationMixer(model);
|
||||
const btnHost = document.getElementById('clip-buttons');
|
||||
for (const clip of clips) {
|
||||
const action = mixer.clipAction(clip);
|
||||
clipActions[clip.name] = action;
|
||||
const btn = document.createElement('button');
|
||||
btn.textContent = clip.name || 'clip-' + Object.keys(clipActions).length;
|
||||
btn.addEventListener('click', () => playClip(clip.name));
|
||||
btnHost.appendChild(btn);
|
||||
}
|
||||
playClip(clips[0].name);
|
||||
}
|
||||
|
||||
// 4. Face point cloud
|
||||
if (headBone) buildFacePointCloud();
|
||||
|
||||
document.getElementById('loading').classList.add('hidden');
|
||||
}, (xhr) => {
|
||||
const total = xhr.total || 1750032;
|
||||
const pct = (xhr.loaded / total * 100).toFixed(0);
|
||||
const txt = document.querySelector('#loading .text');
|
||||
if (txt) txt.textContent = `▸ Loading skinned subject · X Bot.fbx · ${pct} %`;
|
||||
}, (err) => {
|
||||
console.error('FBX load failed', err);
|
||||
const txt = document.querySelector('#loading .text');
|
||||
if (txt) txt.textContent = '⚠ Load failed — see console';
|
||||
});
|
||||
|
||||
function playClip(name) {
|
||||
for (const k in clipActions) {
|
||||
const a = clipActions[k];
|
||||
if (k === name) {
|
||||
a.reset(); a.play(); currentClip = name;
|
||||
document.getElementById('anim-name').textContent = name;
|
||||
for (const btn of document.querySelectorAll('#anim button[data-base], #anim button')) {
|
||||
if (btn.dataset.base !== undefined || !btn.textContent) continue;
|
||||
btn.classList.toggle('active', btn.textContent === name);
|
||||
}
|
||||
} else a.stop();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Face point cloud — anchored to head bone, same shimmer shader
|
||||
// ---------------------------------------------------------------------
|
||||
const FACE_POINTS = 220; // fewer points so each dot is visible as a tracked landmark
|
||||
const facePositions = new Float32Array(FACE_POINTS * 3);
|
||||
const faceOffsets = new Float32Array(FACE_POINTS * 3);
|
||||
const facePhases = new Float32Array(FACE_POINTS);
|
||||
let facePoints = null;
|
||||
function buildFacePointCloud() {
|
||||
// Front-hemisphere only — points scattered on the +Z half of an
|
||||
// ellipsoid so the cloud reads as a FACE projection forward from
|
||||
// the head bone, not a halo wrapping the skull. Local coords:
|
||||
// +Z = forward (face direction), +Y = up, +X = right.
|
||||
for (let i = 0; i < FACE_POINTS; i++) {
|
||||
// theta in [0, 2π) around the local Z axis, phi in [0, π/2]
|
||||
// (front hemisphere only — no points behind the head)
|
||||
const theta = Math.random() * Math.PI * 2;
|
||||
const phi = Math.acos(1 - Math.random() * 0.95); // dense near face front
|
||||
const sinPhi = Math.sin(phi), cosPhi = Math.cos(phi);
|
||||
// ellipsoid radii (taller than wide, slightly squashed F-B)
|
||||
const rx = 0.085, ry = 0.108, rz = 0.075;
|
||||
// local coords with +Z = face forward
|
||||
faceOffsets[i*3+0] = rx * sinPhi * Math.cos(theta);
|
||||
faceOffsets[i*3+1] = ry * sinPhi * Math.sin(theta) * 1.05; // taller
|
||||
faceOffsets[i*3+2] = rz * cosPhi; // forward
|
||||
facePhases[i] = Math.random() * Math.PI * 2;
|
||||
}
|
||||
const geom = new THREE.BufferGeometry();
|
||||
geom.setAttribute('position', new THREE.BufferAttribute(facePositions, 3));
|
||||
geom.setAttribute('aPhase', new THREE.BufferAttribute(facePhases, 1));
|
||||
const mat = new THREE.ShaderMaterial({
|
||||
uniforms: { time: { value: 0 } },
|
||||
vertexShader: `
|
||||
attribute float aPhase; uniform float time;
|
||||
varying float vAlpha;
|
||||
void main() {
|
||||
vec4 mv = modelViewMatrix * vec4(position, 1.0);
|
||||
// Slow per-point shimmer + occasional "scan-lit" spike
|
||||
// so the cloud reads as discrete tracked landmarks
|
||||
// rather than a fluffy halo.
|
||||
float shimmer = 0.5 + 0.5 * sin(time * 3.0 + aPhase);
|
||||
float spark = step(0.95, fract(sin(aPhase * 17.0 + time * 0.5) * 43.0));
|
||||
vAlpha = 0.10 + 0.25 * shimmer + 0.55 * spark;
|
||||
gl_Position = projectionMatrix * mv;
|
||||
// 6× smaller — tracked dots, not a cloud
|
||||
gl_PointSize = (1.0 + shimmer * 0.6 + spark * 1.5) * (32.0 / -mv.z);
|
||||
}`,
|
||||
fragmentShader: `
|
||||
varying float vAlpha;
|
||||
void main() {
|
||||
vec2 c = gl_PointCoord - 0.5;
|
||||
float d = length(c);
|
||||
if (d > 0.5) discard;
|
||||
float falloff = smoothstep(0.5, 0.0, d);
|
||||
vec3 col = vec3(0.40, 0.78, 1.00);
|
||||
gl_FragColor = vec4(col, vAlpha * falloff);
|
||||
}`,
|
||||
transparent: true, depthWrite: false,
|
||||
});
|
||||
facePoints = new THREE.Points(geom, mat);
|
||||
scene.add(facePoints);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Pings + tomography + CSI driver — copied wholesale from skinned-glb
|
||||
// ---------------------------------------------------------------------
|
||||
const PING_POOL = 24;
|
||||
const pings = [];
|
||||
const pingGeo = new THREE.TorusGeometry(1, 0.012, 8, 48);
|
||||
for (let i = 0; i < PING_POOL; i++) {
|
||||
const mat = new THREE.MeshBasicMaterial({ color: 0x4cf, transparent: true, opacity: 0, depthWrite: false });
|
||||
const mesh = new THREE.Mesh(pingGeo, mat); mesh.visible = false; scene.add(mesh);
|
||||
pings.push({ mesh, active: false, t0: 0, duration: 0,
|
||||
origin: new THREE.Vector3(), target: new THREE.Vector3() });
|
||||
}
|
||||
let pingIndex = 0;
|
||||
function emitPing(origin, target) {
|
||||
const p = pings[pingIndex]; pingIndex = (pingIndex + 1) % PING_POOL;
|
||||
p.active = true; p.t0 = performance.now() * 0.001;
|
||||
p.duration = 0.55 + Math.random() * 0.20;
|
||||
p.origin.copy(origin); p.target.copy(target);
|
||||
p.mesh.position.copy(origin); p.mesh.visible = true; p.mesh.material.opacity = 0;
|
||||
const dir = new THREE.Vector3().subVectors(target, origin).normalize();
|
||||
p.mesh.quaternion.setFromUnitVectors(new THREE.Vector3(0, 0, 1), dir);
|
||||
}
|
||||
|
||||
const tomoMat = new THREE.ShaderMaterial({
|
||||
uniforms: { time: { value: 0 }, intensity: { value: 0 } },
|
||||
vertexShader: `varying vec2 vUv; void main() { vUv = uv; gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0); }`,
|
||||
fragmentShader: `
|
||||
uniform float time, intensity; varying vec2 vUv;
|
||||
void main() {
|
||||
float band = exp(-pow((vUv.x - 0.5) * 14.0, 2.0));
|
||||
float lines = 0.5 + 0.5 * sin(vUv.y * 90.0 + time * 4.0);
|
||||
vec3 col = vec3(1.0, 0.3, 0.78) * band * (0.6 + 0.4 * lines);
|
||||
gl_FragColor = vec4(col, intensity * band * 0.75);
|
||||
}`,
|
||||
transparent: true, blending: THREE.AdditiveBlending, depthWrite: false, side: THREE.DoubleSide,
|
||||
});
|
||||
const tomoPlane = new THREE.Mesh(new THREE.PlaneGeometry(8, 6), tomoMat);
|
||||
tomoPlane.rotation.y = Math.PI / 2;
|
||||
tomoPlane.position.set(-2, 1.0, 0); tomoPlane.visible = false;
|
||||
scene.add(tomoPlane);
|
||||
let tomoActive = false, tomoT0 = 0, tomoNextAt = 4 + Math.random() * 4;
|
||||
|
||||
const csiAmp = [0, 0, 0, 0];
|
||||
let csiCoherence = 0.5;
|
||||
const csiNoise = [0, 0, 0, 0];
|
||||
function tickCsi(t, targetWorld) {
|
||||
for (let i = 0; i < 4; i++) csiNoise[i] = csiNoise[i] * 0.92 + (Math.random() - 0.5) * 0.08;
|
||||
let mean = 0; const amps = [];
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const np = NODE_POSITIONS[i];
|
||||
const dx = np[0] - targetWorld.x, dy = np[1] - targetWorld.y, dz = np[2] - targetWorld.z;
|
||||
const r2 = dx*dx + dy*dy + dz*dz;
|
||||
const fall = 1.0 / (1.0 + r2 * 0.18);
|
||||
const breath = Math.sin(t * 0.27 * Math.PI * 2) * 0.10;
|
||||
const heart = Math.sin(t * 1.18 * Math.PI * 2) * 0.04;
|
||||
const walk = Math.sin(t * 1.9 + i * 0.5) * 0.12;
|
||||
const a = Math.max(0, Math.min(1, fall + breath + heart + walk + csiNoise[i] * 0.30));
|
||||
amps.push(a);
|
||||
csiAmp[i] = csiAmp[i] * 0.7 + a * 0.3;
|
||||
mean += a;
|
||||
}
|
||||
mean /= 4;
|
||||
let v = 0; for (let i = 0; i < 4; i++) v += (amps[i] - mean) ** 2;
|
||||
v = Math.sqrt(v / 4);
|
||||
csiCoherence = csiCoherence * 0.85 + Math.max(0, Math.min(1, 1.0 - v * 2.5)) * 0.15;
|
||||
}
|
||||
|
||||
let lastPingT = [0, 0, 0, 0];
|
||||
// Subject hit-flash: when a sonar ping lands, briefly raise the
|
||||
// emissive on every mesh in the model. Decays each frame.
|
||||
let subjectFlash = 0;
|
||||
const modelMeshes = [];
|
||||
function collectModelMeshes() {
|
||||
if (!model || modelMeshes.length) return;
|
||||
model.traverse(o => {
|
||||
if (o.isMesh && o.material && o.material.isMeshStandardMaterial) {
|
||||
o.material.emissive = new THREE.Color(0xffb840);
|
||||
o.material.emissiveIntensity = 0;
|
||||
modelMeshes.push(o);
|
||||
}
|
||||
});
|
||||
}
|
||||
function updateSubjectFlash() {
|
||||
collectModelMeshes();
|
||||
subjectFlash *= 0.86;
|
||||
for (const m of modelMeshes) {
|
||||
m.material.emissiveIntensity = subjectFlash;
|
||||
}
|
||||
}
|
||||
|
||||
// Subtle root motion — even with a stationary Idle clip, give the
|
||||
// figure a gentle drift + look-around so it doesn't feel pinned.
|
||||
function updateRootMotion(t) {
|
||||
if (!model) return;
|
||||
model.position.x = Math.sin(t * 0.18) * 0.06;
|
||||
model.position.z = Math.cos(t * 0.13) * 0.05;
|
||||
model.rotation.y = Math.sin(t * 0.11) * 0.18;
|
||||
}
|
||||
|
||||
function updateNodes() {
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const ring = nodeRings[i];
|
||||
const amp = csiAmp[i];
|
||||
ring.material.opacity = 0.32 + 0.55 * amp;
|
||||
ring.scale.setScalar(1 + 0.30 * amp);
|
||||
rimLights[i].intensity = 0.30 + 0.60 * amp * csiCoherence;
|
||||
}
|
||||
}
|
||||
function maybeEmitPings(t, modelCenter) {
|
||||
if (!document.getElementById('t-pings').checked || !model) return;
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const interval = 1.2 / (0.25 + csiAmp[i]);
|
||||
if (t - lastPingT[i] > interval) {
|
||||
lastPingT[i] = t;
|
||||
const target = modelCenter.clone();
|
||||
target.y += (Math.random() - 0.3) * 0.8;
|
||||
target.x += (Math.random() - 0.5) * 0.2;
|
||||
const origin = nodeAnchors[i].getWorldPosition(new THREE.Vector3());
|
||||
emitPing(origin, target);
|
||||
}
|
||||
}
|
||||
}
|
||||
function updatePings(t) {
|
||||
for (const p of pings) {
|
||||
if (!p.active) continue;
|
||||
const u = (t - p.t0) / p.duration;
|
||||
if (u >= 1) {
|
||||
p.active = false; p.mesh.visible = false;
|
||||
// ping landed — flash the subject (drives emissiveIntensity)
|
||||
subjectFlash = Math.min(0.42, subjectFlash + 0.18);
|
||||
continue;
|
||||
}
|
||||
p.mesh.position.lerpVectors(p.origin, p.target, u);
|
||||
p.mesh.scale.setScalar(0.03 + u * 0.18);
|
||||
p.mesh.material.opacity = (1.0 - u) * 0.40 * csiCoherence;
|
||||
}
|
||||
}
|
||||
function updateTomography(t) {
|
||||
if (!document.getElementById('t-tomo').checked) { tomoActive = false; tomoPlane.visible = false; return; }
|
||||
if (!tomoActive && t > tomoNextAt) {
|
||||
tomoActive = true; tomoT0 = t; tomoPlane.visible = true;
|
||||
const sf = document.getElementById('scan-flash');
|
||||
sf.style.animation = 'none';
|
||||
requestAnimationFrame(() => { sf.style.animation = 'scanFlash 1.6s ease-out'; });
|
||||
}
|
||||
if (tomoActive) {
|
||||
const dur = 2.4;
|
||||
const e = (t - tomoT0) / dur;
|
||||
if (e >= 1) {
|
||||
tomoActive = false; tomoPlane.visible = false;
|
||||
tomoNextAt = t + 4 + Math.random() * 5;
|
||||
} else {
|
||||
tomoPlane.position.x = -3 + e * 6;
|
||||
tomoMat.uniforms.intensity.value = Math.sin(e * Math.PI);
|
||||
tomoMat.uniforms.time.value = t;
|
||||
}
|
||||
}
|
||||
}
|
||||
function updateBbox() {
|
||||
const want = document.getElementById('t-bbox').checked && model;
|
||||
if (!want) {
|
||||
if (bboxHelper) { scene.remove(bboxHelper); bboxHelper = null; }
|
||||
document.getElementById('bbox-vol').textContent = '—';
|
||||
return;
|
||||
}
|
||||
if (!bboxHelper) {
|
||||
bboxHelper = new THREE.BoxHelper(model, 0xffe09f);
|
||||
bboxHelper.material.transparent = true; bboxHelper.material.opacity = 0.55;
|
||||
scene.add(bboxHelper);
|
||||
} else bboxHelper.setFromObject(model);
|
||||
const box = new THREE.Box3().setFromObject(model);
|
||||
const size = box.getSize(new THREE.Vector3());
|
||||
document.getElementById('bbox-vol').textContent = (size.x * size.y * size.z).toFixed(3) + ' m³';
|
||||
}
|
||||
const tmpHeadPos = new THREE.Vector3();
|
||||
const tmpHeadQuat = new THREE.Quaternion();
|
||||
const tmpHeadScl = new THREE.Vector3();
|
||||
const tmpOffset = new THREE.Vector3();
|
||||
function updateFaceCloud(t) {
|
||||
if (!facePoints || !headBone) return;
|
||||
// Decompose the head bone's world matrix so we can apply its
|
||||
// orientation (face direction) to each local offset. This way
|
||||
// the cloud rotates with the head — turn left/right and the
|
||||
// face points stay in front of the face.
|
||||
headBone.updateMatrixWorld(true);
|
||||
headBone.matrixWorld.decompose(tmpHeadPos, tmpHeadQuat, tmpHeadScl);
|
||||
// Mixamo head bone forward is along +Y in some rigs (head looks up the
|
||||
// bone chain) — project the cloud along the model's actual forward
|
||||
// vector, which for Mixamo X Bot facing camera is world +Z.
|
||||
// Use the model's root rotation as the source of "forward".
|
||||
const forward = new THREE.Vector3(0, 0, 1);
|
||||
if (model) forward.applyQuaternion(model.getWorldQuaternion(new THREE.Quaternion()));
|
||||
const up = new THREE.Vector3(0, 1, 0);
|
||||
const right = new THREE.Vector3().crossVectors(up, forward).normalize();
|
||||
const facingUp = up.clone();
|
||||
// anchor the cloud just in front of the head
|
||||
const anchor = tmpHeadPos.clone().addScaledVector(forward, 0.04);
|
||||
anchor.y += 0.04; // nudge up so cloud sits over the face, not the chin
|
||||
const pos = facePoints.geometry.attributes.position;
|
||||
for (let i = 0; i < FACE_POINTS; i++) {
|
||||
const ox = faceOffsets[i*3+0];
|
||||
const oy = faceOffsets[i*3+1];
|
||||
const oz = faceOffsets[i*3+2];
|
||||
// map local (ox, oy, oz) into world via (right, up, forward)
|
||||
tmpOffset.copy(right).multiplyScalar(ox)
|
||||
.addScaledVector(facingUp, oy)
|
||||
.addScaledVector(forward, oz);
|
||||
pos.array[i*3+0] = anchor.x + tmpOffset.x;
|
||||
pos.array[i*3+1] = anchor.y + tmpOffset.y;
|
||||
pos.array[i*3+2] = anchor.z + tmpOffset.z;
|
||||
}
|
||||
pos.needsUpdate = true;
|
||||
facePoints.material.uniforms.time.value = t;
|
||||
}
|
||||
|
||||
let hudT = 0;
|
||||
function updateHud(t, fps) {
|
||||
if (t - hudT < 0.1) return;
|
||||
hudT = t;
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const pct = Math.round(csiAmp[i] * 100);
|
||||
document.getElementById('bar-' + i).style.width = pct + '%';
|
||||
document.getElementById('val-' + i).textContent = pct + '%';
|
||||
}
|
||||
document.getElementById('coh-val').textContent = (csiCoherence * 100).toFixed(0) + ' %';
|
||||
document.getElementById('hr-val').textContent = (68 + Math.sin(t * 0.3) * 4).toFixed(0) + ' bpm';
|
||||
document.getElementById('fps-val').textContent = fps.toFixed(0) + ' fps';
|
||||
}
|
||||
|
||||
// UI wiring
|
||||
document.getElementById('time-scale').addEventListener('input', (e) => {
|
||||
const ts = parseFloat(e.target.value);
|
||||
document.getElementById('time-scale-val').textContent = ts.toFixed(2);
|
||||
if (mixer) mixer.timeScale = ts;
|
||||
});
|
||||
function bindToggle(id, obj) {
|
||||
document.getElementById(id).addEventListener('change', e => {
|
||||
if (e.target.checked && !scene.children.includes(obj)) scene.add(obj);
|
||||
else if (!e.target.checked) scene.remove(obj);
|
||||
});
|
||||
}
|
||||
bindToggle('t-grid', gridHelper);
|
||||
bindToggle('t-polar', polarHelper);
|
||||
document.getElementById('t-skel').addEventListener('change', e => {
|
||||
if (skeletonHelper) skeletonHelper.visible = e.target.checked;
|
||||
});
|
||||
document.getElementById('t-nodebox').addEventListener('change', e => {
|
||||
for (const bb of nodeBboxHelpers) {
|
||||
if (e.target.checked && !scene.children.includes(bb)) scene.add(bb);
|
||||
else if (!e.target.checked) scene.remove(bb);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Main loop
|
||||
// ---------------------------------------------------------------------
|
||||
const clock = new THREE.Clock();
|
||||
let lastMs = performance.now();
|
||||
let fpsEma = 60;
|
||||
function tick() {
|
||||
const nowMs = performance.now();
|
||||
const dt = nowMs - lastMs;
|
||||
lastMs = nowMs;
|
||||
fpsEma = fpsEma * 0.92 + (1000 / Math.max(dt, 1)) * 0.08;
|
||||
const t = nowMs * 0.001;
|
||||
const delta = clock.getDelta();
|
||||
|
||||
if (mixer) mixer.update(delta);
|
||||
floorMat.uniforms.time.value = t;
|
||||
filmShader.uniforms.time.value = t;
|
||||
|
||||
const center = new THREE.Vector3();
|
||||
if (model) {
|
||||
const box = new THREE.Box3().setFromObject(model);
|
||||
box.getCenter(center);
|
||||
} else center.set(0, 0.9, 0);
|
||||
|
||||
tickCsi(t, center);
|
||||
updateRootMotion(t);
|
||||
updateNodes();
|
||||
updateGodRays(t);
|
||||
maybeEmitPings(t, center);
|
||||
updatePings(t);
|
||||
updateSubjectFlash();
|
||||
updateTomography(t);
|
||||
updateBbox();
|
||||
updateFaceCloud(t);
|
||||
|
||||
controls.update();
|
||||
composer.render();
|
||||
updateHud(t, fpsEma);
|
||||
requestAnimationFrame(tick);
|
||||
}
|
||||
requestAnimationFrame(tick);
|
||||
|
||||
window.addEventListener('resize', () => {
|
||||
camera.aspect = window.innerWidth / window.innerHeight;
|
||||
camera.updateProjectionMatrix();
|
||||
renderer.setSize(window.innerWidth, window.innerHeight);
|
||||
composer.setSize(window.innerWidth, window.innerHeight);
|
||||
bloom.setSize(window.innerWidth, window.innerHeight);
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
After Width: | Height: | Size: 96 KiB |
|
After Width: | Height: | Size: 598 KiB |
|
After Width: | Height: | Size: 632 KiB |
|
After Width: | Height: | Size: 682 KiB |
|
After Width: | Height: | Size: 596 KiB |
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env python3
|
||||
"""ruvultra → browser CSI bridge.
|
||||
|
||||
Reads adaptive_ctrl tick lines from the ESP32-S3 RuView firmware on
|
||||
/dev/ttyACM0 and forwards normalized per-node metrics over a WebSocket
|
||||
that the helpers-skinned-realtime demo can subscribe to via Tailscale.
|
||||
|
||||
Sample serial line (1 Hz cadence from firmware):
|
||||
I (22890561) adaptive_ctrl: medium tick: state=6 yield=15pps motion=1.00 presence=5.35 rssi=-33
|
||||
|
||||
Output JSON (per tick):
|
||||
{
|
||||
"ts": 1716830400.123,
|
||||
"node": 0, # always 0 (single node), client expands to 4
|
||||
"motion": 1.00, # raw firmware metric
|
||||
"presence": 5.35,
|
||||
"rssi": -33,
|
||||
"yield_pps": 15,
|
||||
"amp": 0.78 # synthesized CSI amplitude in [0..1] for the bar
|
||||
}
|
||||
|
||||
Run on ruvultra:
|
||||
python3 -u ruvultra-csi-bridge.py
|
||||
"""
|
||||
import asyncio
|
||||
import builtins
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from contextlib import suppress
|
||||
|
||||
# Force every print to flush — we're often piped to a log file
|
||||
_orig_print = builtins.print
|
||||
def _print(*a, **kw):
|
||||
kw.setdefault("flush", True)
|
||||
return _orig_print(*a, **kw)
|
||||
builtins.print = _print
|
||||
|
||||
import serial
|
||||
import websockets
|
||||
|
||||
PORT = "/dev/ttyACM0"
|
||||
BAUD = 115200
|
||||
WS_HOST = "0.0.0.0"
|
||||
WS_PORT = 8766
|
||||
|
||||
TICK_RE = re.compile(
|
||||
r"adaptive_ctrl:\s*\w+\s+tick:\s*"
|
||||
r"state=(?P<state>\d+)\s+"
|
||||
r"yield=(?P<yield>\d+)pps\s+"
|
||||
r"motion=(?P<motion>[\d.]+)\s+"
|
||||
r"presence=(?P<presence>[\d.]+)\s+"
|
||||
r"rssi=(?P<rssi>-?\d+)"
|
||||
)
|
||||
|
||||
clients = set()
|
||||
last_payload = None
|
||||
|
||||
|
||||
def amp_from_metrics(motion, presence, rssi):
|
||||
"""Map firmware metrics to a [0..1] CSI-style amplitude."""
|
||||
rssi_norm = max(0.0, min(1.0, (rssi + 80) / 50)) # -80..-30 → 0..1
|
||||
presence_norm = max(0.0, min(1.0, presence / 8.0)) # cap at 8
|
||||
motion_norm = max(0.0, min(1.0, motion)) # already 0..1ish
|
||||
return 0.40 * rssi_norm + 0.35 * presence_norm + 0.25 * motion_norm
|
||||
|
||||
|
||||
async def serial_reader_loop():
|
||||
global last_payload
|
||||
print(f"[bridge] opening {PORT} @ {BAUD}…")
|
||||
while True:
|
||||
try:
|
||||
ser = serial.Serial(PORT, BAUD, timeout=1)
|
||||
except (serial.SerialException, OSError) as e:
|
||||
print(f"[bridge] serial open failed ({e}); retry in 3s")
|
||||
await asyncio.sleep(3)
|
||||
continue
|
||||
|
||||
print(f"[bridge] connected to {PORT}")
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
while True:
|
||||
line = await loop.run_in_executor(None, ser.readline)
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
text = line.decode(errors="replace").strip()
|
||||
except Exception:
|
||||
continue
|
||||
m = TICK_RE.search(text)
|
||||
if not m:
|
||||
continue
|
||||
motion = float(m["motion"])
|
||||
presence = float(m["presence"])
|
||||
rssi = int(m["rssi"])
|
||||
payload = {
|
||||
"ts": time.time(),
|
||||
"node": 0,
|
||||
"state": int(m["state"]),
|
||||
"yield_pps": int(m["yield"]),
|
||||
"motion": motion,
|
||||
"presence": presence,
|
||||
"rssi": rssi,
|
||||
"amp": amp_from_metrics(motion, presence, rssi),
|
||||
}
|
||||
last_payload = payload
|
||||
msg = json.dumps(payload)
|
||||
if clients:
|
||||
dead = []
|
||||
for ws in list(clients):
|
||||
try:
|
||||
await ws.send(msg)
|
||||
except websockets.ConnectionClosed:
|
||||
dead.append(ws)
|
||||
for d in dead:
|
||||
clients.discard(d)
|
||||
print(
|
||||
f"[tick] motion={motion:.2f} presence={presence:5.2f} "
|
||||
f"rssi={rssi:+d} yield={int(m['yield']):3d}pps "
|
||||
f"amp={payload['amp']:.2f} clients={len(clients)}"
|
||||
)
|
||||
except (serial.SerialException, OSError) as e:
|
||||
print(f"[bridge] serial error ({e}); reopen in 1s")
|
||||
with suppress(Exception):
|
||||
ser.close()
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
async def ws_handler(ws):
|
||||
addr = ws.remote_address
|
||||
clients.add(ws)
|
||||
print(f"[ws] client connected: {addr} total={len(clients)}")
|
||||
try:
|
||||
if last_payload is not None:
|
||||
await ws.send(json.dumps(last_payload))
|
||||
await ws.wait_closed()
|
||||
finally:
|
||||
clients.discard(ws)
|
||||
print(f"[ws] client gone: {addr} total={len(clients)}")
|
||||
|
||||
|
||||
async def main():
|
||||
print(f"[bridge] websocket on ws://{WS_HOST}:{WS_PORT}")
|
||||
async with websockets.serve(ws_handler, WS_HOST, WS_PORT):
|
||||
await serial_reader_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(main())
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Tiny threaded HTTP server for the three.js demos that fetch local files.
|
||||
|
||||
Why a sibling helper script instead of `python -m http.server`?
|
||||
The stdlib SimpleHTTPServer is single-threaded; Chrome opens many parallel
|
||||
connections (HTML + 9 script tags + FBX), the first eats the worker, the
|
||||
rest time out with net::ERR_EMPTY_RESPONSE. ThreadingHTTPServer fixes it.
|
||||
|
||||
Usage:
|
||||
python examples/three.js/server/serve-demo.py
|
||||
open http://localhost:8765/examples/three.js/demos/05-skinned-realtime.html
|
||||
"""
|
||||
from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler
|
||||
import os, sys
|
||||
|
||||
PORT = int(os.environ.get("PORT", 8765))
|
||||
# Always serve from the repo root regardless of where the script is launched.
|
||||
# This file lives at examples/three.js/server/serve-demo.py — three levels deep.
|
||||
os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")))
|
||||
|
||||
class NoCacheHandler(SimpleHTTPRequestHandler):
|
||||
def end_headers(self):
|
||||
# Aggressive no-cache so browser ALWAYS fetches the latest .html
|
||||
# after we edit it. Otherwise stale code sticks around even on hard
|
||||
# refresh and you debug a phantom.
|
||||
self.send_header("Cache-Control", "no-store, no-cache, must-revalidate, max-age=0")
|
||||
self.send_header("Pragma", "no-cache")
|
||||
self.send_header("Expires", "0")
|
||||
super().end_headers()
|
||||
|
||||
DEMOS = [
|
||||
"01-helpers.html",
|
||||
"02-cinematic.html",
|
||||
"03-skinned.html",
|
||||
"04-skinned-fbx.html",
|
||||
"05-skinned-realtime.html",
|
||||
]
|
||||
|
||||
with ThreadingHTTPServer(("127.0.0.1", PORT), NoCacheHandler) as srv:
|
||||
print(f"serving {os.getcwd()} on http://127.0.0.1:{PORT}/")
|
||||
print("demos:")
|
||||
for d in DEMOS:
|
||||
print(f" http://127.0.0.1:{PORT}/examples/three.js/demos/{d}")
|
||||
try:
|
||||
srv.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(0)
|
||||
@@ -15,7 +15,7 @@ This firmware captures WiFi Channel State Information (CSI) from an ESP32-S3 and
|
||||
> | **CSI streaming** | Per-subcarrier I/Q capture over UDP | ~20 Hz, ADR-018 binary format |
|
||||
> | **Breathing detection** | Bandpass 0.1-0.5 Hz, zero-crossing BPM | 6-30 BPM |
|
||||
> | **Heart rate** | Bandpass 0.8-2.0 Hz, zero-crossing BPM | 40-120 BPM |
|
||||
> | **Presence sensing** | Phase variance + adaptive calibration | < 1 ms latency |
|
||||
> | **Presence indicator** (heuristic) | Phase variance + adaptive threshold (60 s ambient learning) | < 1 ms latency, false-positives under strong RF interference — see [Tier 2 caveats](#what-this-firmware-does-not-do-tier-2-caveats) |
|
||||
> | **Fall detection** | Phase acceleration threshold | Configurable sensitivity |
|
||||
> | **Programmable sensing** | WASM modules loaded over HTTP | Hot-swap, no reflash |
|
||||
|
||||
@@ -37,18 +37,22 @@ MSYS_NO_PATHCONV=1 docker run --rm \
|
||||
|
||||
### 2. Flash
|
||||
|
||||
Offsets must match `partitions_display.csv` (8 MB) or `partitions_4mb.csv` (4 MB):
|
||||
`bootloader=0x0`, `partition-table=0x8000`, `otadata=0xf000`, `app (ota_0)=0x20000`.
|
||||
|
||||
```bash
|
||||
python -m esptool --chip esp32s3 --port COM7 --baud 460800 \
|
||||
write_flash --flash_mode dio --flash_size 8MB \
|
||||
0x0 firmware/esp32-csi-node/build/bootloader/bootloader.bin \
|
||||
0x8000 firmware/esp32-csi-node/build/partition_table/partition-table.bin \
|
||||
0x10000 firmware/esp32-csi-node/build/esp32-csi-node.bin
|
||||
0x0 firmware/esp32-csi-node/build/bootloader/bootloader.bin \
|
||||
0x8000 firmware/esp32-csi-node/build/partition_table/partition-table.bin \
|
||||
0xf000 firmware/esp32-csi-node/build/ota_data_initial.bin \
|
||||
0x20000 firmware/esp32-csi-node/build/esp32-csi-node.bin
|
||||
```
|
||||
|
||||
### 3. Provision WiFi credentials (no reflash needed)
|
||||
|
||||
```bash
|
||||
python scripts/provision.py --port COM7 \
|
||||
python firmware/esp32-csi-node/provision.py --port COM7 \
|
||||
--ssid "YourSSID" --password "YourPass" --target-ip 192.168.1.20
|
||||
```
|
||||
|
||||
@@ -129,11 +133,32 @@ Adds real-time health and safety monitoring.
|
||||
|
||||
- **Breathing rate** -- biquad IIR bandpass 0.1-0.5 Hz, zero-crossing BPM (6-30 BPM)
|
||||
- **Heart rate** -- biquad IIR bandpass 0.8-2.0 Hz, zero-crossing BPM (40-120 BPM)
|
||||
- **Presence detection** -- adaptive threshold calibration (60 s ambient learning)
|
||||
- **Presence indicator** -- phase variance vs an adaptively-calibrated threshold (60 s ambient learning at boot). Heuristic, not a learned classifier — strong RF interferers (fans, microwaves, transmit-power swings) can push variance above threshold without anyone in the room. See "What this firmware does NOT do" below.
|
||||
- **Fall detection** -- phase acceleration exceeds configurable threshold
|
||||
- **Multi-person estimation** -- subcarrier group clustering (up to 4 persons)
|
||||
- **Multi-person slot count** -- partitions the top-K subcarriers into `top_k / 2` groups (clamped to `[1, EDGE_MAX_PERSONS]`), computes per-group filtered breathing/heart-rate estimates, and reports the slot count as `pkt.n_persons`. This is a **slot-capacity heuristic**, not a learned counter — the reported count tracks subcarrier diversity, not actual occupancy. See [`edge_processing.c:481-548`](main/edge_processing.c#L481-L548).
|
||||
- **Vitals packet** -- 32-byte UDP packet at 1 Hz (magic `0xC5110002`)
|
||||
|
||||
### What this firmware does NOT do (Tier 2 caveats)
|
||||
|
||||
- It does **not** run a trained neural model. The "person count" is an
|
||||
arithmetic slot-capacity heuristic over the top-K subcarrier groups
|
||||
(`firmware/esp32-csi-node/main/edge_processing.c:481`). It tracks
|
||||
subcarrier diversity, not actual occupancy.
|
||||
- It does **not** run pose estimation. Pose-related features in the host
|
||||
UI come from the Rust `wifi-densepose-sensing-server` running a separate
|
||||
pipeline. When no `.rvf` model file is loaded via `--model`, the server
|
||||
drives the on-screen skeleton from signal-based heuristics (amplitude
|
||||
variance, motion-band power), not from learned keypoint inference. The
|
||||
repository does not ship pre-trained weights — see issues
|
||||
[#509](../../issues/509) and [#506](../../issues/506) for context, and
|
||||
[ADR-079](../../docs/adr/ADR-079-camera-supervised-pose-finetune.md) for
|
||||
the planned training path (phases P7-P9 are `Pending`).
|
||||
- The presence indicator is a calibrated variance threshold and **will
|
||||
false-positive** under strong RF interference from non-human sources
|
||||
(fans near the antenna, microwave duty cycles, neighbouring AP power
|
||||
swings) without re-running the 60-second ambient calibration. If you
|
||||
see ghost detections, re-calibrate by power-cycling in an empty room.
|
||||
|
||||
### Tier 3 -- WASM Programmable Sensing (Alpha)
|
||||
|
||||
Turns the ESP32 from a fixed-function sensor into a programmable sensing computer. Instead of reflashing firmware to change algorithms, you upload new sensing logic as small WASM modules -- compiled from Rust, packaged in signed RVF containers.
|
||||
@@ -254,9 +279,10 @@ Find your serial port: `COM7` on Windows, `/dev/ttyUSB0` on Linux, `/dev/cu.SLAB
|
||||
```bash
|
||||
python -m esptool --chip esp32s3 --port COM7 --baud 460800 \
|
||||
write_flash --flash_mode dio --flash_size 8MB \
|
||||
0x0 firmware/esp32-csi-node/build/bootloader/bootloader.bin \
|
||||
0x8000 firmware/esp32-csi-node/build/partition_table/partition-table.bin \
|
||||
0x10000 firmware/esp32-csi-node/build/esp32-csi-node.bin
|
||||
0x0 firmware/esp32-csi-node/build/bootloader/bootloader.bin \
|
||||
0x8000 firmware/esp32-csi-node/build/partition_table/partition-table.bin \
|
||||
0xf000 firmware/esp32-csi-node/build/ota_data_initial.bin \
|
||||
0x20000 firmware/esp32-csi-node/build/esp32-csi-node.bin
|
||||
```
|
||||
|
||||
### Serial Monitor
|
||||
@@ -285,7 +311,7 @@ All settings can be changed at runtime via Non-Volatile Storage (NVS) without re
|
||||
The easiest way to write NVS settings:
|
||||
|
||||
```bash
|
||||
python scripts/provision.py --port COM7 \
|
||||
python firmware/esp32-csi-node/provision.py --port COM7 \
|
||||
--ssid "MyWiFi" \
|
||||
--password "MyPassword" \
|
||||
--target-ip 192.168.1.20
|
||||
|
||||
@@ -11,7 +11,26 @@ set(SRCS
|
||||
"adaptive_controller.c"
|
||||
)
|
||||
|
||||
set(REQUIRES "")
|
||||
# ESP-IDF v6+: headers must resolve via explicit REQUIRES (no implicit deps).
|
||||
set(REQUIRES
|
||||
esp_wifi
|
||||
esp_netif
|
||||
esp_event
|
||||
nvs_flash
|
||||
app_update
|
||||
esp_http_server
|
||||
esp_http_client
|
||||
esp_app_format
|
||||
esp_timer
|
||||
esp_pm
|
||||
esp_driver_uart
|
||||
esp_driver_gpio
|
||||
esp_driver_spi
|
||||
esp_driver_i2c
|
||||
driver
|
||||
lwip
|
||||
mbedtls
|
||||
)
|
||||
|
||||
# ADR-061: Mock CSI generator for QEMU testing + ADR-081 mock radio binding
|
||||
if(CONFIG_CSI_MOCK_ENABLED)
|
||||
@@ -21,7 +40,11 @@ endif()
|
||||
# ADR-045: AMOLED display support (compile-time optional)
|
||||
if(CONFIG_DISPLAY_ENABLE)
|
||||
list(APPEND SRCS "display_hal.c" "display_ui.c" "display_task.c")
|
||||
set(REQUIRES esp_lcd esp_lcd_touch lvgl)
|
||||
list(APPEND REQUIRES esp_lcd esp_lcd_touch lvgl)
|
||||
endif()
|
||||
|
||||
if(CONFIG_WASM_ENABLE)
|
||||
list(APPEND REQUIRES wasm3)
|
||||
endif()
|
||||
|
||||
idf_component_register(
|
||||
|
||||
@@ -336,6 +336,21 @@ void csi_collector_init(void)
|
||||
/* Update the hop table's first channel to match. */
|
||||
s_hop_channels[0] = csi_channel;
|
||||
|
||||
/* Disable WiFi modem sleep — reliable CSI capture needs the radio awake.
|
||||
* The ESP-IDF STA default is WIFI_PS_MIN_MODEM, which lets the modem
|
||||
* sleep between DTIM beacons; with the MGMT-only promiscuous filter
|
||||
* (RuView#396) that starves the CSI callback and the per-second yield
|
||||
* collapses toward 0 pps (RuView#521). Operators who want battery
|
||||
* duty-cycling opt back in via power_mgmt_init() (provision.py
|
||||
* --duty-cycle <N>), which runs after this and re-enables modem sleep. */
|
||||
esp_err_t ps_err = esp_wifi_set_ps(WIFI_PS_NONE);
|
||||
if (ps_err != ESP_OK) {
|
||||
ESP_LOGW(TAG, "esp_wifi_set_ps(WIFI_PS_NONE) failed: %s — CSI yield may be low",
|
||||
esp_err_to_name(ps_err));
|
||||
} else {
|
||||
ESP_LOGI(TAG, "WiFi modem sleep disabled (WIFI_PS_NONE) for CSI capture");
|
||||
}
|
||||
|
||||
/* Enable promiscuous mode — required for reliable CSI callbacks.
|
||||
* Without this, CSI only fires on frames destined to this station,
|
||||
* which may be very infrequent on a quiet network. */
|
||||
@@ -356,6 +371,30 @@ void csi_collector_init(void)
|
||||
|
||||
ESP_LOGI(TAG, "Promiscuous mode enabled (MGMT-only, RuView#396)");
|
||||
|
||||
#if CONFIG_SOC_WIFI_HE_SUPPORT
|
||||
/* Wi-Fi 6 targets (e.g. ESP32-C6): wifi_csi_config_t is wifi_csi_acquire_config_t
|
||||
* (bitfields), not the legacy 802.11n bool layout used on ESP32-S3. */
|
||||
wifi_csi_config_t csi_config;
|
||||
memset(&csi_config, 0, sizeof(csi_config));
|
||||
csi_config.enable = 1U;
|
||||
csi_config.acquire_csi_legacy = 1U;
|
||||
csi_config.acquire_csi_ht20 = 1U;
|
||||
csi_config.acquire_csi_ht40 = 1U;
|
||||
csi_config.acquire_csi_su = 1U;
|
||||
csi_config.acquire_csi_mu = 1U;
|
||||
csi_config.acquire_csi_dcm = 1U;
|
||||
csi_config.acquire_csi_beamformed = 1U;
|
||||
#if CONFIG_SOC_WIFI_MAC_VERSION_NUM >= 3
|
||||
csi_config.acquire_csi_force_lltf = 1U;
|
||||
csi_config.acquire_csi_vht = 1U;
|
||||
csi_config.acquire_csi_he_stbc_mode = ESP_CSI_ACQUIRE_STBC_SAMPLE_HELTFS;
|
||||
csi_config.val_scale_cfg = 0U;
|
||||
#else
|
||||
csi_config.acquire_csi_he_stbc = ESP_CSI_ACQUIRE_STBC_SAMPLE_HELTFS;
|
||||
csi_config.val_scale_cfg = 0U;
|
||||
#endif
|
||||
csi_config.dump_ack_en = 0U;
|
||||
#else
|
||||
wifi_csi_config_t csi_config = {
|
||||
.lltf_en = true,
|
||||
.htltf_en = true,
|
||||
@@ -365,6 +404,7 @@ void csi_collector_init(void)
|
||||
.manu_scale = false,
|
||||
.shift = false,
|
||||
};
|
||||
#endif
|
||||
|
||||
ESP_ERROR_CHECK(esp_wifi_set_csi_config(&csi_config));
|
||||
ESP_ERROR_CHECK(esp_wifi_set_csi_rx_cb(wifi_csi_callback, NULL));
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
* @file edge_processing.c
|
||||
* @brief ADR-039 Edge Intelligence — dual-core CSI processing pipeline.
|
||||
*
|
||||
* Core 0 (WiFi task): Pushes raw CSI frames into lock-free SPSC ring buffer.
|
||||
* Core 1 (DSP task): Pops frames, runs signal processing pipeline:
|
||||
* Core 0 (WiFi path): Pushes raw CSI frames into lock-free SPSC ring buffer.
|
||||
* Second core when present (DSP task): pops frames, runs signal processing pipeline.
|
||||
* On unicore targets (e.g. ESP32-C6), the DSP task is pinned to core 0.
|
||||
* 1. Phase extraction from I/Q pairs
|
||||
* 2. Phase unwrapping (continuous phase)
|
||||
* 3. Welford variance tracking per subcarrier
|
||||
@@ -1050,7 +1051,9 @@ esp_err_t edge_processing_init(const edge_config_t *cfg)
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
/* Start DSP task on Core 1. */
|
||||
/* Pin DSP off WiFi's preferred core when SMP; else core 0 only (ESP32-C6). */
|
||||
const BaseType_t dsp_core = (portNUM_PROCESSORS > 1) ? (BaseType_t)1 : (BaseType_t)0;
|
||||
|
||||
BaseType_t ret = xTaskCreatePinnedToCore(
|
||||
edge_task,
|
||||
"edge_dsp",
|
||||
@@ -1058,14 +1061,14 @@ esp_err_t edge_processing_init(const edge_config_t *cfg)
|
||||
NULL,
|
||||
5, /* Priority 5 — above idle, below WiFi. */
|
||||
NULL,
|
||||
1 /* Pin to Core 1. */
|
||||
);
|
||||
dsp_core);
|
||||
|
||||
if (ret != pdPASS) {
|
||||
ESP_LOGE(TAG, "Failed to create edge DSP task");
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
|
||||
ESP_LOGI(TAG, "Edge DSP task created on Core 1 (stack=8192, priority=5)");
|
||||
ESP_LOGI(TAG, "Edge DSP task created on core %d (stack=8192, priority=5)",
|
||||
(int)dsp_core);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
@@ -8,3 +8,6 @@ dependencies:
|
||||
|
||||
## LCD touch abstraction
|
||||
espressif/esp_lcd_touch: "^1.0"
|
||||
|
||||
## Onboard WS2812 LED Disabling
|
||||
espressif/led_strip: "^3.0.0"
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
#include "nvs_flash.h"
|
||||
#include "esp_app_desc.h"
|
||||
#include "sdkconfig.h"
|
||||
#include "led_strip.h"
|
||||
|
||||
#include "csi_collector.h"
|
||||
#include "stream_sender.h"
|
||||
@@ -149,6 +150,23 @@ void app_main(void)
|
||||
ESP_LOGI(TAG, "ESP32-S3 CSI Node (ADR-018) — v%s — Node ID: %d",
|
||||
app_desc->version, g_nvs_config.node_id);
|
||||
|
||||
/* Turn off onboard WS2812 LED on GPIO 38 */
|
||||
led_strip_handle_t led_strip;
|
||||
led_strip_config_t strip_config = {
|
||||
.strip_gpio_num = 38,
|
||||
.max_leds = 1,
|
||||
.led_model = LED_MODEL_WS2812,
|
||||
.color_component_format = LED_STRIP_COLOR_COMPONENT_FMT_GRB,
|
||||
.flags.invert_out = false,
|
||||
};
|
||||
led_strip_rmt_config_t rmt_config = {
|
||||
.resolution_hz = 10 * 1000 * 1000, // 10MHz
|
||||
.flags.with_dma = false,
|
||||
};
|
||||
if (led_strip_new_rmt_device(&strip_config, &rmt_config, &led_strip) == ESP_OK) {
|
||||
led_strip_clear(led_strip);
|
||||
}
|
||||
|
||||
/* Initialize WiFi STA (skip entirely under QEMU mock — no RF hardware) */
|
||||
#ifndef CONFIG_CSI_MOCK_SKIP_WIFI_CONNECT
|
||||
wifi_init_sta();
|
||||
|
||||
@@ -109,7 +109,7 @@ static void mr60_process_frame(uint16_t type, const uint8_t *data, uint16_t len)
|
||||
|
||||
switch (type) {
|
||||
case MR60_TYPE_BREATHING:
|
||||
if (len >= 4) {
|
||||
if (len >= sizeof(float)) {
|
||||
/* Breathing rate as float32 (little-endian in payload). */
|
||||
float br;
|
||||
memcpy(&br, data, sizeof(float));
|
||||
@@ -120,7 +120,7 @@ static void mr60_process_frame(uint16_t type, const uint8_t *data, uint16_t len)
|
||||
break;
|
||||
|
||||
case MR60_TYPE_HEARTRATE:
|
||||
if (len >= 4) {
|
||||
if (len >= sizeof(float)) {
|
||||
float hr;
|
||||
memcpy(&hr, data, sizeof(float));
|
||||
if (hr >= 0.0f && hr <= 250.0f) {
|
||||
@@ -130,13 +130,13 @@ static void mr60_process_frame(uint16_t type, const uint8_t *data, uint16_t len)
|
||||
break;
|
||||
|
||||
case MR60_TYPE_DISTANCE:
|
||||
if (len >= 8) {
|
||||
if (len >= sizeof(uint32_t) + sizeof(float)) {
|
||||
/* Bytes 0-3: range flag (uint32 LE). 0 = no valid distance. */
|
||||
uint32_t range_flag;
|
||||
memcpy(&range_flag, data, sizeof(uint32_t));
|
||||
if (range_flag != 0 && len >= 8) {
|
||||
if (range_flag != 0) {
|
||||
float dist;
|
||||
memcpy(&dist, &data[4], sizeof(float));
|
||||
memcpy(&dist, &data[sizeof(uint32_t)], sizeof(float));
|
||||
s_state.distance_cm = dist;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,14 +38,24 @@ static char s_ota_psk[OTA_PSK_MAX_LEN] = {0};
|
||||
|
||||
/**
|
||||
* ADR-050: Verify the Authorization header contains the correct PSK.
|
||||
* Returns true if auth is disabled (no PSK provisioned) or if the
|
||||
* Bearer token matches the stored PSK.
|
||||
* Returns true only when a PSK is provisioned AND the Bearer token
|
||||
* matches it. An unprovisioned node refuses all OTA requests
|
||||
* (fail-closed, see RuView#596 audit). The OTA server still starts so
|
||||
* the operator can `provision.py --ota-psk <hex>` over USB-CDC without
|
||||
* a reflash, but the upload endpoint will reject every request until
|
||||
* the PSK is set.
|
||||
*/
|
||||
static bool ota_check_auth(httpd_req_t *req)
|
||||
{
|
||||
if (s_ota_psk[0] == '\0') {
|
||||
/* No PSK provisioned — auth disabled (permissive for dev). */
|
||||
return true;
|
||||
/* No PSK provisioned — fail closed. Previously this returned
|
||||
* true ("permissive for dev"), which let any host on the WiFi
|
||||
* push attacker-controlled firmware to a freshly-flashed node.
|
||||
* Plain HTTP transport + no Secure Boot V2 + no signed-image
|
||||
* verification meant a single LAN call could brick or back-
|
||||
* door a node. Reject until provisioned. */
|
||||
ESP_LOGW(TAG, "OTA rejected: no PSK in NVS (run provision.py --ota-psk <hex>)");
|
||||
return false;
|
||||
}
|
||||
|
||||
char auth_header[128] = {0};
|
||||
@@ -241,26 +251,45 @@ static esp_err_t ota_start_server(httpd_handle_t *out_handle)
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t ota_update_init(void)
|
||||
/**
|
||||
* Load the OTA PSK from NVS into the module-local s_ota_psk cache and log
|
||||
* the resulting posture. Called by both ota_update_init() and
|
||||
* ota_update_init_ex() so the per-boot diagnostic prints no matter which
|
||||
* entry point main.c uses — historically only ota_update_init() loaded the
|
||||
* PSK, which left ota_update_init_ex() with an empty s_ota_psk and an
|
||||
* invisible fail-closed posture (RuView#596 follow-up).
|
||||
*/
|
||||
static void ota_load_psk_from_nvs(void)
|
||||
{
|
||||
/* ADR-050: Load OTA PSK from NVS if provisioned. */
|
||||
nvs_handle_t nvs;
|
||||
if (nvs_open(OTA_NVS_NAMESPACE, NVS_READONLY, &nvs) == ESP_OK) {
|
||||
size_t len = sizeof(s_ota_psk);
|
||||
if (nvs_get_str(nvs, OTA_NVS_KEY, s_ota_psk, &len) == ESP_OK) {
|
||||
ESP_LOGI(TAG, "OTA PSK loaded from NVS (%d chars) — authentication enabled", (int)len - 1);
|
||||
} else {
|
||||
ESP_LOGW(TAG, "No OTA PSK in NVS — OTA authentication DISABLED (provision with nvs_set)");
|
||||
ESP_LOGW(TAG, "No OTA PSK in NVS — OTA upload endpoint will REJECT all requests until "
|
||||
"provisioned (provision.py --ota-psk <hex>). Fail-closed per RuView#596.");
|
||||
}
|
||||
nvs_close(nvs);
|
||||
} else {
|
||||
ESP_LOGW(TAG, "NVS namespace '%s' not found — OTA authentication DISABLED", OTA_NVS_NAMESPACE);
|
||||
ESP_LOGW(TAG, "NVS namespace '%s' not found — OTA upload endpoint will REJECT all "
|
||||
"requests until provisioned. Fail-closed per RuView#596.", OTA_NVS_NAMESPACE);
|
||||
}
|
||||
}
|
||||
|
||||
esp_err_t ota_update_init(void)
|
||||
{
|
||||
/* ADR-050: Load OTA PSK from NVS if provisioned. */
|
||||
ota_load_psk_from_nvs();
|
||||
return ota_start_server(NULL);
|
||||
}
|
||||
|
||||
esp_err_t ota_update_init_ex(void **out_server)
|
||||
{
|
||||
/* ADR-050: Load OTA PSK from NVS if provisioned. main.c uses this
|
||||
* variant (not ota_update_init), so without this call s_ota_psk
|
||||
* stayed empty forever and the fail-closed posture was invisible
|
||||
* in serial logs. */
|
||||
ota_load_psk_from_nvs();
|
||||
return ota_start_server((httpd_handle_t *)out_server);
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
|
||||
#include <string.h>
|
||||
#include "esp_log.h"
|
||||
#include "mbedtls/sha256.h"
|
||||
#include "psa/crypto.h"
|
||||
|
||||
static const char *TAG = "rvf";
|
||||
|
||||
@@ -125,9 +125,13 @@ esp_err_t rvf_parse(const uint8_t *data, uint32_t data_len, rvf_parsed_t *out)
|
||||
|
||||
/* ---- Verify build hash (SHA-256 of WASM payload) ---- */
|
||||
uint8_t computed_hash[32];
|
||||
int ret = mbedtls_sha256(wasm_data, hdr->wasm_len, computed_hash, 0);
|
||||
if (ret != 0) {
|
||||
ESP_LOGE(TAG, "SHA-256 computation failed: %d", ret);
|
||||
size_t hash_len = 0;
|
||||
psa_status_t psa_st = psa_hash_compute(PSA_ALG_SHA_256, wasm_data,
|
||||
hdr->wasm_len, computed_hash,
|
||||
sizeof(computed_hash), &hash_len);
|
||||
if (psa_st != PSA_SUCCESS || hash_len != 32) {
|
||||
ESP_LOGE(TAG, "SHA-256 computation failed: psa=%d len=%u",
|
||||
(int)psa_st, (unsigned)hash_len);
|
||||
return ESP_FAIL;
|
||||
}
|
||||
|
||||
@@ -186,8 +190,7 @@ esp_err_t rvf_verify_signature(const rvf_parsed_t *parsed, const uint8_t *data,
|
||||
/*
|
||||
* Ed25519 verification.
|
||||
*
|
||||
* ESP-IDF v5.2 mbedtls does NOT include Ed25519 (Curve25519 is
|
||||
* for ECDH/X25519 only). We use a SHA-256-HMAC integrity check:
|
||||
* Legacy mbedtls Ed25519 is optional. We use a SHA-256 keyed digest:
|
||||
*
|
||||
* expected = SHA-256(pubkey || signed_region)
|
||||
*
|
||||
@@ -196,35 +199,34 @@ esp_err_t rvf_verify_signature(const rvf_parsed_t *parsed, const uint8_t *data,
|
||||
* pubkey produces a different expected hash, so unauthorized
|
||||
* publishers cannot forge a valid signature.
|
||||
*
|
||||
* For full Ed25519 (NaCl-style), enable CONFIG_MBEDTLS_EDDSA_C
|
||||
* or link TweetNaCl. The RVF builder should match this scheme.
|
||||
* For full Ed25519, enable CONFIG_MBEDTLS_EDDSA_C or equivalent.
|
||||
* The RVF builder should match this scheme.
|
||||
*/
|
||||
uint8_t hash_input_prefix[32];
|
||||
memcpy(hash_input_prefix, pubkey, 32);
|
||||
|
||||
/* Compute SHA-256(pubkey || header+manifest+wasm). */
|
||||
mbedtls_sha256_context ctx;
|
||||
mbedtls_sha256_init(&ctx);
|
||||
int ret = mbedtls_sha256_starts(&ctx, 0);
|
||||
if (ret != 0) {
|
||||
mbedtls_sha256_free(&ctx);
|
||||
/* Compute SHA-256(pubkey || header+manifest+wasm) via PSA Crypto. */
|
||||
psa_hash_operation_t op = PSA_HASH_OPERATION_INIT;
|
||||
psa_status_t st = psa_hash_setup(&op, PSA_ALG_SHA_256);
|
||||
if (st != PSA_SUCCESS) {
|
||||
return ESP_FAIL;
|
||||
}
|
||||
ret = mbedtls_sha256_update(&ctx, hash_input_prefix, 32);
|
||||
if (ret != 0) {
|
||||
mbedtls_sha256_free(&ctx);
|
||||
st = psa_hash_update(&op, hash_input_prefix, 32);
|
||||
if (st != PSA_SUCCESS) {
|
||||
(void)psa_hash_abort(&op);
|
||||
return ESP_FAIL;
|
||||
}
|
||||
ret = mbedtls_sha256_update(&ctx, data, signed_len);
|
||||
if (ret != 0) {
|
||||
mbedtls_sha256_free(&ctx);
|
||||
st = psa_hash_update(&op, data, signed_len);
|
||||
if (st != PSA_SUCCESS) {
|
||||
(void)psa_hash_abort(&op);
|
||||
return ESP_FAIL;
|
||||
}
|
||||
|
||||
uint8_t expected[32];
|
||||
ret = mbedtls_sha256_finish(&ctx, expected);
|
||||
mbedtls_sha256_free(&ctx);
|
||||
if (ret != 0) {
|
||||
size_t out_len = 0;
|
||||
st = psa_hash_finish(&op, expected, sizeof(expected), &out_len);
|
||||
if (st != PSA_SUCCESS || out_len != 32) {
|
||||
(void)psa_hash_abort(&op);
|
||||
return ESP_FAIL;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
ESP32-S3 CSI Node Provisioning Script
|
||||
ESP32 CSI node provisioning (ESP32-S3, ESP32-C6, other targets).
|
||||
|
||||
Writes WiFi credentials and aggregator target to the ESP32's NVS partition
|
||||
so users can configure a pre-built firmware binary without recompiling.
|
||||
|
||||
Usage:
|
||||
python provision.py --port COM7 --ssid "MyWiFi" --password "secret" --target-ip 192.168.1.20
|
||||
python provision.py --port /dev/ttyUSB0 --chip esp32c6 --ssid "..." \\
|
||||
--password "..." --target-ip 192.168.1.20
|
||||
|
||||
Requirements:
|
||||
pip install 'esptool>=5.0' nvs-partition-gen
|
||||
@@ -35,6 +37,39 @@ NVS_PARTITION_OFFSET = 0x9000
|
||||
NVS_PARTITION_SIZE = 0x6000 # 24 KiB
|
||||
|
||||
|
||||
CONFIG_VALUE_CHECKS = [
|
||||
("ssid", bool),
|
||||
("password", lambda value: value is not None),
|
||||
("target_ip", bool),
|
||||
("target_port", lambda value: value is not None),
|
||||
("node_id", lambda value: value is not None),
|
||||
("tdm_slot", lambda value: value is not None),
|
||||
("tdm_total", lambda value: value is not None),
|
||||
("edge_tier", lambda value: value is not None),
|
||||
("pres_thresh", lambda value: value is not None),
|
||||
("fall_thresh", lambda value: value is not None),
|
||||
("vital_win", lambda value: value is not None),
|
||||
("vital_int", lambda value: value is not None),
|
||||
("subk_count", lambda value: value is not None),
|
||||
("channel", lambda value: value is not None),
|
||||
("filter_mac", lambda value: value is not None),
|
||||
("hop_channels", lambda value: value is not None),
|
||||
("seed_url", lambda value: value is not None),
|
||||
("seed_token", lambda value: value is not None),
|
||||
("zone", lambda value: value is not None),
|
||||
("swarm_hb", lambda value: value is not None),
|
||||
("swarm_ingest", lambda value: value is not None),
|
||||
]
|
||||
|
||||
|
||||
def has_config_value(args):
|
||||
"""Return True when args include at least one NVS-writing config value."""
|
||||
return any(
|
||||
check(getattr(args, name, None))
|
||||
for name, check in CONFIG_VALUE_CHECKS
|
||||
)
|
||||
|
||||
|
||||
def build_nvs_csv(args):
|
||||
"""Build an NVS CSV string for the csi_cfg namespace."""
|
||||
buf = io.StringIO()
|
||||
@@ -143,7 +178,7 @@ def generate_nvs_binary(csv_content, size):
|
||||
os.unlink(p)
|
||||
|
||||
|
||||
def flash_nvs(port, baud, nvs_bin):
|
||||
def flash_nvs(port, baud, nvs_bin, chip):
|
||||
"""Flash the NVS partition binary to the ESP32."""
|
||||
with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f:
|
||||
f.write(nvs_bin)
|
||||
@@ -152,16 +187,13 @@ def flash_nvs(port, baud, nvs_bin):
|
||||
try:
|
||||
cmd = [
|
||||
sys.executable, "-m", "esptool",
|
||||
"--chip", "esp32s3",
|
||||
"--chip", chip,
|
||||
"--port", port,
|
||||
"--baud", str(baud),
|
||||
# Keep underscore form — ESP-IDF v5.4 bundles esptool 4.10.0 which only
|
||||
# accepts "write_flash". pip's esptool >=5.x accepts both (hyphenated
|
||||
# form preferred) but keeps underscore working. Do not "correct" this.
|
||||
"write_flash",
|
||||
"write-flash",
|
||||
hex(NVS_PARTITION_OFFSET), bin_path,
|
||||
]
|
||||
print(f"Flashing NVS partition ({len(nvs_bin)} bytes) to {port}...")
|
||||
print(f"Flashing NVS partition ({len(nvs_bin)} bytes) to {port} (chip={chip})...")
|
||||
subprocess.check_call(cmd)
|
||||
print("NVS provisioning complete!")
|
||||
finally:
|
||||
@@ -170,10 +202,20 @@ def flash_nvs(port, baud, nvs_bin):
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Provision ESP32-S3 CSI Node with WiFi and aggregator settings",
|
||||
epilog="Example: python provision.py --port COM7 --ssid MyWiFi --password secret --target-ip 192.168.1.20",
|
||||
description="Provision CSI node NVS (WiFi + aggregator); works on S3, C6, etc.",
|
||||
epilog=(
|
||||
"Example: python provision.py --port COM7 --ssid MyWiFi --password secret "
|
||||
"--target-ip 192.168.1.20\n"
|
||||
"ESP32-C6: same, or pass --chip esp32c6 if auto-detect fails "
|
||||
"(default chip is auto for esptool v5+)."
|
||||
),
|
||||
)
|
||||
parser.add_argument("--port", required=True, help="Serial port (e.g. COM7, /dev/ttyUSB0)")
|
||||
parser.add_argument(
|
||||
"--chip",
|
||||
default="auto",
|
||||
help="esptool target: auto (default), esp32s3, esp32c6, ... (must match connected chip)",
|
||||
)
|
||||
parser.add_argument("--baud", type=int, default=460800, help="Flash baud rate (default: 460800)")
|
||||
parser.add_argument("--ssid", help="WiFi SSID")
|
||||
parser.add_argument("--password", help="WiFi password")
|
||||
@@ -214,17 +256,7 @@ def main():
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
has_value = any([
|
||||
args.ssid, args.password is not None, args.target_ip,
|
||||
args.target_port, args.node_id is not None,
|
||||
args.tdm_slot is not None, args.tdm_total is not None,
|
||||
args.edge_tier is not None, args.pres_thresh is not None,
|
||||
args.fall_thresh is not None, args.vital_win is not None,
|
||||
args.vital_int is not None, args.subk_count is not None,
|
||||
args.channel is not None, args.filter_mac is not None,
|
||||
args.seed_url is not None, args.zone is not None,
|
||||
])
|
||||
if not has_value:
|
||||
if not has_config_value(args):
|
||||
parser.error("At least one config value must be specified")
|
||||
|
||||
# Bug 2 (#391): Prevent silent wipe of WiFi credentials on partial invocations.
|
||||
@@ -281,7 +313,7 @@ def main():
|
||||
if args.ssid:
|
||||
print(f" WiFi SSID: {args.ssid}")
|
||||
if args.password is not None:
|
||||
print(f" WiFi Password: {'*' * len(args.password)}")
|
||||
print(f" WiFi Password: {'(set)' if args.password else '(empty)'}")
|
||||
if args.target_ip:
|
||||
print(f" Target IP: {args.target_ip}")
|
||||
if args.target_port:
|
||||
@@ -337,11 +369,11 @@ def main():
|
||||
with open(out, "wb") as f:
|
||||
f.write(nvs_bin)
|
||||
print(f"NVS binary saved to {out} ({len(nvs_bin)} bytes)")
|
||||
print(f"Flash manually: python -m esptool --chip esp32s3 --port {args.port} "
|
||||
print(f"Flash manually: python -m esptool --chip {args.chip} --port {args.port} "
|
||||
f"write-flash 0x9000 {out}")
|
||||
return
|
||||
|
||||
flash_nvs(args.port, args.baud, nvs_bin)
|
||||
flash_nvs(args.port, args.baud, nvs_bin, args.chip)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -34,3 +34,11 @@ CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192
|
||||
|
||||
# Extra WiFi IRAM placement (defense-in-depth for RuView#396 SPI cache race)
|
||||
CONFIG_ESP_WIFI_EXTRA_IRAM_OPT=y
|
||||
|
||||
# ADR-081: adaptive_controller runs emit_feature_state + stream_sender
|
||||
# network I/O inside Timer Svc callbacks, exceeding the 2 KiB default.
|
||||
# Without this, the device bootloops with
|
||||
# "***ERROR*** A stack overflow in task Tmr Svc has been detected."
|
||||
# Was present in sdkconfig.defaults.template but missing here — fixed
|
||||
# in the v0.6.5-esp32 release.
|
||||
CONFIG_FREERTOS_TIMER_TASK_STACK_DEPTH=8192
|
||||
|
||||
@@ -153,6 +153,13 @@ typedef struct {
|
||||
uint8_t primary;
|
||||
} wifi_ap_record_t;
|
||||
|
||||
typedef enum {
|
||||
WIFI_PS_NONE = 0,
|
||||
WIFI_PS_MIN_MODEM = 1,
|
||||
WIFI_PS_MAX_MODEM = 2,
|
||||
} wifi_ps_type_t;
|
||||
|
||||
static inline esp_err_t esp_wifi_set_ps(wifi_ps_type_t type) { (void)type; return ESP_OK; }
|
||||
static inline esp_err_t esp_wifi_set_promiscuous(bool en) { (void)en; return ESP_OK; }
|
||||
static inline esp_err_t esp_wifi_set_promiscuous_rx_cb(void *cb) { (void)cb; return ESP_OK; }
|
||||
static inline esp_err_t esp_wifi_set_promiscuous_filter(wifi_promiscuous_filter_t *f) { (void)f; return ESP_OK; }
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import csv
|
||||
import importlib.util
|
||||
import io
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PROVISION_PATH = Path(__file__).resolve().parents[1] / "provision.py"
|
||||
SPEC = importlib.util.spec_from_file_location("provision", PROVISION_PATH)
|
||||
provision = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(provision)
|
||||
|
||||
|
||||
def make_args(**overrides):
|
||||
values = {name: None for name, _ in provision.CONFIG_VALUE_CHECKS}
|
||||
values["hop_dwell"] = 200
|
||||
values.update(overrides)
|
||||
return types.SimpleNamespace(**values)
|
||||
|
||||
|
||||
def csv_rows(content):
|
||||
return list(csv.DictReader(io.StringIO(content)))
|
||||
|
||||
|
||||
class ProvisionConfigValueTests(unittest.TestCase):
|
||||
def test_swarm_and_hopping_flags_count_as_config_values(self):
|
||||
cases = [
|
||||
{"hop_channels": "1,6,11"},
|
||||
{"seed_token": "token-123"},
|
||||
{"swarm_hb": 15},
|
||||
{"swarm_ingest": 3},
|
||||
]
|
||||
|
||||
for values in cases:
|
||||
with self.subTest(values=values):
|
||||
self.assertTrue(provision.has_config_value(make_args(**values)))
|
||||
|
||||
def test_operational_flags_alone_do_not_count_as_config_values(self):
|
||||
self.assertFalse(provision.has_config_value(make_args()))
|
||||
|
||||
def test_swarm_and_hopping_values_are_written_to_csv(self):
|
||||
args = make_args(
|
||||
hop_channels="1,6,11",
|
||||
hop_dwell=250,
|
||||
seed_token="token-123",
|
||||
swarm_hb=15,
|
||||
swarm_ingest=3,
|
||||
)
|
||||
|
||||
rows = csv_rows(provision.build_nvs_csv(args))
|
||||
values_by_key = {row["key"]: row["value"] for row in rows}
|
||||
|
||||
self.assertEqual(values_by_key["hop_count"], "3")
|
||||
self.assertEqual(values_by_key["chan_list"], "01060b")
|
||||
self.assertEqual(values_by_key["dwell_ms"], "250")
|
||||
self.assertEqual(values_by_key["seed_token"], "token-123")
|
||||
self.assertEqual(values_by_key["swarm_hb"], "15")
|
||||
self.assertEqual(values_by_key["swarm_ingest"], "3")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1 +1 @@
|
||||
0.6.2
|
||||
0.6.5
|
||||
@@ -1,4 +1,4 @@
|
||||
# ESP32-S3 Hello World — Capability Discovery
|
||||
# ESP32 Hello World — Capability Discovery (S3 / C6 targets)
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
|
||||
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
/**
|
||||
* @file main.c
|
||||
* @brief ESP32-S3 Hello World — Full Capability Discovery
|
||||
* @brief ESP32 Hello World — Full Capability Discovery
|
||||
*
|
||||
* Boots up, prints "Hello World!", then probes and reports every major
|
||||
* hardware/software capability of the ESP32-S3: chip info, flash, PSRAM,
|
||||
* WiFi (including CSI), Bluetooth, GPIOs, peripherals, FreeRTOS stats,
|
||||
* and power management features. No WiFi connection required.
|
||||
* Boots up, prints "Hello World!", then probes chip info, flash, PSRAM,
|
||||
* WiFi (including CSI where enabled), 802.15.4/BLE on C6, GPIOs,
|
||||
* peripherals, FreeRTOS stats, and power management. No WiFi connection
|
||||
* required. Supports ESP32-S3 and ESP32-C6 (set IDF target accordingly).
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
@@ -18,7 +18,6 @@
|
||||
#include "esp_chip_info.h"
|
||||
#include "esp_flash.h"
|
||||
#include "esp_mac.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_wifi.h"
|
||||
#include "esp_event.h"
|
||||
#include "esp_timer.h"
|
||||
@@ -33,7 +32,24 @@
|
||||
#include "driver/temperature_sensor.h"
|
||||
#include "sdkconfig.h"
|
||||
|
||||
static const char *TAG = "hello";
|
||||
/*
|
||||
* Peripheral counts: ESP-IDF v6+ dropped some SOC_* macros; values below
|
||||
* match each target's HAL (esp_hal_* *_ll.h) where applicable.
|
||||
*/
|
||||
#if CONFIG_IDF_TARGET_ESP32S3
|
||||
#define PROBE_I2S_CTRL_NUM 2
|
||||
#define PROBE_RMT_CHAN_NUM 8
|
||||
#define PROBE_MCPWM_GROUPS 2
|
||||
#define PROBE_PCNT_UNITS 4
|
||||
#define PROBE_TOUCH_CHAN_NUM ((int)(SOC_TOUCH_MAX_CHAN_ID - SOC_TOUCH_MIN_CHAN_ID + 1))
|
||||
#elif CONFIG_IDF_TARGET_ESP32C6
|
||||
#define PROBE_I2S_CTRL_NUM 1
|
||||
#define PROBE_RMT_CHAN_NUM 4
|
||||
#define PROBE_MCPWM_GROUPS 1
|
||||
#define PROBE_PCNT_UNITS 4
|
||||
#else
|
||||
#error "hello-world: add PROBE_* peripheral counts for this IDF target in main.c"
|
||||
#endif
|
||||
|
||||
/* ── Helpers ─────────────────────────────────────────────────────────── */
|
||||
|
||||
@@ -46,6 +62,7 @@ static const char *chip_model_str(esp_chip_model_t model)
|
||||
case CHIP_ESP32C3: return "ESP32-C3";
|
||||
case CHIP_ESP32H2: return "ESP32-H2";
|
||||
case CHIP_ESP32C2: return "ESP32-C2";
|
||||
case CHIP_ESP32C6: return "ESP32-C6";
|
||||
default: return "Unknown";
|
||||
}
|
||||
}
|
||||
@@ -168,7 +185,11 @@ static void probe_wifi_capabilities(void)
|
||||
ESP_ERROR_CHECK(esp_wifi_start());
|
||||
|
||||
/* Protocol capabilities */
|
||||
#if CONFIG_IDF_TARGET_ESP32C6
|
||||
printf(" Protocols: 802.11 b/g/n/ax (Wi-Fi 6, 2.4 GHz)\n");
|
||||
#else
|
||||
printf(" Protocols: 802.11 b/g/n\n");
|
||||
#endif
|
||||
|
||||
/* CSI (Channel State Information) */
|
||||
#ifdef CONFIG_ESP_WIFI_CSI_ENABLED
|
||||
@@ -246,7 +267,7 @@ static void probe_bluetooth(void)
|
||||
esp_chip_info(&info);
|
||||
|
||||
if (info.features & CHIP_FEATURE_BLE) {
|
||||
printf(" BLE: Supported (Bluetooth 5.0 LE)\n");
|
||||
printf(" BLE: Supported (Bluetooth LE)\n");
|
||||
printf(" - GATT Server/Client\n");
|
||||
printf(" - Advertising & Scanning\n");
|
||||
printf(" - Mesh Networking\n");
|
||||
@@ -256,10 +277,16 @@ static void probe_bluetooth(void)
|
||||
printf(" BLE: Not supported on this chip\n");
|
||||
}
|
||||
|
||||
#if CONFIG_IDF_TARGET_ESP32C6
|
||||
if (info.features & CHIP_FEATURE_IEEE802154) {
|
||||
printf(" 802.15.4: Supported (Thread / Zigbee style MAC)\n");
|
||||
}
|
||||
#endif
|
||||
|
||||
if (info.features & CHIP_FEATURE_BT) {
|
||||
printf(" BT Classic: Supported (A2DP, SPP, HFP)\n");
|
||||
} else {
|
||||
printf(" BT Classic: Not available (ESP32-S3 is BLE-only)\n");
|
||||
printf(" BT Classic: Not available (BLE-only on this chip)\n");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -269,24 +296,52 @@ static void probe_peripherals(void)
|
||||
|
||||
printf(" GPIOs: %d total\n", SOC_GPIO_PIN_COUNT);
|
||||
printf(" ADC:\n");
|
||||
#if CONFIG_IDF_TARGET_ESP32C6
|
||||
printf(" - SAR ADC: %d channels (12-bit, one controller)\n",
|
||||
(int)SOC_ADC_CHANNEL_NUM(0));
|
||||
#else
|
||||
printf(" - ADC1: %d channels (12-bit SAR)\n", SOC_ADC_CHANNEL_NUM(0));
|
||||
printf(" - ADC2: %d channels (shared with WiFi)\n", SOC_ADC_CHANNEL_NUM(1));
|
||||
printf(" DAC: Not available on ESP32-S3\n");
|
||||
printf(" Touch Sensors: %d channels (capacitive)\n", SOC_TOUCH_SENSOR_NUM);
|
||||
printf(" SPI: %d controllers (SPI2/SPI3 for user)\n", SOC_SPI_PERIPH_NUM);
|
||||
printf(" I2C: %d controllers\n", SOC_I2C_NUM);
|
||||
printf(" I2S: %d controllers (audio/PDM/TDM)\n", SOC_I2S_NUM);
|
||||
printf(" UART: %d controllers\n", SOC_UART_NUM);
|
||||
#endif
|
||||
printf(" DAC: Not available on this chip\n");
|
||||
#if CONFIG_IDF_TARGET_ESP32S3
|
||||
printf(" Touch Sensors: %d channels (capacitive)\n", PROBE_TOUCH_CHAN_NUM);
|
||||
#elif CONFIG_IDF_TARGET_ESP32C6
|
||||
printf(" Touch Sensors: Not available (no capacitive touch on ESP32-C6)\n");
|
||||
#endif
|
||||
printf(" SPI: %d controllers\n", SOC_SPI_PERIPH_NUM);
|
||||
#if CONFIG_IDF_TARGET_ESP32S3
|
||||
printf(" (SPI2/SPI3 typical for user apps)\n");
|
||||
#endif
|
||||
printf(" I2C: %d controllers\n", (int)SOC_I2C_NUM);
|
||||
printf(" I2S: %d controller(s) (audio/PDM/TDM)\n", PROBE_I2S_CTRL_NUM);
|
||||
printf(" UART: %d controllers\n", (int)SOC_UART_NUM);
|
||||
#if CONFIG_IDF_TARGET_ESP32S3
|
||||
printf(" USB: USB-OTG 1.1 (Host & Device)\n");
|
||||
printf(" USB-Serial: Built-in USB-JTAG/Serial (this console)\n");
|
||||
#elif CONFIG_IDF_TARGET_ESP32C6
|
||||
printf(" USB: No native USB-OTG (use SPI/USB bridge or off-chip PHY)\n");
|
||||
printf(" USB-Serial: Built-in USB Serial/JTAG (this console)\n");
|
||||
#endif
|
||||
#if CONFIG_IDF_TARGET_ESP32S3
|
||||
printf(" TWAI (CAN): 1 controller (CAN 2.0B compatible)\n");
|
||||
printf(" RMT: %d channels (IR/WS2812/NeoPixel)\n", SOC_RMT_TX_CANDIDATES_PER_GROUP + SOC_RMT_RX_CANDIDATES_PER_GROUP);
|
||||
#elif CONFIG_IDF_TARGET_ESP32C6
|
||||
printf(" TWAI (CAN): %d controller(s) (CAN 2.0B compatible)\n",
|
||||
(int)SOC_TWAI_CONTROLLER_NUM);
|
||||
#endif
|
||||
printf(" RMT: %d channels (IR/WS2812/NeoPixel)\n", PROBE_RMT_CHAN_NUM);
|
||||
printf(" LEDC (PWM): %d channels\n", SOC_LEDC_CHANNEL_NUM);
|
||||
printf(" MCPWM: %d groups (motor control)\n", SOC_MCPWM_GROUPS);
|
||||
printf(" PCNT: %d units (pulse counter / encoder)\n", SOC_PCNT_UNITS_PER_GROUP);
|
||||
printf(" MCPWM: %d group(s) (motor control)\n", PROBE_MCPWM_GROUPS);
|
||||
printf(" PCNT: %d units (pulse counter / encoder)\n", PROBE_PCNT_UNITS);
|
||||
#if CONFIG_IDF_TARGET_ESP32S3
|
||||
printf(" LCD: Parallel 8/16-bit + SPI + I2C interfaces\n");
|
||||
printf(" Camera: DVP 8/16-bit parallel interface\n");
|
||||
printf(" SDMMC: SD/MMC host controller (1-bit / 4-bit)\n");
|
||||
#elif CONFIG_IDF_TARGET_ESP32C6
|
||||
printf(" PARLIO: Parallel TX/RX (e.g. LED matrix / custom buses)\n");
|
||||
printf(" Camera: SPI / external bridge (no native DVP)\n");
|
||||
printf(" SDIO: SDIO slave peripheral (see TRM for capabilities)\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
static void probe_security(void)
|
||||
@@ -309,17 +364,29 @@ static void probe_power(void)
|
||||
{
|
||||
print_separator("POWER MANAGEMENT");
|
||||
|
||||
#if CONFIG_IDF_TARGET_ESP32C6
|
||||
printf(" Clock Modes:\n");
|
||||
printf(" - 160 MHz (max CPU on ESP32-C6)\n");
|
||||
printf(" - 120 MHz (balanced)\n");
|
||||
printf(" - 80 MHz (low power)\n");
|
||||
#else
|
||||
printf(" Clock Modes:\n");
|
||||
printf(" - 240 MHz (max performance)\n");
|
||||
printf(" - 160 MHz (balanced)\n");
|
||||
printf(" - 80 MHz (low power)\n");
|
||||
#endif
|
||||
printf(" Sleep Modes:\n");
|
||||
printf(" - Modem Sleep (WiFi off, CPU active)\n");
|
||||
printf(" - Light Sleep (CPU paused, fast wake)\n");
|
||||
printf(" - Deep Sleep (RTC only, ~10 uA)\n");
|
||||
printf(" - Hibernation (RTC timer only, ~5 uA)\n");
|
||||
#if CONFIG_IDF_TARGET_ESP32C6
|
||||
printf(" Wake Sources: GPIO, LP timer, UART, etc.\n");
|
||||
printf(" LP domain: LP core / LP peripherals (see TRM)\n");
|
||||
#else
|
||||
printf(" Wake Sources: GPIO, timer, touch, ULP, UART\n");
|
||||
printf(" ULP Coprocessor: RISC-V + FSM (runs in deep sleep)\n");
|
||||
printf(" ULP Coprocessor: FSM (runs in deep sleep)\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
static void probe_temperature(void)
|
||||
@@ -389,6 +456,9 @@ static void probe_csi_details(void)
|
||||
|
||||
void app_main(void)
|
||||
{
|
||||
esp_chip_info_t chip;
|
||||
esp_chip_info(&chip);
|
||||
|
||||
/* NVS required for WiFi */
|
||||
esp_err_t ret = nvs_flash_init();
|
||||
if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
|
||||
@@ -401,7 +471,7 @@ void app_main(void)
|
||||
printf("\n");
|
||||
printf(" ╭─────────────────────────────────────────────────╮\n");
|
||||
printf(" │ │\n");
|
||||
printf(" │ HELLO WORLD from ESP32-S3! │\n");
|
||||
printf(" │ HELLO WORLD from %-24s │\n", chip_model_str(chip.model));
|
||||
printf(" │ │\n");
|
||||
printf(" │ WiFi-DensePose Capability Discovery v1.0 │\n");
|
||||
printf(" │ │\n");
|
||||
@@ -422,8 +492,9 @@ void app_main(void)
|
||||
probe_csi_details();
|
||||
|
||||
print_separator("DONE — ALL CAPABILITIES REPORTED");
|
||||
printf("\n This ESP32-S3 is ready for WiFi-DensePose!\n");
|
||||
printf(" Flash the full firmware (esp32-csi-node) to begin CSI sensing.\n\n");
|
||||
printf("\n This %s is ready for WiFi-DensePose experiments.\n",
|
||||
chip_model_str(chip.model));
|
||||
printf(" For production CSI on S3, flash esp32-csi-node; C6 path may differ.\n\n");
|
||||
|
||||
/* Keep alive — blink a status message every 10 seconds */
|
||||
int tick = 0;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# ESP32-S3 Hello World — SDK Configuration
|
||||
CONFIG_IDF_TARGET="esp32s3"
|
||||
# ESP32 Hello World — SDK Configuration (default: ESP32-C6)
|
||||
CONFIG_IDF_TARGET="esp32c6"
|
||||
|
||||
# Flash: 4MB (this chip has Embedded Flash 4MB)
|
||||
CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "ruview",
|
||||
"description": "End-to-end RuView (WiFi-DensePose) toolkit for Claude Code: onboarding, ESP32 hardware setup, configuration, sensing applications, model training, advanced multistatic sensing, and witness verification — from practical to advanced.",
|
||||
"version": "0.1.0",
|
||||
"author": {
|
||||
"name": "ruvnet",
|
||||
"url": "https://github.com/ruvnet/RuView"
|
||||
},
|
||||
"homepage": "https://github.com/ruvnet/RuView",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"ruview",
|
||||
"wifi-densepose",
|
||||
"wifi-sensing",
|
||||
"csi",
|
||||
"esp32",
|
||||
"pose-estimation",
|
||||
"vital-signs",
|
||||
"edge-ai",
|
||||
"model-training",
|
||||
"onboarding"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
# ruview — Claude Code + Codex plugin for WiFi sensing
|
||||
|
||||
End-to-end toolkit for **RuView** (WiFi-DensePose): onboarding, ESP32 hardware setup, configuration, sensing applications, model training, advanced multistatic sensing, and witness verification — from practical to advanced.
|
||||
|
||||
Part of the **`ruview` marketplace** — manifest at the repo root: `.claude-plugin/marketplace.json` (this plugin's `source` is `./plugins/ruview`).
|
||||
|
||||
## Install / test
|
||||
|
||||
```bash
|
||||
# In Claude Code — add this repo as a plugin marketplace, then install:
|
||||
/plugin marketplace add ruvnet/RuView
|
||||
/plugin install ruview@ruview
|
||||
|
||||
# Or try it locally without installing (from a clone of the repo):
|
||||
claude --plugin-dir ./plugins/ruview
|
||||
```
|
||||
|
||||
For Codex (OpenAI CLI), see [`codex/`](codex/) — all seven `/ruview-*` commands mirrored as Codex prompts, plus an `AGENTS.md` and install instructions in [`codex/README.md`](codex/README.md).
|
||||
|
||||
## What's inside
|
||||
|
||||
### Skills (auto-discovered from `skills/`)
|
||||
|
||||
| Skill | What it does |
|
||||
|-------|--------------|
|
||||
| `ruview-quickstart` | Onboarding & first run — Docker demo, repo build, fastest path to a live dashboard |
|
||||
| `ruview-hardware-setup` | ESP32-S3 / C6 firmware build, flash, WiFi provisioning, serial monitoring |
|
||||
| `ruview-configure` | sdkconfig variants, NVS provisioning, channel/MAC overrides (ADR-060), edge modules (ADR-041), sensing-server flags, mesh, Cognitum Seed |
|
||||
| `ruview-applications` | Run presence, vitals, pose (WiFlow), sleep, environment mapping, MAT, point-cloud fusion, novel RF apps |
|
||||
| `ruview-model-training` | Camera-free pose, camera-supervised pose (92.9% PCK@20, ADR-079), RuVector embeddings (AETHER), domain generalization (MERIDIAN), local SNN, GPU on GCloud, HF publishing |
|
||||
| `ruview-advanced-sensing` | RuvSense multistatic, cross-viewpoint fusion, RF tomography, persistent field model, intention signals, adversarial detection, mesh security |
|
||||
| `ruview-cli-api` | `wifi-densepose` CLI binary (incl. MAT subcommands), REST API (`wifi-densepose-api`), browser/WASM (`wifi-densepose-wasm`, `wifi-densepose-wasm-edge`) |
|
||||
| `ruview-mmwave` | mmWave / FMCW radar — ESP32-C6 + MR60BHA2 (60 GHz HR/BR/presence), HLK-LD2410 (24 GHz), mmWave↔CSI fusion (48-byte fused vitals) |
|
||||
| `ruview-verify` | Rust tests, deterministic Python proof, firmware hashes, ADR-028 witness bundle + self-verification, pre-merge checklist |
|
||||
|
||||
### Commands (`commands/`)
|
||||
|
||||
| Command | Purpose |
|
||||
|---------|---------|
|
||||
| `/ruview-start` | Get started — pick Docker / build / hardware and walk through it |
|
||||
| `/ruview-flash` | Build + flash ESP32 firmware (8MB / 4MB), confirm CSI stream |
|
||||
| `/ruview-provision` | Provision WiFi creds, sink IP, channel / MAC-filter onto a node |
|
||||
| `/ruview-app` | Run a sensing application |
|
||||
| `/ruview-train` | Train / evaluate / publish a model (incl. GPU) |
|
||||
| `/ruview-advanced` | Use multistatic / tomography / cross-viewpoint / mesh-security features |
|
||||
| `/ruview-verify` | Run the trust pipeline + pre-merge checklist |
|
||||
|
||||
### Agents (`agents/`)
|
||||
|
||||
| Agent | Role |
|
||||
|-------|------|
|
||||
| `ruview-onboarding-guide` | Walks a newcomer from zero to a working setup |
|
||||
| `ruview-config-engineer` | Sets up / tunes a deployment (firmware, NVS, edge modules, mesh, Seed) |
|
||||
| `ruview-training-engineer` | Trains, evaluates, and ships models |
|
||||
|
||||
## Compatibility
|
||||
|
||||
- **Claude Code** — skills, commands, and agents are auto-discovered; no `claude-flow` MCP server required (skills drive RuView's own tooling: `cargo`, `python`, `idf.py`, `docker`, `node`). Optional: `npx @claude-flow/cli@latest security scan` is referenced for security changes.
|
||||
- **Codex (OpenAI CLI)** — workflows mirrored under `codex/prompts/`; drop them in `~/.codex/prompts/` (or point Codex at `codex/`). `codex/AGENTS.md` carries the project rules.
|
||||
- **Target repo** — assumes the [`ruvnet/RuView`](https://github.com/ruvnet/RuView) / `wifi-densepose` layout: `v2/crates/`, `firmware/esp32-csi-node/`, `archive/v1/`, `scripts/`, `docs/adr/`. On Windows, ESP-IDF builds go through the Python-subprocess pattern in `CLAUDE.local.md`.
|
||||
|
||||
## Namespace coordination
|
||||
|
||||
This plugin claims the kebab-case `ruview-*` namespace for its skills, commands, and agents (skills: `ruview-quickstart`, `ruview-hardware-setup`, `ruview-configure`, `ruview-applications`, `ruview-model-training`, `ruview-advanced-sensing`, `ruview-cli-api`, `ruview-mmwave`, `ruview-verify`; commands: `/ruview-start`, `/ruview-flash`, `/ruview-provision`, `/ruview-app`, `/ruview-train`, `/ruview-advanced`, `/ruview-verify`; agents: `ruview-onboarding-guide`, `ruview-config-engineer`, `ruview-training-engineer`). It does not write to any `claude-flow` memory namespace. If combined with the `ruflo` marketplace, defer to `ruflo-agentdb` ADR-0001 §"Namespace convention" — there is no overlap (`ruview-*` vs. `ruflo-*`).
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
bash plugins/ruview/scripts/smoke.sh
|
||||
```
|
||||
|
||||
Structural contract: plugin.json has `version` + `keywords` and does **not** enumerate skills/commands/agents; every skill/command/agent file exists with valid frontmatter; README has a Compatibility section and a Namespace coordination block; ADR-0001 exists with status `Proposed`; no wildcard tools in skills; Codex mirror present **and parity** — every `commands/<name>.md` has a matching `codex/prompts/<name>.md`.
|
||||
|
||||
## Architecture Decisions
|
||||
|
||||
- [`docs/adrs/0001-ruview-plugin-contract.md`](docs/adrs/0001-ruview-plugin-contract.md) — plugin contract (Proposed): structure, namespace, compatibility surface, smoke scope, Codex mirror policy.
|
||||
|
||||
## Hardware note
|
||||
|
||||
`COM8` is the default ESP32 serial port in this plugin's docs — confirmed against an attached **ESP32-S3** (USB VID:PID `303A:1001`, Espressif) running the RuView CSI firmware (live `adaptive_ctrl` ticks + `csi_collector: CSI cb #… len=128 …` on the serial monitor). The repo's `CLAUDE.local.md` historically referenced `COM7`; some README snippets reference `COM9`. Always confirm the actual port (`python -c "import serial.tools.list_ports as l; print([p.device for p in l.comports()])"`, or Device Manager) before flashing. On Windows, `provision.py --help` needs `PYTHONUTF8=1` to print (non-ASCII in the help text); the build/flash path goes through the Python-subprocess pattern in `CLAUDE.local.md` (ESP-IDF v5.4 ≠ Git Bash).
|
||||
@@ -0,0 +1,29 @@
|
||||
---
|
||||
name: ruview-config-engineer
|
||||
description: Configures RuView deployments — ESP32 firmware variants (8MB/4MB/Heltec), sdkconfig, NVS provisioning, WiFi channel / MAC-filter overrides (ADR-060), edge intelligence modules (ADR-041), sensing-server flags, multi-node mesh, and Cognitum Seed integration. Use to set up or tune a RuView system without changing source code.
|
||||
model: sonnet
|
||||
---
|
||||
|
||||
# RuView Config Engineer
|
||||
|
||||
You own everything tunable in a RuView deployment — from a single provision flag to a full mesh + Cognitum Seed.
|
||||
|
||||
## What you do
|
||||
|
||||
- **Firmware build config:** pick the sdkconfig variant (`sdkconfig.defaults.template` for 8MB no-mock, `sdkconfig.defaults.4mb`, `sdkconfig.defaults.heltec_n16r2`), copy it to `sdkconfig.defaults`, rebuild via the Windows Python-subprocess command (`CLAUDE.local.md`). **Never test in mock mode.**
|
||||
- **Device runtime config (`provision.py`):** writes the `csi_cfg` NVS namespace over serial. Always check `python firmware/esp32-csi-node/provision.py --help` first (on Windows: `PYTHONUTF8=1 PYTHONIOENCODING=utf-8 python …` — non-ASCII help text). Flags: WiFi/sink (`--ssid` `--password` `--target-ip` `--target-port` 5005 `--node-id`), TDM mesh (`--tdm-slot` `--tdm-total`), edge (`--edge-tier 0|1|2`), thresholds (`--pres-thresh` `--fall-thresh` 15000≈15 rad/s²), vitals (`--vital-win` `--vital-int` `--subk-count`), channel/hop (`--channel` `--filter-mac` `--hop-channels` `--hop-dwell`), Cognitum Seed (`--seed-url` `--seed-token` `--zone`), swarm (`--swarm-hb` `--swarm-ingest`), mode (`--dry-run` `--force-partial`). ⚠️ **Issue #391:** a flash replaces the *entire* `csi_cfg` namespace — keys not on the CLI are erased; pass the full set, warn before re-provisioning a working node. Fleet: `scripts/generate_nvs_matrix.py`.
|
||||
- **Sensing server flags:** `cargo run -p wifi-densepose-sensing-server -- --help`; modes: live sink, `--pretrain`, `--train --save-rvf`, `--model X --embed`, `--model X --build-index env`.
|
||||
- **Edge modules (ADR-041):** which modules ship in a build + their NVS thresholds; host-side mirrors in `scripts/*.js` (apnea, gait, material, passive-radar, mincut, fingerprint).
|
||||
- **Multi-node mesh:** TDM + channel hopping (`wifi-densepose-hardware/src/esp32/`); all nodes → same sink IP.
|
||||
- **Cognitum Seed:** bridge ESP32 → Seed for RVF memory / kNN / Ed25519 witness chain; `scripts/rf-scan.js`, `scripts/snn-csi-processor.js`; `docs/tutorials/cognitum-seed-pretraining.md`.
|
||||
|
||||
## Workflow
|
||||
|
||||
1. Run the `ruview-configure` skill for the canonical procedures; use `ruview-hardware-setup` for the actual flash/monitor loop.
|
||||
2. Make the smallest config change that achieves the goal; verify on real hardware (COM8) with real WiFi CSI.
|
||||
3. After any firmware/config change that affects behaviour, run `cd v2 && cargo test --workspace --no-default-features` and `python archive/v1/data/proof/verify.py`, then regenerate the witness bundle if needed (`/ruview-verify`).
|
||||
|
||||
## Ground rules
|
||||
|
||||
- Read before edit. No new files unless required. No secrets / `.env` in commits.
|
||||
- Reference ADR-022, 028, 041, 060, 061, 081; `CLAUDE.md` / `CLAUDE.local.md`; `example.env`.
|
||||
@@ -0,0 +1,28 @@
|
||||
---
|
||||
name: ruview-onboarding-guide
|
||||
description: Walks a newcomer through RuView (WiFi-DensePose) from zero to a working sensing setup — picks the right path (Docker demo / repo build / live ESP32), explains the physics and the hardware caveats, and points to the next steps. Use when someone is new to the project or asks "how do I get started".
|
||||
model: sonnet
|
||||
---
|
||||
|
||||
# RuView Onboarding Guide
|
||||
|
||||
You help people get started with **RuView** — WiFi-based human sensing from Channel State Information (CSI). Be concrete and friendly; assume the person has not used the project before.
|
||||
|
||||
## Your job
|
||||
|
||||
1. **Figure out what they have.** No hardware? → Docker demo. Want to build? → Rust workspace + Python proof. Have an ESP32-S3/C6? → flash + provision + sensing server.
|
||||
2. **Run the `ruview-quickstart` skill** for the canonical steps. For hardware, hand to `ruview-hardware-setup`.
|
||||
3. **Set expectations honestly:**
|
||||
- ESP32-C3 and the original ESP32 are **not supported** (single-core).
|
||||
- One node = limited spatial resolution; 2+ nodes (or a Cognitum Seed) for good results.
|
||||
- Camera-free pose is modest; camera-supervised training reaches 92.9% PCK@20 (ADR-079).
|
||||
- Everything runs on the edge — no cloud, no cameras, no internet required.
|
||||
4. **Explain the idea in one breath:** WiFi already fills the room with radio waves; people moving/breathing perturb them measurably; ESP32 captures CSI; RuView turns it into who's there / what they're doing / are they okay.
|
||||
5. **Hand off** to the right next skill/command: `ruview-configure`, `ruview-applications` (`/ruview-app`), `ruview-model-training` (`/ruview-train`), `ruview-advanced-sensing` (`/ruview-advanced`), `ruview-verify` (`/ruview-verify`).
|
||||
|
||||
## Ground rules
|
||||
|
||||
- Read a file before editing it. Don't create files unless asked.
|
||||
- Don't commit secrets or `.env`.
|
||||
- Use the project's own tooling: `cargo`, `python`, `idf.py` (via the Python-subprocess on Windows — see `CLAUDE.local.md`), `docker`, `node` scripts.
|
||||
- Reference, don't paraphrase: `README.md`, `docs/user-guide.md`, `docs/build-guide.md`, `docs/TROUBLESHOOTING.md`, `docs/tutorials/`, `examples/`.
|
||||
@@ -0,0 +1,40 @@
|
||||
---
|
||||
name: ruview-training-engineer
|
||||
description: Trains, evaluates, and ships RuView models — camera-free WiFlow pose, camera-supervised pose (MediaPipe + ESP32 CSI → 92.9% PCK@20, ADR-079), RuVector contrastive embeddings (AETHER, ADR-024), domain generalization (MERIDIAN, ADR-027), local SNN environment adaptation, GPU training on GCloud, and Hugging Face publishing. Use for any model-building task.
|
||||
model: sonnet
|
||||
---
|
||||
|
||||
# RuView Training Engineer
|
||||
|
||||
You build and ship RuView models. Know the tracks, the data layout, and the validation gate.
|
||||
|
||||
## Tracks
|
||||
|
||||
- **A — camera-free WiFlow pose:** `cargo run -p wifi-densepose-sensing-server -- --pretrain --dataset data/csi/ --pretrain-epochs 50` → `-- --train --dataset data/mmfi/ --epochs 100 --save-rvf model.rvf`. ~84 s on M4 Pro; modest accuracy. Bench: `node scripts/benchmark-wiflow.js`; eval: `node scripts/eval-wiflow.js`.
|
||||
- **B — camera-supervised pose (ADR-079):** `python scripts/collect-ground-truth.py` (MediaPipe), `python scripts/collect-training-data.py` (CSI), `node scripts/align-ground-truth.js`, train on `data/paired/`, eval `eval-wiflow.js` → reports PCK@20. ~19 min on a laptop; 92.9% PCK@20. Needs `data/pose_landmarker_lite.task`.
|
||||
- **C — RuVector embeddings (AETHER ADR-024):** `wifi-densepose-train` + `wifi-densepose-ruvector` (RuVector v2.0.4); `-- --model model.rvf --embed`, `-- --build-index env`. Spectrogram embeddings: ADR-076.
|
||||
- **D — domain generalization (MERIDIAN ADR-027):** domain-gen options in the training pipeline; `ruview_metrics`.
|
||||
- **E — local SNN adaptation:** `node scripts/snn-csi-processor.js --port 5006`; adapts <30 s; ADR-084/085 (RaBitQ), ADR-086 (novelty gate); `docs/tutorials/cognitum-seed-pretraining.md`.
|
||||
|
||||
## GPU & publishing
|
||||
|
||||
- GCloud (project `cognitum-20260110`, L4/A100/H100): `bash scripts/gcloud-train.sh [--dry-run] [--gpu l4|a100|h100] [--hours N] [--config FILE] [--sweep] [--keep-vm]`. VM auto-deletes. Local Mac: `bash scripts/mac-mini-train.sh`. Bench: `python scripts/benchmark-model.py`.
|
||||
- Publish: `python scripts/publish-huggingface.py` (or the `.sh`); `docs/huggingface/`.
|
||||
|
||||
## Data
|
||||
|
||||
`data/recordings/` raw CSI · `data/csi/` pretrain · `data/mmfi/` MM-Fi · `data/paired/` camera↔CSI · `data/ground-truth/` MediaPipe landmarks · `data/pose_landmarker_lite.task` · `models/`. Record more: `python scripts/record-csi-udp.py`.
|
||||
|
||||
## Validation gate (always, after a training change)
|
||||
|
||||
1. `cd v2 && cargo test --workspace --no-default-features` — 1,400+ pass, 0 fail.
|
||||
2. `cd .. && python archive/v1/data/proof/verify.py` — VERDICT: PASS.
|
||||
3. Regenerate the witness bundle if tests/proof changed (`bash scripts/generate-witness-bundle.sh`; self-verify 7/7).
|
||||
|
||||
## Workflow
|
||||
|
||||
Run the `ruview-model-training` skill for canonical commands. Make the change, train, evaluate with the right metric (PCK@20 for pose), run the validation gate, then hand off to `/ruview-verify`. Read before edit; no new files unless required; no secrets in commits.
|
||||
|
||||
## Reference
|
||||
|
||||
ADRs 015, 016, 017, 024, 027, 076, 079, 084, 085, 095, 096; crates `wifi-densepose-train`, `-nn`, `-ruvector`, `-sensing-server`; `CLAUDE.md` build/test section.
|
||||
@@ -0,0 +1,55 @@
|
||||
# AGENTS.md — RuView (WiFi-DensePose)
|
||||
|
||||
Project rules for Codex (and any agent) working in the `ruvnet/RuView` / `wifi-densepose` repo. Mirrors the Claude Code `ruview` plugin.
|
||||
|
||||
## What this repo is
|
||||
|
||||
WiFi-based human sensing from Channel State Information (CSI). Dual codebase: Rust port in `v2/` (15 crates), Python v1 in `archive/v1/`. ESP32-S3 / ESP32-C6 firmware in `firmware/esp32-csi-node/`. 96 ADRs in `docs/adr/`.
|
||||
|
||||
## Hard rules
|
||||
|
||||
- Do exactly what's asked — nothing more, nothing less.
|
||||
- Never create files (especially `*.md`/README) unless required for the task. Prefer editing an existing file.
|
||||
- Never save working files/tests/notes to the repo root — use `v2/crates/`, `tests/`, `docs/`, `scripts/`, `examples/`.
|
||||
- Read a file before editing it.
|
||||
- Never commit secrets, credentials, or `.env`.
|
||||
- Validate user input at system boundaries; sanitize file paths.
|
||||
- ESP32-C3 and the original ESP32 are **not supported** (single-core). Use ESP32-S3 (8MB/4MB) or ESP32-C6.
|
||||
|
||||
## Build & test
|
||||
|
||||
```bash
|
||||
# Rust workspace (1,400+ tests, ~2 min)
|
||||
cd v2 && cargo test --workspace --no-default-features
|
||||
# Single crate, no GPU
|
||||
cargo check -p wifi-densepose-train --no-default-features
|
||||
# Deterministic Python pipeline proof (SHA-256 Trust Kill Switch)
|
||||
python archive/v1/data/proof/verify.py # must print VERDICT: PASS
|
||||
# Python v1 tests
|
||||
cd archive/v1 && python -m pytest tests/ -x -q
|
||||
```
|
||||
|
||||
## ESP32 firmware (Windows)
|
||||
|
||||
ESP-IDF v5.4 does **not** work under Git Bash/MSYS2 and `cmd.exe /C` hangs when called from bash. Build/flash via the **Espressif Python venv as a subprocess with `MSYSTEM*` env vars stripped** — the exact command is in `CLAUDE.local.md`. Default ESP32 serial port: **COM8** (confirm with `mode` / Device Manager — older docs say COM7 or COM9). Provision WiFi: `python firmware/esp32-csi-node/provision.py --port COM8 --ssid ... --password ... --target-ip ... [--channel N] [--filter-mac MAC]`. Serial monitor via pyserial, not `idf.py monitor`. Always test with real WiFi CSI, never mock mode.
|
||||
|
||||
## Witness verification (ADR-028)
|
||||
|
||||
After significant changes: run the Rust tests + Python proof, then `bash scripts/generate-witness-bundle.sh`, then `cd dist/witness-bundle-ADR028-*/ && bash VERIFY.sh` (7/7 PASS). Pre-merge checklist lives in `CLAUDE.md`.
|
||||
|
||||
## Prompt files in `codex/prompts/`
|
||||
|
||||
| Prompt | Purpose |
|
||||
|--------|---------|
|
||||
| `ruview-start` | Onboarding — Docker demo / repo build / live ESP32 |
|
||||
| `ruview-flash` | Build + flash ESP32 firmware (8MB / 4MB) |
|
||||
| `ruview-provision` | Provision WiFi creds + sink IP + channel/MAC overrides |
|
||||
| `ruview-app` | Run a sensing application (presence / vitals / pose / sleep / MAT / point cloud) |
|
||||
| `ruview-train` | Train / evaluate / publish a model (incl. GPU on GCloud) |
|
||||
| `ruview-verify` | Run the trust pipeline + pre-merge checklist |
|
||||
|
||||
Install: copy `codex/prompts/*.md` into `~/.codex/prompts/`, or run Codex with this directory on its prompt path.
|
||||
|
||||
## Reference
|
||||
|
||||
`README.md`, `docs/user-guide.md`, `docs/wifi-mat-user-guide.md`, `docs/build-guide.md`, `docs/TROUBLESHOOTING.md`, `docs/adr/`, `docs/tutorials/`, `examples/`, `CLAUDE.md`, `CLAUDE.local.md`.
|
||||
@@ -0,0 +1,47 @@
|
||||
# RuView prompts for Codex (OpenAI CLI)
|
||||
|
||||
This directory mirrors the Claude Code `ruview` plugin's operator commands as Codex prompts, plus an `AGENTS.md` carrying the RuView project rules.
|
||||
|
||||
## Contents
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `AGENTS.md` | Project rules — repo layout, hard rules, build/test, ESP32 firmware on Windows, witness verification |
|
||||
| `prompts/ruview-start.md` | Onboarding — Docker demo / repo build / live ESP32 |
|
||||
| `prompts/ruview-flash.md` | Build + flash ESP32 firmware (8MB / 4MB) |
|
||||
| `prompts/ruview-provision.md` | Provision WiFi creds + sink IP + channel/MAC overrides |
|
||||
| `prompts/ruview-app.md` | Run a sensing application (presence / vitals / pose / sleep / MAT / point cloud) |
|
||||
| `prompts/ruview-train.md` | Train / evaluate / publish a model (incl. GPU on GCloud) |
|
||||
| `prompts/ruview-advanced.md` | Multistatic / tomography / cross-viewpoint / field-model / mesh-security |
|
||||
| `prompts/ruview-verify.md` | Run the trust pipeline + pre-merge checklist |
|
||||
|
||||
Prompt parity with the Claude Code plugin is enforced by `plugins/ruview/scripts/smoke.sh` (every `commands/<name>.md` must have a matching `codex/prompts/<name>.md`).
|
||||
|
||||
## Install
|
||||
|
||||
**Per-user prompts** — copy the prompt files into Codex's prompt directory:
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.codex/prompts
|
||||
cp plugins/ruview/codex/prompts/*.md ~/.codex/prompts/
|
||||
# now in the codex TUI: /ruview-start /ruview-flash /ruview-app /ruview-train /ruview-verify /ruview-advanced
|
||||
```
|
||||
|
||||
**Project rules** — point Codex at the `AGENTS.md`. Codex auto-discovers an `AGENTS.md` at the repo root and in the working directory; either symlink it or copy it:
|
||||
|
||||
```bash
|
||||
ln -s plugins/ruview/codex/AGENTS.md AGENTS.md # repo root (if you don't already have one)
|
||||
# — or, if a root AGENTS.md exists, append the relevant sections from plugins/ruview/codex/AGENTS.md
|
||||
```
|
||||
|
||||
**Config (optional)** — to keep prompts in-repo instead of `~/.codex/prompts`, add to `~/.codex/config.toml`:
|
||||
|
||||
```toml
|
||||
# Codex reads prompts from ~/.codex/prompts by default; symlinking keeps them versioned with the repo:
|
||||
# ln -s "$PWD/plugins/ruview/codex/prompts" ~/.codex/prompts/ruview (then prompts appear as /ruview/ruview-start, etc.)
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- The Codex mirror is the **operator-facing subset** — the seven `/ruview-*` commands. The Claude Code plugin additionally ships skills (`ruview-quickstart`, `ruview-hardware-setup`, `ruview-configure`, `ruview-applications`, `ruview-model-training`, `ruview-advanced-sensing`, `ruview-cli-api`, `ruview-mmwave`, `ruview-verify`) and agents (`ruview-onboarding-guide`, `ruview-config-engineer`, `ruview-training-engineer`) that have no Codex equivalent — their content is folded into `AGENTS.md` and the prompt files.
|
||||
- On Windows, ESP-IDF firmware builds go through the Python-subprocess pattern documented in `CLAUDE.local.md` (Git Bash / MSYS2 is not supported by ESP-IDF v5.4). Default ESP32 serial port: **COM8**.
|
||||
@@ -0,0 +1,15 @@
|
||||
# /ruview-advanced — advanced RuView capabilities
|
||||
|
||||
Drive RuView's research-grade / multi-node features. Topic: `$ARGUMENTS` (one of `multistatic`, `cross-viewpoint`, `tomography`, `field-model`, `intention`, `adversarial`, `security`; if empty, ask).
|
||||
|
||||
- **multistatic** (ADR-029) — treat every WiFi link in range (incl. neighbours' APs) as a bistatic radar pair, then fuse. `v2/crates/wifi-densepose-signal/src/ruvsense/multistatic.rs` (attention-weighted fusion, geometric diversity), `phase_align.rs` (iterative LO phase-offset, circular mean), `multiband.rs`, `coherence.rs` / `coherence_gate.rs` (Z-score scoring; Accept / PredictOnly / Reject / Recalibrate).
|
||||
- **cross-viewpoint** (ADR-016 viewpoint module) — combine 2+ nodes geometrically. `v2/crates/wifi-densepose-ruvector/src/viewpoint/`: `attention.rs` (CrossViewpointAttention, GeometricBias, softmax with `G_bias`), `geometry.rs` (GeometricDiversityIndex, Cramér–Rao bounds, Fisher Information), `coherence.rs` (phase-phasor coherence, hysteresis gate), `fusion.rs` (MultistaticArray aggregate root). Explore geometry first: `node scripts/mesh-graph-transformer.js`, `node scripts/deep-scan.js`.
|
||||
- **tomography** — `ruvsense/tomography.rs` reconstructs a voxel occupancy grid via an ISTA L1 solver (sparse — most voxels empty); pair with cross-viewpoint geometry for through-wall volumetric imaging. RuVector solver crates back the 114→56 subcarrier sparse interpolation.
|
||||
- **field-model** (ADR-030) — `ruvsense/field_model.rs` builds an SVD eigenstructure of the room, persists it (RVF, ideally on a Cognitum Seed); new frames are projected against it and the residual is the perturbation. Survives restarts; answers "what's different from the empty-room baseline?"
|
||||
- **intention** — `ruvsense/intention.rs`, pre-movement lead signals 200–500 ms ahead.
|
||||
- **adversarial** — `ruvsense/adversarial.rs`, rejects physically impossible signals + cross-checks multi-link consistency.
|
||||
- **security** (ADR-032, multistatic mesh hardening) — using neighbour APs and pooling links across a mesh expands the attack surface. Mitigations: `adversarial.rs` + `coherence_gate.rs` quarantine (Reject / Recalibrate) + Ed25519 witness chain (ADR-028). Run a security review (`docs/security-audit-wasm-edge-vendor.md`); see `/ruview-verify`.
|
||||
|
||||
Also relevant: ADR-031 (sensing-first RF mode), ADR-081 (adaptive CSI mesh firmware kernel), ADR-083 (per-cluster π compute hop), ADR-095/096 (on-ESP32 temporal modeling, sparse GQA).
|
||||
|
||||
Validate: `cd v2 && cargo test -p wifi-densepose-signal --no-default-features && cargo test -p wifi-densepose-ruvector --no-default-features`, then `cd .. && python archive/v1/data/proof/verify.py`.
|
||||
@@ -0,0 +1,13 @@
|
||||
# /ruview-app — run a RuView sensing application
|
||||
|
||||
Run a RuView application. Which one: `$ARGUMENTS` (one of `presence`, `vitals`, `pose`, `sleep`, `environment`, `mat`, `pointcloud`, or a novel-RF app name; if empty, show the catalogue and ask).
|
||||
|
||||
- **presence / vitals / pose / environment** → `cd v2 && cargo run -p wifi-densepose-sensing-server` against a live ESP32 sink, or the Docker demo (`docker run -p 3000:3000 ruvnet/wifi-densepose:latest`) for simulated CSI. For environment also `-- --model model.rvf --build-index env`. Vitals: breathing 6–30 BPM (bandpass 0.1–0.5 Hz), heart rate 40–120 BPM (bandpass 0.8–2.0 Hz), `wifi-densepose-vitals` crate (ADR-021). Pose: 17 COCO keypoints via WiFlow (ADR-059 live pipeline) — train for accuracy (`/ruview-train`).
|
||||
- **sleep** → `examples/sleep/` + `node scripts/apnea-detector.js` (sleep-stage classification, apnea screening).
|
||||
- **mat** (Mass Casualty Assessment — disaster survivor detection) → `wifi-densepose-mat` crate, `docs/wifi-mat-user-guide.md`.
|
||||
- **pointcloud** → `python scripts/mmwave_fusion_bridge.py` (camera depth via MiDaS + WiFi CSI + mmWave radar → unified spatial model, ~22 ms, 19K+ pts/frame; ADR-094).
|
||||
- **novel RF** → `scripts/passive-radar.js`, `material-classifier.js`, `device-fingerprint.js`, `mincut-person-counter.js`, `gait-analyzer.js` (ADR-077/078).
|
||||
|
||||
No hardware? Fall back to the Docker demo or `python examples/ruview_live.py`. Visualisers: `node scripts/csi-spectrogram.js`, `node scripts/csi-graph-visualizer.js`.
|
||||
|
||||
Help me pick: through-wall → presence/activity (≤5 m depth); stationary subject → vitals/sleep; need skeletons → pose (train it); search & rescue → MAT; best spatial accuracy → 2+ ESP32 nodes + cross-viewpoint fusion (`v2/crates/wifi-densepose-ruvector/src/viewpoint/`), optionally + Cognitum Seed. Examples: `examples/{environment,medical,sleep,stress,happiness-vector}/`.
|
||||
@@ -0,0 +1,17 @@
|
||||
# /ruview-flash — build + flash ESP32 firmware
|
||||
|
||||
Build and flash RuView ESP32 firmware. Variant + port: `$ARGUMENTS` (default `8mb`, port `COM8`).
|
||||
|
||||
1. **Variant.** `8mb` → ensure it builds from `firmware/esp32-csi-node/sdkconfig.defaults.template` (no mock — real WiFi CSI). `4mb` → `cp firmware/esp32-csi-node/sdkconfig.defaults.4mb firmware/esp32-csi-node/sdkconfig.defaults` first (display disabled, dual OTA via `partitions_4mb.csv`). `heltec` → `sdkconfig.defaults.heltec_n16r2`.
|
||||
2. **Build (Windows).** ESP-IDF v5.4 does NOT work under Git Bash; `cmd.exe /C` hangs. Use the Espressif Python venv as a subprocess with `MSYSTEM*` env vars stripped — the exact command is in `CLAUDE.local.md` (`[python, idf_py, 'build']`, cwd = `firmware/esp32-csi-node`). Outputs in `firmware/esp32-csi-node/build/{bootloader/bootloader.bin, partition_table/partition-table.bin, esp32-csi-node.bin, ota_data_initial.bin}`.
|
||||
3. **Flash.** Same subprocess with `[python, idf_py, '-p', 'COM8', 'flash']`, or:
|
||||
```
|
||||
python -m esptool --chip esp32s3 --port COM8 --baud 460800 write_flash \
|
||||
0x0 firmware/esp32-csi-node/build/bootloader/bootloader.bin \
|
||||
0x8000 firmware/esp32-csi-node/build/partition_table/partition-table.bin \
|
||||
0xf000 firmware/esp32-csi-node/build/ota_data_initial.bin \
|
||||
0x20000 firmware/esp32-csi-node/build/esp32-csi-node.bin
|
||||
```
|
||||
4. **Confirm.** Serial monitor via pyserial on `COM8` @ 115200 (NOT `idf.py monitor` — it hangs in a subprocess). Then `cd v2 && cargo run -p wifi-densepose-sensing-server` — frames should arrive. If not: re-run `/ruview-provision`, match the AP channel, drop any `--filter-mac`.
|
||||
|
||||
Never test in mock mode — the Kconfig fall-threshold bug only showed up with real CSI.
|
||||
@@ -0,0 +1,25 @@
|
||||
# /ruview-provision — provision an ESP32 sensing node
|
||||
|
||||
Write NVS config to a RuView ESP32 node. Args: `$ARGUMENTS` (expect `--port`, `--ssid`, `--password`, `--target-ip`, optional `--channel`, `--filter-mac`). Default port `COM8`.
|
||||
|
||||
First get the authoritative flag list: `python firmware/esp32-csi-node/provision.py --help` (on Windows prefix `PYTHONUTF8=1 PYTHONIOENCODING=utf-8` — the help text has non-ASCII and crashes under cp1252). Then run:
|
||||
|
||||
```
|
||||
python firmware/esp32-csi-node/provision.py --port COM8 \
|
||||
--ssid "<SSID>" --password "<PW>" --target-ip <SINK_IP> --target-port 5005 --node-id <0-255> \
|
||||
[--channel <N>] [--filter-mac <AA:BB:CC:DD:EE:FF>] [--hop-channels 1,6,11 --hop-dwell 200] \
|
||||
[--tdm-slot <i> --tdm-total <n>] [--edge-tier 0|1|2] [--pres-thresh 50] [--fall-thresh 15000] \
|
||||
[--vital-win 300] [--vital-int 1000] [--subk-count 32] \
|
||||
[--seed-url http://10.1.10.236 --seed-token <bearer> --zone lobby] [--swarm-hb 30] [--swarm-ingest 5] [--dry-run]
|
||||
```
|
||||
|
||||
Trade-offs:
|
||||
- `--channel <N>` pins the node to one WiFi channel (set it to the AP's channel). Omit it and pass `--hop-channels 1,6,11` for the firmware's multi-band hopping schedule (more sensing bandwidth, uses neighbour APs as illuminators; `--hop-dwell` ms per channel).
|
||||
- `--filter-mac <MAC>` restricts CSI capture to one transmitter (cleaner signal); omit for all transmitters (more data, more noise).
|
||||
- `--edge-tier` 0/1/2 = off / stats / vitals (ADR-041). `--tdm-slot`/`--tdm-total` slot a multi-node mesh. `--fall-thresh 15000` ≈ 15.0 rad/s² (raise to cut false falls).
|
||||
|
||||
⚠️ **Issue #391:** flashing rewrites the *entire* `csi_cfg` NVS namespace — every key not on the CLI is erased. Pass the full set you want; warn before re-provisioning a working node. `--dry-run` builds the NVS binary without flashing; `--force-partial` allows config without WiFi creds (knowingly).
|
||||
|
||||
Fleet provisioning: `python scripts/generate_nvs_matrix.py` (subprocess-first — the `esp_idf_nvs_partition_gen` API changed across versions).
|
||||
|
||||
Verify: serial monitor (pyserial on `COM8`, 115200) should show `adaptive_ctrl` ticks + `csi_collector: CSI cb #… len=128 rssi=… ch=…` lines; the sink `cd v2 && cargo run -p wifi-densepose-sensing-server` should report incoming UDP frames if `--target-ip` points at this host. If no frames: wrong channel, MAC filter too tight, target-ip not this host, or WiFi creds wrong — re-run with corrected args.
|
||||
@@ -0,0 +1,11 @@
|
||||
# /ruview-start — onboard onto RuView
|
||||
|
||||
Help me get started with RuView (WiFi-DensePose). Path: `$ARGUMENTS` (one of `docker`, `build`, `hardware`; if empty, ask which hardware I have).
|
||||
|
||||
- **docker** (no hardware): `docker pull ruvnet/wifi-densepose:latest && docker run -p 3000:3000 ruvnet/wifi-densepose:latest`, then open http://localhost:3000 (simulated CSI, full UI).
|
||||
- **build** (from source): `cd v2 && cargo test --workspace --no-default-features`, then `cd .. && python archive/v1/data/proof/verify.py` (expect `VERDICT: PASS`). Single-crate sanity: `cargo check -p wifi-densepose-train --no-default-features`.
|
||||
- **hardware** (ESP32-S3/C6): use `/ruview-flash` then `/ruview-provision`, then `cd v2 && cargo run -p wifi-densepose-sensing-server` to consume the UDP CSI stream. Also: `node scripts/rf-scan.js --port 5006`, `node scripts/snn-csi-processor.js --port 5006`.
|
||||
|
||||
Warn me about: ESP32-C3 / original ESP32 are unsupported (single-core); one node = limited spatial resolution (use 2+ or add a Cognitum Seed); camera-free pose is modest — camera-supervised training reaches 92.9% PCK@20 (ADR-079); no cloud/cameras/internet needed.
|
||||
|
||||
Then point me at next steps: `/ruview-app`, `/ruview-train`, `/ruview-verify`, and the configuration workflow (sdkconfig variants, NVS provisioning, edge modules, mesh, Cognitum Seed). Reference `README.md`, `docs/user-guide.md`, `docs/build-guide.md`, `docs/TROUBLESHOOTING.md`, `examples/`.
|
||||
@@ -0,0 +1,14 @@
|
||||
# /ruview-train — train a RuView model
|
||||
|
||||
Train / evaluate / publish a RuView model. Track: `$ARGUMENTS` (one of `camera-free`, `camera-supervised`, `embeddings`, `domain-gen`, `snn`, `gpu`; if empty, ask).
|
||||
|
||||
- **camera-free** (WiFlow pose, no labels): `cd v2 && cargo run -p wifi-densepose-sensing-server -- --pretrain --dataset data/csi/ --pretrain-epochs 50`, then `-- --train --dataset data/mmfi/ --epochs 100 --save-rvf model.rvf`. ~84 s on M4 Pro, modest accuracy. Bench `node scripts/benchmark-wiflow.js`, eval `node scripts/eval-wiflow.js`.
|
||||
- **camera-supervised** (ADR-079, 92.9% PCK@20, ~19 min): `python scripts/collect-ground-truth.py` (MediaPipe landmarks; needs `data/pose_landmarker_lite.task`), `python scripts/collect-training-data.py` (CSI capture), `node scripts/align-ground-truth.js` (timestamp align), then `cd v2 && cargo run -p wifi-densepose-sensing-server -- --train --dataset data/paired/ --epochs <N> --save-rvf model.rvf`, eval `node scripts/eval-wiflow.js` (reports PCK@20).
|
||||
- **embeddings** (AETHER ADR-024 / spectrogram ADR-076): `wifi-densepose-train` + `wifi-densepose-ruvector`; `-- --model model.rvf --embed`, `-- --model model.rvf --build-index env`. 171K emb/s on M4 Pro.
|
||||
- **domain-gen** (MERIDIAN ADR-027): domain-generalization options in the training pipeline + `ruview_metrics`.
|
||||
- **snn** (local env adaptation, <30 s): `node scripts/snn-csi-processor.js --port 5006`; `docs/tutorials/cognitum-seed-pretraining.md`; ADR-084/085 (RaBitQ), ADR-086 (novelty gate).
|
||||
- **gpu**: `gcloud auth login && gcloud config set project cognitum-20260110`, then `bash scripts/gcloud-train.sh --dry-run` (smoke), `bash scripts/gcloud-train.sh --gpu l4 --hours 2` (proto, ~$0.80/hr), `bash scripts/gcloud-train.sh --gpu a100 --config scripts/training-config-sweep.json` (~$3.60/hr), `bash scripts/gcloud-train.sh --sweep` (full sweep). VM auto-deletes unless `--keep-vm`. Local Mac: `bash scripts/mac-mini-train.sh`. Bench: `python scripts/benchmark-model.py`.
|
||||
|
||||
Data: `data/recordings/` raw CSI · `data/csi/` pretrain · `data/mmfi/` MM-Fi · `data/paired/` camera↔CSI · `data/ground-truth/` MediaPipe · `models/` artifacts. Record more: `python scripts/record-csi-udp.py`.
|
||||
|
||||
After training: `cd v2 && cargo test --workspace --no-default-features`, `cd .. && python archive/v1/data/proof/verify.py` (VERDICT: PASS). Publish: `python scripts/publish-huggingface.py` (or `.sh`; `docs/huggingface/`). Then run `/ruview-verify`.
|
||||
@@ -0,0 +1,12 @@
|
||||
# /ruview-verify — run the RuView trust pipeline
|
||||
|
||||
Verify a RuView build. Scope: `$ARGUMENTS` (one of `tests`, `proof`, `bundle`, `all`; default `all`).
|
||||
|
||||
1. **tests** — `cd v2 && cargo test --workspace --no-default-features` → must be 1,400+ passed, 0 failed (~2 min). Single-crate: `cargo test -p wifi-densepose-signal --no-default-features`, etc.
|
||||
2. **proof** — `cd .. && python archive/v1/data/proof/verify.py` → must print `VERDICT: PASS`. If a hash mismatch from a legitimate numpy/scipy bump: `python archive/v1/data/proof/verify.py --generate-hash`, then re-run. Optional: `cd archive/v1 && python -m pytest tests/ -x -q`.
|
||||
3. **bundle** — `bash scripts/generate-witness-bundle.sh` produces `dist/witness-bundle-ADR028-<sha>.tar.gz` (WITNESS-LOG-028.md, ADR-028 audit, proof, rust test log, firmware hash manifest, crate versions, VERIFY.sh). Then `cd dist/witness-bundle-ADR028-*/ && bash VERIFY.sh` → must be 7/7 PASS.
|
||||
4. **all** — do 1→3 in order.
|
||||
|
||||
If this follows a code change, walk the pre-merge checklist from `CLAUDE.md`: Rust tests pass; Python proof passes; README updated if scope changed; CLAUDE.md updated if scope changed; CHANGELOG `[Unreleased]` entry; `docs/user-guide.md` updated if new data sources/CLI flags/setup; ADR count bumped in README if a new ADR added; witness bundle regenerated if tests/proof hash changed; Docker image rebuilt only if Dockerfile/deps/runtime changed; crate publishing only if a published crate's public API changed (publish in dependency order — see CLAUDE.md); `.gitignore` updated for new artifacts; security review for new hardware/network-boundary modules.
|
||||
|
||||
For security-related changes also run `npx @claude-flow/cli@latest security scan`. QEMU firmware CI (ADR-061): local helpers `scripts/qemu-esp32s3-test.sh`, `qemu-mesh-test.sh`, `qemu-chaos-test.sh`, `install-qemu.sh`.
|
||||
@@ -0,0 +1,19 @@
|
||||
---
|
||||
description: Use advanced RuView capabilities — multistatic sensing, cross-viewpoint fusion, RF tomography, persistent field model, intention signals, adversarial detection, mesh security.
|
||||
argument-hint: "[multistatic|cross-viewpoint|tomography|field-model|intention|adversarial|security]"
|
||||
---
|
||||
|
||||
# /ruview-advanced
|
||||
|
||||
Drive RuView's research-grade / multi-node features.
|
||||
|
||||
1. Invoke the **`ruview-advanced-sensing`** skill.
|
||||
2. Route on `$ARGUMENTS`:
|
||||
- **multistatic** (ADR-029) — `wifi-densepose-signal/src/ruvsense/multistatic.rs`, `phase_align.rs`, `coherence_gate.rs`; neighbours' APs as illuminators.
|
||||
- **cross-viewpoint** (ADR-016 viewpoint) — `wifi-densepose-ruvector/src/viewpoint/`; needs 2+ nodes; `node scripts/mesh-graph-transformer.js`.
|
||||
- **tomography** — `ruvsense/tomography.rs` (ISTA L1 voxel solver) + cross-viewpoint geometry; through-wall volumetric.
|
||||
- **field-model** (ADR-030) — `ruvsense/field_model.rs`, SVD room eigenstructure persisted to RVF (Cognitum Seed); residual = perturbation.
|
||||
- **intention** — `ruvsense/intention.rs`, 200–500 ms pre-movement lead signals.
|
||||
- **adversarial** — `ruvsense/adversarial.rs`, physically-impossible-signal + multi-link consistency checks.
|
||||
- **security** (ADR-032) — mesh hardening: adversarial gate + coherence quarantine + Ed25519 witness chain; run a security review (`docs/security-audit-wasm-edge-vendor.md`), see `/ruview-verify`.
|
||||
3. Validate: `cd v2 && cargo test -p wifi-densepose-signal --no-default-features && cargo test -p wifi-densepose-ruvector --no-default-features`, then `python archive/v1/data/proof/verify.py`.
|
||||
@@ -0,0 +1,18 @@
|
||||
---
|
||||
description: Run a RuView sensing application — presence, vitals, pose, sleep, environment mapping, MAT, point cloud, or a novel RF app.
|
||||
argument-hint: "[presence|vitals|pose|sleep|environment|mat|pointcloud|<name>]"
|
||||
---
|
||||
|
||||
# /ruview-app
|
||||
|
||||
Launch a RuView application.
|
||||
|
||||
1. Invoke the **`ruview-applications`** skill.
|
||||
2. Map `$ARGUMENTS` to an application; if empty, show the catalogue and ask. Quick mappings:
|
||||
- `presence` / `vitals` / `pose` / `environment` → `cd v2 && cargo run -p wifi-densepose-sensing-server` (live ESP32 sink) or the Docker demo for simulated CSI; for environment also `--build-index env`.
|
||||
- `sleep` → `examples/sleep/` + `node scripts/apnea-detector.js`.
|
||||
- `mat` (Mass Casualty Assessment) → `wifi-densepose-mat` crate, `docs/wifi-mat-user-guide.md`.
|
||||
- `pointcloud` → `python scripts/mmwave_fusion_bridge.py` (camera depth + CSI + mmWave).
|
||||
- novel RF → `scripts/passive-radar.js`, `material-classifier.js`, `device-fingerprint.js`, `mincut-person-counter.js`.
|
||||
3. If no hardware: fall back to `docker run -p 3000:3000 ruvnet/wifi-densepose:latest` or `python examples/ruview_live.py`.
|
||||
4. Help pick the right modality (through-wall → presence/activity; stationary subject → vitals/sleep; need skeletons → pose, train it for accuracy; search & rescue → MAT; best accuracy → 2+ nodes + cross-viewpoint fusion via `/ruview-advanced`).
|
||||
@@ -0,0 +1,15 @@
|
||||
---
|
||||
description: Build and flash RuView ESP32 firmware (8MB or 4MB), then confirm the CSI stream.
|
||||
argument-hint: "[8mb|4mb] [COM port]"
|
||||
---
|
||||
|
||||
# /ruview-flash
|
||||
|
||||
Build + flash RuView firmware to an ESP32-S3 sensing node.
|
||||
|
||||
1. Invoke the **`ruview-hardware-setup`** skill.
|
||||
2. Determine variant from `$ARGUMENTS` (default `8mb`). For `4mb`: `cp firmware/esp32-csi-node/sdkconfig.defaults.4mb firmware/esp32-csi-node/sdkconfig.defaults` first. For `8mb`: ensure it's built from `sdkconfig.defaults.template` (no mock).
|
||||
3. Build using the **Python-subprocess** command from `CLAUDE.local.md` (ESP-IDF v5.4 does NOT work under Git Bash — strip `MSYSTEM*` env vars). Never use `cmd.exe /C` from bash.
|
||||
4. Flash: same subprocess, `[python, idf_py, '-p', '<COM port>', 'flash']` (default port **COM8**), or `python -m esptool ... write_flash ...` with the four binaries.
|
||||
5. Confirm: serial monitor via pyserial (not `idf.py monitor`), then `cd v2 && cargo run -p wifi-densepose-sensing-server` to see frames arrive.
|
||||
6. If no frames: re-run `/ruview-provision`, check channel matches the AP, drop any `--filter-mac`.
|
||||
@@ -0,0 +1,24 @@
|
||||
---
|
||||
description: Provision WiFi credentials, sink IP, and optional channel / MAC-filter overrides onto a RuView ESP32 node.
|
||||
argument-hint: "--port COM8 --ssid ... --password ... --target-ip ... [--channel N] [--filter-mac AA:BB:..]"
|
||||
---
|
||||
|
||||
# /ruview-provision
|
||||
|
||||
Write NVS config to an ESP32 sensing node.
|
||||
|
||||
1. Invoke the **`ruview-configure`** skill (§"Runtime device config" — has the full `provision.py` flag table).
|
||||
2. Run `python firmware/esp32-csi-node/provision.py --help` for the authoritative options (on Windows: `PYTHONUTF8=1 PYTHONIOENCODING=utf-8 python …` — the help text has non-ASCII). Collect any missing params (port — default **COM8**, SSID, password, target sink IP, `--target-port` default 5005, `--node-id`).
|
||||
3. Run:
|
||||
```bash
|
||||
python firmware/esp32-csi-node/provision.py --port <PORT> \
|
||||
--ssid "<SSID>" --password "<PW>" --target-ip <IP> --target-port 5005 --node-id <0-255> \
|
||||
[--channel <N>] [--filter-mac <MAC>] [--hop-channels 1,6,11 --hop-dwell 200] \
|
||||
[--tdm-slot <i> --tdm-total <n>] [--edge-tier {0|1|2}] [--pres-thresh 50] [--fall-thresh 15000] \
|
||||
[--vital-win 300] [--vital-int 1000] [--subk-count 32] \
|
||||
[--seed-url http://… --seed-token … --zone lobby] [--swarm-hb 30] [--swarm-ingest 5] [--dry-run]
|
||||
```
|
||||
4. Explain trade-offs: `--channel` pins the node (AP's channel) vs. `--hop-channels` for ADR-061 multi-freq hopping; `--filter-mac` restricts to one transmitter vs. omit for all (more data, more noise); `--edge-tier` 0/1/2 = off/stats/vitals; `--tdm-slot`/`--tdm-total` slot a multi-node mesh.
|
||||
5. ⚠️ **Issue #391**: flashing rewrites the *entire* `csi_cfg` NVS namespace — every key not on the CLI is erased. Pass the full set you want; warn the user before re-provisioning a working node. `--force-partial` bypasses the WiFi-creds requirement (knowingly). `--dry-run` builds the NVS binary without flashing.
|
||||
6. Fleet provisioning: `scripts/generate_nvs_matrix.py` (subprocess-first).
|
||||
7. Verify: serial monitor (pyserial on the port, 115200) should show `adaptive_ctrl` ticks + `csi_collector: CSI cb #… len=128 …` lines; the sink (`cd v2 && cargo run -p wifi-densepose-sensing-server`) should report incoming UDP frames if `--target-ip` points at this host.
|
||||
@@ -0,0 +1,16 @@
|
||||
---
|
||||
description: Get started with RuView — pick the fastest path (Docker demo, repo build, or live ESP32) and walk through it.
|
||||
argument-hint: "[docker|build|hardware]"
|
||||
---
|
||||
|
||||
# /ruview-start
|
||||
|
||||
Onboard the user onto RuView (WiFi-DensePose).
|
||||
|
||||
1. Invoke the **`ruview-quickstart`** skill.
|
||||
2. If `$ARGUMENTS` names a tier (`docker`, `build`, `hardware`), go straight to it; otherwise ask which hardware they have:
|
||||
- **No hardware** → Tier 0: `docker run -p 3000:3000 ruvnet/wifi-densepose:latest`, open `http://localhost:3000`.
|
||||
- **Want to build from source** → Tier 1: `cd v2 && cargo test --workspace --no-default-features`, then `python archive/v1/data/proof/verify.py`.
|
||||
- **Have an ESP32-S3 / C6** → Tier 2: hand off to `/ruview-flash` then `/ruview-provision`, then `cargo run -p wifi-densepose-sensing-server`.
|
||||
3. Warn about the gotchas: ESP32-C3 / original ESP32 unsupported; single node = limited spatial resolution; camera-free pose is modest (use camera-supervised for 92.9% PCK@20).
|
||||
4. Point to next steps: `/ruview-app`, `/ruview-train`, `/ruview-advanced`, `/ruview-verify`, and the `ruview-configure` skill.
|
||||
@@ -0,0 +1,18 @@
|
||||
---
|
||||
description: Train a RuView model — camera-free WiFlow pose, camera-supervised pose (92.9% PCK@20), RuVector embeddings, domain generalization, local SNN, with optional GPU on GCloud.
|
||||
argument-hint: "[camera-free|camera-supervised|embeddings|domain-gen|snn|gpu] [--epochs N]"
|
||||
---
|
||||
|
||||
# /ruview-train
|
||||
|
||||
Train, fine-tune, evaluate, or publish a RuView model.
|
||||
|
||||
1. Invoke the **`ruview-model-training`** skill.
|
||||
2. Pick the track from `$ARGUMENTS`; if empty, ask which:
|
||||
- **camera-free** (Track A) — `cargo run -p wifi-densepose-sensing-server -- --pretrain --dataset data/csi/ --pretrain-epochs 50` then `-- --train --dataset data/mmfi/ --epochs 100 --save-rvf model.rvf`. ~84 s on M4 Pro, modest accuracy.
|
||||
- **camera-supervised** (Track B, ADR-079) — `python scripts/collect-ground-truth.py`, `python scripts/collect-training-data.py`, `node scripts/align-ground-truth.js`, then train on `data/paired/`, eval with `node scripts/eval-wiflow.js`. ~19 min, 92.9% PCK@20. Needs `data/pose_landmarker_lite.task`.
|
||||
- **embeddings** (Track C, AETHER ADR-024) — `wifi-densepose-train` + `wifi-densepose-ruvector`; `-- --model model.rvf --embed`, `-- --build-index env`.
|
||||
- **domain-gen** (Track D, MERIDIAN ADR-027) / **snn** (Track E) — `node scripts/snn-csi-processor.js --port 5006`; cognitum-seed-pretraining tutorial.
|
||||
- **gpu** — `gcloud config set project cognitum-20260110`; `bash scripts/gcloud-train.sh --gpu l4 --hours 2` (or `--gpu a100 --sweep`, `--dry-run` to smoke-test). VM auto-deletes unless `--keep-vm`.
|
||||
3. After training: `cd v2 && cargo test --workspace --no-default-features`, `python archive/v1/data/proof/verify.py`. To publish: `python scripts/publish-huggingface.py`.
|
||||
4. Hand off to `/ruview-verify` for the witness bundle.
|
||||
@@ -0,0 +1,17 @@
|
||||
---
|
||||
description: Verify a RuView build — Rust tests, deterministic Python proof, firmware hashes, ADR-028 witness bundle + self-verification, and the pre-merge checklist.
|
||||
argument-hint: "[tests|proof|bundle|all]"
|
||||
---
|
||||
|
||||
# /ruview-verify
|
||||
|
||||
Run RuView's trust pipeline.
|
||||
|
||||
1. Invoke the **`ruview-verify`** skill.
|
||||
2. Based on `$ARGUMENTS` (default `all`):
|
||||
- **tests** — `cd v2 && cargo test --workspace --no-default-features` (1,400+ pass, 0 fail).
|
||||
- **proof** — `python archive/v1/data/proof/verify.py` (must print `VERDICT: PASS`; if hash drift from a legit numpy/scipy bump, `--generate-hash` then re-run). Optionally `cd archive/v1 && python -m pytest tests/ -x -q`.
|
||||
- **bundle** — `bash scripts/generate-witness-bundle.sh`, then `cd dist/witness-bundle-ADR028-*/ && bash VERIFY.sh` (must be 7/7 PASS).
|
||||
- **all** — do all of the above in order.
|
||||
3. If this follows a code change, walk the **pre-merge checklist** from `CLAUDE.md` (README/CLAUDE.md/CHANGELOG/user-guide updates, ADR count, witness bundle regen, Docker rebuild only if needed, crate publishing in dependency order, `.gitignore`, security review for hardware/network modules).
|
||||
4. For security-related changes also run `npx @claude-flow/cli@latest security scan`.
|
||||
@@ -0,0 +1,43 @@
|
||||
# ADR-0001 — ruview plugin contract
|
||||
|
||||
- **Status:** Proposed
|
||||
- **Date:** 2026-05-11
|
||||
- **Scope:** `plugins/ruview` (and the repo-root `.claude-plugin/marketplace.json` that lists it)
|
||||
|
||||
## Context
|
||||
|
||||
RuView (WiFi-DensePose) is a large dual-codebase project (Rust `v2/`, Python `archive/v1/`, ESP32 firmware, 96 ADRs). Newcomers and operators repeatedly re-derive the same workflows: spin up the Docker demo, flash and provision an ESP32, run a sensing application, train a pose model, run the witness verification. We want those workflows packaged as a single discoverable Claude Code plugin (and mirrored for Codex), spanning practical → advanced.
|
||||
|
||||
## Decision
|
||||
|
||||
1. **One mega-plugin, marketplace-listed from the repo root.** A single plugin `ruview` under `plugins/ruview/`, listed by `.claude-plugin/marketplace.json` **at the repo root** (marketplace name `ruview`, plugin `source: "./plugins/ruview"`). The manifest sits at the repo root so `claude plugin marketplace add ruvnet/RuView` (and `/plugin marketplace add ruvnet/RuView` in Claude Code) resolve it — Claude Code looks for `.claude-plugin/marketplace.json` at the cloned repo's root, not in subdirectories. No sub-plugins; the breadth is organized by skill instead.
|
||||
|
||||
2. **Directory contract.**
|
||||
```
|
||||
.claude-plugin/marketplace.json # REPO ROOT — marketplace name `ruview`, plugin source ./plugins/ruview
|
||||
plugins/ruview/.claude-plugin/plugin.json # name, description, version, author, homepage, license, keywords — NO skills/commands/agents arrays
|
||||
plugins/ruview/skills/<name>/SKILL.md # frontmatter: name, description, allowed-tools
|
||||
plugins/ruview/commands/<name>.md # frontmatter: description (+ argument-hint)
|
||||
plugins/ruview/agents/<name>.md # frontmatter: name, description, model
|
||||
plugins/ruview/docs/adrs/0001-ruview-plugin-contract.md
|
||||
plugins/ruview/scripts/smoke.sh # structural contract
|
||||
plugins/ruview/codex/AGENTS.md + codex/README.md + codex/prompts/*.md # Codex mirror
|
||||
plugins/ruview/README.md # Compatibility + Namespace coordination + Verification + ADR sections
|
||||
```
|
||||
Skills/commands/agents are **auto-discovered** from the directory tree — they are deliberately *not* enumerated in `plugin.json`.
|
||||
|
||||
3. **Shell-first skills.** Skills drive RuView's own tooling — `cargo`, `python`, `idf.py` (via the Windows Python-subprocess pattern in `CLAUDE.local.md`), `docker`, `node` scripts. `allowed-tools` is limited to core tools (`Bash Read Write Edit Glob Grep`); **no `mcp__claude-flow__*` dependency** and **no wildcard tools**. The only external CLI referenced is `npx @claude-flow/cli@latest security scan`, and only as an optional step for security changes.
|
||||
|
||||
4. **Namespace.** The plugin claims the `ruview-*` namespace for skills (`ruview-quickstart`, `ruview-hardware-setup`, `ruview-configure`, `ruview-applications`, `ruview-model-training`, `ruview-advanced-sensing`, `ruview-cli-api`, `ruview-mmwave`, `ruview-verify`), commands (`/ruview-*`), and agents (`ruview-*`). It writes to no `claude-flow` memory namespace. Coexists with the `ruflo` marketplace with zero overlap (`ruview-*` vs. `ruflo-*`); if both are present, defer to `ruflo-agentdb` ADR-0001 §"Namespace convention".
|
||||
|
||||
5. **Codex mirror — full command parity.** Every `/ruview-*` command (`ruview-start`, `ruview-flash`, `ruview-provision`, `ruview-app`, `ruview-train`, `ruview-advanced`, `ruview-verify`) has a matching `codex/prompts/<name>.md`; `codex/AGENTS.md` carries the project rules and `codex/README.md` documents installation. The mirror covers the operator-facing **commands** in full; the additional **skills** (`ruview-quickstart`, `ruview-hardware-setup`, `ruview-configure`, `ruview-applications`, `ruview-model-training`, `ruview-advanced-sensing`, `ruview-cli-api`, `ruview-mmwave`, `ruview-verify`) and **agents** have no Codex equivalent — their knowledge is folded into `AGENTS.md` and the prompt files. The smoke script enforces command↔prompt parity.
|
||||
|
||||
6. **Compatibility surface.** Targets the `ruvnet/RuView` / `wifi-densepose` repo layout (`v2/crates/`, `firmware/esp32-csi-node/`, `archive/v1/`, `scripts/`, `docs/adr/`). Hardware docs default to ESP32 on `COM8` and tell the reader to confirm the port.
|
||||
|
||||
7. **Smoke contract** (`scripts/smoke.sh`, ≥13 checks): repo-root `.claude-plugin/marketplace.json` exists + lists `ruview` + points `source` at `./plugins/ruview`; plugin.json has `name`/`description`/`version`/`keywords` and does **not** contain `skills`/`commands`/`agents` arrays; every `skills/*/SKILL.md` has `name` + `description` + `allowed-tools`; no wildcard (`*`) in any `allowed-tools`; the expected skill set is present; every `commands/*.md` has a `description`; every `agents/*.md` has `name` + `description` + `model`; README contains a `## Compatibility` section and a `Namespace coordination` block; this ADR exists with `Status: Proposed`; `codex/AGENTS.md` and `codex/prompts/*.md` exist **and** every `commands/<name>.md` has a matching `codex/prompts/<name>.md` (command↔prompt parity); nothing is misplaced under `.claude-plugin/`.
|
||||
|
||||
## Consequences
|
||||
|
||||
- **Good:** `/plugin marketplace add ruvnet/RuView` + `/plugin install ruview@ruview` (or `claude --plugin-dir ./plugins/ruview` from a clone) gives newcomers and operators the whole RuView workflow surface; no MCP-server prerequisite; Codex users get the same operator commands; the smoke script makes drift visible.
|
||||
- **Cost:** a mega-plugin means coarser install granularity (you get all 9 skills or none); the Codex mirror must be kept in sync by hand (the smoke script checks command↔prompt *presence* parity, not content parity); a skill stem (`ruview-verify`) collides with a command stem — tolerated by Claude Code (both resolve), but `claude plugin details` lists it twice.
|
||||
- **Follow-ups:** if the skill set grows past comfortable browsing (it's at 9), revisit the "one mega-plugin" decision and split by lifecycle (`ruview-edge`, `ruview-train`, …); add a *content*-parity lint between commands and Codex prompts; consider renaming `/ruview-verify` to drop the skill/command stem collision; consider pinning a tested `claude-flow` CLI minor for the security-scan step if that step becomes load-bearing; verify the underlying RuView command flags (`sensing-server --help`, `gcloud-train.sh`, `provision.py`) against the live tree rather than from README/scripts.
|
||||
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env bash
|
||||
# Structural smoke test for the `ruview` Claude Code plugin.
|
||||
# Run from anywhere: bash plugins/ruview/scripts/smoke.sh
|
||||
set -u
|
||||
|
||||
# Resolve plugin root (this file lives in <root>/scripts/smoke.sh).
|
||||
# Plugin lives at <repo>/plugins/ruview ; marketplace manifest is at <repo>/.claude-plugin/marketplace.json
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
REPO="$(cd "$ROOT/../.." && pwd)"
|
||||
MARKET="$REPO/.claude-plugin/marketplace.json"
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
ok() { echo " PASS $1"; PASS=$((PASS+1)); }
|
||||
bad() { echo " FAIL $1"; FAIL=$((FAIL+1)); }
|
||||
has() { grep -q "$1" "$2" 2>/dev/null; }
|
||||
|
||||
echo "ruview plugin smoke test"
|
||||
echo "root: $ROOT"
|
||||
echo "repo: $REPO"
|
||||
echo
|
||||
|
||||
# 1. repo-root marketplace.json exists, lists the ruview plugin, points source at ./plugins/ruview
|
||||
if [ -f "$MARKET" ] && has '"ruview"' "$MARKET" && has '"\./plugins/ruview"' "$MARKET"; then ok "repo-root .claude-plugin/marketplace.json lists 'ruview' with source ./plugins/ruview"; else bad "marketplace.json missing / wrong location / wrong source ($MARKET)"; fi
|
||||
|
||||
# 2. plugin.json exists with required fields
|
||||
PJ="$ROOT/.claude-plugin/plugin.json"
|
||||
if [ -f "$PJ" ] && has '"name"' "$PJ" && has '"description"' "$PJ" && has '"version"' "$PJ"; then ok "plugin.json has name/description/version"; else bad "plugin.json missing or incomplete"; fi
|
||||
|
||||
# 3. plugin.json has keywords
|
||||
if has '"keywords"' "$PJ"; then ok "plugin.json has keywords"; else bad "plugin.json missing keywords"; fi
|
||||
|
||||
# 4. plugin.json does NOT enumerate skills/commands/agents (auto-discovered)
|
||||
if has '"skills"' "$PJ" || has '"commands"' "$PJ" || has '"agents"' "$PJ"; then bad "plugin.json must NOT contain skills/commands/agents arrays"; else ok "plugin.json does not enumerate skills/commands/agents"; fi
|
||||
|
||||
# 5. every skill has SKILL.md with name + description + allowed-tools, and no wildcard tools
|
||||
SKILL_OK=1
|
||||
for d in "$ROOT"/skills/*/; do
|
||||
[ -d "$d" ] || continue
|
||||
f="$d/SKILL.md"
|
||||
if [ ! -f "$f" ]; then bad "missing $f"; SKILL_OK=0; continue; fi
|
||||
has '^name:' "$f" || { bad "$f missing 'name:'"; SKILL_OK=0; }
|
||||
has '^description:' "$f" || { bad "$f missing 'description:'"; SKILL_OK=0; }
|
||||
has '^allowed-tools:' "$f" || { bad "$f missing 'allowed-tools:'"; SKILL_OK=0; }
|
||||
if grep -E '^allowed-tools:.*(\*|\ball tools\b)' "$f" >/dev/null 2>&1; then bad "$f uses wildcard tools"; SKILL_OK=0; fi
|
||||
done
|
||||
[ "$SKILL_OK" = 1 ] && ok "all skills have valid frontmatter, no wildcard tools"
|
||||
|
||||
# 6. expected skills present
|
||||
EXPECTED_SKILLS="ruview-quickstart ruview-hardware-setup ruview-configure ruview-applications ruview-model-training ruview-advanced-sensing ruview-cli-api ruview-mmwave ruview-verify"
|
||||
SKILLS_PRESENT=1
|
||||
for s in $EXPECTED_SKILLS; do
|
||||
[ -f "$ROOT/skills/$s/SKILL.md" ] || { bad "expected skill missing: $s"; SKILLS_PRESENT=0; }
|
||||
done
|
||||
[ "$SKILLS_PRESENT" = 1 ] && ok "expected skill set present ($(echo $EXPECTED_SKILLS | wc -w) skills)"
|
||||
|
||||
# 7. every command has a description in frontmatter
|
||||
CMD_OK=1
|
||||
for f in "$ROOT"/commands/*.md; do
|
||||
[ -f "$f" ] || { bad "no command files found"; CMD_OK=0; break; }
|
||||
has '^description:' "$f" || { bad "$f missing 'description:'"; CMD_OK=0; }
|
||||
done
|
||||
[ "$CMD_OK" = 1 ] && ok "all commands have a description"
|
||||
|
||||
# 8. every agent has name + description + model
|
||||
AG_OK=1
|
||||
for f in "$ROOT"/agents/*.md; do
|
||||
[ -f "$f" ] || { bad "no agent files found"; AG_OK=0; break; }
|
||||
has '^name:' "$f" || { bad "$f missing 'name:'"; AG_OK=0; }
|
||||
has '^description:' "$f" || { bad "$f missing 'description:'"; AG_OK=0; }
|
||||
has '^model:' "$f" || { bad "$f missing 'model:'"; AG_OK=0; }
|
||||
done
|
||||
[ "$AG_OK" = 1 ] && ok "all agents have name/description/model"
|
||||
|
||||
# 9. README has Compatibility + Namespace coordination
|
||||
RM="$ROOT/README.md"
|
||||
if has '## Compatibility' "$RM" && has 'Namespace coordination' "$RM"; then ok "README has Compatibility + Namespace coordination"; else bad "README missing Compatibility or Namespace coordination section"; fi
|
||||
|
||||
# 10. ADR-0001 exists with Status: Proposed
|
||||
ADR="$ROOT/docs/adrs/0001-ruview-plugin-contract.md"
|
||||
if [ -f "$ADR" ] && grep -qi 'Status:.*Proposed' "$ADR"; then ok "ADR-0001 present with Status: Proposed"; else bad "ADR-0001 missing or not 'Proposed'"; fi
|
||||
|
||||
# 11. Codex mirror present
|
||||
if [ -f "$ROOT/codex/AGENTS.md" ] && ls "$ROOT"/codex/prompts/*.md >/dev/null 2>&1; then ok "Codex mirror present (AGENTS.md + prompts/)"; else bad "Codex mirror missing"; fi
|
||||
|
||||
# 11b. command <-> Codex prompt parity
|
||||
PARITY=1
|
||||
for f in "$ROOT"/commands/*.md; do
|
||||
[ -f "$f" ] || continue
|
||||
base="$(basename "$f")"
|
||||
[ -f "$ROOT/codex/prompts/$base" ] || { bad "no Codex prompt for command $base"; PARITY=0; }
|
||||
done
|
||||
[ "$PARITY" = 1 ] && ok "every command has a matching Codex prompt"
|
||||
|
||||
# 12. no skills/commands/agents accidentally placed inside .claude-plugin/
|
||||
if ls "$ROOT"/.claude-plugin/skills "$ROOT"/.claude-plugin/commands "$ROOT"/.claude-plugin/agents >/dev/null 2>&1; then bad "skills/commands/agents must not live under .claude-plugin/"; else ok ".claude-plugin/ contains only plugin.json"; fi
|
||||
|
||||
echo
|
||||
echo "----------------------------------------"
|
||||
echo "PASS: $PASS FAIL: $FAIL"
|
||||
[ "$FAIL" -eq 0 ] || exit 1
|
||||
@@ -0,0 +1,76 @@
|
||||
---
|
||||
name: ruview-advanced-sensing
|
||||
description: Advanced RuView capabilities — RuvSense multistatic sensing (attention-weighted fusion, geometric diversity, persistent field model), cross-viewpoint fusion across multiple nodes, RF tomography (ISTA L1 solver, voxel grids), longitudinal biomechanics drift, pre-movement intention signals, adversarial signal detection, and multistatic mesh security hardening. Use for research-grade or multi-node deployments.
|
||||
allowed-tools: Bash Read Write Edit Glob Grep
|
||||
---
|
||||
|
||||
# RuView Advanced Sensing
|
||||
|
||||
The deep end: multistatic mesh, tomography, persistent field models, and the security model that protects them. Most of this lives in `wifi-densepose-signal/src/ruvsense/` (14 modules) and `wifi-densepose-ruvector/src/viewpoint/` (5 modules).
|
||||
|
||||
## RuvSense multistatic mode (ADR-029)
|
||||
|
||||
Treat every WiFi link in range — including neighbours' APs — as a bistatic radar pair, then fuse them.
|
||||
|
||||
| Module (`signal/src/ruvsense/`) | Purpose |
|
||||
|--------------------------------|---------|
|
||||
| `multiband.rs` | Multi-band CSI frame fusion, cross-channel coherence |
|
||||
| `phase_align.rs` | Iterative LO phase-offset estimation, circular mean |
|
||||
| `multistatic.rs` | Attention-weighted fusion, geometric diversity |
|
||||
| `coherence.rs` / `coherence_gate.rs` | Z-score coherence scoring; Accept / PredictOnly / Reject / Recalibrate gate decisions |
|
||||
| `pose_tracker.rs` | 17-keypoint Kalman tracker with AETHER re-ID embeddings |
|
||||
| `field_model.rs` | SVD room eigenstructure, perturbation extraction |
|
||||
| `tomography.rs` | RF tomography, ISTA L1 solver, voxel grid |
|
||||
| `longitudinal.rs` | Welford stats, biomechanics drift detection |
|
||||
| `intention.rs` | Pre-movement lead signals (200–500 ms ahead) |
|
||||
| `cross_room.rs` | Environment fingerprinting, transition graph |
|
||||
| `gesture.rs` | DTW template-matching gesture classifier |
|
||||
| `adversarial.rs` | Physically-impossible-signal detection, multi-link consistency |
|
||||
|
||||
## Cross-viewpoint fusion (ADR-016 viewpoint module)
|
||||
|
||||
Combine 2+ nodes geometrically — more nodes, more independent looks, tighter localization.
|
||||
|
||||
| Module (`ruvector/src/viewpoint/`) | Purpose |
|
||||
|------------------------------------|---------|
|
||||
| `attention.rs` | CrossViewpointAttention, GeometricBias, softmax with `G_bias` |
|
||||
| `geometry.rs` | GeometricDiversityIndex, Cramér–Rao bounds, Fisher Information |
|
||||
| `coherence.rs` | Phase-phasor coherence, hysteresis gate |
|
||||
| `fusion.rs` | MultistaticArray aggregate root, domain events |
|
||||
|
||||
Host-side helpers to explore the geometry before deploying: `node scripts/mesh-graph-transformer.js`, `node scripts/passive-radar.js`, `node scripts/deep-scan.js`.
|
||||
|
||||
## Persistent field model (ADR-030)
|
||||
|
||||
`field_model.rs` builds an SVD eigenstructure of the room and stores it (RVF, ideally on a Cognitum Seed). New CSI frames are projected against it; the residual *is* the perturbation. Lets you ask "what's different from the empty-room baseline?" and survive restarts.
|
||||
|
||||
## RF tomography
|
||||
|
||||
`tomography.rs` reconstructs a voxel occupancy grid from the multistatic link set via an ISTA L1 solver (sparse — most voxels are empty). Use with cross-viewpoint geometry for through-wall volumetric imaging. RuVector solver crates back the sparse interpolation (114→56 subcarriers).
|
||||
|
||||
## Sensing-first RF mode & adaptive mesh kernel
|
||||
|
||||
- ADR-031 (RuView sensing-first RF mode), ADR-081 (adaptive CSI mesh firmware kernel), ADR-083 (per-cluster π compute hop), ADR-095/096 (on-ESP32 temporal modeling with sparse GQA attention — runs the temporal head on-device).
|
||||
|
||||
## Security (ADR-032 — multistatic mesh hardening)
|
||||
|
||||
Using neighbours' APs as illuminators and pooling links across a mesh expands the attack surface. Mitigations:
|
||||
- `adversarial.rs` rejects physically impossible signals and cross-checks multi-link consistency.
|
||||
- `coherence_gate.rs` quarantines low-coherence / suspicious links (Reject / Recalibrate).
|
||||
- Ed25519 witness chain (ADR-028) attests every measurement.
|
||||
- Run a security review when touching anything on the hardware/network boundary (see `ruview-verify` and `docs/security-audit-wasm-edge-vendor.md`).
|
||||
|
||||
## Validate advanced changes
|
||||
|
||||
```bash
|
||||
cd v2 && cargo test --workspace --no-default-features # incl. ruvsense + viewpoint tests
|
||||
cargo test -p wifi-densepose-signal --no-default-features
|
||||
cargo test -p wifi-densepose-ruvector --no-default-features
|
||||
cd .. && python archive/v1/data/proof/verify.py
|
||||
```
|
||||
|
||||
## Reference
|
||||
|
||||
- ADRs: 014 (SOTA signal processing), 029 (multistatic mode), 030 (persistent field model), 031 (sensing-first RF), 032 (mesh security hardening), 081/083/095/096
|
||||
- `v2/crates/wifi-densepose-signal/src/ruvsense/` · `v2/crates/wifi-densepose-ruvector/src/viewpoint/`
|
||||
- `docs/research/`, `docs/security-audit-wasm-edge-vendor.md`
|
||||
@@ -0,0 +1,69 @@
|
||||
---
|
||||
name: ruview-applications
|
||||
description: Run RuView sensing applications — presence/occupancy, breathing & heart rate, activity & fall detection, 17-keypoint pose estimation (WiFlow), sleep monitoring & apnea screening, environment mapping, Mass Casualty Assessment (MAT), and the 3D point-cloud fusion demo. Use when someone wants to actually *do* something with a working RuView setup.
|
||||
allowed-tools: Bash Read Write Edit Glob Grep
|
||||
---
|
||||
|
||||
# RuView Applications
|
||||
|
||||
What RuView can sense, and how to run each one. Assumes you have either the Docker demo (simulated CSI) or a live ESP32 sink (see `ruview-quickstart` / `ruview-hardware-setup`).
|
||||
|
||||
## Application catalogue
|
||||
|
||||
| Application | What it does | Entry point |
|
||||
|-------------|--------------|-------------|
|
||||
| **Presence / occupancy** | Detect people through walls, count them, track entries/exits (trained model + PIR fusion, ~0.012 ms latency) | sensing-server live mode; `examples/environment/` |
|
||||
| **Vital signs** | Breathing 6–30 BPM (bandpass 0.1–0.5 Hz), heart rate 40–120 BPM (bandpass 0.8–2.0 Hz), contactless while sleeping/sitting | `wifi-densepose-vitals` crate (ADR-021); `examples/medical/` |
|
||||
| **Activity recognition** | Walking, sitting, gestures, falls — from temporal CSI patterns | RuvSense `gesture.rs` (DTW), `pose_tracker.rs`; `scripts/gait-analyzer.js` |
|
||||
| **Pose estimation** | 17 COCO keypoints via WiFlow architecture; dual-modal webcam+WiFi fusion demo | `cargo run -p wifi-densepose-sensing-server` + pose-fusion demo (ADR-059); see `ruview-model-training` to train |
|
||||
| **Sleep monitoring** | Overnight monitoring, sleep-stage classification, apnea screening | `examples/sleep/`; `scripts/apnea-detector.js` |
|
||||
| **Environment mapping** | RF fingerprinting identifies rooms, detects moved furniture, spots new objects | sensing-server `--build-index env`; RuvSense `field_model.rs`, `cross_room.rs` |
|
||||
| **Mass Casualty Assessment (MAT)** | Disaster survivor detection — find people in rubble/smoke | `wifi-densepose-mat` crate; `docs/wifi-mat-user-guide.md`; `examples/medical/` |
|
||||
| **3D point cloud** *(optional fusion)* | Camera depth (MiDaS) + WiFi CSI + mmWave radar → unified spatial model (~22 ms, 19K+ pts/frame) | `scripts/mmwave_fusion_bridge.py`; ADR-094 (GitHub Pages deploy) |
|
||||
| **Novel RF apps** | Passive radar, material classification, device fingerprinting, mincut person-counting | `scripts/passive-radar.js`, `material-classifier.js`, `device-fingerprint.js`, `mincut-person-counter.js` (ADR-077/078) |
|
||||
|
||||
## Quick recipes
|
||||
|
||||
```bash
|
||||
# Docker demo — everything, simulated CSI
|
||||
docker run -p 3000:3000 ruvnet/wifi-densepose:latest # http://localhost:3000
|
||||
|
||||
# Live sensing server (consumes ESP32 UDP CSI)
|
||||
cd v2 && cargo run -p wifi-densepose-sensing-server
|
||||
|
||||
# Live RF room scan (Cognitum Seed on :5006)
|
||||
node scripts/rf-scan.js --port 5006
|
||||
node scripts/snn-csi-processor.js --port 5006
|
||||
|
||||
# Embed a trained model + build an environment index
|
||||
cd v2
|
||||
cargo run -p wifi-densepose-sensing-server -- --model model.rvf --embed
|
||||
cargo run -p wifi-densepose-sensing-server -- --model model.rvf --build-index env
|
||||
|
||||
# Python live demo
|
||||
python examples/ruview_live.py
|
||||
|
||||
# Spectrogram / graph visualisers
|
||||
node scripts/csi-spectrogram.js
|
||||
node scripts/csi-graph-visualizer.js
|
||||
```
|
||||
|
||||
## Picking the right modality
|
||||
|
||||
- **Through a wall, no line of sight** → presence + activity; expect ≤5 m depth (Fresnel-zone geometry).
|
||||
- **Person stationary (sleeping / sitting)** → vitals (breathing first, heart rate needs cleaner signal) + sleep staging.
|
||||
- **Need skeletons** → pose (WiFlow). Camera-free works but is modest; camera-supervised gets 92.9% PCK@20 — train it (`ruview-model-training`).
|
||||
- **Search & rescue** → MAT (`docs/wifi-mat-user-guide.md`).
|
||||
- **"What changed in this room?"** → environment mapping / RF fingerprint index.
|
||||
- **Best spatial accuracy** → 2+ ESP32 nodes + cross-viewpoint fusion (`ruview-advanced-sensing`), optionally + Cognitum Seed.
|
||||
|
||||
## Examples directory map
|
||||
|
||||
`examples/environment/` · `examples/medical/` · `examples/sleep/` · `examples/stress/` · `examples/happiness-vector/` · `examples/ruview_live.py` — each has a README.
|
||||
|
||||
## Reference
|
||||
|
||||
- `README.md` — feature matrix, latency/throughput numbers
|
||||
- `docs/user-guide.md`, `docs/wifi-mat-user-guide.md`
|
||||
- ADRs: 021 (vitals), 024 (AETHER contrastive embeddings), 027 (MERIDIAN domain generalization), 041 (edge modules), 059 (live ESP32 pipeline), 077/078 (novel RF apps), 082 (pose tracker output filter), 094 (point cloud)
|
||||
- RuvSense modules: `v2/crates/wifi-densepose-signal/src/ruvsense/` (14 modules)
|
||||
@@ -0,0 +1,82 @@
|
||||
---
|
||||
name: ruview-cli-api
|
||||
description: Use the RuView `wifi-densepose` CLI binary (incl. MAT scan/status/zones/survivors/alerts/export subcommands), the REST API (`wifi-densepose-api`, Axum), and the browser/WASM build (`wifi-densepose-wasm`, `wifi-densepose-wasm-edge`). Use when integrating RuView into another program, scripting it from the shell, exposing it over HTTP, or shipping it to the browser / ESP32-WASM3.
|
||||
allowed-tools: Bash Read Write Edit Glob Grep
|
||||
---
|
||||
|
||||
# RuView CLI, API & WASM
|
||||
|
||||
The programmatic surfaces of RuView — the `wifi-densepose` binary, the HTTP API, and the WebAssembly builds.
|
||||
|
||||
## 1. The `wifi-densepose` CLI binary (`wifi-densepose-cli`)
|
||||
|
||||
```bash
|
||||
cd v2
|
||||
cargo run -p wifi-densepose-cli -- --help # or: cargo build -p wifi-densepose-cli --release → target/release/wifi-densepose
|
||||
cargo run -p wifi-densepose-cli -- version
|
||||
```
|
||||
|
||||
Top-level subcommands: `version`, and `mat` (Mass Casualty Assessment Tool).
|
||||
|
||||
### `wifi-densepose mat …` — disaster survivor detection
|
||||
|
||||
| Subcommand | Purpose | Key flags |
|
||||
|------------|---------|-----------|
|
||||
| `mat scan [zone]` | Start scanning for survivors | `--disaster-type <…>`, `--sensitivity 0.0–1.0`, `--max-depth <m>`, `--continuous`, `--interval <ms>`, `--simulate` |
|
||||
| `mat status` | Current scan status | `--detailed`, `--format <…>`, `--watch` |
|
||||
| `mat zones …` | Manage scan zones | `zones list [--active-only]`, plus add/remove/update |
|
||||
| `mat survivors` | List detected survivors with triage status | |
|
||||
| `mat alerts` | View / manage alerts | |
|
||||
| `mat export` | Export scan data | JSON or CSV |
|
||||
|
||||
Example:
|
||||
```bash
|
||||
cargo run -p wifi-densepose-cli -- mat scan rubble-A --disaster-type earthquake --sensitivity 0.7 --max-depth 5 --continuous --interval 2000
|
||||
cargo run -p wifi-densepose-cli -- mat survivors --format json
|
||||
cargo run -p wifi-densepose-cli -- mat export --format csv > survivors.csv
|
||||
```
|
||||
|
||||
Use `--simulate` for testing without hardware. Background and user guide: `docs/wifi-mat-user-guide.md`, `wifi-densepose-mat` crate.
|
||||
|
||||
## 2. REST API (`wifi-densepose-api`, Axum)
|
||||
|
||||
Library crate (`v2/crates/wifi-densepose-api/src/lib.rs`) — the Axum router/handlers; configured via the `wifi-densepose-config` crate. It's wired into the server binaries (e.g. the sensing server / Docker image), not a standalone `cargo run` target by itself.
|
||||
|
||||
```bash
|
||||
# Easiest way to exercise it: the Docker image exposes the API + dashboard on :3000
|
||||
docker run -p 3000:3000 ruvnet/wifi-densepose:latest
|
||||
# Then hit the HTTP endpoints (see the API module / docs for routes) and open http://localhost:3000
|
||||
|
||||
# v1 Python service config reference: example.env, pyproject.toml (archive/v1/)
|
||||
```
|
||||
|
||||
When embedding the API crate in your own binary, take the router from `wifi_densepose_api`, supply config via `wifi-densepose-config`, and serve with Axum/Tokio. Keep input validation at the boundary (project rule).
|
||||
|
||||
## 3. WASM / browser & ESP32-WASM3
|
||||
|
||||
- **`wifi-densepose-wasm`** — compiles the stack to `wasm32-unknown-unknown` with a JS-friendly API:
|
||||
```bash
|
||||
cd v2/crates/wifi-densepose-wasm
|
||||
wasm-pack build --target web --features mat # recommended (produces pkg/)
|
||||
cargo build --target wasm32-unknown-unknown --features mat # plain cargo build
|
||||
```
|
||||
See `v2/crates/wifi-densepose-wasm/README.md` for the exported surface.
|
||||
- **`wifi-densepose-wasm-edge`** — 60 edge modules (609 tests) that compile to `wasm32-unknown-unknown` and run on ESP32-S3 via WASM3; shared utils in `src/vendor_common.rs`. These are the ADR-041 edge-intelligence modules in WASM form.
|
||||
- Browser demos: pose-fusion (ADR-059), point-cloud (ADR-094) — deployed via GitHub Pages from the WASM build.
|
||||
|
||||
## 4. Where it fits
|
||||
|
||||
| You want to… | Use |
|
||||
|--------------|-----|
|
||||
| Script a survivor scan / export results | `wifi-densepose mat …` |
|
||||
| Expose sensing over HTTP | `wifi-densepose-api` (via a server binary / Docker) |
|
||||
| Run sensing in a browser | `wifi-densepose-wasm` → `wasm-pack build --target web` |
|
||||
| Run an edge module on an ESP32 in WASM | `wifi-densepose-wasm-edge` + WASM3 |
|
||||
| A long-running CSI sink + training | `wifi-densepose-sensing-server` (see `ruview-applications` / `ruview-model-training`) |
|
||||
|
||||
## Reference
|
||||
|
||||
- Crates: `wifi-densepose-cli`, `wifi-densepose-api`, `wifi-densepose-config`, `wifi-densepose-wasm`, `wifi-densepose-wasm-edge`, `wifi-densepose-mat`
|
||||
- ADRs: 041 (edge modules), 059 (live ESP32 pipeline), 094 (point-cloud GitHub Pages)
|
||||
- `docs/wifi-mat-user-guide.md`, `docs/edge-modules/`, `docs/security-audit-wasm-edge-vendor.md`
|
||||
- Validate after changes: `cd v2 && cargo test -p wifi-densepose-cli -p wifi-densepose-api -p wifi-densepose-wasm --no-default-features`
|
||||